Merge pull request #9 from Sv443/ver/1.2.0

.env.template

@@ -1,2 +1,4 @@
-HTTP_PORT=8074
+HTTP_PORT=8074             # 
+HTTP_HOST=0.0.0.0          # Defaults to 0.0.0.0 (listen on all interfaces)
 GENIUS_ACCESS_TOKEN=abcdef # Gotten from POST https://api.genius.com/oauth/token or from creating a client on https://genius.com/api-clients
+AUTH_TOKENS=               # Comma-separated list of HTTP bearer tokens that are excluded from rate limiting

README.md

@@ -1,7 +1,9 @@
 # geniURL
 
 Simple JSON and XML REST API to search for song metadata and the lyrics URL on [genius.com](https://genius.com/)  
-Obtaining actual lyrics sadly isn't possible due to licensing and copyright reasons
+Authorization is not required and geniURL implements a fuzzy search that will greatly improve search results over the genius.com API.  
+  
+Obtaining actual lyrics sadly isn't possible due to licensing and copyright reasons.  
 
 <br><br>
 
@@ -32,8 +34,8 @@ All routes support gzip and deflate compression.
 
 > ### GET `/search`
 >
-> This endpoint gives you the top 10 results for a search query specified by `search_text`  
-> The returned data contains various data like the lyrics website URL, song and thumbnail metadata and more (see below).
+> This endpoint gives you up to 10 results for a search query specified by `search_text`  
+> The returned payload contains various data like the lyrics website URL, song and thumbnail metadata and more (see below).
 >
 > <br>
 >
@@ -41,17 +43,26 @@ All routes support gzip and deflate compression.
 > `?q=search%20query`  
 > This parameter should contain both the song and artist name (for best result artist name should come first, separate with a whitespace).  
 > Sometimes the song name alone might be enough but the results vary greatly.  
-> Using this parameter instead of `?artist` and `?song` will not modify the search results and so you will sometimes get blatantly wrong top matches.  
+> Using this parameter instead of `?artist` and `?song` means you will get slightly less accurate results.  
 > Make sure the search query is [percent/URL-encoded.](https://en.wikipedia.org/wiki/Percent-encoding)  
->   
+> 
+> **OR**
+> 
 > `?artist=name` and `?song=name`  
-> Instead of `?q`, you can use `?artist` and `?song` to tell geniURL to preemptively filter the search results.  
-> This is done using a fuzzy search to greatly increase the chances the correct search result will be at the top.  
+> Instead of `?q`, you can use `?artist` and `?song` to help geniURL filter the search results better, so your top results will be more accurate.  
 > Make sure these parameters are [percent/URL-encoded.](https://en.wikipedia.org/wiki/Percent-encoding)  
->   
+> 
+> <br>
+> 
+> **Optional URL Parameters:**  
 > `?format=json/xml`  
-> Use this parameter to change the response format from the default (`json`) to `xml`  
+> Use this optional parameter to change the response format from the default (`json`) to `xml`  
 > The structure of the XML data is similar to the shown JSON data.
+>   
+> `?threshold=0.65`  
+> This optional parameter can be used to change the fuzzy search threshold from the default of 0.65  
+> It has to be between 0.0 and 1.0; the lower the number, the less results you'll get but the more accurate the top results will be.  
+> 0.65 is a good middle ground but depending on your use-case you might want to play around with this.
 >
 > <br>
 > 
@@ -64,23 +75,23 @@ All routes support gzip and deflate compression.
 >     "error": false,
 >     "matches": 10,
 >     "top": {
->         "url": "https://genius.com/Artist-1-song-name-lyrics",
->         "path": "/Artist-1-song-name-lyrics",
+>         "url": "https://genius.com/Artist-Foo-song-name-lyrics",
+>         "path": "/Artist-Foo-song-name-lyrics",
 >         "language": "en",
 >         "meta": {
 >             "title": "Song Name",
->             "fullTitle": "Song Name by Artist 1 (ft. Artist 2)",
->             "artists": "Artist 1 (ft. Artist 2)",
+>             "fullTitle": "Song Name by Artist Foo (ft. Artist Bar)",
+>             "artists": "Artist Foo (ft. Artist Bar)",
 >             "primaryArtist": {
->                 "name": "Artist 1",
->                 "url": "https://genius.com/artists/Artist-1",
+>                 "name": "Artist Foo",
+>                 "url": "https://genius.com/artists/Artist-Foo",
 >                 "headerImage": "https://images.genius.com/...",
 >                 "image": "https://images.genius.com/..."
 >             },
 >             "featuredArtists": [
 >                 {
->                     "name": "Featured Artist 1",
->                     "url": "https://genius.com/artists/Featured-Artist-1",
+>                     "name": "Artist Bar",
+>                     "url": "https://genius.com/artists/Artist-Bar",
 >                     "headerImage": "https://images.genius.com/...",
 >                     "image": "https://images.genius.com/..."
 >                 }
@@ -139,7 +150,7 @@ All routes support gzip and deflate compression.
 
 > ### GET `/search/top`
 >
-> This endpoint is the same as `/search`, but it only gives the top result.  
+> This endpoint is similar to `/search`, but it only gives the top result.  
 > Use this if you are only interested in the top result and want to reduce traffic.
 >
 > <br>
@@ -148,17 +159,26 @@ All routes support gzip and deflate compression.
 > `?q=search%20query`  
 > This parameter should contain both the song and artist name (for best result artist name should come first, separate with a whitespace).  
 > Sometimes the song name alone might be enough but the results vary greatly.  
-> Using this parameter instead of `?artist` and `?song` will not modify the search result and so you will sometimes get a blatantly wrong top match.  
+> Using this parameter instead of `?artist` and `?song` means you will get slightly less accurate results.  
 > Make sure the search query is [percent/URL-encoded.](https://en.wikipedia.org/wiki/Percent-encoding)  
->   
+> 
+> **OR**
+> 
 > `?artist=name` and `?song=name`  
-> Instead of `?q`, you can use `?artist` and `?song` to tell geniURL to preemptively filter the search results.  
-> This is done using a fuzzy search to greatly increase the chances the correct search result will be returned.  
+> Instead of `?q`, you can use `?artist` and `?song` to help geniURL filter the search results better, so your top results will be more accurate.  
 > Make sure these parameters are [percent/URL-encoded.](https://en.wikipedia.org/wiki/Percent-encoding)  
->   
+> 
+> <br><br>
+> 
+> **Optional URL Parameters:**  
 > `?format=json/xml`  
-> Use this parameter to change the response format from the default (`json`) to `xml`  
+> Use this optional parameter to change the response format from the default (`json`) to `xml`  
 > The structure of the XML data is similar to the shown JSON data.
+>   
+> `?threshold=0.65`  
+> This optional parameter can be used to change the fuzzy search threshold from the default of 0.65  
+> It has to be between 0.0 and 1.0; the lower the number, the less results you'll get but the more accurate the top results will be.  
+> 0.65 is a good middle ground but depending on your use-case you might want to play around with this.
 >
 > <br>
 > 
@@ -170,23 +190,23 @@ All routes support gzip and deflate compression.
 > {
 >     "error": false,
 >     "matches": 1,
->     "url": "https://genius.com/Artist-1-song-name-lyrics",
->     "path": "/Artist-1-song-name-lyrics",
+>     "url": "https://genius.com/Artist-Foo-song-name-lyrics",
+>     "path": "/Artist-Foo-song-name-lyrics",
 >     "language": "en",
 >     "meta": {
 >         "title": "Song Name",
->         "fullTitle": "Song Name by Artist 1 (ft. Artist 2)",
->         "artists": "Artist 1 (ft. Artist 2)",
+>         "fullTitle": "Song Name by Artist Foo (ft. Artist Bar)",
+>         "artists": "Artist Foo (ft. Artist Bar)",
 >         "primaryArtist": {
->             "name": "Artist 1",
->             "url": "https://genius.com/artists/Artist-1",
+>             "name": "Artist Foo",
+>             "url": "https://genius.com/artists/Artist-Foo",
 >             "headerImage": "https://images.genius.com/...",
 >             "image": "https://images.genius.com/..."
 >         },
 >         "featuredArtists": [
 >             {
->                 "name": "Featured Artist 1",
->                 "url": "https://genius.com/artists/Featured-Artist-1",
+>                 "name": "Artist Bar",
+>                 "url": "https://genius.com/artists/Artist-Bar",
 >                 "headerImage": "https://images.genius.com/...",
 >                 "image": "https://images.genius.com/..."
 >             }

changelog.md

@@ -1,4 +1,5 @@
 ## Version History:
+- [**1.2.0**](#v120)
 - [1.1.1](#v111)
 - [1.1.0](#v110)
 - [1.0.0](#v100)
@@ -7,6 +8,12 @@
 
 <br><br>
 
+### v1.2.0
+- Added `?threshold` parameter to change the fuzzy search threshold from its default of 0.6 ([#7](https://github.com/Sv443/geniURL/issues/7))
+- Added support for fuzzy searching when using `?q` instead of `?artist` and `?song` ([#8](https://github.com/Sv443/geniURL/issues/8))
+
+<br>
+
 ### v1.1.1
 - Minor fixes
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "geniurl",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "description": "Simple JSON and XML REST API to search for song metadata and the lyrics URL on genius.com",
   "main": "src/index.ts",
   "scripts": {
@@ -14,8 +14,8 @@
   "keywords": [
     "rest-api",
     "lyrics",
-    "rest-proxy",
-    "lyrics-search"
+    "lyrics-search",
+    "song-metadata"
   ],
   "author": {
     "name": "Sv443",
@@ -27,15 +27,20 @@
     "url": "https://github.com/Sv443/geniURL/issues"
   },
   "homepage": "https://github.com/Sv443/geniURL",
+  "engines": {
+    "node": ">=16",
+    "npm": ">=8"
+  },
   "dependencies": {
-    "axios": "^0.26.0",
+    "axios": "^1.2.1",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
     "express": "^4.17.3",
     "fuse.js": "^6.6.2",
-    "helmet": "^5.0.2",
-    "js2xmlparser": "^4.0.2",
+    "helmet": "^6.0.1",
+    "js2xmlparser": "^5.0.0",
     "kleur": "^4.1.4",
+    "nanoid": "^3.3.4",
     "rate-limiter-flexible": "^2.3.6",
     "svcorelib": "^1.15.0",
     "tcp-port-used": "^1.0.2"

src/index.ts

@@ -12,7 +12,7 @@ async function init()
 
     try
     {
-        server.init();
+        await server.init();
 
         stage = "(done)";
     }

src/server.ts

@@ -25,45 +25,54 @@ const rateLimiter = new RateLimiterMemory({
     duration: 10,
 });
 
+const authTokens = getAuthTokens();
 
 export async function init()
 {
-    const port = parseInt(String(process.env.HTTP_PORT));
+    const port = parseInt(String(process.env.HTTP_PORT ?? "").trim());
+    const hostRaw = String(process.env.HTTP_HOST ?? "").trim();
+    const host = hostRaw.length < 1 ? "0.0.0.0" : hostRaw;
 
     if(await portUsed(port))
-        return error(`TCP port ${port} is already used`, undefined, true);
+        return error(`TCP port ${port} is already used or invalid`, undefined, true);
 
     // on error
-    app.use((err: any, req: Request, res: Response, next: NextFunction) => {
+    app.use((err: unknown, req: Request, res: Response, next: NextFunction) => {
         if(typeof err === "string" || err instanceof Error)
             return respond(res, "serverError", `General error in HTTP server: ${err.toString()}`, req?.query?.format ? String(req.query.format) : undefined);
         else
             return next();
     });
 
-    const listener = app.listen(port, () => {
+    const listener = app.listen(port, host, () => {
         app.disable("x-powered-by");
 
         // rate limiting
         app.use(async (req, res, next) => {
             const fmt = req?.query?.format ? String(req.query.format) : undefined;
+            const { authorization } = req.headers;
+            const authHeader = authorization?.startsWith("Bearer ") ? authorization.substring(7) : authorization;
 
             res.setHeader("API-Info", `geniURL v${packageJson.version} (${packageJson.homepage})`);
 
+            if(authHeader && authTokens.has(authHeader))
+                return next();
+
             rateLimiter.consume(req.ip)
                 .catch((err) => {
                     if(err instanceof RateLimiterRes) {
                         res.set("Retry-After", String(Math.ceil(err.msBeforeNext / 1000)));
                         return respond(res, 429, { message: "You are being rate limited" }, fmt);
                     }
-                    else return respond(res, 500, { message: "Internal error in rate limiting middleware. Please try again later." }, fmt);
+                    else
+                        return respond(res, 500, { message: "Internal error in rate limiting middleware. Please try again later." }, fmt);
                 })
                 .finally(next);
         });
 
         registerEndpoints();
 
-        console.log(k.green(`Ready on port ${port}`));
+        console.log(k.green(`Listening on ${host}:${port}`));
     });
 
     listener.on("error", (err) => error("General server error", err, true));
@@ -82,17 +91,21 @@ function registerEndpoints()
         app.get("/search", async (req, res) => {
             try
             {
-                const { q, artist, song, format: fmt } = req.query;
+                const { q, artist, song, format: fmt, threshold: thr } = req.query;
 
-                const format = fmt ? String(fmt) : "json";
+                const format: string = fmt ? String(fmt) : "json";
+                const threshold = isNaN(Number(thr)) ? undefined : Number(thr);
 
                 if(hasArg(q) || (hasArg(artist) && hasArg(song)))
                 {
-                    const meta = await getMeta(q ? {
-                        q: String(q),
-                    } : {
-                        artist: String(artist),
-                        song: String(song),
+                    const meta = await getMeta({
+                        ...(q ? {
+                            q: String(q),
+                        } : {
+                            artist: String(artist),
+                            song: String(song),
+                        }),
+                        threshold,
                     });
 
                     if(!meta || meta.all.length < 1)
@@ -115,17 +128,21 @@ function registerEndpoints()
         app.get("/search/top", async (req, res) => {
             try
             {
-                const { q, artist, song, format: fmt } = req.query;
+                const { q, artist, song, format: fmt, threshold: thr } = req.query;
 
-                const format = fmt ? String(fmt) : "json";
+                const format: string = fmt ? String(fmt) : "json";
+                const threshold = isNaN(Number(thr)) ? undefined : Number(thr);
 
                 if(hasArg(q) || (hasArg(artist) && hasArg(song)))
                 {
-                    const meta = await getMeta(q ? {
-                        q: String(q),
-                    } : {
-                        artist: String(artist),
-                        song: String(song),
+                    const meta = await getMeta({
+                        ...(q ? {
+                            q: String(q),
+                        } : {
+                            artist: String(artist),
+                            song: String(song),
+                        }),
+                        threshold,
                     });
 
                     if(!meta || !meta.top)
@@ -148,6 +165,12 @@ function registerEndpoints()
     }
 }
 
+/**
+ * Responds to an incoming request
+ * @param type Type of response or status code
+ * @param data The data to send in the response body
+ * @param format json / xml
+ */
 function respond(res: Response, type: ResponseType | number, data: Stringifiable | Record<string, unknown>, format = "json", matchesAmt = 0)
 {
     let statusCode = 500;
@@ -167,7 +190,7 @@ function respond(res: Response, type: ResponseType | number, data: Stringifiable
             error = false;
             matches = matchesAmt;
             statusCode = 200;
-            resData = { ...data };
+            resData = typeof data === "string" ? data : { ...data };
             break;
         case "clientError":
             error = true;
@@ -187,7 +210,7 @@ function respond(res: Response, type: ResponseType | number, data: Stringifiable
                 error = false;
                 matches = matchesAmt ?? 0;
                 statusCode = type;
-                resData = { ...data };
+                resData = typeof data === "string" ? data : { ...data };
             }
             break;
     }
@@ -204,5 +227,18 @@ function respond(res: Response, type: ResponseType | number, data: Stringifiable
     const finalData = format === "xml" ? jsonToXml.parse("data", resData) : resData;
 
     res.setHeader("Content-Type", mimeType);
-    res.status(statusCode).send(finalData);
+    res.status(statusCode)
+        .send(finalData);
+}
+
+function getAuthTokens() {
+    const envVal = process.env["AUTH_TOKENS"];
+    let tokens: string[] = [];
+
+    if(!envVal || envVal.length === 0)
+        tokens = [];
+    else
+        tokens = envVal.split(/,/g);
+
+    return new Set<string>(tokens);
 }

src/songMeta.ts

@@ -1,25 +1,49 @@
 import axios from "axios";
 import Fuse from "fuse.js";
-import { randomUUID } from "crypto";
-import { JSONCompatible, reserialize } from "svcorelib";
-import { ApiSearchResult, SongMeta } from "./types";
+import { nanoid } from "nanoid";
+import { allOfType, clamp } from "svcorelib";
+import type { ApiSearchResult, SongMeta } from "./types";
 
 type MetaSearchHit = SongMeta & { uuid?: string; };
 
+interface GetMetaProps {
+    q?: string;
+    artist?: string;
+    song?: string;
+    threshold?: number;
+}
+
+interface GetMetaResult {
+    top: SongMeta;
+    all: SongMeta[];
+}
+
+const defaultFuzzyThreshold = 0.65;
+
 /**
  * Returns meta information about the top results of a search using the genius API
  * @param param0 Pass an object with either a `q` prop or the props `artist` and `song` to make use of fuzzy filtering
  */
-export async function getMeta({ q, artist, song }: Partial<Record<"q" | "artist" | "song", string>>): Promise<{ top: SongMeta, all: SongMeta[] } | null>
+export async function getMeta({
+    q,
+    artist,
+    song,
+    threshold,
+}: GetMetaProps): Promise<GetMetaResult | null>
 {
     const accessToken = process.env.GENIUS_ACCESS_TOKEN ?? "ERR_NO_ENV";
 
     const query = q ? q : `${artist} ${song}`;
+    const searchByQuery = allOfType([artist, song], "undefined");
 
     const { data: { response }, status } = await axios.get<ApiSearchResult>(`https://api.genius.com/search?q=${encodeURIComponent(query)}`, {
         headers: { "Authorization": `Bearer ${accessToken}` },
     });
 
+    if(threshold === undefined || isNaN(threshold))
+        threshold = defaultFuzzyThreshold;
+    threshold = clamp(threshold, 0.0, 1.0);
+
     if(status >= 200 && status < 300 && Array.isArray(response?.hits))
     {
         if(response.hits.length === 0)
@@ -36,14 +60,14 @@ export async function getMeta({ q, artist, song }: Partial<Record<"q" | "artist"
                     fullTitle: formatStr(result.full_title),
                     artists: formatStr(result.artist_names),
                     primaryArtist: {
-                        name: formatStr(result.primary_artist.name) ?? null,
+                        name: result.primary_artist.name ? formatStr(result.primary_artist.name) : null,
                         url: result.primary_artist.url ?? null,
                         headerImage: result.primary_artist.header_image_url ?? null,
                         image: result.primary_artist.image_url ?? null,
                     },
                     featuredArtists: Array.isArray(result.featured_artists) && result.featured_artists.length > 0
                         ? result.featured_artists.map((a) => ({
-                            name: a.name ?? null,
+                            name: a.name ? formatStr(a.name) : null,
                             url: a.url ?? null,
                             headerImage: a.header_image_url ?? null,
                             image: a.image_url ?? null,
@@ -59,57 +83,68 @@ export async function getMeta({ q, artist, song }: Partial<Record<"q" | "artist"
                 id: result.id ?? null,
             }));
 
-        if(artist && song)
-        {
-            const scoreMap: Record<string, number> = {};
+        const scoreMap: Record<string, number> = {};
 
-            hits = hits.map(h => {
-                h.uuid = randomUUID();
-                return h;
-            }) as (SongMeta & { uuid: string })[];
+        hits = hits.map(h => {
+            h.uuid = nanoid();
+            return h;
+        }) as (SongMeta & { uuid: string })[];
 
-            const fuseOpts: Fuse.IFuseOptions<MetaSearchHit> = {
-                ignoreLocation: true,
-                includeScore: true,
-                threshold: 0.6,
-            };
+        const fuseOpts: Fuse.IFuseOptions<MetaSearchHit> = {
+            includeScore: true,
+            threshold,
+        };
 
-            const titleFuse = new Fuse(hits, { ...fuseOpts, keys: [ "meta.title" ] });
-            const artistFuse = new Fuse(hits, { ...fuseOpts, keys: [ "meta.primaryArtist.name" ] });
+        const addScores = (searchRes: Fuse.FuseResult<SongMeta & { uuid?: string; }>[]) =>
+            searchRes.forEach(({ item, score }) => {
+                if(!item.uuid || !score)
+                    return;
 
-            /** @param {({ item: { uuid: string }, score: number })[]} searchRes */
-            const addScores = (searchRes: Fuse.FuseResult<SongMeta & { uuid?: string; }>[]) =>
-                searchRes.forEach(({ item, score }) => {
-                    if(!item.uuid || !score)
-                        return;
+                if(!scoreMap[item.uuid])
+                    scoreMap[item.uuid] = score;
+                else
+                    scoreMap[item.uuid] += score;
+            });
 
-                    if(!scoreMap[item.uuid])
-                        scoreMap[item.uuid] = score;
-                    else
-                        scoreMap[item.uuid] += score;
-                });
+        if(song && artist) {
+            const titleFuse = new Fuse(hits, { ...fuseOpts, keys: [ "meta.title" ] });
+            const artistFuse = new Fuse(hits, { ...fuseOpts, keys: [ "meta.primaryArtist.name" ] });
 
             addScores(titleFuse.search(song));
             addScores(artistFuse.search(artist));
+        }
+        else {
+            const queryFuse = new Fuse(hits, {
+                ...fuseOpts,
+                ignoreLocation: true,
+                keys: [ "meta.title", "meta.primaryArtist.name" ],
+            });
+
+            let queryParts = [query];
+            if(query.match(/\s-\s/))
+                queryParts = query.split(/\s-\s/);
 
-            const bestMatches = Object.entries(scoreMap)
-                .sort(([, valA], [, valB]) => valA > valB ? 1 : -1)
-                .map(e => e[0]);
-
-            const oldHits = [...hits];
-
-            hits = bestMatches
-                .map(uuid => oldHits.find(h => h.uuid === uuid))
-                .map(hit => {
-                    if(hit)
-                    {
-                        delete hit.uuid;
-                        return hit;
-                    }
-                })
-                .filter(h => h !== undefined) as MetaSearchHit[];
+            for(const part of queryParts)
+                addScores(queryFuse.search(part.trim()));
         }
 
+        // TODO: reduce the amount of remapping cause it takes long
+
+        const bestMatches = Object.entries(scoreMap)
+            .sort(([, valA], [, valB]) => valA > valB ? 1 : -1)
+            .map(e => e[0]);
+
+        const oldHits = [...hits];
+
+        hits = bestMatches
+            .map(uuid => oldHits.find(h => h.uuid === uuid))
+            .map(hit => {
+                if(!hit) return undefined;
+                delete hit.uuid;
+                return hit;
+            })
+            .filter(h => h !== undefined) as MetaSearchHit[];
+
         return {
             top: hits[0] as MetaSearchHit,
             all: hits.slice(0, 10),
@@ -128,5 +163,7 @@ function formatStr(str: unknown): string
     if(!str || typeof str !== "string")
         throw new TypeError("formatStr(): input is not a string");
 
-    return str.replace(/[\u0000-\u001F\u007F-\u009F\u200B]/g, "").replace(/\u00A0/g, " ");
+    return str
+        .replace(/[\u0000-\u001F\u007F-\u009F\u200B]/g, "") // 0-width spaces & control characters
+        .replace(/\u00A0/g, " "); // non-standard 1-width spaces
 }