fix pool validation regex

search does not need to await imagesheet

README.md

@@ -13,13 +13,14 @@ Get YouTube videos for a search query.
 ### Required query parameters
 
 - `pool`: id of the VRCUrl pool, only letters numbers hyphens or underscores, optionally followed by an integer for pool size.
-- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed. THIS MUST BE THE LAST QUERY PARAMETER AS ALL CHARS AFTER IT ARE CAPTURED VERBATIM (so you can type & etc without encoding)
+- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed. THIS MUST BE THE LAST QUERY PARAMETER AS ALL CHARS AFTER IT ARE CAPTURED VERBATIM (so you can type & etc without encoding). If this contains a url with a playlist ID, playlist results will be loaded instead (which is a bit different, up to 100 results and no descriptions).
 
 ### Optional query parameters
 
 - `thumbnails`: set to `1`, `true`, `yes`, `on` or whatever to load thumbnails
 - `icons`: set to `1`, `true`, `yes`, `on` or whatever to load channel icons
-- `captions`: set to `1`, `true`, `yes`, `on` or whatever if you need access to closed captioning data
+- `mode`: If set to `latestontop` will search without filter to include the "Latest from ..." section (if it exists). Otherwise it uses search filter for Videos only sorted by Relevance.
+- `bp`: Custom YouTube search filter. Go to youtube search site and set some filters and you will see the corresponding `bp` value in the URL. Overrides `mode` option. Use at your own risk (experimental).
 
 ### Example URL
 
@@ -32,11 +33,11 @@ https://api.u2b.cx/search?pool=example10000&input=   Type YouTube search query h
 JSON object:
 
 - `results`: Array of Object
-	- `vrcurl`: (integer) index of VRCUrl that will redirect to the youtube url
+	- `vrcurl`: (integer) index of VRCUrl that will redirect to the youtube url, or serve JSON with captions if used with string loader.
 	- `live`: (boolean) whether it's a live stream
 	- `title`: (string) i.e. `"Nyan Cat! [Official]"`
 	- `id`: (string) YouTube video id i.e. `"2yJgwwDcgV8"`
-	- `description`: (string) short truncated description snippet i.e. `"http://nyan.cat/ Original song : http://momolabo.lolipop.jp/nyancatsong/Nyan/"`
+	- `description`?: (string) short truncated description snippet i.e. `"http://nyan.cat/ Original song : http://momolabo.lolipop.jp/nyancatsong/Nyan/"` (playlist results don't have this)
 	- `lengthText`?: (string) i.e. `"3:37"`
 	- `longLengthText`?: (string) i.e. `"3 minutes, 37 seconds"`
 	- `viewCountText`?: (string) i.e. `"2,552,243 views"` or `"575 watching"` for live streams (playlist results don't have this)
@@ -55,7 +56,6 @@ JSON object:
 		- `y`: (integer) px from top
 		- `width`: (integer)
 		- `height`: (integer)
-	- `captions_vrcurl`?: (integer) index of vrcurl to get the caption data json
 - `imagesheet_vrcurl`?: (integer) index of the vrcurl for the collage of thumbnails and/or icons
 - `nextpage_vrcurl`: (integer) index of the vrcurl that will serve the JSON for the next page of results
 
@@ -73,11 +73,11 @@ Gets Trending YouTube videos. Identical to `/search` but without `input` paramet
 - `{pool}`: must be same as pool param in search endpoint.
 - `{index}`: vrcurl index number
 
-Response may be 302 redirect to youtube url, `image/png` for imagesheet, `application/json` for next page (see response format above) or trending tab or caption data (below).
+Response may be 302 redirect to youtube url, `image/png` for imagesheet, `application/json` for next page (see response format above) or trending tab or video json data (see below).
 
-### Caption JSON format
+### Video metadata JSON format
 
-- Array of Object
+- `captions`: Array of Object
 	- `name`: (string) caption track name like "English" or "English (auto-generated)"
 	- `id`: (string) id like `.en` or `a.en`
 	- `lines`: Array of Object
@@ -108,4 +108,4 @@ All resources (youtube urls etc) referenced in the search results will be substi
 
 Video thumbnails and channel icons are collated together into one image and served at a VRCUrl to be loaded by VRCImageDownloader.
 
-Use the x, y, width and height values from the json to crop the image from the sheet.
+Use the x, y, width and height values from the json to crop the image from the sheet. Do not make any assumptions about these values as the server could arrange the images wherever it wants.

VRCYoutubeSearch.js

@@ -1,36 +1,36 @@
 import { searchYouTubeVideos, continueYouTubeVideoSearch, getYouTubePlaylist, continueYouTubePlaylist, getTrending } from "./youtube.js";
 import { putVrcUrl } from "./vrcurl.js";
-import { makeImageSheetVrcUrl } from "./imagesheet.js";
+import { createImageSheet } from "./imagesheet.js";
 
 var cache = {};
 
-
-
 export async function cachedVRCYoutubeSearch(pool, query, options) {
 	var key = JSON.stringify([pool, query, options]);
 	if (!cache[key]) {
-		cache[key] = VRCYoutubeSearch(pool, query, options);
+		cache[key] = VRCYoutubeSearch(pool, query, options, key);
 		setTimeout(() => {
 			delete cache[key];
 		}, 1000*60*10); // 10 mins
 	}
-	return await cache[key];
+	return (await cache[key])?.response;
+}
+
+export async function getImageSheet(key) {
+	return await (await cache[key])?.imagesheet
 }
 
 
-
+async function VRCYoutubeSearch(pool, query, options = {}, key) {
-
+	console.log("search", pool, JSON.stringify(query), JSON.stringify(options));
-async function VRCYoutubeSearch(pool, query, options = {}) {
+	var response = {results: []};
-	console.log("search:", JSON.stringify(query));
-	var data = {results: []};
 
 	if (typeof query == "object") {
 		switch (query.type) {
 			case "trending":
 				var {videos, tabs} = await getTrending(query.bp);
-				data.tabs = [];
+				response.tabs = [];
 				for (let tab of tabs) {
-					data.tabs.push({
+					response.tabs.push({
 						name: tab.name,
 						vrcurl: await putVrcUrl(pool, {type: "trending", bp: tab.bp, options})
 					});
@@ -48,22 +48,28 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 	} else {
 		var playlistId = query.match(/list=(PL[a-zA-Z0-9-_]{32})/)?.[1];
 		if (playlistId) console.log("playlistId:", playlistId);
-		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query);
+		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query, options.bp || (options.mode == "latestontop" ? null : undefined));
 	}
 	
 	var images = [];
 
 	if (options.thumbnails) {
 		videos.forEach(video => {
-			video.thumbnail = playlistId ? {
+			if (playlistId) video.thumbnail = {
 				url: `https://i.ytimg.com/vi/${video.id}/default.jpg`,
 				width: 120,
 				height: 90
-			} : {
-				url: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`,
-				width: 320,
-				height: 180
 			};
+			else {
+				video.thumbnail ||= {
+					url: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`,
+					width: 320,
+					height: 180
+				};
+				console.debug(video.thumbnail);
+				video.thumbnail.width = 360;
+				video.thumbnail.height = 202;
+			}
 			images.push(video.thumbnail);
 		});
 	}
@@ -80,15 +86,15 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 
 	if (images.length) {
 		try {
-			var {vrcurl: imagesheet_vrcurl} = await makeImageSheetVrcUrl(pool, images);
+			response.imagesheet_vrcurl = await putVrcUrl(pool, {type: "imagesheet", key});
-			data.imagesheet_vrcurl = imagesheet_vrcurl;
+			var imagesheet = createImageSheet(images, !playlistId && !options.icons);
 		} catch (error) {
 			console.error(error.stack);
 		}
 	}
 
 	for (let video of videos) {
-		video.vrcurl = await putVrcUrl(pool, {type: "redirect", url: `https://www.youtube.com/watch?v=${video.id}`});
+		video.vrcurl = await putVrcUrl(pool, {type: "video", id: video.id});
 		let thumbnail = images.find(image => image.url == video.thumbnail.url);
 		video.thumbnail = thumbnail ? {
 			x: thumbnail?.x,
@@ -107,15 +113,15 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 			video.captions_vrcurl = await putVrcUrl(pool, {type: "captions", videoId: video.id});
 		}
 		delete video.channel.iconUrl;
-		data.results.push(video);
+		response.results.push(video);
 	}
 
-	if (continuationData) data.nextpage_vrcurl = await putVrcUrl(pool, {
+	if (continuationData) response.nextpage_vrcurl = await putVrcUrl(pool, {
 		type: "continuation",
 		for: query.for || (playlistId ? "playlist" : "search"),
 		continuationData,
 		options
 	});
 
-	return data;
+	return {response, imagesheet};
 }

imagesheet.js

@@ -1,15 +1,21 @@
 import { createCanvas, loadImage } from 'canvas';
 import potpack from 'potpack';
-import { putVrcUrl } from './vrcurl.js';
 
-var store = {};
+export async function createImageSheet(images /*[{width, height, url}]*/, legacyMode) {
-
-async function createImageSheet(images /*[{width, height, url}]*/) {
 	images.forEach(image => {
 		image.w = image.width;
 		image.h = image.height;
 	});
-	var {w, h, fill} = potpack(images);
+	if (legacyMode) {
+		images.forEach((image, index) => {
+			image.x = index % 5 * 360;
+			image.y = Math.floor(index / 5) * 202;
+		});
+		var w = Math.min(images.length, 5) * 360;
+		var h = images.at(-1).y + 202;
+	} else {
+		var {w, h, fill} = potpack(images);
+	}
 	if (w > 2048) {
 		console.warn("Imagesheet exceeded max width");
 		w = 2048;
@@ -23,36 +29,14 @@ async function createImageSheet(images /*[{width, height, url}]*/) {
 
 	await Promise.all(images.map(({x, y, w, h, url}) => (async function(){
 		if (!url) return;
-		var image = await loadImage(url);
+		try {
+			var image = await loadImage(url);
+		} catch (error) {
+			console.error("failed to load image", url, error.message);
+			return;
+		}
 		ctx.drawImage(image, x, y, w, h);
-	})().catch(error => console.error(error.stack))));
+	})().catch(error => console.error("imageload", error.stack))));
 
-	return {
+	return canvas.toBuffer("image/png");
-		imagesheet: canvas.toBuffer("image/png"),
-		images
-	};
-}
-
-export async function makeImageSheetVrcUrl(pool, images) {
-	var num = await putVrcUrl(pool, {type: "imagesheet"});
-	var key = `${pool}:${num}`;
-	var promise = createImageSheet(images);
-	store[key] = promise;
-	promise.then(() => {
-		setTimeout(() => {
-			if (store[key] === promise) delete store[key];
-		}, 1000*60*10); // 10 mins;
-	});
-	promise.catch(error => {
-		console.error(error.stack);
-	});
-	var {thumbnails, icons} = await promise;
-	return {
-		vrcurl: num,
-		thumbnails, icons
-	}
-}
-
-export async function getImageSheet(pool, num) {
-	return (await store[`${pool}:${num}`])?.imagesheet;
 }

index.js

@@ -1,14 +1,16 @@
 if (process.env.D!="BUG") console.debug = () => {};
+else console.debug(process.env);
 import "./util.js";
 import Koa from "koa";
 import Router from "@koa/router";
 import send from "koa-send";
-import { cachedVRCYoutubeSearch } from "./VRCYoutubeSearch.js"
+import qs from "qs";
-import { getImageSheet } from "./imagesheet.js";
+import { cachedVRCYoutubeSearch, getImageSheet } from "./VRCYoutubeSearch.js"
 import { resolveVrcUrl } from "./vrcurl.js";
 import { getVideoCaptionsCached } from "./youtube-captions.js";
 import { stringToBoolean } from "./util.js";
-import shorturlmap from "./shorturlmap.json" assert { type: "json" };
+import { readFileSync } from "fs";
+var shorturlmap = JSON.parse(readFileSync("./shorturlmap.json", "utf8"));
 
 var app = new Koa();
 var router = new Router();
@@ -16,30 +18,34 @@ var router = new Router();
 
 router.get(["/search", "/trending"], async ctx => {
 	if (ctx.path == "/trending") {
-		var query = {"type":"trending"};
+		var input = {"type":"trending"};
 	} else {
-		var query = ctx.querystring.match(/[?&]input=(.*)/i)?.[1];
+		var input = ctx.querystring.match(/[?&]input=(.*)/i)?.[1];
-		if (!query) {
+		if (!input) {
 			ctx.status = 400;
 			ctx.body = "missing search query";
 			return;
 		}
-		query = decodeURIComponent(query).replace(/^.*→/, '').trim();
+		input = decodeURIComponent(input).replace(/^.*→/, '').replaceAll("\u200b", '').trim();
 	}
 
-	if (!ctx.query.pool || !/^[a-z-_]+\d*$/.test(ctx.query.pool)) {
+	var pqs = qs.parse(ctx.querystring, {duplicates: 'first'});
+
+	if (!pqs.pool || /[^a-z-_0-9]/.test(pqs.pool)) {
 		ctx.status = 400;
 		ctx.body = "invalid pool";
 		return;
 	}
 
 	var options = {
-		thumbnails: stringToBoolean(ctx.query.thumbnails),
+		thumbnails: stringToBoolean(pqs.thumbnails),
-		icons: stringToBoolean(ctx.query.icons),
+		icons: stringToBoolean(pqs.icons),
-		captions: stringToBoolean(ctx.query.captions)
+		captions: stringToBoolean(pqs.captions),
+		mode: pqs.mode,
+		bp: pqs.bp
 	};
 
-	ctx.body = await cachedVRCYoutubeSearch(ctx.query.pool, query, options);
+	ctx.body = await cachedVRCYoutubeSearch(pqs.pool, input, options);
 });
 
 
@@ -53,8 +59,15 @@ router.get("/vrcurl/:pool/:num", async ctx => {
 		case "redirect":
 			ctx.redirect(dest.url);
 			break;
+		case "video":
+			if (ctx.get("User-Agent").includes("UnityWebRequest")) {
+				ctx.body = {captions: await getVideoCaptionsCached(dest.id)};
+			} else {
+				ctx.redirect(`https://www.youtube.com/watch?v=${dest.id}`);
+			}
+			break;
 		case "imagesheet":
-			let buf = await getImageSheet(ctx.params.pool, ctx.params.num);
+			let buf = await getImageSheet(dest.key);
 			if (!buf) {
 				ctx.status = 404;
 				return;
@@ -94,6 +107,18 @@ router.get("/", ctx => {
 
 
 
+app.use(async (ctx, next) => {
+	try {
+		await next();
+	} catch (error) {
+		console.error(ctx.url, error.stack);
+		ctx.status = 500;
+		ctx.type = "text";
+		ctx.body = error.stack;
+	}
+});
+
+
 // short urls to work around https://feedback.vrchat.com/udon/p/vrcurlinputfield-incorrect-focus-issue-on-quest
 app.use(async (ctx, next) => {
 	var subdomain = ctx.hostname.match(/(.*).u2b.cx$/i)?.[1];

package-lock.json

@@ -4,15 +4,18 @@
 	"requires": true,
 	"packages": {
 		"": {
+			"name": "vrchat-youtube-search-api",
 			"dependencies": {
 				"@keyv/sqlite": "^3.6.6",
 				"@koa/router": "^12.0.1",
 				"canvas": "^2.11.2",
 				"fast-xml-parser": "^4.3.4",
+				"got": "^14.4.1",
 				"keyv": "^4.5.4",
 				"koa": "^2.14.2",
 				"koa-send": "^5.0.1",
-				"potpack": "^2.0.0"
+				"potpack": "^2.0.0",
+				"qs": "^6.12.2"
 			},
 			"engines": {
 				"node": ">=18.0.0"
@@ -135,6 +138,33 @@
 				"node": ">=10"
 			}
 		},
+		"node_modules/@sec-ant/readable-stream": {
+			"version": "0.4.1",
+			"resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz",
+			"integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg=="
+		},
+		"node_modules/@sindresorhus/is": {
+			"version": "6.3.1",
+			"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-6.3.1.tgz",
+			"integrity": "sha512-FX4MfcifwJyFOI2lPoX7PQxCqx8BG1HCho7WdiXwpEQx1Ycij0JxkfYtGK7yqNScrZGSlt6RE6sw8QYoH7eKnQ==",
+			"engines": {
+				"node": ">=16"
+			},
+			"funding": {
+				"url": "https://github.com/sindresorhus/is?sponsor=1"
+			}
+		},
+		"node_modules/@szmarczak/http-timer": {
+			"version": "5.0.1",
+			"resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
+			"integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
+			"dependencies": {
+				"defer-to-connect": "^2.0.1"
+			},
+			"engines": {
+				"node": ">=14.16"
+			}
+		},
 		"node_modules/@tootallnate/once": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz",
@@ -144,6 +174,11 @@
 				"node": ">= 6"
 			}
 		},
+		"node_modules/@types/http-cache-semantics": {
+			"version": "4.0.4",
+			"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz",
+			"integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA=="
+		},
 		"node_modules/abbrev": {
 			"version": "1.1.1",
 			"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -277,6 +312,75 @@
 				"node": ">= 6.0.0"
 			}
 		},
+		"node_modules/cacheable-lookup": {
+			"version": "7.0.0",
+			"resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz",
+			"integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==",
+			"engines": {
+				"node": ">=14.16"
+			}
+		},
+		"node_modules/cacheable-request": {
+			"version": "12.0.1",
+			"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-12.0.1.tgz",
+			"integrity": "sha512-Yo9wGIQUaAfIbk+qY0X4cDQgCosecfBe3V9NSyeY4qPC2SAkbCS4Xj79VP8WOzitpJUZKc/wsRCYF5ariDIwkg==",
+			"dependencies": {
+				"@types/http-cache-semantics": "^4.0.4",
+				"get-stream": "^9.0.1",
+				"http-cache-semantics": "^4.1.1",
+				"keyv": "^4.5.4",
+				"mimic-response": "^4.0.0",
+				"normalize-url": "^8.0.1",
+				"responselike": "^3.0.0"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/cacheable-request/node_modules/get-stream": {
+			"version": "9.0.1",
+			"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz",
+			"integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==",
+			"dependencies": {
+				"@sec-ant/readable-stream": "^0.4.1",
+				"is-stream": "^4.0.1"
+			},
+			"engines": {
+				"node": ">=18"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/cacheable-request/node_modules/mimic-response": {
+			"version": "4.0.0",
+			"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",
+			"integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",
+			"engines": {
+				"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/call-bind": {
+			"version": "1.0.7",
+			"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz",
+			"integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==",
+			"dependencies": {
+				"es-define-property": "^1.0.0",
+				"es-errors": "^1.3.0",
+				"function-bind": "^1.1.2",
+				"get-intrinsic": "^1.2.4",
+				"set-function-length": "^1.2.1"
+			},
+			"engines": {
+				"node": ">= 0.4"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
 		"node_modules/canvas": {
 			"version": "2.11.2",
 			"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz",
@@ -398,6 +502,30 @@
 			"resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
 			"integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw=="
 		},
+		"node_modules/defer-to-connect": {
+			"version": "2.0.1",
+			"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
+			"integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==",
+			"engines": {
+				"node": ">=10"
+			}
+		},
+		"node_modules/define-data-property": {
+			"version": "1.1.4",
+			"resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
+			"integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
+			"dependencies": {
+				"es-define-property": "^1.0.0",
+				"es-errors": "^1.3.0",
+				"gopd": "^1.0.1"
+			},
+			"engines": {
+				"node": ">= 0.4"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
 		"node_modules/delegates": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
@@ -482,15 +610,34 @@
 			"integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==",
 			"optional": true
 		},
+		"node_modules/es-define-property": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
+			"integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
+			"dependencies": {
+				"get-intrinsic": "^1.2.4"
+			},
+			"engines": {
+				"node": ">= 0.4"
+			}
+		},
+		"node_modules/es-errors": {
+			"version": "1.3.0",
+			"resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+			"integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+			"engines": {
+				"node": ">= 0.4"
+			}
+		},
 		"node_modules/escape-html": {
 			"version": "1.0.3",
 			"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
 			"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
 		},
 		"node_modules/fast-xml-parser": {
-			"version": "4.3.4",
+			"version": "4.5.0",
-			"resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.4.tgz",
+			"resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz",
-			"integrity": "sha512-utnwm92SyozgA3hhH2I8qldf2lBqm6qHOICawRNRFu1qMe3+oqr+GcXjGqTmXTMGE5T4eC03kr/rlh5C1IRdZA==",
+			"integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==",
 			"funding": [
 				{
 					"type": "github",
@@ -501,6 +648,7 @@
 					"url": "https://paypal.me/naturalintelligence"
 				}
 			],
+			"license": "MIT",
 			"dependencies": {
 				"strnum": "^1.0.5"
 			},
@@ -508,6 +656,14 @@
 				"fxparser": "src/cli/cli.js"
 			}
 		},
+		"node_modules/form-data-encoder": {
+			"version": "4.0.2",
+			"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.0.2.tgz",
+			"integrity": "sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==",
+			"engines": {
+				"node": ">= 18"
+			}
+		},
 		"node_modules/fresh": {
 			"version": "0.5.2",
 			"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
@@ -532,6 +688,14 @@
 			"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
 			"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
 		},
+		"node_modules/function-bind": {
+			"version": "1.1.2",
+			"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+			"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
 		"node_modules/gauge": {
 			"version": "3.0.2",
 			"resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
@@ -551,6 +715,35 @@
 				"node": ">=10"
 			}
 		},
+		"node_modules/get-intrinsic": {
+			"version": "1.2.4",
+			"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
+			"integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
+			"dependencies": {
+				"es-errors": "^1.3.0",
+				"function-bind": "^1.1.2",
+				"has-proto": "^1.0.1",
+				"has-symbols": "^1.0.3",
+				"hasown": "^2.0.0"
+			},
+			"engines": {
+				"node": ">= 0.4"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
+		"node_modules/get-stream": {
+			"version": "8.0.1",
+			"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz",
+			"integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==",
+			"engines": {
+				"node": ">=16"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/glob": {
 			"version": "7.2.3",
 			"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
@@ -570,12 +763,95 @@
 				"url": "https://github.com/sponsors/isaacs"
 			}
 		},
+		"node_modules/gopd": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+			"integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+			"dependencies": {
+				"get-intrinsic": "^1.1.3"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
+		"node_modules/got": {
+			"version": "14.4.1",
+			"resolved": "https://registry.npmjs.org/got/-/got-14.4.1.tgz",
+			"integrity": "sha512-IvDJbJBUeexX74xNQuMIVgCRRuNOm5wuK+OC3Dc2pnSoh1AOmgc7JVj7WC+cJ4u0aPcO9KZ2frTXcqK4W/5qTQ==",
+			"dependencies": {
+				"@sindresorhus/is": "^6.3.1",
+				"@szmarczak/http-timer": "^5.0.1",
+				"cacheable-lookup": "^7.0.0",
+				"cacheable-request": "^12.0.1",
+				"decompress-response": "^6.0.0",
+				"form-data-encoder": "^4.0.2",
+				"get-stream": "^8.0.1",
+				"http2-wrapper": "^2.2.1",
+				"lowercase-keys": "^3.0.0",
+				"p-cancelable": "^4.0.1",
+				"responselike": "^3.0.0",
+				"type-fest": "^4.19.0"
+			},
+			"engines": {
+				"node": ">=20"
+			},
+			"funding": {
+				"url": "https://github.com/sindresorhus/got?sponsor=1"
+			}
+		},
+		"node_modules/got/node_modules/decompress-response": {
+			"version": "6.0.0",
+			"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+			"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+			"dependencies": {
+				"mimic-response": "^3.1.0"
+			},
+			"engines": {
+				"node": ">=10"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/got/node_modules/mimic-response": {
+			"version": "3.1.0",
+			"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+			"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+			"engines": {
+				"node": ">=10"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/graceful-fs": {
 			"version": "4.2.11",
 			"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
 			"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
 			"optional": true
 		},
+		"node_modules/has-property-descriptors": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
+			"integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
+			"dependencies": {
+				"es-define-property": "^1.0.0"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
+		"node_modules/has-proto": {
+			"version": "1.0.3",
+			"resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz",
+			"integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==",
+			"engines": {
+				"node": ">= 0.4"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
 		"node_modules/has-symbols": {
 			"version": "1.0.3",
 			"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
@@ -606,6 +882,17 @@
 			"resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
 			"integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ=="
 		},
+		"node_modules/hasown": {
+			"version": "2.0.2",
+			"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+			"integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+			"dependencies": {
+				"function-bind": "^1.1.2"
+			},
+			"engines": {
+				"node": ">= 0.4"
+			}
+		},
 		"node_modules/http-assert": {
 			"version": "1.5.0",
 			"resolved": "https://registry.npmjs.org/http-assert/-/http-assert-1.5.0.tgz",
@@ -621,8 +908,7 @@
 		"node_modules/http-cache-semantics": {
 			"version": "4.1.1",
 			"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
-			"integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==",
+			"integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="
-			"optional": true
 		},
 		"node_modules/http-errors": {
 			"version": "1.8.1",
@@ -661,6 +947,18 @@
 				"node": ">= 6"
 			}
 		},
+		"node_modules/http2-wrapper": {
+			"version": "2.2.1",
+			"resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz",
+			"integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==",
+			"dependencies": {
+				"quick-lru": "^5.1.1",
+				"resolve-alpn": "^1.2.0"
+			},
+			"engines": {
+				"node": ">=10.19.0"
+			}
+		},
 		"node_modules/https-proxy-agent": {
 			"version": "5.0.1",
 			"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
@@ -761,6 +1059,17 @@
 			"integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==",
 			"optional": true
 		},
+		"node_modules/is-stream": {
+			"version": "4.0.1",
+			"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz",
+			"integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==",
+			"engines": {
+				"node": ">=18"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/isexe": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -860,6 +1169,17 @@
 				"node": ">= 8"
 			}
 		},
+		"node_modules/lowercase-keys": {
+			"version": "3.0.0",
+			"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
+			"integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",
+			"engines": {
+				"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/lru-cache": {
 			"version": "6.0.0",
 			"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
@@ -1184,6 +1504,17 @@
 				"node": ">=6"
 			}
 		},
+		"node_modules/normalize-url": {
+			"version": "8.0.1",
+			"resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.1.tgz",
+			"integrity": "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==",
+			"engines": {
+				"node": ">=14.16"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/npmlog": {
 			"version": "5.0.1",
 			"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
@@ -1203,6 +1534,17 @@
 				"node": ">=0.10.0"
 			}
 		},
+		"node_modules/object-inspect": {
+			"version": "1.13.2",
+			"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz",
+			"integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==",
+			"engines": {
+				"node": ">= 0.4"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
 		"node_modules/on-finished": {
 			"version": "2.4.1",
 			"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@@ -1227,6 +1569,14 @@
 			"resolved": "https://registry.npmjs.org/only/-/only-0.0.2.tgz",
 			"integrity": "sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ=="
 		},
+		"node_modules/p-cancelable": {
+			"version": "4.0.1",
+			"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-4.0.1.tgz",
+			"integrity": "sha512-wBowNApzd45EIKdO1LaU+LrMBwAcjfPaYtVzV3lmfM3gf8Z4CHZsiIqlM8TZZ8okYvh5A1cP6gTfCRQtwUpaUg==",
+			"engines": {
+				"node": ">=14.16"
+			}
+		},
 		"node_modules/p-map": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
@@ -1287,6 +1637,31 @@
 				"node": ">=10"
 			}
 		},
+		"node_modules/qs": {
+			"version": "6.12.2",
+			"resolved": "https://registry.npmjs.org/qs/-/qs-6.12.2.tgz",
+			"integrity": "sha512-x+NLUpx9SYrcwXtX7ob1gnkSems4i/mGZX5SlYxwIau6RrUSODO89TR/XDGGpn5RPWSYIB+aSfuSlV5+CmbTBg==",
+			"dependencies": {
+				"side-channel": "^1.0.6"
+			},
+			"engines": {
+				"node": ">=0.6"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
+		"node_modules/quick-lru": {
+			"version": "5.1.1",
+			"resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
+			"integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==",
+			"engines": {
+				"node": ">=10"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/readable-stream": {
 			"version": "3.6.2",
 			"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -1300,6 +1675,11 @@
 				"node": ">= 6"
 			}
 		},
+		"node_modules/resolve-alpn": {
+			"version": "1.2.1",
+			"resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
+			"integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g=="
+		},
 		"node_modules/resolve-path": {
 			"version": "1.4.0",
 			"resolved": "https://registry.npmjs.org/resolve-path/-/resolve-path-1.4.0.tgz",
@@ -1344,6 +1724,20 @@
 			"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz",
 			"integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ=="
 		},
+		"node_modules/responselike": {
+			"version": "3.0.0",
+			"resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",
+			"integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",
+			"dependencies": {
+				"lowercase-keys": "^3.0.0"
+			},
+			"engines": {
+				"node": ">=14.16"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/retry": {
 			"version": "0.12.0",
 			"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
@@ -1411,11 +1805,44 @@
 			"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
 			"integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw=="
 		},
+		"node_modules/set-function-length": {
+			"version": "1.2.2",
+			"resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
+			"integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==",
+			"dependencies": {
+				"define-data-property": "^1.1.4",
+				"es-errors": "^1.3.0",
+				"function-bind": "^1.1.2",
+				"get-intrinsic": "^1.2.4",
+				"gopd": "^1.0.1",
+				"has-property-descriptors": "^1.0.2"
+			},
+			"engines": {
+				"node": ">= 0.4"
+			}
+		},
 		"node_modules/setprototypeof": {
 			"version": "1.2.0",
 			"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
 			"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
 		},
+		"node_modules/side-channel": {
+			"version": "1.0.6",
+			"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
+			"integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==",
+			"dependencies": {
+				"call-bind": "^1.0.7",
+				"es-errors": "^1.3.0",
+				"get-intrinsic": "^1.2.4",
+				"object-inspect": "^1.13.1"
+			},
+			"engines": {
+				"node": ">= 0.4"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
 		"node_modules/signal-exit": {
 			"version": "3.0.7",
 			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -1574,9 +2001,10 @@
 			"integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA=="
 		},
 		"node_modules/tar": {
-			"version": "6.2.0",
+			"version": "6.2.1",
-			"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz",
+			"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz",
-			"integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==",
+			"integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==",
+			"license": "ISC",
 			"dependencies": {
 				"chownr": "^2.0.0",
 				"fs-minipass": "^2.0.0",
@@ -1618,6 +2046,17 @@
 				"node": ">=0.6.x"
 			}
 		},
+		"node_modules/type-fest": {
+			"version": "4.21.0",
+			"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.21.0.tgz",
+			"integrity": "sha512-ADn2w7hVPcK6w1I0uWnM//y1rLXZhzB9mr0a3OirzclKF1Wp6VzevUmzz/NRAWunOT6E8HrnpGY7xOfc6K57fA==",
+			"engines": {
+				"node": ">=16"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/type-is": {
 			"version": "1.6.18",
 			"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",

package.json

@@ -4,10 +4,12 @@
 		"@koa/router": "^12.0.1",
 		"canvas": "^2.11.2",
 		"fast-xml-parser": "^4.3.4",
+		"got": "^14.4.1",
 		"keyv": "^4.5.4",
 		"koa": "^2.14.2",
 		"koa-send": "^5.0.1",
-		"potpack": "^2.0.0"
+		"potpack": "^2.0.0",
+		"qs": "^6.12.2"
 	},
 	"engines": {
 		"node": ">=18.0.0"

test.html

@@ -14,7 +14,7 @@
 	<br />
 	<label><input id="thumbnails" type="checkbox" checked>thumbnails</label>
 	<label><input id="icons" type="checkbox" checked>icons</label>
-	<label><input id="captions" type="checkbox" checked>captions</label>
+	<label><input id="latestontop" type="checkbox" checked>latestontop</label>
 </div>
 
 <div id="output"></div>
@@ -30,11 +30,11 @@ var lastData;
 
 search.onclick = () => {
 	output.innerHTML = "";
-	loadData(`/search?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&captions=${captions.checked}&input=${encodeURIComponent(input.value)}`);
+	loadData(`/search?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&mode=${latestontop.checked&&"latestontop"}&input=${encodeURIComponent(input.value)}`);
 };
 trending.onclick = () => {
 	output.innerHTML = "";
-	loadData(`/trending?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&captions=${captions.checked}`);
+	loadData(`/trending?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}`);
 }
 
 async function loadData(url) {

util.js

@@ -1,3 +1,6 @@
+import {readFileSync} from "fs";
+import got from "got";
+
 Array.prototype.filterMap = function(fn) {
 	var newarray = [];
 	for (var item of this) {
@@ -7,6 +10,23 @@ Array.prototype.filterMap = function(fn) {
 	return newarray;
 };
 
+Array.prototype.findMap = function(fn) {
+	for (var item of this) {
+		var ret = fn(item);
+		if (ret) return ret;
+	}
+}
+
+Object.prototype.deepFind = function (fn) {
+	return (function crawlObject(object) {
+		for (var key in object) {
+			var value = object[key];
+			if (fn(value)) return value;
+			if (typeof value == "object") crawlObject(value);
+		}
+	})(this);
+};
+
 export function stringToBoolean(str) {
 	if (str) {
 		if (!["0", "false", "off", "no", "null", "undefined", "nan"].includes(str.toLowerCase())) return true;
@@ -14,15 +34,16 @@ export function stringToBoolean(str) {
 	return false;
 }
 
-export function recursiveFind(object, fn) {
+
-	var results = [];
+
-	(function crawlObject(object) {
+
-		for (var key in object) {
+try {
-			var value = object[key];
+	var ips = readFileSync("ips.txt", "utf8").trim().split("\n");
-			var test = fn(value);
+	console.log("using ips", ips);
-			if (test) results.push(test);
+} catch (e) {}
-			if (typeof value == "object") crawlObject(value);
+
-		}
+export function gotw(url, options = {}) {
-	})(object);
+	if (ips) options.localAddress = ips[Math.floor(Math.random() * ips.length)];
-	return results;
+	//options.timeout = {request: 3000}; //"RequestError: Expected values which are `number` or `undefined`. Received values of type `Function`." what the fuck?
+	return got(url, options);
 }

youtube-captions.js

@@ -1,13 +1,14 @@
 import { XMLParser } from "fast-xml-parser";
+import { gotw } from "./util.js";
 
 var xmlParser = new XMLParser({
 	ignoreAttributes: false
 });
 
 async function getVideoData(videoId) {
-	var html = await fetch(`https://www.youtube.com/watch?v=${videoId}`).then(res => res.text());
+	var res = await gotw(`https://www.youtube.com/watch?v=${videoId}`);
 
-	var ytInitialPlayerResponse = html.match(/var ytInitialPlayerResponse = ({.*});/)[1];
+	var ytInitialPlayerResponse = res.body.match(/var ytInitialPlayerResponse = ({.*});/)[1];
 	ytInitialPlayerResponse = JSON.parse(ytInitialPlayerResponse);
 
 	return ytInitialPlayerResponse;
@@ -18,15 +19,21 @@ async function getVideoCaptions(videoId) {
 	if (!ytInitialPlayerResponse.captions) return [];
 	var captionTracks = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks;
 	captionTracks = await Promise.all(captionTracks.map(captionTrack => (async () => {
-		var xml = await fetch(captionTrack.baseUrl).then(res => res.text());
+		try {
-		var parsed = xmlParser.parse(xml);
+			var xml = await gotw(captionTrack.baseUrl, {resolveBodyOnly: true});
-		var lines = parsed.transcript.text.map(({ "#text": text, "@_start": start, "@_dur": dur }) => ({ start: Number(start), dur: Number(dur), text }));
+			var parsed = xmlParser.parse(xml);
-		return {
+			var lines = parsed.transcript.text;
-			name: captionTrack.name.simpleText,
+			if (!Array.isArray(lines)) lines = [lines];
-			id: captionTrack.vssId,
+			lines = lines.map(({ "#text": text, "@_start": start, "@_dur": dur }) => ({ start: Number(start), dur: Number(dur), text }));
-			lines
+			return {
-		};
+				name: captionTrack.name.simpleText,
-	})().catch(error => console.error(error.stack))));
+				id: captionTrack.vssId,
+				lines
+			};
+		} catch (error) {
+			console.error("caption track error", error.stack, videoId, captionTrack, xml, parsed, lines);
+		}
+	})()));
 	return captionTracks;
 }
 
@@ -37,7 +44,7 @@ export async function getVideoCaptionsCached(videoId) {
 		cache[videoId] = getVideoCaptions(videoId);
 		setTimeout(() => {
 			delete cache[videoId];
-		}, 1000*60*60*6); // 6 hours
+		}, 1000*60*10); // 10 minutes
 	}
 	return await cache[videoId];
 }

youtube.js

@@ -1,14 +1,49 @@
+import { gotw } from "./util.js";
 
-export async function searchYouTubeVideos(query) {
-	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}&sp=EgIQAQ%253D%253D`;
-	var html = await fetch(url).then(res => res.text());
 
-	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+
+
+export async function searchYouTubeVideos(query, sp = "EgIQAQ%253D%253D") {
+	console.debug("sp", sp);
+	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}${sp ? `$sp=${sp}` : ''}`;
+	var res = await gotw(url), html = res.body;
+ 
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)?.[1];
+	if (!ytInitialData) {
+		console.error("missing ytInitialData", query, res.status, html);
+	}
 	ytInitialData = JSON.parse(ytInitialData);
 	console.debug(ytInitialData);
 
-	var videos = ytInitialData?.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))?.itemSectionRenderer?.contents?.filterMap(x => x.videoRenderer)?.map(parseVideoRendererData);
+	var sectionListRendererContents = ytInitialData
+		.contents
+		.twoColumnSearchResultsRenderer
+		.primaryContents
+		.sectionListRenderer
+		.contents;
+
+	var videos = sectionListRendererContents
+		?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))
+		?.itemSectionRenderer
+		.contents
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
 	if (!videos) return {videos: []};
+
+	var latest = sectionListRendererContents
+		.find(x => x.itemSectionRenderer?.contents?.find(x => x.shelfRenderer))
+		?.itemSectionRenderer
+		.contents
+		.find(x => x.shelfRenderer?.title?.simpleText?.startsWith("Latest from"))
+		?.shelfRenderer
+		.content
+		.verticalListRenderer
+		.items
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
+	console.debug("latest", latest);
+	if (latest) videos = [...latest, ...videos];
+
 	console.debug(videos.length, "results");
 
 	try {
@@ -16,7 +51,7 @@ export async function searchYouTubeVideos(query) {
 		ytcfg = JSON.parse(ytcfg);
 		var continuationData = {
 			context: ytcfg.INNERTUBE_CONTEXT,
-			continuation: ytInitialData.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
+			continuation: sectionListRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.continuationCommand.token
 		}
 	} catch (error) {
 		console.error(error.stack);
@@ -25,19 +60,33 @@ export async function searchYouTubeVideos(query) {
 	return {videos, continuationData};
 }
 
+
+
+
 export async function continueYouTubeVideoSearch(continuationData) {
-	var data = await fetch("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
+	var res = await gotw("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
 		method: "POST",
-		headers: {
+		json: continuationData,
-			"Content-Type": "application/json"
+		responseType: "json"
-		},
+	});
-		body: JSON.stringify(continuationData)
+	var data = res.body;
-	}).then(res => res.json());
 	console.debug(data);
 
-	var continuationItems = data.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems;
+	var continuationItems = data
-	var videos = continuationItems.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer)).itemSectionRenderer.contents.filterMap(x => x.videoRenderer).map(parseVideoRendererData);
+		.onResponseReceivedCommands[0]
-	var continuationToken = continuationItems.find(x => x.continuationItemRenderer)?.continuationItemRenderer.continuationEndpoint.continuationCommand.token
+		.appendContinuationItemsAction
+		.continuationItems;
+	var videos = continuationItems
+		.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer))
+		.itemSectionRenderer
+		.contents
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
+	var continuationToken = continuationItems
+		.findMap(x => x.continuationItemRenderer)
+		?.continuationEndpoint
+		.continuationCommand
+		.token;
 	console.debug(videos.length, "results");
 
 
@@ -52,14 +101,33 @@ export async function continueYouTubeVideoSearch(continuationData) {
 
 
 
+
+
+
+
+
+
 export async function getYouTubePlaylist(playlistId) {
-	var html = await fetch("https://www.youtube.com/playlist?list=" + playlistId).then(res => res.text());
+	var res = await gotw("https://www.youtube.com/playlist?list=" + playlistId);
+	var html = res.body;
 	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
 	ytInitialData = JSON.parse(ytInitialData);
 	console.debug(ytInitialData);
 
-	var sectionListRendererContents = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.find(tab => tab.tabRenderer.selected).tabRenderer.content.sectionListRenderer.contents;
+	var sectionListRendererContents = ytInitialData
-	var videos = sectionListRendererContents.find(x => x.itemSectionRenderer).itemSectionRenderer.contents.find(x => x.playlistVideoListRenderer).playlistVideoListRenderer.contents.filterMap(x => x.playlistVideoRenderer).map(parseVideoRendererData);
+		.contents
+		.twoColumnBrowseResultsRenderer
+		.tabs
+		.find(tab => tab.tabRenderer.selected)
+		.tabRenderer
+		.content
+		.sectionListRenderer
+		.contents;
+	var videos = sectionListRendererContents
+		.findMap(x => x.itemSectionRenderer?.contents)
+		.findMap(x => x.playlistVideoListRenderer?.contents)
+		.filterMap(x => x.playlistVideoRenderer)
+		.map(parseVideoRendererData);
 	if (!videos) return {videos: []};
 	console.debug(videos.length, "results");
 
@@ -68,7 +136,7 @@ export async function getYouTubePlaylist(playlistId) {
 		ytcfg = JSON.parse(ytcfg);
 		var continuationData = {
 			context: ytcfg.INNERTUBE_CONTEXT,
-			continuation: sectionListRendererContents.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
+			continuation: sectionListRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.continuationCommand.token
 		}
 	} catch (error) {
 		console.error(error.stack);
@@ -76,18 +144,32 @@ export async function getYouTubePlaylist(playlistId) {
 	return {videos, continuationData};
 }
 
+
+
+
 export async function continueYouTubePlaylist(continuationData) {
-	var data = await fetch("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false", {
+	var res = await gotw("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false", {
 		method: "POST",
-		headers: {"Content-Type": "application/json"},
+		json: continuationData,
-		body: JSON.stringify(continuationData)
+		responseType: "json"
-	}).then(res => res.json());
+	});
+	var data = res.body;
 	console.debug(data);
 
 	if (!data.onResponseReceivedActions) return {videos:[]};
-	var continuationItems = data.onResponseReceivedActions[0].appendContinuationItemsAction.continuationItems;
+	var continuationItems = data
-	var videos = continuationItems.find(x => x.itemSectionRenderer).itemSectionRenderer.contents.filterMap(x => x.playlistVideoListRenderer).map(parseVideoRendererData);
+		.onResponseReceivedActions[0]
-	var continuationToken = continuationItems.find(x => x.continuationItemRenderer)?.continuationItemRenderer.continuationEndpoint.continuationCommand.token;
+		.appendContinuationItemsAction
+		.continuationItems;
+	var videos = continuationItems
+		.findMap(x => x.itemSectionRenderer?.contents)
+		.filterMap(x => x.playlistVideoListRenderer)
+		.map(parseVideoRendererData);
+	var continuationToken = continuationItems
+		.findMap(x => x.continuationItemRenderer)
+		?.continuationEndpoint
+		.continuationCommand
+		.token;
 	console.debug(videos.length, "results");
 
 	return {
@@ -110,7 +192,8 @@ export async function continueYouTubePlaylist(continuationData) {
 export async function getTrending(bp) {
 	var url = `https://www.youtube.com/feed/trending`;
 	if (bp) url += `?bp=${bp}`;
-	var html = await fetch(url).then(res => res.text());
+	var res = await gotw(url);
+	var html = res.body;
 	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
 	ytInitialData = JSON.parse(ytInitialData);
 
@@ -133,7 +216,7 @@ export async function getTrending(bp) {
 		.contents
 		// regular trending in sections with shelfRenderer without title
 		.filterMap(x => {
-			var shelfRenderer = x.itemSectionRenderer.contents.find(x => x.shelfRenderer)?.shelfRenderer;
+			var shelfRenderer = x.itemSectionRenderer.contents.findMap(x => x.shelfRenderer);
 			if (shelfRenderer && !shelfRenderer.title) {
 				return shelfRenderer
 					.content
@@ -172,7 +255,7 @@ function parseVideoRendererData(data) {
 		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.concatRunsText() 
 			|| data.descriptionSnippet?.runs?.concatRunsText(),
 		//thumbnailUrl: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138))?.url || data.thumbnail?.thumbnails?.[0]?.url,
-		//thumbnail: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138)) || data.thumbnail?.thumbnails?.[0],
+		thumbnail: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138)) || data.thumbnail?.thumbnails?.[0],
 		/*thumbnail: {
 			url: `https://i.ytimg.com/vi/${data.videoId}/mqdefault.jpg`,
 			width: 320,