youtube antibotted the captions

gives blank file now without special code added to url

.gitignore

@@ -1,2 +1,4 @@
 node_modules
 *.sqlite
+*.sqlite-journal
+ips.txt

.vscode/launch.json

@@ -14,7 +14,10 @@
 			"program": "${workspaceFolder}\\index.js",
 			"outFiles": [
 				"${workspaceFolder}/**/*.js"
-			]
+			],
+			"env": {
+				"D":"BUG"
+			}
 		}
 	]
 }

README.md

@@ -13,13 +13,14 @@ Get YouTube videos for a search query.
 ### Required query parameters
 
 - `pool`: id of the VRCUrl pool, only letters numbers hyphens or underscores, optionally followed by an integer for pool size.
-- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed.
+- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed. THIS MUST BE THE LAST QUERY PARAMETER AS ALL CHARS AFTER IT ARE CAPTURED VERBATIM (so you can type & etc without encoding). If this contains a url with a playlist ID, playlist results will be loaded instead (which is a bit different, up to 100 results and no descriptions).
 
 ### Optional query parameters
 
 - `thumbnails`: set to `1`, `true`, `yes`, `on` or whatever to load thumbnails
 - `icons`: set to `1`, `true`, `yes`, `on` or whatever to load channel icons
-- `captions`: set to `1`, `true`, `yes`, `on` or whatever if you need access to closed captioning data
+- `mode`: If set to `latestontop` will search without filter to include the "Latest from ..." section (if it exists). Otherwise it uses search filter for Videos only sorted by Relevance.
+- `bp`: Custom YouTube search filter. Go to youtube search site and set some filters and you will see the corresponding `bp` value in the URL. Overrides `mode` option. Use at your own risk (experimental).
 
 ### Example URL
 
@@ -32,11 +33,11 @@ https://api.u2b.cx/search?pool=example10000&input=   Type YouTube search query h
 JSON object:
 
 - `results`: Array of Object
-	- `vrcurl`: (integer) index of VRCUrl that will redirect to the youtube url
+	- `vrcurl`: (integer) index of VRCUrl that will redirect to the youtube url, or serve JSON with captions if used with string loader.
 	- `live`: (boolean) whether it's a live stream
 	- `title`: (string) i.e. `"Nyan Cat! [Official]"`
 	- `id`: (string) YouTube video id i.e. `"2yJgwwDcgV8"`
-	- `description`: (string) short truncated description snippet i.e. `"http://nyan.cat/ Original song : http://momolabo.lolipop.jp/nyancatsong/Nyan/"`
+	- `description`?: (string) short truncated description snippet i.e. `"http://nyan.cat/ Original song : http://momolabo.lolipop.jp/nyancatsong/Nyan/"` (playlist results don't have this)
 	- `lengthText`?: (string) i.e. `"3:37"`
 	- `longLengthText`?: (string) i.e. `"3 minutes, 37 seconds"`
 	- `viewCountText`?: (string) i.e. `"2,552,243 views"` or `"575 watching"` for live streams (playlist results don't have this)
@@ -55,7 +56,6 @@ JSON object:
 		- `y`: (integer) px from top
 		- `width`: (integer)
 		- `height`: (integer)
-	- `captions_vrcurl`?: (integer) index of vrcurl to get the caption data json
 - `imagesheet_vrcurl`?: (integer) index of the vrcurl for the collage of thumbnails and/or icons
 - `nextpage_vrcurl`: (integer) index of the vrcurl that will serve the JSON for the next page of results
 
@@ -71,13 +71,15 @@ Gets Trending YouTube videos. Identical to `/search` but without `input` paramet
 ## GET `/vrcurl/{pool}/{index}`
 
 - `{pool}`: must be same as pool param in search endpoint.
-- `{index}`: vrcurl index number
+- `{index}`: vrcurl index number as specified in response data
 
-Response may be 302 redirect to youtube url, `image/png` for imagesheet, `application/json` for next page (see response format above) or trending tab or caption data (below).
+For youtube videos, if `User-Agent` header includes `UnityWebRequest`, then response is `application/json` for video metadata (captions). Otherwise, it is 302 redirect to youtube URL.
 
-### Caption JSON format
+For image sheet, response is `image/png`. For next page or trending tab, response is `application/json`.
 
-- Array of Object
+### Video metadata JSON format
+
+- `captions`: Array of Object
 	- `name`: (string) caption track name like "English" or "English (auto-generated)"
 	- `id`: (string) id like `.en` or `a.en`
 	- `lines`: Array of Object
@@ -108,4 +110,4 @@ All resources (youtube urls etc) referenced in the search results will be substi
 
 Video thumbnails and channel icons are collated together into one image and served at a VRCUrl to be loaded by VRCImageDownloader.
 
-Use the x, y, width and height values from the json to crop the image from the sheet.
+Use the x, y, width and height values from the json to crop the image from the sheet. Do not make any assumptions about these values as the server could arrange the images wherever it wants.

VRCYoutubeSearch.js

@@ -1,44 +1,42 @@
-import { searchYouTubeVideos, continueYouTubeVideoSearch, getYouTubePlaylist, continueYouTubePlaylist } from "./simpleYoutubeSearch.js";
+import { searchYouTubeVideos, continueYouTubeVideoSearch, getYouTubePlaylist, continueYouTubePlaylist, getTrending } from "./youtube.js";
 import { putVrcUrl } from "./vrcurl.js";
-import { makeImageSheetVrcUrl } from "./imagesheet.js";
-import { getTrending } from "./trending.js";
+import { createImageSheet } from "./imagesheet.js";
 
 var cache = {};
 
-
-
 export async function cachedVRCYoutubeSearch(pool, query, options) {
 	var key = JSON.stringify([pool, query, options]);
 	if (!cache[key]) {
-		cache[key] = VRCYoutubeSearch(pool, query, options);
+		cache[key] = VRCYoutubeSearch(pool, query, options, key);
 		setTimeout(() => {
 			delete cache[key];
 		}, 1000*60*10); // 10 mins
 	}
-	return await cache[key];
+	return (await cache[key])?.response;
+}
+
+export async function getImageSheet(key) {
+	return await (await cache[key])?.imagesheet
 }
 
 
-
-
-async function VRCYoutubeSearch(pool, query, options = {}) {
-	console.debug("search:", JSON.stringify(query));
-	var data = {results: []};
+async function VRCYoutubeSearch(pool, query, options = {}, key) {
+	console.log("search", pool, JSON.stringify(query), JSON.stringify(options));
+	var response = {results: []};
 
 	if (typeof query == "object") {
 		switch (query.type) {
 			case "trending":
 				var {videos, tabs} = await getTrending(query.bp);
-				data.tabs = [];
+				response.tabs = [];
 				for (let tab of tabs) {
-					data.tabs.push({
+					response.tabs.push({
 						name: tab.name,
 						vrcurl: await putVrcUrl(pool, {type: "trending", bp: tab.bp, options})
 					});
 				}
 				break;
 			case "continuation":
-				//var {videos, continuationData} = await [query.for == "playlist" ? continueYouTubePlaylist : continueYouTubeVideoSearch](query.continuationData);
 				if (query.for == "playlist") {
 					var {videos, continuationData} = await continueYouTubePlaylist(query.continuationData);
 				} else {
@@ -47,15 +45,32 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 				break;
 		}
 	} else {
-		var playlistId = query.match(/list=(PL[a-zA-Z0-9-_]{32})/)?.[1];
-		if (playlistId) console.debug("playlistId:", playlistId);
-		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query);
+		var playlistId = query.match(/list=(PL(?:[a-zA-Z0-9-_]{32}|[0-9A-F]{16}))/)?.[1];
+		if (playlistId) console.log("playlistId:", playlistId);
+		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query, options.bp || (options.mode == "latestontop" ? null : undefined));
 	}
 	
 	var images = [];
 
 	if (options.thumbnails) {
-		videos.forEach(video => video.thumbnail.url && images.push(video.thumbnail));
+		videos.forEach(video => {
+			if (playlistId || query?.for == "playlist") video.thumbnail = {
+				url: `https://i.ytimg.com/vi/${video.id}/default.jpg`,
+				width: 120,
+				height: 90
+			};
+			else {
+				video.thumbnail ||= {
+					url: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`,
+					width: 320,
+					height: 180
+				};
+				console.debug(video.thumbnail);
+				video.thumbnail.width = 360;
+				video.thumbnail.height = 202;
+			}
+			images.push(video.thumbnail);
+		});
 	}
 
 	if (options.icons) {
@@ -70,15 +85,15 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 
 	if (images.length) {
 		try {
-			var {vrcurl: imagesheet_vrcurl} = await makeImageSheetVrcUrl(pool, images);
-			data.imagesheet_vrcurl = imagesheet_vrcurl;
+			response.imagesheet_vrcurl = await putVrcUrl(pool, {type: "imagesheet", key});
+			var imagesheet = createImageSheet(images, !playlistId && !options.icons);
 		} catch (error) {
 			console.error(error.stack);
 		}
 	}
 
 	for (let video of videos) {
-		video.vrcurl = await putVrcUrl(pool, {type: "redirect", url: `https://www.youtube.com/watch?v=${video.id}`});
+		video.vrcurl = await putVrcUrl(pool, {type: "video", id: video.id});
 		let thumbnail = images.find(image => image.url == video.thumbnail.url);
 		video.thumbnail = thumbnail ? {
 			x: thumbnail?.x,
@@ -97,15 +112,15 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 			video.captions_vrcurl = await putVrcUrl(pool, {type: "captions", videoId: video.id});
 		}
 		delete video.channel.iconUrl;
-		data.results.push(video);
+		response.results.push(video);
 	}
 
-	if (continuationData) data.nextpage_vrcurl = await putVrcUrl(pool, {
+	if (continuationData) response.nextpage_vrcurl = await putVrcUrl(pool, {
 		type: "continuation",
 		for: query.for || (playlistId ? "playlist" : "search"),
 		continuationData,
 		options
 	});
 
-	return data;
+	return {response, imagesheet};
 }

app.js

@@ -1,123 +0,0 @@
-import Koa from "koa";
-import Router from "@koa/router";
-import send from "koa-send";
-import { cachedVRCYoutubeSearch } from "./VRCYoutubeSearch.js"
-import { getImageSheet } from "./imagesheet.js";
-import { resolveVrcUrl } from "./vrcurl.js";
-import { getVideoCaptionsCached } from "./captions.js";
-import { stringToBoolean } from "./util.js";
-import shorturlmap from "./shorturlmap.json" assert { type: "json" };
-
-export var app = new Koa();
-var router = new Router();
-
-
-router.get(["/search", "/trending"], async ctx => {
-	var query = ctx.path == "/trending" ? {"type":"trending"} : ctx.query.input?.replace(/^.*→/, '').trim();
-	if (!query) {
-		ctx.status = 400;
-		ctx.body = "missing search query";
-		return;
-	}
-
-	if (!ctx.query.pool || !/^[a-z-_]+\d*$/.test(ctx.query.pool)) {
-		ctx.status = 400;
-		ctx.body = "invalid pool";
-		return;
-	}
-
-	var options = {
-		thumbnails: stringToBoolean(ctx.query.thumbnails),
-		icons: stringToBoolean(ctx.query.icons),
-		captions: stringToBoolean(ctx.query.captions)
-	};
-
-	ctx.body = await cachedVRCYoutubeSearch(ctx.query.pool, query, options);
-});
-
-
-router.get("/vrcurl/:pool/:num", async ctx => {
-	var dest = await resolveVrcUrl(ctx.params.pool, ctx.params.num);
-	if (!dest) {
-		ctx.status = 404;
-		return;
-	}
-	switch (dest.type) {
-		case "redirect":
-			ctx.redirect(dest.url);
-			break;
-		case "imagesheet":
-			let buf = await getImageSheet(ctx.params.pool, ctx.params.num);
-			if (!buf) {
-				ctx.status = 404;
-				return;
-			}
-			ctx.body = buf;
-			ctx.type = "image/png";
-			break;
-		case "continuation":
-			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "continuation", for: dest.for, continuationData: dest.continuationData}, dest.options);
-			break;
-		case "trending":
-			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "trending", bp: dest.bp}, dest.options);
-			break;
-		case "captions":
-			ctx.body = await getVideoCaptionsCached(dest.videoId);
-			break;
-		default:
-			console.error("unknown vrcurl type", dest.type);
-			ctx.status = 500;
-	}
-});
-
-
-router.get("/robots.txt", ctx => {
-	ctx.body = `User-agent: *\nDisallow: /`;
-});
-
-router.get("/test.html", async ctx => {
-	await send(ctx, "test.html");
-});
-
-router.get("/", ctx => {
-	ctx.redirect("https://www.u2b.cx/");
-});
-
-
-
-
-
-// short urls to work around https://feedback.vrchat.com/udon/p/vrcurlinputfield-incorrect-focus-issue-on-quest
-app.use(async (ctx, next) => {
-	var subdomain = ctx.hostname.match(/(.*).u2b.cx$/i)?.[1];
-	if (subdomain && !["api","api2","dev"].includes(subdomain)) {
-		if (shorturlmap[subdomain]) {
-			ctx.url = shorturlmap[subdomain] + ctx.url.slice(1);
-		} else {
-			ctx.status = 404;
-			return;
-		}
-	}
-	await next();
-});
-
-// work around vrchat json parser bug https://feedback.vrchat.com/udon/p/braces-inside-strings-in-vrcjson-can-fail-to-deserialize
-app.use(async (ctx, next) => {
-	await next();
-	if (ctx.type != "application/json") return;
-	ctx.body = structuredClone(ctx.body);
-	(function iterateObject(obj) {
-		for (var key in obj) {
-			if (typeof obj[key] == "string") {
-				obj[key] = obj[key].replace(/[\[\]{}]/g, chr => "\\u" + chr.charCodeAt(0).toString(16).padStart(4, '0'));
-			} else if (typeof obj[key] == "object") {
-				iterateObject(obj[key]);
-			}
-		}
-	})(ctx.body);
-	ctx.body = JSON.stringify(ctx.body).replaceAll("\\\\u", "\\u");
-});
-
-
-app.use(router.routes());
-app.use(router.allowedMethods());

captions.js

@@ -1,43 +0,0 @@
-import { XMLParser } from "fast-xml-parser";
-
-var xmlParser = new XMLParser({
-	ignoreAttributes: false
-});
-
-async function getVideoData(videoId) {
-	var html = await fetch(`https://www.youtube.com/watch?v=${videoId}`).then(res => res.text());
-
-	var ytInitialPlayerResponse = html.match(/var ytInitialPlayerResponse = ({.*});/)[1];
-	ytInitialPlayerResponse = JSON.parse(ytInitialPlayerResponse);
-
-	return ytInitialPlayerResponse;
-}
-
-async function getVideoCaptions(videoId) {
-	var ytInitialPlayerResponse = await getVideoData(videoId);
-	if (!ytInitialPlayerResponse.captions) return [];
-	var captionTracks = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks;
-	captionTracks = await Promise.all(captionTracks.map(captionTrack => (async () => {
-		var xml = await fetch(captionTrack.baseUrl).then(res => res.text());
-		var parsed = xmlParser.parse(xml);
-		var lines = parsed.transcript.text.map(({ "#text": text, "@_start": start, "@_dur": dur }) => ({ start: Number(start), dur: Number(dur), text }));
-		return {
-			name: captionTrack.name.simpleText,
-			id: captionTrack.vssId,
-			lines
-		};
-	})().catch(error => console.error(error.stack))));
-	return captionTracks;
-}
-
-var cache = {};
-
-export async function getVideoCaptionsCached(videoId) {
-	if (!cache[videoId]) {
-		cache[videoId] = getVideoCaptions(videoId);
-		setTimeout(() => {
-			delete cache[videoId];
-		}, 1000*60*60*6); // 6 hours
-	}
-	return await cache[videoId];
-}

imagesheet.js

@@ -1,15 +1,21 @@
 import { createCanvas, loadImage } from 'canvas';
 import potpack from 'potpack';
-import { putVrcUrl } from './vrcurl.js';
 
-var store = {};
-
-async function createImageSheet(images /*[{width, height, url}]*/) {
+export async function createImageSheet(images /*[{width, height, url}]*/, legacyMode) {
 	images.forEach(image => {
 		image.w = image.width;
 		image.h = image.height;
 	});
-	var {w, h, fill} = potpack(images);
+	if (legacyMode) {
+		images.forEach((image, index) => {
+			image.x = index % 5 * 360;
+			image.y = Math.floor(index / 5) * 202;
+		});
+		var w = Math.min(images.length, 5) * 360;
+		var h = images.at(-1).y + 202;
+	} else {
+		var {w, h, fill} = potpack(images);
+	}
 	if (w > 2048) {
 		console.warn("Imagesheet exceeded max width");
 		w = 2048;
@@ -23,36 +29,14 @@ async function createImageSheet(images /*[{width, height, url}]*/) {
 
 	await Promise.all(images.map(({x, y, w, h, url}) => (async function(){
 		if (!url) return;
-		var image = await loadImage(url);
+		try {
+			var image = await loadImage(url);
+		} catch (error) {
+			console.error("failed to load image", url, error.message);
+			return;
+		}
 		ctx.drawImage(image, x, y, w, h);
-	})().catch(error => console.error(error.stack))));
+	})().catch(error => console.error("imageload", error.stack))));
 
-	return {
-		imagesheet: canvas.toBuffer("image/png"),
-		images
-	};
-}
-
-export async function makeImageSheetVrcUrl(pool, images) {
-	var num = await putVrcUrl(pool, {type: "imagesheet"});
-	var key = `${pool}:${num}`;
-	var promise = createImageSheet(images);
-	store[key] = promise;
-	promise.then(() => {
-		setTimeout(() => {
-			if (store[key] === promise) delete store[key];
-		}, 1000*60*10); // 10 mins;
-	});
-	promise.catch(error => {
-		console.error(error.stack);
-	});
-	var {thumbnails, icons} = await promise;
-	return {
-		vrcurl: num,
-		thumbnails, icons
-	}
-}
-
-export async function getImageSheet(pool, num) {
-	return (await store[`${pool}:${num}`])?.imagesheet;
+	return canvas.toBuffer("image/png");
 }

index.js

@@ -1,4 +1,161 @@
+if (process.env.D!="BUG") console.debug = () => {};
+else console.debug(process.env);
+
 import "./util.js";
-import { app } from "./app.js";
+import Koa from "koa";
+import Router from "@koa/router";
+import send from "koa-send";
+import qs from "qs";
+import { cachedVRCYoutubeSearch, getImageSheet } from "./VRCYoutubeSearch.js"
+import { resolveVrcUrl } from "./vrcurl.js";
+import { getVideoCaptionsCached } from "./youtube-captions.js";
+import { stringToBoolean } from "./util.js";
+import { readFileSync } from "fs";
+var shorturlmap = JSON.parse(readFileSync("./shorturlmap.json", "utf8"));
+
+var app = new Koa();
+var router = new Router();
+
+
+router.get(["/search", "/trending"], async ctx => {
+	if (ctx.path == "/trending") {
+		var input = {"type":"trending"};
+	} else {
+		var input = ctx.querystring.match(/[?&]input=(.*)/i)?.[1];
+		if (!input) {
+			ctx.status = 400;
+			ctx.body = "missing search query";
+			return;
+		}
+		input = decodeURIComponent(input).replace(/^.*→/, '').replaceAll("\u200b", '').trim();
+	}
+
+	var pqs = qs.parse(ctx.querystring, {duplicates: 'first'});
+
+	if (!pqs.pool || /[^a-z-_0-9]/.test(pqs.pool)) {
+		ctx.status = 400;
+		ctx.body = "invalid pool";
+		return;
+	}
+
+	var options = {
+		thumbnails: stringToBoolean(pqs.thumbnails),
+		icons: stringToBoolean(pqs.icons),
+		captions: stringToBoolean(pqs.captions),
+		mode: pqs.mode,
+		bp: pqs.bp
+	};
+
+	ctx.body = await cachedVRCYoutubeSearch(pqs.pool, input, options);
+});
+
+
+router.get("/vrcurl/:pool/:num", async ctx => {
+	var dest = await resolveVrcUrl(ctx.params.pool, ctx.params.num);
+	if (!dest) {
+		ctx.status = 404;
+		return;
+	}
+	switch (dest.type) {
+		case "redirect":
+			ctx.set("Referrer-Policy", "no-referrer");
+			ctx.redirect(dest.url);
+			break;
+		case "video":
+			if (ctx.get("User-Agent").includes("UnityWebRequest")) {
+				ctx.body = {captions: await getVideoCaptionsCached(dest.id)};
+			} else {
+				ctx.set("Referrer-Policy", "no-referrer");
+				ctx.redirect(`https://www.youtube.com/watch?v=${dest.id}`);
+			}
+			break;
+		case "imagesheet":
+			let buf = await getImageSheet(dest.key);
+			if (!buf) {
+				ctx.status = 404;
+				return;
+			}
+			ctx.body = buf;
+			ctx.type = "image/png";
+			break;
+		case "continuation":
+			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "continuation", for: dest.for, continuationData: dest.continuationData}, dest.options);
+			break;
+		case "trending":
+			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "trending", bp: dest.bp}, dest.options);
+			break;
+		case "captions":
+			ctx.body = await getVideoCaptionsCached(dest.videoId);
+			break;
+		default:
+			console.error("unknown vrcurl type", dest.type);
+			ctx.status = 500;
+	}
+});
+
+
+router.get("/robots.txt", ctx => {
+	ctx.body = `User-agent: *\nDisallow: /`;
+});
+
+router.get("/test.html", async ctx => {
+	await send(ctx, "test.html");
+});
+
+router.get("/", ctx => {
+	ctx.redirect("https://www.u2b.cx/");
+});
+
+
+
+
+
+app.use(async (ctx, next) => {
+	console.log("http", ctx.socket.remoteAddress, ctx.get("X-Forwarded-For"), ctx.get("Host") + ctx.originalUrl, '"'+ctx.get("User-Agent")+'"', ctx.get("Referer"));
+	try {
+		await next();
+	} catch (error) {
+		console.error(ctx.url, error.stack);
+		ctx.status = 500;
+		ctx.type = "text";
+		ctx.body = error.stack;
+	}
+});
+
+
+// short urls to work around https://feedback.vrchat.com/udon/p/vrcurlinputfield-incorrect-focus-issue-on-quest
+app.use(async (ctx, next) => {
+	var subdomain = ctx.hostname.match(/(.*).u2b.cx$/i)?.[1];
+	if (subdomain && !["api","api2","dev"].includes(subdomain)) {
+		if (shorturlmap[subdomain]) {
+			ctx.url = shorturlmap[subdomain] + ctx.url.slice(1);
+		} else {
+			ctx.status = 404;
+			return;
+		}
+	}
+	await next();
+});
+
+// work around vrchat json parser bug https://feedback.vrchat.com/udon/p/braces-inside-strings-in-vrcjson-can-fail-to-deserialize
+app.use(async (ctx, next) => {
+	await next();
+	if (ctx.type != "application/json") return;
+	ctx.body = structuredClone(ctx.body);
+	(function iterateObject(obj) {
+		for (var key in obj) {
+			if (typeof obj[key] == "string") {
+				obj[key] = obj[key].replace(/[\[\]{}]/g, chr => "\\u" + chr.charCodeAt(0).toString(16).padStart(4, '0'));
+			} else if (typeof obj[key] == "object") {
+				iterateObject(obj[key]);
+			}
+		}
+	})(ctx.body);
+	ctx.body = JSON.stringify(ctx.body).replaceAll("\\\\u", "\\u");
+});
+
+
+app.use(router.routes());
+app.use(router.allowedMethods());
 
 app.listen(process.env.PORT || 8142, process.env.ADDRESS);

package.json

@@ -2,12 +2,14 @@
 	"dependencies": {
 		"@keyv/sqlite": "^3.6.6",
 		"@koa/router": "^12.0.1",
-		"canvas": "^2.11.2",
+		"canvas": "^3.1.0",
 		"fast-xml-parser": "^4.3.4",
+		"got": "^14.4.1",
 		"keyv": "^4.5.4",
 		"koa": "^2.14.2",
 		"koa-send": "^5.0.1",
-		"potpack": "^2.0.0"
+		"potpack": "^2.0.0",
+		"qs": "^6.12.2"
 	},
 	"engines": {
 		"node": ">=18.0.0"

simpleYoutubeSearch.js

@@ -1,94 +0,0 @@
-import { parseVideoRendererData } from "./util.js";
-
-export async function searchYouTubeVideos(query) {
-	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}&sp=EgIQAQ%253D%253D`;
-	var html = await fetch(url).then(res => res.text());
-
-	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
-	ytInitialData = JSON.parse(ytInitialData);
-
-	var videos = ytInitialData?.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))?.itemSectionRenderer?.contents?.filterMap(x => x.videoRenderer)?.map(parseVideoRendererData);
-	if (!videos) return {videos: []};
-
-	try {
-		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
-		ytcfg = JSON.parse(ytcfg);
-		var continuationData = {
-			context: ytcfg.INNERTUBE_CONTEXT,
-			continuation: ytInitialData.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
-		}
-	} catch (error) {
-		console.error(error.stack);
-	}
-
-	return {videos, continuationData};
-}
-
-
-export async function continueYouTubeVideoSearch(continuationData) {
-	var data = await fetch("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
-		method: "POST",
-		headers: {
-			"Content-Type": "application/json"
-		},
-		body: JSON.stringify(continuationData)
-	}).then(res => res.json());
-
-	var continuationItems = data.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems;
-	var videos = continuationItems.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer)).itemSectionRenderer.contents.filterMap(x => x.videoRenderer).map(parseVideoRendererData);
-	var continuationToken = continuationItems.find(x => x.continuationItemRenderer)?.continuationItemRenderer.continuationEndpoint.continuationCommand.token
-
-	return {
-		videos,
-		continuationData: continuationToken ? {
-			context: continuationData.context,
-			continuation: continuationToken
-		} : null
-	}
-}
-
-
-
-export async function getYouTubePlaylist(playlistId) {
-	var html = await fetch("https://www.youtube.com/playlist?list=" + playlistId).then(res => res.text());
-	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
-	ytInitialData = JSON.parse(ytInitialData);
-
-	var sectionListRendererContents = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.find(tab => tab.tabRenderer.selected).tabRenderer.content.sectionListRenderer.contents;
-	var videos = sectionListRendererContents.find(x => x.itemSectionRenderer).itemSectionRenderer.contents.find(x => x.playlistVideoListRenderer).playlistVideoListRenderer.contents.filterMap(x => x.playlistVideoRenderer).map(parseVideoRendererData);
-	if (!videos) return {videos: []};
-
-	try {
-		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
-		ytcfg = JSON.parse(ytcfg);
-		var continuationData = {
-			context: ytcfg.INNERTUBE_CONTEXT,
-			continuation: sectionListRendererContents.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
-		}
-	} catch (error) {
-		console.error(error.stack);
-	}
-	return {videos, continuationData};
-}
-
-
-export async function continueYouTubePlaylist(continuationData) {
-	var data = await fetch("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false", {
-		method: "POST",
-		headers: {"Content-Type": "application/json"},
-		body: JSON.stringify(continuationData)
-	}).then(res => res.json());
-
-	if (!data.onResponseReceivedActions) return {videos:[]};
-	var continuationItems = data.onResponseReceivedActions[0].appendContinuationItemsAction.continuationItems;
-	var videos = continuationItems.find(x => x.itemSectionRenderer).itemSectionRenderer.contents.filterMap(x => x.playlistVideoListRenderer).map(parseVideoRendererData);
-	var continuationToken = continuationItems.find(x => x.continuationItemRenderer)?.continuationItemRenderer.continuationEndpoint.continuationCommand.token;
-
-	return {
-		videos,
-		continuationData: continuationToken ? {
-			context: continuationData.context,
-			continuation: continuationToken
-		} : null
-	}
-}

test.html

@@ -14,7 +14,7 @@
 	<br />
 	<label><input id="thumbnails" type="checkbox" checked>thumbnails</label>
 	<label><input id="icons" type="checkbox" checked>icons</label>
-	<label><input id="captions" type="checkbox" checked>captions</label>
+	<label><input id="latestontop" type="checkbox">latestontop</label>
 </div>
 
 <div id="output"></div>
@@ -30,11 +30,11 @@ var lastData;
 
 search.onclick = () => {
 	output.innerHTML = "";
-	loadData(`/search?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&captions=${captions.checked}&input=${encodeURIComponent(input.value)}`);
+	loadData(`/search?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&mode=${latestontop.checked&&"latestontop"}&input=${encodeURIComponent(input.value)}`);
 };
 trending.onclick = () => {
 	output.innerHTML = "";
-	loadData(`/trending?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&captions=${captions.checked}`);
+	loadData(`/trending?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}`);
 }
 
 async function loadData(url) {

trending.js

@@ -1,43 +0,0 @@
-import { parseVideoRendererData } from "./util.js";
-
-export async function getTrending(bp) {
-	var url = `https://www.youtube.com/feed/trending`;
-	if (bp) url += `?bp=${bp}`;
-	var html = await fetch(url).then(res => res.text());
-	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
-	ytInitialData = JSON.parse(ytInitialData);
-
-	var tabs = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.map(t => {
-		return {
-			name: t.tabRenderer.title,
-			//url: `https://www.youtube.com` + t.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url
-			bp: t.tabRenderer.endpoint.browseEndpoint.params
-		}
-	});
-
-	var videos = ytInitialData
-		.contents
-		.twoColumnBrowseResultsRenderer
-		.tabs
-		.find(tab => tab.tabRenderer.selected)
-		.tabRenderer
-		.content
-		.sectionListRenderer
-		.contents
-		// regular trending in sections with shelfRenderer without title
-		.filterMap(x => {
-			var shelfRenderer = x.itemSectionRenderer.contents.find(x => x.shelfRenderer)?.shelfRenderer;
-			if (shelfRenderer && !shelfRenderer.title) {
-				return shelfRenderer
-					.content
-					.expandedShelfContentsRenderer
-					.items
-					.filterMap(x => x.videoRenderer)
-					.map(parseVideoRendererData)
-			};
-		})
-		.flat();
-
-
-	return {tabs, videos};
-}

util.js

@@ -1,3 +1,6 @@
+import {readFileSync} from "fs";
+import got from "got";
+
 Array.prototype.filterMap = function(fn) {
 	var newarray = [];
 	for (var item of this) {
@@ -7,6 +10,23 @@ Array.prototype.filterMap = function(fn) {
 	return newarray;
 };
 
+Array.prototype.findMap = function(fn) {
+	for (var item of this) {
+		var ret = fn(item);
+		if (ret) return ret;
+	}
+}
+
+Object.prototype.deepFind = function (fn) {
+	return (function crawlObject(object) {
+		for (var key in object) {
+			var value = object[key];
+			if (fn(value)) return value;
+			if (typeof value == "object") crawlObject(value);
+		}
+	})(this);
+};
+
 export function stringToBoolean(str) {
 	if (str) {
 		if (!["0", "false", "off", "no", "null", "undefined", "nan"].includes(str.toLowerCase())) return true;
@@ -14,47 +34,16 @@ export function stringToBoolean(str) {
 	return false;
 }
 
-export function recursiveFind(object, fn) {
-	var results = [];
-	(function crawlObject(object) {
-		for (var key in object) {
-			var value = object[key];
-			var test = fn(value);
-			if (test) results.push(test);
-			if (typeof value == "object") crawlObject(value);
-		}
-	})(object);
-	return results;
-}
-
-
-Object.prototype.concatRunsText = function concatRunsText() {
-	return this.reduce((str, obj) => str += obj.text, "");
-};
-
-export function parseVideoRendererData(data) {
-	return {
-		id: data.videoId,
-		live: Boolean(data.badges?.find(x => x.metadataBadgeRenderer?.style == "BADGE_STYLE_TYPE_LIVE_NOW")),
-		title: data.title?.runs?.concatRunsText(),
-		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.concatRunsText() 
-			|| data.descriptionSnippet?.runs?.concatRunsText(),
-		//thumbnailUrl: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138))?.url || data.thumbnail?.thumbnails?.[0]?.url,
-		//thumbnail: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138)) || data.thumbnail?.thumbnails?.[0],
-		thumbnail: {
-			url: `https://i.ytimg.com/vi/${data.videoId}/mqdefault.jpg`,
-			width: 320,
-			height: 180
-		},
-		uploaded: data.publishedTimeText?.simpleText || data.videoInfo?.runs?.[2]?.text,
-		lengthText: data.lengthText?.simpleText,
-		longLengthText: data.lengthText?.accessibility?.accessibilityData?.label,
-		viewCountText: data.viewCountText?.runs ? data.viewCountText.runs.concatRunsText() : data.viewCountText?.simpleText,
-		shortViewCountText: data.shortViewCountText?.simpleText || data.videoInfo?.runs?.[0]?.text,
-		channel: {
-			name: (data.ownerText || data.shortBylineText)?.runs?.concatRunsText(),
-			id: (data.ownerText || data.shortBylineText)?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId,
-			iconUrl: data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail?.thumbnails?.[0]?.url
-		}
-	};
+
+
+
+try {
+	var ips = readFileSync("ips.txt", "utf8").trim().split("\n");
+	console.log("using ips", ips);
+} catch (e) {}
+
+export function gotw(url, options = {}) {
+	if (ips) options.localAddress = ips[Math.floor(Math.random() * ips.length)];
+	//options.timeout = {request: 3000}; //"RequestError: Expected values which are `number` or `undefined`. Received values of type `Function`." what the fuck?
+	return got(url, options);
 }

youtube-captions.js

@@ -0,0 +1,54 @@
+import { XMLParser } from "fast-xml-parser";
+import { gotw } from "./util.js";
+
+var xmlParser = new XMLParser({
+	ignoreAttributes: false
+});
+
+async function getVideoData(videoId) {
+	var res = await gotw(`https://www.youtube.com/watch?v=${videoId}`);
+
+	var ytInitialPlayerResponse = res.body.match(/var ytInitialPlayerResponse = ({.*?});/)[1];
+	ytInitialPlayerResponse = JSON.parse(ytInitialPlayerResponse);
+
+	return ytInitialPlayerResponse;
+}
+
+async function getVideoCaptions(videoId) {
+	var ytInitialPlayerResponse = await getVideoData(videoId);
+	if (!ytInitialPlayerResponse.captions) return [];
+	var captionTracks = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks;
+	captionTracks = await Promise.all(captionTracks.map(captionTrack => (async () => {
+		try {
+			var promise = gotw(captionTrack.baseUrl);
+			var res = await promise;
+			if (res.statusCode != 200) throw new Error("unexpected status " + res.statusCode);
+			var xml = await promise.text();
+			if (!xml) throw new Error("empty response for caption track");
+			var parsed = xmlParser.parse(xml);
+			var lines = parsed.transcript.text;
+			if (!Array.isArray(lines)) lines = [lines];
+			lines = lines.map(({ "#text": text, "@_start": start, "@_dur": dur }) => ({ start: Number(start), dur: Number(dur), text }));
+			return {
+				name: captionTrack.name.simpleText,
+				id: captionTrack.vssId,
+				lines
+			};
+		} catch (error) {
+			console.error("caption track error", error.stack, videoId, captionTrack, xml, parsed, lines);
+		}
+	})()));
+	return captionTracks;
+}
+
+var cache = {};
+
+export async function getVideoCaptionsCached(videoId) {
+	if (!cache[videoId]) {
+		cache[videoId] = getVideoCaptions(videoId);
+		setTimeout(() => {
+			delete cache[videoId];
+		}, 1000*60*10); // 10 minutes
+	}
+	return await cache[videoId];
+}

youtube.js

@@ -0,0 +1,274 @@
+import { gotw } from "./util.js";
+
+
+
+
+export async function searchYouTubeVideos(query, sp = "EgIQAQ%253D%253D") {
+	console.debug("sp", sp);
+	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}${sp ? `$sp=${sp}` : ''}`;
+	var res = await gotw(url), html = res.body;
+ 
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)?.[1];
+	if (!ytInitialData) {
+		console.error("missing ytInitialData", query, res.status, html);
+	}
+	ytInitialData = JSON.parse(ytInitialData);
+	console.debug(ytInitialData);
+
+	var sectionListRendererContents = ytInitialData
+		.contents
+		.twoColumnSearchResultsRenderer
+		.primaryContents
+		.sectionListRenderer
+		.contents;
+
+	var videos = sectionListRendererContents
+		?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))
+		?.itemSectionRenderer
+		.contents
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
+	if (!videos) return {videos: []};
+
+	var latest = sectionListRendererContents
+		.find(x => x.itemSectionRenderer?.contents?.find(x => x.shelfRenderer))
+		?.itemSectionRenderer
+		.contents
+		.find(x => x.shelfRenderer?.title?.simpleText?.startsWith("Latest from"))
+		?.shelfRenderer
+		.content
+		.verticalListRenderer
+		.items
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
+	console.debug("latest", latest);
+	if (latest) videos = [...latest, ...videos];
+
+	console.debug(videos.length, "results");
+
+	try {
+		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
+		ytcfg = JSON.parse(ytcfg);
+		var continuationData = {
+			context: ytcfg.INNERTUBE_CONTEXT,
+			continuation: sectionListRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.continuationCommand.token
+		}
+	} catch (error) {
+		console.error(error.stack);
+	}
+
+	return {videos, continuationData};
+}
+
+
+
+
+export async function continueYouTubeVideoSearch(continuationData) {
+	var res = await gotw("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
+		method: "POST",
+		json: continuationData,
+		responseType: "json"
+	});
+	var data = res.body;
+	console.debug(data);
+
+	var continuationItems = data
+		.onResponseReceivedCommands[0]
+		.appendContinuationItemsAction
+		.continuationItems;
+	var videos = continuationItems
+		.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer))
+		.itemSectionRenderer
+		.contents
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
+	var continuationToken = continuationItems
+		.findMap(x => x.continuationItemRenderer)
+		?.continuationEndpoint
+		.continuationCommand
+		.token;
+	console.debug(videos.length, "results");
+
+
+	return {
+		videos,
+		continuationData: continuationToken ? {
+			context: continuationData.context,
+			continuation: continuationToken
+		} : null
+	}
+}
+
+
+
+
+
+
+
+
+
+export async function getYouTubePlaylist(playlistId) {
+	var res = await gotw("https://www.youtube.com/playlist?list=" + playlistId);
+	var html = res.body;
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+	ytInitialData = JSON.parse(ytInitialData);
+	console.debug(ytInitialData);
+
+	var playlistVideoRendererContents = ytInitialData
+		.contents
+		.twoColumnBrowseResultsRenderer
+		.tabs
+		.find(tab => tab.tabRenderer.selected)
+		.tabRenderer
+		.content
+		.sectionListRenderer
+		.contents
+		.findMap(x => x.itemSectionRenderer?.contents)
+		.findMap(x => x.playlistVideoListRenderer?.contents);
+	var videos = playlistVideoRendererContents
+		.filterMap(x => x.playlistVideoRenderer)
+		.map(parseVideoRendererData);
+	if (!videos) return {videos: []};
+	console.debug(videos.length, "results");
+
+	try {
+		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
+		ytcfg = JSON.parse(ytcfg);
+		var continuationData = {
+			context: ytcfg.INNERTUBE_CONTEXT,
+			continuation: playlistVideoRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.commandExecutorCommand.commands.findMap(x=>x.continuationCommand).token
+		}
+	} catch (error) {
+		console.error(error.stack);
+	}
+	return {videos, continuationData};
+}
+
+
+
+
+export async function continueYouTubePlaylist(continuationData) {
+	var res = await gotw("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false", {
+		method: "POST",
+		json: continuationData,
+		responseType: "json"
+	});
+	var data = res.body;
+	console.debug(data);
+
+	if (!data.onResponseReceivedActions) return {videos:[]};
+	var continuationItems = data
+		.onResponseReceivedActions[0]
+		.appendContinuationItemsAction
+		.continuationItems;
+	var videos = continuationItems
+		.filterMap(x => x.playlistVideoRenderer)
+		.map(parseVideoRendererData);
+	var continuationToken = continuationItems
+		.findMap(x => x.continuationItemRenderer)
+		?.continuationEndpoint
+		.continuationCommand
+		.token;
+	console.debug(videos.length, "results");
+
+	return {
+		videos,
+		continuationData: continuationToken ? {
+			context: continuationData.context,
+			continuation: continuationToken
+		} : null
+	}
+}
+
+
+
+
+
+
+
+
+
+export async function getTrending(bp) {
+	var url = `https://www.youtube.com/feed/trending`;
+	if (bp) url += `?bp=${bp}`;
+	var res = await gotw(url);
+	var html = res.body;
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+	ytInitialData = JSON.parse(ytInitialData);
+
+	var tabs = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.map(t => {
+		return {
+			name: t.tabRenderer.title,
+			//url: `https://www.youtube.com` + t.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url
+			bp: t.tabRenderer.endpoint.browseEndpoint.params
+		}
+	});
+
+	var videos = ytInitialData
+		.contents
+		.twoColumnBrowseResultsRenderer
+		.tabs
+		.find(tab => tab.tabRenderer.selected)
+		.tabRenderer
+		.content
+		.sectionListRenderer
+		.contents
+		// regular trending in sections with shelfRenderer without title
+		.filterMap(x => {
+			var shelfRenderer = x.itemSectionRenderer.contents.findMap(x => x.shelfRenderer);
+			if (shelfRenderer && !shelfRenderer.title) {
+				return shelfRenderer
+					.content
+					.expandedShelfContentsRenderer
+					.items
+					.filterMap(x => x.videoRenderer)
+					.map(parseVideoRendererData)
+			};
+		})
+		.flat();
+
+
+	return {tabs, videos};
+}
+
+
+
+
+
+
+
+
+
+
+
+
+Object.prototype.concatRunsText = function concatRunsText() {
+	return this.reduce((str, obj) => str += obj.text, "");
+};
+
+function parseVideoRendererData(data) {
+	return {
+		id: data.videoId,
+		live: Boolean(data.badges?.find(x => x.metadataBadgeRenderer?.style == "BADGE_STYLE_TYPE_LIVE_NOW")),
+		title: data.title?.runs?.concatRunsText(),
+		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.concatRunsText() 
+			|| data.descriptionSnippet?.runs?.concatRunsText(),
+		//thumbnailUrl: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138))?.url || data.thumbnail?.thumbnails?.[0]?.url,
+		thumbnail: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138)) || data.thumbnail?.thumbnails?.[0],
+		/*thumbnail: {
+			url: `https://i.ytimg.com/vi/${data.videoId}/mqdefault.jpg`,
+			width: 320,
+			height: 180
+		},*/
+		uploaded: data.publishedTimeText?.simpleText || data.videoInfo?.runs?.[2]?.text,
+		lengthText: data.lengthText?.simpleText,
+		longLengthText: data.lengthText?.accessibility?.accessibilityData?.label,
+		viewCountText: data.viewCountText?.runs ? data.viewCountText.runs.concatRunsText() : data.viewCountText?.simpleText,
+		shortViewCountText: data.shortViewCountText?.simpleText || data.videoInfo?.runs?.[0]?.text,
+		channel: {
+			name: (data.ownerText || data.shortBylineText)?.runs?.concatRunsText(),
+			id: (data.ownerText || data.shortBylineText)?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId,
+			iconUrl: data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail?.thumbnails?.[0]?.url
+		}
+	};
+}