youtube antibotted the captions

gives blank file now without special code added to url

.gitignore

@@ -1,2 +1,4 @@
 node_modules
-vrcurl.sqlite
+*.sqlite
+*.sqlite-journal
+ips.txt

.vscode/launch.json

@@ -14,7 +14,10 @@
 			"program": "${workspaceFolder}\\index.js",
 			"outFiles": [
 				"${workspaceFolder}/**/*.js"
-			]
+			],
+			"env": {
+				"D":"BUG"
+			}
 		}
 	]
 }

README.md

@@ -8,15 +8,19 @@ Test in browser: https://api.u2b.cx/test.html
 
 ## GET `/search`
 
+Get YouTube videos for a search query.
+
 ### Required query parameters
 
 - `pool`: id of the VRCUrl pool, only letters numbers hyphens or underscores, optionally followed by an integer for pool size.
-- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed.
+- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed. THIS MUST BE THE LAST QUERY PARAMETER AS ALL CHARS AFTER IT ARE CAPTURED VERBATIM (so you can type & etc without encoding). If this contains a url with a playlist ID, playlist results will be loaded instead (which is a bit different, up to 100 results and no descriptions).
 
 ### Optional query parameters
 
 - `thumbnails`: set to `1`, `true`, `yes`, `on` or whatever to load thumbnails
 - `icons`: set to `1`, `true`, `yes`, `on` or whatever to load channel icons
+- `mode`: If set to `latestontop` will search without filter to include the "Latest from ..." section (if it exists). Otherwise it uses search filter for Videos only sorted by Relevance.
+- `bp`: Custom YouTube search filter. Go to youtube search site and set some filters and you will see the corresponding `bp` value in the URL. Overrides `mode` option. Use at your own risk (experimental).
 
 ### Example URL
 
@@ -29,20 +33,20 @@ https://api.u2b.cx/search?pool=example10000&input=   Type YouTube search query h
 JSON object:
 
 - `results`: Array of Object
-	- `vrcurl`: (integer) index of VRCUrl that will redirect to the youtube url
+	- `vrcurl`: (integer) index of VRCUrl that will redirect to the youtube url, or serve JSON with captions if used with string loader.
 	- `live`: (boolean) whether it's a live stream
 	- `title`: (string) i.e. `"Nyan Cat! [Official]"`
 	- `id`: (string) YouTube video id i.e. `"2yJgwwDcgV8"`
-	- `description`: (string) short truncated description snippet i.e. `"http://nyan.cat/ Original song : http://momolabo.lolipop.jp/nyancatsong/Nyan/"`
+	- `description`?: (string) short truncated description snippet i.e. `"http://nyan.cat/ Original song : http://momolabo.lolipop.jp/nyancatsong/Nyan/"` (playlist results don't have this)
 	- `lengthText`?: (string) i.e. `"3:37"`
 	- `longLengthText`?: (string) i.e. `"3 minutes, 37 seconds"`
-	- `viewCountText`: (string) i.e. `"2,552,243 views"` or `"575 watching"` for live streams
-	- `shortViewCountText`?: (string) i.e. `"2.5M views"`
+	- `viewCountText`?: (string) i.e. `"2,552,243 views"` or `"575 watching"` for live streams (playlist results don't have this)
+	- `shortViewCountText`?: (string) i.e. `"2.5M views"` (streams don't have this)
 	- `uploaded`: (string) i.e. `"12 years ago"`
 	- `channel`: (object)
 		- `name`: (string) i.e. `"NyanCat"`
 		- `id`: (string) i.e. `"UCsW85RAS2_Twg_lEPyv7G8A"`
-		- `icon`?: (object)
+		- `icon`?: (object) (playlist results don't have this)
 			- `x`: (integer) px from left
 			- `y`: (integer) px from top
 			- `width`: (integer)
@@ -56,12 +60,32 @@ JSON object:
 - `nextpage_vrcurl`: (integer) index of the vrcurl that will serve the JSON for the next page of results
 
 
+## GET `/trending`
+
+Gets Trending YouTube videos. Identical to `/search` but without `input` parameter, and response includes additional field:
+
+- `tabs`: Array of Object
+	- `name`: (string) Tab title ("Now", "Music", "Gaming", "Movies")
+	- `vrcurl`: (integer) index of vrcurl to load that tab (same response format)
+
 ## GET `/vrcurl/{pool}/{index}`
 
 - `{pool}`: must be same as pool param in search endpoint.
-- `{index}`: vrcurl index number
+- `{index}`: vrcurl index number as specified in response data
 
-Response may be 302 redirect to youtube url, `image/png` for imagesheet or `application/json` for next page
+For youtube videos, if `User-Agent` header includes `UnityWebRequest`, then response is `application/json` for video metadata (captions). Otherwise, it is 302 redirect to youtube URL.
+
+For image sheet, response is `image/png`. For next page or trending tab, response is `application/json`.
+
+### Video metadata JSON format
+
+- `captions`: Array of Object
+	- `name`: (string) caption track name like "English" or "English (auto-generated)"
+	- `id`: (string) id like `.en` or `a.en`
+	- `lines`: Array of Object
+		- `start`: (float) video seconds when the caption is displayed
+		- `dur`: (float) seconds to display the caption
+		- `text`: (string) caption text
 
 # VRCUrls
 
@@ -86,4 +110,4 @@ All resources (youtube urls etc) referenced in the search results will be substi
 
 Video thumbnails and channel icons are collated together into one image and served at a VRCUrl to be loaded by VRCImageDownloader.
 
-Use the x, y, width and height values from the json to crop the image from the sheet.
+Use the x, y, width and height values from the json to crop the image from the sheet. Do not make any assumptions about these values as the server could arrange the images wherever it wants.

VRCYoutubeSearch.js

@@ -1,80 +1,126 @@
-import { searchYouTubeVideos, continueYouTubeVideoSearch } from "./simpleYoutubeSearch.js";
+import { searchYouTubeVideos, continueYouTubeVideoSearch, getYouTubePlaylist, continueYouTubePlaylist, getTrending } from "./youtube.js";
 import { putVrcUrl } from "./vrcurl.js";
-import { makeImageSheetVrcUrl, thumbnailWidth, thumbnailHeight, iconWidth, iconHeight } from "./imagesheet.js";
+import { createImageSheet } from "./imagesheet.js";
 
 var cache = {};
 
-
-
-export async function cachedVRCYoutubeSearch(pool, queryOrContinuation, options) {
-	var key = JSON.stringify([pool, queryOrContinuation, options]);
+export async function cachedVRCYoutubeSearch(pool, query, options) {
+	var key = JSON.stringify([pool, query, options]);
 	if (!cache[key]) {
-		cache[key] = VRCYoutubeSearch(pool, queryOrContinuation, options);
+		cache[key] = VRCYoutubeSearch(pool, query, options, key);
 		setTimeout(() => {
 			delete cache[key];
 		}, 1000*60*10); // 10 mins
 	}
-	return await cache[key];
+	return (await cache[key])?.response;
+}
+
+export async function getImageSheet(key) {
+	return await (await cache[key])?.imagesheet
 }
 
 
+async function VRCYoutubeSearch(pool, query, options = {}, key) {
+	console.log("search", pool, JSON.stringify(query), JSON.stringify(options));
+	var response = {results: []};
 
-
-async function VRCYoutubeSearch(pool, query, options = {}) {
-	console.debug("search:", query);
-	var data = {results: []};
-
-	var {videos, continuationData} = typeof query == "object" ? await continueYouTubeVideoSearch(query) : await searchYouTubeVideos(query);
+	if (typeof query == "object") {
+		switch (query.type) {
+			case "trending":
+				var {videos, tabs} = await getTrending(query.bp);
+				response.tabs = [];
+				for (let tab of tabs) {
+					response.tabs.push({
+						name: tab.name,
+						vrcurl: await putVrcUrl(pool, {type: "trending", bp: tab.bp, options})
+					});
+				}
+				break;
+			case "continuation":
+				if (query.for == "playlist") {
+					var {videos, continuationData} = await continueYouTubePlaylist(query.continuationData);
+				} else {
+					var {videos, continuationData} = await continueYouTubeVideoSearch(query.continuationData);
+				}
+				break;
+		}
+	} else {
+		var playlistId = query.match(/list=(PL(?:[a-zA-Z0-9-_]{32}|[0-9A-F]{16}))/)?.[1];
+		if (playlistId) console.log("playlistId:", playlistId);
+		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query, options.bp || (options.mode == "latestontop" ? null : undefined));
+	}
+	
+	var images = [];
 
 	if (options.thumbnails) {
-		var thumbnailUrls = videos.map(video => video.thumbnailUrl);
+		videos.forEach(video => {
+			if (playlistId || query?.for == "playlist") video.thumbnail = {
+				url: `https://i.ytimg.com/vi/${video.id}/default.jpg`,
+				width: 120,
+				height: 90
+			};
+			else {
+				video.thumbnail ||= {
+					url: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`,
+					width: 320,
+					height: 180
+				};
+				console.debug(video.thumbnail);
+				video.thumbnail.width = 360;
+				video.thumbnail.height = 202;
+			}
+			images.push(video.thumbnail);
+		});
 	}
 
 	if (options.icons) {
-		var iconUrls = new Set();
-		for (let video of videos) {
-			iconUrls.add(video.channel.iconUrl);
-		}
-		iconUrls = [...iconUrls];
+		let iconUrls = new Set();
+		videos.forEach(video => video.channel.iconUrl && iconUrls.add(video.channel.iconUrl));
+		iconUrls.forEach(url => images.push({
+			width: 68,//todo pass from yt data not hardcode
+			height: 68,
+			url
+		}));
 	}
 
-	if (thumbnailUrls || iconUrls) {
-		var {vrcurl: imagesheet_vrcurl, thumbnails, icons} = await makeImageSheetVrcUrl(pool, thumbnailUrls, iconUrls);
-		data.imagesheet_vrcurl = imagesheet_vrcurl;
+	if (images.length) {
+		try {
+			response.imagesheet_vrcurl = await putVrcUrl(pool, {type: "imagesheet", key});
+			var imagesheet = createImageSheet(images, !playlistId && !options.icons);
+		} catch (error) {
+			console.error(error.stack);
+		}
 	}
 
 	for (let video of videos) {
-		video.vrcurl = await putVrcUrl(pool, {type: "redirect", url: `https://www.youtube.com/watch?v=${video.id}`});
-		if (thumbnails?.length) {
-			let thumbnail = thumbnails.find(x => x.url == video.thumbnailUrl);
-			video.thumbnail = {
-				x: thumbnail?.x,
-				y: thumbnail?.y,
-				width: thumbnailWidth,
-				height: thumbnailHeight
-			};
+		video.vrcurl = await putVrcUrl(pool, {type: "video", id: video.id});
+		let thumbnail = images.find(image => image.url == video.thumbnail.url);
+		video.thumbnail = thumbnail ? {
+			x: thumbnail?.x,
+			y: thumbnail?.y,
+			width: thumbnail?.width,
+			height: thumbnail?.height
+		} : undefined;
+		let icon = images.find(image => image.url == video.channel.iconUrl);
+		video.channel.icon = icon ? {
+			x: icon?.x,
+			y: icon?.y,
+			width: icon?.width,
+			height: icon?.height
+		} : undefined;
+		if (options.captions) {
+			video.captions_vrcurl = await putVrcUrl(pool, {type: "captions", videoId: video.id});
 		}
-		if (icons?.length) {
-			let icon = icons.find(x => x.url == video.channel.iconUrl);
-			video.channel.icon = {
-				x: icon?.x,
-				y: icon?.y,
-				width: iconWidth,
-				height: iconHeight
-			};
-		}
-		delete video.thumbnailUrl;
 		delete video.channel.iconUrl;
-		data.results.push(video);
+		response.results.push(video);
 	}
 
-	
-
-	if (continuationData) data.nextpage_vrcurl = await putVrcUrl(pool, {
-		type: "ytContinuation",
+	if (continuationData) response.nextpage_vrcurl = await putVrcUrl(pool, {
+		type: "continuation",
+		for: query.for || (playlistId ? "playlist" : "search"),
 		continuationData,
 		options
 	});
 
-	return data;
+	return {response, imagesheet};
 }

app.js

@@ -1,79 +0,0 @@
-import Koa from "koa";
-import Router from "@koa/router";
-import send from "koa-send";
-import { cachedVRCYoutubeSearch } from "./VRCYoutubeSearch.js"
-import { getImageSheet } from "./imagesheet.js";
-import { resolveVrcUrl } from "./vrcurl.js";
-import { stringToBoolean } from "./util.js";
-
-export var app = new Koa();
-var router = new Router();
-
-
-router.get("/search", async ctx => {
-	var query = ctx.query.input?.replace(/^.*→/, '').trim();
-	if (!query) {
-		ctx.status = 400;
-		ctx.body = "missing search query";
-		return;
-	}
-
-	if (!ctx.query.pool || !/^[a-z-_]+\d*$/.test(ctx.query.pool)) {
-		ctx.status = 400;
-		ctx.body = "invalid pool";
-		return;
-	}
-
-	var options = {
-		thumbnails: stringToBoolean(ctx.query.thumbnails),
-		icons: stringToBoolean(ctx.query.icons)
-	};
-
-	ctx.body = await cachedVRCYoutubeSearch(ctx.query.pool, query, options);
-});
-
-
-router.get("/vrcurl/:pool/:num", async ctx => {
-	var dest = await resolveVrcUrl(ctx.params.pool, ctx.params.num);
-	if (!dest) {
-		ctx.status = 404;
-		return;
-	}
-	switch (dest.type) {
-		case "redirect":
-			ctx.redirect(dest.url);
-			break;
-		case "imagesheet":
-			let buf = await getImageSheet(ctx.params.pool, ctx.params.num);
-			if (!buf) {
-				ctx.status = 404;
-				return;
-			}
-			ctx.body = buf;
-			ctx.type = "image/png";
-			break;
-		case "ytContinuation":
-			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, dest.continuationData, dest.options);
-			break;
-		default:
-			ctx.status = 500;
-	}
-});
-
-
-router.get("/robots.txt", ctx => {
-	ctx.body = `User-agent: *\nDisallow: /`;
-});
-
-router.get("/test.html", async ctx => {
-	await send(ctx, "test.html");
-});
-
-router.get("/", ctx => {
-	ctx.redirect("https://www.u2b.cx/");
-});
-
-
-
-app.use(router.routes());
-app.use(router.allowedMethods());

imagesheet.js

@@ -1,91 +1,42 @@
 import { createCanvas, loadImage } from 'canvas';
-import { putVrcUrl } from './vrcurl.js';
+import potpack from 'potpack';
 
-var store = {};
-
-
-export const thumbnailWidth = 360;
-export const thumbnailHeight = 202;
-export const iconWidth = 68;
-export const iconHeight = 68;
-const maxSheetWidth = 2048;
-const maxSheetHeight = 2048;
-const maxThumbnailRowLen = Math.floor(maxSheetWidth / thumbnailWidth); // 5
-const maxThumbnailColLen = Math.floor(maxSheetHeight / thumbnailHeight); // 10
-const maxIconRowLen = Math.floor(maxSheetWidth / iconWidth); // 30
-const maxIconColLen = Math.floor(maxSheetHeight / iconHeight); // 30
-
-
-async function createImageSheet(thumbnailUrls = [], iconUrls = []) {
-	
-
-	var thumbnails = thumbnailUrls.map((url, index) => {
-		const x = index % maxThumbnailRowLen * thumbnailWidth;
-		const y = Math.floor(index / maxThumbnailRowLen) * thumbnailHeight;
-		return {x, y, url};
+export async function createImageSheet(images /*[{width, height, url}]*/, legacyMode) {
+	images.forEach(image => {
+		image.w = image.width;
+		image.h = image.height;
 	});
-
-	const iconStartY = thumbnails.length ? thumbnails.at(-1).y + thumbnailHeight : 0;
-
-	var icons = iconUrls.map((url, index) => {
-		const x = index % maxIconRowLen * iconWidth;
-		const y = iconStartY + Math.floor(index / maxIconRowLen);
-		return {x, y, url};
-	});
-
-	const canvasWidth = Math.max(
-		Math.min(thumbnails.length, maxThumbnailRowLen) * thumbnailWidth,
-		Math.min(icons.length, maxIconRowLen) * iconWidth
-	);
-	const canvasHeight = icons.length ? icons.at(-1).y + iconHeight : thumbnails.length ? thumbnails.at(-1).y + thumbnailHeight : 0;
-
-	var canvas = createCanvas(Math.min(maxSheetWidth, canvasWidth), Math.min(maxSheetHeight, canvasHeight));
+	if (legacyMode) {
+		images.forEach((image, index) => {
+			image.x = index % 5 * 360;
+			image.y = Math.floor(index / 5) * 202;
+		});
+		var w = Math.min(images.length, 5) * 360;
+		var h = images.at(-1).y + 202;
+	} else {
+		var {w, h, fill} = potpack(images);
+	}
+	if (w > 2048) {
+		console.warn("Imagesheet exceeded max width");
+		w = 2048;
+	}
+	if (h > 2048) {
+		console.warn("Imagesheet exceeded max height");
+		h = 2048;
+	}
+	var canvas = createCanvas(w, h);
 	var ctx = canvas.getContext('2d');
 
-	var promises = [];
-
-	if (thumbnails.length) {
-		promises = promises.concat(thumbnails.map(({x, y, url}) => (async function(){
+	await Promise.all(images.map(({x, y, w, h, url}) => (async function(){
+		if (!url) return;
+		try {
 			var image = await loadImage(url);
-			ctx.drawImage(image, x, y, thumbnailWidth, thumbnailHeight);
-		})().catch(error => console.error(error.stack))));
-	}
+		} catch (error) {
+			console.error("failed to load image", url, error.message);
+			return;
+		}
+		ctx.drawImage(image, x, y, w, h);
+	})().catch(error => console.error("imageload", error.stack))));
 
-	if (icons.length) {
-		promises = promises.concat(icons.map(({x, y, url}) => (async function(){
-			var image = await loadImage(url);
-			ctx.drawImage(image, x, y, iconWidth, iconHeight);
-		})().catch(error => console.error(error.stack))));
-	}
-
-	await Promise.all(promises);
-	return {
-		imagesheet: canvas.toBuffer("image/png"),
-		thumbnails, icons
-	};
+	return canvas.toBuffer("image/png");
 }
-
-
-export async function makeImageSheetVrcUrl(pool, thumbnailUrls, iconUrls) {
-	var num = await putVrcUrl(pool, {type: "imagesheet"});
-	var key = `${pool}:${num}`;
-	var promise = createImageSheet(thumbnailUrls, iconUrls);
-	store[key] = promise;
-	promise.then(() => {
-		setTimeout(() => {
-			if (store[key] === promise) delete store[key];
-		}, 1000*60*10); // 10 mins;
-	});
-	promise.catch(error => {
-		console.error(error.stack);
-	});
-	var {thumbnails, icons} = await promise;
-	return {
-		vrcurl: num,
-		thumbnails, icons
-	}
-}
-
-export async function getImageSheet(pool, num) {
-	return (await store[`${pool}:${num}`])?.imagesheet;
-}

index.js

@@ -1,3 +1,161 @@
-import { app } from "./app.js";
+if (process.env.D!="BUG") console.debug = () => {};
+else console.debug(process.env);
 
-app.listen(process.env.PORT || 8142, process.env.ADDRESS);
+import "./util.js";
+import Koa from "koa";
+import Router from "@koa/router";
+import send from "koa-send";
+import qs from "qs";
+import { cachedVRCYoutubeSearch, getImageSheet } from "./VRCYoutubeSearch.js"
+import { resolveVrcUrl } from "./vrcurl.js";
+import { getVideoCaptionsCached } from "./youtube-captions.js";
+import { stringToBoolean } from "./util.js";
+import { readFileSync } from "fs";
+var shorturlmap = JSON.parse(readFileSync("./shorturlmap.json", "utf8"));
+
+var app = new Koa();
+var router = new Router();
+
+
+router.get(["/search", "/trending"], async ctx => {
+	if (ctx.path == "/trending") {
+		var input = {"type":"trending"};
+	} else {
+		var input = ctx.querystring.match(/[?&]input=(.*)/i)?.[1];
+		if (!input) {
+			ctx.status = 400;
+			ctx.body = "missing search query";
+			return;
+		}
+		input = decodeURIComponent(input).replace(/^.*→/, '').replaceAll("\u200b", '').trim();
+	}
+
+	var pqs = qs.parse(ctx.querystring, {duplicates: 'first'});
+
+	if (!pqs.pool || /[^a-z-_0-9]/.test(pqs.pool)) {
+		ctx.status = 400;
+		ctx.body = "invalid pool";
+		return;
+	}
+
+	var options = {
+		thumbnails: stringToBoolean(pqs.thumbnails),
+		icons: stringToBoolean(pqs.icons),
+		captions: stringToBoolean(pqs.captions),
+		mode: pqs.mode,
+		bp: pqs.bp
+	};
+
+	ctx.body = await cachedVRCYoutubeSearch(pqs.pool, input, options);
+});
+
+
+router.get("/vrcurl/:pool/:num", async ctx => {
+	var dest = await resolveVrcUrl(ctx.params.pool, ctx.params.num);
+	if (!dest) {
+		ctx.status = 404;
+		return;
+	}
+	switch (dest.type) {
+		case "redirect":
+			ctx.set("Referrer-Policy", "no-referrer");
+			ctx.redirect(dest.url);
+			break;
+		case "video":
+			if (ctx.get("User-Agent").includes("UnityWebRequest")) {
+				ctx.body = {captions: await getVideoCaptionsCached(dest.id)};
+			} else {
+				ctx.set("Referrer-Policy", "no-referrer");
+				ctx.redirect(`https://www.youtube.com/watch?v=${dest.id}`);
+			}
+			break;
+		case "imagesheet":
+			let buf = await getImageSheet(dest.key);
+			if (!buf) {
+				ctx.status = 404;
+				return;
+			}
+			ctx.body = buf;
+			ctx.type = "image/png";
+			break;
+		case "continuation":
+			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "continuation", for: dest.for, continuationData: dest.continuationData}, dest.options);
+			break;
+		case "trending":
+			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "trending", bp: dest.bp}, dest.options);
+			break;
+		case "captions":
+			ctx.body = await getVideoCaptionsCached(dest.videoId);
+			break;
+		default:
+			console.error("unknown vrcurl type", dest.type);
+			ctx.status = 500;
+	}
+});
+
+
+router.get("/robots.txt", ctx => {
+	ctx.body = `User-agent: *\nDisallow: /`;
+});
+
+router.get("/test.html", async ctx => {
+	await send(ctx, "test.html");
+});
+
+router.get("/", ctx => {
+	ctx.redirect("https://www.u2b.cx/");
+});
+
+
+
+
+
+app.use(async (ctx, next) => {
+	console.log("http", ctx.socket.remoteAddress, ctx.get("X-Forwarded-For"), ctx.get("Host") + ctx.originalUrl, '"'+ctx.get("User-Agent")+'"', ctx.get("Referer"));
+	try {
+		await next();
+	} catch (error) {
+		console.error(ctx.url, error.stack);
+		ctx.status = 500;
+		ctx.type = "text";
+		ctx.body = error.stack;
+	}
+});
+
+
+// short urls to work around https://feedback.vrchat.com/udon/p/vrcurlinputfield-incorrect-focus-issue-on-quest
+app.use(async (ctx, next) => {
+	var subdomain = ctx.hostname.match(/(.*).u2b.cx$/i)?.[1];
+	if (subdomain && !["api","api2","dev"].includes(subdomain)) {
+		if (shorturlmap[subdomain]) {
+			ctx.url = shorturlmap[subdomain] + ctx.url.slice(1);
+		} else {
+			ctx.status = 404;
+			return;
+		}
+	}
+	await next();
+});
+
+// work around vrchat json parser bug https://feedback.vrchat.com/udon/p/braces-inside-strings-in-vrcjson-can-fail-to-deserialize
+app.use(async (ctx, next) => {
+	await next();
+	if (ctx.type != "application/json") return;
+	ctx.body = structuredClone(ctx.body);
+	(function iterateObject(obj) {
+		for (var key in obj) {
+			if (typeof obj[key] == "string") {
+				obj[key] = obj[key].replace(/[\[\]{}]/g, chr => "\\u" + chr.charCodeAt(0).toString(16).padStart(4, '0'));
+			} else if (typeof obj[key] == "object") {
+				iterateObject(obj[key]);
+			}
+		}
+	})(ctx.body);
+	ctx.body = JSON.stringify(ctx.body).replaceAll("\\\\u", "\\u");
+});
+
+
+app.use(router.routes());
+app.use(router.allowedMethods());
+
+app.listen(process.env.PORT || 8142, process.env.ADDRESS);

package.json

@@ -2,11 +2,17 @@
 	"dependencies": {
 		"@keyv/sqlite": "^3.6.6",
 		"@koa/router": "^12.0.1",
-		"canvas": "^2.11.2",
+		"canvas": "^3.1.0",
+		"fast-xml-parser": "^4.3.4",
+		"got": "^14.4.1",
 		"keyv": "^4.5.4",
 		"koa": "^2.14.2",
 		"koa-send": "^5.0.1",
-		"node-fetch": "^3.3.2"
+		"potpack": "^2.0.0",
+		"qs": "^6.12.2"
+	},
+	"engines": {
+		"node": ">=18.0.0"
 	},
 	"type": "module"
 }

shorturlmap.json

@@ -0,0 +1,3 @@
+{
+	"hal": "/search?pool=hdays10000&thumbnails=yes&input="
+}

simpleYoutubeSearch.js

@@ -1,84 +0,0 @@
-var fetch = global.fetch || (await import("node-fetch")).default;
-
-
-export async function searchYouTubeVideos(query) {
-	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}&sp=EgIQAQ%253D%253D`;
-	var html = await fetch(url).then(res => res.text());
-
-	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
-	ytInitialData = JSON.parse(ytInitialData);
-
-	var videos = ytInitialData?.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))?.itemSectionRenderer?.contents?.filter(x => x.videoRenderer).map(parseVideoRendererData);
-	if (!videos) return {videos: []};
-
-	try {
-		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
-		ytcfg = JSON.parse(ytcfg);
-		var continuationData = {
-			context: ytcfg.INNERTUBE_CONTEXT,
-			continuation: ytInitialData.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
-		}
-	} catch (error) {
-		console.error(error.stack);
-	}
-
-	return {videos, continuationData};
-}
-
-
-export async function continueYouTubeVideoSearch(continuationData) {
-	var data = await fetch("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
-		method: "POST",
-		headers: {
-			"Content-Type": "application/json"
-		},
-		body: JSON.stringify(continuationData)
-	}).then(res => res.json());
-
-	var videos = data.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer)).itemSectionRenderer.contents.filter(x => x.videoRenderer).map(parseVideoRendererData);
-	var continuationToken = data.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
-
-	return {
-		videos,
-		continuationData: {
-			context: continuationData.context,
-			continuation: continuationToken
-		}
-	}
-}
-
-
-
-function parseVideoRendererData(data) {
-	data = data.videoRenderer;
-	return {
-		id: data.videoId,
-		live: Boolean(data.badges?.find(x => x.metadataBadgeRenderer?.style == "BADGE_STYLE_TYPE_LIVE_NOW")),
-		title: data.title?.runs?.[0]?.text,
-		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.reduce((str, obj) => str += obj.text, ""),
-		thumbnailUrl: data.thumbnail?.thumbnails?.find(x => x.width == 360 && x.height == 202)?.url || data.thumbnail?.thumbnails?.[0]?.url,
-		uploaded: data.publishedTimeText?.simpleText,
-		lengthText: data.lengthText?.simpleText,
-		longLengthText: data.lengthText?.accessibility?.accessibilityData?.label,
-		viewCountText: data.viewCountText?.runs ? data.viewCountText.runs.reduce((str, obj) => str += obj.text, "") : data.viewCountText?.simpleText,
-		shortViewCountText: data.shortViewCountText?.simpleText,
-		channel: {
-			name: data.ownerText?.runs?.[0]?.text,
-			id: data.ownerText?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId,
-			iconUrl: data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail?.thumbnails?.[0]?.url
-		}
-	};
-}
-
-
-
-/*
-var page1 = await searchYouTubeVideos("test video");
-console.log("page1", page1);
-var page2 = await continueYouTubeVideoSearch(page1.continuationData);
-console.log("page2", page2);
-var page3 = await continueYouTubeVideoSearch(page2.continuationData);
-console.log("page3", page3);
-console.log("videos", [...page1.videos, ...page2.videos, ...page3.videos]);
-debugger;
-*/

test.html

@@ -9,28 +9,36 @@
 </head><body>
 
 <div>
-	<label>search: <input id="input" type="text" value="nyan cat" /></label>
+	<label><button id="search">search:</button> <input id="input" type="text" value="nyan cat" /></label>
+	<br /><button id="trending">trending</button>
+	<br />
 	<label><input id="thumbnails" type="checkbox" checked>thumbnails</label>
 	<label><input id="icons" type="checkbox" checked>icons</label>
-	<button id="start">start</button>
+	<label><input id="latestontop" type="checkbox">latestontop</label>
 </div>
 
 <div id="output"></div>
 
-<button id="nextpage">next page</button>
+
+<div id="buttons"></div>
+<button id="nextpage" style="display: none">next page</button>
 
 <script>
 
 var num = 0;
 var lastData;
 
-start.onclick = () => {
+search.onclick = () => {
 	output.innerHTML = "";
-	loadData(`/search?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&input=${encodeURIComponent(input.value)}`);
+	loadData(`/search?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&mode=${latestontop.checked&&"latestontop"}&input=${encodeURIComponent(input.value)}`);
 };
-nextpage.onclick = () => loadData(`/vrcurl/test1000/${lastData.nextpage_vrcurl}`);
+trending.onclick = () => {
+	output.innerHTML = "";
+	loadData(`/trending?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}`);
+}
 
 async function loadData(url) {
+	buttons.innerHTML = '';
 	var data = await fetch(url).then(res => res.json());
 	var pre = document.createElement("pre");
 	pre.innerHTML = hljs.highlight(JSON.stringify(data, null, 4), {language: "json"}).value;
@@ -38,6 +46,12 @@ async function loadData(url) {
 	var img = document.createElement("img");
 	img.src = `/vrcurl/test1000/${data.imagesheet_vrcurl}`;
 	output.appendChild(img);
+	if (data.nextpage_vrcurl) {
+		buttons.innerHTML = `<button onclick="loadData('/vrcurl/test1000/${data.nextpage_vrcurl}')">next page</button>`;
+	}
+	if (data.tabs) {
+		buttons.innerHTML = data.tabs.map(t => `<button onclick="loadData('/vrcurl/test1000/${t.vrcurl}')">${t.name}</button>`).join('');
+	}
 	lastData = data;
 }
 

util.js

@@ -1,6 +1,49 @@
+import {readFileSync} from "fs";
+import got from "got";
+
+Array.prototype.filterMap = function(fn) {
+	var newarray = [];
+	for (var item of this) {
+		var ret = fn(item);
+		if (ret) newarray.push(ret);
+	}
+	return newarray;
+};
+
+Array.prototype.findMap = function(fn) {
+	for (var item of this) {
+		var ret = fn(item);
+		if (ret) return ret;
+	}
+}
+
+Object.prototype.deepFind = function (fn) {
+	return (function crawlObject(object) {
+		for (var key in object) {
+			var value = object[key];
+			if (fn(value)) return value;
+			if (typeof value == "object") crawlObject(value);
+		}
+	})(this);
+};
+
 export function stringToBoolean(str) {
 	if (str) {
 		if (!["0", "false", "off", "no", "null", "undefined", "nan"].includes(str.toLowerCase())) return true;
 	}
 	return false;
+}
+
+
+
+
+try {
+	var ips = readFileSync("ips.txt", "utf8").trim().split("\n");
+	console.log("using ips", ips);
+} catch (e) {}
+
+export function gotw(url, options = {}) {
+	if (ips) options.localAddress = ips[Math.floor(Math.random() * ips.length)];
+	//options.timeout = {request: 3000}; //"RequestError: Expected values which are `number` or `undefined`. Received values of type `Function`." what the fuck?
+	return got(url, options);
 }

youtube-captions.js

@@ -0,0 +1,54 @@
+import { XMLParser } from "fast-xml-parser";
+import { gotw } from "./util.js";
+
+var xmlParser = new XMLParser({
+	ignoreAttributes: false
+});
+
+async function getVideoData(videoId) {
+	var res = await gotw(`https://www.youtube.com/watch?v=${videoId}`);
+
+	var ytInitialPlayerResponse = res.body.match(/var ytInitialPlayerResponse = ({.*?});/)[1];
+	ytInitialPlayerResponse = JSON.parse(ytInitialPlayerResponse);
+
+	return ytInitialPlayerResponse;
+}
+
+async function getVideoCaptions(videoId) {
+	var ytInitialPlayerResponse = await getVideoData(videoId);
+	if (!ytInitialPlayerResponse.captions) return [];
+	var captionTracks = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks;
+	captionTracks = await Promise.all(captionTracks.map(captionTrack => (async () => {
+		try {
+			var promise = gotw(captionTrack.baseUrl);
+			var res = await promise;
+			if (res.statusCode != 200) throw new Error("unexpected status " + res.statusCode);
+			var xml = await promise.text();
+			if (!xml) throw new Error("empty response for caption track");
+			var parsed = xmlParser.parse(xml);
+			var lines = parsed.transcript.text;
+			if (!Array.isArray(lines)) lines = [lines];
+			lines = lines.map(({ "#text": text, "@_start": start, "@_dur": dur }) => ({ start: Number(start), dur: Number(dur), text }));
+			return {
+				name: captionTrack.name.simpleText,
+				id: captionTrack.vssId,
+				lines
+			};
+		} catch (error) {
+			console.error("caption track error", error.stack, videoId, captionTrack, xml, parsed, lines);
+		}
+	})()));
+	return captionTracks;
+}
+
+var cache = {};
+
+export async function getVideoCaptionsCached(videoId) {
+	if (!cache[videoId]) {
+		cache[videoId] = getVideoCaptions(videoId);
+		setTimeout(() => {
+			delete cache[videoId];
+		}, 1000*60*10); // 10 minutes
+	}
+	return await cache[videoId];
+}

youtube.js

@@ -0,0 +1,274 @@
+import { gotw } from "./util.js";
+
+
+
+
+export async function searchYouTubeVideos(query, sp = "EgIQAQ%253D%253D") {
+	console.debug("sp", sp);
+	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}${sp ? `$sp=${sp}` : ''}`;
+	var res = await gotw(url), html = res.body;
+ 
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)?.[1];
+	if (!ytInitialData) {
+		console.error("missing ytInitialData", query, res.status, html);
+	}
+	ytInitialData = JSON.parse(ytInitialData);
+	console.debug(ytInitialData);
+
+	var sectionListRendererContents = ytInitialData
+		.contents
+		.twoColumnSearchResultsRenderer
+		.primaryContents
+		.sectionListRenderer
+		.contents;
+
+	var videos = sectionListRendererContents
+		?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))
+		?.itemSectionRenderer
+		.contents
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
+	if (!videos) return {videos: []};
+
+	var latest = sectionListRendererContents
+		.find(x => x.itemSectionRenderer?.contents?.find(x => x.shelfRenderer))
+		?.itemSectionRenderer
+		.contents
+		.find(x => x.shelfRenderer?.title?.simpleText?.startsWith("Latest from"))
+		?.shelfRenderer
+		.content
+		.verticalListRenderer
+		.items
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
+	console.debug("latest", latest);
+	if (latest) videos = [...latest, ...videos];
+
+	console.debug(videos.length, "results");
+
+	try {
+		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
+		ytcfg = JSON.parse(ytcfg);
+		var continuationData = {
+			context: ytcfg.INNERTUBE_CONTEXT,
+			continuation: sectionListRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.continuationCommand.token
+		}
+	} catch (error) {
+		console.error(error.stack);
+	}
+
+	return {videos, continuationData};
+}
+
+
+
+
+export async function continueYouTubeVideoSearch(continuationData) {
+	var res = await gotw("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
+		method: "POST",
+		json: continuationData,
+		responseType: "json"
+	});
+	var data = res.body;
+	console.debug(data);
+
+	var continuationItems = data
+		.onResponseReceivedCommands[0]
+		.appendContinuationItemsAction
+		.continuationItems;
+	var videos = continuationItems
+		.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer))
+		.itemSectionRenderer
+		.contents
+		.filterMap(x => x.videoRenderer)
+		.map(parseVideoRendererData);
+	var continuationToken = continuationItems
+		.findMap(x => x.continuationItemRenderer)
+		?.continuationEndpoint
+		.continuationCommand
+		.token;
+	console.debug(videos.length, "results");
+
+
+	return {
+		videos,
+		continuationData: continuationToken ? {
+			context: continuationData.context,
+			continuation: continuationToken
+		} : null
+	}
+}
+
+
+
+
+
+
+
+
+
+export async function getYouTubePlaylist(playlistId) {
+	var res = await gotw("https://www.youtube.com/playlist?list=" + playlistId);
+	var html = res.body;
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+	ytInitialData = JSON.parse(ytInitialData);
+	console.debug(ytInitialData);
+
+	var playlistVideoRendererContents = ytInitialData
+		.contents
+		.twoColumnBrowseResultsRenderer
+		.tabs
+		.find(tab => tab.tabRenderer.selected)
+		.tabRenderer
+		.content
+		.sectionListRenderer
+		.contents
+		.findMap(x => x.itemSectionRenderer?.contents)
+		.findMap(x => x.playlistVideoListRenderer?.contents);
+	var videos = playlistVideoRendererContents
+		.filterMap(x => x.playlistVideoRenderer)
+		.map(parseVideoRendererData);
+	if (!videos) return {videos: []};
+	console.debug(videos.length, "results");
+
+	try {
+		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
+		ytcfg = JSON.parse(ytcfg);
+		var continuationData = {
+			context: ytcfg.INNERTUBE_CONTEXT,
+			continuation: playlistVideoRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.commandExecutorCommand.commands.findMap(x=>x.continuationCommand).token
+		}
+	} catch (error) {
+		console.error(error.stack);
+	}
+	return {videos, continuationData};
+}
+
+
+
+
+export async function continueYouTubePlaylist(continuationData) {
+	var res = await gotw("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false", {
+		method: "POST",
+		json: continuationData,
+		responseType: "json"
+	});
+	var data = res.body;
+	console.debug(data);
+
+	if (!data.onResponseReceivedActions) return {videos:[]};
+	var continuationItems = data
+		.onResponseReceivedActions[0]
+		.appendContinuationItemsAction
+		.continuationItems;
+	var videos = continuationItems
+		.filterMap(x => x.playlistVideoRenderer)
+		.map(parseVideoRendererData);
+	var continuationToken = continuationItems
+		.findMap(x => x.continuationItemRenderer)
+		?.continuationEndpoint
+		.continuationCommand
+		.token;
+	console.debug(videos.length, "results");
+
+	return {
+		videos,
+		continuationData: continuationToken ? {
+			context: continuationData.context,
+			continuation: continuationToken
+		} : null
+	}
+}
+
+
+
+
+
+
+
+
+
+export async function getTrending(bp) {
+	var url = `https://www.youtube.com/feed/trending`;
+	if (bp) url += `?bp=${bp}`;
+	var res = await gotw(url);
+	var html = res.body;
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+	ytInitialData = JSON.parse(ytInitialData);
+
+	var tabs = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.map(t => {
+		return {
+			name: t.tabRenderer.title,
+			//url: `https://www.youtube.com` + t.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url
+			bp: t.tabRenderer.endpoint.browseEndpoint.params
+		}
+	});
+
+	var videos = ytInitialData
+		.contents
+		.twoColumnBrowseResultsRenderer
+		.tabs
+		.find(tab => tab.tabRenderer.selected)
+		.tabRenderer
+		.content
+		.sectionListRenderer
+		.contents
+		// regular trending in sections with shelfRenderer without title
+		.filterMap(x => {
+			var shelfRenderer = x.itemSectionRenderer.contents.findMap(x => x.shelfRenderer);
+			if (shelfRenderer && !shelfRenderer.title) {
+				return shelfRenderer
+					.content
+					.expandedShelfContentsRenderer
+					.items
+					.filterMap(x => x.videoRenderer)
+					.map(parseVideoRendererData)
+			};
+		})
+		.flat();
+
+
+	return {tabs, videos};
+}
+
+
+
+
+
+
+
+
+
+
+
+
+Object.prototype.concatRunsText = function concatRunsText() {
+	return this.reduce((str, obj) => str += obj.text, "");
+};
+
+function parseVideoRendererData(data) {
+	return {
+		id: data.videoId,
+		live: Boolean(data.badges?.find(x => x.metadataBadgeRenderer?.style == "BADGE_STYLE_TYPE_LIVE_NOW")),
+		title: data.title?.runs?.concatRunsText(),
+		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.concatRunsText() 
+			|| data.descriptionSnippet?.runs?.concatRunsText(),
+		//thumbnailUrl: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138))?.url || data.thumbnail?.thumbnails?.[0]?.url,
+		thumbnail: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138)) || data.thumbnail?.thumbnails?.[0],
+		/*thumbnail: {
+			url: `https://i.ytimg.com/vi/${data.videoId}/mqdefault.jpg`,
+			width: 320,
+			height: 180
+		},*/
+		uploaded: data.publishedTimeText?.simpleText || data.videoInfo?.runs?.[2]?.text,
+		lengthText: data.lengthText?.simpleText,
+		longLengthText: data.lengthText?.accessibility?.accessibilityData?.label,
+		viewCountText: data.viewCountText?.runs ? data.viewCountText.runs.concatRunsText() : data.viewCountText?.simpleText,
+		shortViewCountText: data.shortViewCountText?.simpleText || data.videoInfo?.runs?.[0]?.text,
+		channel: {
+			name: (data.ownerText || data.shortBylineText)?.runs?.concatRunsText(),
+			id: (data.ownerText || data.shortBylineText)?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId,
+			iconUrl: data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail?.thumbnails?.[0]?.url
+		}
+	};
+}