youtube antibotted the captions

gives blank file now without special code added to url

.gitignore

@@ -1,2 +1,4 @@
 node_modules
-*.sqlite
+*.sqlite
+*.sqlite-journal
+ips.txt

README.md

@@ -71,9 +71,11 @@ Gets Trending YouTube videos. Identical to `/search` but without `input` paramet
 ## GET `/vrcurl/{pool}/{index}`
 
 - `{pool}`: must be same as pool param in search endpoint.
-- `{index}`: vrcurl index number
+- `{index}`: vrcurl index number as specified in response data
 
-Response may be 302 redirect to youtube url, `image/png` for imagesheet, `application/json` for next page (see response format above) or trending tab or video json data (see below).
+For youtube videos, if `User-Agent` header includes `UnityWebRequest`, then response is `application/json` for video metadata (captions). Otherwise, it is 302 redirect to youtube URL.
+
+For image sheet, response is `image/png`. For next page or trending tab, response is `application/json`.
 
 ### Video metadata JSON format
 

VRCYoutubeSearch.js

@@ -1,43 +1,42 @@
 import { searchYouTubeVideos, continueYouTubeVideoSearch, getYouTubePlaylist, continueYouTubePlaylist, getTrending } from "./youtube.js";
 import { putVrcUrl } from "./vrcurl.js";
-import { makeImageSheetVrcUrl } from "./imagesheet.js";
+import { createImageSheet } from "./imagesheet.js";
 
 var cache = {};
 
-
-
 export async function cachedVRCYoutubeSearch(pool, query, options) {
 	var key = JSON.stringify([pool, query, options]);
 	if (!cache[key]) {
-		cache[key] = VRCYoutubeSearch(pool, query, options);
+		cache[key] = VRCYoutubeSearch(pool, query, options, key);
 		setTimeout(() => {
 			delete cache[key];
 		}, 1000*60*10); // 10 mins
 	}
-	return await cache[key];
+	return (await cache[key])?.response;
+}
+
+export async function getImageSheet(key) {
+	return await (await cache[key])?.imagesheet
 }
 
 
-
+async function VRCYoutubeSearch(pool, query, options = {}, key) {
-
-async function VRCYoutubeSearch(pool, query, options = {}) {
 	console.log("search", pool, JSON.stringify(query), JSON.stringify(options));
-	var data = {results: []};
+	var response = {results: []};
 
 	if (typeof query == "object") {
 		switch (query.type) {
 			case "trending":
 				var {videos, tabs} = await getTrending(query.bp);
-				data.tabs = [];
+				response.tabs = [];
 				for (let tab of tabs) {
-					data.tabs.push({
+					response.tabs.push({
 						name: tab.name,
 						vrcurl: await putVrcUrl(pool, {type: "trending", bp: tab.bp, options})
 					});
 				}
 				break;
 			case "continuation":
-				//var {videos, continuationData} = await [query.for == "playlist" ? continueYouTubePlaylist : continueYouTubeVideoSearch](query.continuationData);
 				if (query.for == "playlist") {
 					var {videos, continuationData} = await continueYouTubePlaylist(query.continuationData);
 				} else {
@@ -46,7 +45,7 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 				break;
 		}
 	} else {
-		var playlistId = query.match(/list=(PL[a-zA-Z0-9-_]{32})/)?.[1];
+		var playlistId = query.match(/list=(PL(?:[a-zA-Z0-9-_]{32}|[0-9A-F]{16}))/)?.[1];
 		if (playlistId) console.log("playlistId:", playlistId);
 		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query, options.bp || (options.mode == "latestontop" ? null : undefined));
 	}
@@ -55,14 +54,14 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 
 	if (options.thumbnails) {
 		videos.forEach(video => {
-			if (playlistId) video.thumbnail = {
+			if (playlistId || query?.for == "playlist") video.thumbnail = {
 				url: `https://i.ytimg.com/vi/${video.id}/default.jpg`,
 				width: 120,
 				height: 90
 			};
 			else {
 				video.thumbnail ||= {
-					url: `https://i.ytimg.com/vi/${data.videoId}/mqdefault.jpg`,
+					url: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`,
 					width: 320,
 					height: 180
 				};
@@ -86,7 +85,8 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 
 	if (images.length) {
 		try {
-			data.imagesheet_vrcurl = await makeImageSheetVrcUrl(pool, images, !playlistId && !options.icons);
+			response.imagesheet_vrcurl = await putVrcUrl(pool, {type: "imagesheet", key});
+			var imagesheet = createImageSheet(images, !playlistId && !options.icons);
 		} catch (error) {
 			console.error(error.stack);
 		}
@@ -112,15 +112,15 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 			video.captions_vrcurl = await putVrcUrl(pool, {type: "captions", videoId: video.id});
 		}
 		delete video.channel.iconUrl;
-		data.results.push(video);
+		response.results.push(video);
 	}
 
-	if (continuationData) data.nextpage_vrcurl = await putVrcUrl(pool, {
+	if (continuationData) response.nextpage_vrcurl = await putVrcUrl(pool, {
 		type: "continuation",
 		for: query.for || (playlistId ? "playlist" : "search"),
 		continuationData,
 		options
 	});
 
-	return data;
+	return {response, imagesheet};
 }

imagesheet.js

@@ -1,10 +1,7 @@
 import { createCanvas, loadImage } from 'canvas';
 import potpack from 'potpack';
-import { putVrcUrl } from './vrcurl.js';
 
-var store = {};
+export async function createImageSheet(images /*[{width, height, url}]*/, legacyMode) {
-
-async function createImageSheet(images /*[{width, height, url}]*/, legacyMode) {
 	images.forEach(image => {
 		image.w = image.width;
 		image.h = image.height;
@@ -41,28 +38,5 @@ async function createImageSheet(images /*[{width, height, url}]*/, legacyMode) {
 		ctx.drawImage(image, x, y, w, h);
 	})().catch(error => console.error("imageload", error.stack))));
 
-	return {
+	return canvas.toBuffer("image/png");
-		imagesheet: canvas.toBuffer("image/png"),
-		images
-	};
 }
-
-export async function makeImageSheetVrcUrl(pool, images, legacyMode) {
-	var num = await putVrcUrl(pool, {type: "imagesheet"});
-	var key = `${pool}:${num}`;
-	var promise = createImageSheet(images, legacyMode);
-	store[key] = promise;
-	promise.then(() => {
-		setTimeout(() => {
-			if (store[key] === promise) delete store[key];
-		}, 1000*60*10); // 10 min;
-	});
-	promise.catch(error => {
-		console.error(error.stack);
-	});
-	return num;
-}
-
-export async function getImageSheet(pool, num) {
-	return (await store[`${pool}:${num}`])?.imagesheet;
-}

index.js

@@ -1,12 +1,12 @@
 if (process.env.D!="BUG") console.debug = () => {};
 else console.debug(process.env);
+
 import "./util.js";
 import Koa from "koa";
 import Router from "@koa/router";
 import send from "koa-send";
 import qs from "qs";
-import { cachedVRCYoutubeSearch } from "./VRCYoutubeSearch.js"
+import { cachedVRCYoutubeSearch, getImageSheet } from "./VRCYoutubeSearch.js"
-import { getImageSheet } from "./imagesheet.js";
 import { resolveVrcUrl } from "./vrcurl.js";
 import { getVideoCaptionsCached } from "./youtube-captions.js";
 import { stringToBoolean } from "./util.js";
@@ -32,7 +32,7 @@ router.get(["/search", "/trending"], async ctx => {
 
 	var pqs = qs.parse(ctx.querystring, {duplicates: 'first'});
 
-	if (!pqs.pool || !/^[a-z-_]+\d*$/.test(pqs.pool)) {
+	if (!pqs.pool || /[^a-z-_0-9]/.test(pqs.pool)) {
 		ctx.status = 400;
 		ctx.body = "invalid pool";
 		return;
@@ -58,17 +58,19 @@ router.get("/vrcurl/:pool/:num", async ctx => {
 	}
 	switch (dest.type) {
 		case "redirect":
+			ctx.set("Referrer-Policy", "no-referrer");
 			ctx.redirect(dest.url);
 			break;
 		case "video":
 			if (ctx.get("User-Agent").includes("UnityWebRequest")) {
 				ctx.body = {captions: await getVideoCaptionsCached(dest.id)};
 			} else {
+				ctx.set("Referrer-Policy", "no-referrer");
 				ctx.redirect(`https://www.youtube.com/watch?v=${dest.id}`);
 			}
 			break;
 		case "imagesheet":
-			let buf = await getImageSheet(ctx.params.pool, ctx.params.num);
+			let buf = await getImageSheet(dest.key);
 			if (!buf) {
 				ctx.status = 404;
 				return;
@@ -109,6 +111,7 @@ router.get("/", ctx => {
 
 
 app.use(async (ctx, next) => {
+	console.log("http", ctx.socket.remoteAddress, ctx.get("X-Forwarded-For"), ctx.get("Host") + ctx.originalUrl, '"'+ctx.get("User-Agent")+'"', ctx.get("Referer"));
 	try {
 		await next();
 	} catch (error) {
@@ -155,4 +158,4 @@ app.use(async (ctx, next) => {
 app.use(router.routes());
 app.use(router.allowedMethods());
 
-app.listen(process.env.PORT || 8142, process.env.ADDRESS);
+app.listen(process.env.PORT || 8142, process.env.ADDRESS);

package-lock.json

@@ -7,7 +7,7 @@
 			"dependencies": {
 				"@keyv/sqlite": "^3.6.6",
 				"@koa/router": "^12.0.1",
-				"canvas": "^2.11.2",
+				"canvas": "^3.1.0",
 				"fast-xml-parser": "^4.3.4",
 				"got": "^14.4.1",
 				"keyv": "^4.5.4",
@@ -261,6 +261,37 @@
 			"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
 			"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
 		},
+		"node_modules/base64-js": {
+			"version": "1.5.1",
+			"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+			"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/feross"
+				},
+				{
+					"type": "patreon",
+					"url": "https://www.patreon.com/feross"
+				},
+				{
+					"type": "consulting",
+					"url": "https://feross.org/support"
+				}
+			],
+			"license": "MIT"
+		},
+		"node_modules/bl": {
+			"version": "4.1.0",
+			"resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+			"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+			"license": "MIT",
+			"dependencies": {
+				"buffer": "^5.5.0",
+				"inherits": "^2.0.4",
+				"readable-stream": "^3.4.0"
+			}
+		},
 		"node_modules/brace-expansion": {
 			"version": "1.1.11",
 			"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -270,6 +301,30 @@
 				"concat-map": "0.0.1"
 			}
 		},
+		"node_modules/buffer": {
+			"version": "5.7.1",
+			"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+			"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/feross"
+				},
+				{
+					"type": "patreon",
+					"url": "https://www.patreon.com/feross"
+				},
+				{
+					"type": "consulting",
+					"url": "https://feross.org/support"
+				}
+			],
+			"license": "MIT",
+			"dependencies": {
+				"base64-js": "^1.3.1",
+				"ieee754": "^1.1.13"
+			}
+		},
 		"node_modules/cacache": {
 			"version": "15.3.0",
 			"resolved": "https://registry.npmjs.org/cacache/-/cacache-15.3.0.tgz",
@@ -381,19 +436,25 @@
 			}
 		},
 		"node_modules/canvas": {
-			"version": "2.11.2",
+			"version": "3.1.0",
-			"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz",
+			"resolved": "https://registry.npmjs.org/canvas/-/canvas-3.1.0.tgz",
-			"integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==",
+			"integrity": "sha512-tTj3CqqukVJ9NgSahykNwtGda7V33VLObwrHfzT0vqJXu7J4d4C/7kQQW3fOEGDfZZoILPut5H00gOjyttPGyg==",
 			"hasInstallScript": true,
+			"license": "MIT",
 			"dependencies": {
-				"@mapbox/node-pre-gyp": "^1.0.0",
+				"node-addon-api": "^7.0.0",
-				"nan": "^2.17.0",
+				"prebuild-install": "^7.1.1"
-				"simple-get": "^3.0.3"
 			},
 			"engines": {
-				"node": ">=6"
+				"node": "^18.12.0 || >= 20.9.0"
 			}
 		},
+		"node_modules/canvas/node_modules/node-addon-api": {
+			"version": "7.1.1",
+			"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz",
+			"integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==",
+			"license": "MIT"
+		},
 		"node_modules/chownr": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
@@ -486,14 +547,18 @@
 			}
 		},
 		"node_modules/decompress-response": {
-			"version": "4.2.1",
+			"version": "6.0.0",
-			"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
+			"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
-			"integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
+			"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+			"license": "MIT",
 			"dependencies": {
-				"mimic-response": "^2.0.0"
+				"mimic-response": "^3.1.0"
 			},
 			"engines": {
-				"node": ">=8"
+				"node": ">=10"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
 		"node_modules/deep-equal": {
@@ -501,6 +566,15 @@
 			"resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
 			"integrity": "sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw=="
 		},
+		"node_modules/deep-extend": {
+			"version": "0.6.0",
+			"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+			"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
+			"license": "MIT",
+			"engines": {
+				"node": ">=4.0.0"
+			}
+		},
 		"node_modules/defer-to-connect": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
@@ -573,25 +647,13 @@
 				"node": ">= 0.8"
 			}
 		},
-		"node_modules/encoding": {
+		"node_modules/end-of-stream": {
-			"version": "0.1.13",
+			"version": "1.4.4",
-			"resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
+			"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
-			"integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
+			"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
-			"optional": true,
+			"license": "MIT",
 			"dependencies": {
-				"iconv-lite": "^0.6.2"
+				"once": "^1.4.0"
-			}
-		},
-		"node_modules/encoding/node_modules/iconv-lite": {
-			"version": "0.6.3",
-			"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
-			"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
-			"optional": true,
-			"dependencies": {
-				"safer-buffer": ">= 2.1.2 < 3.0.0"
-			},
-			"engines": {
-				"node": ">=0.10.0"
 			}
 		},
 		"node_modules/env-paths": {
@@ -633,10 +695,19 @@
 			"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
 			"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
 		},
+		"node_modules/expand-template": {
+			"version": "2.0.3",
+			"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+			"integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+			"license": "(MIT OR WTFPL)",
+			"engines": {
+				"node": ">=6"
+			}
+		},
 		"node_modules/fast-xml-parser": {
-			"version": "4.3.4",
+			"version": "4.5.0",
-			"resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.4.tgz",
+			"resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz",
-			"integrity": "sha512-utnwm92SyozgA3hhH2I8qldf2lBqm6qHOICawRNRFu1qMe3+oqr+GcXjGqTmXTMGE5T4eC03kr/rlh5C1IRdZA==",
+			"integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==",
 			"funding": [
 				{
 					"type": "github",
@@ -647,6 +718,7 @@
 					"url": "https://paypal.me/naturalintelligence"
 				}
 			],
+			"license": "MIT",
 			"dependencies": {
 				"strnum": "^1.0.5"
 			},
@@ -670,6 +742,12 @@
 				"node": ">= 0.6"
 			}
 		},
+		"node_modules/fs-constants": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+			"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+			"license": "MIT"
+		},
 		"node_modules/fs-minipass": {
 			"version": "2.1.0",
 			"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -742,6 +820,12 @@
 				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
+		"node_modules/github-from-package": {
+			"version": "0.0.0",
+			"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+			"integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
+			"license": "MIT"
+		},
 		"node_modules/glob": {
 			"version": "7.2.3",
 			"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
@@ -797,31 +881,6 @@
 				"url": "https://github.com/sindresorhus/got?sponsor=1"
 			}
 		},
-		"node_modules/got/node_modules/decompress-response": {
-			"version": "6.0.0",
-			"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
-			"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
-			"dependencies": {
-				"mimic-response": "^3.1.0"
-			},
-			"engines": {
-				"node": ">=10"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/got/node_modules/mimic-response": {
-			"version": "3.1.0",
-			"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
-			"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
-			"engines": {
-				"node": ">=10"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
 		"node_modules/graceful-fs": {
 			"version": "4.2.11",
 			"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -978,6 +1037,26 @@
 				"ms": "^2.0.0"
 			}
 		},
+		"node_modules/ieee754": {
+			"version": "1.2.1",
+			"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+			"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/feross"
+				},
+				{
+					"type": "patreon",
+					"url": "https://www.patreon.com/feross"
+				},
+				{
+					"type": "consulting",
+					"url": "https://feross.org/support"
+				}
+			],
+			"license": "BSD-3-Clause"
+		},
 		"node_modules/imurmurhash": {
 			"version": "0.1.4",
 			"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
@@ -1016,6 +1095,12 @@
 			"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
 			"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
 		},
+		"node_modules/ini": {
+			"version": "1.3.8",
+			"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+			"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
+			"license": "ISC"
+		},
 		"node_modules/ip-address": {
 			"version": "9.0.5",
 			"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
@@ -1274,11 +1359,12 @@
 			}
 		},
 		"node_modules/mimic-response": {
-			"version": "2.1.0",
+			"version": "3.1.0",
-			"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
+			"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
-			"integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==",
+			"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
+			"license": "MIT",
 			"engines": {
-				"node": ">=8"
+				"node": ">=10"
 			},
 			"funding": {
 				"url": "https://github.com/sponsors/sindresorhus"
@@ -1295,6 +1381,15 @@
 				"node": "*"
 			}
 		},
+		"node_modules/minimist": {
+			"version": "1.2.8",
+			"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+			"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+			"license": "MIT",
+			"funding": {
+				"url": "https://github.com/sponsors/ljharb"
+			}
+		},
 		"node_modules/minipass": {
 			"version": "3.3.6",
 			"resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
@@ -1394,15 +1489,22 @@
 				"node": ">=10"
 			}
 		},
+		"node_modules/mkdirp-classic": {
+			"version": "0.5.3",
+			"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
+			"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
+			"license": "MIT"
+		},
 		"node_modules/ms": {
 			"version": "2.1.2",
 			"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
 			"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
 		},
-		"node_modules/nan": {
+		"node_modules/napi-build-utils": {
-			"version": "2.18.0",
+			"version": "2.0.0",
-			"resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz",
+			"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",
-			"integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w=="
+			"integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==",
+			"license": "MIT"
 		},
 		"node_modules/negotiator": {
 			"version": "0.6.3",
@@ -1412,6 +1514,18 @@
 				"node": ">= 0.6"
 			}
 		},
+		"node_modules/node-abi": {
+			"version": "3.74.0",
+			"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.74.0.tgz",
+			"integrity": "sha512-c5XK0MjkGBrQPGYG24GBADZud0NCbznxNx0ZkS+ebUTrmV1qTDxPxSL8zEAPURXSbLRWVexxmP4986BziahL5w==",
+			"license": "MIT",
+			"dependencies": {
+				"semver": "^7.3.5"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/node-addon-api": {
 			"version": "4.3.0",
 			"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz",
@@ -1616,6 +1730,32 @@
 			"resolved": "https://registry.npmjs.org/potpack/-/potpack-2.0.0.tgz",
 			"integrity": "sha512-Q+/tYsFU9r7xoOJ+y/ZTtdVQwTWfzjbiXBDMM/JKUux3+QPP02iUuIoeBQ+Ot6oEDlC+/PGjB/5A3K7KKb7hcw=="
 		},
+		"node_modules/prebuild-install": {
+			"version": "7.1.3",
+			"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
+			"integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==",
+			"license": "MIT",
+			"dependencies": {
+				"detect-libc": "^2.0.0",
+				"expand-template": "^2.0.3",
+				"github-from-package": "0.0.0",
+				"minimist": "^1.2.3",
+				"mkdirp-classic": "^0.5.3",
+				"napi-build-utils": "^2.0.0",
+				"node-abi": "^3.3.0",
+				"pump": "^3.0.0",
+				"rc": "^1.2.7",
+				"simple-get": "^4.0.0",
+				"tar-fs": "^2.0.0",
+				"tunnel-agent": "^0.6.0"
+			},
+			"bin": {
+				"prebuild-install": "bin.js"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/promise-inflight": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
@@ -1635,6 +1775,16 @@
 				"node": ">=10"
 			}
 		},
+		"node_modules/pump": {
+			"version": "3.0.2",
+			"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz",
+			"integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==",
+			"license": "MIT",
+			"dependencies": {
+				"end-of-stream": "^1.1.0",
+				"once": "^1.3.1"
+			}
+		},
 		"node_modules/qs": {
 			"version": "6.12.2",
 			"resolved": "https://registry.npmjs.org/qs/-/qs-6.12.2.tgz",
@@ -1660,6 +1810,21 @@
 				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
+		"node_modules/rc": {
+			"version": "1.2.8",
+			"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+			"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+			"license": "(BSD-2-Clause OR MIT OR Apache-2.0)",
+			"dependencies": {
+				"deep-extend": "^0.6.0",
+				"ini": "~1.3.0",
+				"minimist": "^1.2.0",
+				"strip-json-comments": "~2.0.1"
+			},
+			"bin": {
+				"rc": "cli.js"
+			}
+		},
 		"node_modules/readable-stream": {
 			"version": "3.6.2",
 			"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -1863,14 +2028,30 @@
 					"type": "consulting",
 					"url": "https://feross.org/support"
 				}
-			]
+			],
+			"license": "MIT"
 		},
 		"node_modules/simple-get": {
-			"version": "3.1.1",
+			"version": "4.0.1",
-			"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz",
+			"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
-			"integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==",
+			"integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/feross"
+				},
+				{
+					"type": "patreon",
+					"url": "https://www.patreon.com/feross"
+				},
+				{
+					"type": "consulting",
+					"url": "https://feross.org/support"
+				}
+			],
+			"license": "MIT",
 			"dependencies": {
-				"decompress-response": "^4.2.0",
+				"decompress-response": "^6.0.0",
 				"once": "^1.3.1",
 				"simple-concat": "^1.0.0"
 			}
@@ -1993,15 +2174,25 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/strip-json-comments": {
+			"version": "2.0.1",
+			"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+			"integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
+			"license": "MIT",
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
 		"node_modules/strnum": {
 			"version": "1.0.5",
 			"resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz",
 			"integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA=="
 		},
 		"node_modules/tar": {
-			"version": "6.2.0",
+			"version": "6.2.1",
-			"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz",
+			"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz",
-			"integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==",
+			"integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==",
+			"license": "ISC",
 			"dependencies": {
 				"chownr": "^2.0.0",
 				"fs-minipass": "^2.0.0",
@@ -2014,6 +2205,40 @@
 				"node": ">=10"
 			}
 		},
+		"node_modules/tar-fs": {
+			"version": "2.1.2",
+			"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
+			"integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
+			"license": "MIT",
+			"dependencies": {
+				"chownr": "^1.1.1",
+				"mkdirp-classic": "^0.5.2",
+				"pump": "^3.0.0",
+				"tar-stream": "^2.1.4"
+			}
+		},
+		"node_modules/tar-fs/node_modules/chownr": {
+			"version": "1.1.4",
+			"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+			"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+			"license": "ISC"
+		},
+		"node_modules/tar-stream": {
+			"version": "2.2.0",
+			"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+			"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+			"license": "MIT",
+			"dependencies": {
+				"bl": "^4.0.3",
+				"end-of-stream": "^1.4.1",
+				"fs-constants": "^1.0.0",
+				"inherits": "^2.0.3",
+				"readable-stream": "^3.1.1"
+			},
+			"engines": {
+				"node": ">=6"
+			}
+		},
 		"node_modules/tar/node_modules/minipass": {
 			"version": "5.0.0",
 			"resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
@@ -2043,6 +2268,18 @@
 				"node": ">=0.6.x"
 			}
 		},
+		"node_modules/tunnel-agent": {
+			"version": "0.6.0",
+			"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+			"integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"safe-buffer": "^5.0.1"
+			},
+			"engines": {
+				"node": "*"
+			}
+		},
 		"node_modules/type-fest": {
 			"version": "4.21.0",
 			"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.21.0.tgz",

package.json

@@ -2,7 +2,7 @@
 	"dependencies": {
 		"@keyv/sqlite": "^3.6.6",
 		"@koa/router": "^12.0.1",
-		"canvas": "^2.11.2",
+		"canvas": "^3.1.0",
 		"fast-xml-parser": "^4.3.4",
 		"got": "^14.4.1",
 		"keyv": "^4.5.4",

test.html

@@ -14,7 +14,7 @@
 	<br />
 	<label><input id="thumbnails" type="checkbox" checked>thumbnails</label>
 	<label><input id="icons" type="checkbox" checked>icons</label>
-	<label><input id="latestontop" type="checkbox" checked>latestontop</label>
+	<label><input id="latestontop" type="checkbox">latestontop</label>
 </div>
 
 <div id="output"></div>

youtube-captions.js

@@ -8,7 +8,7 @@ var xmlParser = new XMLParser({
 async function getVideoData(videoId) {
 	var res = await gotw(`https://www.youtube.com/watch?v=${videoId}`);
 
-	var ytInitialPlayerResponse = res.body.match(/var ytInitialPlayerResponse = ({.*});/)[1];
+	var ytInitialPlayerResponse = res.body.match(/var ytInitialPlayerResponse = ({.*?});/)[1];
 	ytInitialPlayerResponse = JSON.parse(ytInitialPlayerResponse);
 
 	return ytInitialPlayerResponse;
@@ -20,7 +20,11 @@ async function getVideoCaptions(videoId) {
 	var captionTracks = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks;
 	captionTracks = await Promise.all(captionTracks.map(captionTrack => (async () => {
 		try {
-			var xml = await gotw(captionTrack.baseUrl, {resolveBodyOnly: true});
+			var promise = gotw(captionTrack.baseUrl);
+			var res = await promise;
+			if (res.statusCode != 200) throw new Error("unexpected status " + res.statusCode);
+			var xml = await promise.text();
+			if (!xml) throw new Error("empty response for caption track");
 			var parsed = xmlParser.parse(xml);
 			var lines = parsed.transcript.text;
 			if (!Array.isArray(lines)) lines = [lines];

youtube.js

@@ -114,7 +114,7 @@ export async function getYouTubePlaylist(playlistId) {
 	ytInitialData = JSON.parse(ytInitialData);
 	console.debug(ytInitialData);
 
-	var sectionListRendererContents = ytInitialData
+	var playlistVideoRendererContents = ytInitialData
 		.contents
 		.twoColumnBrowseResultsRenderer
 		.tabs
@@ -122,10 +122,10 @@ export async function getYouTubePlaylist(playlistId) {
 		.tabRenderer
 		.content
 		.sectionListRenderer
-		.contents;
+		.contents
-	var videos = sectionListRendererContents
 		.findMap(x => x.itemSectionRenderer?.contents)
-		.findMap(x => x.playlistVideoListRenderer?.contents)
+		.findMap(x => x.playlistVideoListRenderer?.contents);
+	var videos = playlistVideoRendererContents
 		.filterMap(x => x.playlistVideoRenderer)
 		.map(parseVideoRendererData);
 	if (!videos) return {videos: []};
@@ -136,7 +136,7 @@ export async function getYouTubePlaylist(playlistId) {
 		ytcfg = JSON.parse(ytcfg);
 		var continuationData = {
 			context: ytcfg.INNERTUBE_CONTEXT,
-			continuation: sectionListRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.continuationCommand.token
+			continuation: playlistVideoRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.commandExecutorCommand.commands.findMap(x=>x.continuationCommand).token
 		}
 	} catch (error) {
 		console.error(error.stack);
@@ -162,8 +162,7 @@ export async function continueYouTubePlaylist(continuationData) {
 		.appendContinuationItemsAction
 		.continuationItems;
 	var videos = continuationItems
-		.findMap(x => x.itemSectionRenderer?.contents)
+		.filterMap(x => x.playlistVideoRenderer)
-		.filterMap(x => x.playlistVideoListRenderer)
 		.map(parseVideoRendererData);
 	var continuationToken = continuationItems
 		.findMap(x => x.continuationItemRenderer)