manual input parsing

regression in imagesheet need bin packing

.gitignore

@@ -1,2 +1,2 @@
 node_modules
-vrcurl.sqlite
+*.sqlite

.vscode/launch.json

@@ -14,7 +14,10 @@
 			"program": "${workspaceFolder}\\index.js",
 			"outFiles": [
 				"${workspaceFolder}/**/*.js"
-			]
+			],
+			"env": {
+				"D":"BUG"
+			}
 		}
 	]
 }

README.md

@@ -8,15 +8,18 @@ Test in browser: https://api.u2b.cx/test.html
 
 ## GET `/search`
 
+Get YouTube videos for a search query.
+
 ### Required query parameters
 
 - `pool`: id of the VRCUrl pool, only letters numbers hyphens or underscores, optionally followed by an integer for pool size.
-- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed.
+- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed. THIS MUST BE THE LAST QUERY PARAMETER AS ALL CHARS AFTER IT ARE CAPTURED VERBATIM (so you can type & etc without encoding)
 
 ### Optional query parameters
 
 - `thumbnails`: set to `1`, `true`, `yes`, `on` or whatever to load thumbnails
 - `icons`: set to `1`, `true`, `yes`, `on` or whatever to load channel icons
+- `captions`: set to `1`, `true`, `yes`, `on` or whatever if you need access to closed captioning data
 
 ### Example URL
 
@@ -36,13 +39,13 @@ JSON object:
 	- `description`: (string) short truncated description snippet i.e. `"http://nyan.cat/ Original song : http://momolabo.lolipop.jp/nyancatsong/Nyan/"`
 	- `lengthText`?: (string) i.e. `"3:37"`
 	- `longLengthText`?: (string) i.e. `"3 minutes, 37 seconds"`
-	- `viewCountText`: (string) i.e. `"2,552,243 views"` or `"575 watching"` for live streams
-	- `shortViewCountText`?: (string) i.e. `"2.5M views"`
+	- `viewCountText`?: (string) i.e. `"2,552,243 views"` or `"575 watching"` for live streams (playlist results don't have this)
+	- `shortViewCountText`?: (string) i.e. `"2.5M views"` (streams don't have this)
 	- `uploaded`: (string) i.e. `"12 years ago"`
 	- `channel`: (object)
 		- `name`: (string) i.e. `"NyanCat"`
 		- `id`: (string) i.e. `"UCsW85RAS2_Twg_lEPyv7G8A"`
-		- `icon`?: (object)
+		- `icon`?: (object) (playlist results don't have this)
 			- `x`: (integer) px from left
 			- `y`: (integer) px from top
 			- `width`: (integer)
@@ -52,16 +55,35 @@ JSON object:
 		- `y`: (integer) px from top
 		- `width`: (integer)
 		- `height`: (integer)
+	- `captions_vrcurl`?: (integer) index of vrcurl to get the caption data json
 - `imagesheet_vrcurl`?: (integer) index of the vrcurl for the collage of thumbnails and/or icons
 - `nextpage_vrcurl`: (integer) index of the vrcurl that will serve the JSON for the next page of results
 
 
+## GET `/trending`
+
+Gets Trending YouTube videos. Identical to `/search` but without `input` parameter, and response includes additional field:
+
+- `tabs`: Array of Object
+	- `name`: (string) Tab title ("Now", "Music", "Gaming", "Movies")
+	- `vrcurl`: (integer) index of vrcurl to load that tab (same response format)
+
 ## GET `/vrcurl/{pool}/{index}`
 
 - `{pool}`: must be same as pool param in search endpoint.
 - `{index}`: vrcurl index number
 
-Response may be 302 redirect to youtube url, `image/png` for imagesheet or `application/json` for next page
+Response may be 302 redirect to youtube url, `image/png` for imagesheet, `application/json` for next page (see response format above) or trending tab or caption data (below).
+
+### Caption JSON format
+
+- Array of Object
+	- `name`: (string) caption track name like "English" or "English (auto-generated)"
+	- `id`: (string) id like `.en` or `a.en`
+	- `lines`: Array of Object
+		- `start`: (float) video seconds when the caption is displayed
+		- `dur`: (float) seconds to display the caption
+		- `text`: (string) caption text
 
 # VRCUrls
 

VRCYoutubeSearch.js

@@ -1,15 +1,15 @@
-import { searchYouTubeVideos, continueYouTubeVideoSearch } from "./simpleYoutubeSearch.js";
+import { searchYouTubeVideos, continueYouTubeVideoSearch, getYouTubePlaylist, continueYouTubePlaylist, getTrending } from "./youtube.js";
 import { putVrcUrl } from "./vrcurl.js";
-import { makeImageSheetVrcUrl, thumbnailWidth, thumbnailHeight, iconWidth, iconHeight } from "./imagesheet.js";
+import { makeImageSheetVrcUrl } from "./imagesheet.js";
 
 var cache = {};
 
 
 
-export async function cachedVRCYoutubeSearch(pool, queryOrContinuation, options) {
-	var key = JSON.stringify([pool, queryOrContinuation, options]);
+export async function cachedVRCYoutubeSearch(pool, query, options) {
+	var key = JSON.stringify([pool, query, options]);
 	if (!cache[key]) {
-		cache[key] = VRCYoutubeSearch(pool, queryOrContinuation, options);
+		cache[key] = VRCYoutubeSearch(pool, query, options);
 		setTimeout(() => {
 			delete cache[key];
 		}, 1000*60*10); // 10 mins
@@ -21,57 +21,98 @@ export async function cachedVRCYoutubeSearch(pool, queryOrContinuation, options)
 
 
 async function VRCYoutubeSearch(pool, query, options = {}) {
-	console.debug("search:", query);
+	console.log("search:", JSON.stringify(query));
 	var data = {results: []};
 
-	var {videos, continuationData} = typeof query == "object" ? await continueYouTubeVideoSearch(query) : await searchYouTubeVideos(query);
+	if (typeof query == "object") {
+		switch (query.type) {
+			case "trending":
+				var {videos, tabs} = await getTrending(query.bp);
+				data.tabs = [];
+				for (let tab of tabs) {
+					data.tabs.push({
+						name: tab.name,
+						vrcurl: await putVrcUrl(pool, {type: "trending", bp: tab.bp, options})
+					});
+				}
+				break;
+			case "continuation":
+				//var {videos, continuationData} = await [query.for == "playlist" ? continueYouTubePlaylist : continueYouTubeVideoSearch](query.continuationData);
+				if (query.for == "playlist") {
+					var {videos, continuationData} = await continueYouTubePlaylist(query.continuationData);
+				} else {
+					var {videos, continuationData} = await continueYouTubeVideoSearch(query.continuationData);
+				}
+				break;
+		}
+	} else {
+		var playlistId = query.match(/list=(PL[a-zA-Z0-9-_]{32})/)?.[1];
+		if (playlistId) console.log("playlistId:", playlistId);
+		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query);
+	}
+	
+	var images = [];
 
 	if (options.thumbnails) {
-		var thumbnailUrls = videos.map(video => video.thumbnailUrl);
+		videos.forEach(video => {
+			video.thumbnail = playlistId ? {
+				url: `https://i.ytimg.com/vi/${video.id}/default.jpg`,
+				width: 120,
+				height: 90
+			} : {
+				url: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`,
+				width: 320,
+				height: 180
+			};
+			images.push(video.thumbnail);
+		});
 	}
 
 	if (options.icons) {
-		var iconUrls = new Set();
-		for (let video of videos) {
-			iconUrls.add(video.channel.iconUrl);
-		}
-		iconUrls = [...iconUrls];
+		let iconUrls = new Set();
+		videos.forEach(video => video.channel.iconUrl && iconUrls.add(video.channel.iconUrl));
+		iconUrls.forEach(url => images.push({
+			width: 68,//todo pass from yt data not hardcode
+			height: 68,
+			url
+		}));
 	}
 
-	if (thumbnailUrls || iconUrls) {
-		var {vrcurl: imagesheet_vrcurl, thumbnails, icons} = await makeImageSheetVrcUrl(pool, thumbnailUrls, iconUrls);
-		data.imagesheet_vrcurl = imagesheet_vrcurl;
+	if (images.length) {
+		try {
+			var {vrcurl: imagesheet_vrcurl} = await makeImageSheetVrcUrl(pool, images);
+			data.imagesheet_vrcurl = imagesheet_vrcurl;
+		} catch (error) {
+			console.error(error.stack);
+		}
 	}
 
 	for (let video of videos) {
 		video.vrcurl = await putVrcUrl(pool, {type: "redirect", url: `https://www.youtube.com/watch?v=${video.id}`});
-		if (thumbnails?.length) {
-			let thumbnail = thumbnails.find(x => x.url == video.thumbnailUrl);
-			video.thumbnail = {
-				x: thumbnail?.x,
-				y: thumbnail?.y,
-				width: thumbnailWidth,
-				height: thumbnailHeight
-			};
+		let thumbnail = images.find(image => image.url == video.thumbnail.url);
+		video.thumbnail = thumbnail ? {
+			x: thumbnail?.x,
+			y: thumbnail?.y,
+			width: thumbnail?.width,
+			height: thumbnail?.height
+		} : undefined;
+		let icon = images.find(image => image.url == video.channel.iconUrl);
+		video.channel.icon = icon ? {
+			x: icon?.x,
+			y: icon?.y,
+			width: icon?.width,
+			height: icon?.height
+		} : undefined;
+		if (options.captions) {
+			video.captions_vrcurl = await putVrcUrl(pool, {type: "captions", videoId: video.id});
 		}
-		if (icons?.length) {
-			let icon = icons.find(x => x.url == video.channel.iconUrl);
-			video.channel.icon = {
-				x: icon?.x,
-				y: icon?.y,
-				width: iconWidth,
-				height: iconHeight
-			};
-		}
-		delete video.thumbnailUrl;
 		delete video.channel.iconUrl;
 		data.results.push(video);
 	}
 
-	
-
 	if (continuationData) data.nextpage_vrcurl = await putVrcUrl(pool, {
-		type: "ytContinuation",
+		type: "continuation",
+		for: query.for || (playlistId ? "playlist" : "search"),
 		continuationData,
 		options
 	});

app.js

@@ -1,79 +0,0 @@
-import Koa from "koa";
-import Router from "@koa/router";
-import send from "koa-send";
-import { cachedVRCYoutubeSearch } from "./VRCYoutubeSearch.js"
-import { getImageSheet } from "./imagesheet.js";
-import { resolveVrcUrl } from "./vrcurl.js";
-import { stringToBoolean } from "./util.js";
-
-export var app = new Koa();
-var router = new Router();
-
-
-router.get("/search", async ctx => {
-	var query = ctx.query.input?.replace(/^.*→/, '').trim();
-	if (!query) {
-		ctx.status = 400;
-		ctx.body = "missing search query";
-		return;
-	}
-
-	if (!ctx.query.pool || !/^[a-z-_]+\d*$/.test(ctx.query.pool)) {
-		ctx.status = 400;
-		ctx.body = "invalid pool";
-		return;
-	}
-
-	var options = {
-		thumbnails: stringToBoolean(ctx.query.thumbnails),
-		icons: stringToBoolean(ctx.query.icons)
-	};
-
-	ctx.body = await cachedVRCYoutubeSearch(ctx.query.pool, query, options);
-});
-
-
-router.get("/vrcurl/:pool/:num", async ctx => {
-	var dest = await resolveVrcUrl(ctx.params.pool, ctx.params.num);
-	if (!dest) {
-		ctx.status = 404;
-		return;
-	}
-	switch (dest.type) {
-		case "redirect":
-			ctx.redirect(dest.url);
-			break;
-		case "imagesheet":
-			let buf = await getImageSheet(ctx.params.pool, ctx.params.num);
-			if (!buf) {
-				ctx.status = 404;
-				return;
-			}
-			ctx.body = buf;
-			ctx.type = "image/png";
-			break;
-		case "ytContinuation":
-			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, dest.continuationData, dest.options);
-			break;
-		default:
-			ctx.status = 500;
-	}
-});
-
-
-router.get("/robots.txt", ctx => {
-	ctx.body = `User-agent: *\nDisallow: /`;
-});
-
-router.get("/test.html", async ctx => {
-	await send(ctx, "test.html");
-});
-
-router.get("/", ctx => {
-	ctx.redirect("https://www.u2b.cx/");
-});
-
-
-
-app.use(router.routes());
-app.use(router.allowedMethods());

imagesheet.js

@@ -1,75 +1,42 @@
 import { createCanvas, loadImage } from 'canvas';
+import potpack from 'potpack';
 import { putVrcUrl } from './vrcurl.js';
 
 var store = {};
 
-
-export const thumbnailWidth = 360;
-export const thumbnailHeight = 202;
-export const iconWidth = 68;
-export const iconHeight = 68;
-const maxSheetWidth = 2048;
-const maxSheetHeight = 2048;
-const maxThumbnailRowLen = Math.floor(maxSheetWidth / thumbnailWidth); // 5
-const maxThumbnailColLen = Math.floor(maxSheetHeight / thumbnailHeight); // 10
-const maxIconRowLen = Math.floor(maxSheetWidth / iconWidth); // 30
-const maxIconColLen = Math.floor(maxSheetHeight / iconHeight); // 30
-
-
-async function createImageSheet(thumbnailUrls = [], iconUrls = []) {
-	
-
-	var thumbnails = thumbnailUrls.map((url, index) => {
-		const x = index % maxThumbnailRowLen * thumbnailWidth;
-		const y = Math.floor(index / maxThumbnailRowLen) * thumbnailHeight;
-		return {x, y, url};
+async function createImageSheet(images /*[{width, height, url}]*/) {
+	images.forEach(image => {
+		image.w = image.width;
+		image.h = image.height;
 	});
-
-	const iconStartY = thumbnails.length ? thumbnails.at(-1).y + thumbnailHeight : 0;
-
-	var icons = iconUrls.map((url, index) => {
-		const x = index % maxIconRowLen * iconWidth;
-		const y = iconStartY + Math.floor(index / maxIconRowLen);
-		return {x, y, url};
-	});
-
-	const canvasWidth = Math.max(
-		Math.min(thumbnails.length, maxThumbnailRowLen) * thumbnailWidth,
-		Math.min(icons.length, maxIconRowLen) * iconWidth
-	);
-	const canvasHeight = icons.length ? icons.at(-1).y + iconHeight : thumbnails.length ? thumbnails.at(-1).y + thumbnailHeight : 0;
-
-	var canvas = createCanvas(Math.min(maxSheetWidth, canvasWidth), Math.min(maxSheetHeight, canvasHeight));
+	var {w, h, fill} = potpack(images);
+	if (w > 2048) {
+		console.warn("Imagesheet exceeded max width");
+		w = 2048;
+	}
+	if (h > 2048) {
+		console.warn("Imagesheet exceeded max height");
+		h = 2048;
+	}
+	var canvas = createCanvas(w, h);
 	var ctx = canvas.getContext('2d');
 
-	var promises = [];
+	await Promise.all(images.map(({x, y, w, h, url}) => (async function(){
+		if (!url) return;
+		var image = await loadImage(url);
+		ctx.drawImage(image, x, y, w, h);
+	})().catch(error => console.error(error.stack))));
 
-	if (thumbnails.length) {
-		promises = promises.concat(thumbnails.map(({x, y, url}) => (async function(){
-			var image = await loadImage(url);
-			ctx.drawImage(image, x, y, thumbnailWidth, thumbnailHeight);
-		})().catch(error => console.error(error.stack))));
-	}
-
-	if (icons.length) {
-		promises = promises.concat(icons.map(({x, y, url}) => (async function(){
-			var image = await loadImage(url);
-			ctx.drawImage(image, x, y, iconWidth, iconHeight);
-		})().catch(error => console.error(error.stack))));
-	}
-
-	await Promise.all(promises);
 	return {
 		imagesheet: canvas.toBuffer("image/png"),
-		thumbnails, icons
+		images
 	};
 }
 
-
-export async function makeImageSheetVrcUrl(pool, thumbnailUrls, iconUrls) {
+export async function makeImageSheetVrcUrl(pool, images) {
 	var num = await putVrcUrl(pool, {type: "imagesheet"});
 	var key = `${pool}:${num}`;
-	var promise = createImageSheet(thumbnailUrls, iconUrls);
+	var promise = createImageSheet(images);
 	store[key] = promise;
 	promise.then(() => {
 		setTimeout(() => {

index.js

@@ -1,3 +1,132 @@
-import { app } from "./app.js";
+if (process.env.D!="BUG") console.debug = () => {};
+import "./util.js";
+import Koa from "koa";
+import Router from "@koa/router";
+import send from "koa-send";
+import { cachedVRCYoutubeSearch } from "./VRCYoutubeSearch.js"
+import { getImageSheet } from "./imagesheet.js";
+import { resolveVrcUrl } from "./vrcurl.js";
+import { getVideoCaptionsCached } from "./youtube-captions.js";
+import { stringToBoolean } from "./util.js";
+import shorturlmap from "./shorturlmap.json" assert { type: "json" };
+
+var app = new Koa();
+var router = new Router();
+
+
+router.get(["/search", "/trending"], async ctx => {
+	if (ctx.path == "/trending") {
+		var query = {"type":"trending"};
+	} else {
+		var query = ctx.querystring.match(/[?&]input=(.*)/i)?.[1];
+		if (!query) {
+			ctx.status = 400;
+			ctx.body = "missing search query";
+			return;
+		}
+		query = decodeURIComponent(query).replace(/^.*→/, '').trim();
+	}
+
+	if (!ctx.query.pool || !/^[a-z-_]+\d*$/.test(ctx.query.pool)) {
+		ctx.status = 400;
+		ctx.body = "invalid pool";
+		return;
+	}
+
+	var options = {
+		thumbnails: stringToBoolean(ctx.query.thumbnails),
+		icons: stringToBoolean(ctx.query.icons),
+		captions: stringToBoolean(ctx.query.captions)
+	};
+
+	ctx.body = await cachedVRCYoutubeSearch(ctx.query.pool, query, options);
+});
+
+
+router.get("/vrcurl/:pool/:num", async ctx => {
+	var dest = await resolveVrcUrl(ctx.params.pool, ctx.params.num);
+	if (!dest) {
+		ctx.status = 404;
+		return;
+	}
+	switch (dest.type) {
+		case "redirect":
+			ctx.redirect(dest.url);
+			break;
+		case "imagesheet":
+			let buf = await getImageSheet(ctx.params.pool, ctx.params.num);
+			if (!buf) {
+				ctx.status = 404;
+				return;
+			}
+			ctx.body = buf;
+			ctx.type = "image/png";
+			break;
+		case "continuation":
+			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "continuation", for: dest.for, continuationData: dest.continuationData}, dest.options);
+			break;
+		case "trending":
+			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "trending", bp: dest.bp}, dest.options);
+			break;
+		case "captions":
+			ctx.body = await getVideoCaptionsCached(dest.videoId);
+			break;
+		default:
+			console.error("unknown vrcurl type", dest.type);
+			ctx.status = 500;
+	}
+});
+
+
+router.get("/robots.txt", ctx => {
+	ctx.body = `User-agent: *\nDisallow: /`;
+});
+
+router.get("/test.html", async ctx => {
+	await send(ctx, "test.html");
+});
+
+router.get("/", ctx => {
+	ctx.redirect("https://www.u2b.cx/");
+});
+
+
+
+
+
+// short urls to work around https://feedback.vrchat.com/udon/p/vrcurlinputfield-incorrect-focus-issue-on-quest
+app.use(async (ctx, next) => {
+	var subdomain = ctx.hostname.match(/(.*).u2b.cx$/i)?.[1];
+	if (subdomain && !["api","api2","dev"].includes(subdomain)) {
+		if (shorturlmap[subdomain]) {
+			ctx.url = shorturlmap[subdomain] + ctx.url.slice(1);
+		} else {
+			ctx.status = 404;
+			return;
+		}
+	}
+	await next();
+});
+
+// work around vrchat json parser bug https://feedback.vrchat.com/udon/p/braces-inside-strings-in-vrcjson-can-fail-to-deserialize
+app.use(async (ctx, next) => {
+	await next();
+	if (ctx.type != "application/json") return;
+	ctx.body = structuredClone(ctx.body);
+	(function iterateObject(obj) {
+		for (var key in obj) {
+			if (typeof obj[key] == "string") {
+				obj[key] = obj[key].replace(/[\[\]{}]/g, chr => "\\u" + chr.charCodeAt(0).toString(16).padStart(4, '0'));
+			} else if (typeof obj[key] == "object") {
+				iterateObject(obj[key]);
+			}
+		}
+	})(ctx.body);
+	ctx.body = JSON.stringify(ctx.body).replaceAll("\\\\u", "\\u");
+});
+
+
+app.use(router.routes());
+app.use(router.allowedMethods());
 
 app.listen(process.env.PORT || 8142, process.env.ADDRESS);

package-lock.json

@@ -8,10 +8,14 @@
 				"@keyv/sqlite": "^3.6.6",
 				"@koa/router": "^12.0.1",
 				"canvas": "^2.11.2",
+				"fast-xml-parser": "^4.3.4",
 				"keyv": "^4.5.4",
 				"koa": "^2.14.2",
 				"koa-send": "^5.0.1",
-				"node-fetch": "^3.3.2"
+				"potpack": "^2.0.0"
+			},
+			"engines": {
+				"node": ">=18.0.0"
 			}
 		},
 		"node_modules/@gar/promisify": {
@@ -362,14 +366,6 @@
 				"node": ">= 0.8"
 			}
 		},
-		"node_modules/data-uri-to-buffer": {
-			"version": "4.0.1",
-			"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
-			"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
-			"engines": {
-				"node": ">= 12"
-			}
-		},
 		"node_modules/debug": {
 			"version": "4.3.4",
 			"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -491,37 +487,25 @@
 			"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
 			"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
 		},
-		"node_modules/fetch-blob": {
-			"version": "3.2.0",
-			"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
-			"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
+		"node_modules/fast-xml-parser": {
+			"version": "4.3.4",
+			"resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.4.tgz",
+			"integrity": "sha512-utnwm92SyozgA3hhH2I8qldf2lBqm6qHOICawRNRFu1qMe3+oqr+GcXjGqTmXTMGE5T4eC03kr/rlh5C1IRdZA==",
 			"funding": [
 				{
 					"type": "github",
-					"url": "https://github.com/sponsors/jimmywarting"
+					"url": "https://github.com/sponsors/NaturalIntelligence"
 				},
 				{
 					"type": "paypal",
-					"url": "https://paypal.me/jimmywarting"
+					"url": "https://paypal.me/naturalintelligence"
 				}
 			],
 			"dependencies": {
-				"node-domexception": "^1.0.0",
-				"web-streams-polyfill": "^3.0.3"
+				"strnum": "^1.0.5"
 			},
-			"engines": {
-				"node": "^12.20 || >= 14.13"
-			}
-		},
-		"node_modules/formdata-polyfill": {
-			"version": "4.0.10",
-			"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
-			"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
-			"dependencies": {
-				"fetch-blob": "^3.1.2"
-			},
-			"engines": {
-				"node": ">=12.20.0"
+			"bin": {
+				"fxparser": "src/cli/cli.js"
 			}
 		},
 		"node_modules/fresh": {
@@ -736,11 +720,18 @@
 			"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
 			"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
 		},
-		"node_modules/ip": {
-			"version": "2.0.0",
-			"resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
-			"integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
-			"optional": true
+		"node_modules/ip-address": {
+			"version": "9.0.5",
+			"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
+			"integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==",
+			"optional": true,
+			"dependencies": {
+				"jsbn": "1.1.0",
+				"sprintf-js": "^1.1.3"
+			},
+			"engines": {
+				"node": ">= 12"
+			}
 		},
 		"node_modules/is-fullwidth-code-point": {
 			"version": "3.0.0",
@@ -776,6 +767,12 @@
 			"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
 			"optional": true
 		},
+		"node_modules/jsbn": {
+			"version": "1.1.0",
+			"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz",
+			"integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==",
+			"optional": true
+		},
 		"node_modules/json-buffer": {
 			"version": "3.0.1",
 			"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
@@ -1102,41 +1099,6 @@
 			"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz",
 			"integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="
 		},
-		"node_modules/node-domexception": {
-			"version": "1.0.0",
-			"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
-			"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
-			"funding": [
-				{
-					"type": "github",
-					"url": "https://github.com/sponsors/jimmywarting"
-				},
-				{
-					"type": "github",
-					"url": "https://paypal.me/jimmywarting"
-				}
-			],
-			"engines": {
-				"node": ">=10.5.0"
-			}
-		},
-		"node_modules/node-fetch": {
-			"version": "3.3.2",
-			"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
-			"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
-			"dependencies": {
-				"data-uri-to-buffer": "^4.0.0",
-				"fetch-blob": "^3.1.4",
-				"formdata-polyfill": "^4.0.10"
-			},
-			"engines": {
-				"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-			},
-			"funding": {
-				"type": "opencollective",
-				"url": "https://opencollective.com/node-fetch"
-			}
-		},
 		"node_modules/node-gyp": {
 			"version": "8.4.1",
 			"resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz",
@@ -1301,6 +1263,11 @@
 			"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz",
 			"integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw=="
 		},
+		"node_modules/potpack": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/potpack/-/potpack-2.0.0.tgz",
+			"integrity": "sha512-Q+/tYsFU9r7xoOJ+y/ZTtdVQwTWfzjbiXBDMM/JKUux3+QPP02iUuIoeBQ+Ot6oEDlC+/PGjB/5A3K7KKb7hcw=="
+		},
 		"node_modules/promise-inflight": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
@@ -1494,16 +1461,16 @@
 			}
 		},
 		"node_modules/socks": {
-			"version": "2.7.1",
-			"resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz",
-			"integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
+			"version": "2.7.3",
+			"resolved": "https://registry.npmjs.org/socks/-/socks-2.7.3.tgz",
+			"integrity": "sha512-vfuYK48HXCTFD03G/1/zkIls3Ebr2YNa4qU9gHDZdblHLiqhJrJGkY3+0Nx0JpN9qBhJbVObc1CNciT1bIZJxw==",
 			"optional": true,
 			"dependencies": {
-				"ip": "^2.0.0",
+				"ip-address": "^9.0.5",
 				"smart-buffer": "^4.2.0"
 			},
 			"engines": {
-				"node": ">= 10.13.0",
+				"node": ">= 10.0.0",
 				"npm": ">= 3.0.0"
 			}
 		},
@@ -1521,6 +1488,12 @@
 				"node": ">= 10"
 			}
 		},
+		"node_modules/sprintf-js": {
+			"version": "1.1.3",
+			"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
+			"integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
+			"optional": true
+		},
 		"node_modules/sqlite3": {
 			"version": "5.1.6",
 			"resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.1.6.tgz",
@@ -1595,6 +1568,11 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/strnum": {
+			"version": "1.0.5",
+			"resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz",
+			"integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA=="
+		},
 		"node_modules/tar": {
 			"version": "6.2.0",
 			"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz",
@@ -1683,14 +1661,6 @@
 				"node": ">= 0.8"
 			}
 		},
-		"node_modules/web-streams-polyfill": {
-			"version": "3.2.1",
-			"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
-			"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
-			"engines": {
-				"node": ">= 8"
-			}
-		},
 		"node_modules/webidl-conversions": {
 			"version": "3.0.1",
 			"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",

package.json

@@ -3,10 +3,14 @@
 		"@keyv/sqlite": "^3.6.6",
 		"@koa/router": "^12.0.1",
 		"canvas": "^2.11.2",
+		"fast-xml-parser": "^4.3.4",
 		"keyv": "^4.5.4",
 		"koa": "^2.14.2",
 		"koa-send": "^5.0.1",
-		"node-fetch": "^3.3.2"
+		"potpack": "^2.0.0"
+	},
+	"engines": {
+		"node": ">=18.0.0"
 	},
 	"type": "module"
 }

shorturlmap.json

@@ -0,0 +1,3 @@
+{
+	"hal": "/search?pool=hdays10000&thumbnails=yes&input="
+}

simpleYoutubeSearch.js

@@ -1,84 +0,0 @@
-var fetch = global.fetch || (await import("node-fetch")).default;
-
-
-export async function searchYouTubeVideos(query) {
-	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}&sp=EgIQAQ%253D%253D`;
-	var html = await fetch(url).then(res => res.text());
-
-	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
-	ytInitialData = JSON.parse(ytInitialData);
-
-	var videos = ytInitialData?.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))?.itemSectionRenderer?.contents?.filter(x => x.videoRenderer).map(parseVideoRendererData);
-	if (!videos) return {videos: []};
-
-	try {
-		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
-		ytcfg = JSON.parse(ytcfg);
-		var continuationData = {
-			context: ytcfg.INNERTUBE_CONTEXT,
-			continuation: ytInitialData.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
-		}
-	} catch (error) {
-		console.error(error.stack);
-	}
-
-	return {videos, continuationData};
-}
-
-
-export async function continueYouTubeVideoSearch(continuationData) {
-	var data = await fetch("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
-		method: "POST",
-		headers: {
-			"Content-Type": "application/json"
-		},
-		body: JSON.stringify(continuationData)
-	}).then(res => res.json());
-
-	var videos = data.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer)).itemSectionRenderer.contents.filter(x => x.videoRenderer).map(parseVideoRendererData);
-	var continuationToken = data.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
-
-	return {
-		videos,
-		continuationData: {
-			context: continuationData.context,
-			continuation: continuationToken
-		}
-	}
-}
-
-
-
-function parseVideoRendererData(data) {
-	data = data.videoRenderer;
-	return {
-		id: data.videoId,
-		live: Boolean(data.badges?.find(x => x.metadataBadgeRenderer?.style == "BADGE_STYLE_TYPE_LIVE_NOW")),
-		title: data.title?.runs?.[0]?.text,
-		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.reduce((str, obj) => str += obj.text, ""),
-		thumbnailUrl: data.thumbnail?.thumbnails?.find(x => x.width == 360 && x.height == 202)?.url || data.thumbnail?.thumbnails?.[0]?.url,
-		uploaded: data.publishedTimeText?.simpleText,
-		lengthText: data.lengthText?.simpleText,
-		longLengthText: data.lengthText?.accessibility?.accessibilityData?.label,
-		viewCountText: data.viewCountText?.runs ? data.viewCountText.runs.reduce((str, obj) => str += obj.text, "") : data.viewCountText?.simpleText,
-		shortViewCountText: data.shortViewCountText?.simpleText,
-		channel: {
-			name: data.ownerText?.runs?.[0]?.text,
-			id: data.ownerText?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId,
-			iconUrl: data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail?.thumbnails?.[0]?.url
-		}
-	};
-}
-
-
-
-/*
-var page1 = await searchYouTubeVideos("test video");
-console.log("page1", page1);
-var page2 = await continueYouTubeVideoSearch(page1.continuationData);
-console.log("page2", page2);
-var page3 = await continueYouTubeVideoSearch(page2.continuationData);
-console.log("page3", page3);
-console.log("videos", [...page1.videos, ...page2.videos, ...page3.videos]);
-debugger;
-*/

test.html

@@ -9,28 +9,36 @@
 </head><body>
 
 <div>
-	<label>search: <input id="input" type="text" value="nyan cat" /></label>
+	<label><button id="search">search:</button> <input id="input" type="text" value="nyan cat" /></label>
+	<br /><button id="trending">trending</button>
+	<br />
 	<label><input id="thumbnails" type="checkbox" checked>thumbnails</label>
 	<label><input id="icons" type="checkbox" checked>icons</label>
-	<button id="start">start</button>
+	<label><input id="captions" type="checkbox" checked>captions</label>
 </div>
 
 <div id="output"></div>
 
-<button id="nextpage">next page</button>
+
+<div id="buttons"></div>
+<button id="nextpage" style="display: none">next page</button>
 
 <script>
 
 var num = 0;
 var lastData;
 
-start.onclick = () => {
+search.onclick = () => {
 	output.innerHTML = "";
-	loadData(`/search?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&input=${encodeURIComponent(input.value)}`);
+	loadData(`/search?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&captions=${captions.checked}&input=${encodeURIComponent(input.value)}`);
 };
-nextpage.onclick = () => loadData(`/vrcurl/test1000/${lastData.nextpage_vrcurl}`);
+trending.onclick = () => {
+	output.innerHTML = "";
+	loadData(`/trending?pool=test1000&thumbnails=${thumbnails.checked}&icons=${icons.checked}&captions=${captions.checked}`);
+}
 
 async function loadData(url) {
+	buttons.innerHTML = '';
 	var data = await fetch(url).then(res => res.json());
 	var pre = document.createElement("pre");
 	pre.innerHTML = hljs.highlight(JSON.stringify(data, null, 4), {language: "json"}).value;
@@ -38,6 +46,12 @@ async function loadData(url) {
 	var img = document.createElement("img");
 	img.src = `/vrcurl/test1000/${data.imagesheet_vrcurl}`;
 	output.appendChild(img);
+	if (data.nextpage_vrcurl) {
+		buttons.innerHTML = `<button onclick="loadData('/vrcurl/test1000/${data.nextpage_vrcurl}')">next page</button>`;
+	}
+	if (data.tabs) {
+		buttons.innerHTML = data.tabs.map(t => `<button onclick="loadData('/vrcurl/test1000/${t.vrcurl}')">${t.name}</button>`).join('');
+	}
 	lastData = data;
 }
 

util.js

@@ -1,6 +1,28 @@
+Array.prototype.filterMap = function(fn) {
+	var newarray = [];
+	for (var item of this) {
+		var ret = fn(item);
+		if (ret) newarray.push(ret);
+	}
+	return newarray;
+};
+
 export function stringToBoolean(str) {
 	if (str) {
 		if (!["0", "false", "off", "no", "null", "undefined", "nan"].includes(str.toLowerCase())) return true;
 	}
 	return false;
-}
+}
+
+export function recursiveFind(object, fn) {
+	var results = [];
+	(function crawlObject(object) {
+		for (var key in object) {
+			var value = object[key];
+			var test = fn(value);
+			if (test) results.push(test);
+			if (typeof value == "object") crawlObject(value);
+		}
+	})(object);
+	return results;
+}

youtube-captions.js

@@ -0,0 +1,43 @@
+import { XMLParser } from "fast-xml-parser";
+
+var xmlParser = new XMLParser({
+	ignoreAttributes: false
+});
+
+async function getVideoData(videoId) {
+	var html = await fetch(`https://www.youtube.com/watch?v=${videoId}`).then(res => res.text());
+
+	var ytInitialPlayerResponse = html.match(/var ytInitialPlayerResponse = ({.*});/)[1];
+	ytInitialPlayerResponse = JSON.parse(ytInitialPlayerResponse);
+
+	return ytInitialPlayerResponse;
+}
+
+async function getVideoCaptions(videoId) {
+	var ytInitialPlayerResponse = await getVideoData(videoId);
+	if (!ytInitialPlayerResponse.captions) return [];
+	var captionTracks = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks;
+	captionTracks = await Promise.all(captionTracks.map(captionTrack => (async () => {
+		var xml = await fetch(captionTrack.baseUrl).then(res => res.text());
+		var parsed = xmlParser.parse(xml);
+		var lines = parsed.transcript.text.map(({ "#text": text, "@_start": start, "@_dur": dur }) => ({ start: Number(start), dur: Number(dur), text }));
+		return {
+			name: captionTrack.name.simpleText,
+			id: captionTrack.vssId,
+			lines
+		};
+	})().catch(error => console.error(error.stack))));
+	return captionTracks;
+}
+
+var cache = {};
+
+export async function getVideoCaptionsCached(videoId) {
+	if (!cache[videoId]) {
+		cache[videoId] = getVideoCaptions(videoId);
+		setTimeout(() => {
+			delete cache[videoId];
+		}, 1000*60*60*6); // 6 hours
+	}
+	return await cache[videoId];
+}

youtube.js

@@ -0,0 +1,192 @@
+
+export async function searchYouTubeVideos(query) {
+	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}&sp=EgIQAQ%253D%253D`;
+	var html = await fetch(url).then(res => res.text());
+
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+	ytInitialData = JSON.parse(ytInitialData);
+	console.debug(ytInitialData);
+
+	var videos = ytInitialData?.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))?.itemSectionRenderer?.contents?.filterMap(x => x.videoRenderer)?.map(parseVideoRendererData);
+	if (!videos) return {videos: []};
+	console.debug(videos.length, "results");
+
+	try {
+		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
+		ytcfg = JSON.parse(ytcfg);
+		var continuationData = {
+			context: ytcfg.INNERTUBE_CONTEXT,
+			continuation: ytInitialData.contents.twoColumnSearchResultsRenderer.primaryContents.sectionListRenderer.contents.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
+		}
+	} catch (error) {
+		console.error(error.stack);
+	}
+
+	return {videos, continuationData};
+}
+
+export async function continueYouTubeVideoSearch(continuationData) {
+	var data = await fetch("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json"
+		},
+		body: JSON.stringify(continuationData)
+	}).then(res => res.json());
+	console.debug(data);
+
+	var continuationItems = data.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems;
+	var videos = continuationItems.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer)).itemSectionRenderer.contents.filterMap(x => x.videoRenderer).map(parseVideoRendererData);
+	var continuationToken = continuationItems.find(x => x.continuationItemRenderer)?.continuationItemRenderer.continuationEndpoint.continuationCommand.token
+	console.debug(videos.length, "results");
+
+
+	return {
+		videos,
+		continuationData: continuationToken ? {
+			context: continuationData.context,
+			continuation: continuationToken
+		} : null
+	}
+}
+
+
+
+export async function getYouTubePlaylist(playlistId) {
+	var html = await fetch("https://www.youtube.com/playlist?list=" + playlistId).then(res => res.text());
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+	ytInitialData = JSON.parse(ytInitialData);
+	console.debug(ytInitialData);
+
+	var sectionListRendererContents = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.find(tab => tab.tabRenderer.selected).tabRenderer.content.sectionListRenderer.contents;
+	var videos = sectionListRendererContents.find(x => x.itemSectionRenderer).itemSectionRenderer.contents.find(x => x.playlistVideoListRenderer).playlistVideoListRenderer.contents.filterMap(x => x.playlistVideoRenderer).map(parseVideoRendererData);
+	if (!videos) return {videos: []};
+	console.debug(videos.length, "results");
+
+	try {
+		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
+		ytcfg = JSON.parse(ytcfg);
+		var continuationData = {
+			context: ytcfg.INNERTUBE_CONTEXT,
+			continuation: sectionListRendererContents.find(x => x.continuationItemRenderer).continuationItemRenderer.continuationEndpoint.continuationCommand.token
+		}
+	} catch (error) {
+		console.error(error.stack);
+	}
+	return {videos, continuationData};
+}
+
+export async function continueYouTubePlaylist(continuationData) {
+	var data = await fetch("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false", {
+		method: "POST",
+		headers: {"Content-Type": "application/json"},
+		body: JSON.stringify(continuationData)
+	}).then(res => res.json());
+	console.debug(data);
+
+	if (!data.onResponseReceivedActions) return {videos:[]};
+	var continuationItems = data.onResponseReceivedActions[0].appendContinuationItemsAction.continuationItems;
+	var videos = continuationItems.find(x => x.itemSectionRenderer).itemSectionRenderer.contents.filterMap(x => x.playlistVideoListRenderer).map(parseVideoRendererData);
+	var continuationToken = continuationItems.find(x => x.continuationItemRenderer)?.continuationItemRenderer.continuationEndpoint.continuationCommand.token;
+	console.debug(videos.length, "results");
+
+	return {
+		videos,
+		continuationData: continuationToken ? {
+			context: continuationData.context,
+			continuation: continuationToken
+		} : null
+	}
+}
+
+
+
+
+
+
+
+
+
+export async function getTrending(bp) {
+	var url = `https://www.youtube.com/feed/trending`;
+	if (bp) url += `?bp=${bp}`;
+	var html = await fetch(url).then(res => res.text());
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+	ytInitialData = JSON.parse(ytInitialData);
+
+	var tabs = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.map(t => {
+		return {
+			name: t.tabRenderer.title,
+			//url: `https://www.youtube.com` + t.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url
+			bp: t.tabRenderer.endpoint.browseEndpoint.params
+		}
+	});
+
+	var videos = ytInitialData
+		.contents
+		.twoColumnBrowseResultsRenderer
+		.tabs
+		.find(tab => tab.tabRenderer.selected)
+		.tabRenderer
+		.content
+		.sectionListRenderer
+		.contents
+		// regular trending in sections with shelfRenderer without title
+		.filterMap(x => {
+			var shelfRenderer = x.itemSectionRenderer.contents.find(x => x.shelfRenderer)?.shelfRenderer;
+			if (shelfRenderer && !shelfRenderer.title) {
+				return shelfRenderer
+					.content
+					.expandedShelfContentsRenderer
+					.items
+					.filterMap(x => x.videoRenderer)
+					.map(parseVideoRendererData)
+			};
+		})
+		.flat();
+
+
+	return {tabs, videos};
+}
+
+
+
+
+
+
+
+
+
+
+
+
+Object.prototype.concatRunsText = function concatRunsText() {
+	return this.reduce((str, obj) => str += obj.text, "");
+};
+
+function parseVideoRendererData(data) {
+	return {
+		id: data.videoId,
+		live: Boolean(data.badges?.find(x => x.metadataBadgeRenderer?.style == "BADGE_STYLE_TYPE_LIVE_NOW")),
+		title: data.title?.runs?.concatRunsText(),
+		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.concatRunsText() 
+			|| data.descriptionSnippet?.runs?.concatRunsText(),
+		//thumbnailUrl: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138))?.url || data.thumbnail?.thumbnails?.[0]?.url,
+		//thumbnail: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138)) || data.thumbnail?.thumbnails?.[0],
+		/*thumbnail: {
+			url: `https://i.ytimg.com/vi/${data.videoId}/mqdefault.jpg`,
+			width: 320,
+			height: 180
+		},*/
+		uploaded: data.publishedTimeText?.simpleText || data.videoInfo?.runs?.[2]?.text,
+		lengthText: data.lengthText?.simpleText,
+		longLengthText: data.lengthText?.accessibility?.accessibilityData?.label,
+		viewCountText: data.viewCountText?.runs ? data.viewCountText.runs.concatRunsText() : data.viewCountText?.simpleText,
+		shortViewCountText: data.shortViewCountText?.simpleText || data.videoInfo?.runs?.[0]?.text,
+		channel: {
+			name: (data.ownerText || data.shortBylineText)?.runs?.concatRunsText(),
+			id: (data.ownerText || data.shortBylineText)?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId,
+			iconUrl: data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail?.thumbnails?.[0]?.url
+		}
+	};
+}