manual input parsing

reorganize youtube stuff
combine app.js into index.js
2024-06-19 23:25:30 -07:00 · 2024-06-17 16:24:26 -07:00 · 2024-06-17 16:15:01 -07:00 · 2024-06-17 16:04:19 -07:00 · 2024-06-17 15:50:03 -07:00
9 changed files with 250 additions and 209 deletions
@@ -14,7 +14,10 @@
 			"program": "${workspaceFolder}\\index.js",
 			"outFiles": [
 				"${workspaceFolder}/**/*.js"
-			]
+			],
+			"env": {
+				"D":"BUG"
+			}
 		}
 	]
 }
@@ -13,7 +13,7 @@ Get YouTube videos for a search query.
 ### Required query parameters

 - `pool`: id of the VRCUrl pool, only letters numbers hyphens or underscores, optionally followed by an integer for pool size.
- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed.
+- `input`: youtube search query. All chars up to and including this exact unicode char `→` are ignored, and then whitespace is trimmed. THIS MUST BE THE LAST QUERY PARAMETER AS ALL CHARS AFTER IT ARE CAPTURED VERBATIM (so you can type & etc without encoding)

 ### Optional query parameters

@@ -1,7 +1,6 @@
-import { searchYouTubeVideos, continueYouTubeVideoSearch, getYouTubePlaylist, continueYouTubePlaylist } from "./simpleYoutubeSearch.js";
+import { searchYouTubeVideos, continueYouTubeVideoSearch, getYouTubePlaylist, continueYouTubePlaylist, getTrending } from "./youtube.js";
 import { putVrcUrl } from "./vrcurl.js";
 import { makeImageSheetVrcUrl } from "./imagesheet.js";
-import { getTrending } from "./trending.js";

 var cache = {};

@@ -22,7 +21,7 @@ export async function cachedVRCYoutubeSearch(pool, query, options) {


 async function VRCYoutubeSearch(pool, query, options = {}) {
-	console.debug("search:", JSON.stringify(query));
+	console.log("search:", JSON.stringify(query));
 	var data = {results: []};

 	if (typeof query == "object") {
@@ -48,14 +47,25 @@ async function VRCYoutubeSearch(pool, query, options = {}) {
 		}
 	} else {
 		var playlistId = query.match(/list=(PL[a-zA-Z0-9-_]{32})/)?.[1];
-		if (playlistId) console.debug("playlistId:", playlistId);
+		if (playlistId) console.log("playlistId:", playlistId);
 		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query);
 	}
 	
 	var images = [];

 	if (options.thumbnails) {
-		videos.forEach(video => video.thumbnail.url && images.push(video.thumbnail));
+		videos.forEach(video => {
+			video.thumbnail = playlistId ? {
+				url: `https://i.ytimg.com/vi/${video.id}/default.jpg`,
+				width: 120,
+				height: 90
+			} : {
+				url: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`,
+				width: 320,
+				height: 180
+			};
+			images.push(video.thumbnail);
+		});
 	}

 	if (options.icons) {
@@ -1,123 +0,0 @@
-import Koa from "koa";
-import Router from "@koa/router";
-import send from "koa-send";
-import { cachedVRCYoutubeSearch } from "./VRCYoutubeSearch.js"
-import { getImageSheet } from "./imagesheet.js";
-import { resolveVrcUrl } from "./vrcurl.js";
-import { getVideoCaptionsCached } from "./captions.js";
-import { stringToBoolean } from "./util.js";
-import shorturlmap from "./shorturlmap.json" assert { type: "json" };
-
-export var app = new Koa();
-var router = new Router();
-
-
-router.get(["/search", "/trending"], async ctx => {
-	var query = ctx.path == "/trending" ? {"type":"trending"} : ctx.query.input?.replace(/^.*→/, '').trim();
-	if (!query) {
-		ctx.status = 400;
-		ctx.body = "missing search query";
-		return;
-	}
-
-	if (!ctx.query.pool || !/^[a-z-_]+\d*$/.test(ctx.query.pool)) {
-		ctx.status = 400;
-		ctx.body = "invalid pool";
-		return;
-	}
-
-	var options = {
-		thumbnails: stringToBoolean(ctx.query.thumbnails),
-		icons: stringToBoolean(ctx.query.icons),
-		captions: stringToBoolean(ctx.query.captions)
-	};
-
-	ctx.body = await cachedVRCYoutubeSearch(ctx.query.pool, query, options);
-});
-
-
-router.get("/vrcurl/:pool/:num", async ctx => {
-	var dest = await resolveVrcUrl(ctx.params.pool, ctx.params.num);
-	if (!dest) {
-		ctx.status = 404;
-		return;
-	}
-	switch (dest.type) {
-		case "redirect":
-			ctx.redirect(dest.url);
-			break;
-		case "imagesheet":
-			let buf = await getImageSheet(ctx.params.pool, ctx.params.num);
-			if (!buf) {
-				ctx.status = 404;
-				return;
-			}
-			ctx.body = buf;
-			ctx.type = "image/png";
-			break;
-		case "continuation":
-			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "continuation", for: dest.for, continuationData: dest.continuationData}, dest.options);
-			break;
-		case "trending":
-			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "trending", bp: dest.bp}, dest.options);
-			break;
-		case "captions":
-			ctx.body = await getVideoCaptionsCached(dest.videoId);
-			break;
-		default:
-			console.error("unknown vrcurl type", dest.type);
-			ctx.status = 500;
-	}
-});
-
-
-router.get("/robots.txt", ctx => {
-	ctx.body = `User-agent: *\nDisallow: /`;
-});
-
-router.get("/test.html", async ctx => {
-	await send(ctx, "test.html");
-});
-
-router.get("/", ctx => {
-	ctx.redirect("https://www.u2b.cx/");
-});
-
-
-
-
-
-// short urls to work around https://feedback.vrchat.com/udon/p/vrcurlinputfield-incorrect-focus-issue-on-quest
-app.use(async (ctx, next) => {
-	var subdomain = ctx.hostname.match(/(.*).u2b.cx$/i)?.[1];
-	if (subdomain && !["api","api2","dev"].includes(subdomain)) {
-		if (shorturlmap[subdomain]) {
-			ctx.url = shorturlmap[subdomain] + ctx.url.slice(1);
-		} else {
-			ctx.status = 404;
-			return;
-		}
-	}
-	await next();
-});
-
-// work around vrchat json parser bug https://feedback.vrchat.com/udon/p/braces-inside-strings-in-vrcjson-can-fail-to-deserialize
-app.use(async (ctx, next) => {
-	await next();
-	if (ctx.type != "application/json") return;
-	ctx.body = structuredClone(ctx.body);
-	(function iterateObject(obj) {
-		for (var key in obj) {
-			if (typeof obj[key] == "string") {
-				obj[key] = obj[key].replace(/[\[\]{}]/g, chr => "\\u" + chr.charCodeAt(0).toString(16).padStart(4, '0'));
-			} else if (typeof obj[key] == "object") {
-				iterateObject(obj[key]);
-			}
-		}
-	})(ctx.body);
-	ctx.body = JSON.stringify(ctx.body).replaceAll("\\\\u", "\\u");
-});
-
-
-app.use(router.routes());
-app.use(router.allowedMethods());
@@ -1,4 +1,132 @@
+if (process.env.D!="BUG") console.debug = () => {};
 import "./util.js";
-import { app } from "./app.js";
+import Koa from "koa";
+import Router from "@koa/router";
+import send from "koa-send";
+import { cachedVRCYoutubeSearch } from "./VRCYoutubeSearch.js"
+import { getImageSheet } from "./imagesheet.js";
+import { resolveVrcUrl } from "./vrcurl.js";
+import { getVideoCaptionsCached } from "./youtube-captions.js";
+import { stringToBoolean } from "./util.js";
+import shorturlmap from "./shorturlmap.json" assert { type: "json" };
+
+var app = new Koa();
+var router = new Router();
+
+
+router.get(["/search", "/trending"], async ctx => {
+	if (ctx.path == "/trending") {
+		var query = {"type":"trending"};
+	} else {
+		var query = ctx.querystring.match(/[?&]input=(.*)/i)?.[1];
+		if (!query) {
+			ctx.status = 400;
+			ctx.body = "missing search query";
+			return;
+		}
+		query = decodeURIComponent(query).replace(/^.*→/, '').trim();
+	}
+
+	if (!ctx.query.pool || !/^[a-z-_]+\d*$/.test(ctx.query.pool)) {
+		ctx.status = 400;
+		ctx.body = "invalid pool";
+		return;
+	}
+
+	var options = {
+		thumbnails: stringToBoolean(ctx.query.thumbnails),
+		icons: stringToBoolean(ctx.query.icons),
+		captions: stringToBoolean(ctx.query.captions)
+	};
+
+	ctx.body = await cachedVRCYoutubeSearch(ctx.query.pool, query, options);
+});
+
+
+router.get("/vrcurl/:pool/:num", async ctx => {
+	var dest = await resolveVrcUrl(ctx.params.pool, ctx.params.num);
+	if (!dest) {
+		ctx.status = 404;
+		return;
+	}
+	switch (dest.type) {
+		case "redirect":
+			ctx.redirect(dest.url);
+			break;
+		case "imagesheet":
+			let buf = await getImageSheet(ctx.params.pool, ctx.params.num);
+			if (!buf) {
+				ctx.status = 404;
+				return;
+			}
+			ctx.body = buf;
+			ctx.type = "image/png";
+			break;
+		case "continuation":
+			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "continuation", for: dest.for, continuationData: dest.continuationData}, dest.options);
+			break;
+		case "trending":
+			ctx.body = await cachedVRCYoutubeSearch(ctx.params.pool, {type: "trending", bp: dest.bp}, dest.options);
+			break;
+		case "captions":
+			ctx.body = await getVideoCaptionsCached(dest.videoId);
+			break;
+		default:
+			console.error("unknown vrcurl type", dest.type);
+			ctx.status = 500;
+	}
+});
+
+
+router.get("/robots.txt", ctx => {
+	ctx.body = `User-agent: *\nDisallow: /`;
+});
+
+router.get("/test.html", async ctx => {
+	await send(ctx, "test.html");
+});
+
+router.get("/", ctx => {
+	ctx.redirect("https://www.u2b.cx/");
+});
+
+
+
+
+
+// short urls to work around https://feedback.vrchat.com/udon/p/vrcurlinputfield-incorrect-focus-issue-on-quest
+app.use(async (ctx, next) => {
+	var subdomain = ctx.hostname.match(/(.*).u2b.cx$/i)?.[1];
+	if (subdomain && !["api","api2","dev"].includes(subdomain)) {
+		if (shorturlmap[subdomain]) {
+			ctx.url = shorturlmap[subdomain] + ctx.url.slice(1);
+		} else {
+			ctx.status = 404;
+			return;
+		}
+	}
+	await next();
+});
+
+// work around vrchat json parser bug https://feedback.vrchat.com/udon/p/braces-inside-strings-in-vrcjson-can-fail-to-deserialize
+app.use(async (ctx, next) => {
+	await next();
+	if (ctx.type != "application/json") return;
+	ctx.body = structuredClone(ctx.body);
+	(function iterateObject(obj) {
+		for (var key in obj) {
+			if (typeof obj[key] == "string") {
+				obj[key] = obj[key].replace(/[\[\]{}]/g, chr => "\\u" + chr.charCodeAt(0).toString(16).padStart(4, '0'));
+			} else if (typeof obj[key] == "object") {
+				iterateObject(obj[key]);
+			}
+		}
+	})(ctx.body);
+	ctx.body = JSON.stringify(ctx.body).replaceAll("\\\\u", "\\u");
+});
+
+
+app.use(router.routes());
+app.use(router.allowedMethods());

 app.listen(process.env.PORT || 8142, process.env.ADDRESS);
@@ -1,43 +0,0 @@
-import { parseVideoRendererData } from "./util.js";
-
-export async function getTrending(bp) {
-	var url = `https://www.youtube.com/feed/trending`;
-	if (bp) url += `?bp=${bp}`;
-	var html = await fetch(url).then(res => res.text());
-	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
-	ytInitialData = JSON.parse(ytInitialData);
-
-	var tabs = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.map(t => {
-		return {
-			name: t.tabRenderer.title,
-			//url: `https://www.youtube.com` + t.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url
-			bp: t.tabRenderer.endpoint.browseEndpoint.params
-		}
-	});
-
-	var videos = ytInitialData
-		.contents
-		.twoColumnBrowseResultsRenderer
-		.tabs
-		.find(tab => tab.tabRenderer.selected)
-		.tabRenderer
-		.content
-		.sectionListRenderer
-		.contents
-		// regular trending in sections with shelfRenderer without title
-		.filterMap(x => {
-			var shelfRenderer = x.itemSectionRenderer.contents.find(x => x.shelfRenderer)?.shelfRenderer;
-			if (shelfRenderer && !shelfRenderer.title) {
-				return shelfRenderer
-					.content
-					.expandedShelfContentsRenderer
-					.items
-					.filterMap(x => x.videoRenderer)
-					.map(parseVideoRendererData)
-			};
-		})
-		.flat();
-
-
-	return {tabs, videos};
-}
@@ -26,35 +26,3 @@ export function recursiveFind(object, fn) {
 	})(object);
 	return results;
 }
-
-
-Object.prototype.concatRunsText = function concatRunsText() {
-	return this.reduce((str, obj) => str += obj.text, "");
-};
-
-export function parseVideoRendererData(data) {
-	return {
-		id: data.videoId,
-		live: Boolean(data.badges?.find(x => x.metadataBadgeRenderer?.style == "BADGE_STYLE_TYPE_LIVE_NOW")),
-		title: data.title?.runs?.concatRunsText(),
-		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.concatRunsText() 
-			|| data.descriptionSnippet?.runs?.concatRunsText(),
-		//thumbnailUrl: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138))?.url || data.thumbnail?.thumbnails?.[0]?.url,
-		//thumbnail: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138)) || data.thumbnail?.thumbnails?.[0],
-		thumbnail: {
-			url: `https://i.ytimg.com/vi/${data.videoId}/mqdefault.jpg`,
-			width: 320,
-			height: 180
-		},
-		uploaded: data.publishedTimeText?.simpleText || data.videoInfo?.runs?.[2]?.text,
-		lengthText: data.lengthText?.simpleText,
-		longLengthText: data.lengthText?.accessibility?.accessibilityData?.label,
-		viewCountText: data.viewCountText?.runs ? data.viewCountText.runs.concatRunsText() : data.viewCountText?.simpleText,
-		shortViewCountText: data.shortViewCountText?.simpleText || data.videoInfo?.runs?.[0]?.text,
-		channel: {
-			name: (data.ownerText || data.shortBylineText)?.runs?.concatRunsText(),
-			id: (data.ownerText || data.shortBylineText)?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId,
-			iconUrl: data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail?.thumbnails?.[0]?.url
-		}
-	};
-}
@@ -1,4 +1,3 @@
-import { parseVideoRendererData } from "./util.js";

 export async function searchYouTubeVideos(query) {
 	var url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query.replaceAll(' ', '+'))}&sp=EgIQAQ%253D%253D`;
@@ -6,9 +5,11 @@ export async function searchYouTubeVideos(query) {

 	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
 	ytInitialData = JSON.parse(ytInitialData);
+	console.debug(ytInitialData);

 	var videos = ytInitialData?.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents?.find(x => x.itemSectionRenderer?.contents?.find(x => x.videoRenderer))?.itemSectionRenderer?.contents?.filterMap(x => x.videoRenderer)?.map(parseVideoRendererData);
 	if (!videos) return {videos: []};
+	console.debug(videos.length, "results");

 	try {
 		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
@@ -24,7 +25,6 @@ export async function searchYouTubeVideos(query) {
 	return {videos, continuationData};
 }

-
 export async function continueYouTubeVideoSearch(continuationData) {
 	var data = await fetch("https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false", {
 		method: "POST",
@@ -33,10 +33,13 @@ export async function continueYouTubeVideoSearch(continuationData) {
 		},
 		body: JSON.stringify(continuationData)
 	}).then(res => res.json());
+	console.debug(data);

 	var continuationItems = data.onResponseReceivedCommands[0].appendContinuationItemsAction.continuationItems;
 	var videos = continuationItems.find(x => x.itemSectionRenderer?.contents.find(x => x.videoRenderer)).itemSectionRenderer.contents.filterMap(x => x.videoRenderer).map(parseVideoRendererData);
 	var continuationToken = continuationItems.find(x => x.continuationItemRenderer)?.continuationItemRenderer.continuationEndpoint.continuationCommand.token
+	console.debug(videos.length, "results");
+

 	return {
 		videos,
@@ -53,10 +56,12 @@ export async function getYouTubePlaylist(playlistId) {
 	var html = await fetch("https://www.youtube.com/playlist?list=" + playlistId).then(res => res.text());
 	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
 	ytInitialData = JSON.parse(ytInitialData);
+	console.debug(ytInitialData);

 	var sectionListRendererContents = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.find(tab => tab.tabRenderer.selected).tabRenderer.content.sectionListRenderer.contents;
 	var videos = sectionListRendererContents.find(x => x.itemSectionRenderer).itemSectionRenderer.contents.find(x => x.playlistVideoListRenderer).playlistVideoListRenderer.contents.filterMap(x => x.playlistVideoRenderer).map(parseVideoRendererData);
 	if (!videos) return {videos: []};
+	console.debug(videos.length, "results");

 	try {
 		var ytcfg = html.match(/ytcfg.set\(({.*})\);/)[1];
@@ -71,18 +76,19 @@ export async function getYouTubePlaylist(playlistId) {
 	return {videos, continuationData};
 }

-
 export async function continueYouTubePlaylist(continuationData) {
 	var data = await fetch("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false", {
 		method: "POST",
 		headers: {"Content-Type": "application/json"},
 		body: JSON.stringify(continuationData)
 	}).then(res => res.json());
+	console.debug(data);

 	if (!data.onResponseReceivedActions) return {videos:[]};
 	var continuationItems = data.onResponseReceivedActions[0].appendContinuationItemsAction.continuationItems;
 	var videos = continuationItems.find(x => x.itemSectionRenderer).itemSectionRenderer.contents.filterMap(x => x.playlistVideoListRenderer).map(parseVideoRendererData);
 	var continuationToken = continuationItems.find(x => x.continuationItemRenderer)?.continuationItemRenderer.continuationEndpoint.continuationCommand.token;
+	console.debug(videos.length, "results");

 	return {
 		videos,
@@ -92,3 +98,95 @@ export async function continueYouTubePlaylist(continuationData) {
 		} : null
 	}
 }
+
+
+
+
+
+
+
+
+
+export async function getTrending(bp) {
+	var url = `https://www.youtube.com/feed/trending`;
+	if (bp) url += `?bp=${bp}`;
+	var html = await fetch(url).then(res => res.text());
+	var ytInitialData = html.match(/ytInitialData = ({.*});<\/script>/)[1];
+	ytInitialData = JSON.parse(ytInitialData);
+
+	var tabs = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.map(t => {
+		return {
+			name: t.tabRenderer.title,
+			//url: `https://www.youtube.com` + t.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url
+			bp: t.tabRenderer.endpoint.browseEndpoint.params
+		}
+	});
+
+	var videos = ytInitialData
+		.contents
+		.twoColumnBrowseResultsRenderer
+		.tabs
+		.find(tab => tab.tabRenderer.selected)
+		.tabRenderer
+		.content
+		.sectionListRenderer
+		.contents
+		// regular trending in sections with shelfRenderer without title
+		.filterMap(x => {
+			var shelfRenderer = x.itemSectionRenderer.contents.find(x => x.shelfRenderer)?.shelfRenderer;
+			if (shelfRenderer && !shelfRenderer.title) {
+				return shelfRenderer
+					.content
+					.expandedShelfContentsRenderer
+					.items
+					.filterMap(x => x.videoRenderer)
+					.map(parseVideoRendererData)
+			};
+		})
+		.flat();
+
+
+	return {tabs, videos};
+}
+
+
+
+
+
+
+
+
+
+
+
+
+Object.prototype.concatRunsText = function concatRunsText() {
+	return this.reduce((str, obj) => str += obj.text, "");
+};
+
+function parseVideoRendererData(data) {
+	return {
+		id: data.videoId,
+		live: Boolean(data.badges?.find(x => x.metadataBadgeRenderer?.style == "BADGE_STYLE_TYPE_LIVE_NOW")),
+		title: data.title?.runs?.concatRunsText(),
+		description: data.detailedMetadataSnippets?.[0]?.snippetText?.runs?.concatRunsText() 
+			|| data.descriptionSnippet?.runs?.concatRunsText(),
+		//thumbnailUrl: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138))?.url || data.thumbnail?.thumbnails?.[0]?.url,
+		//thumbnail: data.thumbnail?.thumbnails?.find(x => (x.width == 360 && x.height == 202) || (x.width == 246 && x.height == 138)) || data.thumbnail?.thumbnails?.[0],
+		/*thumbnail: {
+			url: `https://i.ytimg.com/vi/${data.videoId}/mqdefault.jpg`,
+			width: 320,
+			height: 180
+		},*/
+		uploaded: data.publishedTimeText?.simpleText || data.videoInfo?.runs?.[2]?.text,
+		lengthText: data.lengthText?.simpleText,
+		longLengthText: data.lengthText?.accessibility?.accessibilityData?.label,
+		viewCountText: data.viewCountText?.runs ? data.viewCountText.runs.concatRunsText() : data.viewCountText?.simpleText,
+		shortViewCountText: data.shortViewCountText?.simpleText || data.videoInfo?.runs?.[0]?.text,
+		channel: {
+			name: (data.ownerText || data.shortBylineText)?.runs?.concatRunsText(),
+			id: (data.ownerText || data.shortBylineText)?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId,
+			iconUrl: data.channelThumbnailSupportedRenderers?.channelThumbnailWithLinkRenderer?.thumbnail?.thumbnails?.[0]?.url
+		}
+	};
+}
Author	SHA1	Message	Date
lamp	e7d8a4b737	manual input parsing	2024-06-19 23:25:30 -07:00
lamp	8ac07acf3b	reorganize youtube stuff	2024-06-17 16:24:26 -07:00
lamp	393c073072	combine app.js into index.js	2024-06-17 16:15:01 -07:00
lamp	40c63daccd	debug	2024-06-17 16:04:19 -07:00
lamp	7c25cfb6a4	use smaller thumbnails for playlist	2024-06-17 15:50:03 -07:00