youtube antibotted the captions

gives blank file now without special code added to url

README.md

@@ -71,9 +71,11 @@ Gets Trending YouTube videos. Identical to `/search` but without `input` paramet
 ## GET `/vrcurl/{pool}/{index}`
 
 - `{pool}`: must be same as pool param in search endpoint.
-- `{index}`: vrcurl index number
+- `{index}`: vrcurl index number as specified in response data
 
-Response may be 302 redirect to youtube url, `image/png` for imagesheet, `application/json` for next page (see response format above) or trending tab or video json data (see below).
+For youtube videos, if `User-Agent` header includes `UnityWebRequest`, then response is `application/json` for video metadata (captions). Otherwise, it is 302 redirect to youtube URL.
+
+For image sheet, response is `image/png`. For next page or trending tab, response is `application/json`.
 
 ### Video metadata JSON format
 

VRCYoutubeSearch.js

@@ -37,7 +37,6 @@ async function VRCYoutubeSearch(pool, query, options = {}, key) {
 				}
 				break;
 			case "continuation":
-				//var {videos, continuationData} = await [query.for == "playlist" ? continueYouTubePlaylist : continueYouTubeVideoSearch](query.continuationData);
 				if (query.for == "playlist") {
 					var {videos, continuationData} = await continueYouTubePlaylist(query.continuationData);
 				} else {
@@ -46,7 +45,7 @@ async function VRCYoutubeSearch(pool, query, options = {}, key) {
 				break;
 		}
 	} else {
-		var playlistId = query.match(/list=(PL[a-zA-Z0-9-_]{32})/)?.[1];
+		var playlistId = query.match(/list=(PL(?:[a-zA-Z0-9-_]{32}|[0-9A-F]{16}))/)?.[1];
 		if (playlistId) console.log("playlistId:", playlistId);
 		var {videos, continuationData} = playlistId ? await getYouTubePlaylist(playlistId) : await searchYouTubeVideos(query, options.bp || (options.mode == "latestontop" ? null : undefined));
 	}
@@ -55,7 +54,7 @@ async function VRCYoutubeSearch(pool, query, options = {}, key) {
 
 	if (options.thumbnails) {
 		videos.forEach(video => {
-			if (playlistId) video.thumbnail = {
+			if (playlistId || query?.for == "playlist") video.thumbnail = {
 				url: `https://i.ytimg.com/vi/${video.id}/default.jpg`,
 				width: 120,
 				height: 90

index.js

@@ -1,5 +1,6 @@
 if (process.env.D!="BUG") console.debug = () => {};
 else console.debug(process.env);
+
 import "./util.js";
 import Koa from "koa";
 import Router from "@koa/router";
@@ -57,12 +58,14 @@ router.get("/vrcurl/:pool/:num", async ctx => {
 	}
 	switch (dest.type) {
 		case "redirect":
+			ctx.set("Referrer-Policy", "no-referrer");
 			ctx.redirect(dest.url);
 			break;
 		case "video":
 			if (ctx.get("User-Agent").includes("UnityWebRequest")) {
 				ctx.body = {captions: await getVideoCaptionsCached(dest.id)};
 			} else {
+				ctx.set("Referrer-Policy", "no-referrer");
 				ctx.redirect(`https://www.youtube.com/watch?v=${dest.id}`);
 			}
 			break;

package-lock.json

@@ -647,27 +647,6 @@
 				"node": ">= 0.8"
 			}
 		},
-		"node_modules/encoding": {
-			"version": "0.1.13",
-			"resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
-			"integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
-			"optional": true,
-			"dependencies": {
-				"iconv-lite": "^0.6.2"
-			}
-		},
-		"node_modules/encoding/node_modules/iconv-lite": {
-			"version": "0.6.3",
-			"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
-			"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
-			"optional": true,
-			"dependencies": {
-				"safer-buffer": ">= 2.1.2 < 3.0.0"
-			},
-			"engines": {
-				"node": ">=0.10.0"
-			}
-		},
 		"node_modules/end-of-stream": {
 			"version": "1.4.4",
 			"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",

test.html

@@ -14,7 +14,7 @@
 	<br />
 	<label><input id="thumbnails" type="checkbox" checked>thumbnails</label>
 	<label><input id="icons" type="checkbox" checked>icons</label>
-	<label><input id="latestontop" type="checkbox" checked>latestontop</label>
+	<label><input id="latestontop" type="checkbox">latestontop</label>
 </div>
 
 <div id="output"></div>

youtube-captions.js

@@ -8,7 +8,7 @@ var xmlParser = new XMLParser({
 async function getVideoData(videoId) {
 	var res = await gotw(`https://www.youtube.com/watch?v=${videoId}`);
 
-	var ytInitialPlayerResponse = res.body.match(/var ytInitialPlayerResponse = ({.*});/)[1];
+	var ytInitialPlayerResponse = res.body.match(/var ytInitialPlayerResponse = ({.*?});/)[1];
 	ytInitialPlayerResponse = JSON.parse(ytInitialPlayerResponse);
 
 	return ytInitialPlayerResponse;
@@ -20,7 +20,11 @@ async function getVideoCaptions(videoId) {
 	var captionTracks = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks;
 	captionTracks = await Promise.all(captionTracks.map(captionTrack => (async () => {
 		try {
-			var xml = await gotw(captionTrack.baseUrl, {resolveBodyOnly: true});
+			var promise = gotw(captionTrack.baseUrl);
+			var res = await promise;
+			if (res.statusCode != 200) throw new Error("unexpected status " + res.statusCode);
+			var xml = await promise.text();
+			if (!xml) throw new Error("empty response for caption track");
 			var parsed = xmlParser.parse(xml);
 			var lines = parsed.transcript.text;
 			if (!Array.isArray(lines)) lines = [lines];

youtube.js

@@ -114,7 +114,7 @@ export async function getYouTubePlaylist(playlistId) {
 	ytInitialData = JSON.parse(ytInitialData);
 	console.debug(ytInitialData);
 
-	var sectionListRendererContents = ytInitialData
+	var playlistVideoRendererContents = ytInitialData
 		.contents
 		.twoColumnBrowseResultsRenderer
 		.tabs
@@ -122,10 +122,10 @@ export async function getYouTubePlaylist(playlistId) {
 		.tabRenderer
 		.content
 		.sectionListRenderer
-		.contents;
-	var videos = sectionListRendererContents
+		.contents
 		.findMap(x => x.itemSectionRenderer?.contents)
-		.findMap(x => x.playlistVideoListRenderer?.contents)
+		.findMap(x => x.playlistVideoListRenderer?.contents);
+	var videos = playlistVideoRendererContents
 		.filterMap(x => x.playlistVideoRenderer)
 		.map(parseVideoRendererData);
 	if (!videos) return {videos: []};
@@ -136,7 +136,7 @@ export async function getYouTubePlaylist(playlistId) {
 		ytcfg = JSON.parse(ytcfg);
 		var continuationData = {
 			context: ytcfg.INNERTUBE_CONTEXT,
-			continuation: sectionListRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.continuationCommand.token
+			continuation: playlistVideoRendererContents.findMap(x => x.continuationItemRenderer).continuationEndpoint.commandExecutorCommand.commands.findMap(x=>x.continuationCommand).token
 		}
 	} catch (error) {
 		console.error(error.stack);
@@ -162,8 +162,7 @@ export async function continueYouTubePlaylist(continuationData) {
 		.appendContinuationItemsAction
 		.continuationItems;
 	var videos = continuationItems
-		.findMap(x => x.itemSectionRenderer?.contents)
-		.filterMap(x => x.playlistVideoListRenderer)
+		.filterMap(x => x.playlistVideoRenderer)
 		.map(parseVideoRendererData);
 	var continuationToken = continuationItems
 		.findMap(x => x.continuationItemRenderer)