update for ytdl ejs

can't force ipv6 because some sites like niconico don't have it, change to debian so that ula can be preferred

.dockerignore

@@ -0,0 +1,4 @@
+.git
+node_modules
+downloads
+metadata

.gitignore

@@ -1,2 +1,3 @@
 node_modules
-v
+downloads
+metadata

Dockerfile

@@ -0,0 +1,15 @@
+FROM node:20-bookworm
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+RUN useradd -r -m -s /bin/bash -u 369 ytdl
+RUN apt update; apt install -y python3-pip ffmpeg mediainfo make g++ iproute2
+ADD . /app
+WORKDIR /app
+RUN npm ci
+ADD ./gai.conf /etc/gai.conf
+ENV DOWNLOADS=/downloads META_DIR=/metadata
+RUN mkdir $DOWNLOADS $META_DIR; chown -R ytdl $DOWNLOADS $META_DIR
+USER ytdl
+ENV PATH="/home/ytdl/.local/bin:$PATH"
+VOLUME $DOWNLOADS $META_DIR
+EXPOSE 8080
+CMD ["sh", "-c", "pip install --break-system-packages --upgrade yt-dlp[default]; exec node ."]

gai.conf

@@ -0,0 +1,67 @@
+# prefer ipv6 ULA over ipv4 because docker uses NATed ULA
+
+# Configuration for getaddrinfo(3).
+#
+# So far only configuration for the destination address sorting is needed.
+# RFC 3484 governs the sorting.  But the RFC also says that system
+# administrators should be able to overwrite the defaults.  This can be
+# achieved here.
+#
+# All lines have an initial identifier specifying the option followed by
+# up to two values.  Information specified in this file replaces the
+# default information.  Complete absence of data of one kind causes the
+# appropriate default information to be used.  The supported commands include:
+#
+# reload  <yes|no>
+#    If set to yes, each getaddrinfo(3) call will check whether this file
+#    changed and if necessary reload.  This option should not really be
+#    used.  There are possible runtime problems.  The default is no.
+#
+# label   <mask>   <value>
+#    Add another rule to the RFC 3484 label table.  See section 2.1 in
+#    RFC 3484.  The default is:
+#
+label ::1/128       0
+label ::/0          1
+label 2002::/16     2
+label ::/96         3
+label ::ffff:0:0/96 4
+#label fec0::/10     5
+#label fc00::/7      6
+#label 2001:0::/32   7
+#
+#    This default differs from the tables given in RFC 3484 by handling
+#    (now obsolete) site-local IPv6 addresses and Unique Local Addresses.
+#    The reason for this difference is that these addresses are never
+#    NATed while IPv4 site-local addresses most probably are.  Given           # *NEVER* NATed huh? You CLUELESS FUCKING RETARDS
+#    the precedence of IPv6 over IPv4 (see below) on machines having only
+#    site-local IPv4 and IPv6 addresses a lookup for a global address would
+#    see the IPv6 be preferred.  The result is a long delay because the
+#    site-local IPv6 addresses cannot be used while the IPv4 address is
+#    (at least for the foreseeable future) NATed.  We also treat Teredo
+#    tunnels special.
+#
+# precedence  <mask>   <value>
+#    Add another rule to the RFC 3484 precedence table.  See section 2.1
+#    and 10.3 in RFC 3484.  The default is:
+#
+#precedence  ::1/128       50
+#precedence  ::/0          40
+#precedence  2002::/16     30
+#precedence ::/96          20
+#precedence ::ffff:0:0/96  10
+#
+#    For sites which prefer IPv4 connections change the last line to
+#
+#precedence ::ffff:0:0/96  100
+
+#
+# scopev4  <mask>  <value>
+#    Add another rule to the RFC 6724 scope table for IPv4 addresses.
+#    By default the scope IDs described in section 3.2 in RFC 6724 are
+#    used.  Changing these defaults should hardly ever be necessary.
+#    The defaults are equivalent to:
+#
+#scopev4 ::ffff:169.254.0.0/112  2
+#scopev4 ::ffff:127.0.0.0/104    2
+#scopev4 ::ffff:0.0.0.0/96       14

index.html

@@ -1,65 +0,0 @@
-<meta name="og:title" content="ytdl host" />
-<meta name="og:description" content="paste youtube url, get direct mp4 url" />
-<title>ytdl host</title>
-<h1>YouTube direct MP4 server</h1>
-<p>Oculus VRChat does not support <a href="https://docs.vrchat.com/docs/video-players" target="_blank">playing videos</a> from YouTube (and <a href="https://docs.vrchat.com/docs/www-whitelist" target="_blank">other video sites</a>) because it requires the <a href="https://github.com/yt-dlp/yt-dlp" target="_blank">yt-dlp</a> python program which can't be built into the quest build, so the only way to see video in quest vrchat is with simple direct links to regular video files (requires allowing untrusted URLs). This service makes it easier to get your YouTube videos to show for quest users: Paste a YouTube video URL below, and the server will download and serve a raw mp4 file which you can paste into VRChat. Also useful for whatever other similar scenario.</p>
-<label>YouTube URL: <input type="text" placeholder="https://www.youtube.com/watch?v=dQw4w9WgXcQ" id="u" /></label>
-<input type="submit" id="s" />
-<div id="x"></div>
-<p>The <a href="v/">downloads</a> hosted on this server are listed in the frame below (right click to copy urls).</p>
-<iframe id="v" src="v/"></iframe>
-<footer><small><a href="https://gitea.moe/lamp/ytdl-h" target="_blank">code repository</a> (you can file any issues there)</small></footer>
-
-
-<script>
-	var u = document.getElementById("u");
-	var s = document.getElementById("s");
-	var x = document.getElementById("x");
-	var v = document.getElementById("v");
-	s.onclick = function () {
-		u.disabled = true;
-		s.disabled = true;
-		x.innerText = '';
-		var ws = new WebSocket(`${location.protocol.replace('http','ws')}//${location.host}/w?url=${encodeURIComponent(u.value)}`);
-		function p(m) {
-			x.innerHTML += m + '\n';
-			x.scrollTop = x.scrollHeight;
-		}
-		ws.onerror = function (event) {
-			p(`<span style="color: red">WebSocket connection failed</span>`);
-		};
-		ws.onmessage = function (evt) {
-			console.log(evt.data)
-			p("server: " + evt.data);
-		};
-		ws.onclose = function () {
-			u.value = "";
-			u.disabled = false;
-			s.disabled = false;
-			v.src = v.src;
-		};
-	};
-</script>
-
-<style>
-	body {
-		text-align: center;
-		font-family: sans-serif;
-	}
-	#u {
-		width: 320px;
-	}
-	#x {
-		text-align: left;
-		white-space: pre;
-		overflow-y: scroll;
-		max-height: 160px;
-	}
-	#v {
-		width: 100%;
-		height: 640px;
-		border: none;
-		
-	}
-
-</style>

index.js

@@ -1,2 +1,6 @@
-require("./web.js")
-require("./wss.js")
+process.env.DOWNLOADS ||= "downloads";
+process.env.META_DIR ||= "metadata";
+process.env.PORT ||= 8080;
+
+require("./web.js");
+require("./wss.js");

metalist.js

@@ -0,0 +1,47 @@
+var getVideoLength = v => require('video-length')(v, {bin: "mediainfo"});
+var getDiskInfo = require('diskusage').check;
+var fs = require("fs");
+var path = require("path");
+
+var cached = {};
+
+async function getList() {
+	if (!cached.list || Date.now() - cached.time > 30000) {
+		cached.list = await generateList();
+		cached.diskinfo = await getDiskInfo(process.env.DOWNLOADS);
+		cached.time = Date.now();
+	}
+	return cached;
+}
+
+async function generateList() {
+	var files = fs.readdirSync(process.env.DOWNLOADS, {withFileTypes: true}).filter(f => f.isFile()).map(f => f.name);
+	var list = await Promise.all(files.map(async name => {
+		var filepath = path.join(process.env.DOWNLOADS, name);
+		var metapath = path.join(process.env.META_DIR, name) + ".json";
+
+		try {
+			var metadata = JSON.parse(fs.readFileSync(metapath, "utf8"));
+		} catch (error) {
+			var metadata = {};
+			try {
+				metadata.duration = await getVideoLength(filepath);
+			} catch(error) {console.error(error.stack)};
+			fs.writeFileSync(metapath, JSON.stringify(metadata));
+		}
+
+		var {mtime, size} = fs.statSync(filepath);
+		return {name, mtime, size, duration: metadata.duration};
+	}));
+
+	list = list.sort((a,b) => a.mtime - b.mtime);
+	return list;
+}
+
+function uncacheList() {
+	cached = {};
+}
+
+
+
+module.exports = {getList, generateList, uncacheList};

package.json

@@ -1,10 +1,13 @@
 {
 	"dependencies": {
-    "express": "^4.17.3",
+		"diskusage": "^1.2.0",
+		"express": "^4.18.2",
+		"express-async-errors": "^3.1.1",
 		"morgan": "^1.10.0",
 		"proxy-addr": "^2.0.7",
-    "qs": "^6.10.3",
-    "serve-index": "^1.9.1",
-    "ws": "^8.5.0"
+		"qs": "^6.11.1",
+		"serve-favicon": "^2.5.0",
+		"video-length": "^2.0.6",
+		"ws": "^8.13.0"
 	}
 }

public/index.html

@@ -0,0 +1,115 @@
+<!DOCTYPE html><html><head>
+	<title>ytdl server</title>
+	<style>
+		
+		#url_input {
+			width: 320px;
+		}
+		#alias_input {
+			width: 64px;
+		}
+
+		#server_output {
+			max-height: 160px;
+			white-space: pre;
+			overflow-y: scroll;
+			font-family: monospace;
+			border: 1px gray solid;
+			padding: 3px;
+		}
+
+		table, th, td {
+			border: 1px black solid;
+			border-collapse: collapse;
+			padding: 3px;
+		}
+
+	</style>
+</head><body>
+	<h1>ytdl server</h1>
+	<div>paste url from one of <a href="https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md" target="_blank">supported sites</a> to import and serve best quality media file for things like vrchat where a streamable url is required.</div>
+	<p>
+		<div>url: <input id="url_input" type="text" placeholder="https://www.youtube.com/watch?v=dQw4w9WgXcQ" /></label><input id="submit_button" type="submit" /></div>
+	</p>
+	<p id="server_output"></p>
+
+	<script>
+		var url_input = document.getElementById("url_input");
+		var submit_button = document.getElementById("submit_button");
+		var server_output = document.getElementById("server_output");
+		submit_button.onclick = function() {
+			url_input.disabled = true;
+			submit_button.disabled = true;
+			server_output.innerText = '';
+			var ws = new WebSocket(`${location.protocol.replace('http','ws')}//${location.host}/w?url=${encodeURIComponent(url_input.value)}`);
+			function print(html) {
+				server_output.innerHTML += html + '\n';
+				server_output.scrollTop = server_output.scrollHeight;
+			}
+			ws.onerror = function (event) {
+				print(`<span style="color: red">WebSocket connection failed</span>`);
+			};
+			ws.onmessage = function (evt) {
+				console.log(evt.data)
+				print(evt.data);
+			};
+			ws.onclose = function () {
+				url_input.value = "";
+				url_input.disabled = false;
+				submit_button.disabled = false;
+				loadList();
+			}
+		}
+	</script>
+
+	<h2>Downloads</h2>
+	<table>
+		<thead>
+			<tr>
+				<th></th>
+				<th>File name</th>
+				<th>Duration</th>
+				<th>Size</th>
+				<th>Date added</th>
+			</tr>
+		</thead>
+		<tbody id="tbody">
+		</tbody>
+	</table>
+	<p id="diskinfo"></p>
+	<script>
+		function loadList() {
+			fetch("/api/list").then(res => res.json()).then(data => {
+				tbody.innerHTML = '';
+				data.list.forEach((file, index) => {
+					var url = `${location.origin}/v/${encodeURIComponent(file.name)}`;
+					var row = tbody.insertRow(0);
+					row.insertCell().innerHTML = `<button title="Copy URL" onclick="navigator.clipboard.writeText('${url}')">📋</button>`;
+					row.insertCell().innerHTML = `<a href="${url}">${file.name}</a>`;
+					row.insertCell().innerHTML = `<span title="${file.duration} seconds">${formatDuration(file.duration)}</span>`;
+					row.insertCell().innerHTML = `<span title="${file.size} bytes">${formatBytes(file.size)}</span>`
+					row.insertCell().innerHTML = `<span title="${file.mtime}">${new Date(file.mtime).toLocaleString()}</span>`;
+				});
+				diskinfo.innerText = `Disk free: ${formatBytes(data.diskinfo.available)}`;
+			});
+		}
+
+		loadList();
+		
+		function formatDuration(seconds) {
+			if (!seconds) return "?";
+			var d = new Date(0);
+			d.setSeconds(seconds);
+			var hms = d.toISOString().substring(11, 19);
+			if (hms.startsWith("00:")) hms = hms.substring(3);
+			return hms;
+		}
+
+		function formatBytes(bytes) {
+			if (bytes == 0) return "0 B";
+			var e = Math.floor(Math.log(bytes) / Math.log(1000));
+			return (bytes / Math.pow(1000, e)).toFixed(2) + ' ' + ' KMGTP'.charAt(e) + 'B';
+		}
+
+	</script>
+</body></html>

readme.txt

@@ -0,0 +1,5 @@
+docker build -t ytdl-server .
+
+docker run -d --restart=unless-stopped --name ytdl-server -p 8660:8080 -v "/zpool1/ytdls:/downloads" -v "./metadata/:/metadata" ytdl-server
+
+docker ipv6 required

web.js

@@ -1,15 +1,22 @@
 var express = require("express");
-var serveIndex = require("serve-index");
+require("express-async-errors");
+var serveFavicon = require("serve-favicon");
 var morgan = require("morgan");
-var http = require("http");
-var fs = require("fs");
+var {getList} = require("./metalist");
 
-var app = express();
-var server = http.createServer(app);
-//try { fs.unlinkSync("web.sock") } catch (e) {}
-server.listen(8024, "127.0.0.1");
+var app = module.exports.app = express();
+app.set("env", process.env.NODE_ENV || "production");
+app.set("trust proxy", ["loopback","linklocal","uniquelocal","173.245.48.0/20","103.21.244.0/22","103.22.200.0/22","103.31.4.0/22","141.101.64.0/18","108.162.192.0/18","190.93.240.0/20","188.114.96.0/20","197.234.240.0/22","198.41.128.0/17","162.158.0.0/15","104.16.0.0/13","104.24.0.0/14","172.64.0.0/13","131.0.72.0/22","2400:cb00::/32","2606:4700::/32","2803:f800::/32","2405:b500::/32","2405:8100::/32","2a06:98c0::/29","2c0f:f248::/32"]);
+module.exports.server = app.listen(process.env.PORT, process.env.ADDR);
 
-app.use(morgan(`:date[iso] :req[cf-connecting-ip] :method :url ":req[user-agent]" :referrer`));
-app.use(express.static('.'), serveIndex('.', {'icons': true}));
+app.use(morgan(`:date[iso] :remote-addr :method :url ":req[user-agent]" :referrer`));
 
-module.exports = server;
+app.use(serveFavicon("favicon.ico"));
+
+app.use("/v/", express.static(process.env.DOWNLOADS));
+
+app.get("/api/list", async (req, res) => {
+	res.send(await getList());
+});
+
+app.use(express.static("public"));

wss.js

@@ -1,24 +1,45 @@
 var {WebSocketServer} = require("ws");
 var qs = require("qs");
+var proxyaddr = require("proxy-addr");
 var child_process = require("child_process");
-var fs = require("fs");
+var {app, server} = require("./web");
+var {uncacheList} = require("./metalist");
 
-var wss = new WebSocketServer({ server: require("./web") });
+var lastUpdate = Date.now();
+
+var wss = new WebSocketServer({ server });
 
 wss.on("connection", function(ws, req) {
+	req.ip = proxyaddr(req, app.get("trust proxy"));
 	req.query = qs.parse(req.url.substring(req.url.indexOf('?')+1));
 
-	var log = (msg, e) => console[e?'error':'log'](`[${new Date().toLocaleString()}] ${req.headers["cf-connecting-ip"]} - ${msg}`);
+	var log = (msg, e) => console[e?'error':'log'](`[${new Date().toLocaleString()}] ${req.ip} - ${msg}`);
 	log("socket open: " + req.url);
 
-	var yturl = req.query.url;
-	if (!yturl) { ws.send(color("red", "missing url")); ws.close(); return }
+	var url = req.query.url;
+	if (!url) { ws.send(color("red", "missing url")); ws.close(); return }
 
-	var ytid = yturl.match(/https:\/\/(?:(?:www\.)?youtube\.com\/(?:watch\?v=|shorts\/)|youtu\.be\/)([A-Za-z0-9_-]{11})/)?.[1];
-	if (!ytid) { ws.send(color("red", "youtube url not valid")); ws.close(); return }
+	if (Date.now() - lastUpdate > 8.64e7) {
+		ws.send("updating yt-dlp...");
+		var cp = makeCp("pip", ["install", "--break-system-packages", "--upgrade", "yt-dlp"]);
+		cp.on("close", next);
+		lastUpdate = Date.now();
+	} else next();
 
-	ws.send(color("green", "spawning yt-dlp..."));
-	var cp = child_process.spawn("yt-dlp", ["--format=mp4", "--max-filesize=1G", `--match-filter=duration<7200`, "--no-mtime", `https://www.youtube.com/watch?v=${ytid}`], {cwd: "v", shell: false});
+	function next() {
+		var cp = makeCp("yt-dlp", ["--js-runtimes", "node", "--no-mtime", "--no-playlist", "--progress-delta", "1", url], {cwd: process.env.DOWNLOADS, shell: false});
+		cp.on("close", (code, signal) => {
+			uncacheList();
+			ws.close();
+		});
+		ws.on("close", () => {
+			log("socket closed");
+		});
+	}
+
+	function makeCp(command, args, options) {
+		ws.send(color("green", `spawning ${command}...`));
+		var cp = child_process.spawn(command, args, options);
 	
 		cp.on("error", error => {
 			log(error.message, true);
@@ -28,40 +49,24 @@ wss.on("connection", function(ws, req) {
 			var msg = data.toString().trim();
 			if (msg) {
 				log(msg);
-			ws.send("yt-dlp: " + color("blue", msg));
+				ws.send(command + ": " + color("blue", msg));
 			}
 		});
 		cp.stderr.on("data", data => {
 			var msg = data.toString().trim();
 			if (msg) {
 				log(msg, true);
-			ws.send(color("orange", "yt-dlp: " + color("blue", msg)));
+				ws.send(color("orange", command + ": " + color("blue", msg)));
 			}
 		});
 		cp.on("close", (code, signal) => {
-		log(`cp exit ${code} (${signal})`);
-		ws.send(color("green", `yt-dlp exited with code ${code}`));
-		var f = findFileForYtid(ytid);``
-		if (f) {
-			var g = `https://${req.headers.host}/v/${encodeURIComponent(f)}`;
-			ws.send(`MP4 URL (<button onclick="navigator.clipboard.writeText(\`${g}\`)">Copy</button>): <a href="${g}" target="_blank">${g}</a>`)
+			log(`${command} exit ${code} (${signal})`);
+			ws.send(color("green", `${command} exited with code ${code}`));
+		});
+		return cp;
 	}
-		ws.close();
-	});
-
-	ws.on("close", () => {
-		log("socket closed");
-		cp.kill();
-	});
 });
 
 function color(color, text) {
 	return `<span style="color: ${color}">${text}</span>`;
 }
-
-function findFileForYtid(ytid) {
-	var d = fs.readdirSync("v");
-	var e = `[${ytid}].mp4`
-	var f = d.find(f => f.endsWith(e));
-	return f;
-}