update for ytdl ejs

can't force ipv6 because some sites like niconico don't have it, change to debian so that ula can be preferred

.dockerignore

@@ -0,0 +1,4 @@
+.git
+node_modules
+downloads
+metadata

.gitignore

@@ -1,4 +1,3 @@
 node_modules
-videos
+downloads
-metadata
+metadata
-aliases

Dockerfile

@@ -0,0 +1,15 @@
+FROM node:20-bookworm
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+RUN useradd -r -m -s /bin/bash -u 369 ytdl
+RUN apt update; apt install -y python3-pip ffmpeg mediainfo make g++ iproute2
+ADD . /app
+WORKDIR /app
+RUN npm ci
+ADD ./gai.conf /etc/gai.conf
+ENV DOWNLOADS=/downloads META_DIR=/metadata
+RUN mkdir $DOWNLOADS $META_DIR; chown -R ytdl $DOWNLOADS $META_DIR
+USER ytdl
+ENV PATH="/home/ytdl/.local/bin:$PATH"
+VOLUME $DOWNLOADS $META_DIR
+EXPOSE 8080
+CMD ["sh", "-c", "pip install --break-system-packages --upgrade yt-dlp[default]; exec node ."]

gai.conf

@@ -0,0 +1,67 @@
+# prefer ipv6 ULA over ipv4 because docker uses NATed ULA
+
+# Configuration for getaddrinfo(3).
+#
+# So far only configuration for the destination address sorting is needed.
+# RFC 3484 governs the sorting.  But the RFC also says that system
+# administrators should be able to overwrite the defaults.  This can be
+# achieved here.
+#
+# All lines have an initial identifier specifying the option followed by
+# up to two values.  Information specified in this file replaces the
+# default information.  Complete absence of data of one kind causes the
+# appropriate default information to be used.  The supported commands include:
+#
+# reload  <yes|no>
+#    If set to yes, each getaddrinfo(3) call will check whether this file
+#    changed and if necessary reload.  This option should not really be
+#    used.  There are possible runtime problems.  The default is no.
+#
+# label   <mask>   <value>
+#    Add another rule to the RFC 3484 label table.  See section 2.1 in
+#    RFC 3484.  The default is:
+#
+label ::1/128       0
+label ::/0          1
+label 2002::/16     2
+label ::/96         3
+label ::ffff:0:0/96 4
+#label fec0::/10     5
+#label fc00::/7      6
+#label 2001:0::/32   7
+#
+#    This default differs from the tables given in RFC 3484 by handling
+#    (now obsolete) site-local IPv6 addresses and Unique Local Addresses.
+#    The reason for this difference is that these addresses are never
+#    NATed while IPv4 site-local addresses most probably are.  Given           # *NEVER* NATed huh? You CLUELESS FUCKING RETARDS
+#    the precedence of IPv6 over IPv4 (see below) on machines having only
+#    site-local IPv4 and IPv6 addresses a lookup for a global address would
+#    see the IPv6 be preferred.  The result is a long delay because the
+#    site-local IPv6 addresses cannot be used while the IPv4 address is
+#    (at least for the foreseeable future) NATed.  We also treat Teredo
+#    tunnels special.
+#
+# precedence  <mask>   <value>
+#    Add another rule to the RFC 3484 precedence table.  See section 2.1
+#    and 10.3 in RFC 3484.  The default is:
+#
+#precedence  ::1/128       50
+#precedence  ::/0          40
+#precedence  2002::/16     30
+#precedence ::/96          20
+#precedence ::ffff:0:0/96  10
+#
+#    For sites which prefer IPv4 connections change the last line to
+#
+#precedence ::ffff:0:0/96  100
+
+#
+# scopev4  <mask>  <value>
+#    Add another rule to the RFC 6724 scope table for IPv4 addresses.
+#    By default the scope IDs described in section 3.2 in RFC 6724 are
+#    used.  Changing these defaults should hardly ever be necessary.
+#    The defaults are equivalent to:
+#
+#scopev4 ::ffff:169.254.0.0/112  2
+#scopev4 ::ffff:127.0.0.0/104    2
+#scopev4 ::ffff:0.0.0.0/96       14

index.js

@@ -1,2 +1,6 @@
-require("./web.js")
+process.env.DOWNLOADS ||= "downloads";
-require("./wss.js")
+process.env.META_DIR ||= "metadata";
+process.env.PORT ||= 8080;
+
+require("./web.js");
+require("./wss.js");

metalist.js

@@ -1,31 +1,32 @@
 var getVideoLength = v => require('video-length')(v, {bin: "mediainfo"});
+var getDiskInfo = require('diskusage').check;
 var fs = require("fs");
+var path = require("path");
 
 var cached = {};
 
 async function getList() {
-	if (!cached.list || Date.now() - cached.list.time > 30000) {
+	if (!cached.list || Date.now() - cached.time > 30000) {
-		var list = await generateList();
+		cached.list = await generateList();
+		cached.diskinfo = await getDiskInfo(process.env.DOWNLOADS);
 		cached.time = Date.now();
-		cached.list = list;
-		return list;
 	}
-	return cached.list;
+	return cached;
 }
 
 async function generateList() {
-	var files = fs.readdirSync("videos");
+	var files = fs.readdirSync(process.env.DOWNLOADS, {withFileTypes: true}).filter(f => f.isFile()).map(f => f.name);
 	var list = await Promise.all(files.map(async name => {
-		var filepath = `videos/${name}`;
+		var filepath = path.join(process.env.DOWNLOADS, name);
-		var metapath = `./metadata/${name}.json`;
+		var metapath = path.join(process.env.META_DIR, name) + ".json";
 
 		try {
-			var metadata = require(metapath);
+			var metadata = JSON.parse(fs.readFileSync(metapath, "utf8"));
 		} catch (error) {
 			var metadata = {};
 			try {
 				metadata.duration = await getVideoLength(filepath);
-			} catch(error) {console.error(error.message)};
+			} catch(error) {console.error(error.stack)};
 			fs.writeFileSync(metapath, JSON.stringify(metadata));
 		}
 
@@ -33,15 +34,6 @@ async function generateList() {
 		return {name, mtime, size, duration: metadata.duration};
 	}));
 
-	var aliases = fs.readdirSync("aliases").map(alias => [alias, fs.readFileSync(`aliases/${alias}`,"utf8")]);
-	for (let [alias, target] of aliases) {
-		let item = list.find(f => f.name == target);
-		if (item) {
-			item.aliases ||= [];
-			item.aliases.push(alias);
-		}
-	}
-
 	list = list.sort((a,b) => a.mtime - b.mtime);
 	return list;
 }

package-lock.json

@@ -5,6 +5,7 @@
 	"packages": {
 		"": {
 			"dependencies": {
+				"diskusage": "^1.2.0",
 				"express": "^4.18.2",
 				"express-async-errors": "^3.1.1",
 				"morgan": "^1.10.0",
@@ -173,6 +174,16 @@
 				"npm": "1.2.8000 || >= 1.4.16"
 			}
 		},
+		"node_modules/diskusage": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/diskusage/-/diskusage-1.2.0.tgz",
+			"integrity": "sha512-2u3OG3xuf5MFyzc4MctNRUKjjwK+UkovRYdD2ed/NZNZPrt0lqHnLKxGhlFVvAb4/oufIgQG3nWgwmeTbHOvXA==",
+			"hasInstallScript": true,
+			"dependencies": {
+				"es6-promise": "^4.2.8",
+				"nan": "^2.18.0"
+			}
+		},
 		"node_modules/ee-first": {
 			"version": "1.1.1",
 			"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -186,6 +197,11 @@
 				"node": ">= 0.8"
 			}
 		},
+		"node_modules/es6-promise": {
+			"version": "4.2.8",
+			"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
+			"integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w=="
+		},
 		"node_modules/escape-html": {
 			"version": "1.0.3",
 			"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -515,6 +531,11 @@
 			"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
 			"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
 		},
+		"node_modules/nan": {
+			"version": "2.18.0",
+			"resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz",
+			"integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w=="
+		},
 		"node_modules/negotiator": {
 			"version": "0.6.3",
 			"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -939,6 +960,15 @@
 			"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
 			"integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="
 		},
+		"diskusage": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/diskusage/-/diskusage-1.2.0.tgz",
+			"integrity": "sha512-2u3OG3xuf5MFyzc4MctNRUKjjwK+UkovRYdD2ed/NZNZPrt0lqHnLKxGhlFVvAb4/oufIgQG3nWgwmeTbHOvXA==",
+			"requires": {
+				"es6-promise": "^4.2.8",
+				"nan": "^2.18.0"
+			}
+		},
 		"ee-first": {
 			"version": "1.1.1",
 			"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -949,6 +979,11 @@
 			"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
 			"integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="
 		},
+		"es6-promise": {
+			"version": "4.2.8",
+			"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
+			"integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w=="
+		},
 		"escape-html": {
 			"version": "1.0.3",
 			"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -1197,6 +1232,11 @@
 			"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
 			"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
 		},
+		"nan": {
+			"version": "2.18.0",
+			"resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz",
+			"integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w=="
+		},
 		"negotiator": {
 			"version": "0.6.3",
 			"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",

package.json

@@ -1,5 +1,6 @@
 {
 	"dependencies": {
+		"diskusage": "^1.2.0",
 		"express": "^4.18.2",
 		"express-async-errors": "^3.1.1",
 		"morgan": "^1.10.0",

public/index.html

@@ -1,7 +1,5 @@
 <!DOCTYPE html><html><head>
-	<meta name="og:title" content="Quest VRChat Video Server" />
+	<title>ytdl server</title>
-	<meta name="og:description" content="Import YouTube videos and get short link for Quest VRChat" />
-	<title>Quest VRChat Video Server</title>
 	<style>
 		
 		#url_input {
@@ -28,30 +26,22 @@
 
 	</style>
 </head><body>
-	<h1>Quest VRChat Video Server</h1>
+	<h1>ytdl server</h1>
-	<div>Import YouTube videos onto this server to...</div>
+	<div>paste url from one of <a href="https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md" target="_blank">supported sites</a> to import and serve best quality media file for things like vrchat where a streamable url is required.</div>
-	<ul>
-		<li>Make the videos work for your friends on Quest</li>
-		<li><b>NEW:</b> Get memorable short URLs that you can type in Quest!</li>
-	</ul>
 	<p>
-		<div>Just paste a YouTube video URL here: <input id="url_input" type="text" placeholder="https://www.youtube.com/watch?v=dQw4w9WgXcQ" /></label></div>
+		<div>url: <input id="url_input" type="text" placeholder="https://www.youtube.com/watch?v=dQw4w9WgXcQ" /></label><input id="submit_button" type="submit" /></div>
-		<div>and optionally type a short memorable alias for it: <input id="alias_input" type="text" placeholder="rick" maxlength="32" /></div>
-		<div>and click <input id="submit_button" type="submit" />!</div>
 	</p>
 	<p id="server_output"></p>
 
 	<script>
 		var url_input = document.getElementById("url_input");
-		var alias_input = document.getElementById("alias_input");
 		var submit_button = document.getElementById("submit_button");
 		var server_output = document.getElementById("server_output");
 		submit_button.onclick = function() {
 			url_input.disabled = true;
-			alias_input.disabled = true;
 			submit_button.disabled = true;
 			server_output.innerText = '';
-			var ws = new WebSocket(`${location.protocol.replace('http','ws')}//${location.host}/w?url=${encodeURIComponent(url_input.value)}&alias=${encodeURIComponent(alias_input.value)}`);
+			var ws = new WebSocket(`${location.protocol.replace('http','ws')}//${location.host}/w?url=${encodeURIComponent(url_input.value)}`);
 			function print(html) {
 				server_output.innerHTML += html + '\n';
 				server_output.scrollTop = server_output.scrollHeight;
@@ -65,20 +55,17 @@
 			};
 			ws.onclose = function () {
 				url_input.value = "";
-				alias_input.value = "";
 				url_input.disabled = false;
-				alias_input.disabled = false;
 				submit_button.disabled = false;
 				loadList();
 			}
 		}
 	</script>
 
-	<h2>Videos</h2>
+	<h2>Downloads</h2>
 	<table>
 		<thead>
 			<tr>
-				<th>Short links</th>
 				<th></th>
 				<th>File name</th>
 				<th>Duration</th>
@@ -88,30 +75,29 @@
 		</thead>
 		<tbody id="tbody">
 		</tbody>
-		
 	</table>
+	<p id="diskinfo"></p>
 	<script>
-		var tbody = document.getElementById("tbody");
-
 		function loadList() {
-			fetch("/api/list").then(res => res.json()).then(list => {
+			fetch("/api/list").then(res => res.json()).then(data => {
 				tbody.innerHTML = '';
-				list.forEach((file, index) => {
+				data.list.forEach((file, index) => {
 					var url = `${location.origin}/v/${encodeURIComponent(file.name)}`;
 					var row = tbody.insertRow(0);
-					row.insertCell().innerText = [index, ...file.aliases||[]].map(x => `http://${location.host}/${x}`).join('\n');
 					row.insertCell().innerHTML = `<button title="Copy URL" onclick="navigator.clipboard.writeText('${url}')">📋</button>`;
 					row.insertCell().innerHTML = `<a href="${url}">${file.name}</a>`;
 					row.insertCell().innerHTML = `<span title="${file.duration} seconds">${formatDuration(file.duration)}</span>`;
 					row.insertCell().innerHTML = `<span title="${file.size} bytes">${formatBytes(file.size)}</span>`
 					row.insertCell().innerHTML = `<span title="${file.mtime}">${new Date(file.mtime).toLocaleString()}</span>`;
 				});
+				diskinfo.innerText = `Disk free: ${formatBytes(data.diskinfo.available)}`;
 			});
 		}
 
 		loadList();
 		
 		function formatDuration(seconds) {
+			if (!seconds) return "?";
 			var d = new Date(0);
 			d.setSeconds(seconds);
 			var hms = d.toISOString().substring(11, 19);
@@ -126,4 +112,4 @@
 		}
 
 	</script>
-</body></html>
+</body></html>

readme.txt

@@ -0,0 +1,5 @@
+docker build -t ytdl-server .
+
+docker run -d --restart=unless-stopped --name ytdl-server -p 8660:8080 -v "/zpool1/ytdls:/downloads" -v "./metadata/:/metadata" ytdl-server
+
+docker ipv6 required

web.js

@@ -3,41 +3,20 @@ require("express-async-errors");
 var serveFavicon = require("serve-favicon");
 var morgan = require("morgan");
 var {getList} = require("./metalist");
-var {readFile} = require("fs/promises");
 
 var app = module.exports.app = express();
 app.set("env", process.env.NODE_ENV || "production");
 app.set("trust proxy", ["loopback","linklocal","uniquelocal","173.245.48.0/20","103.21.244.0/22","103.22.200.0/22","103.31.4.0/22","141.101.64.0/18","108.162.192.0/18","190.93.240.0/20","188.114.96.0/20","197.234.240.0/22","198.41.128.0/17","162.158.0.0/15","104.16.0.0/13","104.24.0.0/14","172.64.0.0/13","131.0.72.0/22","2400:cb00::/32","2606:4700::/32","2803:f800::/32","2405:b500::/32","2405:8100::/32","2a06:98c0::/29","2c0f:f248::/32"]);
-module.exports.server = app.listen(process.env.PORT || 8024, process.env.ADDR);
+module.exports.server = app.listen(process.env.PORT, process.env.ADDR);
 
-app.use(morgan(`:date[iso] :req[cf-connecting-ip] :method :url ":req[user-agent]" :referrer`));
+app.use(morgan(`:date[iso] :remote-addr :method :url ":req[user-agent]" :referrer`));
 
 app.use(serveFavicon("favicon.ico"));
 
-app.use("/v/", express.static("videos"));
+app.use("/v/", express.static(process.env.DOWNLOADS));
-
-app.get("/:num", async (req, res, next) => {
-	var num = Number(req.params.num);
-	if (isNaN(num)) return next();
-	var list = await getList();
-	var item = list[num];
-	if (!item) return next();
-	res.sendFile(item.name, {root: "videos"});
-});
-
-app.get("/:alias", async (req, res, next) => {
-	try {
-		var target = await readFile(`aliases/${req.params.alias}`, "utf8");
-		if (!target) next();
-		res.sendFile(target, {root: "videos"});
-	} catch(error) {
-		next();
-	}
-});
 
 app.get("/api/list", async (req, res) => {
-	var list = await getList();
+	res.send(await getList());
-	res.send(list);
 });
 
 app.use(express.static("public"));

wss.js

@@ -3,8 +3,9 @@ var qs = require("qs");
 var proxyaddr = require("proxy-addr");
 var child_process = require("child_process");
 var {app, server} = require("./web");
-var {getList, uncacheList} = require("./metalist");
+var {uncacheList} = require("./metalist");
-var fs = require("fs");
+
+var lastUpdate = Date.now();
 
 var wss = new WebSocketServer({ server });
 
@@ -15,63 +16,57 @@ wss.on("connection", function(ws, req) {
 	var log = (msg, e) => console[e?'error':'log'](`[${new Date().toLocaleString()}] ${req.ip} - ${msg}`);
 	log("socket open: " + req.url);
 
-	var yturl = req.query.url;
+	var url = req.query.url;
-	if (!yturl) { ws.send(color("red", "missing url")); ws.close(); return }
+	if (!url) { ws.send(color("red", "missing url")); ws.close(); return }
 
-	var alias = req.query.alias;
+	if (Date.now() - lastUpdate > 8.64e7) {
-	if (alias) {
+		ws.send("updating yt-dlp...");
-		alias = alias.toLowerCase();
+		var cp = makeCp("pip", ["install", "--break-system-packages", "--upgrade", "yt-dlp"]);
-		if (!/^[a-z0-9-_]{1,32}$/.test(alias)) { ws.send(color("red", "alias must be no more than 32 chars of only letters, numbers, hypens or underscores.")); ws.close(); return }
+		cp.on("close", next);
-		if (fs.existsSync(`aliases/${alias}`)) { ws.send(color("red", `the alias ${alias} already exists.`)); ws.close(); return }
+		lastUpdate = Date.now();
+	} else next();
+
+	function next() {
+		var cp = makeCp("yt-dlp", ["--js-runtimes", "node", "--no-mtime", "--no-playlist", "--progress-delta", "1", url], {cwd: process.env.DOWNLOADS, shell: false});
+		cp.on("close", (code, signal) => {
+			uncacheList();
+			ws.close();
+		});
+		ws.on("close", () => {
+			log("socket closed");
+		});
 	}
 
-	var ytid = yturl.match(/https:\/\/(?:(?:www\.)?youtube\.com\/(?:watch\?v=|shorts\/)|youtu\.be\/)([A-Za-z0-9_-]{11})/)?.[1];
+	function makeCp(command, args, options) {
-	if (!ytid) { ws.send(color("red", "youtube url not valid")); ws.close(); return }
+		ws.send(color("green", `spawning ${command}...`));
-
+		var cp = child_process.spawn(command, args, options);
-	ws.send(color("green", "spawning yt-dlp..."));
+	
-	var cp = child_process.spawn("yt-dlp", ["--format=mp4", "--max-filesize=1G", `--match-filter=duration<7200`, "--no-mtime", `https://www.youtube.com/watch?v=${ytid}`], {cwd: "videos", shell: false});
+		cp.on("error", error => {
-
+			log(error.message, true);
-	cp.on("error", error => {
+			ws.send(color("red", error.message));
-		log(error.message, true);
+		});
-		ws.send(color("red", error.message));
+		cp.stdout.on("data", data => {
-	});
+			var msg = data.toString().trim();
-	cp.stdout.on("data", data => {
+			if (msg) {
-		var msg = data.toString().trim();
+				log(msg);
-		if (msg) {
+				ws.send(command + ": " + color("blue", msg));
-			log(msg);
+			}
-			ws.send("yt-dlp: " + color("blue", msg));
+		});
-		}
+		cp.stderr.on("data", data => {
-	});
+			var msg = data.toString().trim();
-	cp.stderr.on("data", data => {
+			if (msg) {
-		var msg = data.toString().trim();
+				log(msg, true);
-		if (msg) {
+				ws.send(color("orange", command + ": " + color("blue", msg)));
-			log(msg, true);
+			}
-			ws.send(color("orange", "yt-dlp: " + color("blue", msg)));
+		});
-		}
+		cp.on("close", (code, signal) => {
-	});
+			log(`${command} exit ${code} (${signal})`);
-	cp.on("close", (code, signal) => {
+			ws.send(color("green", `${command} exited with code ${code}`));
-		log(`cp exit ${code} (${signal})`);
+		});
-		ws.send(color("green", `yt-dlp exited with code ${code}`));
+		return cp;
-		let files = fs.readdirSync("videos");
+	}
-		let file = files.find(file => file.includes(`[${ytid}]`));
-		if (file) {
-			if (alias) fs.writeFileSync(`aliases/${alias}`, file);
-			let url = `http://${req.headers.host}/v/${file}`;
-			ws.send(`MP4 URL (<button onclick="navigator.clipboard.writeText(\`${url}\`)">Copy</button>): <a href="${url}">${url}</a>`)
-		} else {
-			ws.send(color("red", "error: couldn't find file to alias"));
-		}
-		
-		uncacheList();
-		ws.close();
-	});
-
-	ws.on("close", () => {
-		log("socket closed");
-		cp.kill();
-	});
 });
 
 function color(color, text) {
 	return `<span style="color: ${color}">${text}</span>`;
-}
+}