Merge branch 'proxy' of gitea.moe:lamp/random-miku-server into proxy

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 .env
 output.log
+logs

README.md

@@ -1,21 +1,25 @@
-Simply go to https://random-miku.owo39.me/ and it will redirect to `https://random-miku.owo39.me/https://www.pixiv.net/en/artworks/<randomid>` which will serve the raw image directly, and the url as such allows you to delete the left half to get to the source. Another request to the same URL from the same IP address will be redirected to a different random image so you can click the reload button in your browser to see different images.
+A simple, random Hatsune Miku image server using no HTML*: https://random-miku.owo39.me/
 
-Some query parameters are available to change the functionality:
+The `/` URL will 302-redirect you** to a random URL like this: https://random-miku.owo39.me/https://www.pixiv.net/en/artworks/92388757, which then proxies the first original-size image from that post. The url is such that you can select and delete the left half to easily get to the Pixiv post.
 
+Refresh the window and you'll get another random image***. To do this the server remembers the last URL your IP address requested.
 
-| param | values | description | example |
-|-------|--------|-------------|---------|
-| `size` | `original`, `regular`, `small`, `thumb_mini` | Get a different image size. The default is `original`. | https://random-miku.owo39.me/?size=small |
-| `noredirect` |  | If used at `/`, server will respond the random image directly to that request instead of redirecting to a URL with the source URL in it (In this case the source id is only recoverable from `X-Pixiv-Id` header.). If used for the other, disables random redirection on reload. | https://random-miku.owo39.me/?noredirect, https://random-miku.owo39.me/https://www.pixiv.net/en/artworks/55814670?noredirect |
-| `allow_r18` |  | Enables random selection of pornographic illustrations. | https://random-miku.owo39.me/?allow_r18 |
+There are also some other parameters you can change.
 
+To include pornographic results: https://random-miku.owo39.me/?allow_r18
 
-For programatic purposes you can GET `/api` which responds just a random ID in plain text.
+Auto-refresh the page every 3 seconds using HTTP refresh header: https://random-miku.owo39.me/?auto=3 Note that this way the browser won't log them to history, but you can see your IPA's history at https://random-miku.owo39.me/log (*the only thing HTML is used for).
 
-<hr>
+Get different sizes (regular, small, or thumb_mini; default is original): https://random-miku.owo39.me/?size=small
 
-here's an embedded example.
+**Use noredirect parameter to get served a random image directly: https://random-miku.owo39.me/?noredirect This enables it to be embedded in Discord, but makes it harder to find the source. 
 
-unfortunately there's no way to get source like this (browsers don't expose redirected urls of embedded resources).
+***Use noredirect parameter to disable: https://random-miku.owo39.me/noredirect/https://www.pixiv.net/en/artworks/72389090 That's right, querystring is relocated to the left of pixiv url.
 
-![random miku image](https://random-miku.owo39.me/)
+Combining opts: https://random-miku.owo39.me/?auto=0&size=small&allow_r18 ![:sunglas:](https://cdn.discordapp.com/emojis/707727355057012806.png?size=24)
+
+To get just a random ID: https://random-miku.owo39.me/api. Useful for embedding into a website with JavaScript. There is also an `X-Pixiv-Id` header.
+
+----------------------------------------------------------------------------------------
+
+Data set consists of the top 99,495 popular 初音ミク Pixiv illustrations (10,242 of which are r18), scraped in September 2021. Originally served data from a mongo db, but was changed to proxy from Pixiv when lost access to db and because it is faster and more flexible. The old mongodb version is available at http://hmbp.gq:39/, which you can use if you encounter a 404 with the primary one (because some pixiv posts are disappearing apparently).

index.js

@@ -1,29 +1,36 @@
 var express = require("express");
 var fetch = require("node-fetch");
+var qs = require("qs");
+var serveFavicon = require("serve-favicon");
+var fs = require("fs");
+
 var app = express();
 app.set("trust proxy", "127.0.0.1");
 app.listen(process.env.PORT || 39);
+app.use(serveFavicon("ミク.png"));
+app.use((req,res,next)=>{req.rawQuery = req.url.includes('?') && req.url.substr(req.url.indexOf('?')+1);next();});
 
 var list = require("./list.json");
 var ipa_lastload_map = {};
 
+
 function getRandomId(allow_r18) {
-	let x = list[Math.floor(Math.random() * list.length)];
-	if (x[2] && !allow_r18) return getRandomId();
-	return x[0];
+	var i = list[Math.floor(Math.random() * list.length)];
+	if (i[2] && !allow_r18) return getRandomId();
+	return i[0];
 };
 
-function redirectRandom(req, res) {
-	res.redirect(`/https://www.pixiv.net/en/artworks/${getRandomId(req.query.allow_r18!=null)}${req.url.includes('?')?req.url.substr(req.url.indexOf('?')):''}`);
+
+function getRandomUrl(req) {
+	var s = req?.params.settings ? `/${req?.params.settings}` : '';
+	var d = getRandomId(req?.query.allow_r18!=null);
+	return `${s}/https://www.pixiv.net/en/artworks/${d}`;
 }
 
+
 async function serveId(id, req, res, next) {
 	try {
-		if (ipa_lastload_map[req.ip] == req.url && req.query.noredirect == null)
-			return redirectRandom(req, res); 
-		else ipa_lastload_map[req.ip] = req.url;
-
-		console.log(new Date().toLocaleString(), req.ip, id, Object.keys(req.query).length ? req.query : '', req.headers['referer'] || '');
+		console.log(new Date().toLocaleString(), req.ip, req.url);
 		
 		let pages = list.find(x => x[0] == id)?.[1];
 		if (!pages) {
@@ -43,29 +50,56 @@ async function serveId(id, req, res, next) {
 		var pxreq = await fetch(url, { headers: {"Referer": "https://www.pixiv.net"} });
 		res.type(pxreq.headers.get("Content-Type"));
 		res.header("X-Pixiv-Id", id);
+		res.header("Cache-Control", "max-age=99999999999999");
+		if (req.query.auto) res.header("Refresh", `${Number(req.query.auto)}; url=${getRandomUrl(req)}`);
 		pxreq.body.pipe(res);
 
+		fs.appendFileSync(`logs/${req.ip.replace(/[:\/]/g, '-')}.csv`, `${new Date().toISOString()},${id}\n`);
+
 	} catch(error) {
-		next(error);
+		console.log(error);
 	}
 }
 
-app.get("/https://www.pixiv.net/en/artworks/:id", function (req, res, next) {
+
+app.get("(/:settings)?/https://www.pixiv.net/en/artworks/:id", function (req, res, next) {
+	if (req.rawQuery) {
+		let settings = req.params.settings || '';
+		if (settings) settings += "&";
+		settings += req.rawQuery;
+		return res.redirect(`/${settings}/https://www.pixiv.net/en/artworks/${req.params.id}`);
+	}
+	
+	if (req.params.settings) req.query = qs.parse(req.params.settings);
+	
+	if (ipa_lastload_map[req.ip] == req.url && req.query.noredirect == null)
+		return res.redirect(getRandomUrl(req)); 
+	else ipa_lastload_map[req.ip] = req.url;
+
 	serveId(req.params.id, req, res, next);
 });
 
+
 app.get('/', (req, res, next) => {
 	if (req.query.noredirect != null) {
 		serveId(getRandomId(req.query.allow_r18!=null), req, res, next)
 	} else {
-		redirectRandom(req, res);
+		res.redirect((req.rawQuery ? `/${req.rawQuery}` : '') + getRandomUrl(req));
 	}
 });
 
-app.get("/favicon.ico", function (req, res) {
-	res.sendFile(process.cwd() + "/ミク.png");
-});
 
 app.get("/api", function (req, res) {
 	res.type("text/plain").send(getRandomId(req.query.allow_r18!=null));
 });
+
+app.get("/log", function (req, res) {
+	res.redirect(`/log/${req.ip}`);
+});
+var log_html = fs.readFileSync("log.html", "utf8");
+app.get("/log/:ipa", function (req, res) {
+	res.send(log_html);
+});
+app.get("/log/:ipa/csv", function (req, res) {
+	res.sendFile(req.params.ipa.replace(/[:\/]/g, '-') + '.csv', {root: process.cwd() + "/logs/"});
+});

log.html

@@ -0,0 +1,34 @@
+<title>history</title>
+<style>
+	.i {
+		width: 128px;
+		height: 128px;
+		display: inline-block;
+	}
+</style>
+
+<h1>view history for ip address <span id="ipa">???</span></h1>
+<p>in reverse-chronological order. hover for dates</p>
+
+<script>
+	let ipa = location.pathname.split("/").pop().trim();
+	document.getElementById("ipa").innerText = ipa;
+	fetch(location.pathname + "/csv").then(r => r.text()).then(csv => {
+		csv = csv.trim();
+		csv = csv.split("\n").map(x => x.split(','));
+
+		var imgs = [];
+
+		for (let row of csv) {
+			let date = new Date(row[0]).toLocaleString();
+			let pixiv_id = row[1];
+			let pixiv_url = "https://www.pixiv.net/en/artworks/" + pixiv_id;
+			let thumb_url = "/size=thumb_mini/" + pixiv_url;
+			//imgs.unshift(`<a href="${pixiv_url}"><img src="${thumb_url}" alt="${pixiv_id}" title="${date}" /></a>`);
+			imgs.unshift(`<a href="${pixiv_url}"><div class="i" style="background-image: url(${thumb_url})" title="${date}">${pixiv_id}</div></a>`);
+		}
+
+		document.body.innerHTML += imgs.join('');
+
+	});
+</script>

package-lock.json

@@ -6,7 +6,8 @@
     "": {
       "dependencies": {
         "express": "^4.17.1",
-        "node-fetch": "^2.6.5"
+        "node-fetch": "^2.6.5",
+        "serve-favicon": "^2.5.0"
       }
     },
     "node_modules/accepts": {
@@ -443,6 +444,31 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
       "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
     },
+    "node_modules/serve-favicon": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/serve-favicon/-/serve-favicon-2.5.0.tgz",
+      "integrity": "sha1-k10kDN/g9YBTB/3+ln2IlCosvPA=",
+      "dependencies": {
+        "etag": "~1.8.1",
+        "fresh": "0.5.2",
+        "ms": "2.1.1",
+        "parseurl": "~1.3.2",
+        "safe-buffer": "5.1.1"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/serve-favicon/node_modules/ms": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+      "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
+    },
+    "node_modules/serve-favicon/node_modules/safe-buffer": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
+      "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
+    },
     "node_modules/serve-static": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
@@ -861,6 +887,30 @@
         }
       }
     },
+    "serve-favicon": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/serve-favicon/-/serve-favicon-2.5.0.tgz",
+      "integrity": "sha1-k10kDN/g9YBTB/3+ln2IlCosvPA=",
+      "requires": {
+        "etag": "~1.8.1",
+        "fresh": "0.5.2",
+        "ms": "2.1.1",
+        "parseurl": "~1.3.2",
+        "safe-buffer": "5.1.1"
+      },
+      "dependencies": {
+        "ms": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
+          "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
+        },
+        "safe-buffer": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
+          "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
+        }
+      }
+    },
     "serve-static": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",

package.json

@@ -1,6 +1,7 @@
 {
   "dependencies": {
     "express": "^4.17.1",
-    "node-fetch": "^2.6.5"
+    "node-fetch": "^2.6.5",
+    "serve-favicon": "^2.5.0"
   }
 }