.gitignore

@@ -0,0 +1,2 @@
+credentials.json
+known_ids.csv

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+	"deno.enable": true,
+	"deno.lint": false
+}

catchup.js

@@ -0,0 +1,28 @@
+import { credentials, pixivToPleroma, known_ids } from "./common.js";
+var {pixiv_cookie, pixiv_user_id} = credentials;
+
+var new_ids = [];
+top: for (var offset = 0;;) {
+	let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
+	console.log("get", url);
+	let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
+	let ids = data.body.works.map(x => x.id);
+	if (!ids.length) break;
+	let this_page_has_known_ids = false;
+	for (let id of ids) {
+		if (id == Deno.args[0]) break top; //init empty db
+		if (id in known_ids) {
+			this_page_has_known_ids = true;
+		} else {
+			new_ids.push(id);
+		}
+	}
+	if (this_page_has_known_ids) break;
+	offset += ids.length;
+}
+
+console.log("new ids", new_ids);
+
+for (let id of new_ids.reverse()) {
+	await pixivToPleroma(id);
+}

common.js

@@ -0,0 +1,138 @@
+export var credentials = JSON.parse(Deno.readTextFileSync("credentials.json"));
+var {client_id, client_secret, username, password, pixiv_cookie, access_token, pleroma_user_id} = credentials;
+
+
+export var known_ids = {};
+try {
+	known_ids = Object.fromEntries(Deno.readTextFileSync("known_ids.csv").trim().split("\n").map(line => line.split(",")));
+} catch (e) {}
+
+
+if (!access_token) {
+	let form = new FormData();
+	form.append("client_id", client_id);
+	form.append("client_secret", client_secret);
+	form.append("grant_type", "password");
+	form.append("username", username);
+	form.append("password", password);
+	let data = await fetch("https://pleroma.lamp.wtf/oauth/token", {
+		method: "POST",
+		body: form
+	}).then(res => res.json());
+	console.log ("logged in", data);
+	credentials.access_token = access_token = data.access_token;
+	Deno.writeTextFileSync("credentials.json", JSON.stringify(credentials, null, '\t'));
+}
+
+export async function getAllStatuses() {
+	var allStatuses = [];
+	await (async function getPage(max_id) {
+		var url = `https://pleroma.lamp.wtf/api/v1/accounts/${pleroma_user_id}/statuses?limit=40`;
+		if (max_id) url += `&max_id=${max_id}`;
+		console.log("get", url);
+		var statuses = await fetch(url, {headers: {"Authorization": `Bearer ${access_token}`}}).then(res => res.json());
+		if (!statuses.length) return;
+		allStatuses.push(...statuses);
+		var last_id = statuses.at(-1).id;
+		await getPage(last_id);
+	})();
+	return allStatuses;
+}
+
+async function uploadFile({data, name}) {
+	var form = new FormData();
+	form.append("file", data, name);
+	var res = await fetch("https://pleroma.lamp.wtf/api/v1/media", {
+		method: "POST",
+		body: form,
+		headers: {"Authorization": `Bearer ${access_token}`}
+	});
+	if (!res.ok) throw new Error("HTTP " + res.status);
+	var json = await res.json();
+	console.log("uploaded file", res.status, json.url);
+	return json;
+}
+
+async function postStatus({status, visibility = "unlisted", content_type = "text/plain", media_ids = [], sensitive, files}) {
+	if (files) {
+		media_ids = (await Promise.all(files.map(file => uploadFile(file)))).map(d => d.id);
+	}
+	var form = new FormData();
+	form.append("status", status);
+	form.append("visibility", visibility);
+	form.append("source", "bot");
+	form.append("content_type", content_type);
+	for (let media_id of media_ids) {
+		form.append("media_ids[]", media_id);
+	}
+	if (sensitive) form.append("sensitive", "true");
+	var res = await fetch("https://pleroma.lamp.wtf/api/v1/statuses", {
+		method: "POST",
+		body: form,
+		headers: {"Authorization": `Bearer ${access_token}`}
+	});
+	if (!res.ok) throw new Error("HTTP " + res.status);
+	var json = await res.json();
+	console.log("posted", res.status, json.uri || json);
+	return json;
+}
+
+async function downloadPixivIllust(illust_id) {
+	var url = `https://www.pixiv.net/en/artworks/${illust_id}`;
+	console.log("fetch", url);
+	var res = await fetch(url, {headers: {"Cookie": pixiv_cookie}});
+	if (!res.ok) throw new Error("HTTP " + res.status);
+	var html = await res.text();
+	var illust = Object.values(JSON.parse(html.match(/<meta name="preload-data" id="meta-preload-data" content='(.*)'>/)[1]).illust)[0];
+	try {
+		let u = `https://www.pixiv.net/ajax/illust/${illust_id}/pages`;
+		console.log("fetch", u);
+		let res = await fetch(u, {headers: {"Cookie": pixiv_cookie}});
+		if (!res.ok) throw new Error("HTTP " + res.status);
+		let json = await res.json();
+		var images = json.body.map(x => ({
+			url: x.urls.original,
+			width: x.width,
+			height: x.height
+		}));
+	} catch (error) {
+		console.error(error.stack);
+		var images = [];
+		for (let i = 0; i < illust.pageCount; i++) {
+			images.push({
+				url: illust.urls.original.replace('p0', 'p'+i)
+			});
+		}
+	}
+	for (let image of images) {
+		image.name = image.url.split('/').pop();
+		console.log("fetch", image.url);
+		image.data = await fetch(image.url, {headers: {"Referer": "https://www.pixiv.net/"}}).then(res => res.blob());
+	};
+	return {illust, images};
+}
+
+export async function pixivToPleroma(illust_id) {
+	try {
+		var {illust, images} = await downloadPixivIllust(illust_id);
+		var {url} = await postStatus({
+			status: `https://www.pixiv.net/en/artworks/${illust_id}`
+				+ ((images.length > 4) ? `\n⚠ There are ${images.length} attachments` : '')
+				+ ((illust.illustType == 2) ? '\n⚠ ugoira conversion not implemented, view on Pixiv' : ''),
+			files: images,
+			visibility: "unlisted",
+			sensitive: Boolean(illust.xRestrict)
+		});
+		Deno.writeTextFileSync("known_ids.csv", `${illust_id},${url}\n`, {append: true});
+
+		var {postPixivIllustToMatrix} = await import("./matrix.js"); //circular dependency -.-
+		await postPixivIllustToMatrix(illust, images, url);
+		
+	} catch (error) {
+		console.error(error.stack);
+		postStatus({
+			status: `https://www.pixiv.net/en/artworks/${illust_id}` +
+				`\n#error:\n${error.stack}`
+		});
+	}
+}

matrix.js

@@ -1,6 +1,5 @@
 import ky from 'https://esm.sh/ky';
-import credentials from "./credentials.json" with { type: "json" };
+import { credentials } from "./common.js";
-var {matrix_token, pixiv_cookie, pixiv_user_id} = credentials;
 
 var ke = ky.create({
 	hooks: {
@@ -14,7 +13,6 @@ var ke = ky.create({
 	timeout: false
 });
 
-
 var matrix = {
 	ky: ke.extend({headers: {"Authorization": `Bearer ${credentials.matrix_token}`}}),
 	rooms: {
@@ -35,34 +33,12 @@ var matrix = {
 		var content_uri = await this.uploadFile(filename, data);
 		return await this.sendMessage({room, type: "m.image", body: body || filename, filename, url: content_uri, info});
 	}
-}
+};
 
-
+export async function postPixivIllustToMatrix(illust, images, msg) {
-
-
-
-
-
-
-
-
-async function downloadPixivIllust(illust_id) {
-	var html = await ke.get(`https://www.pixiv.net/en/artworks/${illust_id}`, {headers: {"Cookie": credentials.pixiv_cookie}}).text();
-	var illust = Object.values(JSON.parse(html.match(/<meta name="preload-data" id="meta-preload-data" content='(.*)'>/)[1]).illust)[0];
-	var images = [];
-	for (let i = 0; i < illust.pageCount; i++) {
-		let url = illust.urls.original.replace('p0', 'p'+i);
-		let data = await ke.get(url, {headers: {"Referer": "https://www.pixiv.net/"}}).then(res => res.blob());
-		images.push({url, name: url.split('/').pop(), data});
-	};
-	return {illust, images};
-}
-
-export async function pixivToMatrix(illust_id) {
 	try {
-		var {illust, images} = await downloadPixivIllust(illust_id);
 		var room = illust.xRestrict ? matrix.rooms.pixiv_nsfw : matrix.rooms.pixiv_sfw;
-		await matrix.sendMessage({room, body: `https://www.pixiv.net/en/artworks/${illust_id}`});
+		await matrix.sendMessage({room, body: msg || `https://www.pixiv.net/en/artworks/${illust.id}`});
 		for (var image of images) {
 			try {
 				await matrix.sendImage({
@@ -71,7 +47,9 @@ export async function pixivToMatrix(illust_id) {
 					data: image.data,
 					info: {
 						mimetype: image.data.type,
-						size: image.data.size
+						size: image.data.size,
+						w: image.width,
+						h: image.height
 					}
 				});
 			} catch (error) {
@@ -87,27 +65,4 @@ export async function pixivToMatrix(illust_id) {
 			console.error(error.stack);
 		});
 	}
-}
-
-
-
-
-var new_ids = [];
-top: for (var offset = 0;;) {
-	let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
-	console.log("get", url);
-	let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
-	let ids = data.body.works.map(x => x.id);
-	if (!ids.length) break;
-	for (let id of ids) {
-		if (["119320087", "119534122"].includes(id)) break top;
-		new_ids.push(id);
-	}
-	offset += ids.length;
-}
-
-console.log("new ids", new_ids);
-
-for (let id of new_ids.reverse()) {
-	await pixivToMatrix(id);
 }

missing.js

@@ -0,0 +1,39 @@
+import { credentials, pixivToPleroma, getAllStatuses } from "./common.js";
+var {pixiv_cookie, pixiv_user_id} = credentials;
+
+console.log("get all pixiv bookmarks");
+
+var all_bookmark_ids = [];
+for (var offset = 0;;) {
+	let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
+	console.log("get", url);
+	let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
+	let works = data.body.works;
+	if (!works.length) break;
+	offset += works.length;
+	let ids = works.filter(w => !w.isMasked).map(x => x.id.toString());
+	all_bookmark_ids.push(...ids);
+}
+
+console.log(`${all_bookmark_ids.length} ummasked bookmarks`);
+console.log("get all pleroma statuses");
+
+var statuses = await getAllStatuses();
+console.log(`${statuses.length} statuses`);
+
+statuses = statuses.map(x => x.content);
+
+var missing_ids = [];
+
+for (let id of all_bookmark_ids.reverse()) {
+	if (statuses.some(status => status.includes(id) && !status.includes("#error"))) continue;
+	missing_ids.push(id);
+}
+
+console.log("missing_ids", missing_ids);
+
+confirm();
+
+for (let id of missing_ids) {
+	await pixivToPleroma(id);
+}

server.js

@@ -0,0 +1,50 @@
+import { pixivToPleroma } from "./common.js";
+
+Deno.serve({
+	port: 18024,
+	hostname: "127.0.0.1"
+}, async (req, info) => {
+	var {pathname} = new URL(req.url);
+	console.log(info.remoteAddr.hostname, req.headers.get('x-forwarded-for'), pathname);
+
+	try {
+		switch (pathname) {
+			case "/ajax/illusts/bookmarks/add":
+				var payload = await req.clone().json();
+				console.log(payload);
+				var {comment, illust_id, restrict, tags} = payload;
+				pixivToPleroma(illust_id);
+				break;
+			case "/ajax/illusts/bookmarks/delete":
+				var payload = await req.clone().formData();
+				console.log(payload);
+				// todo
+				break;
+			case "/touch/ajax_api/ajax_api.php":
+				var payload = await req.clone().formData();
+				console.log(payload);
+				var mode = payload.get("mode");
+				if (mode == "add_bookmark_illust") {
+					//var restrict = payload.get("restrict");
+					//var tag = payload.get("tag");
+					var id = payload.get("id");
+					//var comment = payload.get("comment");
+					pixivToPleroma(id);
+				} else if (mode == "delete_bookmark_illust") {
+					var id = payload.get("id");
+					// todo
+				}
+				break;
+			//todo mobile app
+		}
+	} catch (error) {
+		console.error("payload error:", error.stack);
+	}
+
+	return fetch("https://www.pixiv.net"+pathname, {
+		method: req.method,
+		headers: req.headers,
+		body: req.body,
+		redirect: "manual"
+	});
+});