fix errored posts

Bot.js

@@ -1,4 +1,5 @@
 import {readFileSync, writeFileSync} from "fs";
+import {fetch} from "./common.js";
 
 export default class Bot {
 	constructor(opts) {
@@ -22,12 +23,11 @@ export default class Bot {
 			method: "POST",
 			body: form
 		});
-		if (!res.ok) throw new Error("HTTP "+ res.status);
 		this.loginData = await res.json();
 		writeFileSync(`logindata/${this.username}.json`, JSON.stringify(this.loginData));
 	}
 
-	async post({status, visibility = "unlisted", content_type = "text/plain", media_ids = [], sensitive, files}) {
+	async post({status, visibility = "unlisted", content_type = "text/plain", media_ids = [], sensitive, files, edit}) {
 		if (files) {
 			media_ids = (await Promise.all(files.map(file => this.uploadFile(file)))).map(d => d.id);
 		}
@@ -40,16 +40,17 @@ export default class Bot {
 			form.append("media_ids[]", media_id);
 		}
 		if (sensitive) form.append("sensitive", "true");
-		var res = await fetch(this.url + "/api/v1/statuses", {
-			method: "POST",
+		var url = this.url + "/api/v1/statuses";
+		if (edit) url += "/" + edit;
+		var res = await fetch(url, {
+			method: edit ? "PUT" : "POST",
 			body: form,
 			headers: {
 				"Authorization": `${this.loginData.token_type} ${this.loginData.access_token}`
 			}
 		});
-		if (!res.ok) throw new Error("HTTP "+ res.status);
 		var json = await res.json();
-		console.log("posted", res.status, json.uri || json);
+		console.log(edit ? "edited" : "posted", res.status, json.uri || json);
 		return json;
 	}
 
@@ -63,7 +64,6 @@ export default class Bot {
 				"Authorization": `${this.loginData.token_type} ${this.loginData.access_token}`
 			}
 		});
-		if (!res.ok) throw new Error("HTTP "+ res.status);
 		var json = await res.json();
 		console.log("uploaded file", res.status, json.url);
 		return json;

bots.js

@@ -0,0 +1,20 @@
+import credentials from "./credentials.json" assert { type: 'json' };
+import Bot from "./Bot.js";
+
+export var mikubot = new Bot({
+	url: "https://mikuobsession.net",
+	app: credentials.app,
+	username: credentials.accounts[0].username,
+	password: credentials.accounts[0].password
+});
+
+export var mikubot_nsfw = new Bot({
+	url: "https://mikuobsession.net",
+	app: credentials.app,
+	username: credentials.accounts[1].username,
+	password: credentials.accounts[1].password
+});
+
+
+await mikubot.login();
+await mikubot_nsfw.login();

common.js

@@ -0,0 +1,58 @@
+import {unescape} from 'html-escaper';
+import { getPostDataById } from "./pixiv.js";
+
+
+export async function fetch(url, options, nothrow) {
+	for (var a = 0, ma = 10, error; a < ma; a++) {
+		console.log(options.method || "GET", url);
+		try {
+			var res = await global.fetch(url, options);
+			error = null;
+			break;
+		} catch (error) {
+			console.error("fetch error", error);
+		}
+	}
+	if (error) throw error;
+	if (!res.ok && !nothrow) throw new Error(`HTTP ${res.status} ${await res.text()}`);
+	return res;
+}
+
+export async function sleep(seconds) {
+	await new Promise(resolve => setTimeout(resolve, seconds * 1000));
+}
+
+export async function pixivToPleroma(bot, pixiv_id, edit) {
+	let url = `https://www.pixiv.net/en/artworks/${pixiv_id}`;
+	try {
+		let {images, illust} = await getPostDataById(pixiv_id);
+		let date = new Date(illust.createDate);
+		let dateString = date.toLocaleDateString("en-US", {timeZone: "JST", month: "long", day: "numeric", year: "numeric"});
+		let timeString = date.toLocaleTimeString("en-US", {timeZone: "JST", hour12: true, hour: "numeric", "minute": "numeric"});
+		let tags = illust.tags.tags.map(tag => `#${tag.tag}`).join(" ");
+		if (illust.aiType == 2) tags = `#AIgenerated ${tags}`;
+		let status = `<b>${illust.title}</b> / ${illust.userName} / ${dateString} ${timeString}<br>${unescape(illust.description)}<br>${tags}<br>${url}<br>https://www.pixiv.net/en/users/${illust.userId}`;
+		if (images.length > 4) {
+			status += `<br>⚠ There are ${images.length} images.`;
+		}
+		if (illust.illustType == 2) {
+			status += `<br>⚠ This is ugoria, you have to view on Pixiv.`;
+		}
+		await bot.post({
+			status,
+			content_type: "text/html",
+			files: images,
+			sensitive: Boolean(illust.xRestrict),
+			visibility: "public",
+			edit
+		});
+		return true;
+	} catch(error) {
+		console.error(error.stack);
+		await bot.post({
+			status: `${url}\n#error\n${error.stack}`,
+			visibility: "public",
+			edit
+		});
+	}
+}

fix-errored.js

@@ -0,0 +1,24 @@
+import {readFileSync} from "fs";
+import { mikubot, mikubot_nsfw } from "./bots.js";
+import { pixivToPleroma } from "./common.js";
+
+// psql -d pleroma39 -t -c "select data from objects where data->>'type' = 'Note' and data->>'actor' like '%pixiv_hatsunemiku%'" -o dump.txt
+
+var dump = readFileSync("dump.txt", "utf8");
+dump = dump.trim().split("\n");
+dump = dump.filter(line => line.includes("#error"));
+dump = dump.map(line => JSON.parse(line.trim()));
+dump = dump.filter(obj => obj.content.includes("#error"));
+
+console.log(`${dump.length} errored posts`);
+
+for (let {id, content, actor} of dump) {
+	// how u supposed to convert this uuid to the id the api uses???
+	let ap_id = (await fetch(id, {redirect: "manual"})).headers.get("Location").split('/').at(-1);
+	console.log(`${id} => ${ap_id}`);
+
+	let pixiv_id = content.match(/https:\/\/www.pixiv.net\/en\/artworks\/(\d+)/)[1];
+	let bot = actor.includes("hentai") ? mikubot_nsfw : mikubot;
+	
+	await pixivToPleroma(bot, pixiv_id, ap_id);
+}

index.js

@@ -1,62 +1,14 @@
-import {unescape} from 'html-escaper';
-import fetchRetry from "fetch-retry";
-import { appendFileSync } from "fs";
-import credentials from "./credentials.json" assert { type: 'json' };
-import Bot from "./Bot.js";
-import { getPostDataById, getNewPixivPosts } from "./pixiv.js";
-
 process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
-global.fetch = fetchRetry(global.fetch);
-
-var mikubot = new Bot({
-	url: "https://mikuobsession.net",
-	app: credentials.app,
-	username: credentials.accounts[0].username,
-	password: credentials.accounts[0].password
-});
-
-var mikubot_nsfw = new Bot({
-	url: "https://mikuobsession.net",
-	app: credentials.app,
-	username: credentials.accounts[1].username,
-	password: credentials.accounts[1].password
-});
-
-
-await mikubot.login();
-await mikubot_nsfw.login();
-
+import { appendFileSync } from "fs";
+import { getNewPixivPosts } from "./pixiv.js";
+import { pixivToPleroma } from './common.js';
+import { mikubot, mikubot_nsfw } from "./bots.js";
 
 var newPosts = await getNewPixivPosts("初音ミク");
 
 for (let post of newPosts) {
 	let bot = post.xRestrict ? mikubot_nsfw : mikubot;
-	let url = `https://www.pixiv.net/en/artworks/${post.id}`;
-	try {
-		let {images, illust} = await getPostDataById(post.id);
-		let date = new Date(illust.createDate);
-		let dateString = date.toLocaleDateString("en-US", {timeZone: "JST", month: "long", day: "numeric", year: "numeric"});
-		let timeString = date.toLocaleTimeString("en-US", {timeZone: "JST", hour12: true, hour: "numeric", "minute": "numeric"});
-		let status = `<b>${illust.title}</b> / ${illust.userName} / ${dateString} ${timeString}<br>${unescape(illust.description)}<br>${illust.aiType == 2 ? `#AIgenerated ` : ''}${post.tags.map(tag => `#${tag}`).join(" ")}<br>${url}<br>https://www.pixiv.net/en/users/${illust.userId}`;
-		if (images.length > 4) {
-			status += `<br>⚠ There are ${images.length} images.`;
-		}
-		if (illust.illustType == 2) {
-			status += `<br>⚠ This is ugoria, you have to view on Pixiv.`;
-		}
-		await bot.post({
-			status,
-			content_type: "text/html",
-			files: images,
-			sensitive: Boolean(post.xRestrict),
-			visibility: "public"
-		});
+	if (await pixivToPleroma(bot, post.id)) {
 		appendFileSync(`known_ids/初音ミク.txt`, "\n" + post.id);
-	} catch(error) {
-		console.error(error.stack);
-		await bot.post({
-			status: `${url}\n#error\n${error.stack}`,
-			visibility: "public"
-		});
 	}
 }

package-lock.json

@@ -5,15 +5,9 @@
 	"packages": {
 		"": {
 			"dependencies": {
-				"fetch-retry": "^6.0.0",
 				"html-escaper": "^3.0.3"
 			}
 		},
-		"node_modules/fetch-retry": {
-			"version": "6.0.0",
-			"resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-6.0.0.tgz",
-			"integrity": "sha512-BUFj1aMubgib37I3v4q78fYo63Po7t4HUPTpQ6/QE6yK6cIQrP+W43FYToeTEyg5m2Y7eFUtijUuAv/PDlWuag=="
-		},
 		"node_modules/html-escaper": {
 			"version": "3.0.3",
 			"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
@@ -21,11 +15,6 @@
 		}
 	},
 	"dependencies": {
-		"fetch-retry": {
-			"version": "6.0.0",
-			"resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-6.0.0.tgz",
-			"integrity": "sha512-BUFj1aMubgib37I3v4q78fYo63Po7t4HUPTpQ6/QE6yK6cIQrP+W43FYToeTEyg5m2Y7eFUtijUuAv/PDlWuag=="
-		},
 		"html-escaper": {
 			"version": "3.0.3",
 			"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",

package.json

@@ -1,7 +1,6 @@
 {
 	"type": "module",
 	"dependencies": {
-		"fetch-retry": "^6.0.0",
 		"html-escaper": "^3.0.3"
 	}
 }

pixiv.js

@@ -1,14 +1,15 @@
-import { readFileSync, appendFileSync } from "fs";
+import { readFileSync } from "fs";
 import credentials from "./credentials.json" assert { type: 'json' };
+import { sleep, fetch } from "./common.js";
 
 
 async function cfetch(url, options) {
-	console.log("cfetch", url);
+	//console.log("cfetch", url);
 	options ||= {};
 	options.headers ||= {};
 	options.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
 	for (var i = 0; i < 8; i++) {
-		var res = await fetch(url, options);
+		var res = await fetch(url, options, true);
 		if (res.ok) return res;
 		if (res.status == 403) {
 			console.log("403 retry");
@@ -70,10 +71,6 @@ export async function getNewPixivPosts(tag = "初音ミク") {
 	}*/
 }
 
-async function sleep(seconds) {
-	await new Promise(resolve => setTimeout(resolve, seconds * 1000));
-}
-
 
 export async function getPostDataById(id) {
 	var url = `https://www.pixiv.net/en/artworks/${id}`;