mkdir

.gitignore

@@ -2,3 +2,4 @@ collection
 config.json
 lastartwork.json
 .DS_Store
+._*

index.js

@@ -1,26 +1,33 @@
 var fs = require("fs");
-var {Readable} = require("stream");
-
-if (!fs.existsSync("collection")) fs.mkdirSync("collection");
 var {UserID, Cookie} = require("./config.json");
 
-try {
-	var lastartwork = require("./lastartwork.json");
-} catch (error) {
-	var lastartwork = {
-		id: undefined,
-		index: -1
-	};
+async function sleep(seconds) {
+	await new Promise(resolve => setTimeout(resolve, seconds * 1000));
 }
 
 async function get(url) {
+	var res;
+	await (async function tryRequest() {
 		console.log("get", url);
-	var res = await fetch(url, {
+		try {
+			res = await fetch(url, {
 				headers: {
 					Cookie,
 					Referer: "https://www.pixiv.net/"
 				}
 			});
+		} catch(error) {
+			console.error(error.stack);
+			await sleep(1);
+			await tryRequest();
+			return;
+		}
+		if (res.status == 429) {
+			console.warn(res.status, res.statusText);
+			await sleep(10);
+			await tryRequest();
+		}
+	})();
 	if (!res.ok) {
 		throw new Error(`${res.url} ${res.status} ${res.statusText} ${await res.text()}`);
 	}
@@ -37,9 +44,14 @@ async function archiveArtwork(id, index) {
 		for (let i = 0; i < illust.pageCount; i++) {
 			let url = illust.urls.original.replace('p0', 'p'+i);
 			let filename = url.split('/').pop();
-			let writeStream = fs.createWriteStream(`collection/${prefix}_${filename}`, {flags: "wx"});
+			try {
 				let res = await get(url);
-			Readable.fromWeb(res.body).pipe(writeStream);
+				let buffer = Buffer.from(await res.arrayBuffer());
+				fs.writeFileSync(`collection/${prefix}_${filename}`, buffer, {flag: "wx"});
+			} catch(error) {
+				console.error(error.stack);
+				fs.writeFileSync(`collection/${prefix}_${filename}_error.txt`, error.stack, {flag: "wx"});
+			}
 		}
 	} catch (error) {
 		fs.writeFileSync(`collection/${prefix}_${id}_error.txt`, error.stack, {flag: "wx"});
@@ -47,9 +59,51 @@ async function archiveArtwork(id, index) {
 	}
 }
 
-(async function archiveBookmarks() {
+(async function main() {
+	var lastartwork = {
+		id: undefined,
+		index: -1
+	};
+	try {
+		var files = fs.readdirSync("collection");
+	} catch (e) {
+		if (e.code == "ENOENT") {
+			fs.mkdirSync("collection");
+			var files = [];
+		} else throw e;
+	}
+	for (let file of files) {
+		let match = file.match(/^(\d{5})_(\d+)/);
+		if (!match) continue;
+		let prefix = match[1], index = Number(prefix), id = Number(match[2])
+		// find last saved artwork
+		if (index > lastartwork.index) {
+			lastartwork.index = index;
+			lastartwork.id = id;
+		}
+		// fetch missing images
+		if (file.endsWith(".json")) {
+			let illust = JSON.parse(fs.readFileSync(`collection/${file}`, "utf8"));
+			for (let i = 0; i < illust.pageCount; i++) {
+				let img_url = illust.urls.original.replace('p0', 'p'+i);
+				let img_filename = img_url.split('/').pop();
+				let img_local_filename = `${prefix}_${img_filename}`;
+				if (!files.includes(img_local_filename)) {
+					console.log(`missing ${img_local_filename}`);
+					try {
+						let res = await get(img_url);
+						let buffer = Buffer.from(await res.arrayBuffer());
+						fs.writeFileSync(`collection/${img_local_filename}`, buffer, {flag: "wx"});
+					} catch(error) {
+						console.error(error.stack);
+						fs.writeFileSync(`collection/${img_local_filename}_error.txt`, error.stack, {flag: "wx"});
+					}
+				}
+			}
+		}
+	}
+	console.debug("lastartwork", lastartwork);
 	var newIds = [];
-
 	var offset = 0;
 	l1: while (true) {
 		let data = await get(`https://www.pixiv.net/ajax/user/${UserID}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`)
@@ -63,14 +117,12 @@ async function archiveArtwork(id, index) {
 		if (!artworkIds.length) break;
 		offset += artworkIds.length;
 	}
-
 	newIds = newIds.reverse();
 	console.log("archiving", newIds.length);
 	for (let id of newIds) {
 		await archiveArtwork(id, lastartwork.index + 1);
 		lastartwork.id = id;
 		lastartwork.index++;
-		fs.writeFileSync("lastartwork.json", JSON.stringify(lastartwork));
+		await sleep(1);
 	}
-
 })();

readme.md

@@ -0,0 +1,20 @@
+A script to download your Pixiv bookmarks, because some of them disappear!
+
+## How to use
+
+Bypass Cloudflare or it might block the script. Find origin IPs with `nslookup pixiv.net` and add to your local DNS or hosts file for `www.pixiv.net`.
+
+Create `config.json` file:
+
+```json
+{
+	"UserID": "your_pixiv_user_id",
+	"Cookie": "paste_cookies_here"
+}
+```
+
+User ID is in your Pixiv profile url: `https://www.pixiv.net/en/users/<user id>`.
+
+Use devtools to get Cookie, select a www.pixiv.net request and copy the entire contents of Cookie header. (todo: not sure which exact cookies are actually needed)
+
+Run Node.js 18+ `node .`. All bookmarked images plus meta json are saved in `collection` folder. Run again whenever you save new bookmarks.