buffer

index.js

@@ -14,13 +14,29 @@ try {
 }
 
 async function get(url) {
-	console.log("get", url);
-	var res = await fetch(url, {
-		headers: {
-			Cookie,
-			Referer: "https://www.pixiv.net/"
+	var res;
+	await (async function tryRequest() {
+		console.log("get", url);
+		try {
+			res = await fetch(url, {
+				headers: {
+					Cookie,
+					Referer: "https://www.pixiv.net/"
+				}
+			});
+		} catch(error) {
+			console.error(error.stack);
+			await sleep(1);
+			await tryRequest();
+			return;
 		}
-	});
+		if (res.status == 429) {
+			console.warn(res.status, res.statusText);
+			await sleep(10);
+			await tryRequest();
+		}
+	})();
+	
 	if (!res.ok) {
 		throw new Error(`${res.url} ${res.status} ${res.statusText} ${await res.text()}`);
 	}
@@ -37,9 +53,14 @@ async function archiveArtwork(id, index) {
 		for (let i = 0; i < illust.pageCount; i++) {
 			let url = illust.urls.original.replace('p0', 'p'+i);
 			let filename = url.split('/').pop();
-			let writeStream = fs.createWriteStream(`collection/${prefix}_${filename}`, {flags: "wx"});
-			let res = await get(url);
-			Readable.fromWeb(res.body).pipe(writeStream);
+			try {
+				let res = await get(url);
+				let buffer = Buffer.from(await res.arrayBuffer());
+				fs.writeFileSync(`collection/${prefix}_${filename}`, buffer, {flag: "wx"});
+			} catch(error) {
+				console.error(error.stack);
+				fs.writeFileSync(`collection/${prefix}_${filename}_error.txt`, error.stack, {flag: "wx"});
+			}
 		}
 	} catch (error) {
 		fs.writeFileSync(`collection/${prefix}_${id}_error.txt`, error.stack, {flag: "wx"});
@@ -71,6 +92,11 @@ async function archiveArtwork(id, index) {
 		lastartwork.id = id;
 		lastartwork.index++;
 		fs.writeFileSync("lastartwork.json", JSON.stringify(lastartwork));
+		await sleep(1);
 	}
 
 })();
+
+async function sleep(seconds) {
+	await new Promise(resolve => setTimeout(resolve, seconds * 1000));
+}