lamp/archive-pixiv-bookmarks
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: lamp:9d8653e0ac24d6b31c4b23ad0db8f7841b578b27
Branches Tags
lamp:master
..
compare: lamp:b7642eea4dc620194b4361ee1ca24520b80d4f91
Branches Tags
lamp:master

No commits in common. "9d8653e0ac24d6b31c4b23ad0db8f7841b578b27" and "b7642eea4dc620194b4361ee1ca24520b80d4f91" have entirely different histories.
9d8653e0ac ... b7642eea4d

1 changed files with 9 additions and 35 deletions
Show Stats Expand all files Collapse all files

32
index.js
View File

@ -14,29 +14,13 @@ try {
} }
async function get(url) { async function get(url) {
var res;
await (async function tryRequest() {
console.log("get", url); console.log("get", url);
try { var res = await fetch(url, {
res = await fetch(url, {
headers: { headers: {
Cookie, Cookie,
Referer: "https://www.pixiv.net/" Referer: "https://www.pixiv.net/"
} }
}); });
} catch(error) {
console.error(error.stack);
await sleep(1);
await tryRequest();
return;
}
if (res.status == 429) {
console.warn(res.status, res.statusText);
await sleep(10);
await tryRequest();
}
})();
if (!res.ok) { if (!res.ok) {
throw new Error(`${res.url} ${res.status} ${res.statusText} ${await res.text()}`); throw new Error(`${res.url} ${res.status} ${res.statusText} ${await res.text()}`);
} }
@ -53,14 +37,9 @@ async function archiveArtwork(id, index) {
for (let i = 0; i < illust.pageCount; i++) { for (let i = 0; i < illust.pageCount; i++) {
let url = illust.urls.original.replace('p0', 'p'+i); let url = illust.urls.original.replace('p0', 'p'+i);
let filename = url.split('/').pop(); let filename = url.split('/').pop();
try { let writeStream = fs.createWriteStream(`collection/${prefix}_${filename}`, {flags: "wx"});
let res = await get(url); let res = await get(url);
let buffer = Buffer.from(await res.arrayBuffer()); Readable.fromWeb(res.body).pipe(writeStream);
fs.writeFileSync(`collection/${prefix}_${filename}`, buffer, {flag: "wx"});
} catch(error) {
console.error(error.stack);
fs.writeFileSync(`collection/${prefix}_${filename}_error.txt`, error.stack, {flag: "wx"});
}
} }
} catch (error) { } catch (error) {
fs.writeFileSync(`collection/${prefix}_${id}_error.txt`, error.stack, {flag: "wx"}); fs.writeFileSync(`collection/${prefix}_${id}_error.txt`, error.stack, {flag: "wx"});
@ -92,11 +71,6 @@ async function archiveArtwork(id, index) {
lastartwork.id = id; lastartwork.id = id;
lastartwork.index++; lastartwork.index++;
fs.writeFileSync("lastartwork.json", JSON.stringify(lastartwork)); fs.writeFileSync("lastartwork.json", JSON.stringify(lastartwork));
await sleep(1);
} }
})(); })();
async function sleep(seconds) {
await new Promise(resolve => setTimeout(resolve, seconds * 1000));
}