129 lines
3.8 KiB
JavaScript
129 lines
3.8 KiB
JavaScript
var fs = require("fs");
|
|
var {UserID, Cookie} = require("./config.json");
|
|
|
|
async function sleep(seconds) {
|
|
await new Promise(resolve => setTimeout(resolve, seconds * 1000));
|
|
}
|
|
|
|
async function get(url) {
|
|
var res;
|
|
await (async function tryRequest() {
|
|
console.log("get", url);
|
|
try {
|
|
res = await fetch(url, {
|
|
headers: {
|
|
Cookie,
|
|
Referer: "https://www.pixiv.net/"
|
|
}
|
|
});
|
|
} catch(error) {
|
|
console.error(error.stack);
|
|
await sleep(1);
|
|
await tryRequest();
|
|
return;
|
|
}
|
|
if (res.status == 429) {
|
|
console.warn(res.status, res.statusText);
|
|
await sleep(10);
|
|
await tryRequest();
|
|
}
|
|
})();
|
|
if (!res.ok) {
|
|
throw new Error(`${res.url} ${res.status} ${res.statusText} ${await res.text()}`);
|
|
}
|
|
return res;
|
|
}
|
|
|
|
async function archiveArtwork(id, index) {
|
|
console.log("archive", id, index);
|
|
var prefix = index.toString().padStart(5, '0');
|
|
try {
|
|
var html = await get(`https://www.pixiv.net/en/artworks/${id}`).then(res => res.text());
|
|
var illust = Object.values(JSON.parse(html.match(/<meta name="preload-data" id="meta-preload-data" content='(.*)'>/)[1]).illust)[0];
|
|
fs.writeFileSync(`collection/${prefix}_${id}.json`, JSON.stringify(illust), {flag: "wx"});
|
|
for (let i = 0; i < illust.pageCount; i++) {
|
|
let url = illust.urls.original.replace('p0', 'p'+i);
|
|
let filename = url.split('/').pop();
|
|
try {
|
|
let res = await get(url);
|
|
let buffer = Buffer.from(await res.arrayBuffer());
|
|
fs.writeFileSync(`collection/${prefix}_${filename}`, buffer, {flag: "wx"});
|
|
} catch(error) {
|
|
console.error(error.stack);
|
|
fs.writeFileSync(`collection/${prefix}_${filename}_error.txt`, error.stack, {flag: "wx"});
|
|
}
|
|
}
|
|
} catch (error) {
|
|
fs.writeFileSync(`collection/${prefix}_${id}_error.txt`, error.stack, {flag: "wx"});
|
|
console.error(error.stack);
|
|
}
|
|
}
|
|
|
|
(async function main() {
|
|
var lastartwork = {
|
|
id: undefined,
|
|
index: -1
|
|
};
|
|
try {
|
|
var files = fs.readdirSync("collection");
|
|
} catch (e) {
|
|
if (e.code == "ENOENT") {
|
|
fs.mkdirSync("collection");
|
|
var files = [];
|
|
} else throw e;
|
|
}
|
|
for (let file of files) {
|
|
let match = file.match(/^(\d{5})_(\d+)/);
|
|
if (!match) continue;
|
|
let prefix = match[1], index = Number(prefix), id = Number(match[2])
|
|
// find last saved artwork
|
|
if (index > lastartwork.index) {
|
|
lastartwork.index = index;
|
|
lastartwork.id = id;
|
|
}
|
|
// fetch missing images
|
|
if (file.endsWith(".json")) {
|
|
let illust = JSON.parse(fs.readFileSync(`collection/${file}`, "utf8"));
|
|
for (let i = 0; i < illust.pageCount; i++) {
|
|
let img_url = illust.urls.original.replace('p0', 'p'+i);
|
|
let img_filename = img_url.split('/').pop();
|
|
let img_local_filename = `${prefix}_${img_filename}`;
|
|
if (!files.includes(img_local_filename)) {
|
|
console.log(`missing ${img_local_filename}`);
|
|
try {
|
|
let res = await get(img_url);
|
|
let buffer = Buffer.from(await res.arrayBuffer());
|
|
fs.writeFileSync(`collection/${img_local_filename}`, buffer, {flag: "wx"});
|
|
} catch(error) {
|
|
console.error(error.stack);
|
|
fs.writeFileSync(`collection/${img_local_filename}_error.txt`, error.stack, {flag: "wx"});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
console.debug("lastartwork", lastartwork);
|
|
var newIds = [];
|
|
var offset = 0;
|
|
l1: while (true) {
|
|
let data = await get(`https://www.pixiv.net/ajax/user/${UserID}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`)
|
|
.then(res => res.json());
|
|
let artworkIds = data.body.works.map(x => x.id);
|
|
if (!artworkIds.length) break;
|
|
for (let id of artworkIds) {
|
|
if (id == lastartwork.id) break l1;
|
|
newIds.push(id);
|
|
}
|
|
if (!artworkIds.length) break;
|
|
offset += artworkIds.length;
|
|
}
|
|
newIds = newIds.reverse();
|
|
console.log("archiving", newIds.length);
|
|
for (let id of newIds) {
|
|
await archiveArtwork(id, lastartwork.index + 1);
|
|
lastartwork.id = id;
|
|
lastartwork.index++;
|
|
await sleep(1);
|
|
}
|
|
})();
|