import credentials from "./credentials.json" with {type: 'json'}; import {sleep, fetch} from "./common.js"; import JSZip from "jszip"; import {rimraf} from "rimraf"; import {tmpdir} from "os"; import {join} from "path"; import {readFileSync, writeFileSync, mkdirSync} from "fs"; import {execFileSync} from "child_process"; async function cfetch(url, options) { //console.log("cfetch", url); options ||= {}; options.headers ||= {}; options.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"; for (var i = 0; i < 8; i++) { var res = await fetch(url, options, true); if (res.ok) return res; if (res.status == 403) { console.log("403 retry"); await sleep(1); } else { throw new Error(`HTTP ${res.status} ${res.statusText}`); } } throw new Error("403"); } export async function getNewPixivPosts(tag = "初音ミク") { try { var known_ids = readFileSync(`known_ids/${tag}.txt`, "utf8").trim().split("\n"); //todo tail n lines } catch (error) { var known_ids = []; } var newPosts = []; var encodedTag = encodeURIComponent(tag); iteratePages: for (var page = 1; page < 500; page++) { let url = `https://www.pixiv.net/ajax/search/artworks/${encodedTag}?word=${encodedTag}&order=date_d&mode=all&p=${page}&csw=0&s_mode=s_tag&type=all&lang=en&version=59e08d0871b7c68569ebe89084d52eca68a1685d`; let data = await cfetch(url, { headers: { "Cookie": credentials.pixiv_cookie } }).then(res => res.json()); if (data.error) { throw new Error(JSON.stringify(data)); } for (let post of data.body.illustManga.data) { if (known_ids.includes(post.id) || new Date(post.createDate) < new Date("2024-01-01T00:00:00+09:00")) { break iteratePages; } newPosts.push(post); if (post.xRestrict) console.log("r18"); } await sleep(1); } return newPosts.reverse(); /*return { [Symbol.iterator]() { return this; }, next() { var value = newPosts.pop(); if (value) { appendFileSync(`known_ids/${tag}.txt`, "\n" + value.id); } return {value, done: !Boolean(value)} } }*/ } export async function getPostDataById(id) { var data = await cfetch(`https://www.pixiv.net/ajax/illust/${id}?lang=en`, {headers: {"Cookie": credentials.pixiv_cookie}}).then(res => res.json()); if (data.error) throw new Error(JSON.stringify(data)); var illust = data.body; if (illust.illustType == 2) { let data = await cfetch(`https://www.pixiv.net/ajax/illust/${id}/ugoira_meta?lang=en`, {headers: {"Cookie": credentials.pixiv_cookie}}).then(res => res.json()); if (data.error) throw new Error(JSON.stringify(data)); let url = data.body.originalSrc; let zip = await fetch(url, {headers: {"Referer": "https://www.pixiv.net/"}}).then(res => res.arrayBuffer()); let webp = await ugoira2webp(zip, data.body.frames); return {illust, images: [{name: url.split('/').pop()+".webp", data: new Blob([webp], {type:"image/webp"})}]} } var images = []; for (let i = 0; i < illust.pageCount; i++) { let url = illust.urls.original.replace('p0', 'p'+i); let data = await fetch(url, {headers: {"Referer": "https://www.pixiv.net/"}}).then(res => res.blob()); images.push({url, name: url.split('/').pop(), data}); }; return {illust, images}; } async function ugoira2webp(zip, frames) { console.log("convert ugoira"); var cwd = join(tmpdir(), Math.random().toString(36)); mkdirSync(cwd); var jsz = new JSZip(); await jsz.loadAsync(zip); var webpmux_args = []; for (let {file, delay} of frames) { let file_data = await jsz.file(file).async("nodebuffer"); writeFileSync(join(cwd, file), file_data); execFileSync("cwebp", ["-quiet", file, "-o", `${file}.webp`], {cwd}); webpmux_args.push("-frame", `${file}.webp`, `+${delay}`); } execFileSync("webpmux", [...webpmux_args, "-o", "output.webp"], {cwd}); var webp = readFileSync(join(cwd, "output.webp")); rimraf(cwd).catch(console.error); return webp; }