pixiv-discord-dumper/main.js

process.on("unhandledRejection", error => console.error(error.stack));
require("dotenv").config();
var Discord = require("discord.js");
var puppeteer = require("puppeteer");
var fetch = require("node-fetch");
var exitHook = require('async-exit-hook');


var browser;
puppeteer.launch({
	headless: !Boolean(process.env.INIT),
	userDataDir: process.cwd() + "/chromium_data"
}).then(x => {
	browser = x;
	exitHook(cb => browser.close().then(cb))
});



var client = new Discord.Client({intents: 32767, restRequestTimeout: 10*60*1000});
client.login(process.env.TOKEN);
client.once("ready", () => {
	client.guilds.resolve("672956423545815040").commands.set([
		{
			name: "dump",
			description: "Dump popular results of a Pixiv tag",
			options: [
				{
					name: "tag",
					description: "Pixiv tag",
					type: "STRING",
					required: true
				}
			]
		}
	]);
});

client.on("interactionCreate", async i => {
	if (i.isCommand() && i.commandName == "dump") {
		if (i.channel.id != "889598863944609882") return i.reply({content: "in <#889598863944609882> only pls", ephemeral: true});

		let tag = i.options.getString("tag");
		if (!tag) return i.reply({content: "bruh no tag", ephemeral: true});

		let reply = await i.reply({content: `${i.user} started Pixiv dump for tag **${tag}**`, fetchReply: true});
		let thread = await reply.startThread({name: tag, autoArchiveDuration: 'MAX'});
		dump(tag, thread);
	} 
});






async function dump(tag, thread) {
	try {
		var page_number = 1;
		var page = await browser.newPage();

		let sefg8uyaogse8ygiluahrgihseiufhguisdg = true;
		let lastpagefirstpostid;

		// for each page of search
		while (true) {
			let url = `https://www.pixiv.net/ajax/search/artworks/${encodeURIComponent(tag)}?order=popular_d&mode=all&p=${page_number}`;
			console.log(`p get ${url}`);
			await page.goto(url);
			let data = JSON.parse(await page.evaluate(() => document.querySelector("body").innerText));
			if (data.error) throw data.message;

			if (data.body.illustManga.data.length == 0) {
				await thread.send(`No data on next page. Abort.`);
				break;
			}

			if (sefg8uyaogse8ygiluahrgihseiufhguisdg) { await thread.send(`${data.body.illustManga.total?.toLocaleString()} works`);  sefg8uyaogse8ygiluahrgihseiufhguisdg = false;   }	

			if (data.body.illustManga.data[0].id == lastpagefirstpostid) {
				await thread.send("It appears the next page has the same data, so I guess I've reached the end.\n\nAbort.");
				break;
			} else lastpagefirstpostid = data.body.illustManga.data[0].id;

			
			// for each post on page
			for (let i = 0; i < data.body.illustManga.data.length; i++) {
				let item = data.body.illustManga.data[i];
				let post_url = `https://www.pixiv.net/en/artworks/${item.id}`;
				try {

					let url = `https://www.pixiv.net/ajax/illust/${item.id}/pages`;
					console.log(`p get ${url}`);
					await page.goto(url);
					let data = JSON.parse(await page.evaluate(() => document.querySelector("body").innerText));
					if (data.error) throw data.message;
					let attachments = [];

					// for each image on post
					for (let image of data.body) {
						let url = image.urls.original;
						console.log(`f get ${url}`);

						attachments.push({
							attachment: await (await fetch(url, {
								headers: { "Referer": "https://www.pixiv.net" }
							})).buffer(),
							name: (item.xRestrict ? "SPOILER_" : '') + url.split('/').pop(),
							x: image
						});
					}

					// chunk for posts with more than 10 images
					let chunked_attachments = chunkArray(attachments, 10);
					let m = `**${item.alt}**\n<${post_url}>`

					// if all chunks small enough send in fewest messages
					console.log("uploading");
					if (chunked_attachments.some(x => {
						let totalsize = 0;
						x.forEach(x => totalsize += x.attachment.length);
						return totalsize < 50000000 // 8000000 if not boost lvl 2
					})) {
						for (let i = 0; i < chunked_attachments.length; i++) {
							await thread.send({
								content: i == 0 ? m : undefined,
								files: chunked_attachments[i]
							});
						}
					}

					// else send each image separate message
					else {
						for (let i = 0; i < attachments.length; i++) {
							let attachment = attachments[i]
							try {
								await thread.send({
									content: i == 0 ? m : undefined,
									files: [attachment]
								});
							} catch (error) {
								await thread.send(`for ${attachment.x.urls.original} (page ${page_number} item ${i}:): \`\`\`\n${error.stack}\n\`\`\``);
								// if too large try the regular edition
								if (error.message == "Request entity too large") {
									await thread.send(`trying regular size instead`);
									let url = attachment.x.urls.regular;
									console.log(`f get 2 ${url}`);
									try {
										await thread.send({files:[{
											attachment: (await fetch(url, {headers:{"Referer":"https://www.pixiv.net"}})).buffer(),
											name: (item.xRestrict ? "SPOILER_" : '') + url.split('/').pop()
										}]});
									} catch (err0r) {
										await thread.send(`still: ${err0r.message}`);
									}
								}
							}
							
						}
					}
					







				} catch (error) {
					// error for post
					await thread.send(`for <${post_url}>:\n\`\`\`\n${error.stack || error}\n\`\`\``);
				}

			}
			page_number++;
		}
	} catch (error) {
		// error on this dump
		await thread.send(`\`\`\`\n${error.stack || error}\n\`\`\`\n\nAborted.`);
	}
}






function chunkArray(array, n) {
	return array.reduce((resultArray, item, index) => { 
		const chunkIndex = Math.floor(index/n)
		if(!resultArray[chunkIndex]) {
			resultArray[chunkIndex] = []
		}
		resultArray[chunkIndex].push(item)
		return resultArray
	}, [])
}