lamp/pixiv-bookmarks-to-pleroma
lamp/pixiv-bookmarks-to-pleroma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

merge into: lamp:master
Branches Tags
lamp:master
lamp:to-matrix
...
pull from: lamp:to-matrix
Branches Tags
lamp:master
lamp:to-matrix

No commits in common. "master" and "to-matrix" have entirely different histories.
master ... to-matrix

8 changed files with 52 additions and 297 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +0,0 @@
credentials.json
known_ids.csv

4
.vscode/settings.json vendored
View File

@ -1,4 +0,0 @@
{
"deno.enable": true,
"deno.lint": false
}

59
matrix.js → app.deno.js
View File

@ -1,5 +1,6 @@
import ky from 'https://esm.sh/ky';
import { credentials } from "./common.js";
import credentials from "./credentials.json" with { type: "json" };
var {matrix_token, pixiv_cookie, pixiv_user_id} = credentials;
var ke = ky.create({
hooks: {
@ -13,6 +14,7 @@ var ke = ky.create({
timeout: false
});
var matrix = {
ky: ke.extend({headers: {"Authorization": `Bearer ${credentials.matrix_token}`}}),
rooms: {
@ -33,12 +35,34 @@ var matrix = {
var content_uri = await this.uploadFile(filename, data);
return await this.sendMessage({room, type: "m.image", body: body || filename, filename, url: content_uri, info});
}
};
}
export async function postPixivIllustToMatrix(illust, images, msg) {
async function downloadPixivIllust(illust_id) {
var html = await ke.get(`https://www.pixiv.net/en/artworks/${illust_id}`, {headers: {"Cookie": credentials.pixiv_cookie}}).text();
var illust = Object.values(JSON.parse(html.match(/<meta name="preload-data" id="meta-preload-data" content='(.*)'>/)[1]).illust)[0];
var images = [];
for (let i = 0; i < illust.pageCount; i++) {
let url = illust.urls.original.replace('p0', 'p'+i);
let data = await ke.get(url, {headers: {"Referer": "https://www.pixiv.net/"}}).then(res => res.blob());
images.push({url, name: url.split('/').pop(), data});
};
return {illust, images};
}
export async function pixivToMatrix(illust_id) {
try {
var {illust, images} = await downloadPixivIllust(illust_id);
var room = illust.xRestrict ? matrix.rooms.pixiv_nsfw : matrix.rooms.pixiv_sfw;
await matrix.sendMessage({room, body: msg || `https://www.pixiv.net/en/artworks/${illust.id}`});
await matrix.sendMessage({room, body: `https://www.pixiv.net/en/artworks/${illust_id}`});
for (var image of images) {
try {
await matrix.sendImage({
@ -47,9 +71,7 @@ export async function postPixivIllustToMatrix(illust, images, msg) {
data: image.data,
info: {
mimetype: image.data.type,
size: image.data.size,
w: image.width,
h: image.height
size: image.data.size
}
});
} catch (error) {
@ -65,4 +87,27 @@ export async function postPixivIllustToMatrix(illust, images, msg) {
console.error(error.stack);
});
}
}
var new_ids = [];
top: for (var offset = 0;;) {
let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
console.log("get", url);
let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
let ids = data.body.works.map(x => x.id);
if (!ids.length) break;
for (let id of ids) {
if (["119320087", "119534122"].includes(id)) break top;
new_ids.push(id);
}
offset += ids.length;
}
console.log("new ids", new_ids);
for (let id of new_ids.reverse()) {
await pixivToMatrix(id);
}

27
catchup.js
View File

@ -1,27 +0,0 @@
import { fetch, credentials, pixivToPleroma, known_ids } from "./common.js";
var {pixiv_cookie, pixiv_user_id} = credentials;
var new_ids = [];
top: for (var offset = 0;;) {
let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
let ids = data.body.works.map(x => x.id);
if (!ids.length) break;
let this_page_has_known_ids = false;
for (let id of ids) {
if (id == Deno.args[0]) break top; //init empty db
if (id in known_ids) {
this_page_has_known_ids = true;
} else {
new_ids.push(id);
}
}
if (this_page_has_known_ids) break;
offset += ids.length;
}
console.log("new ids", new_ids);
for (let id of new_ids.reverse()) {
await pixivToPleroma(id);
}

155
common.js
View File

@ -1,155 +0,0 @@
export async function fetch(url, options = {}, nothrow) {
console.log("fetch", url);
options.headers ||= {};
options.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
for (var i = 3; i; i--) {
try {
var res = await globalThis.fetch(url, options);
break;
} catch (error) {
console.error(error.stack);
if (i <= 1) throw error;
}
console.log("retry");
}
if (!nothrow && !res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`);
return res;
}
export var credentials = JSON.parse(Deno.readTextFileSync("credentials.json"));
var {client_id, client_secret, username, password, pixiv_cookie, access_token, pleroma_user_id} = credentials;
export var known_ids = {};
try {
known_ids = Object.fromEntries(Deno.readTextFileSync("known_ids.csv").trim().split("\n").map(line => line.split(",")));
} catch (e) {}
if (!access_token) {
let form = new FormData();
form.append("client_id", client_id);
form.append("client_secret", client_secret);
form.append("grant_type", "password");
form.append("username", username);
form.append("password", password);
let data = await fetch("https://pleroma.lamp.wtf/oauth/token", {
method: "POST",
body: form
}).then(res => res.json());
console.log ("logged in", data);
credentials.access_token = access_token = data.access_token;
Deno.writeTextFileSync("credentials.json", JSON.stringify(credentials, null, '\t'));
}
export async function getAllStatuses() {
var allStatuses = [];
await (async function getPage(max_id) {
var url = `https://pleroma.lamp.wtf/api/v1/accounts/${pleroma_user_id}/statuses?limit=40`;
if (max_id) url += `&max_id=${max_id}`;
var statuses = await fetch(url, {headers: {"Authorization": `Bearer ${access_token}`}}).then(res => res.json());
if (!statuses.length) return;
allStatuses.push(...statuses);
var last_id = statuses.at(-1).id;
await getPage(last_id);
})();
return allStatuses;
}
async function uploadFile({data, name}) {
var form = new FormData();
form.append("file", data, name);
var res = await fetch("https://pleroma.lamp.wtf/api/v1/media", {
method: "POST",
body: form,
headers: {"Authorization": `Bearer ${access_token}`}
});
var json = await res.json();
console.log("uploaded file", res.status, json.url);
return json;
}
async function postStatus({status, visibility = "unlisted", content_type = "text/plain", media_ids = [], sensitive, files, edit}) {
if (files) {
media_ids = (await Promise.all(files.map(file => uploadFile(file)))).map(d => d.id);
}
var form = new FormData();
form.append("status", status);
form.append("visibility", visibility);
form.append("source", "bot");
form.append("content_type", content_type);
for (let media_id of media_ids) {
form.append("media_ids[]", media_id);
}
if (sensitive) form.append("sensitive", "true");
var res = await fetch("https://pleroma.lamp.wtf/api/v1/statuses" + (edit ? `/${edit}` : ''), {
method: edit ? "PUT" : "POST",
body: form,
headers: {"Authorization": `Bearer ${access_token}`}
});
var json = await res.json();
console.log(edit ? "edited" : "posted", res.status, json.uri || json);
return json;
}
async function downloadPixivIllust(illust_id) {
var url = `https://www.pixiv.net/en/artworks/${illust_id}`;
var res = await fetch(url, {headers: {"Cookie": pixiv_cookie}}, true);
if (!res.ok) {
console.error(res.status);
var res = await fetch(url);
}
var html = await res.text();
var illust = Object.values(JSON.parse(html.match(/<meta name="preload-data" id="meta-preload-data" content='(.*)'>/)[1]).illust)[0];
try {
let res = await fetch(`https://www.pixiv.net/ajax/illust/${illust_id}/pages`, {headers: {"Cookie": pixiv_cookie}});
let json = await res.json();
var images = json.body.map(x => ({
url: x.urls.original,
width: x.width,
height: x.height
}));
} catch (error) {
console.error(error.stack);
if (!illust.urls.original) {
console.error("missing original urls", illust.urls);
throw error;
}
var images = [];
for (let i = 0; i < illust.pageCount; i++) {
images.push({
url: illust.urls.original.replace('p0', 'p'+i)
});
}
}
for (let image of images) {
image.name = image.url.split('/').pop();
image.data = await fetch(image.url, {headers: {"Referer": "https://www.pixiv.net/"}}).then(res => res.blob());
};
return {illust, images};
}
export async function pixivToPleroma(illust_id, status_id) {
try {
var {illust, images} = await downloadPixivIllust(illust_id);
var {url} = await postStatus({
status: `https://www.pixiv.net/en/artworks/${illust_id}`
+ ((images.length > 4) ? `\n⚠ There are ${images.length} attachments` : '')
+ ((illust.illustType == 2) ? '\n⚠ ugoira conversion not implemented, view on Pixiv' : ''),
files: images,
visibility: "unlisted",
sensitive: Boolean(illust.xRestrict),
edit: status_id
});
Deno.writeTextFileSync("known_ids.csv", `${illust_id},${url}\n`, {append: true});
var {postPixivIllustToMatrix} = await import("./matrix.js"); //circular dependency -.-
await postPixivIllustToMatrix(illust, images, url);
} catch (error) {
console.error(error.stack);
postStatus({
status: `https://www.pixiv.net/en/artworks/${illust_id}\n#error:\n${error.stack}`,
edit: status_id
});
}
}

38
missing.js
View File

@ -1,38 +0,0 @@
import { fetch, credentials, pixivToPleroma, getAllStatuses } from "./common.js";
var {pixiv_cookie, pixiv_user_id} = credentials;
console.log("get all pixiv bookmarks");
var all_bookmark_ids = [];
for (var offset = 0;;) {
let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
let works = data.body.works;
if (!works.length) break;
offset += works.length;
let ids = works.filter(w => !w.isMasked).map(x => x.id.toString());
all_bookmark_ids.push(...ids);
}
console.log(`${all_bookmark_ids.length} ummasked bookmarks`);
console.log("get all pleroma statuses");
var statuses = await getAllStatuses();
console.log(`${statuses.length} statuses`);
statuses = statuses.map(x => x.content);
var missing_ids = [];
for (let id of all_bookmark_ids.reverse()) {
if (statuses.some(status => status.includes(id) && !status.includes("#error"))) continue;
missing_ids.push(id);
}
console.log("missing_ids", missing_ids);
confirm();
for (let id of missing_ids) {
await pixivToPleroma(id);
}

13
retry-errored.js
View File

@ -1,13 +0,0 @@
import { getAllStatuses, pixivToPleroma } from "./common.js";
var statuses = await getAllStatuses();
statuses = statuses.filter(status => status.content.includes("#error"));
statuses = statuses.map(status => {
var pixiv_id = status.content.match(/https:\/\/www.pixiv.net\/en\/artworks\/(\d+)/)[1];
return [status.id, pixiv_id];
});
console.log(statuses.map(([s,p]) => `https://pleroma.lamp.wtf/notice/${s} => https://www.pixiv.net/en/artworks/${p}`).join("\n"));
for (let [status_id, pixiv_id] of statuses) {
await pixivToPleroma(pixiv_id, status_id);
}

51
server.js
View File

@ -1,51 +0,0 @@
import { pixivToPleroma } from "./common.js";
Deno.serve({
port: 18024,
hostname: "127.0.0.1"
}, async (req, info) => {
var {pathname} = new URL(req.url);
console.log(info.remoteAddr.hostname, req.headers.get('x-forwarded-for'), pathname);
try {
switch (pathname) {
case "/ajax/illusts/bookmarks/add":
var payload = await req.clone().json();
console.log(payload);
var {comment, illust_id, restrict, tags} = payload;
pixivToPleroma(illust_id);
break;
case "/ajax/illusts/bookmarks/delete":
var payload = await req.clone().formData();
console.log(payload);
// todo
break;
case "/touch/ajax_api/ajax_api.php":
var payload = await req.clone().formData();
console.log(payload);
var mode = payload.get("mode");
if (mode == "add_bookmark_illust") {
//var restrict = payload.get("restrict");
//var tag = payload.get("tag");
var id = payload.get("id");
//var comment = payload.get("comment");
pixivToPleroma(id);
} else if (mode == "delete_bookmark_illust") {
var id = payload.get("id");
// todo
}
break;
//todo mobile app
}
} catch (error) {
console.error("payload error:", error.stack);
console.debug(req.headers.get("content-type"));
}
return fetch("https://www.pixiv.net"+pathname, {
method: req.method,
headers: req.headers,
body: req.body,
redirect: "manual"
});
});