lamp/pixiv-bookmarks-to-pleroma
lamp/pixiv-bookmarks-to-pleroma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

merge into: lamp:to-matrix
Branches Tags
lamp:master
lamp:to-matrix
...
pull from: lamp:master
Branches Tags
lamp:master
lamp:to-matrix

No commits in common. "to-matrix" and "master" have entirely different histories.
to-matrix ... master

7 changed files with 268 additions and 52 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
credentials.json
known_ids.csv

4
.vscode/settings.json vendored Executable file
View File

@ -0,0 +1,4 @@
{
"deno.enable": true,
"deno.lint": false
}

28
catchup.js Normal file
View File

@ -0,0 +1,28 @@
import { credentials, pixivToPleroma, known_ids } from "./common.js";
var {pixiv_cookie, pixiv_user_id} = credentials;
var new_ids = [];
top: for (var offset = 0;;) {
let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
console.log("get", url);
let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
let ids = data.body.works.map(x => x.id);
if (!ids.length) break;
let this_page_has_known_ids = false;
for (let id of ids) {
if (id == Deno.args[0]) break top; //init empty db
if (id in known_ids) {
this_page_has_known_ids = true;
} else {
new_ids.push(id);
}
}
if (this_page_has_known_ids) break;
offset += ids.length;
}
console.log("new ids", new_ids);
for (let id of new_ids.reverse()) {
await pixivToPleroma(id);
}

138
common.js Normal file
View File

@ -0,0 +1,138 @@
export var credentials = JSON.parse(Deno.readTextFileSync("credentials.json"));
var {client_id, client_secret, username, password, pixiv_cookie, access_token, pleroma_user_id} = credentials;
export var known_ids = {};
try {
known_ids = Object.fromEntries(Deno.readTextFileSync("known_ids.csv").trim().split("\n").map(line => line.split(",")));
} catch (e) {}
if (!access_token) {
let form = new FormData();
form.append("client_id", client_id);
form.append("client_secret", client_secret);
form.append("grant_type", "password");
form.append("username", username);
form.append("password", password);
let data = await fetch("https://pleroma.lamp.wtf/oauth/token", {
method: "POST",
body: form
}).then(res => res.json());
console.log ("logged in", data);
credentials.access_token = access_token = data.access_token;
Deno.writeTextFileSync("credentials.json", JSON.stringify(credentials, null, '\t'));
}
export async function getAllStatuses() {
var allStatuses = [];
await (async function getPage(max_id) {
var url = `https://pleroma.lamp.wtf/api/v1/accounts/${pleroma_user_id}/statuses?limit=40`;
if (max_id) url += `&max_id=${max_id}`;
console.log("get", url);
var statuses = await fetch(url, {headers: {"Authorization": `Bearer ${access_token}`}}).then(res => res.json());
if (!statuses.length) return;
allStatuses.push(...statuses);
var last_id = statuses.at(-1).id;
await getPage(last_id);
})();
return allStatuses;
}
async function uploadFile({data, name}) {
var form = new FormData();
form.append("file", data, name);
var res = await fetch("https://pleroma.lamp.wtf/api/v1/media", {
method: "POST",
body: form,
headers: {"Authorization": `Bearer ${access_token}`}
});
if (!res.ok) throw new Error("HTTP " + res.status);
var json = await res.json();
console.log("uploaded file", res.status, json.url);
return json;
}
async function postStatus({status, visibility = "unlisted", content_type = "text/plain", media_ids = [], sensitive, files}) {
if (files) {
media_ids = (await Promise.all(files.map(file => uploadFile(file)))).map(d => d.id);
}
var form = new FormData();
form.append("status", status);
form.append("visibility", visibility);
form.append("source", "bot");
form.append("content_type", content_type);
for (let media_id of media_ids) {
form.append("media_ids[]", media_id);
}
if (sensitive) form.append("sensitive", "true");
var res = await fetch("https://pleroma.lamp.wtf/api/v1/statuses", {
method: "POST",
body: form,
headers: {"Authorization": `Bearer ${access_token}`}
});
if (!res.ok) throw new Error("HTTP " + res.status);
var json = await res.json();
console.log("posted", res.status, json.uri || json);
return json;
}
async function downloadPixivIllust(illust_id) {
var url = `https://www.pixiv.net/en/artworks/${illust_id}`;
console.log("fetch", url);
var res = await fetch(url, {headers: {"Cookie": pixiv_cookie}});
if (!res.ok) throw new Error("HTTP " + res.status);
var html = await res.text();
var illust = Object.values(JSON.parse(html.match(/<meta name="preload-data" id="meta-preload-data" content='(.*)'>/)[1]).illust)[0];
try {
let u = `https://www.pixiv.net/ajax/illust/${illust_id}/pages`;
console.log("fetch", u);
let res = await fetch(u, {headers: {"Cookie": pixiv_cookie}});
if (!res.ok) throw new Error("HTTP " + res.status);
let json = await res.json();
var images = json.body.map(x => ({
url: x.urls.original,
width: x.width,
height: x.height
}));
} catch (error) {
console.error(error.stack);
var images = [];
for (let i = 0; i < illust.pageCount; i++) {
images.push({
url: illust.urls.original.replace('p0', 'p'+i)
});
}
}
for (let image of images) {
image.name = image.url.split('/').pop();
console.log("fetch", image.url);
image.data = await fetch(image.url, {headers: {"Referer": "https://www.pixiv.net/"}}).then(res => res.blob());
};
return {illust, images};
}
export async function pixivToPleroma(illust_id) {
try {
var {illust, images} = await downloadPixivIllust(illust_id);
var {url} = await postStatus({
status: `https://www.pixiv.net/en/artworks/${illust_id}`
+ ((images.length > 4) ? `\n⚠ There are ${images.length} attachments` : '')
+ ((illust.illustType == 2) ? '\n⚠ ugoira conversion not implemented, view on Pixiv' : ''),
files: images,
visibility: "unlisted",
sensitive: Boolean(illust.xRestrict)
});
Deno.writeTextFileSync("known_ids.csv", `${illust_id},${url}\n`, {append: true});
var {postPixivIllustToMatrix} = await import("./matrix.js"); //circular dependency -.-
await postPixivIllustToMatrix(illust, images, url);
} catch (error) {
console.error(error.stack);
postStatus({
status: `https://www.pixiv.net/en/artworks/${illust_id}` +
`\n#error:\n${error.stack}`
});
}
}

59
app.deno.js → matrix.js
View File

@ -1,6 +1,5 @@
import ky from 'https://esm.sh/ky';
import credentials from "./credentials.json" with { type: "json" };
var {matrix_token, pixiv_cookie, pixiv_user_id} = credentials;
import { credentials } from "./common.js";
var ke = ky.create({
hooks: {
@ -14,7 +13,6 @@ var ke = ky.create({
timeout: false
});
var matrix = {
ky: ke.extend({headers: {"Authorization": `Bearer ${credentials.matrix_token}`}}),
rooms: {
@ -35,34 +33,12 @@ var matrix = {
var content_uri = await this.uploadFile(filename, data);
return await this.sendMessage({room, type: "m.image", body: body || filename, filename, url: content_uri, info});
}
}
};
async function downloadPixivIllust(illust_id) {
var html = await ke.get(`https://www.pixiv.net/en/artworks/${illust_id}`, {headers: {"Cookie": credentials.pixiv_cookie}}).text();
var illust = Object.values(JSON.parse(html.match(/<meta name="preload-data" id="meta-preload-data" content='(.*)'>/)[1]).illust)[0];
var images = [];
for (let i = 0; i < illust.pageCount; i++) {
let url = illust.urls.original.replace('p0', 'p'+i);
let data = await ke.get(url, {headers: {"Referer": "https://www.pixiv.net/"}}).then(res => res.blob());
images.push({url, name: url.split('/').pop(), data});
};
return {illust, images};
}
export async function pixivToMatrix(illust_id) {
export async function postPixivIllustToMatrix(illust, images, msg) {
try {
var {illust, images} = await downloadPixivIllust(illust_id);
var room = illust.xRestrict ? matrix.rooms.pixiv_nsfw : matrix.rooms.pixiv_sfw;
await matrix.sendMessage({room, body: `https://www.pixiv.net/en/artworks/${illust_id}`});
await matrix.sendMessage({room, body: msg || `https://www.pixiv.net/en/artworks/${illust.id}`});
for (var image of images) {
try {
await matrix.sendImage({
@ -71,7 +47,9 @@ export async function pixivToMatrix(illust_id) {
data: image.data,
info: {
mimetype: image.data.type,
size: image.data.size
size: image.data.size,
w: image.width,
h: image.height
}
});
} catch (error) {
@ -87,27 +65,4 @@ export async function pixivToMatrix(illust_id) {
console.error(error.stack);
});
}
}
var new_ids = [];
top: for (var offset = 0;;) {
let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
console.log("get", url);
let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
let ids = data.body.works.map(x => x.id);
if (!ids.length) break;
for (let id of ids) {
if (["119320087", "119534122"].includes(id)) break top;
new_ids.push(id);
}
offset += ids.length;
}
console.log("new ids", new_ids);
for (let id of new_ids.reverse()) {
await pixivToMatrix(id);
}

39
missing.js Normal file
View File

@ -0,0 +1,39 @@
import { credentials, pixivToPleroma, getAllStatuses } from "./common.js";
var {pixiv_cookie, pixiv_user_id} = credentials;
console.log("get all pixiv bookmarks");
var all_bookmark_ids = [];
for (var offset = 0;;) {
let url = `https://www.pixiv.net/ajax/user/${pixiv_user_id}/illusts/bookmarks?tag=&offset=${offset}&limit=100&rest=show&lang=en&version=5dc84ab282403a049abea4e2f2214b6a69d31da6`;
console.log("get", url);
let data = await fetch(url, {headers: {Cookie: pixiv_cookie}}).then(res => res.json());
let works = data.body.works;
if (!works.length) break;
offset += works.length;
let ids = works.filter(w => !w.isMasked).map(x => x.id.toString());
all_bookmark_ids.push(...ids);
}
console.log(`${all_bookmark_ids.length} ummasked bookmarks`);
console.log("get all pleroma statuses");
var statuses = await getAllStatuses();
console.log(`${statuses.length} statuses`);
statuses = statuses.map(x => x.content);
var missing_ids = [];
for (let id of all_bookmark_ids.reverse()) {
if (statuses.some(status => status.includes(id) && !status.includes("#error"))) continue;
missing_ids.push(id);
}
console.log("missing_ids", missing_ids);
confirm();
for (let id of missing_ids) {
await pixivToPleroma(id);
}

50
server.js Normal file
View File

@ -0,0 +1,50 @@
import { pixivToPleroma } from "./common.js";
Deno.serve({
port: 18024,
hostname: "127.0.0.1"
}, async (req, info) => {
var {pathname} = new URL(req.url);
console.log(info.remoteAddr.hostname, req.headers.get('x-forwarded-for'), pathname);
try {
switch (pathname) {
case "/ajax/illusts/bookmarks/add":
var payload = await req.clone().json();
console.log(payload);
var {comment, illust_id, restrict, tags} = payload;
pixivToPleroma(illust_id);
break;
case "/ajax/illusts/bookmarks/delete":
var payload = await req.clone().formData();
console.log(payload);
// todo
break;
case "/touch/ajax_api/ajax_api.php":
var payload = await req.clone().formData();
console.log(payload);
var mode = payload.get("mode");
if (mode == "add_bookmark_illust") {
//var restrict = payload.get("restrict");
//var tag = payload.get("tag");
var id = payload.get("id");
//var comment = payload.get("comment");
pixivToPleroma(id);
} else if (mode == "delete_bookmark_illust") {
var id = payload.get("id");
// todo
}
break;
//todo mobile app
}
} catch (error) {
console.error("payload error:", error.stack);
}
return fetch("https://www.pixiv.net"+pathname, {
method: req.method,
headers: req.headers,
body: req.body,
redirect: "manual"
});
});