bskybackup.deno.js

@@ -1,6 +1,3 @@
-try { Deno.mkdirSync("blobs") } catch (error) {}
-try { Deno.mkdirSync("repos") } catch (error) {}
-
 async function get(url, options) {
 	console.debug("get", url);
 	var res = await fetch(url, options);

common.mjs

@@ -0,0 +1,15 @@
+import fetchRetry from "fetch-retry";
+
+var _fetch = fetchRetry(global.fetch);
+export async function fetch(url, options) {
+	console.log("fetch", url);
+	var res = await _fetch(url, options);
+	if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText} ${await res.text()}`);
+	return res;
+}
+
+export async function getPds(did) {
+	var doc = await fetch(`https://plc.directory/${did}`).then(res => res.json());
+	var {serviceEndpoint} = doc.service.find(s => s.id == "#atproto_pds");
+	return serviceEndpoint;
+}

getblobs.mjs

@@ -0,0 +1,36 @@
+import * as fs from "fs";
+import {fetch, getPds} from "./common.mjs";
+
+let dids = fs.readdirSync("repos");
+dids = dids.map(did => did.replaceAll('-',':'));
+
+for (let did of dids) {
+	console.log(did);
+
+	let allCids = [];
+
+	let pds = await getPds(did);
+	let cursor;
+	do {
+		try {
+			let {cids, cursor: _cursor} = await fetch(`${pds}/xrpc/com.atproto.sync.listBlobs?did=${did}&limit=1000${cursor?`&cursor=${cursor}`:''}`).then(res => res.json());
+			cursor = _cursor;
+			allCids.push(...cids);
+		} catch (error) {
+			console.error(error);
+		}
+	} while (cursor);
+	console.log(`${allCids.length} files`);
+
+	for (let cid of allCids) {
+		if (fs.existsSync(`blobs/${cid}`)) continue;
+		try {
+			let res = await fetch(`${pds}/xrpc/com.atproto.sync.getBlob?did=${did}&cid=${cid}`);
+			let ab = await res.arrayBuffer();
+			let dv = new DataView(ab);
+			fs.writeFileSync(`blobs/${cid}`, dv);
+		} catch (error) {
+			console.error(error);
+		}
+	}
+}

getrepos.mjs

@@ -0,0 +1,29 @@
+import * as fs from "fs";
+import {fetch, getPds} from "./common.mjs";
+
+async function getRepo(did) {
+	console.log("get", did);
+	var fp = `repos/${did.replaceAll(':','-')}`;
+	//try {
+	//	return fs.readFileSync(fp);
+	//} catch(e) {}
+	var pds = await getPds(did);
+	var res = await fetch(`${pds}/xrpc/com.atproto.sync.getRepo?did=${did}`);
+	var repo = Buffer.from(await res.arrayBuffer());
+	fs.writeFileSync(fp, repo);
+	return repo;
+}
+
+
+var repo = await getRepo("did:plc:u4gngygg2w5egsigxu5g7byu");
+repo = repo.toString("ascii");
+var dids = new Set(repo.match(/did:plc:[a-z2-7]{24}/g));
+
+console.log(`${dids.size} related dids`);
+for (var did of dids) {
+	try {
+		await getRepo(did);
+	} catch (error) {
+		console.error(error);
+	}
+}

package-lock.json

@@ -0,0 +1,17 @@
+{
+  "name": "bskybackup",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "dependencies": {
+        "fetch-retry": "^6.0.0"
+      }
+    },
+    "node_modules/fetch-retry": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-6.0.0.tgz",
+      "integrity": "sha512-BUFj1aMubgib37I3v4q78fYo63Po7t4HUPTpQ6/QE6yK6cIQrP+W43FYToeTEyg5m2Y7eFUtijUuAv/PDlWuag=="
+    }
+  }
+}

package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "fetch-retry": "^6.0.0"
+  }
+}