Update readme.md

bskybackup.deno.js

@@ -1,3 +1,6 @@
+try { Deno.mkdirSync("blobs") } catch (error) {}
+try { Deno.mkdirSync("repos") } catch (error) {}
+
 async function get(url, options) {
 	console.debug("get", url);
 	var res = await fetch(url, options);
@@ -83,12 +86,16 @@ async function backupRecursive(dids, depth = 1) {
 	}
 	var allrdids = new Set();
 	for (var did of dids) {
-		var repo = await backup(did);
+		try {
-		repo = ascii.decode(repo);
+			var repo = await backup(did);
-		var rdids = new Set(repo.match(/did:plc:[a-z2-7]{24}/g));
+			repo = ascii.decode(repo);
-		rdids.delete(did);
+			var rdids = new Set(repo.match(/did:plc:[a-z2-7]{24}/g));
-		console.log(`${rdids.size} related didplcs`);
+			rdids.delete(did);
-		rdids.forEach(rdid => allrdids.add(rdid));
+			console.log(`${rdids.size} related didplcs`);
+			rdids.forEach(rdid => allrdids.add(rdid));
+		} catch (error) {
+			console.error(error);
+		}
 	}
 	await backupRecursive(allrdids, depth - 1);
 }

common.mjs

@@ -1,15 +0,0 @@
-import fetchRetry from "fetch-retry";
-
-var _fetch = fetchRetry(global.fetch);
-export async function fetch(url, options) {
-	console.log("fetch", url);
-	var res = await _fetch(url, options);
-	if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText} ${await res.text()}`);
-	return res;
-}
-
-export async function getPds(did) {
-	var doc = await fetch(`https://plc.directory/${did}`).then(res => res.json());
-	var {serviceEndpoint} = doc.service.find(s => s.id == "#atproto_pds");
-	return serviceEndpoint;
-}

getblobs.mjs

@@ -1,36 +0,0 @@
-import * as fs from "fs";
-import {fetch, getPds} from "./common.mjs";
-
-let dids = fs.readdirSync("repos");
-dids = dids.map(did => did.replaceAll('-',':'));
-
-for (let did of dids) {
-	console.log(did);
-
-	let allCids = [];
-
-	let pds = await getPds(did);
-	let cursor;
-	do {
-		try {
-			let {cids, cursor: _cursor} = await fetch(`${pds}/xrpc/com.atproto.sync.listBlobs?did=${did}&limit=1000${cursor?`&cursor=${cursor}`:''}`).then(res => res.json());
-			cursor = _cursor;
-			allCids.push(...cids);
-		} catch (error) {
-			console.error(error);
-		}
-	} while (cursor);
-	console.log(`${allCids.length} files`);
-
-	for (let cid of allCids) {
-		if (fs.existsSync(`blobs/${cid}`)) continue;
-		try {
-			let res = await fetch(`${pds}/xrpc/com.atproto.sync.getBlob?did=${did}&cid=${cid}`);
-			let ab = await res.arrayBuffer();
-			let dv = new DataView(ab);
-			fs.writeFileSync(`blobs/${cid}`, dv);
-		} catch (error) {
-			console.error(error);
-		}
-	}
-}

getrepos.mjs

@@ -1,29 +0,0 @@
-import * as fs from "fs";
-import {fetch, getPds} from "./common.mjs";
-
-async function getRepo(did) {
-	console.log("get", did);
-	var fp = `repos/${did.replaceAll(':','-')}`;
-	//try {
-	//	return fs.readFileSync(fp);
-	//} catch(e) {}
-	var pds = await getPds(did);
-	var res = await fetch(`${pds}/xrpc/com.atproto.sync.getRepo?did=${did}`);
-	var repo = Buffer.from(await res.arrayBuffer());
-	fs.writeFileSync(fp, repo);
-	return repo;
-}
-
-
-var repo = await getRepo("did:plc:u4gngygg2w5egsigxu5g7byu");
-repo = repo.toString("ascii");
-var dids = new Set(repo.match(/did:plc:[a-z2-7]{24}/g));
-
-console.log(`${dids.size} related dids`);
-for (var did of dids) {
-	try {
-		await getRepo(did);
-	} catch (error) {
-		console.error(error);
-	}
-}

package-lock.json

@@ -1,17 +0,0 @@
-{
-  "name": "bskybackup",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "dependencies": {
-        "fetch-retry": "^6.0.0"
-      }
-    },
-    "node_modules/fetch-retry": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-6.0.0.tgz",
-      "integrity": "sha512-BUFj1aMubgib37I3v4q78fYo63Po7t4HUPTpQ6/QE6yK6cIQrP+W43FYToeTEyg5m2Y7eFUtijUuAv/PDlWuag=="
-    }
-  }
-}

package.json

@@ -1,5 +0,0 @@
-{
-  "dependencies": {
-    "fetch-retry": "^6.0.0"
-  }
-}

readme.md

@@ -20,4 +20,33 @@ $ deno run --allow-all bskybackup.deno.js did:plc:u4gngygg2w5egsigxu5g7byu --dep
 
 This will additionally save all profiles related to the profiles related to them, so you will have a copy of everything their friends liked and reposted as well. This may be a LOT of data!
 
-Running the script again will refresh the repositories and download new blobs. Existing blobs are not re-downloaded.
+Running the script again will refresh the repositories and download new blobs. Existing blobs are not re-downloaded. Run the script regularly to keep the backups up to date.
+
+## Example SystemD
+
+systemd timer is better than cron because it won't start it again if it's still running.
+
+/etc/systemd/system/bskybackup.service
+```systemd
+[Unit]
+Description=Bluesky Backup script
+After=network-online.target
+
+[Service]
+WorkingDirectory=/zpool1/FileStorage/srv/bskybackup/
+User=bskybackup
+ExecStart=deno run --allow-all bskybackup.deno.js did:plc:u4gngygg2w5egsigxu5g7byu
+Type=exec
+```
+
+/etc/systemd/system/bskybackup.timer
+```systemd
+[Unit]
+Description=Daily Bluesky backup
+
+[Timer]
+OnCalendar=05:00:00
+
+[Install]
+WantedBy=timers.target
+```