lamp/bskybackup
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize sqlite insert

Browse Source
This commit is contained in:
Lamp 2025-05-12 02:48:16 -07:00
parent 5b060c6a23
commit 245e2bfe22
1 changed files with 6 additions and 7 deletions

13
index.js

@ -16,9 +16,9 @@ var db = await open({
filename: "repos.sqlite",
driver: sqlite3.Database
});
db.getDatabaseInstance().serialize(); // :/
await db.exec(`CREATE TABLE IF NOT EXISTS records (did TEXT, collection TEXT, rkey TEXT, cid TEXT, record BLOB, deleted BOOLEAN, insertedAt INTEGER, updatedAt INTEGER);`);
var insert_statement = await db.prepare(`INSERT INTO records VALUES (?,?,?,?,?,?,?,?);`);
async function fetch(url, options) {
console.debug("get", url);
@ -48,6 +48,7 @@ async function processRepo(did, repo) {
still_exists: null
};
}
await db.exec(`BEGIN TRANSACTION`);
for (let entry of iterateAtpRepo(repo)) {
recordCounts[entry.collection] ||= {now: 0, new: 0};
recordCounts[entry.collection].now++;
@ -78,9 +79,7 @@ async function processRepo(did, repo) {
continue;
}
recordCounts[entry.collection].new++;
await db.run(`INSERT INTO records VALUES (?,?,?,?,?,?,?,?);`, [
did, entry.collection, entry.rkey, entry.cid.$link, entry.bytes, false, Date.now(), null
]);
await insert_statement.run([did, entry.collection, entry.rkey, entry.cid.$link, entry.bytes, false, Date.now(), null]);
} catch (error) {
console.error("record error", error.stack)
}
@ -95,6 +94,7 @@ async function processRepo(did, repo) {
]);
}
}
await db.exec(`COMMIT`);
console.log(did, `${rdids.size} related dids`);
rdids.delete(did);
return rdids;
@ -171,5 +171,4 @@ for (let arg of process.argv) {
}
}
await backupRecursive(didsToBackup, depth);
await db.close();
await backupRecursive(didsToBackup, depth);