read/write CAR files

This commit is contained in:
Daniel Holmgren 2021-12-01 21:09:47 -06:00
parent f255178936
commit 03ed5c886f
6 changed files with 86 additions and 1 deletions

@ -1,4 +1,7 @@
import { CarWriter } from '@ipld/car'
import { BlockWriter } from '@ipld/car/lib/writer-browser'
import { CID } from 'multiformats/cid'
import { flattenUint8Arrays } from './util'
export default class MemoryDB {
@ -15,4 +18,26 @@ export default class MemoryDB {
async put(k: CID, v: Uint8Array) {
this.map.set(k.toString(), v)
}
getCarStream(roots: CID): AsyncIterable<Uint8Array> {
const writeDB = async (car: BlockWriter) => {
for await (const [cid, bytes] of this.map.entries()) {
car.put({ cid: CID.parse(cid), bytes })
}
car.close()
}
const { writer, out } = CarWriter.create(roots)
writeDB(writer)
return out
}
async getCarFile(roots: CID): Promise<Uint8Array> {
const arrays = []
for await (const chunk of this.getCarStream(roots)) {
arrays.push(chunk)
}
return flattenUint8Arrays(arrays)
}
}

@ -7,6 +7,7 @@ import * as blockCodec from '@ipld/dag-cbor'
import * as hashmap from 'ipld-hashmap'
import { User, Post } from "./types"
import { CarWriter } from '@ipld/car'
export default class UserStore {
@ -71,4 +72,13 @@ export default class UserStore {
this.posts = posts
return posts
}
getCarStream(): AsyncIterable<Uint8Array> {
return this.db.getCarStream(this.root)
}
async getCarFile(): Promise<Uint8Array> {
return this.db.getCarFile(this.root)
}
}

12
common/util.ts Normal file

@ -0,0 +1,12 @@
export const flattenUint8Arrays = (arrs: Uint8Array[]): Uint8Array => {
const length = arrs.reduce((acc, cur) => {
return acc + cur.length
}, 0)
const flattened = new Uint8Array(length)
let offset = 0
arrs.forEach((arr) => {
flattened.set(arr, offset)
offset += arr.length
})
return flattened
}

@ -5,10 +5,12 @@
"dev": "next -p 3005",
"build": "next build",
"start": "next start",
"server": "ts-node --skip-project ts-server/index.ts"
"server": "nodemon --skip-project ts-server/index.ts"
},
"dependencies": {
"@ipld/car": "^3.2.0",
"@ipld/dag-cbor": "^6.0.12",
"axios": "^0.24.0",
"cors": "^2.8.5",
"express": "^4.17.1",
"ipld-hashmap": "^2.1.7",
@ -22,6 +24,7 @@
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",
"@types/react": "^17.0.20",
"nodemon": "^2.0.15",
"ts-node": "^10.4.0",
"typescript": "^4.4.2"
}

@ -2,6 +2,8 @@ import styles from "@components/App.module.scss";
import * as React from "react";
import axios from 'axios'
import App from "@components/App"
import UserStore from "@root/common/user-store";
import { Post } from "@root/common/types";
@ -13,6 +15,8 @@ function Home(props: {}) {
const addPost = async (post: Post) => {
await store.addPost(post)
setPosts([...posts, post])
const car = await store.getCarFile()
await axios.post('http://localhost:1337/update', car, { headers: { 'Content-Type': 'application/octet-stream' }})
}
const setupPostsMap = async () => {

@ -1,11 +1,42 @@
import express from 'express'
import { Request, Response } from 'express'
import cors from 'cors'
import { CarReader } from '@ipld/car'
const app = express()
app.use(express.json())
app.use(cors())
const readReqStream = (req: Request): Promise<Buffer> => {
return new Promise(resolve => {
const bufs = [] as Buffer[]
req.on('data', (chunk: Buffer) => {
bufs.push(chunk)
})
req.on('end', () => {
resolve(Buffer.concat(bufs))
})
})
}
app.post('/update', async (req: Request, res: Response) => {
const buf = await readReqStream(req)
const reader = await CarReader.fromBytes(buf)
console.log(reader)
const roots = await reader.getRoots()
console.log(roots)
// req.on('data', (chunk) => {
// console.log(chunk)
// console.log(chunk.toString())
// })
// req.on('end', () => {
// console.log("DONE")
// res.status(200).send()
// })
})
const runServer = (port = 1337) => {
app.listen(port, () => {