redoing a bunch of stuff, all in memory
This commit is contained in:
dholms
parent
197b75f116
commit
82e9334294
packages/common
@ -1,434 +0,0 @@
|
||||
import { CID } from 'multiformats'
|
||||
import * as uint8arrays from 'uint8arrays'
|
||||
import IpldStore from '../blockstore/ipld-store'
|
||||
import { sha256 } from '@adxp/crypto'
|
||||
|
||||
import z from 'zod'
|
||||
import { schema } from '../common/types'
|
||||
import * as check from '../common/check'
|
||||
|
||||
const leafPointer = z.tuple([z.string(), schema.cid])
|
||||
const treePointer = schema.cid
|
||||
const treeEntry = z.union([leafPointer, treePointer])
|
||||
const nodeSchema = z.array(treeEntry)
|
||||
|
||||
type LeafPointer = z.infer<typeof leafPointer>
|
||||
type TreePointer = z.infer<typeof treePointer>
|
||||
type TreeEntry = z.infer<typeof treeEntry>
|
||||
type Node = z.infer<typeof nodeSchema>
|
||||
|
||||
export const leadingZerosOnHash = async (key: string): Promise<number> => {
|
||||
const hash = await sha256(key)
|
||||
const b32 = uint8arrays.toString(hash, 'base32')
|
||||
let count = 0
|
||||
for (const char of b32) {
|
||||
if (char === 'a') {
|
||||
// 'a' is 0 in b32
|
||||
count++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
const spliceIn = <T>(array: T[], item: T, index: number): T[] => {
|
||||
return [...array.slice(0, index), item, ...array.slice(index)]
|
||||
}
|
||||
|
||||
export class MST {
|
||||
blockstore: IpldStore
|
||||
cid: CID
|
||||
node: Node
|
||||
zeros: number
|
||||
|
||||
constructor(blockstore: IpldStore, cid: CID, node: Node, zeros: number) {
|
||||
this.blockstore = blockstore
|
||||
this.cid = cid
|
||||
this.node = node
|
||||
this.zeros = zeros
|
||||
}
|
||||
|
||||
static async create(blockstore: IpldStore, zeros = 0): Promise<MST> {
|
||||
return MST.fromData(blockstore, [], zeros)
|
||||
}
|
||||
|
||||
static async fromData(
|
||||
blockstore: IpldStore,
|
||||
node: Node,
|
||||
zeros: number,
|
||||
): Promise<MST> {
|
||||
const cid = await blockstore.put(node as any)
|
||||
return new MST(blockstore, cid, node, zeros)
|
||||
}
|
||||
|
||||
static async load(
|
||||
blockstore: IpldStore,
|
||||
cid: CID,
|
||||
zeros?: number,
|
||||
): Promise<MST> {
|
||||
const node = await blockstore.get(cid, nodeSchema)
|
||||
if (zeros === undefined) {
|
||||
const firstLeaf = node.find((entry) => check.is(entry, leafPointer))
|
||||
if (!firstLeaf) {
|
||||
throw new Error('not a valid mst node: no leaves')
|
||||
}
|
||||
zeros = await leadingZerosOnHash(firstLeaf[0])
|
||||
}
|
||||
return new MST(blockstore, cid, node, zeros)
|
||||
}
|
||||
|
||||
async put(): Promise<CID> {
|
||||
this.cid = await this.blockstore.put(this.node as any) // @TODO no any
|
||||
return this.cid
|
||||
}
|
||||
|
||||
async add(key: string, value: CID): Promise<CID> {
|
||||
const keyZeros = await leadingZerosOnHash(key)
|
||||
if (keyZeros === this.zeros) {
|
||||
// it belongs in this layer
|
||||
const index = this.findGtOrEqualLeafIndex(key)
|
||||
const found = this.node[index]
|
||||
if (found && found[0] === key) {
|
||||
throw new Error(`There is already a value at key: ${key}`)
|
||||
}
|
||||
const prevNode = this.node[index - 1]
|
||||
if (!prevNode || check.is(prevNode, leafPointer)) {
|
||||
// if entry before is a leaf, (or we're on far left) we can just splice in
|
||||
this.node = spliceIn(this.node, [key, value], index)
|
||||
return this.put()
|
||||
} else {
|
||||
// else we need to investigate the subtree
|
||||
const subTree = await MST.load(
|
||||
this.blockstore,
|
||||
prevNode,
|
||||
this.zeros - 1,
|
||||
)
|
||||
// we try to split the subtree around the key
|
||||
const splitSubTree = await subTree.splitAround(key)
|
||||
const newNode = this.node.slice(0, index - 1)
|
||||
if (splitSubTree[0]) newNode.push(splitSubTree[0])
|
||||
newNode.push([key, value])
|
||||
if (splitSubTree[1]) newNode.push(splitSubTree[1])
|
||||
newNode.push(...this.node.slice(index))
|
||||
this.node = newNode
|
||||
return this.put()
|
||||
}
|
||||
} else if (keyZeros < this.zeros) {
|
||||
// it belongs on a lower layer
|
||||
const index = this.findGtOrEqualLeafIndex(key)
|
||||
const prevNode = this.node[index - 1]
|
||||
if (check.is(prevNode, treePointer)) {
|
||||
// if entry before is a tree, we add it to that tree
|
||||
const subTree = await MST.load(
|
||||
this.blockstore,
|
||||
prevNode,
|
||||
this.zeros - 1,
|
||||
)
|
||||
const newSubTreeCid = await subTree.add(key, value)
|
||||
this.node[index - 1] = newSubTreeCid
|
||||
return this.put()
|
||||
} else {
|
||||
// else we need to create the subtree for it to go in
|
||||
const subTree = await MST.create(this.blockstore, this.zeros - 1)
|
||||
const newSubTreeCid = await subTree.add(key, value)
|
||||
this.node = spliceIn(this.node, newSubTreeCid, index)
|
||||
return this.put()
|
||||
}
|
||||
} else {
|
||||
// it belongs on a higher layer & we must push the rest of the tree down
|
||||
let split = await this.splitAround(key)
|
||||
// if the newly added key has >=2 more leading zeros than the current highest layer
|
||||
// then we need to add in structural nodes in between as well
|
||||
let left: CID | null = split[0]
|
||||
let right: CID | null = split[1]
|
||||
const extraLayersToAdd = keyZeros - this.zeros
|
||||
// intentionally starting at 1, since first layer is taken care of by split
|
||||
for (let i = 1; i < extraLayersToAdd; i++) {
|
||||
if (left !== null) {
|
||||
const leftNode = await MST.fromData(
|
||||
this.blockstore,
|
||||
[left],
|
||||
this.zeros + i,
|
||||
)
|
||||
left = leftNode.cid
|
||||
}
|
||||
if (right !== null) {
|
||||
const rightNode = await MST.fromData(
|
||||
this.blockstore,
|
||||
[right],
|
||||
this.zeros + i,
|
||||
)
|
||||
right = rightNode.cid
|
||||
}
|
||||
}
|
||||
let newNode: Node = []
|
||||
if (left) newNode.push(left)
|
||||
newNode.push([key, value])
|
||||
if (right) newNode.push(right)
|
||||
this.node = newNode
|
||||
this.zeros = keyZeros
|
||||
return this.put()
|
||||
}
|
||||
}
|
||||
|
||||
// finds first leaf node that is greater than or equal to the value
|
||||
findGtOrEqualLeafIndex(key: string): number {
|
||||
const maybeIndex = this.node.findIndex(
|
||||
(entry) => check.is(entry, leafPointer) && entry[0] >= key,
|
||||
)
|
||||
// if we can't find, we're on the end
|
||||
return maybeIndex >= 0 ? maybeIndex : this.node.length
|
||||
}
|
||||
|
||||
async splitAround(key: string): Promise<[CID | null, CID | null]> {
|
||||
const index = this.findGtOrEqualLeafIndex(key)
|
||||
const leftData = this.node.slice(0, index)
|
||||
const rightData = this.node.slice(index)
|
||||
|
||||
if (leftData.length === 0) {
|
||||
return [null, this.cid]
|
||||
}
|
||||
if (rightData.length === 0) {
|
||||
return [this.cid, null]
|
||||
}
|
||||
const left = await MST.fromData(this.blockstore, leftData, this.zeros)
|
||||
const right = await MST.fromData(this.blockstore, rightData, this.zeros)
|
||||
const prev = leftData[leftData.length - 1]
|
||||
if (check.is(prev, treePointer)) {
|
||||
const prevSubtree = await MST.load(this.blockstore, prev, this.zeros - 1)
|
||||
const prevSplit = await prevSubtree.splitAround(key)
|
||||
if (prevSplit[0]) {
|
||||
await left.append(prev)
|
||||
}
|
||||
if (prevSplit[1]) {
|
||||
await right.prepend(prev)
|
||||
}
|
||||
}
|
||||
|
||||
return [left.cid, right.cid]
|
||||
}
|
||||
|
||||
async append(entry: TreeEntry): Promise<CID> {
|
||||
this.node = [...this.node, entry]
|
||||
return this.put()
|
||||
}
|
||||
|
||||
async prepend(entry: TreeEntry): Promise<CID> {
|
||||
this.node = [entry, ...this.node]
|
||||
return this.put()
|
||||
}
|
||||
|
||||
async get(key: string): Promise<CID | null> {
|
||||
const index = this.findGtOrEqualLeafIndex(key)
|
||||
const found = this.node[index]
|
||||
if (found && check.is(found, leafPointer) && found[0] === key) {
|
||||
return found[1]
|
||||
}
|
||||
const prev = this.node[index - 1]
|
||||
if (check.is(prev, treePointer)) {
|
||||
const subTree = await MST.load(this.blockstore, prev, this.zeros - 1)
|
||||
return subTree.get(key)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
async edit(key: string, value: CID): Promise<CID> {
|
||||
const index = this.findGtOrEqualLeafIndex(key)
|
||||
const found = this.node[index]
|
||||
if (found && check.is(found, leafPointer) && found[0] === key) {
|
||||
this.node[index][1] = value
|
||||
return await this.put()
|
||||
}
|
||||
const prev = this.node[index - 1]
|
||||
if (check.is(prev, treePointer)) {
|
||||
const subTree = await MST.load(this.blockstore, prev, this.zeros - 1)
|
||||
const subTreeCid = await subTree.edit(key, value)
|
||||
this.node[index - 1] = subTreeCid
|
||||
return await this.put()
|
||||
}
|
||||
throw new Error(`Could not find a record with key: ${key}`)
|
||||
}
|
||||
|
||||
// async delete(key: string): Promise<void> {}
|
||||
|
||||
layerHasEntry(entry: TreeEntry): boolean {
|
||||
let found: TreeEntry | undefined
|
||||
if (check.is(entry, leafPointer)) {
|
||||
found = this.node.find((e) => {
|
||||
return (
|
||||
check.is(e, leafPointer) && entry[0] === e[0] && entry[1].equals(e[1])
|
||||
)
|
||||
})
|
||||
} else {
|
||||
found = this.node.find((e) => {
|
||||
return check.is(e, treePointer) && entry.equals(e)
|
||||
})
|
||||
}
|
||||
return found !== undefined
|
||||
}
|
||||
|
||||
async loadChild(cid: CID): Promise<MST> {
|
||||
return MST.load(this.blockstore, cid, this.zeros - 1)
|
||||
}
|
||||
|
||||
async mergeIn(toMerge: MST): Promise<CID> {
|
||||
let newNode: Node = []
|
||||
let thisI = 0,
|
||||
toMergeI = 0
|
||||
while (thisI < this.node.length && toMergeI < toMerge.node.length) {
|
||||
const thisHead = this.node[thisI]
|
||||
const toMergeHead = toMerge.node[toMergeI]
|
||||
if (!thisHead) {
|
||||
newNode.push(toMergeHead)
|
||||
toMergeI++
|
||||
} else if (!toMergeHead) {
|
||||
newNode.push(thisHead)
|
||||
thisI++
|
||||
} else if (
|
||||
check.is(thisHead, leafPointer) &&
|
||||
check.is(toMergeHead, leafPointer)
|
||||
) {
|
||||
if (thisHead[0] === toMergeHead[0]) {
|
||||
// on same, toMerge wins
|
||||
newNode.push(toMergeHead)
|
||||
thisI++
|
||||
toMergeI++
|
||||
} else if (thisHead[0] < toMergeHead[0]) {
|
||||
newNode.push(thisHead)
|
||||
thisI++
|
||||
} else {
|
||||
newNode.push(toMergeHead)
|
||||
toMergeI++
|
||||
}
|
||||
} else if (
|
||||
check.is(thisHead, treePointer) &&
|
||||
check.is(toMergeHead, leafPointer)
|
||||
) {
|
||||
const toSplit = await this.loadChild(thisHead)
|
||||
const split = await toSplit.splitAround(toMergeHead[0])
|
||||
if (split[0] !== null) {
|
||||
const prev = newNode[newNode.length - 1]
|
||||
if (check.is(prev, treePointer)) {
|
||||
const toMerge = await this.loadChild(split[0])
|
||||
const toMergeIn = await this.loadChild(prev)
|
||||
await toMerge.mergeIn(toMergeIn)
|
||||
newNode.push(toMerge.cid)
|
||||
} else {
|
||||
newNode.push(split[0])
|
||||
}
|
||||
}
|
||||
newNode.push(toMergeHead)
|
||||
if (split[1] !== null) newNode.push(split[1])
|
||||
thisI++
|
||||
toMergeI++
|
||||
} else if (
|
||||
check.is(thisHead, leafPointer) &&
|
||||
check.is(toMergeHead, treePointer)
|
||||
) {
|
||||
const toSplit = await this.loadChild(toMergeHead)
|
||||
const split = await toSplit.splitAround(thisHead[0])
|
||||
if (split[0] !== null) {
|
||||
const prev = newNode[newNode.length - 1]
|
||||
if (check.is(prev, treePointer)) {
|
||||
const toMerge = await this.loadChild(prev)
|
||||
const toMergeIn = await this.loadChild(split[0])
|
||||
await toMerge.mergeIn(toMergeIn)
|
||||
newNode.push(toMerge.cid)
|
||||
} else {
|
||||
newNode.push(split[0])
|
||||
}
|
||||
}
|
||||
newNode.push(toMergeHead)
|
||||
if (split[1] !== null) newNode.push(split[1])
|
||||
thisI++
|
||||
toMergeI++
|
||||
} else if (
|
||||
check.is(thisHead, treePointer) &&
|
||||
check.is(toMergeHead, treePointer)
|
||||
) {
|
||||
const toMerge = await this.loadChild(thisHead)
|
||||
const toMergeIn = await this.loadChild(toMergeHead)
|
||||
await toMerge.mergeIn(toMergeIn)
|
||||
newNode.push(toMerge.cid)
|
||||
thisI++
|
||||
toMergeI++
|
||||
} else {
|
||||
throw new Error('SHOULDNT ever reach this')
|
||||
}
|
||||
}
|
||||
return this.put()
|
||||
}
|
||||
|
||||
// toMerge wins on merge conflicts
|
||||
async mergeInOld(toMerge: MST): Promise<CID> {
|
||||
let lastIndex = 0
|
||||
for (const entry of toMerge.node) {
|
||||
if (check.is(entry, leafPointer)) {
|
||||
lastIndex = this.findGtOrEqualLeafIndex(entry[0])
|
||||
const found = this.node[lastIndex]
|
||||
if (found && found[0] === entry[0]) {
|
||||
// does nothing if same, overwrites if different
|
||||
this.node[lastIndex] = entry
|
||||
lastIndex++
|
||||
} else {
|
||||
this.node = spliceIn(this.node, entry, lastIndex)
|
||||
lastIndex++
|
||||
}
|
||||
} else {
|
||||
const nextEntryInNode = this.node[lastIndex]
|
||||
if (!check.is(nextEntryInNode, treePointer)) {
|
||||
// if the next is a leaf, we splice in before
|
||||
this.node = spliceIn(this.node, entry, lastIndex)
|
||||
lastIndex++
|
||||
} else if (!nextEntryInNode.equals(entry)) {
|
||||
// if it's a new subtree, then we have to merge the two children
|
||||
const nodeChild = await MST.load(
|
||||
this.blockstore,
|
||||
nextEntryInNode,
|
||||
this.zeros - 1,
|
||||
)
|
||||
const toMergeChild = await MST.load(
|
||||
this.blockstore,
|
||||
entry,
|
||||
this.zeros - 1,
|
||||
)
|
||||
const mergedCid = await nodeChild.mergeIn(toMergeChild)
|
||||
this.node[lastIndex] = mergedCid
|
||||
lastIndex++
|
||||
} else {
|
||||
// if it's the same subtree, do nothing & increment index
|
||||
lastIndex++
|
||||
}
|
||||
}
|
||||
}
|
||||
return this.put()
|
||||
}
|
||||
|
||||
async walk(fn: (level: number, key: string | null) => void) {
|
||||
for (const entry of this.node) {
|
||||
if (check.is(entry, treePointer)) {
|
||||
const subTree = await MST.load(this.blockstore, entry, this.zeros - 1)
|
||||
fn(this.zeros, null)
|
||||
await subTree.walk(fn)
|
||||
} else {
|
||||
fn(this.zeros, entry[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async structure() {
|
||||
const tree: any = []
|
||||
for (const entry of this.node) {
|
||||
if (check.is(entry, treePointer)) {
|
||||
const subTree = await MST.load(this.blockstore, entry, this.zeros - 1)
|
||||
tree.push(['LINK', await subTree.structure()])
|
||||
} else {
|
||||
tree.push([entry[0], entry[1].toString()])
|
||||
}
|
||||
}
|
||||
return tree
|
||||
}
|
||||
}
|
||||
|
||||
export default MST
|
||||
518
packages/common/src/repo/mst/mst.ts
Normal file
518
packages/common/src/repo/mst/mst.ts
Normal file
@ -0,0 +1,518 @@
|
||||
import * as Block from 'multiformats/block'
|
||||
import { sha256 as blockHasher } from 'multiformats/hashes/sha2'
|
||||
import * as blockCodec from '@ipld/dag-cbor'
|
||||
import { CID } from 'multiformats'
|
||||
import * as uint8arrays from 'uint8arrays'
|
||||
import IpldStore from '../../blockstore/ipld-store'
|
||||
import { sha256 } from '@adxp/crypto'
|
||||
|
||||
import z from 'zod'
|
||||
import { schema } from '../../common/types'
|
||||
import * as check from '../../common/check'
|
||||
|
||||
const leafPointer = z.tuple([z.string(), schema.cid])
|
||||
const treePointer = schema.cid
|
||||
const treeEntry = z.union([leafPointer, treePointer])
|
||||
const nodeDataSchema = z.array(treeEntry)
|
||||
|
||||
// type LeafPointer = z.infer<typeof leafPointer>
|
||||
// type TreePointer = z.infer<typeof treePointer>
|
||||
// type TreeEntry = z.infer<typeof treeEntry>
|
||||
type NodeData = z.infer<typeof nodeDataSchema>
|
||||
|
||||
export const leadingZerosOnHash = async (key: string): Promise<number> => {
|
||||
const hash = await sha256(key)
|
||||
const b32 = uint8arrays.toString(hash, 'base32')
|
||||
let count = 0
|
||||
for (const char of b32) {
|
||||
if (char === 'a') {
|
||||
// 'a' is 0 in b32
|
||||
count++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
class MST {
|
||||
blockstore: IpldStore
|
||||
entries: NodeEntry[] | null
|
||||
layer: number | null
|
||||
pointer: CID
|
||||
|
||||
constructor(
|
||||
blockstore: IpldStore,
|
||||
pointer: CID,
|
||||
entries: NodeEntry[] | null,
|
||||
layer: number | null,
|
||||
) {
|
||||
this.blockstore = blockstore
|
||||
this.entries = entries
|
||||
this.layer = layer
|
||||
this.pointer = pointer
|
||||
}
|
||||
|
||||
static async getCid(entries: NodeEntry[]): Promise<CID> {
|
||||
const data = entries.map((entry) => {
|
||||
if (entry.isLeaf()) {
|
||||
return [entry.key, entry.value]
|
||||
} else {
|
||||
return entry.pointer
|
||||
}
|
||||
})
|
||||
const block = await Block.encode({
|
||||
value: data as any,
|
||||
codec: blockCodec,
|
||||
hasher: blockHasher,
|
||||
})
|
||||
return block.cid
|
||||
}
|
||||
|
||||
static async create(
|
||||
blockstore: IpldStore,
|
||||
entries: NodeEntry[] = [],
|
||||
layer = 0,
|
||||
): Promise<MST> {
|
||||
const pointer = await MST.getCid(entries)
|
||||
return new MST(blockstore, pointer, entries, layer)
|
||||
}
|
||||
|
||||
static async fromData(
|
||||
blockstore: IpldStore,
|
||||
data: NodeData,
|
||||
layer?: number,
|
||||
): Promise<MST> {
|
||||
const entries = data.map((entry) => {
|
||||
if (check.is(entry, treePointer)) {
|
||||
return MST.fromCid(blockstore, entry, layer ? layer - 1 : undefined)
|
||||
} else {
|
||||
return new Leaf(entry[0], entry[1])
|
||||
}
|
||||
})
|
||||
const pointer = await MST.getCid(entries)
|
||||
return new MST(blockstore, pointer, entries, layer ?? null)
|
||||
}
|
||||
|
||||
static fromCid(blockstore: IpldStore, cid: CID, layer?: number): MST {
|
||||
return new MST(blockstore, cid, null, layer ?? null)
|
||||
}
|
||||
|
||||
async getEntries(): Promise<NodeEntry[]> {
|
||||
if (this.entries) return this.entries
|
||||
if (this.pointer) {
|
||||
const data = await this.blockstore.get(this.pointer, nodeDataSchema)
|
||||
this.entries = data.map((entry) => {
|
||||
if (check.is(entry, treePointer)) {
|
||||
// @TODO using this.layer instead of getLayer here??
|
||||
return MST.fromCid(
|
||||
this.blockstore,
|
||||
entry,
|
||||
this.layer ? this.layer - 1 : undefined,
|
||||
)
|
||||
} else {
|
||||
return new Leaf(entry[0], entry[1])
|
||||
}
|
||||
})
|
||||
|
||||
return this.entries
|
||||
}
|
||||
throw new Error('No entries or CID provided')
|
||||
}
|
||||
|
||||
async getLayer(): Promise<number> {
|
||||
if (this.layer !== null) return this.layer
|
||||
const entries = await this.getEntries()
|
||||
const firstLeaf = entries.find((entry) => entry.isLeaf())
|
||||
if (!firstLeaf) {
|
||||
throw new Error('not a valid mst node: no leaves')
|
||||
}
|
||||
this.layer = await leadingZerosOnHash(firstLeaf[0])
|
||||
return this.layer
|
||||
}
|
||||
|
||||
async add(key: string, value: CID): Promise<MST> {
|
||||
const keyZeros = await leadingZerosOnHash(key)
|
||||
const layer = await this.getLayer()
|
||||
const newLeaf = new Leaf(key, value)
|
||||
if (keyZeros === layer) {
|
||||
// it belongs in this layer
|
||||
const index = await this.findGtOrEqualLeafIndex(key)
|
||||
const found = await this.atIndex(index)
|
||||
if (found && found.equals(newLeaf)) {
|
||||
throw new Error(`There is already a value at key: ${key}`)
|
||||
}
|
||||
const prevNode = await this.atIndex(index - 1)
|
||||
if (!prevNode || prevNode.isLeaf()) {
|
||||
// if entry before is a leaf, (or we're on far left) we can just splice in
|
||||
return this.spliceIn(newLeaf, index)
|
||||
} else {
|
||||
// else we try to split the subtree around the key
|
||||
const splitSubTree = await prevNode.splitAround(key)
|
||||
return this.replaceWithSplit(
|
||||
index - 1,
|
||||
splitSubTree[0],
|
||||
newLeaf,
|
||||
splitSubTree[1],
|
||||
)
|
||||
}
|
||||
} else if (keyZeros < layer) {
|
||||
// it belongs on a lower layer
|
||||
const index = await this.findGtOrEqualLeafIndex(key)
|
||||
const prevNode = await this.atIndex(index - 1)
|
||||
if (prevNode && prevNode.isTree()) {
|
||||
// if entry before is a tree, we add it to that tree
|
||||
const newSubtree = await prevNode.add(key, value)
|
||||
return this.updateEntry(index - 1, newSubtree)
|
||||
} else {
|
||||
const subTree = await this.createChild()
|
||||
const newSubTree = await subTree.add(key, value)
|
||||
return this.spliceIn(newSubTree, index)
|
||||
}
|
||||
} else {
|
||||
// it belongs on a higher layer & we must push the rest of the tree down
|
||||
let split = await this.splitAround(key)
|
||||
// if the newly added key has >=2 more leading zeros than the current highest layer
|
||||
// then we need to add in structural nodes in between as well
|
||||
let left: MST | null = split[0]
|
||||
let right: MST | null = split[1]
|
||||
const layer = await this.getLayer()
|
||||
const extraLayersToAdd = keyZeros - layer
|
||||
// intentionally starting at 1, since first layer is taken care of by split
|
||||
for (let i = 1; i < extraLayersToAdd; i++) {
|
||||
if (left !== null) {
|
||||
left = await left.createParent()
|
||||
}
|
||||
if (right !== null) {
|
||||
right = await right.createParent()
|
||||
}
|
||||
}
|
||||
const updated: NodeEntry[] = []
|
||||
if (left) updated.push(left)
|
||||
updated.push(new Leaf(key, value))
|
||||
if (right) updated.push(right)
|
||||
return MST.create(this.blockstore, updated, keyZeros)
|
||||
}
|
||||
}
|
||||
|
||||
async get(key: string): Promise<CID | null> {
|
||||
const index = await this.findGtOrEqualLeafIndex(key)
|
||||
const found = await this.atIndex(index)
|
||||
if (found && found.isLeaf() && found.key === key) {
|
||||
return found.value
|
||||
}
|
||||
const prev = await this.atIndex(index - 1)
|
||||
if (prev && prev.isTree()) {
|
||||
return prev.get(key)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
async edit(key: string, value: CID): Promise<MST> {
|
||||
const index = await this.findGtOrEqualLeafIndex(key)
|
||||
const found = await this.atIndex(index)
|
||||
if (found && found.isLeaf() && found.key === key) {
|
||||
return this.updateEntry(index, new Leaf(key, value))
|
||||
}
|
||||
const prev = await this.atIndex(index - 1)
|
||||
if (prev && prev.isTree()) {
|
||||
const updatedTree = await prev.edit(key, value)
|
||||
return this.updateEntry(index - 1, updatedTree)
|
||||
}
|
||||
throw new Error(`Could not find a record with key: ${key}`)
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<MST> {
|
||||
const index = await this.findGtOrEqualLeafIndex(key)
|
||||
const found = await this.atIndex(index)
|
||||
if (found?.isLeaf() && found.key === key) {
|
||||
const prev = await this.atIndex(index - 1)
|
||||
const next = await this.atIndex(index + 10)
|
||||
if (prev?.isTree() && next?.isTree()) {
|
||||
const merged = await prev.appendMerge(next)
|
||||
return this.newTree([
|
||||
...(await this.slice(0, index - 1)),
|
||||
merged,
|
||||
...(await this.slice(0, index + 1)),
|
||||
])
|
||||
} else {
|
||||
return this.removeEntry(index)
|
||||
}
|
||||
}
|
||||
const prev = await this.atIndex(index - 1)
|
||||
if (prev?.isTree()) {
|
||||
const subtree = await prev.delete(key)
|
||||
return this.updateEntry(index - 1, subtree)
|
||||
} else {
|
||||
throw new Error(`Could not find a record with key: ${key}`)
|
||||
}
|
||||
}
|
||||
|
||||
// the simple merge case where every key in the right tree is greater than every key in the left tree (ie deletes)
|
||||
async appendMerge(toMerge: MST): Promise<MST> {
|
||||
if (!(await this.isSameLayer(toMerge))) {
|
||||
throw new Error(
|
||||
'Trying to merge two nodes from different layers of the MST',
|
||||
)
|
||||
}
|
||||
const thisEntries = await this.getEntries()
|
||||
const toMergeEntries = await toMerge.getEntries()
|
||||
const lastInLeft = thisEntries[toMergeEntries.length - 1]
|
||||
const firstInRight = toMergeEntries[0]
|
||||
if (lastInLeft?.isTree() && firstInRight?.isTree()) {
|
||||
const merged = await lastInLeft.appendMerge(firstInRight)
|
||||
return this.newTree([
|
||||
...thisEntries.slice(0, thisEntries.length - 1),
|
||||
merged,
|
||||
...toMergeEntries.slice(1),
|
||||
])
|
||||
} else {
|
||||
return this.newTree([...thisEntries, ...toMergeEntries])
|
||||
}
|
||||
}
|
||||
|
||||
async isSameLayer(other: MST): Promise<boolean> {
|
||||
const thisLayer = await this.getLayer()
|
||||
const otherLayer = await other.getLayer()
|
||||
return thisLayer === otherLayer
|
||||
}
|
||||
|
||||
async createChild(): Promise<MST> {
|
||||
const layer = await this.getLayer()
|
||||
return MST.create(this.blockstore, [], layer - 1)
|
||||
}
|
||||
|
||||
async createParent(): Promise<MST> {
|
||||
const layer = await this.getLayer()
|
||||
return MST.create(this.blockstore, [this], layer + 1)
|
||||
}
|
||||
|
||||
async updateEntry(index: number, entry: NodeEntry): Promise<MST> {
|
||||
const entries = await this.getEntries()
|
||||
entries[index] = entry
|
||||
return this.newTree(entries)
|
||||
}
|
||||
|
||||
async removeEntry(index: number): Promise<MST> {
|
||||
const entries = await this.getEntries()
|
||||
const updated = entries.splice(index, 1)
|
||||
return this.newTree(updated)
|
||||
}
|
||||
|
||||
newTree(entries: NodeEntry[]): MST {
|
||||
return new MST(this.blockstore, this.pointer, entries, this.layer)
|
||||
}
|
||||
|
||||
async splitAround(key: string): Promise<[MST | null, MST | null]> {
|
||||
const index = await this.findGtOrEqualLeafIndex(key)
|
||||
const leftData = await this.slice(0, index)
|
||||
const rightData = await this.slice(index)
|
||||
|
||||
if (leftData.length === 0) {
|
||||
return [null, this]
|
||||
}
|
||||
if (rightData.length === 0) {
|
||||
return [this, null]
|
||||
}
|
||||
const left = this.newTree(leftData)
|
||||
const right = this.newTree(rightData)
|
||||
const prev = leftData[leftData.length - 1]
|
||||
if (prev.isTree()) {
|
||||
const prevSplit = await prev.splitAround(key)
|
||||
if (prevSplit[0]) {
|
||||
left.append(prev)
|
||||
}
|
||||
if (prevSplit[1]) {
|
||||
right.prepend(prev)
|
||||
}
|
||||
}
|
||||
|
||||
return [left, right]
|
||||
}
|
||||
|
||||
async append(entry: NodeEntry): Promise<MST> {
|
||||
const entries = await this.getEntries()
|
||||
return this.newTree([...entries, entry])
|
||||
}
|
||||
|
||||
async prepend(entry: NodeEntry): Promise<MST> {
|
||||
const entries = await this.getEntries()
|
||||
return this.newTree([entry, ...entries])
|
||||
}
|
||||
|
||||
async atIndex(index: number): Promise<NodeEntry | null> {
|
||||
const entries = await this.getEntries()
|
||||
return entries[index] ?? null
|
||||
}
|
||||
|
||||
async slice(
|
||||
start?: number | undefined,
|
||||
end?: number | undefined,
|
||||
): Promise<NodeEntry[]> {
|
||||
const entries = await this.getEntries()
|
||||
return entries.slice(start, end)
|
||||
}
|
||||
|
||||
async spliceIn(entry: NodeEntry, index: number): Promise<MST> {
|
||||
const update = [
|
||||
...(await this.slice(0, index)),
|
||||
entry,
|
||||
...(await this.slice(index)),
|
||||
]
|
||||
return this.newTree(update)
|
||||
}
|
||||
|
||||
async replaceWithSplit(
|
||||
index: number,
|
||||
left: MST | null,
|
||||
leaf: Leaf,
|
||||
right: MST | null,
|
||||
): Promise<MST> {
|
||||
const update = await this.slice(0, index)
|
||||
if (left) update.push(left)
|
||||
update.push(leaf)
|
||||
if (right) update.push(right)
|
||||
update.push(...(await this.slice(index + 1)))
|
||||
return this.newTree(update)
|
||||
}
|
||||
|
||||
async findLeafOrPriorSubTree(key: string): Promise<NodeEntry | null> {
|
||||
const index = await this.findGtOrEqualLeafIndex(key)
|
||||
const found = await this.atIndex(index)
|
||||
if (found && found.isLeaf() && found.key === key) {
|
||||
return found
|
||||
}
|
||||
const prev = await this.atIndex(index - 1)
|
||||
if (prev && prev.isTree()) {
|
||||
return prev
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
// finds first leaf node that is greater than or equal to the value
|
||||
async findGtOrEqualLeafIndex(key: string): Promise<number> {
|
||||
const entries = await this.getEntries()
|
||||
const maybeIndex = entries.findIndex(
|
||||
(entry) => entry.isLeaf() && entry.key >= key,
|
||||
)
|
||||
// if we can't find, we're on the end
|
||||
return maybeIndex >= 0 ? maybeIndex : entries.length
|
||||
}
|
||||
|
||||
isTree(): this is MST {
|
||||
return true
|
||||
}
|
||||
|
||||
isLeaf(): this is Leaf {
|
||||
return false
|
||||
}
|
||||
|
||||
equals(entry: NodeEntry): boolean {
|
||||
if (entry.isTree()) {
|
||||
return entry.pointer.equals(this.pointer)
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type NodeEntry = MST | Leaf
|
||||
|
||||
// class Subtree {
|
||||
|
||||
// constructor(public pointer: CID) {}
|
||||
|
||||
// isSubtree(): this is Subtree {
|
||||
// return true
|
||||
// }
|
||||
|
||||
// isLeaf(): this is Leaf {
|
||||
// return false
|
||||
// }
|
||||
|
||||
// equals(entry: NodeEntry): boolean {
|
||||
// if(entry.isSubtree()) {
|
||||
// return entry.pointer.equals(this.pointer)
|
||||
// } else {
|
||||
// return false
|
||||
// }
|
||||
// }
|
||||
|
||||
// }
|
||||
|
||||
class Leaf {
|
||||
constructor(public key: string, public value: CID) {}
|
||||
|
||||
isTree(): this is MST {
|
||||
return false
|
||||
}
|
||||
|
||||
isLeaf(): this is Leaf {
|
||||
return true
|
||||
}
|
||||
|
||||
equals(entry: NodeEntry): boolean {
|
||||
if (entry.isLeaf()) {
|
||||
return this.key === entry.key && this.value.equals(entry.value)
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// class DiffTracker {
|
||||
// adds: Record<string, Add> = {}
|
||||
// updates: Record<string, Update> = {}
|
||||
// deletes: Record<string, Delete> = {}
|
||||
|
||||
// recordDelete(key: string): void {
|
||||
// if (this.adds[key]) {
|
||||
// delete this.adds[key]
|
||||
// } else {
|
||||
// this.deletes[key] = { key }
|
||||
// }
|
||||
// }
|
||||
|
||||
// recordAdd(key: string, cid: CID): void {
|
||||
// if (this.deletes[key]) {
|
||||
// delete this.deletes[key]
|
||||
// } else {
|
||||
// this.adds[key] = { key, cid }
|
||||
// }
|
||||
// }
|
||||
|
||||
// recordUpdate(key: string, old: CID, cid: CID): void {
|
||||
// this.updates[key] = { key, old, cid }
|
||||
// }
|
||||
|
||||
// getDiff(): Diff {
|
||||
// return {
|
||||
// adds: Object.values(adds),
|
||||
// updates: Object.values(updates),
|
||||
// deletes: Object.values(deletes),
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// type Delete = {
|
||||
// key: string
|
||||
// }
|
||||
|
||||
// type Add = {
|
||||
// key: string
|
||||
// cid: CID
|
||||
// }
|
||||
|
||||
// type Update = {
|
||||
// key: string
|
||||
// old: CID
|
||||
// cid: CID
|
||||
// }
|
||||
|
||||
// type Diff = {
|
||||
// adds: Add[]
|
||||
// updates: Update[]
|
||||
// deletes: Delete[]
|
||||
// }
|
||||
|
||||
export default MST
|
||||
@ -1,4 +1,4 @@
|
||||
import MST from '../src/repo/mst'
|
||||
import MST from '../src/repo/mst/mst'
|
||||
|
||||
import * as util from './_util'
|
||||
import { IpldStore } from '../src'
|
||||
@ -6,108 +6,94 @@ import { CID } from 'multiformats'
|
||||
import fs from 'fs'
|
||||
|
||||
describe('Merkle Search Tree', () => {
|
||||
it('height of all stupidity', async () => {
|
||||
const blockstore = IpldStore.createInMemory()
|
||||
const mst = await MST.create(blockstore)
|
||||
const toMerge = await MST.create(blockstore)
|
||||
const mapping = await util.generateBulkTidMapping(500)
|
||||
const shuffled = shuffle(Object.entries(mapping))
|
||||
|
||||
for (const entry of shuffled.slice(0, 350)) {
|
||||
await mst.add(entry[0], entry[1])
|
||||
await toMerge.add(entry[0], entry[1])
|
||||
}
|
||||
for (const entry of shuffled.slice(350, 400)) {
|
||||
await mst.add(entry[0], entry[1])
|
||||
}
|
||||
for (const entry of shuffled.slice(400)) {
|
||||
await toMerge.add(entry[0], entry[1])
|
||||
}
|
||||
console.log('zeros 1: ', mst.zeros)
|
||||
console.log('zeros 2: ', toMerge.zeros)
|
||||
|
||||
await mst.mergeIn(toMerge)
|
||||
for (const entry of shuffled) {
|
||||
const got = await mst.get(entry[0])
|
||||
expect(entry[1].equals(got)).toBeTruthy()
|
||||
}
|
||||
})
|
||||
|
||||
it('merges', async () => {
|
||||
const blockstore = IpldStore.createInMemory()
|
||||
const mst = await MST.create(blockstore)
|
||||
const toMerge = await MST.create(blockstore)
|
||||
// const mapping = await util.generateBulkTidMapping(500)
|
||||
// const shuffled = shuffle(Object.entries(mapping))
|
||||
const values: Record<string, CID> = {}
|
||||
const layer1 = ['3j6hnk65jju2t']
|
||||
const layer0 = ['3j6hnk65jis2t', '3j6hnk65jit2t']
|
||||
|
||||
const newKeys = ['3j6hnk65jnm2t']
|
||||
|
||||
const all = [...layer0, ...layer1]
|
||||
|
||||
for (const tid of all) {
|
||||
const cid = await util.randomCid()
|
||||
values[tid] = cid
|
||||
await mst.add(tid, cid)
|
||||
await toMerge.add(tid, cid)
|
||||
}
|
||||
|
||||
console.log('ADDING NEW KEYS')
|
||||
for (const tid of newKeys) {
|
||||
const cid = await util.randomCid()
|
||||
values[tid] = cid
|
||||
await toMerge.add(tid, cid)
|
||||
}
|
||||
console.log('MERGING')
|
||||
await mst.mergeIn(toMerge)
|
||||
|
||||
const structure = await mst.structure()
|
||||
|
||||
let output = ''
|
||||
await mst.walk((lvl, key) => {
|
||||
if (key) {
|
||||
output += `${lvl}: ${key}\n`
|
||||
}
|
||||
output += `${lvl}\n`
|
||||
})
|
||||
|
||||
fs.writeFileSync('structure', output)
|
||||
|
||||
// const tree = {
|
||||
// 0: [],
|
||||
// 1: [],
|
||||
// 2: [],
|
||||
// }
|
||||
// await mst.walk((lvl, key) => {
|
||||
// tree[lvl].push(key)
|
||||
// })
|
||||
// console.log(tree)
|
||||
|
||||
const got = await mst.get(newKeys[0])
|
||||
console.log('GOT: ', got)
|
||||
|
||||
// for (const entry of Object.entries(values)) {
|
||||
// const got = await mst.get(entry[0])
|
||||
// expect(entry[1].equals(got)).toBeTruthy()
|
||||
// }
|
||||
})
|
||||
|
||||
// it('works', async () => {
|
||||
// it('height of all stupidity', async () => {
|
||||
// const blockstore = IpldStore.createInMemory()
|
||||
// const mst = await MST.create(blockstore)
|
||||
// const mapping = await util.generateBulkTidMapping(1000)
|
||||
// const toMerge = await MST.create(blockstore)
|
||||
// const mapping = await util.generateBulkTidMapping(500)
|
||||
// const shuffled = shuffle(Object.entries(mapping))
|
||||
// for (const entry of shuffled) {
|
||||
// for (const entry of shuffled.slice(0, 350)) {
|
||||
// await mst.add(entry[0], entry[1])
|
||||
// await toMerge.add(entry[0], entry[1])
|
||||
// }
|
||||
// for (const entry of shuffled.slice(350, 400)) {
|
||||
// await mst.add(entry[0], entry[1])
|
||||
// }
|
||||
|
||||
// for (const entry of shuffled.slice(400)) {
|
||||
// await toMerge.add(entry[0], entry[1])
|
||||
// }
|
||||
// console.log('zeros 1: ', mst.zeros)
|
||||
// console.log('zeros 2: ', toMerge.zeros)
|
||||
// await mst.mergeIn(toMerge)
|
||||
// for (const entry of shuffled) {
|
||||
// const got = await mst.get(entry[0])
|
||||
// expect(entry[1].equals(got)).toBeTruthy()
|
||||
// }
|
||||
// })
|
||||
// it('merges', async () => {
|
||||
// const blockstore = IpldStore.createInMemory()
|
||||
// const mst = await MST.create(blockstore)
|
||||
// const toMerge = await MST.create(blockstore)
|
||||
// // const mapping = await util.generateBulkTidMapping(500)
|
||||
// // const shuffled = shuffle(Object.entries(mapping))
|
||||
// const values: Record<string, CID> = {}
|
||||
// const layer1 = ['3j6hnk65jju2t']
|
||||
// const layer0 = ['3j6hnk65jis2t', '3j6hnk65jit2t']
|
||||
// const newKeys = ['3j6hnk65jnm2t']
|
||||
// const all = [...layer0, ...layer1]
|
||||
// for (const tid of all) {
|
||||
// const cid = await util.randomCid()
|
||||
// values[tid] = cid
|
||||
// await mst.add(tid, cid)
|
||||
// await toMerge.add(tid, cid)
|
||||
// }
|
||||
// console.log('ADDING NEW KEYS')
|
||||
// for (const tid of newKeys) {
|
||||
// const cid = await util.randomCid()
|
||||
// values[tid] = cid
|
||||
// await toMerge.add(tid, cid)
|
||||
// }
|
||||
// console.log('MERGING')
|
||||
// await mst.mergeIn(toMerge)
|
||||
// const structure = await mst.structure()
|
||||
// let output = ''
|
||||
// await mst.walk((lvl, key) => {
|
||||
// if (key) {
|
||||
// output += `${lvl}: ${key}\n`
|
||||
// }
|
||||
// output += `${lvl}\n`
|
||||
// })
|
||||
// fs.writeFileSync('structure', output)
|
||||
// // const tree = {
|
||||
// // 0: [],
|
||||
// // 1: [],
|
||||
// // 2: [],
|
||||
// // }
|
||||
// // await mst.walk((lvl, key) => {
|
||||
// // tree[lvl].push(key)
|
||||
// // })
|
||||
// // console.log(tree)
|
||||
// const got = await mst.get(newKeys[0])
|
||||
// console.log('GOT: ', got)
|
||||
// // for (const entry of Object.entries(values)) {
|
||||
// // const got = await mst.get(entry[0])
|
||||
// // expect(entry[1].equals(got)).toBeTruthy()
|
||||
// // }
|
||||
// })
|
||||
|
||||
it('works', async () => {
|
||||
const blockstore = IpldStore.createInMemory()
|
||||
let mst = await MST.create(blockstore)
|
||||
const mapping = await util.generateBulkTidMapping(1000)
|
||||
const shuffled = shuffle(Object.entries(mapping))
|
||||
for (const entry of shuffled) {
|
||||
mst = await mst.add(entry[0], entry[1])
|
||||
}
|
||||
for (const entry of shuffled) {
|
||||
const got = await mst.get(entry[0])
|
||||
expect(entry[1].equals(got)).toBeTruthy()
|
||||
}
|
||||
})
|
||||
|
||||
// /**
|
||||
// * `f` gets added & it does two node splits (e is no longer grouped with g/h)
|
||||
@ -141,7 +127,6 @@ describe('Merkle Search Tree', () => {
|
||||
// const blockstore = IpldStore.createInMemory()
|
||||
// const mst = await MST.create(blockstore)
|
||||
// const cid = await util.randomCid()
|
||||
|
||||
// for (const tid of layer0) {
|
||||
// await mst.add(tid, cid)
|
||||
// }
|
||||
@ -150,14 +135,12 @@ describe('Merkle Search Tree', () => {
|
||||
// }
|
||||
// await mst.add(layer2, cid)
|
||||
// expect(mst.zeros).toBe(2)
|
||||
|
||||
// const allTids = [...layer0, ...layer1, layer2]
|
||||
// for (const tid of allTids) {
|
||||
// const got = await mst.get(tid)
|
||||
// expect(cid.equals(got)).toBeTruthy()
|
||||
// }
|
||||
// })
|
||||
|
||||
// /**
|
||||
// * `b` gets added & it hashes to 2 levels above any existing laves
|
||||
// *
|
||||
@ -185,7 +168,6 @@ describe('Merkle Search Tree', () => {
|
||||
// for (const tid of layer1) {
|
||||
// await mst.add(tid, cid)
|
||||
// }
|
||||
|
||||
// expect(mst.zeros).toBe(2)
|
||||
// const allTids = [...layer0, ...layer1, layer2]
|
||||
// for (const tid of allTids) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user