lamp/atproto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Disable pds appview indexing ()

Browse Source 
* rm indexing service

* remove message queue & refactor background queue

* wip

* remove all canProxyReadc

* finish cleanup

* clean up tests

* fix up tests

* fix api tests

* fix build

* fix compression test

* update image tests

* fix dev envs

* build branch

* wip - removing labeler

* fix service file

* remove kysely tables

* re-enable getPopular

* format

* cleaning up tests

* rm unused sharp code

* rm pds build

* clean up tests

* fix build

* fix build

* migration

* tidy

* build branch

* tidy

* build branch

* small tidy

* dont build
This commit is contained in:
Daniel Holmgren 2023-09-25 20:09:52 -05:00 committed by GitHub
parent 233a132c11
commit 2fa9088639
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
111 changed files with 173 additions and 5493 deletions
lexicons/app/bsky/unspecced
applyLabels.json
packages
api/src/client
index.tslexicons.ts
types/app/bsky/unspecced
applyLabels.ts
bsky
package.json
src
auto-moderator
index.ts
lexicon
index.tslexicons.ts
types/app/bsky/unspecced
applyLabels.ts
services
indexing/plugins
block.tsfeed-generator.tsfollow.tslike.tslist-block.tslist-item.tslist.tspost.tsrepost.tsthread-gate.ts
label
index.ts
common
package.json
src
dates.tsindex.ts
dev-env
build.jspackage.json
src
bin-network.tsbin.ts
mock
index.ts
pds.tstypes.ts
pds
package.json
src
api/com/atproto/admin
reverseModerationAction.tstakeModerationAction.ts
app-view
api/app/bsky
unspecced.ts
db
index.ts
tables
actor-block.tsalgo.tsduplicate-record.tsfeed-generator.tsfeed-item.tsfollow.tslike.tslist-item.tslist.tspost-agg.tspost-embed.tspost.tsprofile-agg.tsprofile.tsrepost.tssubscription.tssuggested-follow.tsview-param.ts
event-stream
consumers.ts
services
indexing
index.ts
plugins
block.tsfeed-generator.tsfollow.tslike.tslist-item.tslist.tspost.tsprofile.tsrepost.ts
processor.tsutil.ts
label
index.ts
background.tsconfig.tscontext.ts
db
database-schema.ts
migrations
20230922T033938477Z-remove-appview.tsindex.ts
periodic-moderation-action-reversal.ts
tables
label.ts
event-stream
message-queue.tsmessages.tstypes.ts
index.tslabel-cache.ts
labeler
base.tshive.tsindex.tskeyword.tsutil.ts
lexicon
index.tslexicons.ts
types/app/bsky/unspecced
applyLabels.ts
services
account
index.ts
index.ts
moderation
index.tsviews.ts
record
index.ts
repo
blobs.tsindex.ts
util
search.ts
tests
_util.tsaccount-deletion.test.ts
admin/__snapshots__
get-record.test.ts.snapget-repo.test.ts.snap
Show More

23
lexicons/app/bsky/unspecced/applyLabels.json

@ -1,23 +0,0 @@
{
"lexicon": 1,
"id": "app.bsky.unspecced.applyLabels",
"defs": {
"main": {
"type": "procedure",
"description": "Allow a labeler to apply labels directly.",
"input": {
"encoding": "application/json",
"schema": {
"type": "object",
"required": ["labels"],
"properties": {
"labels": {
"type": "array",
"items": { "type": "ref", "ref": "com.atproto.label.defs#label" }
}
}
}
}
}
}
}

13
packages/api/src/client/index.ts

@ -129,7 +129,6 @@ import * as AppBskyNotificationListNotifications from './types/app/bsky/notifica
import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush'
import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
import * as AppBskyRichtextFacet from './types/app/bsky/richtext/facet'
import * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels'
import * as AppBskyUnspeccedDefs from './types/app/bsky/unspecced/defs'
import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular'
import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
@ -259,7 +258,6 @@ export * as AppBskyNotificationListNotifications from './types/app/bsky/notifica
export * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush'
export * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
export * as AppBskyRichtextFacet from './types/app/bsky/richtext/facet'
export * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels'
export * as AppBskyUnspeccedDefs from './types/app/bsky/unspecced/defs'
export * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular'
export * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
@ -2253,17 +2251,6 @@ export class UnspeccedNS {
this._service = service
}
applyLabels(
data?: AppBskyUnspeccedApplyLabels.InputSchema,
opts?: AppBskyUnspeccedApplyLabels.CallOptions,
): Promise<AppBskyUnspeccedApplyLabels.Response> {
return this._service.xrpc
.call('app.bsky.unspecced.applyLabels', opts?.qp, data, opts)
.catch((e) => {
throw AppBskyUnspeccedApplyLabels.toKnownErr(e)
})
}
getPopular(
params?: AppBskyUnspeccedGetPopular.QueryParams,
opts?: AppBskyUnspeccedGetPopular.CallOptions,

27
packages/api/src/client/lexicons.ts

@ -6905,32 +6905,6 @@ export const schemaDict = {
},
},
},
AppBskyUnspeccedApplyLabels: {
lexicon: 1,
id: 'app.bsky.unspecced.applyLabels',
defs: {
main: {
type: 'procedure',
description: 'Allow a labeler to apply labels directly.',
input: {
encoding: 'application/json',
schema: {
type: 'object',
required: ['labels'],
properties: {
labels: {
type: 'array',
items: {
type: 'ref',
ref: 'lex:com.atproto.label.defs#label',
},
},
},
},
},
},
},
},
AppBskyUnspeccedDefs: {
lexicon: 1,
id: 'app.bsky.unspecced.defs',
@ -7359,7 +7333,6 @@ export const ids = {
AppBskyNotificationRegisterPush: 'app.bsky.notification.registerPush',
AppBskyNotificationUpdateSeen: 'app.bsky.notification.updateSeen',
AppBskyRichtextFacet: 'app.bsky.richtext.facet',
AppBskyUnspeccedApplyLabels: 'app.bsky.unspecced.applyLabels',
AppBskyUnspeccedDefs: 'app.bsky.unspecced.defs',
AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular',
AppBskyUnspeccedGetPopularFeedGenerators:

33
packages/api/src/client/types/app/bsky/unspecced/applyLabels.ts

@ -1,33 +0,0 @@
/**
* GENERATED CODE - DO NOT MODIFY
*/
import { Headers, XRPCError } from '@atproto/xrpc'
import { ValidationResult, BlobRef } from '@atproto/lexicon'
import { isObj, hasProp } from '../../../../util'
import { lexicons } from '../../../../lexicons'
import { CID } from 'multiformats/cid'
import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs'
export interface QueryParams {}
export interface InputSchema {
labels: ComAtprotoLabelDefs.Label[]
[k: string]: unknown
}
export interface CallOptions {
headers?: Headers
qp?: QueryParams
encoding: 'application/json'
}
export interface Response {
success: boolean
headers: Headers
}
export function toKnownErr(e: any) {
if (e instanceof XRPCError) {
}
return e
}

1
packages/bsky/package.json

@ -50,7 +50,6 @@
"http-errors": "^2.0.0",
"http-terminator": "^3.2.0",
"ioredis": "^5.3.2",
"iso-datestring-validator": "^2.2.2",
"kysely": "^0.22.0",
"multiformats": "^9.9.0",
"p-queue": "^6.6.2",

18
packages/bsky/src/auto-moderator/index.ts

@ -280,28 +280,12 @@ export class AutoModerator {
async storeLabels(uri: AtUri, cid: CID, labels: string[]): Promise<void> {
if (labels.length < 1) return
const labelSrvc = this.services.label(this.ctx.db)
const formatted = await labelSrvc.formatAndCreate(
await labelSrvc.formatAndCreate(
this.ctx.cfg.labelerDid,
uri.toString(),
cid.toString(),
{ create: labels },
)
if (this.pushAgent) {
const agent = this.pushAgent
try {
await agent.api.app.bsky.unspecced.applyLabels({ labels: formatted })
} catch (err) {
log.error(
{
err,
uri: uri.toString(),
labels,
receiver: agent.service.toString(),
},
'failed to push labels',
)
}
}
}
async processAll() {

12
packages/bsky/src/lexicon/index.ts

@ -107,7 +107,6 @@ import * as AppBskyNotificationGetUnreadCount from './types/app/bsky/notificatio
import * as AppBskyNotificationListNotifications from './types/app/bsky/notification/listNotifications'
import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush'
import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
import * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels'
import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular'
import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton'
@ -1402,17 +1401,6 @@ export class UnspeccedNS {
this._server = server
}
applyLabels<AV extends AuthVerifier>(
cfg: ConfigOf<
AV,
AppBskyUnspeccedApplyLabels.Handler<ExtractAuth<AV>>,
AppBskyUnspeccedApplyLabels.HandlerReqCtx<ExtractAuth<AV>>
>,
) {
const nsid = 'app.bsky.unspecced.applyLabels' // @ts-ignore
return this._server.xrpc.method(nsid, cfg)
}
getPopular<AV extends AuthVerifier>(
cfg: ConfigOf<
AV,

27
packages/bsky/src/lexicon/lexicons.ts

@ -6905,32 +6905,6 @@ export const schemaDict = {
},
},
},
AppBskyUnspeccedApplyLabels: {
lexicon: 1,
id: 'app.bsky.unspecced.applyLabels',
defs: {
main: {
type: 'procedure',
description: 'Allow a labeler to apply labels directly.',
input: {
encoding: 'application/json',
schema: {
type: 'object',
required: ['labels'],
properties: {
labels: {
type: 'array',
items: {
type: 'ref',
ref: 'lex:com.atproto.label.defs#label',
},
},
},
},
},
},
},
},
AppBskyUnspeccedDefs: {
lexicon: 1,
id: 'app.bsky.unspecced.defs',
@ -7359,7 +7333,6 @@ export const ids = {
AppBskyNotificationRegisterPush: 'app.bsky.notification.registerPush',
AppBskyNotificationUpdateSeen: 'app.bsky.notification.updateSeen',
AppBskyRichtextFacet: 'app.bsky.richtext.facet',
AppBskyUnspeccedApplyLabels: 'app.bsky.unspecced.applyLabels',
AppBskyUnspeccedDefs: 'app.bsky.unspecced.defs',
AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular',
AppBskyUnspeccedGetPopularFeedGenerators:

39
packages/bsky/src/lexicon/types/app/bsky/unspecced/applyLabels.ts

@ -1,39 +0,0 @@
/**
* GENERATED CODE - DO NOT MODIFY
*/
import express from 'express'
import { ValidationResult, BlobRef } from '@atproto/lexicon'
import { lexicons } from '../../../../lexicons'
import { isObj, hasProp } from '../../../../util'
import { CID } from 'multiformats/cid'
import { HandlerAuth } from '@atproto/xrpc-server'
import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs'
export interface QueryParams {}
export interface InputSchema {
labels: ComAtprotoLabelDefs.Label[]
[k: string]: unknown
}
export interface HandlerInput {
encoding: 'application/json'
body: InputSchema
}
export interface HandlerError {
status: number
message?: string
}
export type HandlerOutput = HandlerError | void
export type HandlerReqCtx<HA extends HandlerAuth = never> = {
auth: HA
params: QueryParams
input: HandlerInput
req: express.Request
res: express.Response
}
export type Handler<HA extends HandlerAuth = never> = (
ctx: HandlerReqCtx<HA>,
) => Promise<HandlerOutput> | HandlerOutput

2
packages/bsky/src/services/indexing/plugins/block.ts

@ -1,11 +1,11 @@
import { Selectable } from 'kysely'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { CID } from 'multiformats/cid'
import * as Block from '../../../lexicon/types/app/bsky/graph/block'
import * as lex from '../../../lexicon/lexicons'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
import { PrimaryDatabase } from '../../../db'
import { BackgroundQueue } from '../../../background'
import { NotificationServer } from '../../../notifications'

2
packages/bsky/src/services/indexing/plugins/feed-generator.ts

@ -1,5 +1,6 @@
import { Selectable } from 'kysely'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { CID } from 'multiformats/cid'
import * as FeedGenerator from '../../../lexicon/types/app/bsky/feed/generator'
import * as lex from '../../../lexicon/lexicons'
@ -7,7 +8,6 @@ import { PrimaryDatabase } from '../../../db'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import { BackgroundQueue } from '../../../background'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
import { NotificationServer } from '../../../notifications'
const lexId = lex.ids.AppBskyFeedGenerator

2
packages/bsky/src/services/indexing/plugins/follow.ts

@ -1,11 +1,11 @@
import { Selectable } from 'kysely'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { CID } from 'multiformats/cid'
import * as Follow from '../../../lexicon/types/app/bsky/graph/follow'
import * as lex from '../../../lexicon/lexicons'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
import { PrimaryDatabase } from '../../../db'
import { countAll, excluded } from '../../../db/util'
import { BackgroundQueue } from '../../../background'

2
packages/bsky/src/services/indexing/plugins/like.ts

@ -1,11 +1,11 @@
import { Selectable } from 'kysely'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { CID } from 'multiformats/cid'
import * as Like from '../../../lexicon/types/app/bsky/feed/like'
import * as lex from '../../../lexicon/lexicons'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
import { countAll, excluded } from '../../../db/util'
import { PrimaryDatabase } from '../../../db'
import { BackgroundQueue } from '../../../background'

2
packages/bsky/src/services/indexing/plugins/list-block.ts

@ -1,5 +1,6 @@
import { Selectable } from 'kysely'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { CID } from 'multiformats/cid'
import * as ListBlock from '../../../lexicon/types/app/bsky/graph/listblock'
import * as lex from '../../../lexicon/lexicons'
@ -8,7 +9,6 @@ import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { BackgroundQueue } from '../../../background'
import { NotificationServer } from '../../../notifications'
import { toSimplifiedISOSafe } from '../util'
const lexId = lex.ids.AppBskyGraphListblock
type IndexedListBlock = Selectable<DatabaseSchemaType['list_block']>

2
packages/bsky/src/services/indexing/plugins/list-item.ts

@ -1,11 +1,11 @@
import { Selectable } from 'kysely'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { CID } from 'multiformats/cid'
import * as ListItem from '../../../lexicon/types/app/bsky/graph/listitem'
import * as lex from '../../../lexicon/lexicons'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
import { InvalidRequestError } from '@atproto/xrpc-server'
import { PrimaryDatabase } from '../../../db'
import { BackgroundQueue } from '../../../background'

2
packages/bsky/src/services/indexing/plugins/list.ts

@ -1,11 +1,11 @@
import { Selectable } from 'kysely'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { CID } from 'multiformats/cid'
import * as List from '../../../lexicon/types/app/bsky/graph/list'
import * as lex from '../../../lexicon/lexicons'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
import { PrimaryDatabase } from '../../../db'
import { BackgroundQueue } from '../../../background'
import { NotificationServer } from '../../../notifications'

2
packages/bsky/src/services/indexing/plugins/post.ts

@ -1,6 +1,7 @@
import { Insertable, Selectable, sql } from 'kysely'
import { CID } from 'multiformats/cid'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { jsonStringToLex } from '@atproto/lexicon'
import {
Record as PostRecord,
@ -19,7 +20,6 @@ import * as lex from '../../../lexicon/lexicons'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { Notification } from '../../../db/tables/notification'
import { toSimplifiedISOSafe } from '../util'
import { PrimaryDatabase } from '../../../db'
import { countAll, excluded } from '../../../db/util'
import { BackgroundQueue } from '../../../background'

2
packages/bsky/src/services/indexing/plugins/repost.ts

@ -1,11 +1,11 @@
import { Selectable } from 'kysely'
import { CID } from 'multiformats/cid'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import * as Repost from '../../../lexicon/types/app/bsky/feed/repost'
import * as lex from '../../../lexicon/lexicons'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
import { PrimaryDatabase } from '../../../db'
import { countAll, excluded } from '../../../db/util'
import { BackgroundQueue } from '../../../background'

2
packages/bsky/src/services/indexing/plugins/thread-gate.ts

@ -1,11 +1,11 @@
import { AtUri } from '@atproto/syntax'
import { InvalidRequestError } from '@atproto/xrpc-server'
import { toSimplifiedISOSafe } from '@atproto/common'
import { CID } from 'multiformats/cid'
import * as Threadgate from '../../../lexicon/types/app/bsky/feed/threadgate'
import * as lex from '../../../lexicon/lexicons'
import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
import { PrimaryDatabase } from '../../../db'
import { BackgroundQueue } from '../../../background'
import { NotificationServer } from '../../../notifications'

2
packages/bsky/src/services/label/index.ts

@ -1,9 +1,9 @@
import { sql } from 'kysely'
import { AtUri } from '@atproto/syntax'
import { toSimplifiedISOSafe } from '@atproto/common'
import { Database } from '../../db'
import { Label, isSelfLabels } from '../../lexicon/types/com/atproto/label/defs'
import { ids } from '../../lexicon/lexicons'
import { toSimplifiedISOSafe } from '../indexing/util'
import { LabelCache } from '../../label-cache'
export type Labels = Record<string, Label[]>

1
packages/common/package.json

@ -27,6 +27,7 @@
"@atproto/common-web": "workspace:^",
"@ipld/dag-cbor": "^7.0.3",
"cbor-x": "^1.5.1",
"iso-datestring-validator": "^2.2.2",
"multiformats": "^9.9.0",
"pino": "^8.15.0",
"zod": "3.21.4"

0
packages/bsky/src/services/indexing/util.ts → packages/common/src/dates.ts

1
packages/common/src/index.ts

@ -1,4 +1,5 @@
export * from '@atproto/common-web'
export * from './dates'
export * from './fs'
export * from './ipld'
export * from './ipld-multi'

2
packages/dev-env/build.js

@ -6,7 +6,7 @@ const buildShallow =
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts', 'src/bin.ts', 'src/bin-network.ts'],
entryPoints: ['src/index.ts', 'src/bin.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',

3
packages/dev-env/package.json

@ -22,8 +22,7 @@
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/dev-env",
"start": "node dist/bin.js",
"start:network": "../dev-infra/with-test-redis-and-db.sh node dist/bin-network.js"
"start": "../dev-infra/with-test-redis-and-db.sh node dist/bin.js"
},
"dependencies": {
"@atproto/api": "workspace:^",

41
packages/dev-env/src/bin-network.ts

@ -1,41 +0,0 @@
import { generateMockSetup } from './mock'
import { TestNetwork } from './network'
const run = async () => {
console.log(`
██████╗
██╔═══██╗
██║██╗██║
██║██║██║
╚█║████╔╝
╚╝╚═══╝ protocol
[ created by Bluesky ]`)
const network = await TestNetwork.create({
pds: {
port: 2583,
publicUrl: 'http://localhost:2583',
enableLabelsCache: true,
dbPostgresSchema: 'pds',
},
bsky: {
dbPostgresSchema: 'bsky',
},
plc: { port: 2582 },
})
await generateMockSetup(network)
console.log(
`👤 DID Placeholder server started http://localhost:${network.plc.port}`,
)
console.log(
`🌞 Personal Data server started http://localhost:${network.pds.port}`,
)
console.log(`🌅 Bsky Appview started http://localhost:${network.bsky.port}`)
for (const fg of network.feedGens) {
console.log(`🤖 Feed Generator started http://localhost:${fg.port}`)
}
}
run()

10
packages/dev-env/src/bin.ts

@ -1,5 +1,5 @@
import { generateMockSetup } from './mock'
import { TestNetworkNoAppView } from './network-no-appview'
import { TestNetwork } from './network'
const run = async () => {
console.log(`
@ -12,11 +12,14 @@ const run = async () => {
[ created by Bluesky ]`)
const network = await TestNetworkNoAppView.create({
const network = await TestNetwork.create({
pds: {
port: 2583,
enableLabelsCache: true,
publicUrl: 'http://localhost:2583',
dbPostgresSchema: 'pds',
},
bsky: {
dbPostgresSchema: 'bsky',
},
plc: { port: 2582 },
})
@ -28,6 +31,7 @@ const run = async () => {
console.log(
`🌞 Personal Data server started http://localhost:${network.pds.port}`,
)
console.log(`🌅 Bsky Appview started http://localhost:${network.bsky.port}`)
for (const fg of network.feedGens) {
console.log(`🤖 Feed Generator started http://localhost:${fg.port}`)
}

46
packages/dev-env/src/mock/index.ts

@ -4,7 +4,7 @@ import {
REASONSPAM,
REASONOTHER,
} from '@atproto/api/src/client/types/com/atproto/moderation/defs'
import { TestNetworkNoAppView } from '../index'
import { TestNetwork } from '../index'
import { postTexts, replyTexts } from './data'
import labeledImgB64 from './img/labeled-img-b64'
import blurHashB64 from './img/blur-hash-avatar-b64'
@ -23,7 +23,7 @@ function* dateGen() {
return ''
}
export async function generateMockSetup(env: TestNetworkNoAppView) {
export async function generateMockSetup(env: TestNetwork) {
const date = dateGen()
const rand = (n: number) => Math.floor(Math.random() * n)
@ -186,29 +186,27 @@ export async function generateMockSetup(env: TestNetworkNoAppView) {
},
)
const ctx = env.pds.ctx
const ctx = env.bsky.ctx
if (ctx) {
await ctx.db.db
.insertInto('label')
.values([
{
src: ctx.cfg.labelerDid,
uri: labeledPost.uri,
cid: labeledPost.cid,
val: 'nudity',
neg: 0,
cts: new Date().toISOString(),
},
{
src: ctx.cfg.labelerDid,
uri: filteredPost.uri,
cid: filteredPost.cid,
val: 'dmca-violation',
neg: 0,
cts: new Date().toISOString(),
},
])
.execute()
const labelSrvc = ctx.services.label(ctx.db.getPrimary())
await labelSrvc.createLabels([
{
src: ctx.cfg.labelerDid,
uri: labeledPost.uri,
cid: labeledPost.cid,
val: 'nudity',
neg: false,
cts: new Date().toISOString(),
},
{
src: ctx.cfg.labelerDid,
uri: filteredPost.uri,
cid: filteredPost.cid,
val: 'dmca-violation',
neg: false,
cts: new Date().toISOString(),
},
])
}
// a set of replies

12
packages/dev-env/src/pds.ts

@ -2,7 +2,6 @@ import getPort from 'get-port'
import * as ui8 from 'uint8arrays'
import * as pds from '@atproto/pds'
import { Secp256k1Keypair, randomStr } from '@atproto/crypto'
import { MessageDispatcher } from '@atproto/pds/src/event-stream/message-queue'
import { AtpAgent } from '@atproto/api'
import { Client as PlcClient } from '@did-plc/lib'
import { DAY, HOUR } from '@atproto/common-web'
@ -60,9 +59,6 @@ export class TestPds {
maxSubscriptionBuffer: 200,
repoBackfillLimitMs: 1000 * 60 * 60, // 1hr
sequencerLeaderLockId: uniqueLockId(),
labelerDid: 'did:example:labeler',
labelerKeywords: { label_me: 'test-label', label_me_2: 'test-label-2' },
feedGenDid: 'did:example:feedGen',
dbTxLockNonce: await randomStr(32, 'base32'),
bskyAppViewEndpoint: cfg.bskyAppViewEndpoint ?? 'http://fake_address',
bskyAppViewDid: cfg.bskyAppViewDid ?? 'did:example:fake',
@ -80,11 +76,6 @@ export class TestPds {
: pds.Database.memory()
await db.migrateToLatestOrThrow()
if (cfg.bskyAppViewEndpoint && !cfg.enableInProcessAppView) {
// Disable communication to app view within pds
MessageDispatcher.prototype.send = async () => {}
}
const server = pds.PDS.create({
db,
blobstore,
@ -95,8 +86,6 @@ export class TestPds {
await server.start()
// we refresh label cache by hand in `processAll` instead of on a timer
if (!cfg.enableLabelsCache) server.ctx.labelCache.stop()
return new TestPds(url, port, server)
}
@ -129,7 +118,6 @@ export class TestPds {
async processAll() {
await this.ctx.backgroundQueue.processAll()
await this.ctx.labelCache.fullRefresh()
}
async close() {

2
packages/dev-env/src/types.ts

@ -10,8 +10,6 @@ export type PlcConfig = {
export type PdsConfig = Partial<pds.ServerConfig> & {
plcUrl: string
migration?: string
enableInProcessAppView?: boolean
enableLabelsCache?: boolean
}
export type BskyConfig = Partial<bsky.ServerConfig> & {

1
packages/pds/package.json

@ -56,7 +56,6 @@
"http-errors": "^2.0.0",
"http-terminator": "^3.2.0",
"ioredis": "^5.3.2",
"iso-datestring-validator": "^2.2.2",
"jsonwebtoken": "^8.5.1",
"kysely": "^0.22.0",
"multiformats": "^9.9.0",

31
packages/pds/src/api/com/atproto/admin/reverseModerationAction.ts

@ -26,7 +26,6 @@ export default function (server: Server, ctx: AppContext) {
const transact = db.transaction(async (dbTxn) => {
const moderationTxn = services.moderation(dbTxn)
const labelTxn = services.appView.label(dbTxn)
// reverse takedowns
if (result.action === TAKEDOWN && isRepoRef(result.subject)) {
await moderationTxn.reverseTakedownRepo({
@ -38,18 +37,6 @@ export default function (server: Server, ctx: AppContext) {
uri: new AtUri(result.subject.uri),
})
}
// invert label creation & negations
const reverseLabels = (uri: string, cid: string | null) =>
labelTxn.formatAndCreate(ctx.cfg.labelerDid, uri, cid, {
create: result.negateLabelVals,
negate: result.createLabelVals,
})
if (isRepoRef(result.subject)) {
await reverseLabels(result.subject.did, null)
}
if (isStrongRef(result.subject)) {
await reverseLabels(result.subject.uri, result.subject.cid)
}
})
try {
@ -72,7 +59,6 @@ export default function (server: Server, ctx: AppContext) {
const moderationAction = await db.transaction(async (dbTxn) => {
const moderationTxn = services.moderation(dbTxn)
const labelTxn = services.appView.label(dbTxn)
const now = new Date()
const existing = await moderationTxn.getAction(id)
@ -114,23 +100,6 @@ export default function (server: Server, ctx: AppContext) {
reason,
})
// invert creates & negates
const { createLabelVals, negateLabelVals } = result
const negate =
createLabelVals && createLabelVals.length > 0
? createLabelVals.split(' ')
: undefined
const create =
negateLabelVals && negateLabelVals.length > 0
? negateLabelVals.split(' ')
: undefined
await labelTxn.formatAndCreate(
ctx.cfg.labelerDid,
result.subjectUri ?? result.subjectDid,
result.subjectCid,
{ create, negate },
)
return result
})

21
packages/pds/src/api/com/atproto/admin/takeModerationAction.ts

@ -29,7 +29,6 @@ export default function (server: Server, ctx: AppContext) {
const transact = db.transaction(async (dbTxn) => {
const authTxn = services.auth(dbTxn)
const moderationTxn = services.moderation(dbTxn)
const labelTxn = services.appView.label(dbTxn)
// perform takedowns
if (result.action === TAKEDOWN && isRepoRef(result.subject)) {
await authTxn.revokeRefreshTokensByDid(result.subject.did)
@ -45,18 +44,6 @@ export default function (server: Server, ctx: AppContext) {
blobCids: result.subjectBlobCids.map((cid) => CID.parse(cid)),
})
}
// apply label creation & negations
const applyLabels = (uri: string, cid: string | null) =>
labelTxn.formatAndCreate(ctx.cfg.labelerDid, uri, cid, {
create: result.createLabelVals,
negate: result.negateLabelVals,
})
if (isRepoRef(result.subject)) {
await applyLabels(result.subject.did, null)
}
if (isStrongRef(result.subject)) {
await applyLabels(result.subject.uri, result.subject.cid)
}
})
try {
@ -113,7 +100,6 @@ export default function (server: Server, ctx: AppContext) {
const moderationAction = await db.transaction(async (dbTxn) => {
const authTxn = services.auth(dbTxn)
const moderationTxn = services.moderation(dbTxn)
const labelTxn = services.appView.label(dbTxn)
const result = await moderationTxn.logAction({
action: getAction(action),
@ -150,13 +136,6 @@ export default function (server: Server, ctx: AppContext) {
})
}
await labelTxn.formatAndCreate(
ctx.cfg.labelerDid,
result.subjectUri ?? result.subjectDid,
result.subjectCid,
{ create: createLabelVals, negate: negateLabelVals },
)
return result
})

19
packages/pds/src/app-view/api/app/bsky/unspecced.ts

@ -1,4 +1,4 @@
import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server'
import { InvalidRequestError } from '@atproto/xrpc-server'
import { Server } from '../../../../lexicon'
import { GenericKeyset } from '../../../../db/pagination'
import AppContext from '../../../../context'
@ -20,11 +20,6 @@ export default function (server: Server, ctx: AppContext) {
encoding: 'application/json',
body: res.data,
}
return {
encoding: 'application/json',
body: { feed: [] },
}
},
})
@ -43,18 +38,6 @@ export default function (server: Server, ctx: AppContext) {
}
},
})
server.app.bsky.unspecced.applyLabels({
auth: ctx.roleVerifier,
handler: async ({ auth, input }) => {
if (!auth.credentials.admin) {
throw new AuthRequiredError('Insufficient privileges')
}
const { services, db } = ctx
const { labels } = input.body
await services.appView.label(db).createLabels(labels)
},
})
}
type Result = { likeCount: number; cid: string }

38
packages/pds/src/app-view/db/index.ts

@ -1,38 +0,0 @@
import * as duplicateRecords from './tables/duplicate-record'
import * as profile from './tables/profile'
import * as profileAgg from './tables/profile-agg'
import * as post from './tables/post'
import * as postAgg from './tables/post-agg'
import * as postEmbed from './tables/post-embed'
import * as repost from './tables/repost'
import * as feedItem from './tables/feed-item'
import * as follow from './tables/follow'
import * as list from './tables/list'
import * as listItem from './tables/list-item'
import * as actorBlock from './tables/actor-block'
import * as like from './tables/like'
import * as feedGenerator from './tables/feed-generator'
import * as subscription from './tables/subscription'
import * as algo from './tables/algo'
import * as viewParam from './tables/view-param'
import * as suggestedFollow from './tables/suggested-follow'
// @NOTE app-view also shares did-handle, record, and repo-root tables w/ main pds
export type DatabaseSchemaType = duplicateRecords.PartialDB &
profile.PartialDB &
profileAgg.PartialDB &
post.PartialDB &
postAgg.PartialDB &
postEmbed.PartialDB &
repost.PartialDB &
feedItem.PartialDB &
follow.PartialDB &
list.PartialDB &
listItem.PartialDB &
actorBlock.PartialDB &
like.PartialDB &
feedGenerator.PartialDB &
subscription.PartialDB &
algo.PartialDB &
viewParam.PartialDB &
suggestedFollow.PartialDB

11
packages/pds/src/app-view/db/tables/actor-block.ts

@ -1,11 +0,0 @@
export const tableName = 'actor_block'
export interface ActorBlock {
uri: string
cid: string
creator: string
subjectDid: string
createdAt: string
indexedAt: string
}
export type PartialDB = { [tableName]: ActorBlock }

12
packages/pds/src/app-view/db/tables/algo.ts

@ -1,12 +0,0 @@
// @NOTE postgres-only
export const whatsHotViewTableName = 'algo_whats_hot_view'
export interface AlgoWhatsHotView {
uri: string
cid: string
score: number
}
export type PartialDB = {
[whatsHotViewTableName]: AlgoWhatsHotView
}

12
packages/pds/src/app-view/db/tables/duplicate-record.ts

@ -1,12 +0,0 @@
export interface DuplicateRecord {
uri: string
cid: string
duplicateOf: string
indexedAt: string
}
export const tableName = 'duplicate_record'
export type PartialDB = {
[tableName]: DuplicateRecord
}

18
packages/pds/src/app-view/db/tables/feed-generator.ts

@ -1,18 +0,0 @@
export const tableName = 'feed_generator'
export interface FeedGenerator {
uri: string
cid: string
creator: string
feedDid: string
displayName: string
description: string | null
descriptionFacets: string | null
avatarCid: string | null
createdAt: string
indexedAt: string
}
export type PartialDB = {
[tableName]: FeedGenerator
}

12
packages/pds/src/app-view/db/tables/feed-item.ts

@ -1,12 +0,0 @@
export const tableName = 'feed_item'
export interface FeedItem {
uri: string
cid: string
type: 'post' | 'repost'
postUri: string
originatorDid: string
sortAt: string
}
export type PartialDB = { [tableName]: FeedItem }

11
packages/pds/src/app-view/db/tables/follow.ts

@ -1,11 +0,0 @@
export const tableName = 'follow'
export interface Follow {
uri: string
cid: string
creator: string
subjectDid: string
createdAt: string
indexedAt: string
}
export type PartialDB = { [tableName]: Follow }

13
packages/pds/src/app-view/db/tables/like.ts

@ -1,13 +0,0 @@
export interface Like {
uri: string
cid: string
creator: string
subject: string
subjectCid: string
createdAt: string
indexedAt: string
}
const tableName = 'like'
export type PartialDB = { [tableName]: Like }

13
packages/pds/src/app-view/db/tables/list-item.ts

@ -1,13 +0,0 @@
export const tableName = 'list_item'
export interface ListItem {
uri: string
cid: string
creator: string
subjectDid: string
listUri: string
createdAt: string
indexedAt: string
}
export type PartialDB = { [tableName]: ListItem }

16
packages/pds/src/app-view/db/tables/list.ts

@ -1,16 +0,0 @@
export const tableName = 'list'
export interface List {
uri: string
cid: string
creator: string
name: string
purpose: string
description: string | null
descriptionFacets: string | null
avatarCid: string | null
createdAt: string
indexedAt: string
}
export type PartialDB = { [tableName]: List }

14
packages/pds/src/app-view/db/tables/post-agg.ts

@ -1,14 +0,0 @@
import { Generated } from 'kysely'
export const tableName = 'post_agg'
export interface PostAgg {
uri: string
likeCount: Generated<number>
replyCount: Generated<number>
repostCount: Generated<number>
}
export type PartialDB = {
[tableName]: PostAgg
}

30
packages/pds/src/app-view/db/tables/post-embed.ts

@ -1,30 +0,0 @@
export const imageTableName = 'post_embed_image'
export const externalTableName = 'post_embed_external'
export const recordTableName = 'post_embed_record'
export interface PostEmbedImage {
postUri: string
position: number
imageCid: string
alt: string
}
export interface PostEmbedExternal {
postUri: string
uri: string
title: string
description: string
thumbCid: string | null
}
export interface PostEmbedRecord {
postUri: string
embedUri: string
embedCid: string
}
export type PartialDB = {
[imageTableName]: PostEmbedImage
[externalTableName]: PostEmbedExternal
[recordTableName]: PostEmbedRecord
}

18
packages/pds/src/app-view/db/tables/post.ts

@ -1,18 +0,0 @@
export const tableName = 'post'
export interface Post {
uri: string
cid: string
creator: string
text: string
replyRoot: string | null
replyRootCid: string | null
replyParent: string | null
replyParentCid: string | null
createdAt: string
indexedAt: string
}
export type PartialDB = {
[tableName]: Post
}

14
packages/pds/src/app-view/db/tables/profile-agg.ts

@ -1,14 +0,0 @@
import { Generated } from 'kysely'
export const tableName = 'profile_agg'
export interface ProfileAgg {
did: string
followersCount: Generated<number>
followsCount: Generated<number>
postsCount: Generated<number>
}
export type PartialDB = {
[tableName]: ProfileAgg
}

13
packages/pds/src/app-view/db/tables/profile.ts

@ -1,13 +0,0 @@
export const tableName = 'profile'
export interface Profile {
uri: string
cid: string
creator: string
displayName: string | null
description: string | null
avatarCid: string | null
bannerCid: string | null
indexedAt: string
}
export type PartialDB = { [tableName]: Profile }

13
packages/pds/src/app-view/db/tables/repost.ts

@ -1,13 +0,0 @@
export const tableName = 'repost'
export interface Repost {
uri: string
cid: string
creator: string
subject: string
subjectCid: string
createdAt: string
indexedAt: string
}
export type PartialDB = { [tableName]: Repost }

9
packages/pds/src/app-view/db/tables/subscription.ts

@ -1,9 +0,0 @@
export const tableName = 'subscription'
export interface Subscription {
service: string
method: string
state: string
}
export type PartialDB = { [tableName]: Subscription }

10
packages/pds/src/app-view/db/tables/suggested-follow.ts

@ -1,10 +0,0 @@
export const tableName = 'suggested_follow'
export interface SuggestedFollow {
did: string
order: number
}
export type PartialDB = {
[tableName]: SuggestedFollow
}

12
packages/pds/src/app-view/db/tables/view-param.ts

@ -1,12 +0,0 @@
// @NOTE postgres-only
export const tableName = 'view_param'
// materialized views are difficult to change,
// so we parameterize them at runtime with contents of this table.
// its contents are set in migrations, available param names are static.
export interface ViewParam {
name: string
value: string
}
export type PartialDB = { [tableName]: ViewParam }

38
packages/pds/src/app-view/event-stream/consumers.ts

@ -1,38 +0,0 @@
import AppContext from '../../context'
import Database from '../../db'
import {
DeleteRecord,
IndexRecord,
DeleteRepo,
} from '../../event-stream/messages'
// Used w/ in-process PDS as alternative to the repo subscription
export const listen = (ctx: AppContext) => {
ctx.messageDispatcher.listen('index_record', {
async listener(input: { db: Database; message: IndexRecord }) {
const { db, message } = input
const indexingService = ctx.services.appView.indexing(db)
await indexingService.indexRecord(
message.uri,
message.cid,
message.obj,
message.action,
message.timestamp,
)
},
})
ctx.messageDispatcher.listen('delete_record', {
async listener(input: { db: Database; message: DeleteRecord }) {
const { db, message } = input
const indexingService = ctx.services.appView.indexing(db)
await indexingService.deleteRecord(message.uri, message.cascading)
},
})
ctx.messageDispatcher.listen('delete_repo', {
async listener(input: { db: Database; message: DeleteRepo }) {
const { db, message } = input
const indexingService = ctx.services.appView.indexing(db)
await indexingService.deleteForUser(message.did)
},
})
}

136
packages/pds/src/app-view/services/indexing/index.ts

@ -1,136 +0,0 @@
import { CID } from 'multiformats/cid'
import { WriteOpAction } from '@atproto/repo'
import { AtUri } from '@atproto/syntax'
import { ids } from '../../../lexicon/lexicons'
import Database from '../../../db'
import { BackgroundQueue } from '../../../event-stream/background-queue'
import { NoopProcessor } from './processor'
import * as Post from './plugins/post'
import * as Like from './plugins/like'
import * as Repost from './plugins/repost'
import * as Follow from './plugins/follow'
import * as Block from './plugins/block'
import * as List from './plugins/list'
import * as ListItem from './plugins/list-item'
import * as Profile from './plugins/profile'
import * as FeedGenerator from './plugins/feed-generator'
export class IndexingService {
records: {
post: Post.PluginType
like: Like.PluginType
repost: Repost.PluginType
follow: Follow.PluginType
block: Block.PluginType
list: List.PluginType
listItem: ListItem.PluginType
listBlock: NoopProcessor
profile: Profile.PluginType
feedGenerator: FeedGenerator.PluginType
}
constructor(public db: Database, public backgroundQueue: BackgroundQueue) {
this.records = {
post: Post.makePlugin(this.db, backgroundQueue),
like: Like.makePlugin(this.db, backgroundQueue),
repost: Repost.makePlugin(this.db, backgroundQueue),
follow: Follow.makePlugin(this.db, backgroundQueue),
block: Block.makePlugin(this.db, backgroundQueue),
list: List.makePlugin(this.db, backgroundQueue),
listItem: ListItem.makePlugin(this.db, backgroundQueue),
listBlock: new NoopProcessor(
ids.AppBskyGraphListblock,
this.db,
backgroundQueue,
),
profile: Profile.makePlugin(this.db, backgroundQueue),
feedGenerator: FeedGenerator.makePlugin(this.db, backgroundQueue),
}
}
static creator(backgroundQueue: BackgroundQueue) {
return (db: Database) => new IndexingService(db, backgroundQueue)
}
async indexRecord(
uri: AtUri,
cid: CID,
obj: unknown,
action: WriteOpAction.Create | WriteOpAction.Update,
timestamp: string,
) {
this.db.assertTransaction()
const indexer = this.findIndexerForCollection(uri.collection)
if (action === WriteOpAction.Create) {
await indexer.insertRecord(uri, cid, obj, timestamp)
} else {
await indexer.updateRecord(uri, cid, obj, timestamp)
}
}
async deleteRecord(uri: AtUri, cascading = false) {
this.db.assertTransaction()
const indexer = this.findIndexerForCollection(uri.collection)
await indexer.deleteRecord(uri, cascading)
}
findIndexerForCollection(collection: string) {
const found = Object.values(this.records).find(
(plugin) => plugin.collection === collection,
)
if (!found) {
throw new Error('Could not find indexer for collection')
}
return found
}
async deleteForUser(did: string) {
// Not done in transaction because it would be too long, prone to contention.
// Also, this can safely be run multiple times if it fails.
// Omitting updates to profile_agg and post_agg since it's expensive
// and they'll organically update themselves over time.
const postByUser = (qb) =>
qb
.selectFrom('post')
.where('post.creator', '=', did)
.select('post.uri as uri')
await this.db.db
.deleteFrom('post_embed_image')
.where('post_embed_image.postUri', 'in', postByUser)
.execute()
await this.db.db
.deleteFrom('post_embed_external')
.where('post_embed_external.postUri', 'in', postByUser)
.execute()
await this.db.db
.deleteFrom('post_embed_record')
.where('post_embed_record.postUri', 'in', postByUser)
.execute()
await this.db.db
.deleteFrom('duplicate_record')
.where('duplicate_record.duplicateOf', 'in', (qb) =>
// @TODO remove dependency on record table from app view
qb
.selectFrom('record')
.where('record.did', '=', did)
.select('record.uri as uri'),
)
.execute()
await this.db.db
.deleteFrom('actor_block')
.where('creator', '=', did)
.execute()
await this.db.db.deleteFrom('list').where('creator', '=', did).execute()
await this.db.db
.deleteFrom('list_item')
.where('creator', '=', did)
.execute()
await this.db.db.deleteFrom('follow').where('creator', '=', did).execute()
await this.db.db.deleteFrom('post').where('creator', '=', did).execute()
await this.db.db.deleteFrom('profile').where('creator', '=', did).execute()
await this.db.db.deleteFrom('repost').where('creator', '=', did).execute()
await this.db.db.deleteFrom('like').where('creator', '=', did).execute()
}
}

97
packages/pds/src/app-view/services/indexing/plugins/block.ts

@ -1,97 +0,0 @@
import { AtUri } from '@atproto/syntax'
import { CID } from 'multiformats/cid'
import * as Block from '../../../../lexicon/types/app/bsky/graph/block'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
const lexId = lex.ids.AppBskyGraphBlock
type IndexedBlock = DatabaseSchemaType['actor_block']
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: Block.Record,
timestamp: string,
): Promise<IndexedBlock | null> => {
const inserted = await db
.insertInto('actor_block')
.values({
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
subjectDid: obj.subject,
createdAt: toSimplifiedISOSafe(obj.createdAt),
indexedAt: timestamp,
})
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst()
return inserted || null
}
const findDuplicate = async (
db: DatabaseSchema,
uri: AtUri,
obj: Block.Record,
): Promise<AtUri | null> => {
const found = await db
.selectFrom('actor_block')
.where('creator', '=', uri.host)
.where('subjectDid', '=', obj.subject)
.selectAll()
.executeTakeFirst()
return found ? new AtUri(found.uri) : null
}
const notifsForInsert = () => {
return []
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedBlock | null> => {
const deleted = await db
.deleteFrom('actor_block')
.where('uri', '=', uri.toString())
.returningAll()
.executeTakeFirst()
return deleted || null
}
const notifsForDelete = (
deleted: IndexedBlock,
replacedBy: IndexedBlock | null,
) => {
const toDelete = replacedBy ? [] : [deleted.uri]
return { notifs: [], toDelete }
}
const updateAggregates = async () => {}
export type PluginType = RecordProcessor<Block.Record, IndexedBlock>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
updateAggregates,
})
}
export default makePlugin

89
packages/pds/src/app-view/services/indexing/plugins/feed-generator.ts

@ -1,89 +0,0 @@
import { AtUri } from '@atproto/syntax'
import { CID } from 'multiformats/cid'
import * as FeedGenerator from '../../../../lexicon/types/app/bsky/feed/generator'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
const lexId = lex.ids.AppBskyFeedGenerator
type IndexedFeedGenerator = DatabaseSchemaType['feed_generator']
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: FeedGenerator.Record,
timestamp: string,
): Promise<IndexedFeedGenerator | null> => {
const inserted = await db
.insertInto('feed_generator')
.values({
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
feedDid: obj.did,
displayName: obj.displayName,
description: obj.description,
descriptionFacets: obj.descriptionFacets
? JSON.stringify(obj.descriptionFacets)
: undefined,
avatarCid: obj.avatar?.ref.toString(),
createdAt: toSimplifiedISOSafe(obj.createdAt),
indexedAt: timestamp,
})
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst()
return inserted || null
}
const findDuplicate = async (): Promise<AtUri | null> => {
return null
}
const notifsForInsert = () => {
return []
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedFeedGenerator | null> => {
const deleted = await db
.deleteFrom('feed_generator')
.where('uri', '=', uri.toString())
.returningAll()
.executeTakeFirst()
return deleted || null
}
const notifsForDelete = () => {
return { notifs: [], toDelete: [] }
}
export type PluginType = RecordProcessor<
FeedGenerator.Record,
IndexedFeedGenerator
>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
})
}
export default makePlugin

138
packages/pds/src/app-view/services/indexing/plugins/follow.ts

@ -1,138 +0,0 @@
import { AtUri } from '@atproto/syntax'
import { CID } from 'multiformats/cid'
import * as Follow from '../../../../lexicon/types/app/bsky/graph/follow'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { countAll, excluded } from '../../../../db/util'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
const lexId = lex.ids.AppBskyGraphFollow
type IndexedFollow = DatabaseSchemaType['follow']
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: Follow.Record,
timestamp: string,
): Promise<IndexedFollow | null> => {
const inserted = await db
.insertInto('follow')
.values({
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
subjectDid: obj.subject,
createdAt: toSimplifiedISOSafe(obj.createdAt),
indexedAt: timestamp,
})
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst()
return inserted || null
}
const findDuplicate = async (
db: DatabaseSchema,
uri: AtUri,
obj: Follow.Record,
): Promise<AtUri | null> => {
const found = await db
.selectFrom('follow')
.where('creator', '=', uri.host)
.where('subjectDid', '=', obj.subject)
.selectAll()
.executeTakeFirst()
return found ? new AtUri(found.uri) : null
}
const notifsForInsert = (obj: IndexedFollow) => {
return [
{
userDid: obj.subjectDid,
author: obj.creator,
recordUri: obj.uri,
recordCid: obj.cid,
reason: 'follow' as const,
reasonSubject: null,
indexedAt: obj.indexedAt,
},
]
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedFollow | null> => {
const deleted = await db
.deleteFrom('follow')
.where('uri', '=', uri.toString())
.returningAll()
.executeTakeFirst()
return deleted || null
}
const notifsForDelete = (
deleted: IndexedFollow,
replacedBy: IndexedFollow | null,
) => {
const toDelete = replacedBy ? [] : [deleted.uri]
return { notifs: [], toDelete }
}
const updateAggregates = async (db: DatabaseSchema, follow: IndexedFollow) => {
const followersCountQb = db
.insertInto('profile_agg')
.values({
did: follow.subjectDid,
followersCount: db
.selectFrom('follow')
.where('follow.subjectDid', '=', follow.subjectDid)
.select(countAll.as('count')),
})
.onConflict((oc) =>
oc.column('did').doUpdateSet({
followersCount: excluded(db, 'followersCount'),
}),
)
const followsCountQb = db
.insertInto('profile_agg')
.values({
did: follow.creator,
followsCount: db
.selectFrom('follow')
.where('follow.creator', '=', follow.creator)
.select(countAll.as('count')),
})
.onConflict((oc) =>
oc.column('did').doUpdateSet({
followsCount: excluded(db, 'followsCount'),
}),
)
await Promise.all([followersCountQb.execute(), followsCountQb.execute()])
}
export type PluginType = RecordProcessor<Follow.Record, IndexedFollow>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
updateAggregates,
})
}
export default makePlugin

128
packages/pds/src/app-view/services/indexing/plugins/like.ts

@ -1,128 +0,0 @@
import { AtUri } from '@atproto/syntax'
import { CID } from 'multiformats/cid'
import * as Like from '../../../../lexicon/types/app/bsky/feed/like'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { countAll, excluded } from '../../../../db/util'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
const lexId = lex.ids.AppBskyFeedLike
type IndexedLike = DatabaseSchemaType['like']
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: Like.Record,
timestamp: string,
): Promise<IndexedLike | null> => {
const inserted = await db
.insertInto('like')
.values({
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
subject: obj.subject.uri,
subjectCid: obj.subject.cid,
createdAt: toSimplifiedISOSafe(obj.createdAt),
indexedAt: timestamp,
})
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst()
return inserted || null
}
const findDuplicate = async (
db: DatabaseSchema,
uri: AtUri,
obj: Like.Record,
): Promise<AtUri | null> => {
const found = await db
.selectFrom('like')
.where('creator', '=', uri.host)
.where('subject', '=', obj.subject.uri)
.selectAll()
.executeTakeFirst()
return found ? new AtUri(found.uri) : null
}
const notifsForInsert = (obj: IndexedLike) => {
const subjectUri = new AtUri(obj.subject)
// prevent self-notifications
const isSelf = subjectUri.host === obj.creator
return isSelf
? []
: [
{
userDid: subjectUri.host,
author: obj.creator,
recordUri: obj.uri,
recordCid: obj.cid,
reason: 'like' as const,
reasonSubject: subjectUri.toString(),
indexedAt: obj.indexedAt,
},
]
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedLike | null> => {
const deleted = await db
.deleteFrom('like')
.where('uri', '=', uri.toString())
.returningAll()
.executeTakeFirst()
return deleted || null
}
const notifsForDelete = (
deleted: IndexedLike,
replacedBy: IndexedLike | null,
) => {
const toDelete = replacedBy ? [] : [deleted.uri]
return { notifs: [], toDelete }
}
const updateAggregates = async (db: DatabaseSchema, like: IndexedLike) => {
const likeCountQb = db
.insertInto('post_agg')
.values({
uri: like.subject,
likeCount: db
.selectFrom('like')
.where('like.subject', '=', like.subject)
.select(countAll.as('count')),
})
.onConflict((oc) =>
oc.column('uri').doUpdateSet({ likeCount: excluded(db, 'likeCount') }),
)
await likeCountQb.execute()
}
export type PluginType = RecordProcessor<Like.Record, IndexedLike>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
updateAggregates,
})
}
export default makePlugin

102
packages/pds/src/app-view/services/indexing/plugins/list-item.ts

@ -1,102 +0,0 @@
import { AtUri } from '@atproto/syntax'
import { CID } from 'multiformats/cid'
import * as ListItem from '../../../../lexicon/types/app/bsky/graph/listitem'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
import { InvalidRequestError } from '@atproto/xrpc-server'
import { toSimplifiedISOSafe } from '../util'
const lexId = lex.ids.AppBskyGraphListitem
type IndexedListItem = DatabaseSchemaType['list_item']
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: ListItem.Record,
timestamp: string,
): Promise<IndexedListItem | null> => {
const listUri = new AtUri(obj.list)
if (listUri.hostname !== uri.hostname) {
throw new InvalidRequestError(
'Creator of listitem does not match creator of list',
)
}
const inserted = await db
.insertInto('list_item')
.values({
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
subjectDid: obj.subject,
listUri: obj.list,
createdAt: toSimplifiedISOSafe(obj.createdAt),
indexedAt: timestamp,
})
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst()
return inserted || null
}
const findDuplicate = async (
db: DatabaseSchema,
_uri: AtUri,
obj: ListItem.Record,
): Promise<AtUri | null> => {
const found = await db
.selectFrom('list_item')
.where('listUri', '=', obj.list)
.where('subjectDid', '=', obj.subject)
.selectAll()
.executeTakeFirst()
return found ? new AtUri(found.uri) : null
}
const notifsForInsert = () => {
return []
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedListItem | null> => {
const deleted = await db
.deleteFrom('list_item')
.where('uri', '=', uri.toString())
.returningAll()
.executeTakeFirst()
return deleted || null
}
const notifsForDelete = (
deleted: IndexedListItem,
replacedBy: IndexedListItem | null,
) => {
const toDelete = replacedBy ? [] : [deleted.uri]
return { notifs: [], toDelete }
}
export type PluginType = RecordProcessor<ListItem.Record, IndexedListItem>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
})
}
export default makePlugin

86
packages/pds/src/app-view/services/indexing/plugins/list.ts

@ -1,86 +0,0 @@
import { AtUri } from '@atproto/syntax'
import { CID } from 'multiformats/cid'
import * as List from '../../../../lexicon/types/app/bsky/graph/list'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
import { toSimplifiedISOSafe } from '../util'
const lexId = lex.ids.AppBskyGraphList
type IndexedList = DatabaseSchemaType['list']
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: List.Record,
timestamp: string,
): Promise<IndexedList | null> => {
const inserted = await db
.insertInto('list')
.values({
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
name: obj.name,
purpose: obj.purpose,
description: obj.description,
descriptionFacets: obj.descriptionFacets
? JSON.stringify(obj.descriptionFacets)
: undefined,
avatarCid: obj.avatar?.ref.toString(),
createdAt: toSimplifiedISOSafe(obj.createdAt),
indexedAt: timestamp,
})
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst()
return inserted || null
}
const findDuplicate = async (): Promise<AtUri | null> => {
return null
}
const notifsForInsert = () => {
return []
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedList | null> => {
const deleted = await db
.deleteFrom('list')
.where('uri', '=', uri.toString())
.returningAll()
.executeTakeFirst()
return deleted || null
}
const notifsForDelete = () => {
return { notifs: [], toDelete: [] }
}
export type PluginType = RecordProcessor<List.Record, IndexedList>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
})
}
export default makePlugin

326
packages/pds/src/app-view/services/indexing/plugins/post.ts

@ -1,326 +0,0 @@
import { CID } from 'multiformats/cid'
import { AtUri } from '@atproto/syntax'
import { Record as PostRecord } from '../../../../lexicon/types/app/bsky/feed/post'
import { isMain as isEmbedImage } from '../../../../lexicon/types/app/bsky/embed/images'
import { isMain as isEmbedExternal } from '../../../../lexicon/types/app/bsky/embed/external'
import { isMain as isEmbedRecord } from '../../../../lexicon/types/app/bsky/embed/record'
import { isMain as isEmbedRecordWithMedia } from '../../../../lexicon/types/app/bsky/embed/recordWithMedia'
import {
isMention,
isLink,
} from '../../../../lexicon/types/app/bsky/richtext/facet'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
import { UserNotification } from '../../../../db/tables/user-notification'
import { countAll, excluded } from '../../../../db/util'
import { toSimplifiedISOSafe } from '../util'
type Post = DatabaseSchemaType['post']
type PostEmbedImage = DatabaseSchemaType['post_embed_image']
type PostEmbedExternal = DatabaseSchemaType['post_embed_external']
type PostEmbedRecord = DatabaseSchemaType['post_embed_record']
type PostAncestor = {
uri: string
height: number
}
type IndexedPost = {
post: Post
facets: { type: 'mention' | 'link'; value: string }[]
embeds?: (PostEmbedImage[] | PostEmbedExternal | PostEmbedRecord)[]
ancestors?: PostAncestor[]
}
const lexId = lex.ids.AppBskyFeedPost
const REPLY_NOTIF_DEPTH = 5
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: PostRecord,
timestamp: string,
): Promise<IndexedPost | null> => {
const post = {
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
text: obj.text,
createdAt: toSimplifiedISOSafe(obj.createdAt),
replyRoot: obj.reply?.root?.uri || null,
replyRootCid: obj.reply?.root?.cid || null,
replyParent: obj.reply?.parent?.uri || null,
replyParentCid: obj.reply?.parent?.cid || null,
indexedAt: timestamp,
}
const [insertedPost] = await Promise.all([
db
.insertInto('post')
.values(post)
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst(),
db
.insertInto('feed_item')
.values({
type: 'post',
uri: post.uri,
cid: post.cid,
postUri: post.uri,
originatorDid: post.creator,
sortAt:
post.indexedAt < post.createdAt ? post.indexedAt : post.createdAt,
})
.onConflict((oc) => oc.doNothing())
.executeTakeFirst(),
])
if (!insertedPost) {
return null // Post already indexed
}
const facets = (obj.facets || [])
.flatMap((facet) => facet.features)
.flatMap((feature) => {
if (isMention(feature)) {
return {
type: 'mention' as const,
value: feature.did,
}
}
if (isLink(feature)) {
return {
type: 'link' as const,
value: feature.uri,
}
}
return []
})
// Embed indices
const embeds: (PostEmbedImage[] | PostEmbedExternal | PostEmbedRecord)[] = []
const postEmbeds = separateEmbeds(obj.embed)
for (const postEmbed of postEmbeds) {
if (isEmbedImage(postEmbed)) {
const { images } = postEmbed
const imagesEmbed = images.map((img, i) => ({
postUri: uri.toString(),
position: i,
imageCid: img.image.ref.toString(),
alt: img.alt,
}))
embeds.push(imagesEmbed)
await db.insertInto('post_embed_image').values(imagesEmbed).execute()
} else if (isEmbedExternal(postEmbed)) {
const { external } = postEmbed
const externalEmbed = {
postUri: uri.toString(),
uri: external.uri,
title: external.title,
description: external.description,
thumbCid: external.thumb?.ref.toString() || null,
}
embeds.push(externalEmbed)
await db.insertInto('post_embed_external').values(externalEmbed).execute()
} else if (isEmbedRecord(postEmbed)) {
const { record } = postEmbed
const recordEmbed = {
postUri: uri.toString(),
embedUri: record.uri,
embedCid: record.cid,
}
embeds.push(recordEmbed)
await db.insertInto('post_embed_record').values(recordEmbed).execute()
}
}
return { post: insertedPost, facets, embeds }
}
const findDuplicate = async (): Promise<AtUri | null> => {
return null
}
const notifsForInsert = (obj: IndexedPost) => {
const notifs: UserNotification[] = []
const notified = new Set([obj.post.creator])
const maybeNotify = (notif: UserNotification) => {
if (!notified.has(notif.userDid)) {
notified.add(notif.userDid)
notifs.push(notif)
}
}
for (const facet of obj.facets) {
if (facet.type === 'mention') {
maybeNotify({
userDid: facet.value,
reason: 'mention',
reasonSubject: null,
author: obj.post.creator,
recordUri: obj.post.uri,
recordCid: obj.post.cid,
indexedAt: obj.post.indexedAt,
})
}
}
for (const embed of obj.embeds ?? []) {
if ('embedUri' in embed) {
const embedUri = new AtUri(embed.embedUri)
if (embedUri.collection === lex.ids.AppBskyFeedPost) {
maybeNotify({
userDid: embedUri.host,
reason: 'quote',
reasonSubject: embedUri.toString(),
author: obj.post.creator,
recordUri: obj.post.uri,
recordCid: obj.post.cid,
indexedAt: obj.post.indexedAt,
})
}
}
}
for (const ancestor of obj.ancestors ?? []) {
if (ancestor.uri === obj.post.uri) continue // no need to notify for own post
if (ancestor.height < REPLY_NOTIF_DEPTH) {
const ancestorUri = new AtUri(ancestor.uri)
maybeNotify({
userDid: ancestorUri.host,
reason: 'reply',
reasonSubject: ancestorUri.toString(),
author: obj.post.creator,
recordUri: obj.post.uri,
recordCid: obj.post.cid,
indexedAt: obj.post.indexedAt,
})
}
}
return notifs
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedPost | null> => {
const uriStr = uri.toString()
const [deleted] = await Promise.all([
db
.deleteFrom('post')
.where('uri', '=', uriStr)
.returningAll()
.executeTakeFirst(),
db.deleteFrom('feed_item').where('postUri', '=', uriStr).executeTakeFirst(),
])
const deletedEmbeds: (
| PostEmbedImage[]
| PostEmbedExternal
| PostEmbedRecord
)[] = []
const [deletedImgs, deletedExternals, deletedPosts] = await Promise.all([
db
.deleteFrom('post_embed_image')
.where('postUri', '=', uriStr)
.returningAll()
.execute(),
db
.deleteFrom('post_embed_external')
.where('postUri', '=', uriStr)
.returningAll()
.executeTakeFirst(),
db
.deleteFrom('post_embed_record')
.where('postUri', '=', uriStr)
.returningAll()
.executeTakeFirst(),
])
if (deletedImgs.length) {
deletedEmbeds.push(deletedImgs)
}
if (deletedExternals) {
deletedEmbeds.push(deletedExternals)
}
if (deletedPosts) {
deletedEmbeds.push(deletedPosts)
}
return deleted
? {
post: deleted,
facets: [], // Not used
embeds: deletedEmbeds,
}
: null
}
const notifsForDelete = (
deleted: IndexedPost,
replacedBy: IndexedPost | null,
) => {
const notifs = replacedBy ? notifsForInsert(replacedBy) : []
return {
notifs,
toDelete: [deleted.post.uri],
}
}
const updateAggregates = async (db: DatabaseSchema, postIdx: IndexedPost) => {
const replyCountQb = postIdx.post.replyParent
? db
.insertInto('post_agg')
.values({
uri: postIdx.post.replyParent,
replyCount: db
.selectFrom('post')
.where('post.replyParent', '=', postIdx.post.replyParent)
.select(countAll.as('count')),
})
.onConflict((oc) =>
oc
.column('uri')
.doUpdateSet({ replyCount: excluded(db, 'replyCount') }),
)
: null
const postsCountQb = db
.insertInto('profile_agg')
.values({
did: postIdx.post.creator,
postsCount: db
.selectFrom('post')
.where('post.creator', '=', postIdx.post.creator)
.select(countAll.as('count')),
})
.onConflict((oc) =>
oc.column('did').doUpdateSet({ postsCount: excluded(db, 'postsCount') }),
)
await Promise.all([replyCountQb?.execute(), postsCountQb.execute()])
}
export type PluginType = RecordProcessor<PostRecord, IndexedPost>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
updateAggregates,
})
}
export default makePlugin
function separateEmbeds(embed: PostRecord['embed']) {
if (!embed) {
return []
}
if (isEmbedRecordWithMedia(embed)) {
return [{ $type: lex.ids.AppBskyEmbedRecord, ...embed.record }, embed.media]
}
return [embed]
}

82
packages/pds/src/app-view/services/indexing/plugins/profile.ts

@ -1,82 +0,0 @@
import { AtUri } from '@atproto/syntax'
import { CID } from 'multiformats/cid'
import * as Profile from '../../../../lexicon/types/app/bsky/actor/profile'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
const lexId = lex.ids.AppBskyActorProfile
type IndexedProfile = DatabaseSchemaType['profile']
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: Profile.Record,
timestamp: string,
): Promise<IndexedProfile | null> => {
if (uri.rkey !== 'self') return null
const inserted = await db
.insertInto('profile')
.values({
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
displayName: obj.displayName,
description: obj.description,
avatarCid: obj.avatar?.ref.toString(),
bannerCid: obj.banner?.ref.toString(),
indexedAt: timestamp,
})
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst()
return inserted || null
}
const findDuplicate = async (): Promise<AtUri | null> => {
return null
}
const notifsForInsert = () => {
return []
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedProfile | null> => {
const deleted = await db
.deleteFrom('profile')
.where('uri', '=', uri.toString())
.returningAll()
.executeTakeFirst()
return deleted || null
}
const notifsForDelete = () => {
return { notifs: [], toDelete: [] }
}
export type PluginType = RecordProcessor<Profile.Record, IndexedProfile>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
})
}
export default makePlugin

153
packages/pds/src/app-view/services/indexing/plugins/repost.ts

@ -1,153 +0,0 @@
import { CID } from 'multiformats/cid'
import { AtUri } from '@atproto/syntax'
import * as Repost from '../../../../lexicon/types/app/bsky/feed/repost'
import * as lex from '../../../../lexicon/lexicons'
import Database from '../../../../db'
import {
DatabaseSchema,
DatabaseSchemaType,
} from '../../../../db/database-schema'
import { BackgroundQueue } from '../../../../event-stream/background-queue'
import RecordProcessor from '../processor'
import { countAll, excluded } from '../../../../db/util'
import { toSimplifiedISOSafe } from '../util'
const lexId = lex.ids.AppBskyFeedRepost
type IndexedRepost = DatabaseSchemaType['repost']
const insertFn = async (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: Repost.Record,
timestamp: string,
): Promise<IndexedRepost | null> => {
const repost = {
uri: uri.toString(),
cid: cid.toString(),
creator: uri.host,
subject: obj.subject.uri,
subjectCid: obj.subject.cid,
createdAt: toSimplifiedISOSafe(obj.createdAt),
indexedAt: timestamp,
}
const [inserted] = await Promise.all([
db
.insertInto('repost')
.values(repost)
.onConflict((oc) => oc.doNothing())
.returningAll()
.executeTakeFirst(),
db
.insertInto('feed_item')
.values({
type: 'repost',
uri: repost.uri,
cid: repost.cid,
postUri: repost.subject,
originatorDid: repost.creator,
sortAt:
repost.indexedAt < repost.createdAt
? repost.indexedAt
: repost.createdAt,
})
.onConflict((oc) => oc.doNothing())
.executeTakeFirst(),
])
return inserted || null
}
const findDuplicate = async (
db: DatabaseSchema,
uri: AtUri,
obj: Repost.Record,
): Promise<AtUri | null> => {
const found = await db
.selectFrom('repost')
.where('creator', '=', uri.host)
.where('subject', '=', obj.subject.uri)
.selectAll()
.executeTakeFirst()
return found ? new AtUri(found.uri) : null
}
const notifsForInsert = (obj: IndexedRepost) => {
const subjectUri = new AtUri(obj.subject)
// prevent self-notifications
const isSelf = subjectUri.host === obj.creator
return isSelf
? []
: [
{
userDid: subjectUri.host,
author: obj.creator,
recordUri: obj.uri,
recordCid: obj.cid,
reason: 'repost' as const,
reasonSubject: subjectUri.toString(),
indexedAt: obj.indexedAt,
},
]
}
const deleteFn = async (
db: DatabaseSchema,
uri: AtUri,
): Promise<IndexedRepost | null> => {
const uriStr = uri.toString()
const [deleted] = await Promise.all([
db
.deleteFrom('repost')
.where('uri', '=', uriStr)
.returningAll()
.executeTakeFirst(),
db.deleteFrom('feed_item').where('uri', '=', uriStr).executeTakeFirst(),
])
return deleted || null
}
const notifsForDelete = (
deleted: IndexedRepost,
replacedBy: IndexedRepost | null,
) => {
const toDelete = replacedBy ? [] : [deleted.uri]
return { notifs: [], toDelete }
}
const updateAggregates = async (db: DatabaseSchema, repost: IndexedRepost) => {
const repostCountQb = db
.insertInto('post_agg')
.values({
uri: repost.subject,
repostCount: db
.selectFrom('repost')
.where('repost.subject', '=', repost.subject)
.select(countAll.as('count')),
})
.onConflict((oc) =>
oc
.column('uri')
.doUpdateSet({ repostCount: excluded(db, 'repostCount') }),
)
await repostCountQb.execute()
}
export type PluginType = RecordProcessor<Repost.Record, IndexedRepost>
export const makePlugin = (
db: Database,
backgroundQueue: BackgroundQueue,
): PluginType => {
return new RecordProcessor(db, backgroundQueue, {
lexId,
insertFn,
findDuplicate,
deleteFn,
notifsForInsert,
notifsForDelete,
updateAggregates,
})
}
export default makePlugin

259
packages/pds/src/app-view/services/indexing/processor.ts

@ -1,259 +0,0 @@
import { CID } from 'multiformats/cid'
import { AtUri } from '@atproto/syntax'
import { cborToLexRecord } from '@atproto/repo'
import Database from '../../../db'
import DatabaseSchema from '../../../db/database-schema'
import { BackgroundQueue } from '../../../event-stream/background-queue'
import { lexicons } from '../../../lexicon/lexicons'
import { UserNotification } from '../../../db/tables/user-notification'
// @NOTE re: insertions and deletions. Due to how record updates are handled,
// (insertFn) should have the same effect as (insertFn -> deleteFn -> insertFn).
type RecordProcessorParams<T, S> = {
lexId: string
insertFn: (
db: DatabaseSchema,
uri: AtUri,
cid: CID,
obj: T,
timestamp: string,
) => Promise<S | null>
findDuplicate: (
db: DatabaseSchema,
uri: AtUri,
obj: T,
) => Promise<AtUri | null>
deleteFn: (db: DatabaseSchema, uri: AtUri) => Promise<S | null>
notifsForInsert: (obj: S) => UserNotification[]
notifsForDelete: (
prev: S,
replacedBy: S | null,
) => { notifs: UserNotification[]; toDelete: string[] }
updateAggregates?: (db: DatabaseSchema, obj: S) => Promise<void>
}
export class RecordProcessor<T, S> {
collection: string
db: DatabaseSchema
constructor(
private appDb: Database,
private backgroundQueue: BackgroundQueue,
private params: RecordProcessorParams<T, S>,
) {
this.db = appDb.db
this.collection = this.params.lexId
}
matchesSchema(obj: unknown): obj is T {
try {
this.assertValidRecord(obj)
return true
} catch {
return false
}
}
assertValidRecord(obj: unknown): void {
lexicons.assertValidRecord(this.params.lexId, obj)
}
async insertRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string) {
if (!this.matchesSchema(obj)) {
throw new Error(`Record does not match schema: ${this.params.lexId}`)
}
const inserted = await this.params.insertFn(
this.db,
uri,
cid,
obj,
timestamp,
)
// if this was a new record, return events
if (inserted) {
this.aggregateOnCommit(inserted)
await this.handleNotifs({ inserted })
return
}
// if duplicate, insert into duplicates table with no events
const found = await this.params.findDuplicate(this.db, uri, obj)
if (found && found.toString() !== uri.toString()) {
await this.db
.insertInto('duplicate_record')
.values({
uri: uri.toString(),
cid: cid.toString(),
duplicateOf: found.toString(),
indexedAt: timestamp,
})
.onConflict((oc) => oc.doNothing())
.execute()
}
}
// Currently using a very simple strategy for updates: purge the existing index
// for the uri then replace it. The main upside is that this allows the indexer
// for each collection to avoid bespoke logic for in-place updates, which isn't
// straightforward in the general case. We still get nice control over notifications.
async updateRecord(uri: AtUri, cid: CID, obj: unknown, timestamp: string) {
if (!this.matchesSchema(obj)) {
throw new Error(`Record does not match schema: ${this.params.lexId}`)
}
// If the updated record was a dupe, update dupe info for it
const dupe = await this.params.findDuplicate(this.db, uri, obj)
if (dupe) {
await this.db
.updateTable('duplicate_record')
.where('uri', '=', uri.toString())
.set({
cid: cid.toString(),
duplicateOf: dupe.toString(),
indexedAt: timestamp,
})
.execute()
} else {
await this.db
.deleteFrom('duplicate_record')
.where('uri', '=', uri.toString())
.execute()
}
const deleted = await this.params.deleteFn(this.db, uri)
if (!deleted) {
// If a record was updated but hadn't been indexed yet, treat it like a plain insert.
return this.insertRecord(uri, cid, obj, timestamp)
}
this.aggregateOnCommit(deleted)
const inserted = await this.params.insertFn(
this.db,
uri,
cid,
obj,
timestamp,
)
if (!inserted) {
throw new Error(
'Record update failed: removed from index but could not be replaced',
)
}
this.aggregateOnCommit(inserted)
await this.handleNotifs({ inserted, deleted })
}
async deleteRecord(uri: AtUri, cascading = false) {
await this.db
.deleteFrom('duplicate_record')
.where('uri', '=', uri.toString())
.execute()
const deleted = await this.params.deleteFn(this.db, uri)
if (!deleted) return
this.aggregateOnCommit(deleted)
if (cascading) {
await this.db
.deleteFrom('duplicate_record')
.where('duplicateOf', '=', uri.toString())
.execute()
await this.handleNotifs({ deleted })
return
} else {
const found = await this.db
.selectFrom('duplicate_record')
// @TODO remove ipld_block dependency from app-view
.innerJoin('ipld_block', (join) =>
join
.onRef('ipld_block.cid', '=', 'duplicate_record.cid')
.on('ipld_block.creator', '=', uri.host),
)
.where('duplicateOf', '=', uri.toString())
.orderBy('duplicate_record.indexedAt', 'asc')
.limit(1)
.selectAll()
.executeTakeFirst()
if (!found) {
return this.handleNotifs({ deleted })
}
const record = cborToLexRecord(found.content)
if (!this.matchesSchema(record)) {
return this.handleNotifs({ deleted })
}
const inserted = await this.params.insertFn(
this.db,
new AtUri(found.uri),
CID.parse(found.cid),
record,
found.indexedAt,
)
if (inserted) {
this.aggregateOnCommit(inserted)
}
await this.handleNotifs({ deleted, inserted: inserted ?? undefined })
}
}
async handleNotifs(op: { deleted?: S; inserted?: S }) {
let notifs: UserNotification[] = []
const runOnCommit: ((db: Database) => Promise<void>)[] = []
if (op.deleted) {
const forDelete = this.params.notifsForDelete(
op.deleted,
op.inserted ?? null,
)
if (forDelete.toDelete.length > 0) {
// Notifs can be deleted in background: they are expensive to delete and
// listNotifications already excludes notifs with missing records.
runOnCommit.push(async (db) => {
await db.db
.deleteFrom('user_notification')
.where('recordUri', 'in', forDelete.toDelete)
.execute()
})
}
notifs = forDelete.notifs
} else if (op.inserted) {
notifs = this.params.notifsForInsert(op.inserted)
}
if (notifs.length > 0) {
runOnCommit.push(async (db) => {
await db.db.insertInto('user_notification').values(notifs).execute()
})
}
if (runOnCommit.length) {
// Need to ensure notif deletion always happens before creation, otherwise delete may clobber in a race.
this.appDb.onCommit(() => {
this.backgroundQueue.add(async (db) => {
for (const fn of runOnCommit) {
await fn(db)
}
})
})
}
}
aggregateOnCommit(indexed: S) {
const { updateAggregates } = this.params
if (!updateAggregates) return
this.appDb.onCommit(() => {
this.backgroundQueue.add((db) => updateAggregates(db.db, indexed))
})
}
}
export default RecordProcessor
export class NoopProcessor extends RecordProcessor<unknown, unknown> {
constructor(
lexId: string,
appDb: Database,
backgroundQueue: BackgroundQueue,
) {
super(appDb, backgroundQueue, {
lexId,
insertFn: async () => null,
deleteFn: async () => null,
findDuplicate: async () => null,
notifsForInsert: () => [],
notifsForDelete: () => ({ notifs: [], toDelete: [] }),
})
}
}

16
packages/pds/src/app-view/services/indexing/util.ts

@ -1,16 +0,0 @@
import { isValidISODateString } from 'iso-datestring-validator'
// Normalize date strings to simplified ISO so that the lexical sort preserves temporal sort.
// Rather than failing on an invalid date format, returns valid unix epoch.
export function toSimplifiedISOSafe(dateStr: string) {
const date = new Date(dateStr)
if (isNaN(date.getTime())) {
return new Date(0).toISOString()
}
const iso = date.toISOString()
if (!isValidISODateString(iso)) {
// Occurs in rare cases, e.g. where resulting UTC year is negative. These also don't preserve lexical sort.
return new Date(0).toISOString()
}
return iso // YYYY-MM-DDTHH:mm:ss.sssZ
}

171
packages/pds/src/app-view/services/label/index.ts

@ -1,171 +0,0 @@
import { sql } from 'kysely'
import { AtUri } from '@atproto/syntax'
import Database from '../../../db'
import {
Label,
isSelfLabels,
} from '../../../lexicon/types/com/atproto/label/defs'
import { ids } from '../../../lexicon/lexicons'
import { LabelCache } from '../../../label-cache'
import { toSimplifiedISOSafe } from '../indexing/util'
export type Labels = Record<string, Label[]>
export class LabelService {
constructor(public db: Database, public cache: LabelCache) {}
static creator(cache: LabelCache) {
return (db: Database) => new LabelService(db, cache)
}
async formatAndCreate(
src: string,
uri: string,
cid: string | null,
labels: { create?: string[]; negate?: string[] },
) {
const { create = [], negate = [] } = labels
const toCreate = create.map((val) => ({
src,
uri,
cid: cid ?? undefined,
val,
neg: false,
cts: new Date().toISOString(),
}))
const toNegate = negate.map((val) => ({
src,
uri,
cid: cid ?? undefined,
val,
neg: true,
cts: new Date().toISOString(),
}))
await this.createLabels([...toCreate, ...toNegate])
}
async createLabels(labels: Label[]) {
if (labels.length < 1) return
const dbVals = labels.map((l) => ({
...l,
cid: l.cid ?? '',
neg: (l.neg ? 1 : 0) as 1 | 0,
}))
const { ref } = this.db.db.dynamic
const excluded = (col: string) => ref(`excluded.${col}`)
await this.db.db
.insertInto('label')
.values(dbVals)
.onConflict((oc) =>
oc.columns(['src', 'uri', 'cid', 'val']).doUpdateSet({
neg: sql`${excluded('neg')}`,
cts: sql`${excluded('cts')}`,
}),
)
.execute()
}
async getLabelsForUris(
subjects: string[],
opts?: {
includeNeg?: boolean
skipCache?: boolean
},
): Promise<Labels> {
if (subjects.length < 1) return {}
const res = opts?.skipCache
? await this.db.db
.selectFrom('label')
.where('label.uri', 'in', subjects)
.if(!opts?.includeNeg, (qb) => qb.where('neg', '=', 0))
.selectAll()
.execute()
: this.cache.forSubjects(subjects, opts?.includeNeg)
return res.reduce((acc, cur) => {
acc[cur.uri] ??= []
acc[cur.uri].push({
...cur,
cid: cur.cid === '' ? undefined : cur.cid,
neg: cur.neg === 1, // @TODO update in appview
})
return acc
}, {} as Labels)
}
// gets labels for any record. when did is present, combine labels for both did & profile record.
async getLabelsForSubjects(
subjects: string[],
opts?: {
includeNeg?: boolean
skipCache?: boolean
},
): Promise<Labels> {
if (subjects.length < 1) return {}
const expandedSubjects = subjects.flatMap((subject) => {
if (subject.startsWith('did:')) {
return [
subject,
AtUri.make(subject, ids.AppBskyActorProfile, 'self').toString(),
]
}
return subject
})
const labels = await this.getLabelsForUris(expandedSubjects, opts)
return Object.keys(labels).reduce((acc, cur) => {
const uri = cur.startsWith('at://') ? new AtUri(cur) : null
if (
uri &&
uri.collection === ids.AppBskyActorProfile &&
uri.rkey === 'self'
) {
// combine labels for profile + did
const did = uri.hostname
acc[did] ??= []
acc[did].push(...labels[cur])
}
acc[cur] ??= []
acc[cur].push(...labels[cur])
return acc
}, {} as Labels)
}
async getLabels(
subject: string,
opts?: {
includeNeg?: boolean
skipCache?: boolean
},
): Promise<Label[]> {
const labels = await this.getLabelsForUris([subject], opts)
return labels[subject] ?? []
}
async getLabelsForProfile(
did: string,
opts?: {
includeNeg?: boolean
skipCache?: boolean
},
): Promise<Label[]> {
const labels = await this.getLabelsForSubjects([did], opts)
return labels[did] ?? []
}
}
export function getSelfLabels(details: {
uri: string | null
cid: string | null
record: Record<string, unknown> | null
}): Label[] {
const { uri, cid, record } = details
if (!uri || !cid || !record) return []
if (!isSelfLabels(record.labels)) return []
const src = new AtUri(uri).host // record creator
const cts =
typeof record.createdAt === 'string'
? toSimplifiedISOSafe(record.createdAt)
: new Date(0).toISOString()
return record.labels.values.map(({ val }) => {
return { src, uri, cid, val, cts, neg: false }
})
}

4
packages/pds/src/event-stream/background-queue.ts → packages/pds/src/background.ts

@ -1,6 +1,6 @@
import PQueue from 'p-queue'
import Database from '../db'
import { dbLogger } from '../logger'
import Database from './db'
import { dbLogger } from './logger'
// A simple queue for in-process, out-of-band/backgrounded work

32
packages/pds/src/config.ts

@ -50,12 +50,6 @@ export interface ServerConfigValues {
moderationEmailAddress?: string
moderationEmailSmtpUrl?: string
hiveApiKey?: string
labelerDid: string
labelerKeywords: Record<string, string>
feedGenDid?: string
maxSubscriptionBuffer: number
repoBackfillLimitMs: number
sequencerLeaderLockId?: number
@ -179,12 +173,6 @@ export class ServerConfig {
const moderationEmailSmtpUrl =
process.env.MODERATION_EMAIL_SMTP_URL || undefined
const hiveApiKey = process.env.HIVE_API_KEY || undefined
const labelerDid = process.env.LABELER_DID || 'did:example:labeler'
const labelerKeywords = {}
const feedGenDid = process.env.FEED_GEN_DID
const dbPostgresUrl = process.env.DB_POSTGRES_URL
const dbPostgresSchema = process.env.DB_POSTGRES_SCHEMA
@ -271,10 +259,6 @@ export class ServerConfig {
emailNoReplyAddress,
moderationEmailAddress,
moderationEmailSmtpUrl,
hiveApiKey,
labelerDid,
labelerKeywords,
feedGenDid,
maxSubscriptionBuffer,
repoBackfillLimitMs,
sequencerLeaderLockId,
@ -467,22 +451,6 @@ export class ServerConfig {
return this.cfg.moderationEmailSmtpUrl
}
get hiveApiKey() {
return this.cfg.hiveApiKey
}
get labelerDid() {
return this.cfg.labelerDid
}
get labelerKeywords() {
return this.cfg.labelerKeywords
}
get feedGenDid() {
return this.cfg.feedGenDid
}
get maxSubscriptionBuffer() {
return this.cfg.maxSubscriptionBuffer
}

20
packages/pds/src/context.ts

@ -11,13 +11,10 @@ import { ServerMailer } from './mailer'
import { ModerationMailer } from './mailer/moderation'
import { BlobStore } from '@atproto/repo'
import { Services } from './services'
import { MessageDispatcher } from './event-stream/message-queue'
import { Sequencer, SequencerLeader } from './sequencer'
import { Labeler } from './labeler'
import { BackgroundQueue } from './event-stream/background-queue'
import { BackgroundQueue } from './background'
import DidSqlCache from './did-cache'
import { Crawlers } from './crawlers'
import { LabelCache } from './label-cache'
import { RuntimeFlags } from './runtime-flags'
export class AppContext {
@ -35,11 +32,8 @@ export class AppContext {
mailer: ServerMailer
moderationMailer: ModerationMailer
services: Services
messageDispatcher: MessageDispatcher
sequencer: Sequencer
sequencerLeader: SequencerLeader | null
labeler: Labeler
labelCache: LabelCache
runtimeFlags: RuntimeFlags
backgroundQueue: BackgroundQueue
appviewAgent: AtpAgent
@ -115,10 +109,6 @@ export class AppContext {
return this.opts.services
}
get messageDispatcher(): MessageDispatcher {
return this.opts.messageDispatcher
}
get sequencer(): Sequencer {
return this.opts.sequencer
}
@ -127,14 +117,6 @@ export class AppContext {
return this.opts.sequencerLeader
}
get labeler(): Labeler {
return this.opts.labeler
}
get labelCache(): LabelCache {
return this.opts.labelCache
}
get runtimeFlags(): RuntimeFlags {
return this.opts.runtimeFlags
}

6
packages/pds/src/db/database-schema.ts

@ -18,14 +18,11 @@ import * as deleteAccountToken from './tables/delete-account-token'
import * as moderation from './tables/moderation'
import * as mute from './tables/mute'
import * as listMute from './tables/list-mute'
import * as label from './tables/label'
import * as repoSeq from './tables/repo-seq'
import * as appMigration from './tables/app-migration'
import * as runtimeFlag from './tables/runtime-flag'
import * as appView from '../app-view/db'
export type DatabaseSchemaType = appView.DatabaseSchemaType &
runtimeFlag.PartialDB &
export type DatabaseSchemaType = runtimeFlag.PartialDB &
appMigration.PartialDB &
userAccount.PartialDB &
userState.PartialDB &
@ -46,7 +43,6 @@ export type DatabaseSchemaType = appView.DatabaseSchemaType &
moderation.PartialDB &
mute.PartialDB &
listMute.PartialDB &
label.PartialDB &
repoSeq.PartialDB
export type DatabaseSchema = Kysely<DatabaseSchemaType>

29
packages/pds/src/db/migrations/20230922T033938477Z-remove-appview.ts Normal file

@ -0,0 +1,29 @@
import { Kysely } from 'kysely'
export async function up(db: Kysely<unknown>): Promise<void> {
await db.schema.dropView('algo_whats_hot_view').materialized().execute()
await db.schema.dropTable('actor_block').execute()
await db.schema.dropTable('duplicate_record').execute()
await db.schema.dropTable('feed_generator').execute()
await db.schema.dropTable('feed_item').execute()
await db.schema.dropTable('follow').execute()
await db.schema.dropTable('label').execute()
await db.schema.dropTable('like').execute()
await db.schema.dropTable('list_item').execute()
await db.schema.dropTable('list').execute()
await db.schema.dropTable('post_agg').execute()
await db.schema.dropTable('post_embed_image').execute()
await db.schema.dropTable('post_embed_external').execute()
await db.schema.dropTable('post_embed_record').execute()
await db.schema.dropTable('post').execute()
await db.schema.dropTable('profile_agg').execute()
await db.schema.dropTable('profile').execute()
await db.schema.dropTable('repost').execute()
await db.schema.dropTable('subscription').execute()
await db.schema.dropTable('suggested_follow').execute()
await db.schema.dropTable('view_param').execute()
}
export async function down(_db: Kysely<unknown>): Promise<void> {
// Migration code
}

1
packages/pds/src/db/migrations/index.ts

@ -65,3 +65,4 @@ export * as _20230818T134357818Z from './20230818T134357818Z-runtime-flags'
export * as _20230824T182048120Z from './20230824T182048120Z-remove-post-hierarchy'
export * as _20230825T142507884Z from './20230825T142507884Z-blob-tempkey-idx'
export * as _20230828T153013575Z from './20230828T153013575Z-repo-history-rewrite'
export * as _20230922T033938477Z from './20230922T033938477Z-remove-appview'

25
packages/pds/src/db/periodic-moderation-action-reversal.ts

@ -4,7 +4,6 @@ import { Leader } from './leader'
import { dbLogger } from '../logger'
import AppContext from '../context'
import { ModerationActionRow } from '../services/moderation'
import { LabelService } from '../app-view/services/label'
export const MODERATION_ACTION_REVERSAL_ID = 1011
@ -14,39 +13,15 @@ export class PeriodicModerationActionReversal {
constructor(private appContext: AppContext) {}
// invert label creation & negations
async reverseLabels(labelTxn: LabelService, actionRow: ModerationActionRow) {
let uri: string
let cid: string | null = null
if (actionRow.subjectUri && actionRow.subjectCid) {
uri = actionRow.subjectUri
cid = actionRow.subjectCid
} else {
uri = actionRow.subjectDid
}
await labelTxn.formatAndCreate(this.appContext.cfg.labelerDid, uri, cid, {
create: actionRow.negateLabelVals
? actionRow.negateLabelVals.split(' ')
: undefined,
negate: actionRow.createLabelVals
? actionRow.createLabelVals.split(' ')
: undefined,
})
}
async revertAction(actionRow: ModerationActionRow) {
return this.appContext.db.transaction(async (dbTxn) => {
const moderationTxn = this.appContext.services.moderation(dbTxn)
const labelTxn = this.appContext.services.appView.label(dbTxn)
await moderationTxn.revertAction({
id: actionRow.id,
createdBy: actionRow.createdBy,
createdAt: new Date(),
reason: `[SCHEDULED_REVERSAL] Reverting action as originally scheduled`,
})
await this.reverseLabels(labelTxn, actionRow)
})
}

12
packages/pds/src/db/tables/label.ts

@ -1,12 +0,0 @@
export const tableName = 'label'
export interface Label {
src: string
uri: string
cid: string
val: string
neg: 0 | 1 // @TODO convert to boolean in app-view
cts: string
}
export type PartialDB = { [tableName]: Label }

48
packages/pds/src/event-stream/message-queue.ts

@ -1,48 +0,0 @@
import Database from '../db'
import { dbLogger as log } from '../logger'
import { MessageQueue, Listenable, Listener, MessageOfType } from './types'
// @NOTE A message dispatcher for loose coupling within db transactions.
// Messages are handled immediately. This should not be around for long.
export class MessageDispatcher implements MessageQueue {
private destroyed = false
private listeners: Map<string, Listener[]> = new Map()
async send(
tx: Database,
message: MessageOfType | MessageOfType[],
): Promise<void> {
if (this.destroyed) return
const messages = Array.isArray(message) ? message : [message]
for (const msg of messages) {
await this.handleMessage(tx, msg)
}
}
listen<T extends string, M extends MessageOfType<T>>(
topic: T,
listenable: Listenable<M>,
) {
const listeners = this.listeners.get(topic) ?? []
listeners.push(listenable.listener as Listener) // @TODO avoid upcast
this.listeners.set(topic, listeners)
}
destroy(): void {
this.destroyed = true
}
private async handleMessage(db: Database, message: MessageOfType) {
const listeners = this.listeners.get(message.type)
if (!listeners?.length) {
return log.error({ message }, `no listeners for event: ${message.type}`)
}
for (const listener of listeners) {
await listener({ message, db })
}
}
// Unused by MessageDispatcher
async processNext(): Promise<void> {}
async processAll(): Promise<void> {}
}

51
packages/pds/src/event-stream/messages.ts

@ -1,51 +0,0 @@
// Below specific to message dispatcher
import { CID } from 'multiformats/cid'
import { AtUri } from '@atproto/syntax'
import { WriteOpAction } from '@atproto/repo'
export type IndexRecord = {
type: 'index_record'
action: WriteOpAction.Create | WriteOpAction.Update
uri: AtUri
cid: CID
obj: unknown
timestamp: string
}
export type DeleteRecord = {
type: 'delete_record'
uri: AtUri
cascading: boolean
}
export type DeleteRepo = {
type: 'delete_repo'
did: string
}
export const indexRecord = (
uri: AtUri,
cid: CID,
obj: unknown,
action: WriteOpAction.Create | WriteOpAction.Update,
timestamp: string,
): IndexRecord => ({
type: 'index_record',
uri,
cid,
obj,
action,
timestamp,
})
export const deleteRecord = (uri: AtUri, cascading: boolean): DeleteRecord => ({
type: 'delete_record',
uri,
cascading,
})
export const deleteRepo = (did: string): DeleteRepo => ({
type: 'delete_repo',
did,
})

38
packages/pds/src/event-stream/types.ts

@ -1,38 +0,0 @@
import Database from '../db'
export type MessageOfType<T extends string = string> = {
type: T
[s: string]: unknown
}
export type Listener<M extends MessageOfType = MessageOfType> = (ctx: {
message: M
db: Database
}) => Promise<void | MessageOfType[]>
export interface Listenable<M extends MessageOfType = MessageOfType> {
listener: Listener<M>
}
export abstract class Consumer<M extends MessageOfType>
implements Listenable<M>
{
abstract dispatch(ctx: {
db: Database
message: M
}): Promise<void | MessageOfType[]>
get listener() {
return this.dispatch.bind(this)
}
}
export interface MessageQueue {
send(tx: Database, message: MessageOfType | MessageOfType[]): Promise<void>
listen<T extends string, M extends MessageOfType<T>>(
topic: T,
listenable: Listenable<M>,
): void
processNext(): Promise<void>
processAll(): Promise<void>
destroy(): void
}

37
packages/pds/src/index.ts

@ -21,7 +21,6 @@ import {
Options as XrpcServerOptions,
} from '@atproto/xrpc-server'
import { DAY, HOUR, MINUTE } from '@atproto/common'
import * as appviewConsumers from './app-view/event-stream/consumers'
import inProcessAppView from './app-view/api'
import API from './api'
import * as basicRoutes from './basic-routes'
@ -35,16 +34,13 @@ import { ServerConfig } from './config'
import { ServerMailer } from './mailer'
import { ModerationMailer } from './mailer/moderation'
import { createServer } from './lexicon'
import { MessageDispatcher } from './event-stream/message-queue'
import { createServices } from './services'
import { createHttpTerminator, HttpTerminator } from 'http-terminator'
import AppContext from './context'
import { Sequencer, SequencerLeader } from './sequencer'
import { Labeler, HiveLabeler, KeywordLabeler } from './labeler'
import { BackgroundQueue } from './event-stream/background-queue'
import { BackgroundQueue } from './background'
import DidSqlCache from './did-cache'
import { Crawlers } from './crawlers'
import { LabelCache } from './label-cache'
import { getRedisClient } from './redis'
import { RuntimeFlags } from './runtime-flags'
@ -95,7 +91,6 @@ export class PDS {
backupNameservers: config.handleResolveNameservers,
})
const messageDispatcher = new MessageDispatcher()
const sequencer = new Sequencer(db)
const sequencerLeader = config.sequencerLeaderEnabled
? new SequencerLeader(db, config.sequencerLeaderLockId)
@ -129,35 +124,11 @@ export class PDS {
config.crawlersToNotify ?? [],
)
let labeler: Labeler
if (config.hiveApiKey) {
labeler = new HiveLabeler({
db,
blobstore,
backgroundQueue,
labelerDid: config.labelerDid,
hiveApiKey: config.hiveApiKey,
keywords: config.labelerKeywords,
})
} else {
labeler = new KeywordLabeler({
db,
blobstore,
backgroundQueue,
labelerDid: config.labelerDid,
keywords: config.labelerKeywords,
})
}
const labelCache = new LabelCache(db)
const appviewAgent = new AtpAgent({ service: config.bskyAppViewEndpoint })
const services = createServices({
repoSigningKey,
messageDispatcher,
blobstore,
labeler,
labelCache,
appviewAgent,
appviewDid: config.bskyAppViewDid,
appviewCdnUrlPattern: config.bskyAppViewCdnUrlPattern,
@ -185,11 +156,8 @@ export class PDS {
didCache,
cfg: config,
auth,
messageDispatcher,
sequencer,
sequencerLeader,
labeler,
labelCache,
runtimeFlags,
services,
mailer,
@ -293,11 +261,9 @@ export class PDS {
}
}
}, 500)
appviewConsumers.listen(this.ctx)
this.ctx.sequencerLeader?.run()
await this.ctx.sequencer.start()
await this.ctx.db.startListeningToChannels()
this.ctx.labelCache.start()
await this.ctx.runtimeFlags.start()
const server = this.app.listen(this.ctx.cfg.port)
this.server = server
@ -309,7 +275,6 @@ export class PDS {
async destroy(): Promise<void> {
await this.ctx.runtimeFlags.destroy()
this.ctx.labelCache.stop()
await this.ctx.sequencerLeader?.destroy()
await this.terminator?.terminate()
await this.ctx.backgroundQueue.destroy()

90
packages/pds/src/label-cache.ts

@ -1,90 +0,0 @@
import { wait } from '@atproto/common'
import Database from './db'
import { Label } from './db/tables/label'
import { labelerLogger as log } from './logger'
export class LabelCache {
bySubject: Record<string, Label[]> = {}
latestLabel = ''
refreshes = 0
destroyed = false
constructor(public db: Database) {}
start() {
this.poll()
}
async fullRefresh() {
const allLabels = await this.db.db.selectFrom('label').selectAll().execute()
this.wipeCache()
this.processLabels(allLabels)
}
async partialRefresh() {
const labels = await this.db.db
.selectFrom('label')
.selectAll()
.where('cts', '>', this.latestLabel)
.execute()
this.processLabels(labels)
}
async poll() {
try {
if (this.destroyed) return
if (this.refreshes >= 120) {
await this.fullRefresh()
this.refreshes = 0
} else {
await this.partialRefresh()
this.refreshes++
}
} catch (err) {
log.error(
{ err, latestLabel: this.latestLabel, refreshes: this.refreshes },
'label cache failed to refresh',
)
}
await wait(500)
this.poll()
}
processLabels(labels: Label[]) {
for (const label of labels) {
if (label.cts > this.latestLabel) {
this.latestLabel = label.cts
}
this.bySubject[label.uri] ??= []
this.bySubject[label.uri].push(label)
}
}
wipeCache() {
this.bySubject = {}
}
stop() {
this.destroyed = true
}
forSubject(subject: string, includeNeg = false): Label[] {
const labels = this.bySubject[subject] ?? []
return includeNeg ? labels : labels.filter((l) => l.neg === 0)
}
forSubjects(subjects: string[], includeNeg?: boolean): Label[] {
let labels: Label[] = []
const alreadyAdded = new Set<string>()
for (const subject of subjects) {
if (alreadyAdded.has(subject)) {
continue
}
const subLabels = this.forSubject(subject, includeNeg)
labels = [...labels, ...subLabels]
alreadyAdded.add(subject)
}
return labels
}
}

72
packages/pds/src/labeler/base.ts

@ -1,72 +0,0 @@
import Database from '../db'
import { BlobStore, cidForRecord } from '@atproto/repo'
import { dedupe, getFieldsFromRecord } from './util'
import { AtUri } from '@atproto/syntax'
import { labelerLogger as log } from '../logger'
import { BackgroundQueue } from '../event-stream/background-queue'
import { CID } from 'multiformats/cid'
export abstract class Labeler {
public db: Database
public blobstore: BlobStore
public labelerDid: string
public backgroundQueue: BackgroundQueue
constructor(opts: {
db: Database
blobstore: BlobStore
labelerDid: string
backgroundQueue: BackgroundQueue
}) {
this.db = opts.db
this.blobstore = opts.blobstore
this.labelerDid = opts.labelerDid
this.backgroundQueue = opts.backgroundQueue
}
processRecord(uri: AtUri, obj: unknown) {
this.backgroundQueue.add(() =>
this.createAndStoreLabels(uri, obj).catch((err) => {
log.error(
{ err, uri: uri.toString(), record: obj },
'failed to label record',
)
}),
)
}
async createAndStoreLabels(uri: AtUri, obj: unknown): Promise<void> {
const labels = await this.labelRecord(obj)
if (labels.length < 1) return
const cid = await cidForRecord(obj)
const rows = labels.map((val) => ({
src: this.labelerDid,
uri: uri.toString(),
cid: cid.toString(),
val,
neg: 0 as const,
cts: new Date().toISOString(),
}))
await this.db.db
.insertInto('label')
.values(rows)
.onConflict((oc) => oc.doNothing())
.execute()
}
async labelRecord(obj: unknown): Promise<string[]> {
const { text, imgs } = getFieldsFromRecord(obj)
const txtLabels = await this.labelText(text.join(' '))
const imgLabels = await Promise.all(
imgs.map(async (cid) => this.labelImg(cid)),
)
return dedupe([...txtLabels, ...imgLabels.flat()])
}
abstract labelText(text: string): Promise<string[]>
abstract labelImg(cid: CID): Promise<string[]>
async processAll() {
await this.backgroundQueue.processAll()
}
}

191
packages/pds/src/labeler/hive.ts

@ -1,191 +0,0 @@
import stream from 'stream'
import axios from 'axios'
import FormData from 'form-data'
import { Labeler } from './base'
import Database from '../db'
import { BlobStore } from '@atproto/repo'
import { keywordLabeling } from './util'
import { BackgroundQueue } from '../event-stream/background-queue'
import { CID } from 'multiformats/cid'
const HIVE_ENDPOINT = 'https://api.thehive.ai/api/v2/task/sync'
export class HiveLabeler extends Labeler {
hiveApiKey: string
keywords: Record<string, string>
constructor(opts: {
db: Database
blobstore: BlobStore
backgroundQueue: BackgroundQueue
labelerDid: string
hiveApiKey: string
keywords: Record<string, string>
}) {
const { db, blobstore, backgroundQueue, labelerDid, hiveApiKey, keywords } =
opts
super({ db, blobstore, backgroundQueue, labelerDid })
this.hiveApiKey = hiveApiKey
this.keywords = keywords
}
async labelText(text: string): Promise<string[]> {
return keywordLabeling(this.keywords, text)
}
async labelImg(cid: CID): Promise<string[]> {
const stream = await this.blobstore.getStream(cid)
return labelBlob(stream, this.hiveApiKey)
}
}
export const labelBlob = async (
blob: stream.Readable,
hiveApiKey: string,
): Promise<string[]> => {
const classes = await makeHiveReq(blob, hiveApiKey)
return summarizeLabels(classes)
}
export const makeHiveReq = async (
blob: stream.Readable,
hiveApiKey: string,
): Promise<HiveRespClass[]> => {
const form = new FormData()
form.append('media', blob)
const res = await axios.post(HIVE_ENDPOINT, form, {
headers: {
'Content-Type': 'multipart/form-data',
authorization: `token ${hiveApiKey}`,
accept: 'application/json',
},
})
return respToClasses(res.data)
}
export const respToClasses = (res: HiveResp): HiveRespClass[] => {
const classes: HiveRespClass[] = []
for (const status of res.status) {
for (const out of status.response.output) {
for (const cls of out.classes) {
classes.push(cls)
}
}
}
return classes
}
// Matches only one (or none) of: porn, sexual, nudity
//
// porn: sexual and nudity. including both explicit activity or full-frontal and suggestive/intent
// sexual: sexually suggestive, not explicit; may include some forms of nudity
// nudity: non-sexual nudity (eg, artistic, possibly some photographic)
//
// hive docs/definitions: https://docs.thehive.ai/docs/sexual-content
export const sexualLabels = (classes: HiveRespClass[]): string[] => {
const scores = {}
for (const cls of classes) {
scores[cls.class] = cls.score
}
// first check if porn...
for (const pornClass of [
'yes_sexual_activity',
'animal_genitalia_and_human',
'yes_realistic_nsfw',
]) {
if (scores[pornClass] >= 0.9) {
return ['porn']
}
}
if (scores['general_nsfw'] >= 0.9) {
// special case for some anime examples
if (scores['animated_animal_genitalia'] >= 0.5) {
return ['porn']
}
// special case for some pornographic/explicit classic drawings
if (scores['yes_undressed'] >= 0.9 && scores['yes_sexual_activity'] > 0.9) {
return ['porn']
}
}
// then check for sexual suggestive (which may include nudity)...
for (const sexualClass of ['yes_sexual_intent', 'yes_sex_toy']) {
if (scores[sexualClass] >= 0.9) {
return ['sexual']
}
}
if (scores['yes_undressed'] >= 0.9) {
// special case for bondage examples
if (scores['yes_sex_toy'] > 0.75) {
return ['sexual']
}
}
// then non-sexual nudity...
for (const nudityClass of [
'yes_male_nudity',
'yes_female_nudity',
'yes_undressed',
]) {
if (scores[nudityClass] >= 0.9) {
return ['nudity']
}
}
// then finally flag remaining "underwear" images in to sexually suggestive
// (after non-sexual content already labeled above)
for (const nudityClass of ['yes_male_underwear', 'yes_female_underwear']) {
if (scores[nudityClass] >= 0.9) {
return ['sexual']
}
}
return []
}
// gore and violence: https://docs.thehive.ai/docs/class-descriptions-violence-gore
const labelForClass = {
very_bloody: 'gore',
human_corpse: 'corpse',
hanging: 'corpse',
}
const labelForClassLessSensitive = {
yes_self_harm: 'self-harm',
}
export const summarizeLabels = (classes: HiveRespClass[]): string[] => {
const labels: string[] = sexualLabels(classes)
for (const cls of classes) {
if (labelForClass[cls.class] && cls.score >= 0.9) {
labels.push(labelForClass[cls.class])
}
}
for (const cls of classes) {
if (labelForClassLessSensitive[cls.class] && cls.score >= 0.96) {
labels.push(labelForClassLessSensitive[cls.class])
}
}
return labels
}
type HiveResp = {
status: HiveRespStatus[]
}
type HiveRespStatus = {
response: {
output: HiveRespOutput[]
}
}
type HiveRespOutput = {
time: number
classes: HiveRespClass[]
}
type HiveRespClass = {
class: string
score: number
}

3
packages/pds/src/labeler/index.ts

@ -1,3 +0,0 @@
export * from './base'
export * from './hive'
export * from './keyword'

29
packages/pds/src/labeler/keyword.ts

@ -1,29 +0,0 @@
import { BlobStore } from '@atproto/repo'
import Database from '../db'
import { Labeler } from './base'
import { keywordLabeling } from './util'
import { BackgroundQueue } from '../event-stream/background-queue'
export class KeywordLabeler extends Labeler {
keywords: Record<string, string>
constructor(opts: {
db: Database
blobstore: BlobStore
backgroundQueue: BackgroundQueue
labelerDid: string
keywords: Record<string, string>
}) {
const { db, blobstore, backgroundQueue, labelerDid, keywords } = opts
super({ db, blobstore, backgroundQueue, labelerDid })
this.keywords = keywords
}
async labelText(text: string): Promise<string[]> {
return keywordLabeling(this.keywords, text)
}
async labelImg(): Promise<string[]> {
return []
}
}

109
packages/pds/src/labeler/util.ts

@ -1,109 +0,0 @@
import { CID } from 'multiformats/cid'
import * as lex from '../lexicon/lexicons'
import { Record as PostRecord } from '../lexicon/types/app/bsky/feed/post'
import { Record as ProfileRecord } from '../lexicon/types/app/bsky/actor/profile'
import { isMain as isEmbedImage } from '../lexicon/types/app/bsky/embed/images'
import { isMain as isEmbedExternal } from '../lexicon/types/app/bsky/embed/external'
import { isMain as isEmbedRecordWithMedia } from '../lexicon/types/app/bsky/embed/recordWithMedia'
type RecordFields = {
text: string[]
imgs: CID[]
}
export const getFieldsFromRecord = (record: unknown): RecordFields => {
if (isPost(record)) {
return getFieldsFromPost(record)
// @TODO add back in profile labeling
// } else if (isProfile(record)) {
// return getFieldsFromProfile(record)
} else {
return { text: [], imgs: [] }
}
}
export const getFieldsFromPost = (record: PostRecord): RecordFields => {
const text: string[] = []
const imgs: CID[] = []
text.push(record.text)
const embeds = separateEmbeds(record.embed)
for (const embed of embeds) {
if (isEmbedImage(embed)) {
for (const img of embed.images) {
imgs.push(img.image.ref)
text.push(img.alt)
}
} else if (isEmbedExternal(embed)) {
if (embed.external.thumb) {
imgs.push(embed.external.thumb.ref)
}
text.push(embed.external.title)
text.push(embed.external.description)
}
}
return { text, imgs }
}
export const getFieldsFromProfile = (record: ProfileRecord): RecordFields => {
const text: string[] = []
const imgs: CID[] = []
if (record.displayName) {
text.push(record.displayName)
}
if (record.description) {
text.push(record.description)
}
if (record.avatar) {
imgs.push(record.avatar.ref)
}
if (record.banner) {
imgs.push(record.banner.ref)
}
return { text, imgs }
}
export const dedupe = (str: string[]): string[] => {
const set = new Set(str)
return [...set]
}
export const isPost = (obj: unknown): obj is PostRecord => {
return isRecordType(obj, 'app.bsky.feed.post')
}
export const isProfile = (obj: unknown): obj is ProfileRecord => {
return isRecordType(obj, 'app.bsky.actor.profile')
}
export const isRecordType = (obj: unknown, lexId: string): boolean => {
try {
lex.lexicons.assertValidRecord(lexId, obj)
return true
} catch {
return false
}
}
export const keywordLabeling = (
keywords: Record<string, string>,
text: string,
): string[] => {
const lowerText = text.toLowerCase()
const labels: string[] = []
for (const word of Object.keys(keywords)) {
if (lowerText.includes(word)) {
labels.push(keywords[word])
}
}
return labels
}
const separateEmbeds = (embed: PostRecord['embed']) => {
if (!embed) {
return []
}
if (isEmbedRecordWithMedia(embed)) {
return [{ $type: lex.ids.AppBskyEmbedRecord, ...embed.record }, embed.media]
}
return [embed]
}

12
packages/pds/src/lexicon/index.ts

@ -107,7 +107,6 @@ import * as AppBskyNotificationGetUnreadCount from './types/app/bsky/notificatio
import * as AppBskyNotificationListNotifications from './types/app/bsky/notification/listNotifications'
import * as AppBskyNotificationRegisterPush from './types/app/bsky/notification/registerPush'
import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen'
import * as AppBskyUnspeccedApplyLabels from './types/app/bsky/unspecced/applyLabels'
import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular'
import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators'
import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton'
@ -1402,17 +1401,6 @@ export class UnspeccedNS {
this._server = server
}
applyLabels<AV extends AuthVerifier>(
cfg: ConfigOf<
AV,
AppBskyUnspeccedApplyLabels.Handler<ExtractAuth<AV>>,
AppBskyUnspeccedApplyLabels.HandlerReqCtx<ExtractAuth<AV>>
>,
) {
const nsid = 'app.bsky.unspecced.applyLabels' // @ts-ignore
return this._server.xrpc.method(nsid, cfg)
}
getPopular<AV extends AuthVerifier>(
cfg: ConfigOf<
AV,

27
packages/pds/src/lexicon/lexicons.ts

@ -6905,32 +6905,6 @@ export const schemaDict = {
},
},
},
AppBskyUnspeccedApplyLabels: {
lexicon: 1,
id: 'app.bsky.unspecced.applyLabels',
defs: {
main: {
type: 'procedure',
description: 'Allow a labeler to apply labels directly.',
input: {
encoding: 'application/json',
schema: {
type: 'object',
required: ['labels'],
properties: {
labels: {
type: 'array',
items: {
type: 'ref',
ref: 'lex:com.atproto.label.defs#label',
},
},
},
},
},
},
},
},
AppBskyUnspeccedDefs: {
lexicon: 1,
id: 'app.bsky.unspecced.defs',
@ -7359,7 +7333,6 @@ export const ids = {
AppBskyNotificationRegisterPush: 'app.bsky.notification.registerPush',
AppBskyNotificationUpdateSeen: 'app.bsky.notification.updateSeen',
AppBskyRichtextFacet: 'app.bsky.richtext.facet',
AppBskyUnspeccedApplyLabels: 'app.bsky.unspecced.applyLabels',
AppBskyUnspeccedDefs: 'app.bsky.unspecced.defs',
AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular',
AppBskyUnspeccedGetPopularFeedGenerators:

39
packages/pds/src/lexicon/types/app/bsky/unspecced/applyLabels.ts

@ -1,39 +0,0 @@
/**
* GENERATED CODE - DO NOT MODIFY
*/
import express from 'express'
import { ValidationResult, BlobRef } from '@atproto/lexicon'
import { lexicons } from '../../../../lexicons'
import { isObj, hasProp } from '../../../../util'
import { CID } from 'multiformats/cid'
import { HandlerAuth } from '@atproto/xrpc-server'
import * as ComAtprotoLabelDefs from '../../../com/atproto/label/defs'
export interface QueryParams {}
export interface InputSchema {
labels: ComAtprotoLabelDefs.Label[]
[k: string]: unknown
}
export interface HandlerInput {
encoding: 'application/json'
body: InputSchema
}
export interface HandlerError {
status: number
message?: string
}
export type HandlerOutput = HandlerError | void
export type HandlerReqCtx<HA extends HandlerAuth = never> = {
auth: HA
params: QueryParams
input: HandlerInput
req: express.Request
res: express.Response
}
export type Handler<HA extends HandlerAuth = never> = (
ctx: HandlerReqCtx<HA>,
) => Promise<HandlerOutput> | HandlerOutput

30
packages/pds/src/services/account/index.ts

@ -1,18 +1,17 @@
import { WhereInterface, sql } from 'kysely'
import { sql } from 'kysely'
import { dbLogger as log } from '../../logger'
import Database from '../../db'
import * as scrypt from '../../db/scrypt'
import { UserAccountEntry } from '../../db/tables/user-account'
import { DidHandle } from '../../db/tables/did-handle'
import { RepoRoot } from '../../db/tables/repo-root'
import { DbRef, countAll, notSoftDeletedClause } from '../../db/util'
import { countAll, notSoftDeletedClause } from '../../db/util'
import { getUserSearchQueryPg, getUserSearchQuerySqlite } from '../util/search'
import { paginate, TimeCidKeyset } from '../../db/pagination'
import * as sequencer from '../../sequencer'
import { AppPassword } from '../../lexicon/types/com/atproto/server/createAppPassword'
import { randomStr } from '@atproto/crypto'
import { InvalidRequestError } from '@atproto/xrpc-server'
import { NotEmptyArray } from '@atproto/common'
export class AccountService {
constructor(public db: Database) {}
@ -355,27 +354,6 @@ export class AccountService {
.execute()
}
whereNotMuted<W extends WhereInterface<any, any>>(
qb: W,
requester: string,
refs: NotEmptyArray<DbRef>,
) {
const subjectRefs = sql.join(refs)
const actorMute = this.db.db
.selectFrom('mute')
.where('mutedByDid', '=', requester)
.where('did', 'in', sql`(${subjectRefs})`)
.select('did as muted')
const listMute = this.db.db
.selectFrom('list_item')
.innerJoin('list_mute', 'list_mute.listUri', 'list_item.listUri')
.where('list_mute.mutedByDid', '=', requester)
.whereRef('list_item.subjectDid', 'in', sql`(${subjectRefs})`)
.select('list_item.subjectDid as muted')
// Splitting the mute from list-mute checks seems to be more flexible for the query-planner and often quicker
return qb.whereNotExists(actorMute).whereNotExists(listMute)
}
async search(opts: {
searchField?: 'did' | 'handle'
term: string
@ -397,13 +375,9 @@ export class AccountService {
const builder =
this.db.dialect === 'pg'
? getUserSearchQueryPg(this.db, opts)
.innerJoin('repo_root', 'repo_root.did', 'did_handle.did')
.selectAll('did_handle')
.selectAll('repo_root')
.select('results.distance as distance')
: getUserSearchQuerySqlite(this.db, opts)
.leftJoin('profile', 'profile.creator', 'did_handle.did') // @TODO leaky, for getUserSearchQuerySqlite()
.innerJoin('repo_root', 'repo_root.did', 'did_handle.did')
.selectAll('did_handle')
.selectAll('repo_root')
.select(sql<number>`0`.as('distance'))

27
packages/pds/src/services/index.ts

@ -2,26 +2,18 @@ import { AtpAgent } from '@atproto/api'
import * as crypto from '@atproto/crypto'
import { BlobStore } from '@atproto/repo'
import Database from '../db'
import { MessageDispatcher } from '../event-stream/message-queue'
import { AccountService } from './account'
import { AuthService } from './auth'
import { RecordService } from './record'
import { RepoService } from './repo'
import { ModerationService } from './moderation'
import { IndexingService } from '../app-view/services/indexing'
import { Labeler } from '../labeler'
import { LabelService } from '../app-view/services/label'
import { BackgroundQueue } from '../event-stream/background-queue'
import { BackgroundQueue } from '../background'
import { Crawlers } from '../crawlers'
import { LabelCache } from '../label-cache'
import { LocalService } from './local'
export function createServices(resources: {
repoSigningKey: crypto.Keypair
messageDispatcher: MessageDispatcher
blobstore: BlobStore
labeler: Labeler
labelCache: LabelCache
appviewAgent?: AtpAgent
appviewDid?: string
appviewCdnUrlPattern?: string
@ -30,10 +22,7 @@ export function createServices(resources: {
}): Services {
const {
repoSigningKey,
messageDispatcher,
blobstore,
labeler,
labelCache,
appviewAgent,
appviewDid,
appviewCdnUrlPattern,
@ -43,14 +32,12 @@ export function createServices(resources: {
return {
account: AccountService.creator(),
auth: AuthService.creator(),
record: RecordService.creator(messageDispatcher),
record: RecordService.creator(),
repo: RepoService.creator(
repoSigningKey,
messageDispatcher,
blobstore,
backgroundQueue,
crawlers,
labeler,
),
local: LocalService.creator(
repoSigningKey,
@ -58,11 +45,7 @@ export function createServices(resources: {
appviewDid,
appviewCdnUrlPattern,
),
moderation: ModerationService.creator(messageDispatcher, blobstore),
appView: {
indexing: IndexingService.creator(backgroundQueue),
label: LabelService.creator(labelCache),
},
moderation: ModerationService.creator(blobstore),
}
}
@ -73,10 +56,6 @@ export type Services = {
repo: FromDb<RepoService>
local: FromDb<LocalService>
moderation: FromDb<ModerationService>
appView: {
indexing: FromDb<IndexingService>
label: FromDb<LabelService>
}
}
type FromDb<T> = (db: Database) => T

16
packages/pds/src/services/moderation/index.ts

@ -4,7 +4,6 @@ import { BlobStore } from '@atproto/repo'
import { AtUri } from '@atproto/syntax'
import { InvalidRequestError } from '@atproto/xrpc-server'
import Database from '../../db'
import { MessageQueue } from '../../event-stream/types'
import { ModerationAction, ModerationReport } from '../../db/tables/moderation'
import { RecordService } from '../record'
import { ModerationViews } from './views'
@ -13,21 +12,16 @@ import { TAKEDOWN } from '../../lexicon/types/com/atproto/admin/defs'
import { addHoursToDate } from '../../util/date'
export class ModerationService {
constructor(
public db: Database,
public messageDispatcher: MessageQueue,
public blobstore: BlobStore,
) {}
constructor(public db: Database, public blobstore: BlobStore) {}
static creator(messageDispatcher: MessageQueue, blobstore: BlobStore) {
return (db: Database) =>
new ModerationService(db, messageDispatcher, blobstore)
static creator(blobstore: BlobStore) {
return (db: Database) => new ModerationService(db, blobstore)
}
views = new ModerationViews(this.db, this.messageDispatcher)
views = new ModerationViews(this.db)
services = {
record: RecordService.creator(this.messageDispatcher),
record: RecordService.creator(),
}
async getAction(id: number): Promise<ModerationActionRow | undefined> {

45
packages/pds/src/services/moderation/views.ts

@ -2,7 +2,6 @@ import { Selectable } from 'kysely'
import { ArrayEl, cborBytesToRecord } from '@atproto/common'
import { AtUri } from '@atproto/syntax'
import Database from '../../db'
import { MessageQueue } from '../../event-stream/types'
import { DidHandle } from '../../db/tables/did-handle'
import { RepoRoot } from '../../db/tables/repo-root'
import {
@ -17,19 +16,18 @@ import {
BlobView,
} from '../../lexicon/types/com/atproto/admin/defs'
import { OutputSchema as ReportOutput } from '../../lexicon/types/com/atproto/moderation/createReport'
import { Label } from '../../lexicon/types/com/atproto/label/defs'
import { ModerationAction } from '../../db/tables/moderation'
import { AccountService } from '../account'
import { RecordService } from '../record'
import { ModerationReportRowWithHandle } from '.'
import { getSelfLabels } from '../../app-view/services/label'
import { ids } from '../../lexicon/lexicons'
export class ModerationViews {
constructor(private db: Database, private messageDispatcher: MessageQueue) {}
constructor(private db: Database) {}
services = {
account: AccountService.creator(),
record: RecordService.creator(this.messageDispatcher),
record: RecordService.creator(),
}
repo(result: RepoResult, opts: ModViewOptions): Promise<RepoView>
@ -45,10 +43,15 @@ export class ModerationViews {
await this.db.db
.selectFrom('did_handle')
.leftJoin('user_account', 'user_account.did', 'did_handle.did')
.leftJoin('profile', 'profile.creator', 'did_handle.did')
.leftJoin('record as profile_record', (join) =>
join
.onRef('profile_record.did', '=', 'did_handle.did')
.on('profile_record.collection', '=', ids.AppBskyActorProfile)
.on('profile_record.rkey', '=', 'self'),
)
.leftJoin('ipld_block as profile_block', (join) =>
join
.onRef('profile_block.cid', '=', 'profile.cid')
.onRef('profile_block.cid', '=', 'profile_record.cid')
.onRef('profile_block.creator', '=', 'did_handle.did'),
)
.where(
@ -143,10 +146,9 @@ export class ModerationViews {
.execute(),
this.services.account(this.db).getAccountInviteCodes(repo.did),
])
const [reports, actions, labels] = await Promise.all([
const [reports, actions] = await Promise.all([
this.report(reportResults),
this.action(actionResults),
this.labels(repo.did),
])
return {
...repo,
@ -156,7 +158,6 @@ export class ModerationViews {
actions,
},
invites: inviteCodes,
labels,
}
}
@ -270,17 +271,11 @@ export class ModerationViews {
.selectAll()
.execute(),
])
const [reports, actions, blobs, labels] = await Promise.all([
const [reports, actions, blobs] = await Promise.all([
this.report(reportResults),
this.action(actionResults),
this.blob(record.blobCids),
this.labels(record.uri),
])
const selfLabels = getSelfLabels({
uri: result.uri,
cid: result.cid,
record: result.value as Record<string, unknown>,
})
return {
...record,
blobs,
@ -289,7 +284,6 @@ export class ModerationViews {
reports,
actions,
},
labels: [...labels, ...selfLabels],
}
}
@ -609,21 +603,6 @@ export class ModerationViews {
}
})
}
// @TODO: call into label service instead on AppView
async labels(subject: string, includeNeg?: boolean): Promise<Label[]> {
const res = await this.db.db
.selectFrom('label')
.where('label.uri', '=', subject)
.if(!includeNeg, (qb) => qb.where('neg', '=', 0))
.selectAll()
.execute()
return res.map((l) => ({
...l,
cid: l.cid === '' ? undefined : l.cid,
neg: l.neg === 1,
}))
}
}
type RepoResult = DidHandle & RepoRoot

30
packages/pds/src/services/record/index.ts

@ -6,19 +6,13 @@ import { dbLogger as log } from '../../logger'
import Database from '../../db'
import { notSoftDeletedClause } from '../../db/util'
import { Backlink } from '../../db/tables/backlink'
import { MessageQueue } from '../../event-stream/types'
import {
indexRecord,
deleteRecord,
deleteRepo,
} from '../../event-stream/messages'
import { ids } from '../../lexicon/lexicons'
export class RecordService {
constructor(public db: Database, public messageDispatcher: MessageQueue) {}
constructor(public db: Database) {}
static creator(messageDispatcher: MessageQueue) {
return (db: Database) => new RecordService(db, messageDispatcher)
static creator() {
return (db: Database) => new RecordService(db)
}
async indexRecord(
@ -70,30 +64,19 @@ export class RecordService {
}
await this.addBacklinks(backlinks)
// Send to indexers
await this.messageDispatcher.send(
this.db,
indexRecord(uri, cid, obj, action, record.indexedAt),
)
log.info({ uri }, 'indexed record')
}
async deleteRecord(uri: AtUri, cascading = false) {
async deleteRecord(uri: AtUri) {
this.db.assertTransaction()
log.debug({ uri }, 'deleting indexed record')
const deleteQuery = this.db.db
.deleteFrom('record')
.where('uri', '=', uri.toString())
.execute()
await this.db.db
const backlinkQuery = this.db.db
.deleteFrom('backlink')
.where('uri', '=', uri.toString())
.execute()
await Promise.all([
this.messageDispatcher.send(this.db, deleteRecord(uri, cascading)),
deleteQuery,
])
await Promise.all([deleteQuery.execute(), backlinkQuery.execute()])
log.info({ uri }, 'deleted indexed record')
}
@ -233,7 +216,6 @@ export class RecordService {
async deleteForActor(did: string) {
// Not done in transaction because it would be too long, prone to contention.
// Also, this can safely be run multiple times if it fails.
await this.messageDispatcher.send(this.db, deleteRepo(did)) // Needs record table
await this.db.db.deleteFrom('record').where('did', '=', did).execute()
await this.db.db
.deleteFrom('user_notification')

2
packages/pds/src/services/repo/blobs.ts

@ -13,7 +13,7 @@ import { Blob as BlobTable } from '../../db/tables/blob'
import * as img from '../../image'
import { BlobRef } from '@atproto/lexicon'
import { PreparedDelete, PreparedUpdate } from '../../repo'
import { BackgroundQueue } from '../../event-stream/background-queue'
import { BackgroundQueue } from '../../background'
export class RepoBlobs {
constructor(

31
packages/pds/src/services/repo/index.ts

@ -4,7 +4,6 @@ import { BlobStore, CommitData, Repo, WriteOpAction } from '@atproto/repo'
import { InvalidRequestError } from '@atproto/xrpc-server'
import { AtUri } from '@atproto/syntax'
import Database from '../../db'
import { MessageQueue } from '../../event-stream/types'
import SqlRepoStorage from '../../sql-repo-storage'
import {
BadCommitSwapError,
@ -16,9 +15,8 @@ import { RepoBlobs } from './blobs'
import { createWriteToOp, writeToOp } from '../../repo'
import { RecordService } from '../record'
import * as sequencer from '../../sequencer'
import { Labeler } from '../../labeler'
import { wait } from '@atproto/common'
import { BackgroundQueue } from '../../event-stream/background-queue'
import { BackgroundQueue } from '../../background'
import { Crawlers } from '../../crawlers'
export class RepoService {
@ -27,37 +25,25 @@ export class RepoService {
constructor(
public db: Database,
public repoSigningKey: crypto.Keypair,
public messageDispatcher: MessageQueue,
public blobstore: BlobStore,
public backgroundQueue: BackgroundQueue,
public crawlers: Crawlers,
public labeler: Labeler,
) {
this.blobs = new RepoBlobs(db, blobstore, backgroundQueue)
}
static creator(
keypair: crypto.Keypair,
messageDispatcher: MessageQueue,
blobstore: BlobStore,
backgroundQueue: BackgroundQueue,
crawlers: Crawlers,
labeler: Labeler,
) {
return (db: Database) =>
new RepoService(
db,
keypair,
messageDispatcher,
blobstore,
backgroundQueue,
crawlers,
labeler,
)
new RepoService(db, keypair, blobstore, backgroundQueue, crawlers)
}
services = {
record: RecordService.creator(this.messageDispatcher),
record: RecordService.creator(),
}
private async serviceTx<T>(
@ -68,11 +54,9 @@ export class RepoService {
const srvc = new RepoService(
dbTxn,
this.repoSigningKey,
this.messageDispatcher,
this.blobstore,
this.backgroundQueue,
this.crawlers,
this.labeler,
)
return fn(srvc)
})
@ -294,15 +278,6 @@ export class RepoService {
this.backgroundQueue.add(async () => {
await this.crawlers.notifyOfUpdate()
})
writes.forEach((write) => {
if (
write.action === WriteOpAction.Create ||
write.action === WriteOpAction.Update
) {
// @TODO move to appview
this.labeler.processRecord(write.uri, write.record)
}
})
})
const seqEvt = await sequencer.formatSeqCommit(did, commitData, writes)

105
packages/pds/src/services/util/search.ts

@ -1,7 +1,7 @@
import { sql } from 'kysely'
import { InvalidRequestError } from '@atproto/xrpc-server'
import Database from '../../db'
import { notSoftDeletedClause, DbRef, AnyQb } from '../../db/util'
import { notSoftDeletedClause, DbRef } from '../../db/util'
import { GenericKeyset, paginate } from '../../db/pagination'
// @TODO utilized in both pds and app-view
@ -19,58 +19,13 @@ export const getUserSearchQueryPg = (
const { term, limit, cursor, includeSoftDeleted } = opts
// Matching user accounts based on handle
const distanceAccount = distance(term, ref('handle'))
let accountsQb = getMatchingAccountsQb(db, { term, includeSoftDeleted })
accountsQb = paginate(accountsQb, {
const accountsQb = getMatchingAccountsQb(db, { term, includeSoftDeleted })
return paginate(accountsQb, {
limit,
cursor,
direction: 'asc',
keyset: new SearchKeyset(distanceAccount, ref('handle')),
})
// Matching profiles based on display name
const distanceProfile = distance(term, ref('displayName'))
let profilesQb = getMatchingProfilesQb(db, { term, includeSoftDeleted })
profilesQb = paginate(
profilesQb.innerJoin('did_handle', 'did_handle.did', 'profile.creator'), // for handle pagination
{
limit,
cursor,
direction: 'asc',
keyset: new SearchKeyset(distanceProfile, ref('handle')),
},
)
// Combine and paginate result set
return paginate(combineAccountsAndProfilesQb(db, accountsQb, profilesQb), {
limit,
cursor,
direction: 'asc',
keyset: new SearchKeyset(ref('distance'), ref('handle')),
})
}
// Takes maximal advantage of trigram index at the expense of ability to paginate.
export const getUserSearchQuerySimplePg = (
db: Database,
opts: {
term: string
limit: number
},
) => {
const { ref } = db.db.dynamic
const { term, limit } = opts
// Matching user accounts based on handle
const accountsQb = getMatchingAccountsQb(db, { term })
.orderBy('distance', 'asc')
.limit(limit)
// Matching profiles based on display name
const profilesQb = getMatchingProfilesQb(db, { term })
.orderBy('distance', 'asc')
.limit(limit)
// Combine and paginate result set
return paginate(combineAccountsAndProfilesQb(db, accountsQb, profilesQb), {
limit,
direction: 'asc',
keyset: new SearchKeyset(ref('distance'), ref('handle')),
})
}
// Matching user accounts based on handle
@ -91,51 +46,6 @@ const getMatchingAccountsQb = (
.select(['did_handle.did as did', distanceAccount.as('distance')])
}
// Matching profiles based on display name
const getMatchingProfilesQb = (
db: Database,
opts: { term: string; includeSoftDeleted?: boolean },
) => {
const { ref } = db.db.dynamic
const { term, includeSoftDeleted } = opts
const distanceProfile = distance(term, ref('displayName'))
return db.db
.selectFrom('profile')
.innerJoin('repo_root', 'repo_root.did', 'profile.creator')
.if(!includeSoftDeleted, (qb) =>
qb.where(notSoftDeletedClause(ref('repo_root'))),
)
.where(similar(term, ref('displayName'))) // Coarse filter engaging trigram index
.select(['profile.creator as did', distanceProfile.as('distance')])
}
// Combine profile and account result sets
const combineAccountsAndProfilesQb = (
db: Database,
accountsQb: AnyQb,
profilesQb: AnyQb,
) => {
// Combine user account and profile results, taking best matches from each
const emptyQb = db.db
.selectFrom('user_account')
.where(sql`1 = 0`)
.select([sql.literal('').as('did'), sql<number>`0`.as('distance')])
const resultsQb = db.db
.selectFrom(
emptyQb
.unionAll(sql`${accountsQb}`) // The sql`` is adding parens
.unionAll(sql`${profilesQb}`)
.as('accounts_and_profiles'),
)
.selectAll()
.distinctOn('did') // Per did, take whichever of account and profile distance is best
.orderBy('did')
.orderBy('distance')
return db.db
.selectFrom(resultsQb.as('results'))
.innerJoin('did_handle', 'did_handle.did', 'results.did')
}
export const getUserSearchQuerySqlite = (
db: Database,
opts: {
@ -160,7 +70,10 @@ export const getUserSearchQuerySqlite = (
if (!safeWords.length) {
// Return no results. This could happen with weird input like ' % _ '.
return db.db.selectFrom('did_handle').where(sql`1 = 0`)
return db.db
.selectFrom('did_handle')
.innerJoin('repo_root', 'repo_root.did', 'did_handle.did')
.where(sql`1 = 0`)
}
// We'll ensure there's a space before each word in both textForMatch and in safeWords,
@ -174,9 +87,9 @@ export const getUserSearchQuerySqlite = (
return db.db
.selectFrom('did_handle')
.innerJoin('repo_root as _repo_root', '_repo_root.did', 'did_handle.did')
.innerJoin('repo_root', 'repo_root.did', 'did_handle.did')
.if(!includeSoftDeleted, (qb) =>
qb.where(notSoftDeletedClause(ref('_repo_root'))),
qb.where(notSoftDeletedClause(ref('repo_root'))),
)
.where((q) => {
safeWords.forEach((word) => {

7
packages/pds/tests/_util.ts

@ -98,9 +98,6 @@ export const runTestServer = async (
dbPostgresUrl: process.env.DB_POSTGRES_URL,
blobstoreLocation: `${blobstoreLoc}/blobs`,
blobstoreTmp: `${blobstoreLoc}/tmp`,
labelerDid: 'did:example:labeler',
labelerKeywords: { label_me: 'test-label', label_me_2: 'test-label-2' },
feedGenDid: 'did:example:feedGen',
maxSubscriptionBuffer: 200,
repoBackfillLimitMs: HOUR,
sequencerLeaderLockId: uniqueLockId(),
@ -153,9 +150,6 @@ export const runTestServer = async (
const pdsServer = await pds.start()
const pdsPort = (pdsServer.address() as AddressInfo).port
// we refresh label cache by hand in `processAll` instead of on a timer
pds.ctx.labelCache.stop()
return {
url: `http://localhost:${pdsPort}`,
ctx: pds.ctx,
@ -165,7 +159,6 @@ export const runTestServer = async (
},
processAll: async () => {
await pds.ctx.backgroundQueue.processAll()
await pds.ctx.labelCache.fullRefresh()
},
}
}

147
packages/pds/tests/account-deletion.test.ts

@ -11,24 +11,12 @@ import { BlobNotFoundError, BlobStore } from '@atproto/repo'
import { RepoRoot } from '../src/db/tables/repo-root'
import { UserAccount } from '../src/db/tables/user-account'
import { IpldBlock } from '../src/db/tables/ipld-block'
import { Post } from '../src/app-view/db/tables/post'
import { Like } from '../src/app-view/db/tables/like'
import { Repost } from '../src/app-view/db/tables/repost'
import { Follow } from '../src/app-view/db/tables/follow'
import { RepoBlob } from '../src/db/tables/repo-blob'
import { Blob } from '../src/db/tables/blob'
import {
PostEmbedImage,
PostEmbedExternal,
PostEmbedRecord,
} from '../src/app-view/db/tables/post-embed'
import { Record } from '../src/db/tables/record'
import { RepoSeq } from '../src/db/tables/repo-seq'
import { ACKNOWLEDGE } from '../src/lexicon/types/com/atproto/admin/defs'
import { UserState } from '../src/db/tables/user-state'
import { ActorBlock } from '../src/app-view/db/tables/actor-block'
import { List } from '../src/app-view/db/tables/list'
import { ListItem } from '../src/app-view/db/tables/list-item'
describe('account deletion', () => {
let server: util.TestServerInfo
@ -179,43 +167,10 @@ describe('account deletion', () => {
(row) => row.did === carol.did && row.eventType === 'tombstone',
).length,
).toEqual(1)
})
it('no longer stores indexed records from the user', async () => {
expect(updatedDbContents.records).toEqual(
initialDbContents.records.filter((row) => row.did !== carol.did),
)
expect(updatedDbContents.posts).toEqual(
initialDbContents.posts.filter((row) => row.creator !== carol.did),
)
expect(updatedDbContents.likes).toEqual(
initialDbContents.likes.filter((row) => row.creator !== carol.did),
)
expect(updatedDbContents.actorBlocks).toEqual(
initialDbContents.actorBlocks.filter((row) => row.creator !== carol.did),
)
expect(updatedDbContents.lists).toEqual(
initialDbContents.lists.filter((row) => row.creator !== carol.did),
)
expect(updatedDbContents.listItems).toEqual(
initialDbContents.listItems.filter((row) => row.creator !== carol.did),
)
expect(updatedDbContents.reposts).toEqual(
initialDbContents.reposts.filter((row) => row.creator !== carol.did),
)
expect(updatedDbContents.follows).toEqual(
initialDbContents.follows.filter((row) => row.creator !== carol.did),
)
expect(updatedDbContents.postImages).toEqual(
initialDbContents.postImages.filter(
(row) => !row.postUri.includes(carol.did),
),
)
expect(updatedDbContents.postExternals).toEqual(
initialDbContents.postExternals.filter(
(row) => !row.postUri.includes(carol.did),
),
)
})
it('deletes relevant blobs', async () => {
@ -269,82 +224,32 @@ type DbContents = {
blocks: IpldBlock[]
seqs: Selectable<RepoSeq>[]
records: Record[]
posts: Post[]
postImages: PostEmbedImage[]
postExternals: PostEmbedExternal[]
postRecords: PostEmbedRecord[]
likes: Like[]
reposts: Repost[]
follows: Follow[]
actorBlocks: ActorBlock[]
lists: List[]
listItems: ListItem[]
repoBlobs: RepoBlob[]
blobs: Blob[]
}
const getDbContents = async (db: Database): Promise<DbContents> => {
const [
roots,
users,
userState,
blocks,
seqs,
records,
posts,
postImages,
postExternals,
postRecords,
likes,
reposts,
follows,
actorBlocks,
lists,
listItems,
repoBlobs,
blobs,
] = await Promise.all([
db.db.selectFrom('repo_root').orderBy('did').selectAll().execute(),
db.db.selectFrom('user_account').orderBy('did').selectAll().execute(),
db.db.selectFrom('user_state').orderBy('did').selectAll().execute(),
db.db
.selectFrom('ipld_block')
.orderBy('creator')
.orderBy('cid')
.selectAll()
.execute(),
db.db.selectFrom('repo_seq').orderBy('id').selectAll().execute(),
db.db.selectFrom('record').orderBy('uri').selectAll().execute(),
db.db.selectFrom('post').orderBy('uri').selectAll().execute(),
db.db
.selectFrom('post_embed_image')
.orderBy('postUri')
.selectAll()
.execute(),
db.db
.selectFrom('post_embed_external')
.orderBy('postUri')
.selectAll()
.execute(),
db.db
.selectFrom('post_embed_record')
.orderBy('postUri')
.selectAll()
.execute(),
db.db.selectFrom('like').orderBy('uri').selectAll().execute(),
db.db.selectFrom('repost').orderBy('uri').selectAll().execute(),
db.db.selectFrom('follow').orderBy('uri').selectAll().execute(),
db.db.selectFrom('actor_block').orderBy('uri').selectAll().execute(),
db.db.selectFrom('list').orderBy('uri').selectAll().execute(),
db.db.selectFrom('list_item').orderBy('uri').selectAll().execute(),
db.db
.selectFrom('repo_blob')
.orderBy('did')
.orderBy('cid')
.selectAll()
.execute(),
db.db.selectFrom('blob').orderBy('cid').selectAll().execute(),
])
const [roots, users, userState, blocks, seqs, records, repoBlobs, blobs] =
await Promise.all([
db.db.selectFrom('repo_root').orderBy('did').selectAll().execute(),
db.db.selectFrom('user_account').orderBy('did').selectAll().execute(),
db.db.selectFrom('user_state').orderBy('did').selectAll().execute(),
db.db
.selectFrom('ipld_block')
.orderBy('creator')
.orderBy('cid')
.selectAll()
.execute(),
db.db.selectFrom('repo_seq').orderBy('id').selectAll().execute(),
db.db.selectFrom('record').orderBy('uri').selectAll().execute(),
db.db
.selectFrom('repo_blob')
.orderBy('did')
.orderBy('cid')
.selectAll()
.execute(),
db.db.selectFrom('blob').orderBy('cid').selectAll().execute(),
])
return {
roots,
@ -353,16 +258,6 @@ const getDbContents = async (db: Database): Promise<DbContents> => {
blocks,
seqs,
records,
posts,
postImages,
postExternals,
postRecords,
likes,
reposts,
follows,
actorBlocks,
lists,
listItems,
repoBlobs,
blobs,
}

173
packages/pds/tests/admin/__snapshots__/get-record.test.ts.snap

@ -6,16 +6,6 @@ Object {
"blobs": Array [],
"cid": "cids(0)",
"indexedAt": "1970-01-01T00:00:00.000Z",
"labels": Array [
Object {
"cid": "cids(0)",
"cts": "1970-01-01T00:00:00.000Z",
"neg": false,
"src": "user(0)",
"uri": "record(0)",
"val": "self-label",
},
],
"moderation": Object {
"actions": Array [
Object {
@ -143,169 +133,6 @@ Object {
"blobs": Array [],
"cid": "cids(0)",
"indexedAt": "1970-01-01T00:00:00.000Z",
"labels": Array [
Object {
"cid": "cids(0)",
"cts": "1970-01-01T00:00:00.000Z",
"neg": false,
"src": "user(0)",
"uri": "record(0)",
"val": "self-label",
},
],
"moderation": Object {
"actions": Array [
Object {
"action": "com.atproto.admin.defs#takedown",
"createdAt": "1970-01-01T00:00:00.000Z",
"createdBy": "did:example:admin",
"id": 3,
"reason": "X",
"resolvedReportIds": Array [],
"subject": Object {
"$type": "com.atproto.repo.strongRef",
"cid": "cids(0)",
"uri": "record(0)",
},
"subjectBlobCids": Array [],
},
Object {
"action": "com.atproto.admin.defs#acknowledge",
"createdAt": "1970-01-01T00:00:00.000Z",
"createdBy": "did:example:admin",
"id": 2,
"reason": "X",
"resolvedReportIds": Array [],
"reversal": Object {
"createdAt": "1970-01-01T00:00:00.000Z",
"createdBy": "did:example:admin",
"reason": "X",
},
"subject": Object {
"$type": "com.atproto.repo.strongRef",
"cid": "cids(0)",
"uri": "record(0)",
},
"subjectBlobCids": Array [],
},
],
"currentAction": Object {
"action": "com.atproto.admin.defs#takedown",
"id": 3,
},
"reports": Array [
Object {
"createdAt": "1970-01-01T00:00:00.000Z",
"id": 2,
"reason": "defamation",
"reasonType": "com.atproto.moderation.defs#reasonOther",
"reportedBy": "user(1)",
"resolvedByActionIds": Array [],
"subject": Object {
"$type": "com.atproto.repo.strongRef",
"cid": "cids(0)",
"uri": "record(0)",
},
"subjectRepoHandle": "alice.test",
},
Object {
"createdAt": "1970-01-01T00:00:00.000Z",
"id": 1,
"reasonType": "com.atproto.moderation.defs#reasonSpam",
"reportedBy": "user(2)",
"resolvedByActionIds": Array [],
"subject": Object {
"$type": "com.atproto.repo.strongRef",
"cid": "cids(0)",
"uri": "record(0)",
},
"subjectRepoHandle": "alice.test",
},
],
},
"repo": Object {
"did": "user(0)",
"email": "alice@test.com",
"handle": "alice.test",
"indexedAt": "1970-01-01T00:00:00.000Z",
"invitesDisabled": false,
"moderation": Object {},
"relatedRecords": Array [
Object {
"$type": "app.bsky.actor.profile",
"avatar": Object {
"$type": "blob",
"mimeType": "image/jpeg",
"ref": Object {
"$link": "cids(1)",
},
"size": 3976,
},
"description": "its me!",
"displayName": "ali",
"labels": Object {
"$type": "com.atproto.label.defs#selfLabels",
"values": Array [
Object {
"val": "self-label-a",
},
Object {
"val": "self-label-b",
},
],
},
},
],
},
"uri": "record(0)",
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"labels": Object {
"$type": "com.atproto.label.defs#selfLabels",
"values": Array [
Object {
"val": "self-label",
},
],
},
"text": "hey there",
},
}
`;
exports[`pds admin get record view serves labels. 1`] = `
Object {
"blobCids": Array [],
"blobs": Array [],
"cid": "cids(0)",
"indexedAt": "1970-01-01T00:00:00.000Z",
"labels": Array [
Object {
"cid": "cids(0)",
"cts": "1970-01-01T00:00:00.000Z",
"neg": false,
"src": "did:example:labeler",
"uri": "record(0)",
"val": "kittens",
},
Object {
"cid": "cids(0)",
"cts": "1970-01-01T00:00:00.000Z",
"neg": false,
"src": "did:example:labeler",
"uri": "record(0)",
"val": "puppies",
},
Object {
"cid": "cids(0)",
"cts": "1970-01-01T00:00:00.000Z",
"neg": false,
"src": "user(0)",
"uri": "record(0)",
"val": "self-label",
},
],
"moderation": Object {
"actions": Array [
Object {

118
packages/pds/tests/admin/__snapshots__/get-repo.test.ts.snap

@ -8,124 +8,6 @@ Object {
"indexedAt": "1970-01-01T00:00:00.000Z",
"invites": Array [],
"invitesDisabled": false,
"labels": Array [],
"moderation": Object {
"actions": Array [
Object {
"action": "com.atproto.admin.defs#takedown",
"createdAt": "1970-01-01T00:00:00.000Z",
"createdBy": "did:example:admin",
"id": 3,
"reason": "X",
"resolvedReportIds": Array [],
"subject": Object {
"$type": "com.atproto.admin.defs#repoRef",
"did": "user(0)",
},
"subjectBlobCids": Array [],
},
Object {
"action": "com.atproto.admin.defs#acknowledge",
"createdAt": "1970-01-01T00:00:00.000Z",
"createdBy": "did:example:admin",
"id": 2,
"reason": "X",
"resolvedReportIds": Array [],
"reversal": Object {
"createdAt": "1970-01-01T00:00:00.000Z",
"createdBy": "did:example:admin",
"reason": "X",
},
"subject": Object {
"$type": "com.atproto.admin.defs#repoRef",
"did": "user(0)",
},
"subjectBlobCids": Array [],
},
],
"currentAction": Object {
"action": "com.atproto.admin.defs#takedown",
"id": 3,
},
"reports": Array [
Object {
"createdAt": "1970-01-01T00:00:00.000Z",
"id": 2,
"reason": "defamation",
"reasonType": "com.atproto.moderation.defs#reasonOther",
"reportedBy": "user(1)",
"resolvedByActionIds": Array [],
"subject": Object {
"$type": "com.atproto.admin.defs#repoRef",
"did": "user(0)",
},
},
Object {
"createdAt": "1970-01-01T00:00:00.000Z",
"id": 1,
"reasonType": "com.atproto.moderation.defs#reasonSpam",
"reportedBy": "user(2)",
"resolvedByActionIds": Array [],
"subject": Object {
"$type": "com.atproto.admin.defs#repoRef",
"did": "user(0)",
},
},
],
},
"relatedRecords": Array [
Object {
"$type": "app.bsky.actor.profile",
"avatar": Object {
"$type": "blob",
"mimeType": "image/jpeg",
"ref": Object {
"$link": "cids(0)",
},
"size": 3976,
},
"description": "its me!",
"displayName": "ali",
"labels": Object {
"$type": "com.atproto.label.defs#selfLabels",
"values": Array [
Object {
"val": "self-label-a",
},
Object {
"val": "self-label-b",
},
],
},
},
],
}
`;
exports[`pds admin get repo view serves labels. 1`] = `
Object {
"did": "user(0)",
"email": "alice@test.com",
"handle": "alice.test",
"indexedAt": "1970-01-01T00:00:00.000Z",
"invites": Array [],
"invitesDisabled": false,
"labels": Array [
Object {
"cts": "1970-01-01T00:00:00.000Z",
"neg": false,
"src": "did:example:labeler",
"uri": "user(0)",
"val": "kittens",
},
Object {
"cts": "1970-01-01T00:00:00.000Z",
"neg": false,
"src": "did:example:labeler",
"uri": "user(0)",
"val": "puppies",
},
],
"moderation": Object {
"actions": Array [
Object {

Some files were not shown because too many files have changed in this diff Show More