Build system rework ()

* refactor(crypto): remove circular dependency

* refactor(crypto): expose compress/decompress as part of the DidKeyPlugin interface

* fix(crypto): remove import from private file

* refactor: isolate tsconfig

* fix: remove unused bench file

* chore(repo): remove unused deps

* fix(ozone): properly list dependencies

* fix(services): do lint js files

* fix(services/pds): remove unused deps

* chore(pds): remove bench

* chore(dev-env): remove unused deps

* chore(api): remove bench

* remove unused babel.config.js files

* fix: remove .ts extension from import

* fix(pds): remove imports of src files

* fix(tsconfig): properly list all projects

* fix(dev-env): remove imports of src files

* fix(bsky): remove direct import to crypto src

* fix(api): remove imports to api internals

* chore(build): prevent bundling of built output

* chore(dev): add "dev" script to build in watch mode

* chore(deps): move ts-node dependency where it is actually used

* fix(deps): add dev-env as project dependency

* fix(xrpc-server): properly type kexicon

* fix(bsky): improve typings

* fix(pds): fully type formatRecordEmbedInternal return value

* fix(repo): remove imports from @ipld/car/api

* feat(dev-env): re-export BskyIngester

* fix: properly lint & type jest config & test files

* fix(ci): test after build

* fix(types): use NodeJS.Timeout instead of NodeJS.Timer

* fix(bsky): make types exportable

* fix(ozone): make types exportable

* fix(xrpc-server): make types exportable

* fix(xprc-server): make code compliant with "node" types

* fix(xrpc-server): avoid accessing properties of unknown

* chore(deps): update @types/node

* feat(tsconfig): narrow down available types depending on the package's target environment

* fix(pds): remove unused prop

* fix(bsync): Database's migrator not always initialized

* fix(dev-env): remove unreachable code

* fix(xrpc-server): remove unused import

* fix(xrpc-server): mark header property as abstract

* fix(pds): initialize LeakyTxPlugin's txOver property

* fix(bsky): initialize LeakyTxPlugin's txOver property

* fix(bsky): remove unused migrator from DatabaseCoordinator

* fix(bsky): Properly initialize LabelService's cache property

* fix(ozone): Database's migrator not initialized

* fix(ozone): initialize LeakyTxPlugin's txOver property

* fix(crypto): ignore unused variable error

* feat(tsconfig): use stricter rules

* feat(tsconfig): enable useDefineForClassFields

* feat(xrpc-server): add support for brotli incoming payload

* fix(xrpc-server): properly parse & process content-encoding

* fix(common:stream): always call cb in _transform

* tidy/fix tests and service entrypoints

* Revert "fix(xrpc-server): properly parse & process content-encoding"

This reverts commit 2b1c66e153820d3e128fc839fcc1834d52a66686.

* Revert "feat(xrpc-server): add support for brotli incoming payload"

This reverts commit e710c21e6118214ddf215b0515e68cb87299a952.

* remove special node env for tests (defaults to jest val of "test")

* kill mute sync handler on disconnect

* work around connect-es bug w/ request aborts

* style(crypto): rename imports from uint8arrays

* fix update package-lock

* fix lint

* force hbs files to be bundled as cjs

* fix: use concurrently instead of npm-run-all

npm-run-all seems not to be maintained anymore. Additionally, concurrently better forwards signals to child processes.

* remove concurrently alltogether

* ignore sqlite files in services/pds

* fix verify

* fix verify

* tidy, fix verify

* fix blob diversion test

* build rework changeset

---------

Co-authored-by: Devin Ivy <devinivy@gmail.com>
This commit is contained in:
Matthieu Sieben 2024-03-18 22:10:58 +01:00 committed by GitHub
parent 0a2464cb67
commit f689bd51a2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
232 changed files with 1571 additions and 1557 deletions
.changeset
.eslintrc
.github/workflows
babel.config.jsjest.config.base.jsjest.config.jsjest.setup.tspackage.json
packages

@ -0,0 +1,21 @@
---
'@atproto/xrpc-server': minor
'@atproto/common-web': minor
'@atproto/identity': minor
'@atproto/dev-env': minor
'@atproto/lex-cli': minor
'@atproto/lexicon': minor
'@atproto/common': minor
'@atproto/crypto': minor
'@atproto/syntax': minor
'@atproto/repo': minor
'@atproto/xrpc': minor
'@atproto/api': minor
'@atproto/aws': minor
'@atproto/bsync': patch
'@atproto/ozone': patch
'@atproto/bsky': patch
'@atproto/pds': patch
---
Build system rework, stop bundling dependencies.

@ -13,16 +13,7 @@
"plugin:prettier/recommended",
"prettier"
],
"ignorePatterns": [
"dist",
"node_modules",
"jest.config.base.js",
"jest.bench.config.js",
"jest.config.js",
"babel.config.js",
"build.js",
"update-pkg.js"
],
"ignorePatterns": ["dist", "node_modules"],
"rules": {
"no-var": "error",
"prefer-const": "warn",
@ -36,5 +27,21 @@
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/no-empty-function": "off",
"@typescript-eslint/no-explicit-any": "off"
}
},
"overrides": [
{
"files": ["jest.config.js"],
"env": { "commonjs": true }
},
{
"files": ["jest.setup.js"],
"env": { "jest": true }
},
{
"files": "*.js",
"rules": {
"@typescript-eslint/no-var-requires": "off"
}
}
]
}

@ -27,6 +27,7 @@ jobs:
node-version: 18
cache: 'pnpm'
- run: pnpm i --frozen-lockfile
- run: pnpm build
- run: pnpm verify
- name: Publish
id: changesets

@ -24,8 +24,14 @@ jobs:
cache: 'pnpm'
- run: pnpm i --frozen-lockfile
- run: pnpm build
- uses: actions/upload-artifact@v4
with:
name: dist
path: packages/*/dist
retention-days: 1
test:
name: Test
needs: build
strategy:
matrix:
shard: [1/8, 2/8, 3/8, 4/8, 5/8, 6/8, 7/8, 8/8]
@ -40,9 +46,14 @@ jobs:
node-version: 18
cache: 'pnpm'
- run: pnpm i --frozen-lockfile
- uses: actions/download-artifact@v4
with:
name: dist
path: packages
- run: pnpm test:withFlags --maxWorkers=1 --shard=${{ matrix.shard }} --passWithNoTests
verify:
name: Verify
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
@ -54,4 +65,8 @@ jobs:
node-version: 18
cache: 'pnpm'
- run: pnpm install --frozen-lockfile
- uses: actions/download-artifact@v4
with:
name: dist
path: packages
- run: pnpm verify

@ -1,3 +0,0 @@
module.exports = {
presets: [['@babel/preset-env']],
}

@ -1,20 +0,0 @@
// Jest doesn't like ES modules, so we need to transpile them
// For each one, add them to this list, add them to
// "workspaces.nohoist" in the root package.json, and
// make sure that a babel.config.js is in the package root
const esModules = ['get-port', 'node-fetch'].join('|')
// jestconfig.base.js
module.exports = {
roots: ['<rootDir>/src', '<rootDir>/tests'],
transform: {
'^.+\\.(t|j)s?$': '@swc/jest',
'^.+\\.hbs$': require.resolve('handlebars-jest'),
},
transformIgnorePatterns: [`<rootDir>/node_modules/(?!${esModules})`],
testRegex: '(/tests/.*.(test|spec)).(jsx?|tsx?)$',
moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
setupFiles: ['<rootDir>/../../test-setup.ts'],
verbose: true,
testTimeout: 60000,
}

@ -1,7 +1,4 @@
// jest.config.js
const base = require('./jest.config.base.js')
/** @type {import('jest').Config} */
module.exports = {
...base,
projects: ['<rootDir>/packages/*/jest.config.js'],
}

@ -10,44 +10,46 @@
},
"scripts": {
"lint:fix": "pnpm lint --fix",
"lint": "eslint . --ext .ts,.tsx",
"verify": "prettier --check . && pnpm lint",
"format": "prettier --write .",
"build": "pnpm -r --stream build",
"update-main-to-dist": "pnpm -r --stream update-main-to-dist",
"test": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test",
"test:withFlags": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test --",
"lint": "eslint . --ext .ts,.js",
"style:fix": "prettier --write .",
"style": "prettier --check .",
"verify": "pnpm --stream '/^verify:.+$/'",
"verify:style": "pnpm run style",
"verify:lint": "pnpm lint",
"verify:types": "tsc --build tsconfig.json",
"format": "pnpm lint:fix && pnpm style:fix",
"build": "pnpm --recursive --stream build",
"dev": "pnpm --stream '/^dev:.+$/'",
"dev:tsc": "tsc --build tsconfig.json --watch",
"dev:pkg": "pnpm --recursive --parallel --stream dev",
"test": "LOG_ENABLED=false ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test",
"test:withFlags": "LOG_ENABLED=false ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test --",
"changeset": "changeset",
"release": "pnpm build && changeset publish",
"version-packages": "changeset version && git add ."
},
"devDependencies": {
"@atproto/dev-env": "workspace:^",
"@babel/core": "^7.18.6",
"@babel/preset-env": "^7.18.6",
"@changesets/changelog-github": "^0.4.8",
"@changesets/cli": "^2.26.2",
"@npmcli/package-json": "^3.0.0",
"@swc/core": "^1.3.42",
"@swc/jest": "^0.2.24",
"@types/jest": "^28.1.4",
"@types/node": "^18.0.0",
"@types/node": "^18.19.24",
"@typescript-eslint/eslint-plugin": "^6.14.0",
"@typescript-eslint/parser": "^6.14.0",
"babel-eslint": "^10.1.0",
"dotenv": "^16.0.3",
"esbuild": "^0.14.48",
"esbuild-node-externals": "^1.5.0",
"esbuild-plugin-handlebars": "^1.0.2",
"eslint": "^8.24.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-prettier": "^4.2.1",
"handlebars-jest": "^1.0.0",
"jest": "^28.1.2",
"node-gyp": "^9.3.1",
"pino-pretty": "^9.1.0",
"prettier": "^2.7.1",
"prettier-config-standard": "^5.0.0",
"ts-node": "^10.8.2",
"typescript": "^5.3.3"
},
"workspaces": {

@ -1,3 +0,0 @@
module.exports = {
presets: [['@babel/preset-env']],
}

@ -1,15 +0,0 @@
const { nodeExternalsPlugin } = require('esbuild-node-externals')
const buildShallow =
process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',
platform: 'browser',
format: 'cjs',
plugins: buildShallow ? [nodeExternalsPlugin()] : [],
})

@ -1,8 +0,0 @@
const base = require('./jest.config')
module.exports = {
...base,
roots: ['<rootDir>/bench'],
testRegex: '(.*.bench.ts)',
testTimeout: 3000000,
}

@ -1,6 +1,9 @@
const base = require('../../jest.config.base.js')
/** @type {import('jest').Config} */
module.exports = {
...base,
displayName: 'API',
transform: { '^.+\\.(t|j)s$': '@swc/jest' },
transformIgnorePatterns: [`<rootDir>/node_modules/(?!get-port)`],
testTimeout: 60000,
setupFiles: ['<rootDir>/../../jest.setup.ts'],
setupFilesAfterEnv: ['<rootDir>/jest.setup.ts'],
}

20
packages/api/jest.d.ts vendored Normal file

@ -0,0 +1,20 @@
declare namespace jest {
// eslint-disable-next-line
interface Matchers<R, T = {}> {
toBeModerationResult(
expected: ModerationTestSuiteResultFlag[] | undefined,
context?: string,
stringifiedResult?: string,
ignoreCause?: boolean,
): R
}
interface Expect {
toBeModerationResult(
expected: ModerationTestSuiteResultFlag[] | undefined,
context?: string,
stringifiedResult?: string,
ignoreCause?: boolean,
): void
}
}

@ -0,0 +1,97 @@
import { ModerationUI } from './src'
import { ModerationTestSuiteResultFlag } from './tests/util/moderation-behavior'
expect.extend({
toBeModerationResult(
actual: ModerationUI,
expected: ModerationTestSuiteResultFlag[] | undefined,
context: string = '',
stringifiedResult: string | undefined = undefined,
_ignoreCause = false,
) {
const fail = (msg: string) => ({
pass: false,
message: () =>
`${msg}.${
stringifiedResult ? ` Full result: ${stringifiedResult}` : ''
}`,
})
// let cause = actual.causes?.type as string
// if (actual.cause?.type === 'label') {
// cause = `label:${actual.cause.labelDef.id}`
// } else if (actual.cause?.type === 'muted') {
// if (actual.cause.source.type === 'list') {
// cause = 'muted-by-list'
// }
// } else if (actual.cause?.type === 'blocking') {
// if (actual.cause.source.type === 'list') {
// cause = 'blocking-by-list'
// }
// }
if (!expected) {
// if (!ignoreCause && actual.cause) {
// return fail(`${context} expected to be a no-op, got ${cause}`)
// }
if (actual.inform) {
return fail(`${context} expected to be a no-op, got inform=true`)
}
if (actual.alert) {
return fail(`${context} expected to be a no-op, got alert=true`)
}
if (actual.blur) {
return fail(`${context} expected to be a no-op, got blur=true`)
}
if (actual.filter) {
return fail(`${context} expected to be a no-op, got filter=true`)
}
if (actual.noOverride) {
return fail(`${context} expected to be a no-op, got noOverride=true`)
}
} else {
// if (!ignoreCause && cause !== expected.cause) {
// return fail(`${context} expected to be ${expected.cause}, got ${cause}`)
// }
const expectedInform = expected.includes('inform')
if (!!actual.inform !== expectedInform) {
return fail(
`${context} expected to be inform=${expectedInform}, got ${
actual.inform || false
}`,
)
}
const expectedAlert = expected.includes('alert')
if (!!actual.alert !== expectedAlert) {
return fail(
`${context} expected to be alert=${expectedAlert}, got ${
actual.alert || false
}`,
)
}
const expectedBlur = expected.includes('blur')
if (!!actual.blur !== expectedBlur) {
return fail(
`${context} expected to be blur=${expectedBlur}, got ${
actual.blur || false
}`,
)
}
const expectedFilter = expected.includes('filter')
if (!!actual.filter !== expectedFilter) {
return fail(
`${context} expected to be filter=${expectedFilter}, got ${
actual.filter || false
}`,
)
}
const expectedNoOverride = expected.includes('noOverride')
if (!!actual.noOverride !== expectedNoOverride) {
return fail(
`${context} expected to be noOverride=${expectedNoOverride}, got ${
actual.noOverride || false
}`,
)
}
}
return { pass: true, message: () => '' }
},
})

@ -14,19 +14,12 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/api"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"codegen": "node ./scripts/generate-code.mjs && lex gen-api ./src/client ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/* ../../lexicons/tools/ozone/*/*",
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/api",
"test": "jest",
"bench": "jest --config jest.bench.config.js",
"bench:profile": "node --inspect-brk ../../node_modules/.bin/jest --config jest.bench.config.js"
"build": "tsc --build tsconfig.build.json",
"test": "jest"
},
"dependencies": {
"@atproto/common-web": "workspace:^",
@ -38,7 +31,7 @@
},
"devDependencies": {
"@atproto/lex-cli": "workspace:^",
"@atproto/dev-env": "workspace:^",
"get-port": "^6.1.2"
"get-port": "^6.1.2",
"jest": "^28.1.2"
}
}

@ -5,6 +5,7 @@ import {
interpretLabelValueDefinition,
} from '../src'
import './util/moderation-behavior'
import { ModerationOpts } from '../dist'
describe('Moderation', () => {
it('Applies self-labels on profiles according to the global preferences', () => {
@ -30,6 +31,8 @@ describe('Moderation', () => {
porn: 'hide',
},
labelers: [],
hiddenPosts: [],
mutedWords: [],
},
},
)
@ -62,6 +65,8 @@ describe('Moderation', () => {
porn: 'ignore',
},
labelers: [],
hiddenPosts: [],
mutedWords: [],
},
},
)
@ -96,6 +101,8 @@ describe('Moderation', () => {
porn: 'hide',
},
labelers: [],
hiddenPosts: [],
mutedWords: [],
},
},
)
@ -108,7 +115,7 @@ describe('Moderation', () => {
'contentList',
'contentView',
'contentMedia',
]) {
] as const) {
expect(res1.ui(k)).toBeModerationResult(
[],
k,
@ -143,6 +150,8 @@ describe('Moderation', () => {
labels: { porn: 'ignore' },
},
],
hiddenPosts: [],
mutedWords: [],
},
},
)
@ -155,7 +164,7 @@ describe('Moderation', () => {
'contentList',
'contentView',
'contentMedia',
]) {
] as const) {
expect(res2.ui(k)).toBeModerationResult(
[],
k,
@ -188,6 +197,8 @@ describe('Moderation', () => {
labels: {},
},
],
hiddenPosts: [],
mutedWords: [],
},
},
)
@ -238,17 +249,19 @@ describe('Moderation', () => {
labels: {},
},
],
hiddenPosts: [],
mutedWords: [],
},
},
)
expect(res1.ui('contentList').filters[0].label.val).toBe('!hide')
expect(res1.ui('contentList').filters[1].label.val).toBe('porn')
expect(res1.ui('contentList').blurs[0].label.val).toBe('!hide')
expect(res1.ui('contentMedia').blurs[0].label.val).toBe('porn')
expect((res1.ui('contentList').filters[0] as any).label.val).toBe('!hide')
expect((res1.ui('contentList').filters[1] as any).label.val).toBe('porn')
expect((res1.ui('contentList').blurs[0] as any).label.val).toBe('!hide')
expect((res1.ui('contentMedia').blurs[0] as any).label.val).toBe('porn')
})
it('Prioritizes custom label definitions', () => {
const modOpts = {
const modOpts: ModerationOpts = {
userDid: 'did:web:alice.test',
prefs: {
adultContentEnabled: true,
@ -259,6 +272,8 @@ describe('Moderation', () => {
labels: { porn: 'warn' },
},
],
hiddenPosts: [],
mutedWords: [],
},
labelDefs: {
'did:web:labeler.test': [
@ -268,6 +283,7 @@ describe('Moderation', () => {
blurs: 'none',
severity: 'inform',
locales: [],
defaultSetting: 'warn',
},
'did:web:labeler.test',
),
@ -306,7 +322,7 @@ describe('Moderation', () => {
})
it('Doesnt allow custom behaviors to override imperative labels', () => {
const modOpts = {
const modOpts: ModerationOpts = {
userDid: 'did:web:alice.test',
prefs: {
adultContentEnabled: true,
@ -317,6 +333,8 @@ describe('Moderation', () => {
labels: {},
},
],
hiddenPosts: [],
mutedWords: [],
},
labelDefs: {
'did:web:labeler.test': [
@ -326,6 +344,7 @@ describe('Moderation', () => {
blurs: 'none',
severity: 'inform',
locales: [],
defaultSetting: 'warn',
},
'did:web:labeler.test',
),
@ -369,7 +388,7 @@ describe('Moderation', () => {
})
it('Ignores invalid label value names', () => {
const modOpts = {
const modOpts: ModerationOpts = {
userDid: 'did:web:alice.test',
prefs: {
adultContentEnabled: true,
@ -380,6 +399,8 @@ describe('Moderation', () => {
labels: { BadLabel: 'hide', 'bad/label': 'hide' },
},
],
hiddenPosts: [],
mutedWords: [],
},
labelDefs: {
'did:web:labeler.test': [
@ -389,6 +410,7 @@ describe('Moderation', () => {
blurs: 'content',
severity: 'inform',
locales: [],
defaultSetting: 'warn',
},
'did:web:labeler.test',
),
@ -398,6 +420,7 @@ describe('Moderation', () => {
blurs: 'content',
severity: 'inform',
locales: [],
defaultSetting: 'warn',
},
'did:web:labeler.test',
),
@ -443,7 +466,7 @@ describe('Moderation', () => {
})
it('Custom labels can set the default setting', () => {
const modOpts = {
const modOpts: ModerationOpts = {
userDid: 'did:web:alice.test',
prefs: {
adultContentEnabled: true,
@ -454,6 +477,8 @@ describe('Moderation', () => {
labels: {},
},
],
hiddenPosts: [],
mutedWords: [],
},
labelDefs: {
'did:web:labeler.test': [
@ -585,7 +610,7 @@ describe('Moderation', () => {
})
it('Custom labels can require adult content to be enabled', () => {
const modOpts = {
const modOpts: ModerationOpts = {
userDid: 'did:web:alice.test',
prefs: {
adultContentEnabled: false,
@ -598,6 +623,8 @@ describe('Moderation', () => {
},
},
],
hiddenPosts: [],
mutedWords: [],
},
labelDefs: {
'did:web:labeler.test': [
@ -652,7 +679,7 @@ describe('Moderation', () => {
})
it('Adult content disabled forces the preference to hide', () => {
const modOpts = {
const modOpts: ModerationOpts = {
userDid: 'did:web:alice.test',
prefs: {
adultContentEnabled: false,
@ -663,6 +690,8 @@ describe('Moderation', () => {
labels: {},
},
],
hiddenPosts: [],
mutedWords: [],
},
labelDefs: {},
}

@ -65,7 +65,7 @@ expect.extend({
expected: ModerationTestSuiteResultFlag[] | undefined,
context: string = '',
stringifiedResult: string | undefined = undefined,
ignoreCause = false,
_ignoreCause = false,
) {
const fail = (msg: string) => ({
pass: false,

@ -1,4 +1,9 @@
{
"extends": "./tsconfig.json",
"exclude": ["**/*.spec.ts", "**/*.test.ts"]
"extends": "../../tsconfig/isomorphic.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist",
"noUnusedLocals": false
},
"include": ["./src"]
}

@ -1,13 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist", // Your outDir,
"emitDeclarationOnly": true
},
"include": ["./src"],
"include": [],
"references": [
{ "path": "../xrpc/tsconfig.build.json" },
{ "path": "../lex-cli/tsconfig.build.json" }
{ "path": "./tsconfig.build.json" },
{ "path": "./tsconfig.tests.json" }
]
}

@ -0,0 +1,10 @@
{
"extends": "../../tsconfig/tests.json",
"compilerOptions": {
"rootDir": ".",
"types": ["jest", "./jest.d.ts"],
"noEmit": true,
"noUnusedLocals": false
},
"include": ["./tests"]
}

@ -1,14 +0,0 @@
const { nodeExternalsPlugin } = require('esbuild-node-externals')
const buildShallow =
process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',
platform: 'node',
plugins: buildShallow ? [nodeExternalsPlugin()] : [],
})

@ -13,15 +13,10 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/aws"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/src/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/aws"
"build": "tsc --build tsconfig.build.json"
},
"dependencies": {
"@atproto/common": "workspace:^",

@ -1,4 +1,8 @@
{
"extends": "./tsconfig.json",
"exclude": ["**/*.spec.ts", "**/*.test.ts"]
"extends": "../../tsconfig/isomorphic.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist"
},
"include": ["./src"]
}

@ -1,8 +1,4 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"outDir": "./dist", // Your outDir,
"emitDeclarationOnly": true
},
"include": ["./src", "__tests__/**/**.ts"]
"include": [],
"references": [{ "path": "./tsconfig.build.json" }]
}

@ -1,3 +0,0 @@
module.exports = {
presets: [['@babel/preset-env']],
}

@ -3,10 +3,10 @@ plugins:
- plugin: es
opt:
- target=ts
- import_extension=.ts
- import_extension=
out: src/proto
- plugin: connect-es
opt:
- target=ts
- import_extension=.ts
- import_extension=
out: src/proto

@ -1,19 +0,0 @@
const { nodeExternalsPlugin } = require('esbuild-node-externals')
const buildShallow =
process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',
platform: 'node',
external: [
// Referenced in pg driver, but optional and we don't use it
'pg-native',
'sharp',
],
plugins: buildShallow ? [nodeExternalsPlugin()] : [],
})

@ -1,6 +1,8 @@
const base = require('../../jest.config.base.js')
/** @type {import('jest').Config} */
module.exports = {
...base,
displayName: 'Bsky App View',
transform: { '^.+\\.(t|j)s$': '@swc/jest' },
transformIgnorePatterns: [`<rootDir>/node_modules/(?!get-port)`],
testTimeout: 60000,
setupFiles: ['<rootDir>/../../jest.setup.ts'],
}

@ -13,17 +13,12 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/bsky"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"bin": "dist/bin.js",
"scripts": {
"codegen": "lex gen-server ./src/lexicon ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/*",
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/bsky",
"build": "tsc --build tsconfig.build.json",
"start": "node --enable-source-maps dist/bin.js",
"test": "../dev-infra/with-test-redis-and-db.sh jest",
"test:log": "tail -50 test.log | pino-pretty",
@ -65,7 +60,6 @@
},
"devDependencies": {
"@atproto/api": "workspace:^",
"@atproto/dev-env": "workspace:^",
"@atproto/lex-cli": "workspace:^",
"@atproto/pds": "workspace:^",
"@atproto/xrpc": "workspace:^",
@ -79,6 +73,7 @@
"@types/pg": "^8.6.6",
"@types/qs": "^6.9.7",
"axios": "^0.27.2",
"http2-express-bridge": "^1.0.7"
"jest": "^28.1.2",
"ts-node": "^10.8.2"
}
}

@ -169,7 +169,7 @@ const onClientError = (err: Error) => dbLogger.error({ err }, 'db client error')
// -------
class LeakyTxPlugin implements KyselyPlugin {
private txOver: boolean
private txOver = false
endTx() {
this.txOver = true

@ -4,7 +4,13 @@ import os from 'os'
import path from 'path'
import { Readable } from 'stream'
import axios, { AxiosError } from 'axios'
import express, { ErrorRequestHandler, NextFunction } from 'express'
import express, {
Request,
Response,
Express,
ErrorRequestHandler,
NextFunction,
} from 'express'
import createError, { isHttpError } from 'http-errors'
import { BlobNotFoundError } from '@atproto/repo'
import {
@ -20,7 +26,7 @@ import { retryHttp } from '../util/retry'
import { ServerConfig } from '../config'
export class ImageProcessingServer {
app = express()
app: Express = express()
uriBuilder: ImageUriBuilder
constructor(public cfg: ServerConfig, public cache: BlobCache) {
@ -29,11 +35,7 @@ export class ImageProcessingServer {
this.app.use(errorMiddleware)
}
async handler(
req: express.Request,
res: express.Response,
next: NextFunction,
) {
async handler(req: Request, res: Response, next: NextFunction) {
try {
const path = req.path
const options = ImageUriBuilder.getOptions(path)

@ -29,6 +29,7 @@ export { ServerConfig } from './config'
export { Database } from './data-plane/server/db'
export { Redis } from './redis'
export { AppContext } from './context'
export { BackgroundQueue } from './data-plane/server/background'
export class BskyAppView {
public ctx: AppContext

@ -1,4 +1,4 @@
// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension=.ts"
// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension="
// @generated from file bsky.proto (package bsky, syntax proto3)
/* eslint-disable */
// @ts-nocheck
@ -160,7 +160,7 @@ import {
UntakedownRecordResponse,
UpdateNotificationSeenRequest,
UpdateNotificationSeenResponse,
} from './bsky_pb.ts'
} from './bsky_pb'
import { MethodKind } from '@bufbuild/protobuf'
/**

@ -1,4 +1,4 @@
// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension=.ts"
// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension="
// @generated from file bsky.proto (package bsky, syntax proto3)
/* eslint-disable */
// @ts-nocheck

@ -1,4 +1,4 @@
// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension=.ts"
// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension="
// @generated from file bsync.proto (package bsync, syntax proto3)
/* eslint-disable */
// @ts-nocheck
@ -10,7 +10,7 @@ import {
PingResponse,
ScanMuteOperationsRequest,
ScanMuteOperationsResponse,
} from './bsync_pb.ts'
} from './bsync_pb'
import { MethodKind } from '@bufbuild/protobuf'
/**

@ -1,4 +1,4 @@
// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension=.ts"
// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension="
// @generated from file bsync.proto (package bsync, syntax proto3)
/* eslint-disable */
// @ts-nocheck

@ -1,4 +1,4 @@
// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension=.ts"
// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension="
// @generated from file courier.proto (package courier, syntax proto3)
/* eslint-disable */
// @ts-nocheck
@ -10,7 +10,7 @@ import {
PushNotificationsResponse,
RegisterDeviceTokenRequest,
RegisterDeviceTokenResponse,
} from './courier_pb.ts'
} from './courier_pb'
import { MethodKind } from '@bufbuild/protobuf'
/**

@ -1,4 +1,4 @@
// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension=.ts"
// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension="
// @generated from file courier.proto (package courier, syntax proto3)
/* eslint-disable */
// @ts-nocheck

@ -8,12 +8,26 @@ import {
isThreadViewPost,
} from '../src/lexicon/types/app/bsky/feed/defs'
import { isViewRecord } from '../src/lexicon/types/app/bsky/embed/record'
import { AppBskyFeedGetPostThread } from '@atproto/api'
import {
LabelerView,
isLabelerView,
isLabelerViewDetailed,
} from '../src/lexicon/types/app/bsky/labeler/defs'
type ThreadViewPost = Extract<
AppBskyFeedGetPostThread.OutputSchema['thread'],
{ post: { uri: string } }
>
export function assertIsThreadViewPost(
value: unknown,
): asserts value is ThreadViewPost {
expect(value).toMatchObject({
$type: 'app.bsky.feed.defs#threadViewPost',
})
}
// Swap out identifiers and dates with stable
// values for the purpose of snapshot testing
export const forSnapshot = (obj: unknown) => {

@ -3,7 +3,8 @@ import { cidForCbor, TID } from '@atproto/common'
import { WriteOpAction } from '@atproto/repo'
import { TestNetwork } from '@atproto/dev-env'
import * as lex from '../../src/lexicon/lexicons'
import { Database } from '../../src'
type Database = TestNetwork['bsky']['db']
describe('duplicate record', () => {
let network: TestNetwork

@ -1,7 +1,7 @@
import { sql } from 'kysely'
import { CID } from 'multiformats/cid'
import { cidForCbor, TID } from '@atproto/common'
import * as pdsRepo from '@atproto/pds/src/repo/prepare'
import { repoPrepare } from '@atproto/pds'
import { WriteOpAction } from '@atproto/repo'
import { AtUri } from '@atproto/syntax'
import AtpAgent, {
@ -489,12 +489,12 @@ describe('indexing', () => {
// const { db: pdsDb, services: pdsServices } = network.pds.ctx
// Create a good and a bad post record
const writes = await Promise.all([
pdsRepo.prepareCreate({
repoPrepare.prepareCreate({
did: sc.dids.alice,
collection: ids.AppBskyFeedPost,
record: { text: 'valid', createdAt: new Date().toISOString() },
}),
pdsRepo.prepareCreate({
repoPrepare.prepareCreate({
did: sc.dids.alice,
collection: ids.AppBskyFeedPost,
record: { text: 0 },

@ -1,13 +1,13 @@
import AtpAgent from '@atproto/api'
import { TestNetwork, SeedClient, basicSeed } from '@atproto/dev-env'
import { CommitData } from '@atproto/repo'
import { PreparedWrite } from '@atproto/pds/src/repo'
import * as sequencer from '@atproto/pds/src/sequencer'
import { cborDecode, cborEncode } from '@atproto/common'
import { DatabaseSchemaType } from '../../../src/data-plane/server/db/database-schema'
import { SeedClient, TestNetwork, basicSeed } from '@atproto/dev-env'
import { PreparedWrite, sequencer } from '@atproto/pds'
import { CommitData } from '@atproto/repo'
import { ids } from '../../../src/lexicon/lexicons'
import { forSnapshot } from '../../_util'
import { Database } from '../../../src'
type Database = TestNetwork['bsky']['db']
describe('sync', () => {
let network: TestNetwork

@ -1,10 +1,10 @@
import { wait } from '@atproto/common'
import { randomStr } from '@atproto/crypto'
import {
ConsecutiveList,
LatestQueue,
PartitionedQueue,
} from '../../../src/data-plane/server/subscription/util'
import { randomStr } from '../../../../crypto/src'
describe('subscription utils', () => {
describe('ConsecutiveList', () => {

@ -11,8 +11,6 @@ import {
basicSeed,
} from '@atproto/dev-env'
import { Handler as SkeletonHandler } from '../src/lexicon/types/app/bsky/feed/getFeedSkeleton'
import { GeneratorView } from '@atproto/api/src/client/types/app/bsky/feed/defs'
import { UnknownFeedError } from '@atproto/api/src/client/types/app/bsky/feed/getFeed'
import { ids } from '../src/lexicon/lexicons'
import {
FeedViewPost,
@ -197,7 +195,7 @@ describe('feed generation', () => {
return res.data
}
const paginatedAll: GeneratorView[] = results(await paginateAll(paginator))
const paginatedAll = results(await paginateAll(paginator))
expect(paginatedAll.length).toEqual(5)
expect(paginatedAll[0].uri).toEqual(feedUriOdd)
@ -453,7 +451,9 @@ describe('feed generation', () => {
{ feed: feedUriOdd },
{ headers: await network.serviceHeaders(alice, gen.did) },
)
await expect(tryGetFeed).rejects.toThrow(UnknownFeedError)
await expect(tryGetFeed).rejects.toMatchObject({
error: 'UnknownFeed',
})
})
it('resolves contents of taken-down feed.', async () => {

@ -35,7 +35,7 @@ describe('image processing server', () => {
ImageUriBuilder.getPath({
preset: 'feed_fullsize',
did: fileDid,
cid: fileCid,
cid: fileCid.toString(),
}),
{ responseType: 'stream' },
)
@ -61,7 +61,7 @@ describe('image processing server', () => {
const path = ImageUriBuilder.getPath({
preset: 'avatar',
did: fileDid,
cid: fileCid,
cid: fileCid.toString(),
})
const res1 = await client.get(path, { responseType: 'arraybuffer' })
expect(res1.headers['x-cache']).toEqual('miss')
@ -79,7 +79,7 @@ describe('image processing server', () => {
ImageUriBuilder.getPath({
preset: 'feed_fullsize',
did: fileDid,
cid: missingCid,
cid: missingCid.toString(),
}),
)
expect(res.status).toEqual(404)

@ -1,8 +1,6 @@
import AtpAgent, { AtUri } from '@atproto/api'
import { TestNetwork, SeedClient, RecordRef, basicSeed } from '@atproto/dev-env'
import { forSnapshot } from '../_util'
import { BlockedActorError } from '@atproto/api/src/client/types/app/bsky/feed/getAuthorFeed'
import { BlockedByActorError } from '@atproto/api/src/client/types/app/bsky/feed/getAuthorFeed'
describe('pds views with blocking from block lists', () => {
let network: TestNetwork
@ -165,13 +163,17 @@ describe('pds views with blocking from block lists', () => {
{ actor: carol },
{ headers: await network.serviceHeaders(dan) },
)
await expect(attempt1).rejects.toThrow(BlockedActorError)
await expect(attempt1).rejects.toMatchObject({
error: 'BlockedActor',
})
const attempt2 = agent.api.app.bsky.feed.getAuthorFeed(
{ actor: dan },
{ headers: await network.serviceHeaders(carol) },
)
await expect(attempt2).rejects.toThrow(BlockedByActorError)
await expect(attempt2).rejects.toMatchObject({
error: 'BlockedByActor',
})
})
it('strips blocked users out of getTimeline', async () => {

@ -1,14 +1,7 @@
import assert from 'assert'
import { TestNetwork, RecordRef, SeedClient, basicSeed } from '@atproto/dev-env'
import AtpAgent, { AtUri } from '@atproto/api'
import { BlockedActorError } from '@atproto/api/src/client/types/app/bsky/feed/getAuthorFeed'
import { BlockedByActorError } from '@atproto/api/src/client/types/app/bsky/feed/getAuthorFeed'
import { isThreadViewPost } from '@atproto/api/src/client/types/app/bsky/feed/defs'
import {
isViewRecord as isEmbedViewRecord,
isViewBlocked as isEmbedViewBlocked,
} from '@atproto/api/src/client/types/app/bsky/embed/record'
import { forSnapshot } from '../_util'
import { assertIsThreadViewPost, forSnapshot } from '../_util'
describe('pds views with blocking', () => {
let network: TestNetwork
@ -116,9 +109,9 @@ describe('pds views with blocking', () => {
{ depth: 1, uri: carolReplyToDan.ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
if (!isThreadViewPost(thread.thread)) {
throw new Error('Expected thread view post')
}
assertIsThreadViewPost(thread.thread)
expect(thread.thread.post.uri).toEqual(carolReplyToDan.ref.uriStr)
expect(thread.thread.parent).toMatchObject({
$type: 'app.bsky.feed.defs#blockedPost',
@ -149,13 +142,17 @@ describe('pds views with blocking', () => {
{ actor: carol },
{ headers: await network.serviceHeaders(dan) },
)
await expect(attempt1).rejects.toThrow(BlockedActorError)
await expect(attempt1).rejects.toMatchObject({
error: 'BlockedActor',
})
const attempt2 = agent.api.app.bsky.feed.getAuthorFeed(
{ actor: dan },
{ headers: await network.serviceHeaders(carol) },
)
await expect(attempt2).rejects.toThrow(BlockedByActorError)
await expect(attempt2).rejects.toMatchObject({
error: 'BlockedByActor',
})
})
it('strips blocked users out of getTimeline', async () => {
@ -396,7 +393,8 @@ describe('pds views with blocking', () => {
{ depth: 1, uri: sc.posts[dan][0].ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
assert(isThreadViewPost(replyThenBlock.thread))
assertIsThreadViewPost(replyThenBlock.thread)
expect(replyThenBlock.thread.replies?.map(getThreadPostUri)).toEqual([
aliceReplyToDan.ref.uriStr,
])
@ -411,10 +409,12 @@ describe('pds views with blocking', () => {
{ depth: 1, uri: sc.posts[dan][0].ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
assert(isThreadViewPost(unblock.thread))
expect(unblock.thread.replies?.map(getThreadPostUri).sort()).toEqual(
[aliceReplyToDan.ref.uriStr, carolReplyToDan.ref.uriStr].sort(),
)
assertIsThreadViewPost(unblock.thread)
expect(unblock.thread.replies?.map(getThreadPostUri)).toEqual([
carolReplyToDan.ref.uriStr,
aliceReplyToDan.ref.uriStr,
])
// block then reply
danBlockCarol = await pdsAgent.api.app.bsky.graph.block.create(
@ -434,7 +434,9 @@ describe('pds views with blocking', () => {
{ depth: 1, uri: sc.posts[dan][0].ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
assert(isThreadViewPost(blockThenReply.thread))
assertIsThreadViewPost(blockThenReply.thread)
expect(replyThenBlock.thread.replies?.map(getThreadPostUri)).toEqual([
aliceReplyToDan.ref.uriStr,
])
@ -454,8 +456,12 @@ describe('pds views with blocking', () => {
{ depth: 0, uri: sc.posts[dan][1].ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
assert(isThreadViewPost(embedThenBlock.thread))
assert(isEmbedViewBlocked(embedThenBlock.thread.post.embed?.record))
assertIsThreadViewPost(embedThenBlock.thread)
expect(embedThenBlock.thread.post.embed?.record).toMatchObject({
$type: 'app.bsky.embed.record#viewBlocked',
})
// unblock
await pdsAgent.api.app.bsky.graph.block.delete(
@ -467,8 +473,12 @@ describe('pds views with blocking', () => {
{ depth: 0, uri: sc.posts[dan][1].ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
assert(isThreadViewPost(unblock.thread))
assert(isEmbedViewRecord(unblock.thread.post.embed?.record))
assertIsThreadViewPost(unblock.thread)
expect(unblock.thread.post?.embed?.record).toMatchObject({
$type: 'app.bsky.embed.record#viewRecord',
})
// block then embed
danBlockCarol = await pdsAgent.api.app.bsky.graph.block.create(
@ -489,8 +499,11 @@ describe('pds views with blocking', () => {
{ depth: 0, uri: carolEmbedsDan.ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
assert(isThreadViewPost(blockThenEmbed.thread))
assert(isEmbedViewBlocked(blockThenEmbed.thread.post.embed?.record))
assertIsThreadViewPost(blockThenEmbed.thread)
expect(blockThenEmbed.thread.post.embed?.record).toMatchObject({
$type: 'app.bsky.embed.record#viewBlocked',
})
// cleanup
await pdsAgent.api.app.bsky.feed.post.delete(
@ -517,7 +530,9 @@ describe('pds views with blocking', () => {
(item) => item.post.uri === embedBlockedUri,
)
assert(embedBlockedPost)
assert(isEmbedViewBlocked(embedBlockedPost.post.embed?.record))
expect(embedBlockedPost.post.embed?.record).toMatchObject({
$type: 'app.bsky.embed.record#viewBlocked',
})
})
it('returns a list of blocks', async () => {

@ -1,8 +1,10 @@
import AtpAgent, { AppBskyFeedGetPostThread } from '@atproto/api'
import { TestNetwork, SeedClient, basicSeed } from '@atproto/dev-env'
import { forSnapshot, stripViewerFromThread } from '../_util'
import assert from 'assert'
import { isThreadViewPost } from '@atproto/api/src/client/types/app/bsky/feed/defs'
import {
assertIsThreadViewPost,
forSnapshot,
stripViewerFromThread,
} from '../_util'
describe('pds thread views', () => {
let network: TestNetwork
@ -164,12 +166,12 @@ describe('pds thread views', () => {
{ uri: goodReply1.ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
assert(isThreadViewPost(goodReply1Thread.thread))
assert(isThreadViewPost(goodReply1Thread.thread.parent))
assertIsThreadViewPost(goodReply1Thread.thread)
assertIsThreadViewPost(goodReply1Thread.thread.parent)
expect(goodReply1Thread.thread.parent.post.uri).toEqual(goodRoot.ref.uriStr)
expect(
goodReply1Thread.thread.replies?.map((r) => {
assert(isThreadViewPost(r))
assertIsThreadViewPost(r)
return r.post.uri
}),
).toEqual([
@ -182,7 +184,7 @@ describe('pds thread views', () => {
{ uri: badReply.ref.uriStr },
{ headers: await network.serviceHeaders(alice) },
)
assert(isThreadViewPost(badReplyThread.thread))
assertIsThreadViewPost(badReplyThread.thread)
expect(badReplyThread.thread.parent).toBeUndefined() // is not goodReply1
})
@ -192,7 +194,8 @@ describe('pds thread views', () => {
{ headers: await network.serviceHeaders(bob) },
)
assert(isThreadViewPost(thread.thread), 'post does not exist')
assertIsThreadViewPost(thread.thread)
const post = thread.thread.post
const postSelfLabels = post.labels

@ -1,4 +1,9 @@
{
"extends": "./tsconfig.json",
"exclude": ["**/*.spec.ts", "**/*.test.ts"]
"extends": "../../tsconfig/node.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist",
"noUnusedLocals": false
},
"include": ["./src"]
}

@ -1,20 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist",
"emitDeclarationOnly": true
},
"module": "nodenext",
"include": ["./src", "__tests__/**/**.ts"],
"include": [],
"references": [
{ "path": "../api/tsconfig.build.json" },
{ "path": "../common/tsconfig.build.json" },
{ "path": "../crypto/tsconfig.build.json" },
{ "path": "../lexicon/tsconfig.build.json" },
{ "path": "../lex-cli/tsconfig.build.json" },
{ "path": "../repo/tsconfig.build.json" },
{ "path": "../syntax/tsconfig.build.json" },
{ "path": "../xrpc-server/tsconfig.build.json" }
{ "path": "./tsconfig.build.json" },
{ "path": "./tsconfig.tests.json" }
]
}

@ -0,0 +1,8 @@
{
"extends": "../../tsconfig/tests.json",
"compilerOptions": {
"rootDir": ".",
"noUnusedLocals": false
},
"include": ["./tests"]
}

@ -1,3 +0,0 @@
module.exports = {
presets: [['@babel/preset-env']],
}

@ -1,18 +0,0 @@
const { nodeExternalsPlugin } = require('esbuild-node-externals')
const buildShallow =
process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',
platform: 'node',
external: [
// Referenced in pg driver, but optional and we don't use it
'pg-native',
],
plugins: buildShallow ? [nodeExternalsPlugin()] : [],
})

@ -1,6 +1,6 @@
const base = require('../../jest.config.base.js')
/** @type {import('jest').Config} */
module.exports = {
...base,
displayName: 'Bsync',
transform: { '^.+\\.(t|j)s$': '@swc/jest' },
setupFiles: ['<rootDir>/../../jest.setup.ts'],
}

@ -13,15 +13,10 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/bsync"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/bsync",
"build": "tsc --build tsconfig.build.json",
"start": "node --enable-source-maps dist/bin.js",
"test": "../dev-infra/with-test-db.sh jest",
"test:log": "tail -50 test.log | pino-pretty",
@ -45,6 +40,8 @@
"@bufbuild/buf": "^1.28.1",
"@bufbuild/protoc-gen-es": "^1.5.0",
"@connectrpc/protoc-gen-connect-es": "^1.1.4",
"@types/pg": "^8.6.6"
"@types/pg": "^8.6.6",
"jest": "^28.1.2",
"ts-node": "^10.8.2"
}
}

@ -7,21 +7,25 @@ import { EventEmitter } from 'stream'
export type AppContextOptions = {
db: Database
cfg: ServerConfig
shutdown: AbortSignal
}
export class AppContext {
db: Database
cfg: ServerConfig
shutdown: AbortSignal
events: TypedEventEmitter<AppEvents>
constructor(opts: AppContextOptions) {
this.db = opts.db
this.cfg = opts.cfg
this.shutdown = opts.shutdown
this.events = new EventEmitter() as TypedEventEmitter<AppEvents>
}
static async fromConfig(
cfg: ServerConfig,
shutdown: AbortSignal,
overrides?: Partial<AppContextOptions>,
): Promise<AppContext> {
const db = new Database({
@ -31,7 +35,7 @@ export class AppContext {
poolMaxUses: cfg.db.poolMaxUses,
poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs,
})
return new AppContext({ db, cfg, ...overrides })
return new AppContext({ db, cfg, shutdown, ...overrides })
}
}

@ -34,41 +34,43 @@ export class Database {
if (instances) {
this.db = instances.db
this.pool = instances.pool
return
}
} else {
// else create a pool & connect
const { schema, url } = opts
const pool =
opts.pool ??
new PgPool({
connectionString: url,
max: opts.poolSize,
maxUses: opts.poolMaxUses,
idleTimeoutMillis: opts.poolIdleTimeoutMs,
})
// else create a pool & connect
const { schema, url } = opts
const pool =
opts.pool ??
new PgPool({
connectionString: url,
max: opts.poolSize,
maxUses: opts.poolMaxUses,
idleTimeoutMillis: opts.poolIdleTimeoutMs,
// Select count(*) and other pg bigints as js integer
pgTypes.setTypeParser(pgTypes.builtins.INT8, (n) => parseInt(n, 10))
// Setup schema usage, primarily for test parallelism (each test suite runs in its own pg schema)
if (schema && !/^[a-z_]+$/i.test(schema)) {
throw new Error(
`Postgres schema must only contain [A-Za-z_]: ${schema}`,
)
}
pool.on('error', onPoolError)
pool.on('connect', (client) => {
client.on('error', onClientError)
if (schema) {
// Shared objects such as extensions will go in the public schema
client.query(`SET search_path TO "${schema}",public;`)
}
})
// Select count(*) and other pg bigints as js integer
pgTypes.setTypeParser(pgTypes.builtins.INT8, (n) => parseInt(n, 10))
// Setup schema usage, primarily for test parallelism (each test suite runs in its own pg schema)
if (schema && !/^[a-z_]+$/i.test(schema)) {
throw new Error(`Postgres schema must only contain [A-Za-z_]: ${schema}`)
this.pool = pool
this.db = new Kysely<DatabaseSchemaType>({
dialect: new PostgresDialect({ pool }),
})
}
pool.on('error', onPoolError)
pool.on('connect', (client) => {
client.on('error', onClientError)
if (schema) {
// Shared objects such as extensions will go in the public schema
client.query(`SET search_path TO "${schema}",public;`)
}
})
this.pool = pool
this.db = new Kysely<DatabaseSchemaType>({
dialect: new PostgresDialect({ pool }),
})
this.migrator = new Migrator({
db: this.db,
migrationTableSchema: opts.schema,
@ -165,7 +167,7 @@ const onClientError = (err: Error) => dbLogger.error({ err }, 'db client error')
// -------
class LeakyTxPlugin implements KyselyPlugin {
private txOver: boolean
private txOver = false
endTx() {
this.txOver = true

@ -19,7 +19,7 @@ export class BsyncService {
public server: http.Server
private ac: AbortController
private terminator: HttpTerminator
private dbStatsInterval: NodeJS.Timer
private dbStatsInterval?: NodeJS.Timeout
constructor(opts: {
ctx: AppContext
@ -36,8 +36,8 @@ export class BsyncService {
cfg: ServerConfig,
overrides?: Partial<AppContextOptions>,
): Promise<BsyncService> {
const ctx = await AppContext.fromConfig(cfg, overrides)
const ac = new AbortController()
const ctx = await AppContext.fromConfig(cfg, ac.signal, overrides)
const handler = connectNodeAdapter({
routes: routes(ctx),
shutdownSignal: ac.signal,
@ -55,6 +55,9 @@ export class BsyncService {
}
async start(): Promise<http.Server> {
if (this.dbStatsInterval) {
throw new Error(`${this.constructor.name} already started`)
}
this.dbStatsInterval = setInterval(() => {
dbLogger.info(
{
@ -77,6 +80,7 @@ export class BsyncService {
await this.terminator.terminate()
await this.ctx.db.close()
clearInterval(this.dbStatsInterval)
this.dbStatsInterval = undefined
}
async setupAppEvents() {

@ -10,7 +10,7 @@ import {
PingResponse,
ScanMuteOperationsRequest,
ScanMuteOperationsResponse,
} from './bsync_pb.ts'
} from './bsync_pb'
import { MethodKind } from '@bufbuild/protobuf'
/**

@ -13,7 +13,10 @@ export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
const limit = req.limit || 1000
const cursor = validCursor(req.cursor)
const nextMuteOpPromise = once(events, createMuteOpChannel, {
signal: AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs),
signal: combineSignals(
ctx.shutdown,
AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs),
),
})
nextMuteOpPromise.catch(() => null) // ensure timeout is always handled
@ -31,6 +34,7 @@ export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
try {
await nextMuteOpPromise
} catch (err) {
ctx.shutdown.throwIfAborted()
return new ScanMuteOperationsResponse({
operations: [],
cursor: req.cursor,
@ -67,3 +71,18 @@ const validCursor = (cursor: string): number | null => {
}
return int
}
const combineSignals = (a: AbortSignal, b: AbortSignal) => {
const controller = new AbortController()
for (const signal of [a, b]) {
if (signal.aborted) {
controller.abort()
return signal
}
signal.addEventListener('abort', () => controller.abort(signal.reason), {
// @ts-ignore https://github.com/DefinitelyTyped/DefinitelyTyped/pull/68625
signal: controller.signal,
})
}
return controller.signal
}

@ -1,4 +1,8 @@
{
"extends": "./tsconfig.json",
"exclude": ["**/*.spec.ts", "**/*.test.ts"]
"extends": "../../tsconfig/node.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist"
},
"include": ["./src"]
}

@ -1,14 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist",
"emitDeclarationOnly": true
},
"module": "nodenext",
"include": ["./src", "__tests__/**/**.ts"],
"include": [],
"references": [
{ "path": "../common/tsconfig.build.json" },
{ "path": "../common-web/tsconfig.build.json" }
{ "path": "./tsconfig.build.json" },
{ "path": "./tsconfig.tests.json" }
]
}

@ -0,0 +1,7 @@
{
"extends": "../../tsconfig/tests.json",
"compilerOptions": {
"rootDir": "."
},
"include": ["./tests"]
}

@ -1 +0,0 @@
module.exports = require('../../babel.config.js')

@ -1,15 +0,0 @@
const { nodeExternalsPlugin } = require('esbuild-node-externals')
const buildShallow =
process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',
platform: 'browser',
format: 'cjs',
plugins: buildShallow ? [nodeExternalsPlugin()] : [],
})

@ -1,6 +1,6 @@
const base = require('../../jest.config.base.js')
/** @type {import('jest').Config} */
module.exports = {
...base,
displayName: 'Common Web',
transform: { '^.+\\.(t|j)s$': '@swc/jest' },
setupFiles: ['<rootDir>/../../jest.setup.ts'],
}

@ -12,21 +12,19 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/common-web"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"test": "jest",
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/common-web"
"build": "tsc --build tsconfig.build.json"
},
"dependencies": {
"graphemer": "^1.4.0",
"multiformats": "^9.9.0",
"uint8arrays": "3.0.0",
"zod": "^3.21.4"
},
"devDependencies": {
"jest": "^28.1.2"
}
}

@ -1,4 +1,8 @@
{
"extends": "./tsconfig.json",
"exclude": ["**/*.spec.ts", "**/*.test.ts"]
"extends": "../../tsconfig/isomorphic.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist"
},
"include": ["./src"]
}

@ -1,9 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist", // Your outDir,
"emitDeclarationOnly": true
},
"include": ["./src", "__tests__/**/**.ts"]
"include": [],
"references": [
{ "path": "./tsconfig.build.json" },
{ "path": "./tsconfig.tests.json" }
]
}

@ -0,0 +1,7 @@
{
"extends": "../../tsconfig/tests.json",
"compilerOptions": {
"rootDir": "."
},
"include": ["./tests"]
}

@ -1 +0,0 @@
module.exports = require('../../babel.config.js')

@ -1,14 +0,0 @@
const { nodeExternalsPlugin } = require('esbuild-node-externals')
const buildShallow =
process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',
platform: 'node',
plugins: buildShallow ? [nodeExternalsPlugin()] : [],
})

@ -1,6 +1,6 @@
const base = require('../../jest.config.base.js')
/** @type {import('jest').Config} */
module.exports = {
...base,
displayName: 'Common',
transform: { '^.+\\.(t|j)s$': '@swc/jest' },
setupFiles: ['<rootDir>/../../jest.setup.ts'],
}

@ -12,16 +12,11 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/common"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"test": "jest",
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/common"
"build": "tsc --build tsconfig.build.json"
},
"dependencies": {
"@atproto/common-web": "workspace:^",
@ -30,5 +25,10 @@
"iso-datestring-validator": "^2.2.2",
"multiformats": "^9.9.0",
"pino": "^8.15.0"
},
"devDependencies": {
"jest": "^28.1.2",
"typescript": "^5.3.3",
"uint8arrays": "3.0.0"
}
}

@ -59,8 +59,9 @@ export class MaxSizeChecker extends Transform {
_transform(chunk: Uint8Array, _enc: BufferEncoding, cb: TransformCallback) {
this.totalSize += chunk.length
if (this.totalSize > this.maxSize) {
return this.destroy(this.createError())
cb(this.createError())
} else {
cb(null, chunk)
}
return cb(null, chunk)
}
}

@ -1,4 +1,8 @@
{
"extends": "./tsconfig.json",
"exclude": ["**/*.spec.ts", "**/*.test.ts"]
"extends": "../../tsconfig/node.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist"
},
"include": ["./src"]
}

@ -1,10 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist", // Your outDir,
"emitDeclarationOnly": true
},
"include": ["./src", "__tests__/**/**.ts"],
"references": [{ "path": "../common-web/tsconfig.build.json" }]
"include": [],
"references": [
{ "path": "./tsconfig.build.json" },
{ "path": "./tsconfig.tests.json" }
]
}

@ -0,0 +1,7 @@
{
"extends": "../../tsconfig/tests.json",
"compilerOptions": {
"rootDir": "."
},
"include": ["./tests"]
}

@ -1,14 +0,0 @@
const { nodeExternalsPlugin } = require('esbuild-node-externals')
const buildShallow =
process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',
platform: 'node',
plugins: buildShallow ? [nodeExternalsPlugin()] : [],
})

@ -1,6 +1,6 @@
const base = require('../../jest.config.base.js')
/** @type {import('jest').Config} */
module.exports = {
...base,
displayName: 'Crypto',
transform: { '^.+\\.(t|j)s$': '@swc/jest' },
setupFiles: ['<rootDir>/../../jest.setup.ts'],
}

@ -13,16 +13,11 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/crypto"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"test": "jest ",
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/crypto"
"build": "tsc --build tsconfig.build.json"
},
"dependencies": {
"@noble/curves": "^1.1.0",
@ -30,6 +25,7 @@
"uint8arrays": "3.0.0"
},
"devDependencies": {
"@atproto/common": "workspace:^"
"@atproto/common": "workspace:^",
"jest": "^28.1.2"
}
}

@ -1,13 +1,8 @@
import * as uint8arrays from 'uint8arrays'
import * as p256 from './p256/encoding'
import * as secp from './secp256k1/encoding'
import { BASE58_MULTIBASE_PREFIX, DID_KEY_PREFIX } from './const'
import plugins from './plugins'
import {
BASE58_MULTIBASE_PREFIX,
DID_KEY_PREFIX,
P256_JWT_ALG,
SECP256K1_JWT_ALG,
} from './const'
import { extractMultikey, extractPrefixedBytes, hasPrefix } from './utils'
export type ParsedMultikey = {
jwtAlg: string
@ -15,23 +10,14 @@ export type ParsedMultikey = {
}
export const parseMultikey = (multikey: string): ParsedMultikey => {
if (!multikey.startsWith(BASE58_MULTIBASE_PREFIX)) {
throw new Error(`Incorrect prefix for multikey: ${multikey}`)
}
const prefixedBytes = uint8arrays.fromString(
multikey.slice(BASE58_MULTIBASE_PREFIX.length),
'base58btc',
)
const prefixedBytes = extractPrefixedBytes(multikey)
const plugin = plugins.find((p) => hasPrefix(prefixedBytes, p.prefix))
if (!plugin) {
throw new Error('Unsupported key type')
}
let keyBytes = prefixedBytes.slice(plugin.prefix.length)
if (plugin.jwtAlg === P256_JWT_ALG) {
keyBytes = p256.decompressPubkey(keyBytes)
} else if (plugin.jwtAlg === SECP256K1_JWT_ALG) {
keyBytes = secp.decompressPubkey(keyBytes)
}
const keyBytes = plugin.decompressPubkey(
prefixedBytes.slice(plugin.prefix.length),
)
return {
jwtAlg: plugin.jwtAlg,
keyBytes,
@ -46,28 +32,20 @@ export const formatMultikey = (
if (!plugin) {
throw new Error('Unsupported key type')
}
if (jwtAlg === P256_JWT_ALG) {
keyBytes = p256.compressPubkey(keyBytes)
} else if (jwtAlg === SECP256K1_JWT_ALG) {
keyBytes = secp.compressPubkey(keyBytes)
}
const prefixedBytes = uint8arrays.concat([plugin.prefix, keyBytes])
const prefixedBytes = uint8arrays.concat([
plugin.prefix,
plugin.compressPubkey(keyBytes),
])
return (
BASE58_MULTIBASE_PREFIX + uint8arrays.toString(prefixedBytes, 'base58btc')
)
}
export const parseDidKey = (did: string): ParsedMultikey => {
if (!did.startsWith(DID_KEY_PREFIX)) {
throw new Error(`Incorrect prefix for did:key: ${did}`)
}
return parseMultikey(did.slice(DID_KEY_PREFIX.length))
const multikey = extractMultikey(did)
return parseMultikey(multikey)
}
export const formatDidKey = (jwtAlg: string, keyBytes: Uint8Array): string => {
return DID_KEY_PREFIX + formatMultikey(jwtAlg, keyBytes)
}
const hasPrefix = (bytes: Uint8Array, prefix: Uint8Array): boolean => {
return uint8arrays.equals(prefix, bytes.subarray(0, prefix.byteLength))
}

@ -1,7 +1,11 @@
import { p256 } from '@noble/curves/p256'
import { sha256 } from '@noble/hashes/sha256'
import * as uint8arrays from 'uint8arrays'
import { SupportedEncodings } from 'uint8arrays/util/bases'
import { SupportedEncodings } from 'uint8arrays/to-string'
import {
fromString as ui8FromString,
toString as ui8ToString,
} from 'uint8arrays'
import * as did from '../did'
import { P256_JWT_ALG } from '../const'
import { Keypair } from '../types'
@ -32,9 +36,7 @@ export class P256Keypair implements Keypair {
): Promise<P256Keypair> {
const { exportable = false } = opts || {}
const privKeyBytes =
typeof privKey === 'string'
? uint8arrays.fromString(privKey, 'hex')
: privKey
typeof privKey === 'string' ? ui8FromString(privKey, 'hex') : privKey
return new P256Keypair(privKeyBytes, exportable)
}
@ -43,7 +45,7 @@ export class P256Keypair implements Keypair {
}
publicKeyStr(encoding: SupportedEncodings = 'base64pad'): string {
return uint8arrays.toString(this.publicKey, encoding)
return ui8ToString(this.publicKey, encoding)
}
did(): string {

@ -1,9 +1,10 @@
import { p256 } from '@noble/curves/p256'
import { sha256 } from '@noble/hashes/sha256'
import * as ui8 from 'uint8arrays'
import { P256_JWT_ALG } from '../const'
import { parseDidKey } from '../did'
import { equals as ui8equals } from 'uint8arrays'
import { P256_DID_PREFIX } from '../const'
import { VerifyOptions } from '../types'
import { extractMultikey, extractPrefixedBytes, hasPrefix } from '../utils'
export const verifyDidSig = async (
did: string,
@ -11,10 +12,11 @@ export const verifyDidSig = async (
sig: Uint8Array,
opts?: VerifyOptions,
): Promise<boolean> => {
const { jwtAlg, keyBytes } = parseDidKey(did)
if (jwtAlg !== P256_JWT_ALG) {
const prefixedBytes = extractPrefixedBytes(extractMultikey(did))
if (!hasPrefix(prefixedBytes, P256_DID_PREFIX)) {
throw new Error(`Not a P-256 did:key: ${did}`)
}
const keyBytes = prefixedBytes.slice(P256_DID_PREFIX.length)
return verifySig(keyBytes, data, sig, opts)
}
@ -39,7 +41,7 @@ export const verifySig = async (
export const isCompactFormat = (sig: Uint8Array) => {
try {
const parsed = p256.Signature.fromCompact(sig)
return ui8.equals(parsed.toCompactRawBytes(), sig)
return ui8equals(parsed.toCompactRawBytes(), sig)
} catch {
return false
}

@ -1,11 +1,16 @@
import * as operations from './operations'
import { verifyDidSig } from './operations'
import { compressPubkey, decompressPubkey } from './encoding'
import { DidKeyPlugin } from '../types'
import { P256_DID_PREFIX, P256_JWT_ALG } from '../const'
export const p256Plugin: DidKeyPlugin = {
prefix: P256_DID_PREFIX,
jwtAlg: P256_JWT_ALG,
verifySignature: operations.verifyDidSig,
verifySignature: verifyDidSig,
compressPubkey,
decompressPubkey,
}
export default p256Plugin

@ -1,9 +1,13 @@
import { secp256k1 as k256 } from '@noble/curves/secp256k1'
import { sha256 } from '@noble/hashes/sha256'
import * as uint8arrays from 'uint8arrays'
import { SupportedEncodings } from 'uint8arrays/util/bases'
import * as did from '../did'
import {
fromString as ui8FromString,
toString as ui8ToString,
} from 'uint8arrays'
import { SupportedEncodings } from 'uint8arrays/to-string'
import { SECP256K1_JWT_ALG } from '../const'
import * as did from '../did'
import { Keypair } from '../types'
export type Secp256k1KeypairOptions = {
@ -32,9 +36,7 @@ export class Secp256k1Keypair implements Keypair {
): Promise<Secp256k1Keypair> {
const { exportable = false } = opts || {}
const privKeyBytes =
typeof privKey === 'string'
? uint8arrays.fromString(privKey, 'hex')
: privKey
typeof privKey === 'string' ? ui8FromString(privKey, 'hex') : privKey
return new Secp256k1Keypair(privKeyBytes, exportable)
}
@ -43,7 +45,7 @@ export class Secp256k1Keypair implements Keypair {
}
publicKeyStr(encoding: SupportedEncodings = 'base64pad'): string {
return uint8arrays.toString(this.publicKey, encoding)
return ui8ToString(this.publicKey, encoding)
}
did(): string {

@ -1,9 +1,10 @@
import { secp256k1 as k256 } from '@noble/curves/secp256k1'
import { sha256 } from '@noble/hashes/sha256'
import * as ui8 from 'uint8arrays'
import { SECP256K1_JWT_ALG } from '../const'
import { parseDidKey } from '../did'
import { SECP256K1_DID_PREFIX } from '../const'
import { VerifyOptions } from '../types'
import { extractMultikey, extractPrefixedBytes, hasPrefix } from '../utils'
export const verifyDidSig = async (
did: string,
@ -11,10 +12,11 @@ export const verifyDidSig = async (
sig: Uint8Array,
opts?: VerifyOptions,
): Promise<boolean> => {
const { jwtAlg, keyBytes } = parseDidKey(did)
if (jwtAlg !== SECP256K1_JWT_ALG) {
const prefixedBytes = extractPrefixedBytes(extractMultikey(did))
if (!hasPrefix(prefixedBytes, SECP256K1_DID_PREFIX)) {
throw new Error(`Not a secp256k1 did:key: ${did}`)
}
const keyBytes = prefixedBytes.slice(SECP256K1_DID_PREFIX.length)
return verifySig(keyBytes, data, sig, opts)
}

@ -1,11 +1,16 @@
import * as operations from './operations'
import { verifyDidSig } from './operations'
import { compressPubkey, decompressPubkey } from './encoding'
import { DidKeyPlugin } from '../types'
import { SECP256K1_DID_PREFIX, SECP256K1_JWT_ALG } from '../const'
export const secp256k1Plugin: DidKeyPlugin = {
prefix: SECP256K1_DID_PREFIX,
jwtAlg: SECP256K1_JWT_ALG,
verifySignature: operations.verifyDidSig,
verifySignature: verifyDidSig,
compressPubkey,
decompressPubkey,
}
export default secp256k1Plugin

@ -22,6 +22,9 @@ export type DidKeyPlugin = {
data: Uint8Array,
opts?: VerifyOptions,
) => Promise<boolean>
compressPubkey: (uncompressed: Uint8Array) => Uint8Array
decompressPubkey: (compressed: Uint8Array) => Uint8Array
}
export type VerifyOptions = {

@ -0,0 +1,23 @@
import * as uint8arrays from 'uint8arrays'
import { BASE58_MULTIBASE_PREFIX, DID_KEY_PREFIX } from './const'
export const extractMultikey = (did: string): string => {
if (!did.startsWith(DID_KEY_PREFIX)) {
throw new Error(`Incorrect prefix for did:key: ${did}`)
}
return did.slice(DID_KEY_PREFIX.length)
}
export const extractPrefixedBytes = (multikey: string): Uint8Array => {
if (!multikey.startsWith(BASE58_MULTIBASE_PREFIX)) {
throw new Error(`Incorrect prefix for multikey: ${multikey}`)
}
return uint8arrays.fromString(
multikey.slice(BASE58_MULTIBASE_PREFIX.length),
'base58btc',
)
}
export const hasPrefix = (bytes: Uint8Array, prefix: Uint8Array): boolean => {
return uint8arrays.equals(prefix, bytes.subarray(0, prefix.byteLength))
}

@ -137,6 +137,7 @@ describe('signatures', () => {
})
})
// @ts-expect-error
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async function generateTestVectors(): Promise<TestVector[]> {
const p256Key = await EcdsaKeypair.create({ exportable: true })

@ -1,4 +1,8 @@
{
"extends": "./tsconfig.json",
"exclude": ["**/*.spec.ts", "**/*.test.ts"]
"extends": "../../tsconfig/isomorphic.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist"
},
"include": ["./src"]
}

@ -1,9 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist", // Your outDir,
"emitDeclarationOnly": true
},
"include": ["./src", "__tests__/**/**.ts"]
"include": [],
"references": [
{ "path": "./tsconfig.build.json" },
{ "path": "./tsconfig.tests.json" }
]
}

@ -0,0 +1,7 @@
{
"extends": "../../tsconfig/tests.json",
"compilerOptions": {
"rootDir": "."
},
"include": ["./tests"]
}

@ -1,27 +0,0 @@
const { nodeExternalsPlugin } = require('esbuild-node-externals')
const hbsPlugin = require('esbuild-plugin-handlebars')
const buildShallow =
process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
require('esbuild').build({
logLevel: 'info',
entryPoints: ['src/index.ts', 'src/bin.ts'],
bundle: true,
sourcemap: true,
outdir: 'dist',
platform: 'node',
external: [
'better-sqlite3',
// Referenced in pg driver, but optional and we don't use it
'pg-native',
'sharp',
],
plugins: [].concat(buildShallow ? [nodeExternalsPlugin()] : []).concat([
hbsPlugin({
filter: /\.(hbs)$/,
additionalHelpers: {},
precompileOptions: {},
}),
]),
})

@ -12,16 +12,11 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/dev-env"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"bin": "dist/bin.js",
"scripts": {
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/dev-env",
"build": "tsc --build tsconfig.build.json",
"start": "../dev-infra/with-test-redis-and-db.sh node dist/bin.js"
},
"dependencies": {
@ -39,15 +34,10 @@
"@did-plc/lib": "^0.0.1",
"@did-plc/server": "^0.0.1",
"axios": "^0.27.2",
"better-sqlite3": "^9.4.0",
"dotenv": "^16.0.3",
"express": "^4.18.2",
"get-port": "^6.1.2",
"multiformats": "^9.9.0",
"sharp": "^0.32.6",
"uint8arrays": "3.0.0"
},
"devDependencies": {
"ts-node": "^10.8.1"
}
}

@ -3,10 +3,12 @@ import * as ui8 from 'uint8arrays'
import * as bsky from '@atproto/bsky'
import { AtpAgent } from '@atproto/api'
import { Secp256k1Keypair } from '@atproto/crypto'
import { BackgroundQueue } from '@atproto/bsky'
import { Client as PlcClient } from '@did-plc/lib'
import { BskyConfig } from './types'
import { ADMIN_PASSWORD, EXAMPLE_LABELER } from './const'
import { BackgroundQueue } from '@atproto/bsky/src/data-plane/server/background'
export * from '@atproto/bsky'
export class TestBsky {
constructor(

@ -1,12 +1,11 @@
import http from 'http'
import { Secp256k1Keypair } from '@atproto/crypto'
import { SkeletonHandler, createLexiconServer } from '@atproto/pds'
import { InvalidRequestError } from '@atproto/xrpc-server'
import * as plc from '@did-plc/lib'
import events from 'events'
import express from 'express'
import getPort from 'get-port'
import * as plc from '@did-plc/lib'
import { Secp256k1Keypair } from '@atproto/crypto'
import { Handler as SkeletonHandler } from '@atproto/pds/src/lexicon/types/app/bsky/feed/getFeedSkeleton'
import { createServer } from '@atproto/pds/src/lexicon'
import { InvalidRequestError } from '@atproto/xrpc-server'
import http from 'http'
export class TestFeedGen {
destroyed = false
@ -24,7 +23,7 @@ export class TestFeedGen {
const port = await getPort()
const did = await createFgDid(plcUrl, port)
const app = express()
const lexServer = createServer()
const lexServer = createLexiconServer()
lexServer.app.bsky.feed.getFeedSkeleton(async (args) => {
const handler = feeds[args.params.feed]

@ -2,6 +2,7 @@ export * from './bsky'
export * from './bsync'
export * from './network'
export * from './network-no-appview'
export * from './ozone'
export * from './pds'
export * from './plc'
export * from './ozone'

Some files were not shown because too many files have changed in this diff Show More