Lexicon tokens (#292)

* Add token schema types to lexicon package

* Add token-schema support to lex-cli package generation of markdown, client code, and server code

* Add two token lexicons: actorUser and actorScene

* Remove dead code

lexicons/bsky.app/actorScene.json

@@ -0,0 +1,6 @@
+{
+  "lexicon": 1,
+  "id": "app.bsky.actorScene",
+  "type": "token",
+  "description": "Actor type: Scene. Defined for app.bsky.declaration's actorType."
+}

lexicons/bsky.app/actorUser.json

@@ -0,0 +1,6 @@
+{
+  "lexicon": 1,
+  "id": "app.bsky.actorUser",
+  "type": "token",
+  "description": "Actor type: User. Defined for app.bsky.declaration's actorType."
+}

packages/lex-cli/src/codegen/client.ts

@@ -9,7 +9,14 @@ import { NSID } from '@atproto/nsid'
 import * as jsonSchemaToTs from 'json-schema-to-typescript'
 import { gen, schemasTs } from './common'
 import { GeneratedAPI } from '../types'
-import { schemasToNsidTree, NsidNS, toCamelCase, toTitleCase } from './util'
+import {
+  schemasToNsidTree,
+  NsidNS,
+  schemasToNsidTokens,
+  toCamelCase,
+  toTitleCase,
+  toScreamingSnakeCase,
+} from './util'
 
 const ATP_METHODS = {
   list: 'com.atproto.repoListRecords',
@@ -26,6 +33,7 @@ export async function genClientApi(schemas: Schema[]): Promise<GeneratedAPI> {
   })
   const api: GeneratedAPI = { files: [] }
   const nsidTree = schemasToNsidTree(schemas)
+  const nsidTokens = schemasToNsidTokens(schemas)
   for (const schema of schemas) {
     if (schema.type === 'query' || schema.type === 'procedure') {
       api.files.push(await methodSchemaTs(project, schema))
@@ -34,11 +42,16 @@ export async function genClientApi(schemas: Schema[]): Promise<GeneratedAPI> {
     }
   }
   api.files.push(await schemasTs(project, schemas))
-  api.files.push(await indexTs(project, schemas, nsidTree))
+  api.files.push(await indexTs(project, schemas, nsidTree, nsidTokens))
   return api
 }
 
-const indexTs = (project: Project, schemas: Schema[], nsidTree: NsidNS[]) =>
+const indexTs = (
+  project: Project,
+  schemas: Schema[],
+  nsidTree: NsidNS[],
+  nsidTokens: Record<string, string[]>,
+) =>
   gen(project, '/index.ts', async (file) => {
     //= import {Client as XrpcClient, ServiceClient as XrpcServiceClient} from '@atproto/xrpc'
     const xrpcImport = file.addImportDeclaration({
@@ -55,6 +68,7 @@ const indexTs = (project: Project, schemas: Schema[], nsidTree: NsidNS[]) =>
 
     // generate type imports and re-exports
     for (const schema of schemas) {
+      if (schema.type === 'token') continue
       const moduleSpecifier = `./types/${schema.id.split('.').join('/')}`
       file
         .addImportDeclaration({ moduleSpecifier })
@@ -64,6 +78,30 @@ const indexTs = (project: Project, schemas: Schema[], nsidTree: NsidNS[]) =>
         .setNamespaceExport(toTitleCase(schema.id))
     }
 
+    // generate token enums
+    for (const nsidAuthority in nsidTokens) {
+      // export const {THE_AUTHORITY} = {
+      //  {Name}: "{authority.the.name}"
+      // }
+      file.addVariableStatement({
+        isExported: true,
+        declarationKind: VariableDeclarationKind.Const,
+        declarations: [
+          {
+            name: toScreamingSnakeCase(nsidAuthority),
+            initializer: [
+              '{',
+              ...nsidTokens[nsidAuthority].map(
+                (nsidName) =>
+                  `${toTitleCase(nsidName)}: "${nsidAuthority}.${nsidName}",`,
+              ),
+              '}',
+            ].join('\n'),
+          },
+        ],
+      })
+    }
+
     //= export class Client {...}
     const clientCls = file.addClass({
       name: 'Client',

packages/lex-cli/src/codegen/server.ts

@@ -1,10 +1,22 @@
-import { IndentationText, Project, SourceFile } from 'ts-morph'
+import {
+  IndentationText,
+  Project,
+  SourceFile,
+  VariableDeclarationKind,
+} from 'ts-morph'
 import { Schema, MethodSchema, RecordSchema } from '@atproto/lexicon'
 import { NSID } from '@atproto/nsid'
 import * as jsonSchemaToTs from 'json-schema-to-typescript'
 import { gen, schemasTs } from './common'
 import { GeneratedAPI } from '../types'
-import { schemasToNsidTree, NsidNS, toCamelCase, toTitleCase } from './util'
+import {
+  schemasToNsidTree,
+  NsidNS,
+  schemasToNsidTokens,
+  toCamelCase,
+  toTitleCase,
+  toScreamingSnakeCase,
+} from './util'
 
 export async function genServerApi(schemas: Schema[]): Promise<GeneratedAPI> {
   const project = new Project({
@@ -13,6 +25,7 @@ export async function genServerApi(schemas: Schema[]): Promise<GeneratedAPI> {
   })
   const api: GeneratedAPI = { files: [] }
   const nsidTree = schemasToNsidTree(schemas)
+  const nsidTokens = schemasToNsidTokens(schemas)
   for (const schema of schemas) {
     if (schema.type === 'query' || schema.type === 'procedure') {
       api.files.push(await methodSchemaTs(project, schema))
@@ -21,11 +34,16 @@ export async function genServerApi(schemas: Schema[]): Promise<GeneratedAPI> {
     }
   }
   api.files.push(await schemasTs(project, schemas))
-  api.files.push(await indexTs(project, schemas, nsidTree))
+  api.files.push(await indexTs(project, schemas, nsidTree, nsidTokens))
   return api
 }
 
-const indexTs = (project: Project, schemas: Schema[], nsidTree: NsidNS[]) =>
+const indexTs = (
+  project: Project,
+  schemas: Schema[],
+  nsidTree: NsidNS[],
+  nsidTokens: Record<string, string[]>,
+) =>
   gen(project, '/index.ts', async (file) => {
     //= import {createServer as createXrpcServer, Server as XrpcServer} from '@atproto/xrpc-server'
     const xrpcImport = file.addImportDeclaration({
@@ -60,6 +78,30 @@ const indexTs = (project: Project, schemas: Schema[], nsidTree: NsidNS[]) =>
         .setNamespaceImport(toTitleCase(schema.id))
     }
 
+    // generate token enums
+    for (const nsidAuthority in nsidTokens) {
+      // export const {THE_AUTHORITY} = {
+      //  {Name}: "{authority.the.name}"
+      // }
+      file.addVariableStatement({
+        isExported: true,
+        declarationKind: VariableDeclarationKind.Const,
+        declarations: [
+          {
+            name: toScreamingSnakeCase(nsidAuthority),
+            initializer: [
+              '{',
+              ...nsidTokens[nsidAuthority].map(
+                (nsidName) =>
+                  `${toTitleCase(nsidName)}: "${nsidAuthority}.${nsidName}",`,
+              ),
+              '}',
+            ].join('\n'),
+          },
+        ],
+      })
+    }
+
     //= export function createServer() { ... }
     const createServerFn = file.addFunction({
       name: 'createServer',

packages/lex-cli/src/codegen/util.ts

@@ -1,4 +1,5 @@
 import { Schema } from '@atproto/lexicon'
+import { NSID } from '@atproto/nsid'
 
 export interface NsidNS {
   name: string
@@ -11,6 +12,7 @@ export interface NsidNS {
 export function schemasToNsidTree(schemas: Schema[]): NsidNS[] {
   const tree: NsidNS[] = []
   for (const schema of schemas) {
+    if (schema.type === 'token') continue
     const node = getOrCreateNode(tree, schema.id.split('.').slice(0, -1))
     node.schemas.push(schema)
   }
@@ -37,6 +39,23 @@ function getOrCreateNode(tree: NsidNS[], path: string[]): NsidNS {
   return node
 }
 
+export function schemasToNsidTokens(
+  schemas: Schema[],
+): Record<string, string[]> {
+  const nsidTokens: Record<string, string[]> = {}
+  for (const schema of schemas) {
+    if (schema.type !== 'token') {
+      continue
+    }
+    const nsidp = NSID.parse(schema.id)
+    if (!nsidp.name) continue
+    const authority = nsidp.segments.slice(0, -1).join('.')
+    nsidTokens[authority] ??= []
+    nsidTokens[authority].push(nsidp.name)
+  }
+  return nsidTokens
+}
+
 export function toTitleCase(v: string): string {
   v = v.replace(/^([a-z])/gi, (_, g) => g.toUpperCase()) // upper-case first letter
   v = v.replace(/[.-]([a-z])/gi, (_, g) => g.toUpperCase()) // uppercase any dash or dot segments
@@ -47,3 +66,8 @@ export function toCamelCase(v: string): string {
   v = v.replace(/[.-]([a-z])/gi, (_, g) => g.toUpperCase()) // uppercase any dash or dot segments
   return v.replace(/[.-]/g, '') // remove lefover dashes or dots
 }
+
+export function toScreamingSnakeCase(v: string): string {
+  v = v.replace(/[.-]+/gi, '_') // convert dashes and dots into underscores
+  return v.toUpperCase() // and scream!
+}

packages/lex-cli/src/mdgen/index.ts

@@ -4,6 +4,8 @@ import {
   MethodSchema,
   recordSchema,
   RecordSchema,
+  tokenSchema,
+  TokenSchema,
   Schema,
 } from '@atproto/lexicon'
 import * as jsonSchemaToTs from 'json-schema-to-typescript'
@@ -48,7 +50,9 @@ export async function process(outFilePath: string, schemas: Schema[]) {
 async function genMdLines(schemas: Schema[]): Promise<StringTree> {
   let xprcMethods: StringTree = []
   let recordTypes: StringTree = []
+  let tokenTypes: StringTree = []
   for (const schema of schemas) {
+    console.log(schema.id)
     if (methodSchema.safeParse(schema).success) {
       xprcMethods = xprcMethods.concat(
         await genMethodSchemaMd(schema as MethodSchema),
@@ -57,11 +61,14 @@ async function genMdLines(schemas: Schema[]): Promise<StringTree> {
       recordTypes = recordTypes.concat(
         await genRecordSchemaMd(schema as RecordSchema),
       )
+    } else if (tokenSchema.safeParse(schema).success) {
+      tokenTypes = tokenTypes.concat(genTokenSchemaMd(schema as TokenSchema))
     }
   }
   let doc = [
     recordTypes?.length ? recordTypes : undefined,
     xprcMethods?.length ? xprcMethods : undefined,
+    tokenTypes?.length ? tokenTypes : undefined,
   ]
   return doc
 }
@@ -196,6 +203,13 @@ async function genRecordSchemaMd(schema: RecordSchema): Promise<StringTree> {
   return doc
 }
 
+function genTokenSchemaMd(schema: TokenSchema): StringTree {
+  const desc: StringTree = []
+  const doc: StringTree = [`---`, ``, `## ${schema.id}`, '', desc]
+  desc.push(`<mark>Token</mark> ${schema.description || ''}`, ``)
+  return doc
+}
+
 type StringTree = (StringTree | string | undefined)[]
 function merge(arr: StringTree): string {
   return arr

packages/lex-cli/src/refs.ts

@@ -2,6 +2,7 @@ import {
   Schema,
   isValidRecordSchema,
   isValidMethodSchema,
+  isValidTokenSchema,
 } from '@atproto/lexicon'
 import pointer from 'json-pointer'
 import { toCamelCase } from './codegen/util'
@@ -104,6 +105,8 @@ function resolveRefs(
         keysToNameInfo,
       )
     }
+  } else if (isValidTokenSchema(doc)) {
+    // ignore
   } else {
     throw new Error('Unknown lexicon schema')
   }
@@ -169,6 +172,8 @@ function simplifyDefs(doc: Schema, keysToNameInfo: Map<string, NameInfo>) {
   } else if (isValidMethodSchema(doc)) {
     updateDefs(doc.input?.schema?.$defs)
     updateDefs(doc.output?.schema?.$defs)
+  } else if (isValidTokenSchema(doc)) {
+    // ignore
   } else {
     throw new Error('Unknown lexicon schema')
   }

packages/lex-cli/src/util.ts

@@ -1,6 +1,11 @@
 import fs from 'fs'
 import { join } from 'path'
-import { methodSchema, recordSchema, Schema } from '@atproto/lexicon'
+import {
+  tokenSchema,
+  methodSchema,
+  recordSchema,
+  Schema,
+} from '@atproto/lexicon'
 import chalk from 'chalk'
 import { GeneratedAPI, FileDiff } from './types'
 
@@ -52,7 +57,15 @@ export function readSchema(path: string): Schema {
     console.error(`Failed to parse JSON in file`, path)
     throw e
   }
-  if (obj.type === 'query' || obj.type === 'procedure') {
+  if (obj.type === 'token') {
+    try {
+      tokenSchema.parse(obj)
+      return obj
+    } catch (e) {
+      console.error(`Invalid token schema in file`, path)
+      throw e
+    }
+  } else if (obj.type === 'query' || obj.type === 'procedure') {
     try {
       methodSchema.parse(obj)
       return obj

packages/lexicon/src/types.ts

@@ -1,6 +1,22 @@
 import { z } from 'zod'
 import { NSID } from '@atproto/nsid'
 
+export const tokenSchema = z.object({
+  lexicon: z.literal(1),
+  id: z.string().refine((v: string) => NSID.isValid(v), {
+    message: 'Must be a valid NSID',
+  }),
+  type: z.enum(['token']),
+  revision: z.number().optional(),
+  description: z.string().optional(),
+  defs: z.any().optional(),
+})
+export type TokenSchema = z.infer<typeof tokenSchema>
+
+export function isValidTokenSchema(v: unknown): v is TokenSchema {
+  return tokenSchema.safeParse(v).success
+}
+
 export const recordSchema = z.object({
   lexicon: z.literal(1),
   id: z.string().refine((v: string) => NSID.isValid(v), {
@@ -73,6 +89,6 @@ export function isValidMethodSchema(v: unknown): v is MethodSchema {
   return methodSchema.safeParse(v).success
 }
 
-export type Schema = RecordSchema | MethodSchema
+export type Schema = TokenSchema | RecordSchema | MethodSchema
 
 export class SchemaNotFoundError extends Error {}