Post record language tagging, lexicon language format (#1219)

* Add languages field to post record * helper for parsing bcp47 * add language format to lexicon * codegen for post record langs field * re-export language parsing in api package * tests and tidy for lexicon language format * index post langs, in-progress * update snapshots, fixes record-with-media embed issue * index post langs on bsky appview * don't bother indexing post langs in pds appview, tidy
2023-06-23 16:23:16 -04:00 · 2023-06-23 16:23:16 -04:00 · 3da0324873
commit 3da0324873
parent 57b87908c2
32 changed files with 471 additions and 4 deletions
--- a/lexicons/app/bsky/feed/post.json
+++ b/lexicons/app/bsky/feed/post.json
@ -29,6 +29,11 @@
              "app.bsky.embed.recordWithMedia"
            ]
          },
+          "langs": {
+            "type": "array",
+            "maxLength": 3,
+            "items": {"type": "string", "format": "language"}
+          },
          "createdAt": {"type": "string", "format": "datetime"}
        }
      }
--- a/packages/api/src/client/lexicons.ts
+++ b/packages/api/src/client/lexicons.ts
@ -5250,6 +5250,14 @@ export const schemaDict = {
                'lex:app.bsky.embed.recordWithMedia',
              ],
            },
+            langs: {
+              type: 'array',
+              maxLength: 3,
+              items: {
+                type: 'string',
+                format: 'language',
+              },
+            },
            createdAt: {
              type: 'string',
              format: 'datetime',
--- a/packages/api/src/client/types/app/bsky/feed/post.ts
+++ b/packages/api/src/client/types/app/bsky/feed/post.ts
@ -24,6 +24,7 @@ export interface Record {
    | AppBskyEmbedRecord.Main
    | AppBskyEmbedRecordWithMedia.Main
    | { $type: string; [k: string]: unknown }
+  langs?: string[]
  createdAt: string
  [k: string]: unknown
 }
--- a/packages/api/src/index.ts
+++ b/packages/api/src/index.ts
@ -6,6 +6,7 @@ export {
  jsonToLex,
  jsonStringToLex,
 } from '@atproto/lexicon'
+export { parseLanguage } from '@atproto/common-web'
 export * from './types'
 export * from './client'
 export * from './agent'
--- a/packages/bsky/src/db/migrations/20230620T161134972Z-post-langs.ts
+++ b/packages/bsky/src/db/migrations/20230620T161134972Z-post-langs.ts
@ -0,0 +1,9 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post').addColumn('langs', 'jsonb').execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post').dropColumn('langs').execute()
+}
--- a/packages/bsky/src/db/migrations/index.ts
+++ b/packages/bsky/src/db/migrations/index.ts
@ -16,3 +16,4 @@ export * as _20230609T153623961Z from './20230609T153623961Z-blocks'
 export * as _20230609T232122649Z from './20230609T232122649Z-actor-deletion-indexes'
 export * as _20230610T203555962Z from './20230610T203555962Z-suggested-follows'
 export * as _20230611T215300060Z from './20230611T215300060Z-actor-state'
+export * as _20230620T161134972Z from './20230620T161134972Z-post-langs'
--- a/packages/bsky/src/db/tables/post.ts
+++ b/packages/bsky/src/db/tables/post.ts
@ -11,6 +11,7 @@ export interface Post {
  replyRootCid: string | null
  replyParent: string | null
  replyParentCid: string | null
+  langs: string[] | null
  createdAt: string
  indexedAt: string
  sortAt: GeneratedAlways<string>
--- a/packages/bsky/src/lexicon/lexicons.ts
+++ b/packages/bsky/src/lexicon/lexicons.ts
@ -5250,6 +5250,14 @@ export const schemaDict = {
                'lex:app.bsky.embed.recordWithMedia',
              ],
            },
+            langs: {
+              type: 'array',
+              maxLength: 3,
+              items: {
+                type: 'string',
+                format: 'language',
+              },
+            },
            createdAt: {
              type: 'string',
              format: 'datetime',
--- a/packages/bsky/src/lexicon/types/app/bsky/feed/post.ts
+++ b/packages/bsky/src/lexicon/types/app/bsky/feed/post.ts
@ -24,6 +24,7 @@ export interface Record {
    | AppBskyEmbedRecord.Main
    | AppBskyEmbedRecordWithMedia.Main
    | { $type: string; [k: string]: unknown }
+  langs?: string[]
  createdAt: string
  [k: string]: unknown
 }
--- a/packages/bsky/src/services/indexing/plugins/post.ts
+++ b/packages/bsky/src/services/indexing/plugins/post.ts
@ -52,6 +52,9 @@ const insertFn = async (
    replyRootCid: obj.reply?.root?.cid || null,
    replyParent: obj.reply?.parent?.uri || null,
    replyParentCid: obj.reply?.parent?.cid || null,
+    langs: obj.langs?.length
+      ? sql<string[]>`${JSON.stringify(obj.langs)}` // sidesteps kysely's array serialization, which is non-jsonb
+      : null,
    indexedAt: timestamp,
  }
  const [insertedPost] = await Promise.all([
--- a/packages/bsky/tests/snapshots/feed-generation.test.ts.snap
+++ b/packages/bsky/tests/snapshots/feed-generation.test.ts.snap
@ -267,6 +267,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -330,6 +334,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -551,6 +559,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -741,6 +753,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
--- a/packages/bsky/tests/seeds/basic.ts
+++ b/packages/bsky/tests/seeds/basic.ts
@ -24,7 +24,9 @@ export default async (sc: SeedClient, users = true) => {
  await sc.follow(bob, carol, createdAtMicroseconds())
  await sc.follow(dan, bob, createdAtTimezone())
  await sc.post(alice, posts.alice[0])
-  await sc.post(bob, posts.bob[0])
+  await sc.post(bob, posts.bob[0], undefined, undefined, undefined, {
+    langs: ['en-US', 'i-klingon'],
+  })
  const img1 = await sc.uploadFile(
    carol,
    'tests/image/fixtures/key-landscape-small.jpg',
--- a/packages/bsky/tests/views/snapshots/author-feed.test.ts.snap
+++ b/packages/bsky/tests/views/snapshots/author-feed.test.ts.snap
@ -559,6 +559,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -638,6 +642,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -892,6 +900,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -1063,6 +1075,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
--- a/packages/bsky/tests/views/snapshots/posts.test.ts.snap
+++ b/packages/bsky/tests/views/snapshots/posts.test.ts.snap
@ -75,6 +75,10 @@ Array [
    "record": Object {
      "$type": "app.bsky.feed.post",
      "createdAt": "1970-01-01T00:00:00.000Z",
+      "langs": Array [
+        "en-US",
+        "i-klingon",
+      ],
      "text": "bob back at it again!",
    },
    "replyCount": 0,
@ -136,6 +140,10 @@ Array [
          "value": Object {
            "$type": "app.bsky.feed.post",
            "createdAt": "1970-01-01T00:00:00.000Z",
+            "langs": Array [
+              "en-US",
+              "i-klingon",
+            ],
            "text": "bob back at it again!",
          },
        },
@ -261,6 +269,10 @@ Array [
                "value": Object {
                  "$type": "app.bsky.feed.post",
                  "createdAt": "1970-01-01T00:00:00.000Z",
+                  "langs": Array [
+                    "en-US",
+                    "i-klingon",
+                  ],
                  "text": "bob back at it again!",
                },
              },
--- a/packages/bsky/tests/views/snapshots/timeline.test.ts.snap
+++ b/packages/bsky/tests/views/snapshots/timeline.test.ts.snap
@ -698,6 +698,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -795,6 +799,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -957,6 +965,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -1750,6 +1762,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -1937,6 +1953,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -2034,6 +2054,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -2151,6 +2175,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -2947,6 +2975,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -3042,6 +3074,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -3160,6 +3196,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -3782,6 +3822,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -4176,6 +4220,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -4328,6 +4376,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
--- a/packages/common-web/src/strings.ts
+++ b/packages/common-web/src/strings.ts
@ -19,3 +19,38 @@ export const utf8ToB64Url = (utf8: string): string => {
 export const b64UrlToUtf8 = (b64: string): string => {
  return ui8.toString(ui8.fromString(b64, 'base64url'), 'utf8')
 }
+
+export const parseLanguage = (langTag: string): LanguageTag | null => {
+  const parsed = langTag.match(bcp47Regexp)
+  if (!parsed?.groups) return null
+  const parts = parsed.groups
+  return {
+    grandfathered: parts.grandfathered,
+    language: parts.language,
+    extlang: parts.extlang,
+    script: parts.script,
+    region: parts.region,
+    variant: parts.variant,
+    extension: parts.extension,
+    privateUse: parts.privateUseA || parts.privateUseB,
+  }
+}
+
+export const validateLanguage = (langTag: string): boolean => {
+  return bcp47Regexp.test(langTag)
+}
+
+export type LanguageTag = {
+  grandfathered?: string
+  language?: string
+  extlang?: string
+  script?: string
+  region?: string
+  variant?: string
+  extension?: string
+  privateUse?: string
+}
+
+// Validates well-formed BCP 47 syntax: https://www.rfc-editor.org/rfc/rfc5646.html#section-2.1
+const bcp47Regexp =
+  /^((?<grandfathered>(en-GB-oed|i-ami|i-bnn|i-default|i-enochian|i-hak|i-klingon|i-lux|i-mingo|i-navajo|i-pwn|i-tao|i-tay|i-tsu|sgn-BE-FR|sgn-BE-NL|sgn-CH-DE)|(art-lojban|cel-gaulish|no-bok|no-nyn|zh-guoyu|zh-hakka|zh-min|zh-min-nan|zh-xiang))|((?<language>([A-Za-z]{2,3}(-(?<extlang>[A-Za-z]{3}(-[A-Za-z]{3}){0,2}))?)|[A-Za-z]{4}|[A-Za-z]{5,8})(-(?<script>[A-Za-z]{4}))?(-(?<region>[A-Za-z]{2}|[0-9]{3}))?(-(?<variant>[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(-(?<extension>[0-9A-WY-Za-wy-z](-[A-Za-z0-9]{2,8})+))*(-(?<privateUseA>x(-[A-Za-z0-9]{1,8})+))?)|(?<privateUseB>x(-[A-Za-z0-9]{1,8})+))$/
--- a/packages/common-web/tests/strings.test.ts
+++ b/packages/common-web/tests/strings.test.ts
@ -1,4 +1,4 @@
-import { graphemeLen, utf8Len } from '../src'
+import { graphemeLen, parseLanguage, utf8Len, validateLanguage } from '../src'

 describe('string', () => {
  it('calculates utf8 string length', () => {
@ -14,7 +14,7 @@ describe('string', () => {
    expect(utf8Len('👨‍👩‍👧‍👧')).toBe(25)
  })

-  it('caluclates grapheme length', () => {
+  it('calculates grapheme length', () => {
    expect(graphemeLen('a')).toBe(1)
    expect(graphemeLen('~')).toBe(1)
    expect(graphemeLen('ö')).toBe(1)
@ -27,4 +27,88 @@ describe('string', () => {
    expect(graphemeLen('👨‍👩‍👧‍👧')).toBe(1)
    expect(graphemeLen('a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧')).toBe(10)
  })
+
+  describe('languages', () => {
+    it('validates BCP 47', () => {
+      // valid
+      expect(validateLanguage('de')).toEqual(true)
+      expect(validateLanguage('de-CH')).toEqual(true)
+      expect(validateLanguage('de-DE-1901')).toEqual(true)
+      expect(validateLanguage('es-419')).toEqual(true)
+      expect(validateLanguage('sl-IT-nedis')).toEqual(true)
+      expect(validateLanguage('mn-Cyrl-MN')).toEqual(true)
+      expect(validateLanguage('x-fr-CH')).toEqual(true)
+      expect(
+        validateLanguage('en-GB-boont-r-extended-sequence-x-private'),
+      ).toEqual(true)
+      expect(validateLanguage('sr-Cyrl')).toEqual(true)
+      expect(validateLanguage('hy-Latn-IT-arevela')).toEqual(true)
+      expect(validateLanguage('i-klingon')).toEqual(true)
+      // invalid
+      expect(validateLanguage('')).toEqual(false)
+      expect(validateLanguage('x')).toEqual(false)
+      expect(validateLanguage('de-CH-')).toEqual(false)
+      expect(validateLanguage('i-bad-grandfathered')).toEqual(false)
+    })
+
+    it('parses BCP 47', () => {
+      // valid
+      expect(parseLanguage('de')).toEqual({
+        language: 'de',
+      })
+      expect(parseLanguage('de-CH')).toEqual({
+        language: 'de',
+        region: 'CH',
+      })
+      expect(parseLanguage('de-DE-1901')).toEqual({
+        language: 'de',
+        region: 'DE',
+        variant: '1901',
+      })
+      expect(parseLanguage('es-419')).toEqual({
+        language: 'es',
+        region: '419',
+      })
+      expect(parseLanguage('sl-IT-nedis')).toEqual({
+        language: 'sl',
+        region: 'IT',
+        variant: 'nedis',
+      })
+      expect(parseLanguage('mn-Cyrl-MN')).toEqual({
+        language: 'mn',
+        script: 'Cyrl',
+        region: 'MN',
+      })
+      expect(parseLanguage('x-fr-CH')).toEqual({
+        privateUse: 'x-fr-CH',
+      })
+      expect(
+        parseLanguage('en-GB-boont-r-extended-sequence-x-private'),
+      ).toEqual({
+        language: 'en',
+        region: 'GB',
+        variant: 'boont',
+        extension: 'r-extended-sequence',
+        privateUse: 'x-private',
+      })
+      expect(parseLanguage('sr-Cyrl')).toEqual({
+        language: 'sr',
+        script: 'Cyrl',
+      })
+      expect(parseLanguage('hy-Latn-IT-arevela')).toEqual({
+        language: 'hy',
+        script: 'Latn',
+        region: 'IT',
+        variant: 'arevela',
+      })
+      expect(parseLanguage('i-klingon')).toEqual({
+        grandfathered: 'i-klingon',
+      })
+      // invalid
+      expect(parseLanguage('')).toEqual(null)
+      expect(parseLanguage('x')).toEqual(null)
+      expect(parseLanguage('de-CH-')).toEqual(null)
+      expect(parseLanguage('i-bad-grandfathered')).toEqual(null)
+    })
+  })
 })
--- a/packages/lexicon/src/types.ts
+++ b/packages/lexicon/src/types.ts
@ -37,6 +37,7 @@ export const lexStringFormat = z.enum([
  'at-identifier',
  'nsid',
  'cid',
+  'language',
 ])
 export type LexStringFormat = z.infer<typeof lexStringFormat>

--- a/packages/lexicon/src/validators/formats.ts
+++ b/packages/lexicon/src/validators/formats.ts
@ -4,6 +4,7 @@ import { CID } from 'multiformats/cid'
 import { ValidationResult, ValidationError } from '../types'
 import { ensureValidDid, ensureValidHandle } from '@atproto/identifier'
 import { ensureValidNsid } from '@atproto/nsid'
+import { validateLanguage } from '@atproto/common-web'

 export function datetime(path: string, value: string): ValidationResult {
  try {
@ -105,3 +106,16 @@ export function cid(path: string, value: string): ValidationResult {
  }
  return { success: true, value }
 }
+
+// The language format validates well-formed BCP 47 language tags: https://www.rfc-editor.org/info/bcp47
+export function language(path: string, value: string): ValidationResult {
+  if (validateLanguage(value)) {
+    return { success: true, value }
+  }
+  return {
+    success: false,
+    error: new ValidationError(
+      `${path} must be a well-formed BCP 47 language tag`,
+    ),
+  }
+}
--- a/packages/lexicon/src/validators/primitives.ts
+++ b/packages/lexicon/src/validators/primitives.ts
@ -263,6 +263,8 @@ export function string(
        return formats.nsid(path, value)
      case 'cid':
        return formats.cid(path, value)
+      case 'language':
+        return formats.language(path, value)
    }
  }

--- a/packages/lexicon/tests/_scaffolds/lexicons.ts
+++ b/packages/lexicon/tests/_scaffolds/lexicons.ts
@ -486,6 +486,21 @@ export default [
      },
    },
  },
+  {
+    lexicon: 1,
+    id: 'com.example.language',
+    defs: {
+      main: {
+        type: 'record',
+        record: {
+          type: 'object',
+          properties: {
+            language: { type: 'string', format: 'language' },
+          },
+        },
+      },
+    },
+  },
  {
    lexicon: 1,
    id: 'com.example.byteLength',
--- a/packages/lexicon/tests/general.test.ts
+++ b/packages/lexicon/tests/general.test.ts
@ -806,6 +806,19 @@ describe('Record validation', () => {
    ).toThrow('Record/cid must be a cid string')
  })

+  it('Applies language formatting constraint', () => {
+    lex.assertValidRecord('com.example.language', {
+      $type: 'com.example.language',
+      language: 'en-US-boont',
+    })
+    expect(() =>
+      lex.assertValidRecord('com.example.language', {
+        $type: 'com.example.language',
+        language: 'not-a-language-',
+      }),
+    ).toThrow('Record/language must be a well-formed BCP 47 language tag')
+  })
+
  it('Applies bytes length constraints', () => {
    lex.assertValidRecord('com.example.byteLength', {
      $type: 'com.example.byteLength',
--- a/packages/pds/src/lexicon/lexicons.ts
+++ b/packages/pds/src/lexicon/lexicons.ts
@ -5250,6 +5250,14 @@ export const schemaDict = {
                'lex:app.bsky.embed.recordWithMedia',
              ],
            },
+            langs: {
+              type: 'array',
+              maxLength: 3,
+              items: {
+                type: 'string',
+                format: 'language',
+              },
+            },
            createdAt: {
              type: 'string',
              format: 'datetime',
--- a/packages/pds/src/lexicon/types/app/bsky/feed/post.ts
+++ b/packages/pds/src/lexicon/types/app/bsky/feed/post.ts
@ -24,6 +24,7 @@ export interface Record {
    | AppBskyEmbedRecord.Main
    | AppBskyEmbedRecordWithMedia.Main
    | { $type: string; [k: string]: unknown }
+  langs?: string[]
  createdAt: string
  [k: string]: unknown
 }
--- a/packages/pds/tests/snapshots/feed-generation.test.ts.snap
+++ b/packages/pds/tests/snapshots/feed-generation.test.ts.snap
@ -267,6 +267,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -330,6 +334,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -559,6 +567,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -749,6 +761,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
--- a/packages/pds/tests/event-stream/snapshots/sync.test.ts.snap
+++ b/packages/pds/tests/event-stream/snapshots/sync.test.ts.snap
@ -130,6 +130,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -912,6 +916,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -1089,6 +1097,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -1168,6 +1180,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
--- a/packages/pds/tests/proxied/snapshots/feedgen.test.ts.snap
+++ b/packages/pds/tests/proxied/snapshots/feedgen.test.ts.snap
@ -653,6 +653,10 @@ Object {
              "value": Object {
                "$type": "app.bsky.feed.post",
                "createdAt": "1970-01-01T00:00:00.000Z",
+                "langs": Array [
+                  "en-US",
+                  "i-klingon",
+                ],
                "text": "bob back at it again!",
              },
            },
@ -732,6 +736,10 @@ Object {
        "record": Object {
          "$type": "app.bsky.feed.post",
          "createdAt": "1970-01-01T00:00:00.000Z",
+          "langs": Array [
+            "en-US",
+            "i-klingon",
+          ],
          "text": "bob back at it again!",
        },
        "replyCount": 0,
--- a/packages/pds/tests/proxied/snapshots/views.test.ts.snap
+++ b/packages/pds/tests/proxied/snapshots/views.test.ts.snap
@ -393,6 +393,10 @@ Object {
        "record": Object {
          "$type": "app.bsky.feed.post",
          "createdAt": "1970-01-01T00:00:00.000Z",
+          "langs": Array [
+            "en-US",
+            "i-klingon",
+          ],
          "text": "bob back at it again!",
        },
        "replyCount": 0,
@ -528,6 +532,10 @@ Object {
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -589,6 +597,10 @@ Object {
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -785,6 +797,10 @@ Object {
                    "value": Object {
                      "$type": "app.bsky.feed.post",
                      "createdAt": "1970-01-01T00:00:00.000Z",
+                      "langs": Array [
+                        "en-US",
+                        "i-klingon",
+                      ],
                      "text": "bob back at it again!",
                    },
                  },
@ -1551,6 +1567,10 @@ Object {
                    "value": Object {
                      "$type": "app.bsky.feed.post",
                      "createdAt": "1970-01-01T00:00:00.000Z",
+                      "langs": Array [
+                        "en-US",
+                        "i-klingon",
+                      ],
                      "text": "bob back at it again!",
                    },
                  },
@ -1720,6 +1740,10 @@ Object {
              "value": Object {
                "$type": "app.bsky.feed.post",
                "createdAt": "1970-01-01T00:00:00.000Z",
+                "langs": Array [
+                  "en-US",
+                  "i-klingon",
+                ],
                "text": "bob back at it again!",
              },
            },
@ -1799,6 +1823,10 @@ Object {
        "record": Object {
          "$type": "app.bsky.feed.post",
          "createdAt": "1970-01-01T00:00:00.000Z",
+          "langs": Array [
+            "en-US",
+            "i-klingon",
+          ],
          "text": "bob back at it again!",
        },
        "replyCount": 0,
--- a/packages/pds/tests/seeds/basic.ts
+++ b/packages/pds/tests/seeds/basic.ts
@ -26,7 +26,9 @@ export default async (sc: SeedClient) => {
  await sc.follow(bob, carol, createdAtMicroseconds())
  await sc.follow(dan, bob, createdAtTimezone())
  await sc.post(alice, posts.alice[0])
-  await sc.post(bob, posts.bob[0])
+  await sc.post(bob, posts.bob[0], undefined, undefined, undefined, {
+    langs: ['en-US', 'i-klingon'],
+  })
  const img1 = await sc.uploadFile(
    carol,
    'tests/image/fixtures/key-landscape-small.jpg',
--- a/packages/pds/tests/views/snapshots/author-feed.test.ts.snap
+++ b/packages/pds/tests/views/snapshots/author-feed.test.ts.snap
@ -567,6 +567,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -654,6 +658,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -908,6 +916,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -1095,6 +1107,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -1296,6 +1312,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
--- a/packages/pds/tests/views/snapshots/posts.test.ts.snap
+++ b/packages/pds/tests/views/snapshots/posts.test.ts.snap
@ -75,6 +75,10 @@ Array [
    "record": Object {
      "$type": "app.bsky.feed.post",
      "createdAt": "1970-01-01T00:00:00.000Z",
+      "langs": Array [
+        "en-US",
+        "i-klingon",
+      ],
      "text": "bob back at it again!",
    },
    "replyCount": 0,
@ -136,6 +140,10 @@ Array [
          "value": Object {
            "$type": "app.bsky.feed.post",
            "createdAt": "1970-01-01T00:00:00.000Z",
+            "langs": Array [
+              "en-US",
+              "i-klingon",
+            ],
            "text": "bob back at it again!",
          },
        },
@ -269,6 +277,10 @@ Array [
                "value": Object {
                  "$type": "app.bsky.feed.post",
                  "createdAt": "1970-01-01T00:00:00.000Z",
+                  "langs": Array [
+                    "en-US",
+                    "i-klingon",
+                  ],
                  "text": "bob back at it again!",
                },
              },
--- a/packages/pds/tests/views/snapshots/timeline.test.ts.snap
+++ b/packages/pds/tests/views/snapshots/timeline.test.ts.snap
@ -746,6 +746,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -843,6 +847,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -1021,6 +1029,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -1830,6 +1842,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -2025,6 +2041,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -2122,6 +2142,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -2247,6 +2271,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -3051,6 +3079,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -3146,6 +3178,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -3272,6 +3308,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -3902,6 +3942,10 @@ Array [
            "value": Object {
              "$type": "app.bsky.feed.post",
              "createdAt": "1970-01-01T00:00:00.000Z",
+              "langs": Array [
+                "en-US",
+                "i-klingon",
+              ],
              "text": "bob back at it again!",
            },
          },
@ -4312,6 +4356,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },
@ -4472,6 +4520,10 @@ Array [
      "record": Object {
        "$type": "app.bsky.feed.post",
        "createdAt": "1970-01-01T00:00:00.000Z",
+        "langs": Array [
+          "en-US",
+          "i-klingon",
+        ],
        "text": "bob back at it again!",
      },
      "replyCount": 0,
@ -4970,6 +5022,10 @@ Array [
                  "value": Object {
                    "$type": "app.bsky.feed.post",
                    "createdAt": "1970-01-01T00:00:00.000Z",
+                    "langs": Array [
+                      "en-US",
+                      "i-klingon",
+                    ],
                    "text": "bob back at it again!",
                  },
                },