lex refactor hot fixes (#745)

* filter embeds from get popular * Truncate profile info to satisfy validation (#746) * Remove trailing replacement character when utf8-truncating (#747) * Truncate profile info to satisfy validation * Fix utf8 truncation w/ replacement character * filter replies * delete embeds from records on getPopular * Update profile display name and description lengths to be based on graphemes (#748) * @atproto api v0.2.1 * Tidy --------- Co-authored-by: devin ivy <devinivy@gmail.com>
2023-04-04 14:56:53 -05:00 · 2023-04-04 14:56:53 -05:00 · eb488b96f5
commit eb488b96f5
parent 1014938520
9 changed files with 114 additions and 32 deletions
--- a/lexicons/app/bsky/actor/defs.json
+++ b/lexicons/app/bsky/actor/defs.json
@ -11,7 +11,8 @@
        "handle": {"type": "string", "format": "handle"},
        "displayName": {
          "type": "string",
-          "maxLength": 64
+          "maxGraphemes": 64,
+          "maxLength": 640
        },
        "avatar": { "type": "string" },
        "viewer": {"type": "ref", "ref": "#viewerState"}
@ -25,11 +26,13 @@
        "handle": {"type": "string", "format":"handle"},
        "displayName": {
          "type": "string",
-          "maxLength": 64
+          "maxGraphemes": 64,
+          "maxLength": 640
        },
        "description": {
          "type": "string",
-          "maxLength": 256
+          "maxGraphemes": 256,
+          "maxLength": 2560
        },
        "avatar": { "type": "string" },
        "indexedAt": {"type": "string", "format": "datetime"},
@ -44,11 +47,13 @@
        "handle": {"type": "string", "format": "handle"},
        "displayName": {
          "type": "string",
-          "maxLength": 64
+          "maxGraphemes": 64,
+          "maxLength": 640
        },
        "description": {
          "type": "string",
-          "maxLength": 256
+          "maxGraphemes": 256,
+          "maxLength": 2560
        },
        "avatar": { "type": "string" },
        "banner": { "type": "string" },
--- a/lexicons/app/bsky/actor/profile.json
+++ b/lexicons/app/bsky/actor/profile.json
@ -10,11 +10,13 @@
        "properties": {
          "displayName": {
            "type": "string",
-            "maxLength": 64
+            "maxGraphemes": 64,
+            "maxLength": 640
          },
          "description": {
            "type": "string",
-            "maxLength": 256
+            "maxGraphemes": 256,
+            "maxLength": 2560
          },
          "avatar": {
            "type": "blob",
--- a/packages/api/src/client/lexicons.ts
+++ b/packages/api/src/client/lexicons.ts
@ -2533,7 +2533,8 @@ export const schemaDict = {
          },
          displayName: {
            type: 'string',
-            maxLength: 64,
+            maxGraphemes: 64,
+            maxLength: 640,
          },
          avatar: {
            type: 'string',
@ -2558,11 +2559,13 @@ export const schemaDict = {
          },
          displayName: {
            type: 'string',
-            maxLength: 64,
+            maxGraphemes: 64,
+            maxLength: 640,
          },
          description: {
            type: 'string',
-            maxLength: 256,
+            maxGraphemes: 256,
+            maxLength: 2560,
          },
          avatar: {
            type: 'string',
@ -2591,11 +2594,13 @@ export const schemaDict = {
          },
          displayName: {
            type: 'string',
-            maxLength: 64,
+            maxGraphemes: 64,
+            maxLength: 640,
          },
          description: {
            type: 'string',
-            maxLength: 256,
+            maxGraphemes: 256,
+            maxLength: 2560,
          },
          avatar: {
            type: 'string',
@ -2761,11 +2766,13 @@ export const schemaDict = {
          properties: {
            displayName: {
              type: 'string',
-              maxLength: 64,
+              maxGraphemes: 64,
+              maxLength: 640,
            },
            description: {
              type: 'string',
-              maxLength: 256,
+              maxGraphemes: 256,
+              maxLength: 2560,
            },
            avatar: {
              type: 'blob',
--- a/packages/pds/src/app-view/api/app/bsky/unspecced.ts
+++ b/packages/pds/src/app-view/api/app/bsky/unspecced.ts
@ -4,6 +4,7 @@ import { paginate } from '../../../../db/pagination'
 import AppContext from '../../../../context'
 import { FeedRow, FeedItemType } from '../../../services/feed'
 import { sql } from 'kysely'
+import { FeedViewPost } from '../../../../lexicon/types/app/bsky/feed/defs'

 // THIS IS A TEMPORARY UNSPECCED ROUTE
 export default function (server: Server, ctx: AppContext) {
@ -51,12 +52,25 @@ export default function (server: Server, ctx: AppContext) {
      feedQb = paginate(feedQb, { limit, cursor, keyset })

      const feedItems: FeedRow[] = await feedQb.execute()
-      const feed = await composeFeed(feedService, feedItems, requester)
+      const feed: FeedViewPost[] = await composeFeed(
+        feedService,
+        feedItems,
+        requester,
+      )
+
+      const noRecordEmbeds = feed.map((post) => {
+        delete post.post.record['embed']
+        if (post.reply) {
+          delete post.reply.parent.record['embed']
+          delete post.reply.root.record['embed']
+        }
+        return post
+      })

      return {
        encoding: 'application/json',
        body: {
-          feed,
+          feed: noRecordEmbeds,
          cursor: keyset.packFromResult(feedItems),
        },
      }
--- a/packages/pds/src/app-view/services/actor/views.ts
+++ b/packages/pds/src/app-view/services/actor/views.ts
@ -96,8 +96,8 @@ export class ActorViews {
      return {
        did: result.did,
        handle: result.handle,
-        displayName: profileInfo?.displayName || undefined,
-        description: profileInfo?.description || undefined,
+        displayName: truncateUtf8(profileInfo?.displayName, 64) || undefined,
+        description: truncateUtf8(profileInfo?.description, 256) || undefined,
        avatar,
        banner,
        followsCount: profileInfo?.followsCount ?? 0,
@ -174,8 +174,8 @@ export class ActorViews {
      return {
        did: result.did,
        handle: result.handle,
-        displayName: profileInfo?.displayName || undefined,
-        description: profileInfo?.description || undefined,
+        displayName: truncateUtf8(profileInfo?.displayName, 64) || undefined,
+        description: truncateUtf8(profileInfo?.description, 256) || undefined,
        avatar,
        indexedAt: profileInfo?.indexedAt || undefined,
        viewer: {
@ -206,7 +206,7 @@ export class ActorViews {
    const views = profiles.map((view) => ({
      did: view.did,
      handle: view.handle,
-      displayName: view.displayName,
+      displayName: truncateUtf8(view.displayName, 64) || undefined,
      avatar: view.avatar,
      viewer: view.viewer,
    }))
@ -216,3 +216,15 @@ export class ActorViews {
 }

 type ActorResult = DidHandle
+
+function truncateUtf8(str: string | null | undefined, length: number) {
+  if (!str) return str
+  const encoder = new TextEncoder()
+  const utf8 = encoder.encode(str)
+  if (utf8.length > length) {
+    const decoder = new TextDecoder('utf-8', { fatal: false })
+    const truncated = utf8.slice(0, length)
+    return decoder.decode(truncated).replace(/\uFFFD$/, '')
+  }
+  return str
+}
--- a/packages/pds/src/app-view/services/feed/index.ts
+++ b/packages/pds/src/app-view/services/feed/index.ts
@ -109,7 +109,7 @@ export class FeedService {
        [cur.did]: {
          did: cur.did,
          handle: cur.handle,
-          displayName: cur.displayName || undefined,
+          displayName: truncateUtf8(cur.displayName, 64) || undefined,
          avatar: cur.avatarCid
            ? this.imgUriBuilder.getCommonSignedUri('avatar', cur.avatarCid)
            : undefined,
@ -350,3 +350,15 @@ export class FeedService {
    }
  }
 }
+
+function truncateUtf8(str: string | null | undefined, length: number) {
+  if (!str) return str
+  const encoder = new TextEncoder()
+  const utf8 = encoder.encode(str)
+  if (utf8.length > length) {
+    const decoder = new TextDecoder('utf-8', { fatal: false })
+    const truncated = utf8.slice(0, length)
+    return decoder.decode(truncated).replace(/\uFFFD$/, '')
+  }
+  return str
+}
--- a/packages/pds/src/lexicon/lexicons.ts
+++ b/packages/pds/src/lexicon/lexicons.ts
@ -2533,7 +2533,8 @@ export const schemaDict = {
          },
          displayName: {
            type: 'string',
-            maxLength: 64,
+            maxGraphemes: 64,
+            maxLength: 640,
          },
          avatar: {
            type: 'string',
@ -2558,11 +2559,13 @@ export const schemaDict = {
          },
          displayName: {
            type: 'string',
-            maxLength: 64,
+            maxGraphemes: 64,
+            maxLength: 640,
          },
          description: {
            type: 'string',
-            maxLength: 256,
+            maxGraphemes: 256,
+            maxLength: 2560,
          },
          avatar: {
            type: 'string',
@ -2591,11 +2594,13 @@ export const schemaDict = {
          },
          displayName: {
            type: 'string',
-            maxLength: 64,
+            maxGraphemes: 64,
+            maxLength: 640,
          },
          description: {
            type: 'string',
-            maxLength: 256,
+            maxGraphemes: 256,
+            maxLength: 2560,
          },
          avatar: {
            type: 'string',
@ -2761,11 +2766,13 @@ export const schemaDict = {
          properties: {
            displayName: {
              type: 'string',
-              maxLength: 64,
+              maxGraphemes: 64,
+              maxLength: 640,
            },
            description: {
              type: 'string',
-              maxLength: 256,
+              maxGraphemes: 256,
+              maxLength: 2560,
            },
            avatar: {
              type: 'blob',
--- a/packages/pds/tests/views/popular.test.ts
+++ b/packages/pds/tests/views/popular.test.ts
@ -54,24 +54,36 @@ describe('popular views', () => {
  })

  it('returns well liked posts', async () => {
-    const one = await sc.post(alice, 'like this')
+    const img = await sc.uploadFile(
+      alice,
+      'tests/image/fixtures/key-landscape-small.jpg',
+      'image/jpeg',
+    )
+    const one = await sc.post(alice, 'first post', undefined, [img])
    await sc.like(bob, one.ref)
    await sc.like(carol, one.ref)
    await sc.like(dan, one.ref)
    await sc.like(eve, one.ref)
    await sc.like(frank, one.ref)
-    const two = await sc.post(bob, 'like this')
+    const two = await sc.post(bob, 'bobby boi')
    await sc.like(alice, two.ref)
    await sc.like(carol, two.ref)
    await sc.like(dan, two.ref)
    await sc.like(eve, two.ref)
    await sc.like(frank, two.ref)
+    const three = await sc.reply(bob, one.ref, one.ref, 'reply')
+    await sc.like(alice, three.ref)
+    await sc.like(carol, three.ref)
+    await sc.like(dan, three.ref)
+    await sc.like(eve, three.ref)
+    await sc.like(frank, three.ref)
+
    const res = await agent.api.app.bsky.unspecced.getPopular(
      {},
      { headers: sc.getHeaders(alice) },
    )
    const feedUris = res.data.feed.map((i) => i.post.uri).sort()
-    const expected = [one.ref.uriStr, two.ref.uriStr].sort()
+    const expected = [one.ref.uriStr, two.ref.uriStr, three.ref.uriStr].sort()
    expect(feedUris).toEqual(expected)
  })
 })
--- a/yarn.lock
+++ b/yarn.lock
@ -20,6 +20,17 @@
    pino "^8.6.1"
    zod "^3.14.2"

+"@atproto/crypto@0.1.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@atproto/crypto/-/crypto-0.1.0.tgz#bc73a479f9dbe06fa025301c182d7f7ab01bc568"
+  integrity sha512-9xgFEPtsCiJEPt9o3HtJT30IdFTGw5cQRSJVIy5CFhqBA4vDLcdXiRDLCjkzHEVbtNCsHUW6CrlfOgbeLPcmcg==
+  dependencies:
+    "@noble/secp256k1" "^1.7.0"
+    big-integer "^1.6.51"
+    multiformats "^9.6.4"
+    one-webcrypto "^1.0.3"
+    uint8arrays "3.0.0"
+
 "@aws-crypto/crc32@2.0.0":
  version "2.0.0"
  resolved "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-2.0.0.tgz"