Grapheme counting for lex + increase post size

lexicons/app/bsky/feed/post.json

@@ -9,7 +9,7 @@
         "type": "object",
         "required": ["text", "createdAt"],
         "properties": {
-          "text": {"type": "string", "maxLength": 256},
+          "text": {"type": "string", "maxLength": 3000, "maxGraphemes": 300},
           "entities": {
             "type": "array",
             "items": {"type": "ref", "ref": "#entity"}

packages/api/src/client/lexicons.ts

@@ -3524,7 +3524,8 @@ export const schemaDict = {
           properties: {
             text: {
               type: 'string',
-              maxLength: 256,
+              maxLength: 3000,
+              maxGraphemes: 300,
             },
             entities: {
               type: 'array',

packages/common/src/index.ts

@@ -10,3 +10,4 @@ export * from './logger'
 export * from './types'
 export * from './streams'
 export * from './times'
+export * from './strings'

packages/common/src/strings.ts

@@ -0,0 +1,9 @@
+// counts the number of bytes in a utf8 string
+export const utf8Len = (str: string): number => {
+  return new TextEncoder().encode(str).byteLength
+}
+
+// counts the number of graphemes (user-displayed characters) in a string
+export const graphemeLen = (str: string): number => {
+  return [...new Intl.Segmenter().segment(str)].length
+}

packages/common/tests/strings.test.ts

@@ -0,0 +1,30 @@
+import { graphemeLen, utf8Len } from '../src'
+
+describe('string', () => {
+  it('calculates utf8 string length', () => {
+    expect(utf8Len('a')).toBe(1)
+    expect(utf8Len('~')).toBe(1)
+    expect(utf8Len('ö')).toBe(2)
+    expect(utf8Len('ñ')).toBe(2)
+    expect(utf8Len('©')).toBe(2)
+    expect(utf8Len('⽘')).toBe(3)
+    expect(utf8Len('☎')).toBe(3)
+    expect(utf8Len('𓋓')).toBe(4)
+    expect(utf8Len('😀')).toBe(4)
+    expect(utf8Len('👨‍👩‍👧‍👧')).toBe(25)
+  })
+
+  it('caluclates grapheme length', () => {
+    expect(graphemeLen('a')).toBe(1)
+    expect(graphemeLen('~')).toBe(1)
+    expect(graphemeLen('ö')).toBe(1)
+    expect(graphemeLen('ñ')).toBe(1)
+    expect(graphemeLen('©')).toBe(1)
+    expect(graphemeLen('⽘')).toBe(1)
+    expect(graphemeLen('☎')).toBe(1)
+    expect(graphemeLen('𓋓')).toBe(1)
+    expect(graphemeLen('😀')).toBe(1)
+    expect(graphemeLen('👨‍👩‍👧‍👧')).toBe(1)
+    expect(graphemeLen('a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧')).toBe(10)
+  })
+})

packages/lexicon/package.json

@@ -19,7 +19,9 @@
   },
   "license": "MIT",
   "dependencies": {
+    "@atproto/common": "*",
     "@atproto/nsid": "*",
+    "@atproto/uri": "*",
     "iso-datestring-validator": "^2.2.2",
     "zod": "^3.14.2"
   }

packages/lexicon/src/types.ts

@@ -44,6 +44,8 @@ export const lexString = z.object({
   default: z.string().optional(),
   minLength: z.number().int().optional(),
   maxLength: z.number().int().optional(),
+  minGraphemes: z.number().int().optional(),
+  maxGraphemes: z.number().int().optional(),
   enum: z.string().array().optional(),
   const: z.string().optional(),
   knownValues: z.string().array().optional(),

packages/lexicon/src/validators/primitives.ts

@@ -1,4 +1,5 @@
 import { AtUri } from '@atproto/uri'
+import * as common from '@atproto/common'
 import { isValidISODateString } from 'iso-datestring-validator'
 import { CID } from 'multiformats/cid'
 import { Lexicons } from '../lexicons'
@@ -224,7 +225,7 @@ export function string(
 
   // maxLength
   if (typeof def.maxLength === 'number') {
-    if ((value as string).length > def.maxLength) {
+    if (common.utf8Len(value) > def.maxLength) {
       return {
         success: false,
         error: new ValidationError(
@@ -236,7 +237,7 @@ export function string(
 
   // minLength
   if (typeof def.minLength === 'number') {
-    if ((value as string).length < def.minLength) {
+    if (common.utf8Len(value) < def.minLength) {
       return {
         success: false,
         error: new ValidationError(
@@ -246,6 +247,30 @@ export function string(
     }
   }
 
+  // maxGraphemes
+  if (typeof def.maxGraphemes === 'number') {
+    if (common.graphemeLen(value) > def.maxGraphemes) {
+      return {
+        success: false,
+        error: new ValidationError(
+          `${path} must not be longer than ${def.maxGraphemes} graphemes`,
+        ),
+      }
+    }
+  }
+
+  // minGraphemes
+  if (typeof def.minGraphemes === 'number') {
+    if (common.graphemeLen(value) < def.minGraphemes) {
+      return {
+        success: false,
+        error: new ValidationError(
+          `${path} must not be shorter than ${def.minGraphemes} graphemes`,
+        ),
+      }
+    }
+  }
+
   if (typeof def.format === 'string') {
     switch (def.format) {
       case 'datetime':

packages/lexicon/tests/_scaffolds/lexicons.ts

@@ -379,6 +379,25 @@ export default [
       },
     },
   },
+  {
+    lexicon: 1,
+    id: 'com.example.stringLengthGrapheme',
+    defs: {
+      main: {
+        type: 'record',
+        record: {
+          type: 'object',
+          properties: {
+            string: {
+              type: 'string',
+              minGraphemes: 2,
+              maxGraphemes: 4,
+            },
+          },
+        },
+      },
+    },
+  },
   {
     lexicon: 1,
     id: 'com.example.stringEnum',

packages/lexicon/tests/general.test.ts

@@ -567,6 +567,31 @@ describe('Record validation', () => {
         string: '12345',
       }),
     ).toThrow('Record/string must not be longer than 4 characters')
+    expect(() =>
+      lex.assertValidRecord('com.example.stringLength', {
+        $type: 'com.example.stringLength',
+        string: '👨‍👩‍👧‍👧',
+      }),
+    ).toThrow('Record/string must not be longer than 4 characters')
+  })
+
+  it('Applies grapheme string length constraint', () => {
+    lex.assertValidRecord('com.example.stringLengthGrapheme', {
+      $type: 'com.example.stringLengthGrapheme',
+      string: '12👨‍👩‍👧‍👧',
+    })
+    expect(() =>
+      lex.assertValidRecord('com.example.stringLengthGrapheme', {
+        $type: 'com.example.stringLengthGrapheme',
+        string: '👨‍👩‍👧‍👧',
+      }),
+    ).toThrow('Record/string must not be shorter than 2 graphemes')
+    expect(() =>
+      lex.assertValidRecord('com.example.stringLengthGrapheme', {
+        $type: 'com.example.stringLengthGrapheme',
+        string: '12345',
+      }),
+    ).toThrow('Record/string must not be longer than 4 graphemes')
   })
 
   it('Applies string enum constraint', () => {

packages/lexicon/tsconfig.json

@@ -7,6 +7,8 @@
   },
   "include": ["./src"],
   "references": [
-    { "path": "../nsid/tsconfig.build.json" }
+    { "path": "../common/tsconfig.build.json" },
+    { "path": "../nsid/tsconfig.build.json" },
+    { "path": "../uri/tsconfig.build.json" }
   ]
 }

packages/pds/src/lexicon/lexicons.ts

@@ -3524,7 +3524,8 @@ export const schemaDict = {
           properties: {
             text: {
               type: 'string',
-              maxLength: 256,
+              maxLength: 3000,
+              maxGraphemes: 300,
             },
             entities: {
               type: 'array',