Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Post record language tagging, lexicon language format #1219

Merged
merged 10 commits into from
Jun 23, 2023
Merged

Post record language tagging, lexicon language format #1219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4edd1a0
Add languages field to post record
devinivy Jun 19, 2023
ca1e293
helper for parsing bcp47
devinivy Jun 19, 2023
3c9de1b
add language format to lexicon
devinivy Jun 19, 2023
f2ab143
codegen for post record langs field
devinivy Jun 19, 2023
9605bd8
re-export language parsing in api package
devinivy Jun 19, 2023
4f3b679
tests and tidy for lexicon language format
devinivy Jun 19, 2023
d1d27d2
index post langs, in-progress
devinivy Jun 20, 2023
afca83a
update snapshots, fixes record-with-media embed issue
devinivy Jun 20, 2023
29562d3
index post langs on bsky appview
devinivy Jun 20, 2023
9241c0a
don't bother indexing post langs in pds appview, tidy
devinivy Jun 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions lexicons/app/bsky/feed/post.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@
"app.bsky.embed.recordWithMedia"
]
},
"langs": {
"type": "array",
"maxLength": 3,
"items": {"type": "string", "format": "language"}
},
"createdAt": {"type": "string", "format": "datetime"}
}
}
Expand Down
8 changes: 8 additions & 0 deletions packages/api/src/client/lexicons.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5244,6 +5244,14 @@ export const schemaDict = {
'lex:app.bsky.embed.recordWithMedia',
],
},
langs: {
type: 'array',
maxLength: 3,
items: {
type: 'string',
format: 'language',
},
},
createdAt: {
type: 'string',
format: 'datetime',
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/client/types/app/bsky/feed/post.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export interface Record {
| AppBskyEmbedRecord.Main
| AppBskyEmbedRecordWithMedia.Main
| { $type: string; [k: string]: unknown }
langs?: string[]
createdAt: string
[k: string]: unknown
}
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export {
jsonToLex,
jsonStringToLex,
} from '@atproto/lexicon'
export { parseLanguage } from '@atproto/common-web'
export * from './types'
export * from './client'
export * from './agent'
Expand Down
9 changes: 9 additions & 0 deletions packages/bsky/src/db/migrations/20230620T161134972Z-post-langs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { Kysely } from 'kysely'

export async function up(db: Kysely<unknown>): Promise<void> {
await db.schema.alterTable('post').addColumn('langs', 'jsonb').execute()
}

export async function down(db: Kysely<unknown>): Promise<void> {
await db.schema.alterTable('post').dropColumn('langs').execute()
}
1 change: 1 addition & 0 deletions packages/bsky/src/db/migrations/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,4 @@ export * as _20230609T153623961Z from './20230609T153623961Z-blocks'
export * as _20230609T232122649Z from './20230609T232122649Z-actor-deletion-indexes'
export * as _20230610T203555962Z from './20230610T203555962Z-suggested-follows'
export * as _20230611T215300060Z from './20230611T215300060Z-actor-state'
export * as _20230620T161134972Z from './20230620T161134972Z-post-langs'
1 change: 1 addition & 0 deletions packages/bsky/src/db/tables/post.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export interface Post {
replyRootCid: string | null
replyParent: string | null
replyParentCid: string | null
langs: string[] | null
createdAt: string
indexedAt: string
sortAt: GeneratedAlways<string>
Expand Down
8 changes: 8 additions & 0 deletions packages/bsky/src/lexicon/lexicons.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5244,6 +5244,14 @@ export const schemaDict = {
'lex:app.bsky.embed.recordWithMedia',
],
},
langs: {
type: 'array',
maxLength: 3,
items: {
type: 'string',
format: 'language',
},
},
createdAt: {
type: 'string',
format: 'datetime',
Expand Down
1 change: 1 addition & 0 deletions packages/bsky/src/lexicon/types/app/bsky/feed/post.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export interface Record {
| AppBskyEmbedRecord.Main
| AppBskyEmbedRecordWithMedia.Main
| { $type: string; [k: string]: unknown }
langs?: string[]
createdAt: string
[k: string]: unknown
}
Expand Down
3 changes: 3 additions & 0 deletions packages/bsky/src/services/indexing/plugins/post.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ const insertFn = async (
replyRootCid: obj.reply?.root?.cid || null,
replyParent: obj.reply?.parent?.uri || null,
replyParentCid: obj.reply?.parent?.cid || null,
langs: obj.langs?.length
? sql<string[]>`${JSON.stringify(obj.langs)}` // sidesteps kysely's array serialization, which is non-jsonb
: null,
indexedAt: timestamp,
}
const [insertedPost] = await Promise.all([
Expand Down
16 changes: 16 additions & 0 deletions packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -287,6 +291,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -508,6 +516,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -698,6 +710,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down
4 changes: 3 additions & 1 deletion packages/bsky/tests/seeds/basic.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ export default async (sc: SeedClient, users = true) => {
await sc.follow(bob, carol, createdAtMicroseconds())
await sc.follow(dan, bob, createdAtTimezone())
await sc.post(alice, posts.alice[0])
await sc.post(bob, posts.bob[0])
await sc.post(bob, posts.bob[0], undefined, undefined, undefined, {
langs: ['en-US', 'i-klingon'],
})
const img1 = await sc.uploadFile(
carol,
'tests/image/fixtures/key-landscape-small.jpg',
Expand Down
16 changes: 16 additions & 0 deletions packages/bsky/tests/views/__snapshots__/author-feed.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -638,6 +642,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -892,6 +900,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -1063,6 +1075,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down
12 changes: 12 additions & 0 deletions packages/bsky/tests/views/__snapshots__/posts.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -136,6 +140,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -261,6 +269,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down
52 changes: 52 additions & 0 deletions packages/bsky/tests/views/__snapshots__/timeline.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -795,6 +799,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -957,6 +965,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -1750,6 +1762,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -1937,6 +1953,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -2034,6 +2054,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -2151,6 +2175,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -2947,6 +2975,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -3042,6 +3074,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -3160,6 +3196,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -3782,6 +3822,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -4176,6 +4220,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -4328,6 +4376,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down
Loading