Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Post record language tagging, lexicon language format #1219

Merged
merged 10 commits into from
Jun 23, 2023
5 changes: 5 additions & 0 deletions lexicons/app/bsky/feed/post.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@
"app.bsky.embed.recordWithMedia"
]
},
"langs": {
"type": "array",
"maxLength": 3,
"items": {"type": "string", "format": "language"}
},
"createdAt": {"type": "string", "format": "datetime"}
}
}
Expand Down
8 changes: 8 additions & 0 deletions packages/api/src/client/lexicons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5244,6 +5244,14 @@ export const schemaDict = {
'lex:app.bsky.embed.recordWithMedia',
],
},
langs: {
type: 'array',
maxLength: 3,
items: {
type: 'string',
format: 'language',
},
},
createdAt: {
type: 'string',
format: 'datetime',
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/client/types/app/bsky/feed/post.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export interface Record {
| AppBskyEmbedRecord.Main
| AppBskyEmbedRecordWithMedia.Main
| { $type: string; [k: string]: unknown }
langs?: string[]
createdAt: string
[k: string]: unknown
}
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export {
jsonToLex,
jsonStringToLex,
} from '@atproto/lexicon'
export { parseLanguage } from '@atproto/common-web'
export * from './types'
export * from './client'
export * from './agent'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { Kysely } from 'kysely'

export async function up(db: Kysely<unknown>): Promise<void> {
await db.schema.alterTable('post').addColumn('langs', 'jsonb').execute()
}

export async function down(db: Kysely<unknown>): Promise<void> {
await db.schema.alterTable('post').dropColumn('langs').execute()
}
1 change: 1 addition & 0 deletions packages/bsky/src/db/migrations/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,4 @@ export * as _20230609T153623961Z from './20230609T153623961Z-blocks'
export * as _20230609T232122649Z from './20230609T232122649Z-actor-deletion-indexes'
export * as _20230610T203555962Z from './20230610T203555962Z-suggested-follows'
export * as _20230611T215300060Z from './20230611T215300060Z-actor-state'
export * as _20230620T161134972Z from './20230620T161134972Z-post-langs'
1 change: 1 addition & 0 deletions packages/bsky/src/db/tables/post.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export interface Post {
replyRootCid: string | null
replyParent: string | null
replyParentCid: string | null
langs: string[] | null
createdAt: string
indexedAt: string
sortAt: GeneratedAlways<string>
Expand Down
8 changes: 8 additions & 0 deletions packages/bsky/src/lexicon/lexicons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5244,6 +5244,14 @@ export const schemaDict = {
'lex:app.bsky.embed.recordWithMedia',
],
},
langs: {
type: 'array',
maxLength: 3,
items: {
type: 'string',
format: 'language',
},
},
createdAt: {
type: 'string',
format: 'datetime',
Expand Down
1 change: 1 addition & 0 deletions packages/bsky/src/lexicon/types/app/bsky/feed/post.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export interface Record {
| AppBskyEmbedRecord.Main
| AppBskyEmbedRecordWithMedia.Main
| { $type: string; [k: string]: unknown }
langs?: string[]
createdAt: string
[k: string]: unknown
}
Expand Down
3 changes: 3 additions & 0 deletions packages/bsky/src/services/indexing/plugins/post.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ const insertFn = async (
replyRootCid: obj.reply?.root?.cid || null,
replyParent: obj.reply?.parent?.uri || null,
replyParentCid: obj.reply?.parent?.cid || null,
langs: obj.langs?.length
? sql<string[]>`${JSON.stringify(obj.langs)}` // sidesteps kysely's array serialization, which is non-jsonb
: null,
indexedAt: timestamp,
}
const [insertedPost] = await Promise.all([
Expand Down
16 changes: 16 additions & 0 deletions packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -287,6 +291,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -508,6 +516,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -698,6 +710,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down
4 changes: 3 additions & 1 deletion packages/bsky/tests/seeds/basic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ export default async (sc: SeedClient, users = true) => {
await sc.follow(bob, carol, createdAtMicroseconds())
await sc.follow(dan, bob, createdAtTimezone())
await sc.post(alice, posts.alice[0])
await sc.post(bob, posts.bob[0])
await sc.post(bob, posts.bob[0], undefined, undefined, undefined, {
langs: ['en-US', 'i-klingon'],
})
const img1 = await sc.uploadFile(
carol,
'tests/image/fixtures/key-landscape-small.jpg',
Expand Down
16 changes: 16 additions & 0 deletions packages/bsky/tests/views/__snapshots__/author-feed.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -638,6 +642,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -892,6 +900,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -1063,6 +1075,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down
12 changes: 12 additions & 0 deletions packages/bsky/tests/views/__snapshots__/posts.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -136,6 +140,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -261,6 +269,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down
52 changes: 52 additions & 0 deletions packages/bsky/tests/views/__snapshots__/timeline.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -795,6 +799,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -957,6 +965,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -1750,6 +1762,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -1937,6 +1953,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -2034,6 +2054,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -2151,6 +2175,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -2947,6 +2975,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -3042,6 +3074,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down Expand Up @@ -3160,6 +3196,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -3782,6 +3822,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -4176,6 +4220,10 @@ Array [
"value": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
},
Expand Down Expand Up @@ -4328,6 +4376,10 @@ Array [
"record": Object {
"$type": "app.bsky.feed.post",
"createdAt": "1970-01-01T00:00:00.000Z",
"langs": Array [
"en-US",
"i-klingon",
],
"text": "bob back at it again!",
},
"replyCount": 0,
Expand Down
Loading