Appview: apply epoch for indexed-at times in views (#2943)

* appview: start setting up index-at epoch, applying to posts * appview: present all indexed-at times w/ epoch * appview: update Views constructor to take args as object * tidy
bluesky-social · Nov 4, 2024 · 12af21e · 12af21e
1 parent 709ba30
commit 12af21e
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 13 deletions.
diff --git a/packages/bsky/src/config.ts b/packages/bsky/src/config.ts
@@ -36,6 +36,7 @@ export interface ServerConfigValues {
   modServiceDid: string
   adminPasswords: string[]
   labelsFromIssuerDids?: string[]
+  indexedAtEpoch?: Date
   // misc/dev
   blobCacheLocation?: string
   statsigKey?: string
@@ -125,6 +126,13 @@ export class ServerConfig {
         : process.env.BSKY_STATSIG_ENV || 'development'
     const clientCheckEmailConfirmed =
       process.env.BSKY_CLIENT_CHECK_EMAIL_CONFIRMED === 'true'
+    const indexedAtEpoch = process.env.BSKY_INDEXED_AT_EPOCH
+      ? new Date(process.env.BSKY_INDEXED_AT_EPOCH)
+      : undefined
+    assert(
+      !indexedAtEpoch || !isNaN(indexedAtEpoch.getTime()),
+      'invalid BSKY_INDEXED_AT_EPOCH',
+    )
     return new ServerConfig({
       version,
       debugMode,
@@ -161,6 +169,7 @@ export class ServerConfig {
       statsigKey,
       statsigEnv,
       clientCheckEmailConfirmed,
+      indexedAtEpoch,
       ...stripUndefineds(overrides ?? {}),
     })
   }
@@ -317,6 +326,10 @@ export class ServerConfig {
   get clientCheckEmailConfirmed() {
     return this.cfg.clientCheckEmailConfirmed
   }
+
+  get indexedAtEpoch() {
+    return this.cfg.indexedAtEpoch
+  }
 }
 
 function stripUndefineds(

diff --git a/packages/bsky/src/hydration/actor.ts b/packages/bsky/src/hydration/actor.ts
@@ -17,6 +17,7 @@ export type Actor = {
   profileCid?: string
   profileTakedownRef?: string
   sortedAt?: Date
+  indexedAt?: Date
   takedownRef?: string
   isLabeler: boolean
   allowIncomingChatsFrom?: string
@@ -126,6 +127,7 @@ export class ActorHydrator {
         profileCid: profile?.cid,
         profileTakedownRef: safeTakedownRef(profile),
         sortedAt: profile?.sortedAt?.toDate(),
+        indexedAt: profile?.indexedAt?.toDate(),
         takedownRef: safeTakedownRef(actor),
         isLabeler: actor.labeler ?? false,
         allowIncomingChatsFrom: actor.allowIncomingChatsFrom || undefined,

diff --git a/packages/bsky/src/hydration/hydrator.ts b/packages/bsky/src/hydration/hydrator.ts
@@ -976,6 +976,7 @@ export class Hydrator {
         record: actor.profile,
         cid: actor.profileCid,
         sortedAt: actor.sortedAt ?? new Date(0), // @NOTE will be present since profile record is present
+        indexedAt: actor.indexedAt ?? new Date(0), // @NOTE will be present since profile record is present
         takedownRef: actor.profileTakedownRef,
       }
     }

diff --git a/packages/bsky/src/hydration/util.ts b/packages/bsky/src/hydration/util.ts
@@ -22,6 +22,7 @@ export type RecordInfo<T> = {
   record: T
   cid: string
   sortedAt: Date
+  indexedAt: Date
   takedownRef: string | undefined
 }
 
@@ -65,6 +66,7 @@ export const parseRecord = <T>(
   const record = parseRecordBytes<T>(entry.record)
   const cid = entry.cid
   const sortedAt = entry.sortedAt?.toDate() ?? new Date(0)
+  const indexedAt = entry.indexedAt?.toDate() ?? new Date(0)
   if (!record || !cid) return
   if (!isValidRecord(record)) {
     return
@@ -73,6 +75,7 @@ export const parseRecord = <T>(
     record,
     cid,
     sortedAt,
+    indexedAt,
     takedownRef: safeTakedownRef(entry),
   }
 }

diff --git a/packages/bsky/src/index.ts b/packages/bsky/src/index.ts
@@ -101,7 +101,11 @@ export class BskyAppView {
       rejectUnauthorized: !config.dataplaneIgnoreBadTls,
     })
     const hydrator = new Hydrator(dataplane, config.labelsFromIssuerDids)
-    const views = new Views(imgUriBuilder, videoUriBuilder)
+    const views = new Views({
+      imgUriBuilder: imgUriBuilder,
+      videoUriBuilder: videoUriBuilder,
+      indexedAtEpoch: config.indexedAtEpoch,
+    })
 
     const bsyncClient = createBsyncClient({
       baseUrl: config.bsyncUrl,

diff --git a/packages/bsky/src/views/index.ts b/packages/bsky/src/views/index.ts
@@ -75,9 +75,15 @@ import { Notification } from '../proto/bsky_pb'
 import { postUriToThreadgateUri, postUriToPostgateUri } from '../util/uris'
 
 export class Views {
+  public imgUriBuilder: ImageUriBuilder = this.opts.imgUriBuilder
+  public videoUriBuilder: VideoUriBuilder = this.opts.videoUriBuilder
+  public indexedAtEpoch: Date | undefined = this.opts.indexedAtEpoch
   constructor(
-    public imgUriBuilder: ImageUriBuilder,
-    public videoUriBuilder: VideoUriBuilder,
+    private opts: {
+      imgUriBuilder: ImageUriBuilder
+      videoUriBuilder: VideoUriBuilder
+      indexedAtEpoch: Date | undefined
+    },
   ) {}
 
   // Actor
@@ -182,7 +188,13 @@ export class Views {
     return {
       ...basicView,
       description: actor.profile?.description || undefined,
-      indexedAt: actor.sortedAt?.toISOString(),
+      indexedAt:
+        actor.indexedAt && actor.sortedAt
+          ? this.indexedAt({
+              sortedAt: actor.sortedAt,
+              indexedAt: actor.indexedAt,
+            }).toISOString()
+          : undefined,
     }
   }
 
@@ -330,7 +342,7 @@ export class Views {
       creator,
       description: list.record.description,
       descriptionFacets: list.record.descriptionFacets,
-      indexedAt: list.sortedAt.toISOString(),
+      indexedAt: this.indexedAt(list).toISOString(),
     }
   }
 
@@ -356,7 +368,7 @@ export class Views {
           )
         : undefined,
       listItemCount: listAgg?.listItems ?? 0,
-      indexedAt: list.sortedAt.toISOString(),
+      indexedAt: this.indexedAt(list).toISOString(),
       labels,
       viewer: listViewer
         ? {
@@ -386,7 +398,7 @@ export class Views {
       joinedAllTimeCount: agg?.joinedAllTime ?? 0,
       joinedWeekCount: agg?.joinedWeek ?? 0,
       labels,
-      indexedAt: sp.sortedAt.toISOString(),
+      indexedAt: this.indexedAt(sp).toISOString(),
     }
   }
 
@@ -463,7 +475,7 @@ export class Views {
             like: viewer.like,
           }
         : undefined,
-      indexedAt: labeler.sortedAt.toISOString(),
+      indexedAt: this.indexedAt(labeler).toISOString(),
       labels,
     }
   }
@@ -551,7 +563,7 @@ export class Views {
             like: viewer.like,
           }
         : undefined,
-      indexedAt: feedgen.sortedAt.toISOString(),
+      indexedAt: this.indexedAt(feedgen).toISOString(),
     }
   }
 
@@ -600,7 +612,7 @@ export class Views {
       repostCount: aggs?.reposts ?? 0,
       likeCount: aggs?.likes ?? 0,
       quoteCount: aggs?.quotes ?? 0,
-      indexedAt: post.sortedAt.toISOString(),
+      indexedAt: this.indexedAt(post).toISOString(),
       viewer: viewer
         ? {
             repost: viewer.repost,
@@ -724,7 +736,7 @@ export class Views {
     return {
       $type: 'app.bsky.feed.defs#reasonRepost',
       by: creator,
-      indexedAt: repost.sortedAt.toISOString(),
+      indexedAt: this.indexedAt(repost).toISOString(),
     }
   }
 
@@ -1170,11 +1182,12 @@ export class Views {
     } else if (uri.collection === ids.AppBskyActorProfile) {
       const actor = state.actors?.get(authorDid)
       recordInfo =
-        actor && actor.profile && actor.profileCid && actor.sortedAt
+        actor && actor.profile && actor.profileCid
           ? {
               record: actor.profile,
               cid: actor.profileCid,
-              sortedAt: actor.sortedAt,
+              sortedAt: actor.sortedAt ?? new Date(0), // @NOTE will be present since profile record is present
+              indexedAt: actor.indexedAt ?? new Date(0), // @NOTE will be present since profile record is present
               takedownRef: actor.profileTakedownRef,
             }
           : null
@@ -1202,6 +1215,11 @@ export class Views {
       labels: [...labels, ...selfLabels],
     }
   }
+
+  indexedAt({ sortedAt, indexedAt }: { sortedAt: Date; indexedAt: Date }) {
+    if (!this.indexedAtEpoch) return sortedAt
+    return indexedAt && indexedAt > this.indexedAtEpoch ? indexedAt : sortedAt
+  }
 }
 
 const getRootUri = (uri: string, post: Post): string => {