fix: harden twitter timeline review findings (#236)

Author: jackwener

docs/.vitepress/config.mts

@@ -102,9 +102,6 @@ export default defineConfig({
                 { text: 'ChatWise', link: '/adapters/desktop/chatwise' },
                 { text: 'Notion', link: '/adapters/desktop/notion' },
                 { text: 'Discord', link: '/adapters/desktop/discord' },
-                { text: 'Feishu', link: '/adapters/desktop/feishu' },
-                { text: 'WeChat', link: '/adapters/desktop/wechat' },
-                { text: 'NeteaseMusic', link: '/adapters/desktop/neteasemusic' },
               ],
             },
           ],

src/clis/twitter/timeline.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, it } from 'vitest';
+import { __test__ } from './timeline.js';
+
+describe('twitter timeline helpers', () => {
+  it('builds for-you variables with withCommunity', () => {
+    expect(__test__.buildTimelineVariables('for-you', 20)).toEqual({
+      count: 20,
+      includePromotedContent: false,
+      latestControlAvailable: true,
+      requestContext: 'launch',
+      withCommunity: true,
+    });
+  });
+
+  it('builds following variables with seenTweetIds instead of withCommunity', () => {
+    expect(__test__.buildTimelineVariables('following', 20, 'cursor-1')).toEqual({
+      count: 20,
+      includePromotedContent: false,
+      latestControlAvailable: true,
+      requestContext: 'launch',
+      seenTweetIds: [],
+      cursor: 'cursor-1',
+    });
+  });
+
+  it('encodes variables into timeline url', () => {
+    const url = __test__.buildHomeTimelineUrl('query123', 'HomeLatestTimeline', {
+      count: 20,
+      seenTweetIds: [],
+    });
+
+    expect(url).toContain('/i/api/graphql/query123/HomeLatestTimeline');
+    expect(url).toContain('variables=');
+    expect(url).toContain('features=');
+    expect(decodeURIComponent(url)).toContain('"seenTweetIds":[]');
+  });
+
+  it('parses tweets and bottom cursor from home timeline payload', () => {
+    const payload = {
+      data: {
+        home: {
+          home_timeline_urt: {
+            instructions: [
+              {
+                entries: [
+                  {
+                    entryId: 'tweet-1',
+                    content: {
+                      itemContent: {
+                        tweet_results: {
+                          result: {
+                            rest_id: '1',
+                            legacy: {
+                              full_text: 'hello',
+                              favorite_count: 3,
+                              retweet_count: 2,
+                              reply_count: 1,
+                              created_at: 'now',
+                            },
+                            core: {
+                              user_results: {
+                                result: {
+                                  legacy: {
+                                    screen_name: 'alice',
+                                  },
+                                },
+                              },
+                            },
+                            views: {
+                              count: '9',
+                            },
+                          },
+                        },
+                      },
+                    },
+                  },
+                  {
+                    entryId: 'cursor-bottom-1',
+                    content: {
+                      entryType: 'TimelineTimelineCursor',
+                      cursorType: 'Bottom',
+                      value: 'cursor-next',
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+        },
+      },
+    };
+
+    const result = __test__.parseHomeTimeline(payload, new Set());
+
+    expect(result.nextCursor).toBe('cursor-next');
+    expect(result.tweets).toHaveLength(1);
+    expect(result.tweets[0]).toMatchObject({
+      id: '1',
+      author: 'alice',
+      text: 'hello',
+      likes: 3,
+      retweets: 2,
+      replies: 1,
+      views: 9,
+      created_at: 'now',
+      url: 'https://x.com/alice/status/1',
+    });
+  });
+});

src/clis/twitter/timeline.ts

@@ -7,8 +7,16 @@ const BEARER_TOKEN =
 const HOME_TIMELINE_QUERY_ID = 'c-CzHF1LboFilMpsx4ZCrQ';
 const HOME_LATEST_TIMELINE_QUERY_ID = 'BKB7oi212Fi7kQtCBGE4zA';
 
+type TimelineType = 'for-you' | 'following';
+
+interface TimelineEndpointConfig {
+  endpoint: string;
+  method: 'GET' | 'POST';
+  fallbackQueryId: string;
+}
+
 // Endpoint config: for-you uses GET HomeTimeline, following uses POST HomeLatestTimeline
-const TIMELINE_ENDPOINTS: Record<string, { endpoint: string; method: string; fallbackQueryId: string }> = {
+const TIMELINE_ENDPOINTS: Record<TimelineType, TimelineEndpointConfig> = {
   'for-you': { endpoint: 'HomeTimeline', method: 'GET', fallbackQueryId: HOME_TIMELINE_QUERY_ID },
   following: { endpoint: 'HomeLatestTimeline', method: 'POST', fallbackQueryId: HOME_LATEST_TIMELINE_QUERY_ID },
 };
@@ -61,15 +69,20 @@ interface TimelineTweet {
   url: string;
 }
 
-function buildHomeTimelineUrl(queryId: string, endpoint: string, count: number, cursor?: string | null): string {
-  const vars: Record<string, any> = {
+function buildTimelineVariables(type: TimelineType, count: number, cursor?: string | null): Record<string, unknown> {
+  const vars: Record<string, unknown> = {
     count,
     includePromotedContent: false,
     latestControlAvailable: true,
     requestContext: 'launch',
-    withCommunity: true,
   };
+  if (type === 'for-you') vars.withCommunity = true;
+  if (type === 'following') vars.seenTweetIds = [];
   if (cursor) vars.cursor = cursor;
+  return vars;
+}
+
+function buildHomeTimelineUrl(queryId: string, endpoint: string, vars: Record<string, unknown>): string {
 
   return (
     `/i/api/graphql/${queryId}/${endpoint}` +
@@ -170,7 +183,8 @@ cli({
   columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url'],
   func: async (page, kwargs) => {
     const limit = kwargs.limit || 20;
-    const { endpoint, method, fallbackQueryId } = TIMELINE_ENDPOINTS[kwargs.type || 'for-you'];
+    const timelineType: TimelineType = kwargs.type === 'following' ? 'following' : 'for-you';
+    const { endpoint, method, fallbackQueryId } = TIMELINE_ENDPOINTS[timelineType];
 
     // Navigate to x.com for cookie context
     await page.goto('https://x.com');
@@ -212,7 +226,8 @@ cli({
 
     for (let i = 0; i < 5 && allTweets.length < limit; i++) {
       const fetchCount = Math.min(40, limit - allTweets.length + 5); // over-fetch slightly for promoted filtering
-      const apiUrl = buildHomeTimelineUrl(queryId, endpoint, fetchCount, cursor);
+      const variables = buildTimelineVariables(timelineType, fetchCount, cursor);
+      const apiUrl = buildHomeTimelineUrl(queryId, endpoint, variables);
 
       const data = await page.evaluate(`async () => {
         const r = await fetch("${apiUrl}", { method: "${method}", headers: ${headers}, credentials: 'include' });
@@ -235,3 +250,9 @@ cli({
     return allTweets.slice(0, limit);
   },
 });
+
+export const __test__ = {
+  buildTimelineVariables,
+  buildHomeTimelineUrl,
+  parseHomeTimeline,
+};