Skip to content

Commit d8eaee5

Browse files
yuguoruiyuguorui
authored
fix(route/twitter): dynamically resolve GraphQL query IDs and fix production CookieAgent (#21544)
* fix(route/twitter): dynamically resolve GraphQL query IDs from Twitter JS bundles Twitter rotates GraphQL query IDs every 2-4 weeks, causing 404 errors. Instead of relying solely on hardcoded IDs, fetch and cache the latest query IDs from Twitter's client-web main.js bundle at runtime. - Add gql-id-resolver module to extract queryId/operationName pairs - Cache resolved IDs using CACHE_CONTENT_EXPIRE (Redis supported) - Fall back to hardcoded IDs if dynamic resolution fails - Update init() to trigger resolution before first API call Signed-off-by: yuguorui <yuguorui@pku.edu.cn> * fix(route/twitter): use undici.fetch directly to preserve CookieAgent dispatcher Both ofetch and the global wrappedFetch (request-rewriter) drop the dispatcher option when constructing a new Request object, which prevents the CookieAgent from attaching cookies in production builds. Use undici.fetch directly for Twitter API requests to preserve the dispatcher. Signed-off-by: yuguorui <yuguorui@pku.edu.cn> --------- Signed-off-by: yuguorui <yuguorui@pku.edu.cn>
1 parent 8ac41e7 commit d8eaee5

File tree

4 files changed

+205
-71
lines changed

4 files changed

+205
-71
lines changed

‎lib/routes/twitter/api/web-api/api.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import InvalidParameterError from '@/errors/types/invalid-parameter';
33
import cache from '@/utils/cache';
44
import ofetch from '@/utils/ofetch';
55

6-
import { baseUrl, gqlFeatures, gqlMap } from './constants';
6+
import { baseUrl, gqlFeatures, gqlMap, initGqlMap } from './constants';
77
import { gatherLegacyFromData, paginationTweets, twitterGot } from './utils';
88

99
const getUserData = (id) =>
@@ -216,5 +216,5 @@ export default {
216216
getList,
217217
getHomeTimeline,
218218
getHomeLatestTimeline,
219-
init: () => {},
219+
init: initGqlMap,
220220
};

‎lib/routes/twitter/api/web-api/constants.ts‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,14 @@
1+
import { buildGqlMap, fallbackIds, resolveQueryIds } from './gql-id-resolver';
2+
13
const baseUrl = 'https://x.com/i/api';
24

3-
const graphQLEndpointsPlain = [
4-
'/graphql/E3opETHurmVJflFsUBVuUQ/UserTweets',
5-
'/graphql/Yka-W8dz7RaEuQNkroPkYw/UserByScreenName',
6-
'/graphql/HJFjzBgCs16TqxewQOeLNg/HomeTimeline',
7-
'/graphql/DiTkXJgLqBBxCs7zaYsbtA/HomeLatestTimeline',
8-
'/graphql/bt4TKuFz4T7Ckk-VvQVSow/UserTweetsAndReplies',
9-
'/graphql/dexO_2tohK86JDudXXG3Yw/UserMedia',
10-
'/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId',
11-
'/graphql/UN1i3zUiCWa-6r-Uaho4fw/SearchTimeline',
12-
'/graphql/Pa45JvqZuKcW1plybfgBlQ/ListLatestTweetsTimeline',
13-
'/graphql/QuBlQ6SxNAQCt6-kBiCXCQ/TweetDetail',
14-
];
5+
// Initial gqlMap from fallback IDs, updated dynamically via initGqlMap()
6+
let gqlMap: Record<string, string> = buildGqlMap(fallbackIds);
157

16-
const gqlMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3].replace(/V2$|Query$|QueryV2$/, ''), endpoint]));
8+
const initGqlMap = async () => {
9+
const queryIds = await resolveQueryIds();
10+
gqlMap = buildGqlMap(queryIds);
11+
};
1712

1813
const thirdPartySupportedAPI = ['UserByScreenName', 'UserByRestId', 'UserTweets', 'UserTweetsAndReplies', 'ListLatestTweetsTimeline', 'SearchTimeline', 'UserMedia'];
1914

@@ -114,4 +109,4 @@ const timelineParams = {
114109

115110
const bearerToken = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
116111

117-
export { baseUrl, bearerToken, gqlFeatures, gqlMap, thirdPartySupportedAPI, timelineParams };
112+
export { baseUrl, bearerToken, gqlFeatures, gqlMap, initGqlMap, thirdPartySupportedAPI, timelineParams };

‎lib/routes/twitter/api/web-api/gql-id-resolver.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import { config } from '@/config';
2+
import cache from '@/utils/cache';
3+
import logger from '@/utils/logger';
4+
import ofetch from '@/utils/ofetch';
5+
6+
const CACHE_KEY = 'twitter:gql-query-ids';
7+
8+
// Hardcoded fallback IDs (last known working values)
9+
export const fallbackIds: Record<string, string> = {
10+
UserTweets: 'E3opETHurmVJflFsUBVuUQ',
11+
UserByScreenName: 'Yka-W8dz7RaEuQNkroPkYw',
12+
HomeTimeline: 'xhYBF94fPSp8ey64FfYXiA',
13+
HomeLatestTimeline: '0vp2Au9doTKsbn2vIk48Dg',
14+
UserTweetsAndReplies: 'bt4TKuFz4T7Ckk-VvQVSow',
15+
UserMedia: 'dexO_2tohK86JDudXXG3Yw',
16+
UserByRestId: 'Qw77dDjp9xCpUY-AXwt-yQ',
17+
SearchTimeline: 'UN1i3zUiCWa-6r-Uaho4fw',
18+
ListLatestTweetsTimeline: 'Pa45JvqZuKcW1plybfgBlQ',
19+
TweetDetail: 'QuBlQ6SxNAQCt6-kBiCXCQ',
20+
};
21+
22+
const operationNames = Object.keys(fallbackIds);
23+
24+
async function fetchTwitterPage(): Promise<string> {
25+
const response = await ofetch('https://x.com', {
26+
parseResponse: (txt) => txt,
27+
});
28+
return response as unknown as string;
29+
}
30+
31+
function extractQueryIds(scriptContent: string): Record<string, string> {
32+
const ids: Record<string, string> = {};
33+
const matches = scriptContent.matchAll(/queryId:"([^"]+?)".+?operationName:"([^"]+?)"/g);
34+
for (const match of matches) {
35+
const [, queryId, operationName] = match;
36+
if (operationNames.includes(operationName)) {
37+
ids[operationName] = queryId;
38+
}
39+
}
40+
return ids;
41+
}
42+
43+
async function fetchAndExtractIds(): Promise<Record<string, string>> {
44+
const html = await fetchTwitterPage();
45+
46+
// Extract main.hash.js URL — it contains all the GraphQL query IDs we need
47+
const mainMatch = html.match(/\/client-web\/main\.([a-z0-9]+)\./);
48+
if (!mainMatch) {
49+
logger.warn('twitter gql-id-resolver: main.js URL not found in Twitter page');
50+
return {};
51+
}
52+
53+
const mainUrl = `https://abs.twimg.com/responsive-web/client-web/main.${mainMatch[1]}.js`;
54+
logger.debug(`twitter gql-id-resolver: fetching ${mainUrl}`);
55+
56+
const content = await ofetch(mainUrl, {
57+
parseResponse: (txt) => txt,
58+
});
59+
return extractQueryIds(content as unknown as string);
60+
}
61+
62+
let resolvePromise: Promise<Record<string, string>> | null = null;
63+
64+
export async function resolveQueryIds(): Promise<Record<string, string>> {
65+
// Check cache first
66+
const cached = await cache.get(CACHE_KEY);
67+
if (cached) {
68+
try {
69+
const parsed = typeof cached === 'string' ? JSON.parse(cached) : cached;
70+
if (parsed && typeof parsed === 'object' && Object.keys(parsed).length > 0) {
71+
logger.debug(`twitter gql-id-resolver: using cached query IDs`);
72+
return { ...fallbackIds, ...parsed };
73+
}
74+
} catch {
75+
// ignore parse error
76+
}
77+
}
78+
79+
// Deduplicate concurrent requests
80+
if (!resolvePromise) {
81+
resolvePromise = (async () => {
82+
try {
83+
logger.info('twitter gql-id-resolver: fetching fresh query IDs from Twitter JS bundles');
84+
const ids = await fetchAndExtractIds();
85+
86+
if (Object.keys(ids).length > 0) {
87+
await cache.set(CACHE_KEY, JSON.stringify(ids), config.cache.contentExpire);
88+
const found = operationNames.filter((name) => ids[name]);
89+
const missing = operationNames.filter((name) => !ids[name]);
90+
logger.debug(`twitter gql-id-resolver: resolved ${found.length}/${operationNames.length} query IDs. Missing: ${missing.join(', ') || 'none'}`);
91+
} else {
92+
logger.warn('twitter gql-id-resolver: failed to extract any query IDs, using fallback');
93+
}
94+
95+
return ids;
96+
} catch (error) {
97+
logger.warn(`twitter gql-id-resolver: error fetching query IDs: ${error}. Using fallback.`);
98+
return {};
99+
} finally {
100+
resolvePromise = null;
101+
}
102+
})();
103+
}
104+
105+
const ids = await resolvePromise;
106+
return { ...fallbackIds, ...ids };
107+
}
108+
109+
export function buildGqlMap(queryIds: Record<string, string>): Record<string, string> {
110+
const map: Record<string, string> = {};
111+
for (const name of operationNames) {
112+
const id = queryIds[name] || fallbackIds[name];
113+
map[name] = `/graphql/${id}/${name}`;
114+
}
115+
return map;
116+
}

‎lib/routes/twitter/api/web-api/utils.ts‎

Lines changed: 78 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { cookie as HttpCookieAgentCookie, CookieAgent } from 'http-cookie-agent/undici';
22
import queryString from 'query-string';
33
import { Cookie, CookieJar } from 'tough-cookie';
4-
import { Client, ProxyAgent } from 'undici';
4+
import undici, { Client, ProxyAgent } from 'undici';
55

66
import { config } from '@/config';
77
import ConfigNotFoundError from '@/errors/types/config-not-found';
@@ -136,8 +136,21 @@ export const twitterGot = async (
136136
)
137137
: {};
138138

139-
const response = await ofetch.raw(requestUrl, {
140-
retry: 0,
139+
// Use undici.fetch directly instead of ofetch.raw to preserve the CookieAgent
140+
// dispatcher. Two layers drop it in the normal path:
141+
// 1. ofetch does not forward `dispatcher` to its internal fetch() call
142+
// 2. wrappedFetch (request-rewriter) does `new Request(input, init)` which
143+
// discards non-standard options like `dispatcher`
144+
// Additionally, setting `cookie` header manually doesn't work either because
145+
// the Fetch spec treats `cookie` as a forbidden header name, so
146+
// `new Request()` silently strips it.
147+
// The only way to send cookies via CookieAgent is to call undici.fetch with
148+
// the dispatcher option directly.
149+
//
150+
// Because undici.fetch is the standard Fetch API and does not support ofetch's
151+
// `onResponse` callback, the rate-limit and auth error handling that was
152+
// previously in `onResponse` is now inlined below.
153+
const response = await undici.fetch(requestUrl, {
141154
headers: {
142155
authority: 'x.com',
143156
accept: '*/*',
@@ -160,67 +173,77 @@ export const twitterGot = async (
160173
}),
161174
},
162175
dispatcher: dispatchers?.agent,
163-
onResponse: async ({ response }) => {
164-
const remaining = response.headers.get('x-rate-limit-remaining');
165-
const remainingInt = Number.parseInt(remaining || '0');
166-
const reset = response.headers.get('x-rate-limit-reset');
167-
logger.debug(
168-
`twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(response._data?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}`
169-
);
170-
if (auth) {
171-
if (remaining && remainingInt < 2 && reset) {
172-
const resetTime = new Date(Number.parseInt(reset) * 1000);
173-
const delay = (resetTime.getTime() - Date.now()) / 1000;
174-
logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`);
175-
await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2);
176-
} else if (response.status === 429 || JSON.stringify(response._data?.data) === '{"user":{}}') {
177-
logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`);
178-
await cache.set(`${lockPrefix}${auth.token}`, '1', 2000);
179-
} else if (response.status === 403 || response.status === 401) {
180-
const newCookie = await login({
181-
username: auth.username,
182-
password: auth.password,
183-
authenticationSecret: auth.authenticationSecret,
184-
});
185-
if (newCookie) {
186-
logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`);
187-
await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire);
188-
logger.debug(`twitter debug: unlock twitter cookie for token ${auth.token} with error1`);
189-
await cache.set(`${lockPrefix}${auth.token}`, '', 1);
190-
} else {
191-
const tokenIndex = config.twitter.authToken?.indexOf(auth.token);
192-
if (tokenIndex !== undefined && tokenIndex !== -1) {
193-
config.twitter.authToken?.splice(tokenIndex, 1);
194-
}
195-
if (auth.username) {
196-
const usernameIndex = config.twitter.username?.indexOf(auth.username);
197-
if (usernameIndex !== undefined && usernameIndex !== -1) {
198-
config.twitter.username?.splice(usernameIndex, 1);
199-
}
200-
}
201-
if (auth.password) {
202-
const passwordIndex = config.twitter.password?.indexOf(auth.password);
203-
if (passwordIndex !== undefined && passwordIndex !== -1) {
204-
config.twitter.password?.splice(passwordIndex, 1);
205-
}
206-
}
207-
logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`);
208-
await cache.set(`${lockPrefix}${auth.token}`, '1', 3600);
176+
});
177+
178+
let responseData: any;
179+
try {
180+
responseData = await response.json();
181+
} catch {
182+
responseData = null;
183+
}
184+
185+
// Handle rate limiting and auth errors
186+
const remaining = response.headers.get('x-rate-limit-remaining');
187+
const remainingInt = Number.parseInt(remaining || '0');
188+
const reset = response.headers.get('x-rate-limit-reset');
189+
logger.debug(
190+
`twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(responseData?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}`
191+
);
192+
if (auth) {
193+
if (remaining && remainingInt < 2 && reset) {
194+
const resetTime = new Date(Number.parseInt(reset) * 1000);
195+
const delay = (resetTime.getTime() - Date.now()) / 1000;
196+
logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`);
197+
await cache.set(`${lockPrefix}${auth.token}`, '1', Math.ceil(delay) * 2);
198+
} else if (response.status === 429 || JSON.stringify(responseData?.data) === '{"user":{}}') {
199+
logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`);
200+
await cache.set(`${lockPrefix}${auth.token}`, '1', 2000);
201+
} else if (response.status === 403 || response.status === 401) {
202+
const newCookie = await login({
203+
username: auth.username,
204+
password: auth.password,
205+
authenticationSecret: auth.authenticationSecret,
206+
});
207+
if (newCookie) {
208+
logger.debug(`twitter debug: reset twitter cookie for token ${auth.token}, ${newCookie}`);
209+
await cache.set(`twitter:cookie:${auth.token}`, newCookie, config.cache.contentExpire);
210+
await cache.set(`${lockPrefix}${auth.token}`, '', 1);
211+
} else {
212+
const tokenIndex = config.twitter.authToken?.indexOf(auth.token);
213+
if (tokenIndex !== undefined && tokenIndex !== -1) {
214+
config.twitter.authToken?.splice(tokenIndex, 1);
215+
}
216+
if (auth.username) {
217+
const usernameIndex = config.twitter.username?.indexOf(auth.username);
218+
if (usernameIndex !== undefined && usernameIndex !== -1) {
219+
config.twitter.username?.splice(usernameIndex, 1);
220+
}
221+
}
222+
if (auth.password) {
223+
const passwordIndex = config.twitter.password?.indexOf(auth.password);
224+
if (passwordIndex !== undefined && passwordIndex !== -1) {
225+
config.twitter.password?.splice(passwordIndex, 1);
209226
}
210-
} else {
211-
logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`);
212-
await cache.set(`${lockPrefix}${auth.token}`, '', 1);
213227
}
228+
logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`);
229+
await cache.set(`${lockPrefix}${auth.token}`, '1', 3600);
214230
}
215-
},
216-
});
231+
} else {
232+
logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`);
233+
await cache.set(`${lockPrefix}${auth.token}`, '', 1);
234+
}
235+
}
236+
237+
if (response.status >= 400) {
238+
throw new Error(`Twitter API error: ${response.status}`);
239+
}
217240

218241
if (auth?.token) {
219242
logger.debug(`twitter debug: update twitter cookie for token ${auth.token}`);
220243
await cache.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers?.jar.serializeSync()), config.cache.contentExpire);
221244
}
222245

223-
return response._data;
246+
return responseData;
224247
};
225248

226249
export const paginationTweets = async (endpoint: string, userId: number | undefined, variables: Record<string, any>, path?: string[]) => {

0 commit comments

Comments
 (0)