refactor: twitter api base on public token (#10782)

* refactor: twitter api base on public token * refactor: code style * refactor: code style * refactor: lock file
DimensionDev · Sep 19, 2023 · 7d76620 · 7d76620
1 parent b7bbb9e
commit 7d76620
Show file tree

Hide file tree

Showing 10 changed files with 142 additions and 299 deletions.
diff --git a/packages/flags/src/flags/index.ts b/packages/flags/src/flags/index.ts
@@ -26,6 +26,10 @@ export const flags = {
     post_actions_enabled: true,
     sandboxedPluginRuntime: false,
 
+    // twitter
+    twitter_token:
+        'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+
     // sentry
     sentry_earliest_version: env.VERSION,
     sentry_sample_rate: 0.05,

diff --git a/...providers/src/Twitter/apis/getDatabase.ts → ...viders/src/Twitter/apis/getObjectStore.ts b/...providers/src/Twitter/apis/getDatabase.ts → ...viders/src/Twitter/apis/getObjectStore.ts
@@ -4,7 +4,7 @@ import { Sniffings } from '@masknet/shared-base'
 const DB_NAME = 'localforage'
 const DB_VERSION = 2
 
-export async function getDatabase() {
+async function getDatabase() {
     if (!Sniffings.is_firefox) {
         const databases = await indexedDB.databases()
         if (!databases.some((x) => x.name === DB_NAME && x.version === DB_VERSION)) return
@@ -22,7 +22,7 @@ export async function getDatabase() {
     })
 }
 
-export async function getStore(name: string) {
+export async function getObjectStore(name: string) {
     const database = await getDatabase()
     if (!database) throw new Error('Failed to read database.')
 

diff --git a/packages/web3-providers/src/Twitter/apis/getSettings.ts b/packages/web3-providers/src/Twitter/apis/getSettings.ts
@@ -1,23 +1,12 @@
-import urlcat from 'urlcat'
 import { getHeaders } from './getTokens.js'
-import type { TwitterBaseAPI } from '../../entry-types.js'
 import { fetchJSON } from '../../helpers/fetchJSON.js'
+import type { TwitterBaseAPI } from '../../entry-types.js'
 
 export async function getSettings() {
-    return fetchJSON<TwitterBaseAPI.Settings>(
-        urlcat('https://twitter.com/i/api/1.1/account/settings.json', {
-            include_mention_filter: false,
-            include_nsfw_user_flag: false,
-            include_nsfw_admin_flag: false,
-            include_ranked_timeline: false,
-            include_alt_text_compose: false,
-            include_country_code: false,
-            include_ext_dm_nsfw_media_filter: false,
+    return fetchJSON<TwitterBaseAPI.Settings>('https://api.twitter.com/1.1/account/settings.json', {
+        headers: await getHeaders({
+            referer: 'https://twitter.com/home',
         }),
-        {
-            headers: await getHeaders({
-                referer: 'https://twitter.com/home',
-            }),
-        },
-    )
+        credentials: 'include',
+    })
 }
diff --git a/packages/web3-providers/src/Twitter/apis/getTokens.ts b/packages/web3-providers/src/Twitter/apis/getTokens.ts
@@ -1,17 +1,4 @@
-import { escapeRegExp } from 'lodash-es'
-import { squashPromise } from '@masknet/web3-shared-base'
-import { fetchText } from '../../helpers/fetchText.js'
-
-function getScriptURL(content: string, name: string) {
-    const matchURL = new RegExp(
-        `https://abs.twimg.com/responsive-web/(client-web|client-web-\\w+){1}/${escapeRegExp(
-            `${name}.`,
-        )}\\w+${escapeRegExp('.js')}`,
-        'm',
-    )
-    const [url] = content.match(matchURL) ?? []
-    return url
-}
+import { Flags } from '@masknet/flags'
 
 function getCSRFToken() {
     const ct0 = document.cookie.split('; ').find((x) => x.includes('ct0'))
@@ -20,70 +7,12 @@ function getCSRFToken() {
     return value
 }
 
-function getScriptContentMatched(content: string, pattern: RegExp) {
-    const [, matched] = content.match(pattern) ?? []
-    return matched
-}
-
-async function fetchContent(url?: string) {
-    if (!url) return
-    return fetchText(url)
-}
-
-function getAPIScriptURL(content: string) {
-    const matches = content.match(/api:"(\w+)",/)
-    if (!matches) return
-    return `https://abs.twimg.com/responsive-web/client-web/api.${matches[1]}a.js`
-}
-
-async function getScripts() {
-    const indexContent = await fetchContent('https://twitter.com')
-    const swContent = await fetchContent('https://twitter.com/sw.js')
-
-    const allSettled = await Promise.allSettled([
-        fetchContent(getScriptURL(indexContent ?? '', 'main')),
-        fetchContent(getScriptURL(swContent ?? '', 'main')),
-        fetchContent(getScriptURL(swContent ?? '', 'bundle.UserNft')),
-        fetchContent(getAPIScriptURL(indexContent ?? '')),
-    ])
-    const [mainContentPrimary, mainContentSecondary, userNFT_Content, API_Content] = allSettled.map((x) =>
-        x.status === 'fulfilled' ? x.value ?? '' : '',
-    )
-
-    return {
-        mainContent: mainContentPrimary || mainContentSecondary,
-        userNFT_Content,
-        API_Content,
-    }
-}
-
-const getScriptSquashed = squashPromise('GET_TWITTER_SCRIPTS', getScripts, 60_000)
-
-export async function getTokens(operationName?: string) {
-    const { mainContent, userNFT_Content, API_Content } = await getScriptSquashed()
-    const bearerToken = getScriptContentMatched(mainContent ?? '', /Bearer (\w{59,}%3D\w{42,})/)
-    const queryToken = getScriptContentMatched(userNFT_Content ?? '', /{\s?id:\s?"([\w-]+)"/)
-    const csrfToken = getCSRFToken()
-    const queryId = operationName
-        ? getScriptContentMatched(API_Content ?? '', new RegExp(`queryId:"([^"]+)",operationName:"${operationName}"`))
-        : undefined
-
-    return {
-        bearerToken,
-        queryToken,
-        csrfToken,
-        queryId,
-    }
-}
-
 export async function getHeaders(overrides?: Record<string, string>) {
-    const { bearerToken, csrfToken } = await getTokens()
-
     return {
-        authorization: `Bearer ${bearerToken}`,
-        'x-csrf-token': csrfToken,
+        authorization: `Bearer ${Flags.twitter_token}`,
+        'x-csrf-token': getCSRFToken(),
         'x-twitter-auth-type': 'OAuth2Session',
-        referer: 'https://twitter.com/',
+        'x-twitter-client-language': navigator.language ? navigator.language : 'en',
         ...overrides,
     }
 }
diff --git a/packages/web3-providers/src/Twitter/apis/getUserNFTContainer.ts b/packages/web3-providers/src/Twitter/apis/getUserNFTContainer.ts
@@ -1,31 +1,26 @@
 import urlcat from 'urlcat'
-import { getTokens, getHeaders } from './getTokens.js'
+import { getHeaders } from './getTokens.js'
 import { fetchJSON } from '../../helpers/fetchJSON.js'
 import type { TwitterBaseAPI } from '../../entry-types.js'
 
 export async function getUserNFTContainer(screenName: string) {
-    const { queryToken } = await getTokens()
-
     return fetchJSON<{
         data: {
             user: {
                 result: TwitterBaseAPI.NFTContainer
             }
         }
     }>(
-        urlcat('https://twitter.com/i/api/graphql/:queryToken/userNftContainer_Query', {
-            queryToken,
+        urlcat('https://twitter.com/i/api/graphql/z-_uxIiYELU35OzocPdDIw/userNftContainer_Query', {
             variables: JSON.stringify({
                 screenName,
             }),
-            features: JSON.stringify({
-                responsive_web_twitter_blue_verified_badge_is_enabled: false,
-            }),
         }),
         {
             headers: await getHeaders({
                 referer: `https://twitter.com/${screenName}/nft`,
             }),
+            credentials: 'include',
         },
     )
 }
diff --git a/packages/web3-providers/src/Twitter/apis/getUserSettings.ts b/packages/web3-providers/src/Twitter/apis/getUserSettings.ts
@@ -1,5 +1,5 @@
 import { hexToRgb } from '@mui/material'
-import { getStore } from './getDatabase.js'
+import { getObjectStore } from './getObjectStore.js'
 import { TwitterBaseAPI } from '../../entry-types.js'
 
 /* cspell:disable-next-line */
@@ -8,7 +8,7 @@ const STORE_NAME = 'keyvaluepairs'
 const KEY_NAME = 'device:rweb.settings'
 
 export async function getUserSettings() {
-    const store = await getStore(STORE_NAME)
+    const store = await getObjectStore(STORE_NAME)
     const query = store.get(KEY_NAME)
 
     return new Promise<TwitterBaseAPI.UserSettings | undefined>(async (resolve, reject) => {

diff --git a/packages/web3-providers/src/Twitter/apis/getUserViaWebAPI.ts b/packages/web3-providers/src/Twitter/apis/getUserViaWebAPI.ts
@@ -1,7 +1,7 @@
 import urlcat from 'urlcat'
 import { isNull } from 'lodash-es'
 import { attemptTimes } from '@masknet/web3-shared-base'
-import { getTokens, getHeaders } from './getTokens.js'
+import { getHeaders } from './getTokens.js'
 import { fetchGlobal } from '../../helpers/fetchGlobal.js'
 import { Expiration } from '../../helpers/fetchSquashed.js'
 import { Duration, staleCached } from '../../helpers/fetchCached.js'
@@ -24,10 +24,7 @@ const features = {
 }
 
 async function createRequest(screenName: string) {
-    const { queryId } = await getTokens('UserByScreenName')
-    if (!queryId) return
-    const url = urlcat('https://twitter.com/i/api/graphql/:queryId/UserByScreenName', {
-        queryId,
+    const url = urlcat('https://twitter.com/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/UserByScreenName', {
         variables: JSON.stringify({
             screen_name: screenName,
             withSafetyModeUserFields: true,
@@ -38,8 +35,10 @@ async function createRequest(screenName: string) {
 
     return new Request(url, {
         headers: await getHeaders({
+            'content-type': 'application/json',
             referer: `https://twitter.com/${screenName}`,
         }),
+        credentials: 'include',
     })
 }
 
@@ -80,6 +79,7 @@ export const getUserViaWebTimesAPI = (screenName: string) => {
 export async function staleUserViaWebAPI(screenName: string): Promise<TwitterBaseAPI.User | null> {
     const request = await createRequest(screenName)
     if (!request) return null
+
     const response = await staleCached(request)
     if (!response?.ok) return null
 

diff --git a/packages/web3-providers/src/Twitter/apis/index.ts b/packages/web3-providers/src/Twitter/apis/index.ts
@@ -1,4 +1,4 @@
-export * from './getDatabase.js'
+export * from './getObjectStore.js'
 export * from './getSettings.js'
 export * from './getTokens.js'
 export * from './getUserNFTContainer.js'

diff --git a/packages/web3-providers/src/Twitter/index.ts b/packages/web3-providers/src/Twitter/index.ts
@@ -52,7 +52,7 @@ export class TwitterAPI implements TwitterBaseAPI.Provider {
     }
 
     async getUserNftContainer(screenName: string) {
-        const result = await attemptUntil<TwitterBaseAPI.NFT | undefined>(
+        return await attemptUntil<TwitterBaseAPI.NFT | undefined>(
             [
                 async () => {
                     const response = await getUserNFTContainer(screenName)
@@ -76,8 +76,6 @@ export class TwitterAPI implements TwitterBaseAPI.Provider {
             ],
             undefined,
         )
-
-        return result
     }
 
     async uploadUserAvatar(screenName: string, image: File | Blob): Promise<TwitterBaseAPI.TwitterResult> {
@@ -91,8 +89,8 @@ export class TwitterAPI implements TwitterBaseAPI.Provider {
             media_id_string: string
         }>(initURL, {
             method: 'POST',
-            credentials: 'include',
             headers,
+            credentials: 'include',
         })
 
         // APPEND
@@ -102,17 +100,17 @@ export class TwitterAPI implements TwitterBaseAPI.Provider {
         formData.append('media', image)
         await fetch(appendURL, {
             method: 'POST',
-            credentials: 'include',
-            body: formData,
             headers,
+            body: formData,
+            credentials: 'include',
         })
 
         // FINALIZE
         const finalizeURL = `${UPLOAD_AVATAR_URL}?command=FINALIZE&media_id=${mediaId}`
         return fetchJSON<TwitterBaseAPI.TwitterResult>(finalizeURL, {
             method: 'POST',
-            credentials: 'include',
             headers,
+            credentials: 'include',
         })
     }
 
@@ -125,10 +123,10 @@ export class TwitterAPI implements TwitterBaseAPI.Provider {
             }),
             {
                 method: 'POST',
-                credentials: 'include',
                 headers: await getHeaders({
                     referer: `https://twitter.com/${screenName}`,
                 }),
+                credentials: 'include',
             },
         )