Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(route): threads #15239

Merged
merged 1 commit into from
Apr 15, 2024
Merged

fix(route): threads #15239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 36 additions & 38 deletions lib/routes/threads/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,16 @@
import { Route } from '@/types';
import got from '@/utils/got';
import ofetch from '@/utils/ofetch';
import { parseDate } from '@/utils/parse-date';
import { PROFILE_QUERY, REPLIES_QUERY, THREADS_QUERY, apiUrl, threadUrl, profileUrl, extractTokens, makeHeader, buildContent } from './utils';
import { REPLIES_QUERY, THREADS_QUERY, apiUrl, threadUrl, profileUrl, extractTokens, makeHeader, getUserId, buildContent } from './utils';
import { destr } from 'destr';
import cache from '@/utils/cache';
import { config } from '@/config';

export const route: Route = {
path: '/:user/:routeParams?',
categories: ['social-media'],
example: '/threads/zuck',
parameters: { user: 'Username', routeParams: 'Extra parameters, see the table below' },
features: {
requireConfig: false,
requirePuppeteer: false,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
supportScihub: false,
},
name: 'User timeline',
maintainers: ['ninboy'],
handler,
Expand All @@ -40,11 +35,13 @@ export const route: Route = {

async function handler(ctx) {
const { user, routeParams } = ctx.req.param();
const { lsd, userId } = await extractTokens(user, ctx);
const { lsd } = await extractTokens(user);
const userId = await getUserId(user, lsd);

const params = new URLSearchParams(routeParams);
const json = {
const debugJson = {
params: routeParams,
lsd,
};

const options = {
Expand All @@ -57,32 +54,33 @@ async function handler(ctx) {
replies: params.get('replies') ?? false,
};

const { data: profileResponse } = await got.post(apiUrl, {
headers: makeHeader(user, lsd),
form: {
lsd,
variables: JSON.stringify({ userID: userId }),
doc_id: PROFILE_QUERY,
},
});

const { data: threadsResponse, request: threadsRequest } = await got.post(apiUrl, {
headers: makeHeader(user, lsd),
form: {
lsd,
variables: JSON.stringify({ userID: userId }),
doc_id: options.replies ? REPLIES_QUERY : THREADS_QUERY,
},
});
const threadsResponse = await cache.tryGet(
`threads:${userId}:${options.replies}`,
() =>
ofetch(apiUrl, {
method: 'POST',
headers: {
...makeHeader(user, lsd),
'content-type': 'application/x-www-form-urlencoded',
},
body: new URLSearchParams({
lsd,
variables: JSON.stringify({ userID: userId }),
doc_id: String(options.replies ? REPLIES_QUERY : THREADS_QUERY),
}).toString(),
parseResponse: (txt) => destr(txt),
}),
config.cache.routeExpire,
false
);

json.profileData = profileResponse;
json.request = {
headers: threadsRequest.options.headers,
body: threadsRequest.options.body,
debugJson.profileId = userId;
debugJson.response = {
response: threadsResponse,
};

const userData = profileResponse?.data?.userData?.user || {};
const threads = threadsResponse?.data?.mediaData?.threads || [];
const userData = threadsResponse?.data?.mediaData?.threads?.[0]?.thread_items?.[0]?.post?.user || {};

const items = threads.flatMap((thread) =>
thread.thread_items
Expand All @@ -99,14 +97,14 @@ async function handler(ctx) {
})
);

json.items = items;
ctx.set('json', json);
debugJson.items = items;
ctx.set('json', debugJson);

return {
title: `${user} (@${user}) on Threads`,
link: profileUrl(user),
image: userData.hd_profile_pic_versions?.sort((a, b) => b.width - a.width)[0].url ?? userData.profile_pic_url,
description: userData.biography,
image: userData?.profile_pic_url,
// description: userData.biography,
item: items,
};
}
55 changes: 43 additions & 12 deletions lib/routes/threads/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
import got from '@/utils/got';
import ofetch from '@/utils/ofetch';
import { load } from 'cheerio';
import dayjs from 'dayjs';
import cache from '@/utils/cache';
import { destr } from 'destr';
import NotFoundError from '@/errors/types/not-found';

const profileUrl = (user) => `https://www.threads.net/@${user}`;
const threadUrl = (code) => `https://www.threads.net/t/${code}`;
const profileUrl = (user: string) => `https://www.threads.net/@${user}`;
const threadUrl = (code: string) => `https://www.threads.net/t/${code}`;

const apiUrl = 'https://www.threads.net/api/graphql';
const PROFILE_QUERY = 23_996_318_473_300_828;
// const PROFILE_QUERY = 23_996_318_473_300_828; // no longer works
const THREADS_QUERY = 6_232_751_443_445_612;
const REPLIES_QUERY = 6_307_072_669_391_286;
const USER_AGENT = 'Barcelona 289.0.0.77.109 Android';
const appId = '238260118697367';
const asbdId = '129477';

const extractTokens = async (user, ctx) => {
const { data: response } = await got(profileUrl(user), {
const extractTokens = async (user): Promise<{ lsd: string }> => {
const response = await ofetch(profileUrl(user), {
headers: {
'User-Agent': USER_AGENT,
'X-IG-App-ID': appId,
Expand All @@ -24,24 +28,51 @@ const extractTokens = async (user, ctx) => {
const data = $('script:contains("LSD"):first').text();

const lsd = data.match(/"LSD",\[],{"token":"([\w@-]+)"},/)?.[1];
if (!lsd) {
throw new NotFoundError('LSD token not found');
}

const userId = data.match(/{"user_id":"(\d+)"},/)?.[1];
// const userId = data.match(/{"user_id":"(\d+)"},/)?.[1];

const ret = { lsd, userId };
ctx.set('json', ret);
const ret = { lsd };
return ret;
};

const makeHeader = (user, lsd) => ({
Accept: 'application/json',
const makeHeader = (user: string, lsd: string) => ({
Accept: '*/*',
Host: 'www.threads.net',
Origin: 'https://www.threads.net',
Referer: profileUrl(user),
'User-Agent': USER_AGENT,
'X-FB-LSD': lsd,
'X-IG-App-ID': appId,
'Sec-Fetch-Site': 'same-origin',
});

const getUserId = (user: string, lsd: string): Promise<string> =>
cache.tryGet(`threads:userId:${user}`, async () => {
const pathName = `/@${user}`;
const payload: any = {
'route_urls[0]': pathName,
__a: '1',
__comet_req: '29',
lsd,
};
const response = await ofetch('https://www.threads.net/ajax/bulk-route-definitions/', {
method: 'POST',
headers: {
...makeHeader(user, lsd),
'content-type': 'application/x-www-form-urlencoded',
'X-ASBD-ID': asbdId,
},
body: new URLSearchParams(payload).toString(),
parseResponse: (txt) => destr(txt.slice(9)), // remove "for (;;);"
});

const userId = response.payload.payloads[pathName].result.exports.rootView.props.user_id;
return userId;
});

const hasMedia = (post) => post.image_versions2 || post.carousel_media || post.video_versions;
const buildMedia = (post) => {
let html = '';
Expand Down Expand Up @@ -132,4 +163,4 @@ const buildContent = (item, options) => {
return { title, description };
};

export { apiUrl, profileUrl, threadUrl, PROFILE_QUERY, THREADS_QUERY, REPLIES_QUERY, USER_AGENT, extractTokens, makeHeader, hasMedia, buildMedia, buildContent };
export { apiUrl, profileUrl, threadUrl, THREADS_QUERY, REPLIES_QUERY, USER_AGENT, extractTokens, getUserId, makeHeader, hasMedia, buildMedia, buildContent };