From 61ea4679fc2ca61bb7a18f62fadb5c9e05fc53f6 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Wed, 30 Aug 2023 19:18:19 +0800 Subject: [PATCH] Rewrite Feishu HTTP request by use axios with rate limit retrying feature. --- feishu-pages/package.json | 7 +-- feishu-pages/src/doc.ts | 31 ++++--------- feishu-pages/src/feishu.ts | 94 ++++++++++++++++++++++++++++++++++++-- feishu-pages/src/wiki.ts | 70 +++++++++++++--------------- yarn.lock | 16 ++++++- 5 files changed, 149 insertions(+), 69 deletions(-) diff --git a/feishu-pages/package.json b/feishu-pages/package.json index 754cd85..dc4b641 100644 --- a/feishu-pages/package.json +++ b/feishu-pages/package.json @@ -11,13 +11,14 @@ "dependencies": { "@larksuiteoapi/node-sdk": "^1.20.0", "@types/node": "^20.5.7", + "axios": "^1.5.0", "dotenv": "^16.3.1", - "typescript": "^5.2.2", - "feishu-docx": "*" + "feishu-docx": "*", + "typescript": "^5.2.2" }, "devDependencies": { "@jest/globals": "^29.6.4", "jest": "^29.6.4", "ts-jest": "^29.1.1" } -} \ No newline at end of file +} diff --git a/feishu-pages/src/doc.ts b/feishu-pages/src/doc.ts index d5629dc..a7fb9aa 100644 --- a/feishu-pages/src/doc.ts +++ b/feishu-pages/src/doc.ts @@ -1,6 +1,5 @@ -import { withTenantToken } from '@larksuiteoapi/node-sdk'; import { MarkdownRenderer } from 'feishu-docx'; -import { Doc, feishuClient, feishuConfig, feishuRequest } from './feishu'; +import { Doc, feishuFetchWithIterator } from './feishu'; /** * Fetch doc content @@ -11,16 +10,6 @@ import { Doc, feishuClient, feishuConfig, feishuRequest } from './feishu'; export const fetchDocBody = async (document_id: string) => { console.info('Fetching doc: ', document_id, '...'); - let payload: any = { - path: { - document_id: document_id, - }, - params: { - page_size: 500, - document_revision_id: -1, - }, - }; - const doc = { document: { document_id, @@ -28,16 +17,14 @@ export const fetchDocBody = async (document_id: string) => { blocks: [], }; - const options = withTenantToken(feishuConfig.tenantAccessToken); - for await (const data of await feishuRequest( - feishuClient.docx.documentBlock.listWithIterator, - payload, - options - )) { - data.items?.forEach((item) => { - doc.blocks.push(item); - }); - } + doc.blocks = await feishuFetchWithIterator( + 'GET', + `/open-apis/docx/v1/documents/${document_id}/blocks`, + { + page_size: 500, + document_revision_id: -1, + } + ); const render = new MarkdownRenderer(doc as any); diff --git a/feishu-pages/src/feishu.ts b/feishu-pages/src/feishu.ts index 8b0547d..5c21178 100644 --- a/feishu-pages/src/feishu.ts +++ b/feishu-pages/src/feishu.ts @@ -1,8 +1,10 @@ // node-sdk 使用说明：https://github.com/larksuite/node-sdk/blob/main/README.zh.md import { Client } from '@larksuiteoapi/node-sdk'; +import axios from 'axios'; import 'dotenv/config'; -const feishuConfig: Record = { +const feishuConfig = { + endpoint: 'https://open.feishu.cn', /** * App Id of Feishu App * @@ -122,6 +124,32 @@ const requestWait = async (ms?: number) => { RATE_LIMITS[minuteLockKey] += 1; }; +axios.interceptors.response.use( + (response) => { + return response; + }, + async (error) => { + const { headers, data } = error.response; + + // Rate Limit code: 99991400, delay to retry + if (data?.code === 99991400) { + const rateLimitResetSeconds = headers['x-ogw-ratelimit-reset']; + console.warn( + 'Rate Limit: ', + data.code, + data.msg, + `delay ${rateLimitResetSeconds}s to retry...` + ); + + // Delay to retry + await requestWait(rateLimitResetSeconds * 1000); + return await axios.request(error.config); + } + + throw error; + } +); + /** * 带有全局 RateLimit 的 Feishu 网络请求方式 * @param fn @@ -129,9 +157,65 @@ const requestWait = async (ms?: number) => { * @param options * @returns */ -export const feishuRequest = async (fn, payload, options): Promise => { - await requestWait(300); - return await fn(payload, options); +export const feishuFetch = async (method, path, payload): Promise => { + const authorization = `Bearer ${feishuConfig.tenantAccessToken}`; + const headers = { + Authorization: authorization, + 'Content-Type': 'application/json; charset=utf-8', + 'User-Agent': 'feishu-pages', + }; + + const url = `${feishuConfig.endpoint}${path}`; + + const { code, data, msg } = await axios + .request({ + method, + url, + params: payload, + headers, + }) + .then((res) => res.data); + + if (code !== 0) { + console.warn('feishuFetch code:', code, 'msg:', msg); + return null; + } + + return data; +}; + +/** + * Request Feishu List API with iterator + * + * @param method + * @param path + * @param payload + * @param options + * @returns + */ +export const feishuFetchWithIterator = async ( + method: string, + path: string, + payload: Record +): Promise => { + let pageToken = ''; + let hasMore = true; + let results: any[] = []; + + while (hasMore) { + const data = await feishuFetch(method, path, { + ...payload, + page_token: pageToken, + }); + + if (data.items) { + results = results.concat(data.items); + } + hasMore = data.has_more; + pageToken = data.page_token; + } + + return results; }; export interface Doc { @@ -146,4 +230,4 @@ export interface Doc { has_child?: boolean; } -export { checkEnv, feishuClient, feishuConfig }; +export { checkEnv, feishuConfig }; diff --git a/feishu-pages/src/wiki.ts b/feishu-pages/src/wiki.ts index 2a1fb7a..77cb51c 100644 --- a/feishu-pages/src/wiki.ts +++ b/feishu-pages/src/wiki.ts @@ -1,5 +1,4 @@ -import { withTenantToken } from '@larksuiteoapi/node-sdk'; -import { Doc, feishuClient, feishuConfig, feishuRequest } from './feishu'; +import { Doc, feishuFetchWithIterator } from './feishu'; /** * 获取某个空间下的所有文档列表 @@ -14,48 +13,43 @@ export const fetchAllDocs = async ( if (!depth) { depth = 0; } - const docs: Doc[] = []; - const prefix = '|--'.repeat(depth + 1); + const prefix = '|__' + '___'.repeat(depth) + ' '; - let payload = { - path: { - space_id: spaceId, - }, - params: { + let items = await feishuFetchWithIterator( + 'GET', + `/open-apis/wiki/v2/spaces/${spaceId}/nodes`, + { parent_node_token, page_size: 50, - }, - }; - const options = withTenantToken(feishuConfig.tenantAccessToken); - - for await (const result of await feishuRequest( - feishuClient.wiki.spaceNode.listWithIterator, - payload, - options - )) { - const { items = [] } = result; - - items - .filter((item) => item.obj_type == 'doc' || item.obj_type == 'docx') - .map(async (item) => { - const doc: Doc = { - depth: depth, - title: item.title, - node_token: item.node_token, - parent_node_token: parent_node_token, - obj_create_time: item.obj_create_time, - obj_edit_time: item.obj_edit_time, - obj_token: item.obj_token, - children: [], - has_child: item.has_child, - }; + } + ); - docs.push(doc); - }); - } + const docs: Doc[] = []; - console.info(prefix + 'node:', parent_node_token, docs.length, 'children.'); + items + .filter((item) => item.obj_type == 'doc' || item.obj_type == 'docx') + .forEach((item) => { + const doc: Doc = { + depth: depth, + title: item.title, + node_token: item.node_token, + parent_node_token: parent_node_token, + obj_create_time: item.obj_create_time, + obj_edit_time: item.obj_edit_time, + obj_token: item.obj_token, + children: [], + has_child: item.has_child, + }; + + docs.push(doc); + }); + + console.info( + prefix + 'node:', + parent_node_token || 'root', + docs.length > 0 ? `${docs.length} docs` : '' + ); for (const doc of docs) { doc.children = await fetchAllDocs(spaceId, depth + 1, doc.node_token); diff --git a/yarn.lock b/yarn.lock index 521bbbb..1f45562 100644 --- a/yarn.lock +++ b/yarn.lock @@ -739,6 +739,15 @@ asynckit@^0.4.0: follow-redirects "^1.14.9" form-data "^4.0.0" +axios@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.5.0.tgz#f02e4af823e2e46a9768cfc74691fdd0517ea267" + integrity sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + babel-jest@^29.6.4: version "29.6.4" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.6.4.tgz#98dbc45d1c93319c82a8ab4a478b670655dd2585" @@ -1144,7 +1153,7 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" -follow-redirects@^1.14.9: +follow-redirects@^1.14.9, follow-redirects@^1.15.0: version "1.15.2" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== @@ -2085,6 +2094,11 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"