Rewrite Feishu HTTP request by use axios with rate limit retrying fea…

…ture.

feishu-pages/package.json

@@ -11,13 +11,14 @@
   "dependencies": {
     "@larksuiteoapi/node-sdk": "^1.20.0",
     "@types/node": "^20.5.7",
+    "axios": "^1.5.0",
     "dotenv": "^16.3.1",
-    "typescript": "^5.2.2",
-    "feishu-docx": "*"
+    "feishu-docx": "*",
+    "typescript": "^5.2.2"
   },
   "devDependencies": {
     "@jest/globals": "^29.6.4",
     "jest": "^29.6.4",
     "ts-jest": "^29.1.1"
   }
-}
+}

feishu-pages/src/doc.ts

@@ -1,6 +1,5 @@
-import { withTenantToken } from '@larksuiteoapi/node-sdk';
 import { MarkdownRenderer } from 'feishu-docx';
-import { Doc, feishuClient, feishuConfig, feishuRequest } from './feishu';
+import { Doc, feishuFetchWithIterator } from './feishu';
 
 /**
  * Fetch doc content
@@ -11,33 +10,21 @@ import { Doc, feishuClient, feishuConfig, feishuRequest } from './feishu';
 export const fetchDocBody = async (document_id: string) => {
   console.info('Fetching doc: ', document_id, '...');
 
-  let payload: any = {
-    path: {
-      document_id: document_id,
-    },
-    params: {
-      page_size: 500,
-      document_revision_id: -1,
-    },
-  };
-
   const doc = {
     document: {
       document_id,
     },
     blocks: [],
   };
 
-  const options = withTenantToken(feishuConfig.tenantAccessToken);
-  for await (const data of await feishuRequest(
-    feishuClient.docx.documentBlock.listWithIterator,
-    payload,
-    options
-  )) {
-    data.items?.forEach((item) => {
-      doc.blocks.push(item);
-    });
-  }
+  doc.blocks = await feishuFetchWithIterator(
+    'GET',
+    `/open-apis/docx/v1/documents/${document_id}/blocks`,
+    {
+      page_size: 500,
+      document_revision_id: -1,
+    }
+  );
 
   const render = new MarkdownRenderer(doc as any);
 

feishu-pages/src/feishu.ts

@@ -1,8 +1,10 @@
 // node-sdk 使用说明：https://github.com/larksuite/node-sdk/blob/main/README.zh.md
 import { Client } from '@larksuiteoapi/node-sdk';
+import axios from 'axios';
 import 'dotenv/config';
 
-const feishuConfig: Record<string, string> = {
+const feishuConfig = {
+  endpoint: 'https://open.feishu.cn',
   /**
    * App Id of Feishu App
    *
@@ -122,16 +124,98 @@ const requestWait = async (ms?: number) => {
   RATE_LIMITS[minuteLockKey] += 1;
 };
 
+axios.interceptors.response.use(
+  (response) => {
+    return response;
+  },
+  async (error) => {
+    const { headers, data } = error.response;
+
+    // Rate Limit code: 99991400, delay to retry
+    if (data?.code === 99991400) {
+      const rateLimitResetSeconds = headers['x-ogw-ratelimit-reset'];
+      console.warn(
+        'Rate Limit: ',
+        data.code,
+        data.msg,
+        `delay ${rateLimitResetSeconds}s to retry...`
+      );
+
+      // Delay to retry
+      await requestWait(rateLimitResetSeconds * 1000);
+      return await axios.request(error.config);
+    }
+
+    throw error;
+  }
+);
+
 /**
  * 带有全局 RateLimit 的 Feishu 网络请求方式
  * @param fn
  * @param payload
  * @param options
  * @returns
  */
-export const feishuRequest = async (fn, payload, options): Promise<any> => {
-  await requestWait(300);
-  return await fn(payload, options);
+export const feishuFetch = async (method, path, payload): Promise<any> => {
+  const authorization = `Bearer ${feishuConfig.tenantAccessToken}`;
+  const headers = {
+    Authorization: authorization,
+    'Content-Type': 'application/json; charset=utf-8',
+    'User-Agent': 'feishu-pages',
+  };
+
+  const url = `${feishuConfig.endpoint}${path}`;
+
+  const { code, data, msg } = await axios
+    .request({
+      method,
+      url,
+      params: payload,
+      headers,
+    })
+    .then((res) => res.data);
+
+  if (code !== 0) {
+    console.warn('feishuFetch code:', code, 'msg:', msg);
+    return null;
+  }
+
+  return data;
+};
+
+/**
+ * Request Feishu List API with iterator
+ *
+ * @param method
+ * @param path
+ * @param payload
+ * @param options
+ * @returns
+ */
+export const feishuFetchWithIterator = async (
+  method: string,
+  path: string,
+  payload: Record<string, any>
+): Promise<any[]> => {
+  let pageToken = '';
+  let hasMore = true;
+  let results: any[] = [];
+
+  while (hasMore) {
+    const data = await feishuFetch(method, path, {
+      ...payload,
+      page_token: pageToken,
+    });
+
+    if (data.items) {
+      results = results.concat(data.items);
+    }
+    hasMore = data.has_more;
+    pageToken = data.page_token;
+  }
+
+  return results;
 };
 
 export interface Doc {
@@ -146,4 +230,4 @@ export interface Doc {
   has_child?: boolean;
 }
 
-export { checkEnv, feishuClient, feishuConfig };
+export { checkEnv, feishuConfig };

feishu-pages/src/wiki.ts

@@ -1,5 +1,4 @@
-import { withTenantToken } from '@larksuiteoapi/node-sdk';
-import { Doc, feishuClient, feishuConfig, feishuRequest } from './feishu';
+import { Doc, feishuFetchWithIterator } from './feishu';
 
 /**
  * 获取某个空间下的所有文档列表
@@ -14,48 +13,43 @@ export const fetchAllDocs = async (
   if (!depth) {
     depth = 0;
   }
-  const docs: Doc[] = [];
 
-  const prefix = '|--'.repeat(depth + 1);
+  const prefix = '|__' + '___'.repeat(depth) + ' ';
 
-  let payload = {
-    path: {
-      space_id: spaceId,
-    },
-    params: {
+  let items = await feishuFetchWithIterator(
+    'GET',
+    `/open-apis/wiki/v2/spaces/${spaceId}/nodes`,
+    {
       parent_node_token,
       page_size: 50,
-    },
-  };
-  const options = withTenantToken(feishuConfig.tenantAccessToken);
-
-  for await (const result of await feishuRequest(
-    feishuClient.wiki.spaceNode.listWithIterator,
-    payload,
-    options
-  )) {
-    const { items = [] } = result;
-
-    items
-      .filter((item) => item.obj_type == 'doc' || item.obj_type == 'docx')
-      .map(async (item) => {
-        const doc: Doc = {
-          depth: depth,
-          title: item.title,
-          node_token: item.node_token,
-          parent_node_token: parent_node_token,
-          obj_create_time: item.obj_create_time,
-          obj_edit_time: item.obj_edit_time,
-          obj_token: item.obj_token,
-          children: [],
-          has_child: item.has_child,
-        };
+    }
+  );
 
-        docs.push(doc);
-      });
-  }
+  const docs: Doc[] = [];
 
-  console.info(prefix + 'node:', parent_node_token, docs.length, 'children.');
+  items
+    .filter((item) => item.obj_type == 'doc' || item.obj_type == 'docx')
+    .forEach((item) => {
+      const doc: Doc = {
+        depth: depth,
+        title: item.title,
+        node_token: item.node_token,
+        parent_node_token: parent_node_token,
+        obj_create_time: item.obj_create_time,
+        obj_edit_time: item.obj_edit_time,
+        obj_token: item.obj_token,
+        children: [],
+        has_child: item.has_child,
+      };
+
+      docs.push(doc);
+    });
+
+  console.info(
+    prefix + 'node:',
+    parent_node_token || 'root',
+    docs.length > 0 ? `${docs.length} docs` : ''
+  );
 
   for (const doc of docs) {
     doc.children = await fetchAllDocs(spaceId, depth + 1, doc.node_token);

yarn.lock

@@ -739,6 +739,15 @@ asynckit@^0.4.0:
     follow-redirects "^1.14.9"
     form-data "^4.0.0"
 
+axios@^1.5.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.5.0.tgz#f02e4af823e2e46a9768cfc74691fdd0517ea267"
+  integrity sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==
+  dependencies:
+    follow-redirects "^1.15.0"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
 babel-jest@^29.6.4:
   version "29.6.4"
   resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.6.4.tgz#98dbc45d1c93319c82a8ab4a478b670655dd2585"
@@ -1144,7 +1153,7 @@ find-up@^4.0.0, find-up@^4.1.0:
     locate-path "^5.0.0"
     path-exists "^4.0.0"
 
-follow-redirects@^1.14.9:
+follow-redirects@^1.14.9, follow-redirects@^1.15.0:
   version "1.15.2"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
   integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
@@ -2085,6 +2094,11 @@ prompts@^2.0.1:
     kleur "^3.0.3"
     sisteransi "^1.0.5"
 
+proxy-from-env@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
+  integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
+
 psl@^1.1.33:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"