feat(PR-40): auto-page list requests (#100)

When calling `workspaces.list`, `forms.list`, `themes.list` or `responses.list` you can supply `page: "auto"` and the library will fetch all items across all pages for you automatically. It will fetch with maximum available `pageSize` to minimize number of requests.
Typeform · Oct 10, 2023 · a826100 · a826100
1 parent 5bf4e0f
commit a826100
Show file tree

Hide file tree

Showing 14 changed files with 316 additions and 56 deletions.
diff --git a/README.md b/README.md
@@ -156,6 +156,7 @@ Each one of them encapsulates the operations related to it (like listing, updati
 
 - Get a list of your typeforms
 - Returns a list of typeforms with the payload [referenced here](https://developer.typeform.com/create/reference/retrieve-forms/).
+- You can set `page: "auto"` to automatically fetch all pages if there are more. It fetches with maximum `pageSize: 200`. 
 
 #### `forms.get({ uid })`
 
@@ -214,6 +215,7 @@ Each one of them encapsulates the operations related to it (like listing, updati
 
 - Gets your themes collection
 - `page`: default `1`
+  - set `page: "auto"` to automatically fetch all pages if there are more, it fetches with maximum `pageSize: 200` 
 - `pageSize: default `10`
 
 #### `themes.get({ id })`
@@ -262,6 +264,7 @@ Each one of them encapsulates the operations related to it (like listing, updati
 
 - Retrieve all workspaces in your account.
 - `page`: The page of results to retrieve. Default `1` is the first page of results.
+  - set `page: "auto"` to automatically fetch all pages if there are more, it fetches with maximum `pageSize: 200` 
 - `pageSize`: Number of results to retrieve per page. Default is 10. Maximum is 200.
 - `search`: Returns items that contain the specified string.
 
@@ -290,6 +293,7 @@ Each one of them encapsulates the operations related to it (like listing, updati
 - Returns form responses and date and time of form landing and submission.
 - `uid`: Unique ID for the form.
 - `pageSize`: Maximum number of responses. Default value is 25. Maximum value is 1000.
+- `page`: Set to `"auto"` to automatically fetch all pages if there are more. It fetches with maximum `pageSize: 1000`. The `after` value is ignored when automatic paging is enabled. The responses will be sorted in the order that our system processed them (instead of the default order, `submitted_at`). **Note that it does not accept numeric value to identify page number.**
 - `since`: Limit request to responses submitted since the specified date and time. In ISO 8601 format, UTC time, to the second, with T as a delimiter between the date and time.
 - `until`: Limit request to responses submitted until the specified date and time. In ISO 8601 format, UTC time, to the second, with T as a delimiter between the date and time.
 - `after`: Limit request to responses submitted after the specified token. If you use the `after` parameter, the responses will be sorted in the order that our system processed them (instead of the default order, `submitted_at`).

diff --git a/package.json b/package.json
@@ -84,6 +84,7 @@
     "rollup-plugin-typescript2": "^0.24.1",
     "semantic-release": "^17.0.7",
     "ts-jest": "^24.0.2",
+    "tslib": "^2.6.2",
     "typescript": "^4.9.5"
   },
   "jest": {

diff --git a/src/auto-page-items.ts b/src/auto-page-items.ts
@@ -0,0 +1,47 @@
+import { rateLimit } from './utils'
+
+// request with maximum available page size to minimize number of requests
+const MAX_PAGE_SIZE = 200
+
+type RequestItemsFn<Item> = (
+  page: number,
+  pageSize: number
+) => Promise<{
+  total_items: number
+  page_count: number
+  items: Item[]
+}>
+
+const requestPageItems = async <Item>(
+  requestFn: RequestItemsFn<Item>,
+  page = 1
+): Promise<Item[]> => {
+  await rateLimit()
+  const { items = [] } = (await requestFn(page, MAX_PAGE_SIZE)) || {}
+  const moreItems =
+    items.length === MAX_PAGE_SIZE
+      ? await requestPageItems(requestFn, page + 1)
+      : []
+  return [...items, ...moreItems]
+}
+
+export const autoPageItems = async <Item>(
+  requestFn: RequestItemsFn<Item>
+): Promise<{
+  total_items: number
+  page_count: 1
+  items: Item[]
+}> => {
+  const { total_items = 0, items = [] } =
+    (await requestFn(1, MAX_PAGE_SIZE)) || {}
+  return {
+    total_items,
+    page_count: 1,
+    items: [
+      ...items,
+      ...(total_items > items.length
+        ? await requestPageItems(requestFn, 2)
+        : []),
+    ],
+  }
+}
diff --git a/src/bin.ts b/src/bin.ts
@@ -16,7 +16,7 @@ if (!token) {
 const typeformAPI = createClient({ token })
 
 const [, , ...args] = process.argv
-const [methodName, methodParams] = args
+const [methodName, ...methodParams] = args
 
 if (!methodName || methodName === '-h' || methodName === '--help') {
   print('usage: typeform-api <method> [params]')
@@ -35,17 +35,18 @@ if (!typeformAPI[property]?.[method]) {
 
 let parsedParams = undefined
 
-if (methodParams) {
+if (methodParams && methodParams.length > 0) {
+  const methodParamsString = methodParams.join(',')
+  const normalizedParams = methodParamsString.startsWith('{')
+    ? methodParamsString
+    : `{${methodParamsString}}`
+
   try {
     // this eval executes code supplied by user on their own machine, this is safe
     // eslint-disable-next-line no-eval
-    eval(`parsedParams = ${methodParams}`)
+    eval(`parsedParams = ${normalizedParams}`)
   } catch (err) {
-    throw new Error(`Invalid params: ${methodParams}`)
-  }
-
-  if (typeof parsedParams !== 'object') {
-    throw new Error(`Invalid params: ${methodParams}`)
+    throw new Error(`Invalid params: ${normalizedParams}`)
   }
 }
 

diff --git a/src/forms.ts b/src/forms.ts
@@ -1,4 +1,5 @@
 import { Typeform } from './typeform-types'
+import { autoPageItems } from './auto-page-items'
 
 export class Forms {
   private _messages: FormMessages
@@ -40,7 +41,7 @@ export class Forms {
   }
 
   public list(args?: {
-    page?: number
+    page?: number | 'auto'
     pageSize?: number
     search?: string
     workspaceId?: string
@@ -52,16 +53,23 @@ export class Forms {
       workspaceId: null,
     }
 
-    return this._http.request({
-      method: 'get',
-      url: `/forms`,
-      params: {
-        page,
-        page_size: pageSize,
-        search,
-        workspace_id: workspaceId,
-      },
-    })
+    const request = (page: number, pageSize: number) =>
+      this._http.request({
+        method: 'get',
+        url: `/forms`,
+        params: {
+          page,
+          page_size: pageSize,
+          search,
+          workspace_id: workspaceId,
+        },
+      })
+
+    if (page === 'auto') {
+      return autoPageItems(request)
+    }
+
+    return request(page, pageSize)
   }
 
   public update(args: {

diff --git a/src/responses.ts b/src/responses.ts
@@ -1,4 +1,5 @@
 import { Typeform } from './typeform-types'
+import { rateLimit } from './utils'
 
 export class Responses {
   constructor(private _http: Typeform.HTTPClient) {}
@@ -27,6 +28,7 @@ export class Responses {
     sort?: string
     query?: string
     fields?: string | string[]
+    page?: 'auto'
   }): Promise<Typeform.API.Responses.List> {
     const {
       uid,
@@ -40,24 +42,32 @@ export class Responses {
       sort,
       query,
       fields,
+      page,
     } = args
 
-    return this._http.request({
-      method: 'get',
-      url: `/forms/${uid}/responses`,
-      params: {
-        page_size: pageSize,
-        since,
-        until,
-        after,
-        before,
-        included_response_ids: toCSL(ids),
-        completed,
-        sort,
-        query,
-        fields: toCSL(fields),
-      },
-    })
+    const request = (pageSize: number, before: string) =>
+      this._http.request({
+        method: 'get',
+        url: `/forms/${uid}/responses`,
+        params: {
+          page_size: pageSize,
+          since,
+          until,
+          after,
+          before,
+          included_response_ids: toCSL(ids),
+          completed,
+          sort,
+          query,
+          fields: toCSL(fields),
+        },
+      })
+
+    if (page === 'auto') {
+      return autoPageResponses(request)
+    }
+
+    return request(pageSize, before)
   }
 }
 
@@ -68,3 +78,44 @@ const toCSL = (args: string | string[]): string => {
 
   return typeof args === 'string' ? args : args.join(',')
 }
+
+// when auto-paginating, request with maximum available page size to minimize number of requests
+const MAX_RESULTS_PAGE_SIZE = 1000
+
+type RequestResultsFn = (
+  pageSize: number,
+  before?: string
+) => Promise<Typeform.API.Responses.List>
+
+const getLastResponseId = (items: Typeform.Response[]) =>
+  items.length > 0 ? items[items.length - 1]?.response_id : null
+
+const requestPageResponses = async (
+  requestFn: RequestResultsFn,
+  before: string = undefined
+): Promise<Typeform.Response[]> => {
+  await rateLimit()
+  const { items = [] } = (await requestFn(MAX_RESULTS_PAGE_SIZE, before)) || {}
+  const moreItems =
+    items.length === MAX_RESULTS_PAGE_SIZE
+      ? await requestPageResponses(requestFn, getLastResponseId(items))
+      : []
+  return [...items, ...moreItems]
+}
+
+const autoPageResponses = async (
+  requestFn: RequestResultsFn
+): Promise<Typeform.API.Responses.List> => {
+  const { total_items = 0, items = [] } =
+    (await requestFn(MAX_RESULTS_PAGE_SIZE)) || {}
+  return {
+    total_items,
+    page_count: 1,
+    items: [
+      ...items,
+      ...(total_items > items.length
+        ? await requestPageResponses(requestFn, getLastResponseId(items))
+        : []),
+    ],
+  }
+}
diff --git a/src/themes.ts b/src/themes.ts
@@ -1,5 +1,6 @@
 import { Typeform } from './typeform-types'
 import { FONTS_AVAILABLE } from './constants'
+import { autoPageItems } from './auto-page-items'
 
 export class Themes {
   constructor(private _http: Typeform.HTTPClient) {}
@@ -34,19 +35,26 @@ export class Themes {
   }
 
   public list(args?: {
-    page?: number
+    page?: number | 'auto'
     pageSize?: number
   }): Promise<Typeform.API.Themes.List> {
     const { page, pageSize } = args || { page: null, pageSize: null }
 
-    return this._http.request({
-      method: 'get',
-      url: '/themes',
-      params: {
-        page,
-        page_size: pageSize,
-      },
-    })
+    const request = (page: number, pageSize: number) =>
+      this._http.request({
+        method: 'get',
+        url: '/themes',
+        params: {
+          page,
+          page_size: pageSize,
+        },
+      })
+
+    if (page === 'auto') {
+      return autoPageItems(request)
+    }
+
+    return request(page, pageSize)
   }
 
   public update(args: {

diff --git a/src/utils.ts b/src/utils.ts
@@ -16,3 +16,8 @@ export const createMemberPatchQuery = (
 export const isMemberPropValid = (members: string | string[]): boolean => {
   return members && (typeof members === 'string' || Array.isArray(members))
 }
+
+// two requests per second, per Typeform account
+// https://www.typeform.com/developers/get-started/#rate-limits
+export const rateLimit = () =>
+  new Promise((resolve) => setTimeout(resolve, 500))
diff --git a/src/workspaces.ts b/src/workspaces.ts
@@ -1,5 +1,6 @@
 import { Typeform } from './typeform-types'
 import { isMemberPropValid, createMemberPatchQuery } from './utils'
+import { autoPageItems } from './auto-page-items'
 
 export class Workspaces {
   constructor(private _http: Typeform.HTTPClient) {}
@@ -53,7 +54,7 @@ export class Workspaces {
 
   public list(args?: {
     search?: string
-    page?: number
+    page?: number | 'auto'
     pageSize?: number
   }): Promise<Typeform.API.Workspaces.List> {
     const { search, page, pageSize } = args || {
@@ -62,15 +63,22 @@ export class Workspaces {
       pageSize: null,
     }
 
-    return this._http.request({
-      method: 'get',
-      url: '/workspaces',
-      params: {
-        page,
-        page_size: pageSize,
-        search,
-      },
-    })
+    const request = (page: number, pageSize: number) =>
+      this._http.request({
+        method: 'get',
+        url: '/workspaces',
+        params: {
+          page,
+          page_size: pageSize,
+          search,
+        },
+      })
+
+    if (page === 'auto') {
+      return autoPageItems(request)
+    }
+
+    return request(page, pageSize)
   }
 
   public removeMembers(args: {