feat: .paginate() and .paginate.iterator() methods

src/index.ts

@@ -1,13 +1,22 @@
 import { VERSION } from "./version";
 
-type Octokit = any;
-type Options = {
-  [option: string]: any;
-};
+import { paginate } from "./paginate";
+import { iterator } from "./iterator";
+import { PaginateInterface } from "./types";
+
+import { Octokit } from "@octokit/core";
 
 /**
  * @param octokit Octokit instance
  * @param options Options passed to Octokit constructor
  */
-export function paginateRest(octokit: Octokit, options: Options) {}
+export function paginateRest(octokit: Octokit) {
+  return {
+    paginate: Object.assign(paginate.bind(null, octokit), {
+      iterator: iterator.bind(null, octokit)
+    }) as PaginateInterface
+  };
+}
 paginateRest.VERSION = VERSION;
+
+export type PaginateInterface = PaginateInterface;

src/iterator.ts

@@ -0,0 +1,41 @@
+import { Octokit } from "@octokit/core";
+
+import { normalizePaginatedListResponse } from "./normalize-paginated-list-response";
+import { OctokitResponse, RequestParameters, Route } from "./types";
+
+export function iterator(
+  octokit: Octokit,
+  route: Route,
+  parameters?: RequestParameters
+) {
+  const options = octokit.request.endpoint(route, parameters);
+  const method = options.method;
+  const headers = options.headers;
+  let url = options.url;
+
+  return {
+    [Symbol.asyncIterator]: () => ({
+      next() {
+        if (!url) {
+          return Promise.resolve({ done: true });
+        }
+
+        return octokit
+          .request({ method, url, headers })
+
+          .then((response: OctokitResponse<any>) => {
+            normalizePaginatedListResponse(octokit, url, response);
+
+            // `response.headers.link` format:
+            // '<https://api.github.com/users/aseemk/followers?page=2>; rel="next", <https://api.github.com/users/aseemk/followers?page=2>; rel="last"'
+            // sets `url` to undefined if "next" URL is not present or `link` header is not set
+            url = ((response.headers.link || "").match(
+              /<([^>]+)>;\s*rel="next"/
+            ) || [])[1];
+
+            return { value: response };
+          });
+      }
+    })
+  };
+}

src/normalize-paginated-list-response.ts

@@ -0,0 +1,71 @@
+/**
+ * Some “list” response that can be paginated have a different response structure
+ *
+ * They have a `total_count` key in the response (search also has `incomplete_results`,
+ * /installation/repositories also has `repository_selection`), as well as a key with
+ * the list of the items which name varies from endpoint to endpoint:
+ *
+ * - https://developer.github.com/v3/search/#example (key `items`)
+ * - https://developer.github.com/v3/checks/runs/#response-3 (key: `check_runs`)
+ * - https://developer.github.com/v3/checks/suites/#response-1 (key: `check_suites`)
+ * - https://developer.github.com/v3/apps/installations/#list-repositories (key: `repositories`)
+ * - https://developer.github.com/v3/apps/installations/#list-installations-for-a-user (key `installations`)
+ *
+ * Octokit normalizes these responses so that paginated results are always returned following
+ * the same structure. One challenge is that if the list response has only one page, no Link
+ * header is provided, so this header alone is not sufficient to check wether a response is
+ * paginated or not. For the exceptions with the namespace, a fallback check for the route
+ * paths has to be added in order to normalize the response. We cannot check for the total_count
+ * property because it also exists in the response of Get the combined status for a specific ref.
+ */
+
+import { Octokit } from "@octokit/core";
+
+import { OctokitResponse } from "./types";
+
+const REGEX_IS_SEARCH_PATH = /^\/search\//;
+const REGEX_IS_CHECKS_PATH = /^\/repos\/[^/]+\/[^/]+\/commits\/[^/]+\/(check-runs|check-suites)/;
+const REGEX_IS_INSTALLATION_REPOSITORIES_PATH = /^\/installation\/repositories/;
+const REGEX_IS_USER_INSTALLATIONS_PATH = /^\/user\/installations/;
+
+export function normalizePaginatedListResponse(
+  octokit: Octokit,
+  url: string,
+  response: OctokitResponse<any>
+) {
+  const path = url.replace(octokit.request.endpoint.DEFAULTS.baseUrl, "");
+  if (
+    !REGEX_IS_SEARCH_PATH.test(path) &&
+    !REGEX_IS_CHECKS_PATH.test(path) &&
+    !REGEX_IS_INSTALLATION_REPOSITORIES_PATH.test(path) &&
+    !REGEX_IS_USER_INSTALLATIONS_PATH.test(path)
+  ) {
+    if (!Array.isArray(response.data)) {
+      response.data = [response.data];
+    }
+    return;
+  }
+
+  // keep the additional properties intact as there is currently no other way
+  // to retrieve the same information.
+  const incompleteResults = response.data.incomplete_results;
+  const repositorySelection = response.data.repository_selection;
+  const totalCount = response.data.total_count;
+  delete response.data.incomplete_results;
+  delete response.data.repository_selection;
+  delete response.data.total_count;
+
+  const namespaceKey = Object.keys(response.data)[0];
+
+  response.data = response.data[namespaceKey];
+
+  if (typeof incompleteResults !== "undefined") {
+    response.data.incomplete_results = incompleteResults;
+  }
+
+  if (typeof repositorySelection !== "undefined") {
+    response.data.repository_selection = repositorySelection;
+  }
+
+  response.data.total_count = totalCount;
+}

src/paginate.ts

@@ -0,0 +1,58 @@
+import { Octokit } from "@octokit/core";
+
+import { iterator } from "./iterator";
+import {
+  MapFunction,
+  PaginationResults,
+  RequestParameters,
+  Route
+} from "./types";
+
+export function paginate(
+  octokit: Octokit,
+  route: Route,
+  parameters?: RequestParameters,
+  mapFn?: MapFunction
+) {
+  if (typeof parameters === "function") {
+    mapFn = parameters;
+    parameters = undefined;
+  }
+
+  return gather(
+    octokit,
+    [],
+    iterator(octokit, route, parameters)[
+      Symbol.asyncIterator
+    ]() as AsyncIterableIterator<any>,
+    mapFn
+  );
+}
+
+function gather(
+  octokit: Octokit,
+  results: PaginationResults,
+  iterator: AsyncIterableIterator<any>,
+  mapFn?: MapFunction
+): Promise<PaginationResults> {
+  return iterator.next().then(result => {
+    if (result.done) {
+      return results;
+    }
+
+    let earlyExit = false;
+    function done() {
+      earlyExit = true;
+    }
+
+    results = results.concat(
+      mapFn ? mapFn(result.value, done) : result.value.data
+    );
+
+    if (earlyExit) {
+      return results;
+    }
+
+    return gather(octokit, results, iterator, mapFn);
+  });
+}

src/types.ts

@@ -0,0 +1,102 @@
+import * as OctokitTypes from "@octokit/types";
+
+export { EndpointOptions } from "@octokit/types";
+export { OctokitResponse } from "@octokit/types";
+export { RequestParameters } from "@octokit/types";
+export { Route } from "@octokit/types";
+
+export interface PaginateInterface {
+  /**
+   * Sends a request based on endpoint options
+   *
+   * @param {object} endpoint Must set `method` and `url`. Plus URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`.
+   * @param {function} mapFn Optional method to map each response to a custom array
+   */
+  <T, R>(
+    options: OctokitTypes.EndpointOptions,
+    mapFn: MapFunction<T, R>
+  ): Promise<PaginationResults<R>>;
+
+  /**
+   * Sends a request based on endpoint options
+   *
+   * @param {object} endpoint Must set `method` and `url`. Plus URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`.
+   */
+  <T>(options: OctokitTypes.EndpointOptions): Promise<PaginationResults<T>>;
+
+  /**
+   * Sends a request based on endpoint options
+   *
+   * @param {string} route Request method + URL. Example: `'GET /orgs/:org'`
+   * @param {function} mapFn Optional method to map each response to a custom array
+   */
+  <T, R>(route: OctokitTypes.Route, mapFn: MapFunction<T>): Promise<
+    PaginationResults<R>
+  >;
+
+  /**
+   * Sends a request based on endpoint options
+   *
+   * @param {string} route Request method + URL. Example: `'GET /orgs/:org'`
+   * @param {object} parameters URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`.
+   * @param {function} mapFn Optional method to map each response to a custom array
+   */
+  <T, R>(
+    route: OctokitTypes.Route,
+    parameters: OctokitTypes.RequestParameters,
+    mapFn: MapFunction<T>
+  ): Promise<PaginationResults<R>>;
+
+  /**
+   * Sends a request based on endpoint options
+   *
+   * @param {string} route Request method + URL. Example: `'GET /orgs/:org'`
+   * @param {object} parameters URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`.
+   */
+  <T>(
+    route: OctokitTypes.Route,
+    parameters: OctokitTypes.RequestParameters
+  ): Promise<PaginationResults<T>>;
+
+  /**
+   * Sends a request based on endpoint options
+   *
+   * @param {string} route Request method + URL. Example: `'GET /orgs/:org'`
+   */
+  <T>(route: OctokitTypes.Route): Promise<PaginationResults<T>>;
+
+  iterator: {
+    /**
+     * Get an asynchronous iterator for use with `for await()`,
+     *
+     * @see {link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of} for await...of
+     * @param {object} endpoint Must set `method` and `url`. Plus URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`.
+     */
+    <T>(EndpointOptions: OctokitTypes.EndpointOptions): AsyncIterableIterator<
+      OctokitTypes.OctokitResponse<PaginationResults<T>>
+    >;
+
+    /**
+     * Get an asynchronous iterator for use with `for await()`,
+     *
+     * @see {link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of} for await...of
+     * @param {string} route Request method + URL. Example: `'GET /orgs/:org'`
+     * @param {object} [parameters] URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`.
+     */
+    <T>(
+      route: OctokitTypes.Route,
+      parameters?: OctokitTypes.RequestParameters
+    ): AsyncIterableIterator<
+      OctokitTypes.OctokitResponse<PaginationResults<T>>
+    >;
+  };
+}
+
+export interface MapFunction<T = any, R = any> {
+  (
+    response: OctokitTypes.OctokitResponse<PaginationResults<T>>,
+    done: () => void
+  ): R[];
+}
+
+export type PaginationResults<T = any> = T[];