From 5eadc0fad31299b29fd723ba50d861b9a28cee8e Mon Sep 17 00:00:00 2001 From: Gregor Martynus Date: Sun, 27 Oct 2019 15:04:41 -0700 Subject: [PATCH] feat: `.paginate()` and `.paginate.iterator()` methods --- src/index.ts | 19 +++-- src/iterator.ts | 41 +++++++++ src/normalize-paginated-list-response.ts | 71 ++++++++++++++++ src/paginate.ts | 58 +++++++++++++ src/types.ts | 102 +++++++++++++++++++++++ 5 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 src/iterator.ts create mode 100644 src/normalize-paginated-list-response.ts create mode 100644 src/paginate.ts create mode 100644 src/types.ts diff --git a/src/index.ts b/src/index.ts index 9effe0e6..c5a44869 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,13 +1,22 @@ import { VERSION } from "./version"; -type Octokit = any; -type Options = { - [option: string]: any; -}; +import { paginate } from "./paginate"; +import { iterator } from "./iterator"; +import { PaginateInterface } from "./types"; + +import { Octokit } from "@octokit/core"; /** * @param octokit Octokit instance * @param options Options passed to Octokit constructor */ -export function paginateRest(octokit: Octokit, options: Options) {} +export function paginateRest(octokit: Octokit) { + return { + paginate: Object.assign(paginate.bind(null, octokit), { + iterator: iterator.bind(null, octokit) + }) as PaginateInterface + }; +} paginateRest.VERSION = VERSION; + +export type PaginateInterface = PaginateInterface; diff --git a/src/iterator.ts b/src/iterator.ts new file mode 100644 index 00000000..5cb32c75 --- /dev/null +++ b/src/iterator.ts @@ -0,0 +1,41 @@ +import { Octokit } from "@octokit/core"; + +import { normalizePaginatedListResponse } from "./normalize-paginated-list-response"; +import { OctokitResponse, RequestParameters, Route } from "./types"; + +export function iterator( + octokit: Octokit, + route: Route, + parameters?: RequestParameters +) { + const options = octokit.request.endpoint(route, parameters); + const method = options.method; + const headers = options.headers; + let url = options.url; + + return { + [Symbol.asyncIterator]: () => ({ + next() { + if (!url) { + return Promise.resolve({ done: true }); + } + + return octokit + .request({ method, url, headers }) + + .then((response: OctokitResponse) => { + normalizePaginatedListResponse(octokit, url, response); + + // `response.headers.link` format: + // '; rel="next", ; rel="last"' + // sets `url` to undefined if "next" URL is not present or `link` header is not set + url = ((response.headers.link || "").match( + /<([^>]+)>;\s*rel="next"/ + ) || [])[1]; + + return { value: response }; + }); + } + }) + }; +} diff --git a/src/normalize-paginated-list-response.ts b/src/normalize-paginated-list-response.ts new file mode 100644 index 00000000..d7426abc --- /dev/null +++ b/src/normalize-paginated-list-response.ts @@ -0,0 +1,71 @@ +/** + * Some “list” response that can be paginated have a different response structure + * + * They have a `total_count` key in the response (search also has `incomplete_results`, + * /installation/repositories also has `repository_selection`), as well as a key with + * the list of the items which name varies from endpoint to endpoint: + * + * - https://developer.github.com/v3/search/#example (key `items`) + * - https://developer.github.com/v3/checks/runs/#response-3 (key: `check_runs`) + * - https://developer.github.com/v3/checks/suites/#response-1 (key: `check_suites`) + * - https://developer.github.com/v3/apps/installations/#list-repositories (key: `repositories`) + * - https://developer.github.com/v3/apps/installations/#list-installations-for-a-user (key `installations`) + * + * Octokit normalizes these responses so that paginated results are always returned following + * the same structure. One challenge is that if the list response has only one page, no Link + * header is provided, so this header alone is not sufficient to check wether a response is + * paginated or not. For the exceptions with the namespace, a fallback check for the route + * paths has to be added in order to normalize the response. We cannot check for the total_count + * property because it also exists in the response of Get the combined status for a specific ref. + */ + +import { Octokit } from "@octokit/core"; + +import { OctokitResponse } from "./types"; + +const REGEX_IS_SEARCH_PATH = /^\/search\//; +const REGEX_IS_CHECKS_PATH = /^\/repos\/[^/]+\/[^/]+\/commits\/[^/]+\/(check-runs|check-suites)/; +const REGEX_IS_INSTALLATION_REPOSITORIES_PATH = /^\/installation\/repositories/; +const REGEX_IS_USER_INSTALLATIONS_PATH = /^\/user\/installations/; + +export function normalizePaginatedListResponse( + octokit: Octokit, + url: string, + response: OctokitResponse +) { + const path = url.replace(octokit.request.endpoint.DEFAULTS.baseUrl, ""); + if ( + !REGEX_IS_SEARCH_PATH.test(path) && + !REGEX_IS_CHECKS_PATH.test(path) && + !REGEX_IS_INSTALLATION_REPOSITORIES_PATH.test(path) && + !REGEX_IS_USER_INSTALLATIONS_PATH.test(path) + ) { + if (!Array.isArray(response.data)) { + response.data = [response.data]; + } + return; + } + + // keep the additional properties intact as there is currently no other way + // to retrieve the same information. + const incompleteResults = response.data.incomplete_results; + const repositorySelection = response.data.repository_selection; + const totalCount = response.data.total_count; + delete response.data.incomplete_results; + delete response.data.repository_selection; + delete response.data.total_count; + + const namespaceKey = Object.keys(response.data)[0]; + + response.data = response.data[namespaceKey]; + + if (typeof incompleteResults !== "undefined") { + response.data.incomplete_results = incompleteResults; + } + + if (typeof repositorySelection !== "undefined") { + response.data.repository_selection = repositorySelection; + } + + response.data.total_count = totalCount; +} diff --git a/src/paginate.ts b/src/paginate.ts new file mode 100644 index 00000000..4eea4965 --- /dev/null +++ b/src/paginate.ts @@ -0,0 +1,58 @@ +import { Octokit } from "@octokit/core"; + +import { iterator } from "./iterator"; +import { + MapFunction, + PaginationResults, + RequestParameters, + Route +} from "./types"; + +export function paginate( + octokit: Octokit, + route: Route, + parameters?: RequestParameters, + mapFn?: MapFunction +) { + if (typeof parameters === "function") { + mapFn = parameters; + parameters = undefined; + } + + return gather( + octokit, + [], + iterator(octokit, route, parameters)[ + Symbol.asyncIterator + ]() as AsyncIterableIterator, + mapFn + ); +} + +function gather( + octokit: Octokit, + results: PaginationResults, + iterator: AsyncIterableIterator, + mapFn?: MapFunction +): Promise { + return iterator.next().then(result => { + if (result.done) { + return results; + } + + let earlyExit = false; + function done() { + earlyExit = true; + } + + results = results.concat( + mapFn ? mapFn(result.value, done) : result.value.data + ); + + if (earlyExit) { + return results; + } + + return gather(octokit, results, iterator, mapFn); + }); +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 00000000..b4681471 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,102 @@ +import * as OctokitTypes from "@octokit/types"; + +export { EndpointOptions } from "@octokit/types"; +export { OctokitResponse } from "@octokit/types"; +export { RequestParameters } from "@octokit/types"; +export { Route } from "@octokit/types"; + +export interface PaginateInterface { + /** + * Sends a request based on endpoint options + * + * @param {object} endpoint Must set `method` and `url`. Plus URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`. + * @param {function} mapFn Optional method to map each response to a custom array + */ + ( + options: OctokitTypes.EndpointOptions, + mapFn: MapFunction + ): Promise>; + + /** + * Sends a request based on endpoint options + * + * @param {object} endpoint Must set `method` and `url`. Plus URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`. + */ + (options: OctokitTypes.EndpointOptions): Promise>; + + /** + * Sends a request based on endpoint options + * + * @param {string} route Request method + URL. Example: `'GET /orgs/:org'` + * @param {function} mapFn Optional method to map each response to a custom array + */ + (route: OctokitTypes.Route, mapFn: MapFunction): Promise< + PaginationResults + >; + + /** + * Sends a request based on endpoint options + * + * @param {string} route Request method + URL. Example: `'GET /orgs/:org'` + * @param {object} parameters URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`. + * @param {function} mapFn Optional method to map each response to a custom array + */ + ( + route: OctokitTypes.Route, + parameters: OctokitTypes.RequestParameters, + mapFn: MapFunction + ): Promise>; + + /** + * Sends a request based on endpoint options + * + * @param {string} route Request method + URL. Example: `'GET /orgs/:org'` + * @param {object} parameters URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`. + */ + ( + route: OctokitTypes.Route, + parameters: OctokitTypes.RequestParameters + ): Promise>; + + /** + * Sends a request based on endpoint options + * + * @param {string} route Request method + URL. Example: `'GET /orgs/:org'` + */ + (route: OctokitTypes.Route): Promise>; + + iterator: { + /** + * Get an asynchronous iterator for use with `for await()`, + * + * @see {link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of} for await...of + * @param {object} endpoint Must set `method` and `url`. Plus URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`. + */ + (EndpointOptions: OctokitTypes.EndpointOptions): AsyncIterableIterator< + OctokitTypes.OctokitResponse> + >; + + /** + * Get an asynchronous iterator for use with `for await()`, + * + * @see {link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of} for await...of + * @param {string} route Request method + URL. Example: `'GET /orgs/:org'` + * @param {object} [parameters] URL, query or body parameters, as well as `headers`, `mediaType.{format|previews}`, `request`, or `baseUrl`. + */ + ( + route: OctokitTypes.Route, + parameters?: OctokitTypes.RequestParameters + ): AsyncIterableIterator< + OctokitTypes.OctokitResponse> + >; + }; +} + +export interface MapFunction { + ( + response: OctokitTypes.OctokitResponse>, + done: () => void + ): R[]; +} + +export type PaginationResults = T[];