Merged by Peril

Limit concurrent API calls
danger · May 1, 2018 · 388ebc7 · 388ebc7
2 parents 1034b2e + ddd3a9e
commit 388ebc7
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 18 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,8 @@
 
 ## Master
 
+* Automatically rate limit concurrent GitHub API calls to avoid hitting GitHub rate limits [@mxstbr][]
+
 # 3.6.1
 
 * Catch the github api error thrown from @octokit/rest [@Teamop][]

diff --git a/package.json b/package.json
@@ -79,6 +79,7 @@
     "@types/json5": "^0.0.29",
     "@types/lodash.includes": "^4.3.3",
     "@types/node-fetch": "^1.6.6",
+    "@types/p-limit": "^1.1.2",
     "@types/readline-sync": "^1.4.2",
     "@types/voca": "^1.3.0",
     "babel-cli": "7.0.0-alpha.19",
@@ -125,6 +126,7 @@
     "lodash.keys": "^4.0.8",
     "node-cleanup": "^2.1.2",
     "node-fetch": "^2.1.2",
+    "p-limit": "^1.2.0",
     "parse-diff": "^0.4.2",
     "parse-git-config": "^2.0.2",
     "parse-github-url": "^1.0.2",

diff --git a/source/platforms/github/GitHubAPI.ts b/source/platforms/github/GitHubAPI.ts
@@ -3,6 +3,7 @@ import * as debug from "debug"
 import * as node_fetch from "node-fetch"
 import * as parse from "parse-link-header"
 import * as v from "voca"
+import * as pLimit from "p-limit"
 
 import { GitHubPRDSL, GitHubUser } from "../../dsl/GitHubDSL"
 
@@ -15,6 +16,8 @@ import { RepoMetaData } from "../../dsl/BitBucketServerDSL"
 
 export type APIToken = string
 
+const limit = pLimit(25)
+
 // Note that there are parts of this class which don't seem to be
 // used by Danger, they are exposed for Peril support.
 
@@ -369,19 +372,21 @@ export class GitHubAPI {
       // e.g. https://gist.github.com/LTe/5270348
       customAccept = { Accept: `${this.additionalHeaders.Accept}, ${headers.Accept}` }
     }
-    return this.fetch(
-      url,
-      {
-        method: method,
-        body: body,
-        headers: {
-          "Content-Type": "application/json",
-          ...headers,
-          ...this.additionalHeaders,
-          ...customAccept,
+    return limit(() =>
+      this.fetch(
+        url,
+        {
+          method: method,
+          body: body,
+          headers: {
+            "Content-Type": "application/json",
+            ...headers,
+            ...this.additionalHeaders,
+            ...customAccept,
+          },
         },
-      },
-      suppressErrors
+        suppressErrors
+      )
     )
   }
 

diff --git a/yarn.lock b/yarn.lock
@@ -87,6 +87,10 @@
   version "8.0.46"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.46.tgz#6e1766b2d0ed06631d5b5f87bb8e72c8dbb6888e"
 
+"@types/p-limit@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@types/p-limit/-/p-limit-1.1.2.tgz#2af70c9f80fc85ab7502ca92091be0b8f401a774"
+
 "@types/readline-sync@^1.4.2":
   version "1.4.2"
   resolved "https://registry.yarnpkg.com/@types/readline-sync/-/readline-sync-1.4.2.tgz#516b5f4f250534062b6bb08ae8367a088dcf739f"
@@ -2400,9 +2404,9 @@ find@0.2.6:
   dependencies:
     traverse-chain "~0.1.0"
 
-flow-bin@^0.69.0:
-  version "0.69.0"
-  resolved "https://registry.yarnpkg.com/flow-bin/-/flow-bin-0.69.0.tgz#053159a684a6051fcbf0b71a2eb19a9679082da6"
+flow-bin@^0.71.0:
+  version "0.71.0"
+  resolved "https://registry.yarnpkg.com/flow-bin/-/flow-bin-0.71.0.tgz#fd1b27a6458c3ebaa5cb811853182ed631918b70"
 
 for-in@^1.0.1, for-in@^1.0.2:
   version "1.0.2"
@@ -4629,6 +4633,12 @@ p-limit@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.1.0.tgz#b07ff2d9a5d88bec806035895a2bab66a27988bc"
 
+p-limit@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.2.0.tgz#0e92b6bedcb59f022c13d0f1949dc82d15909f1c"
+  dependencies:
+    p-try "^1.0.0"
+
 p-locate@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43"
@@ -4639,6 +4649,10 @@ p-map@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/p-map/-/p-map-1.1.1.tgz#05f5e4ae97a068371bc2a5cc86bfbdbc19c4ae7a"
 
+p-try@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"
+
 package-json@^4.0.0:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/package-json/-/package-json-4.0.1.tgz#8869a0401253661c4c4ca3da6c2121ed555f5eed"
@@ -5952,9 +5966,9 @@ ts-jest@^22.4.2:
     pkg-dir "^2.0.0"
     yargs "^11.0.0"
 
-ts-node@^5.0.1:
-  version "5.0.1"
-  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-5.0.1.tgz#78e5d1cb3f704de1b641e43b76be2d4094f06f81"
+ts-node@^6.0.0:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-6.0.2.tgz#e132d530e53173bc6a8c21ea65f64d8b47bc573e"
   dependencies:
     arrify "^1.0.0"
     chalk "^2.3.0"