feat(git): added retry logic for all git operations. (#136)

* Added retry logic for all git operations.

In the complex CI environments, intermittent issues happen. In order to prevent build failures, it is good to give a server an opportunity to recover from high load spikes.

Put in the ~/.gitconfig following section:
[http]
        proxy = http://192.168.1.101:8888

As you don't have a proxy running on the port you will get a timeout error from git.

* review fixes

* logs are silly

* lint fixes

* added couple of real cases where retry helped to solve problem

lib/util/git.js

@@ -12,6 +12,7 @@ const optCheck = require('./opt-check')
 const osenv = require('osenv')
 const path = require('path')
 const pinflight = require('promise-inflight')
+const promiseRetry = require('promise-retry')
 const uniqueFilename = require('unique-filename')
 const which = BB.promisify(require('which'))
 const semver = require('semver')
@@ -26,6 +27,23 @@ const GOOD_ENV_VARS = new Set([
   'GIT_SSL_NO_VERIFY'
 ])
 
+const GIT_TRANSIENT_ERRORS = [
+  'remote error: Internal Server Error',
+  'The remote end hung up unexpectedly',
+  'Connection timed out',
+  'Operation timed out',
+  'Failed to connect to .* Timed out',
+  'Connection reset by peer',
+  'SSL_ERROR_SYSCALL',
+  'The requested URL returned error: 503'
+].join('|')
+
+const GIT_TRANSIENT_ERROR_RE = new RegExp(GIT_TRANSIENT_ERRORS)
+
+function shouldRetry (error) {
+  return GIT_TRANSIENT_ERROR_RE.test(error)
+}
+
 const GIT_ = 'GIT_'
 let GITENV
 function gitEnv () {
@@ -114,57 +132,51 @@ function revs (repo, opts) {
   return pinflight(`ls-remote:${repo}`, () => {
     return spawnGit(['ls-remote', '-h', '-t', repo], {
       env: gitEnv()
-    }, opts).then(child => {
-      let stdout = ''
-      let stderr = ''
-      child.stdout.on('data', d => { stdout += d })
-      child.stderr.on('data', d => { stderr += d })
-      return finished(child).catch(err => {
-        err.message = `Error while executing:\n${GITPATH} ls-remote -h -t ${repo}\n\n${stderr}\n${err.message}`
-        throw err
-      }).then(() => {
-        return stdout.split('\n').reduce((revs, line) => {
-          const split = line.split(/\s+/, 2)
-          if (split.length < 2) { return revs }
-          const sha = split[0].trim()
-          const ref = split[1].trim().match(/(?:refs\/[^/]+\/)?(.*)/)[1]
-          if (!ref) { return revs } // ???
-          if (ref.endsWith(CARET_BRACES)) { return revs } // refs/tags/x^{} crap
-          const type = refType(line)
-          const doc = {sha, ref, type}
-
-          revs.refs[ref] = doc
-          // We can check out shallow clones on specific SHAs if we have a ref
-          if (revs.shas[sha]) {
-            revs.shas[sha].push(ref)
-          } else {
-            revs.shas[sha] = [ref]
-          }
+    }, opts).then((stdout) => {
+      return stdout.split('\n').reduce((revs, line) => {
+        const split = line.split(/\s+/, 2)
+        if (split.length < 2) { return revs }
+        const sha = split[0].trim()
+        const ref = split[1].trim().match(/(?:refs\/[^/]+\/)?(.*)/)[1]
+        if (!ref) { return revs } // ???
+        if (ref.endsWith(CARET_BRACES)) { return revs } // refs/tags/x^{} crap
+        const type = refType(line)
+        const doc = {sha, ref, type}
 
-          if (type === 'tag') {
-            const match = ref.match(/v?(\d+\.\d+\.\d+(?:[-+].+)?)$/)
-            if (match && semver.valid(match[1], true)) {
-              revs.versions[semver.clean(match[1], true)] = doc
-            }
-          }
+        revs.refs[ref] = doc
+        // We can check out shallow clones on specific SHAs if we have a ref
+        if (revs.shas[sha]) {
+          revs.shas[sha].push(ref)
+        } else {
+          revs.shas[sha] = [ref]
+        }
 
-          return revs
-        }, {versions: {}, 'dist-tags': {}, refs: {}, shas: {}})
-      }).then(revs => {
-        if (revs.refs.HEAD) {
-          const HEAD = revs.refs.HEAD
-          Object.keys(revs.versions).forEach(v => {
-            if (v.sha === HEAD.sha) {
-              revs['dist-tags'].HEAD = v
-              if (!revs.refs.latest) {
-                revs['dist-tags'].latest = revs.refs.HEAD
-              }
-            }
-          })
+        if (type === 'tag') {
+          const match = ref.match(/v?(\d+\.\d+\.\d+(?:[-+].+)?)$/)
+          if (match && semver.valid(match[1], true)) {
+            revs.versions[semver.clean(match[1], true)] = doc
+          }
         }
-        REVS.set(repo, revs)
+
         return revs
-      })
+      }, {versions: {}, 'dist-tags': {}, refs: {}, shas: {}})
+    }, err => {
+      err.message = `Error while executing:\n${GITPATH} ls-remote -h -t ${repo}\n\n${err.stderr}\n${err.message}`
+      throw err
+    }).then(revs => {
+      if (revs.refs.HEAD) {
+        const HEAD = revs.refs.HEAD
+        Object.keys(revs.versions).forEach(v => {
+          if (v.sha === HEAD.sha) {
+            revs['dist-tags'].HEAD = v
+            if (!revs.refs.latest) {
+              revs['dist-tags'].latest = revs.refs.HEAD
+            }
+          }
+        })
+      }
+      REVS.set(repo, revs)
+      return revs
     })
   })
 }
@@ -173,15 +185,47 @@ module.exports._exec = execGit
 function execGit (gitArgs, gitOpts, opts) {
   opts = optCheck(opts)
   return checkGit().then(gitPath => {
-    return execFileAsync(gitPath, gitArgs, mkOpts(gitOpts, opts))
+    return promiseRetry((retry, number) => {
+      if (number !== 1) {
+        opts.log.silly('pacote', 'Retrying git command: ' + gitArgs.join(' ') + ' attempt # ' + number)
+      }
+      return execFileAsync(gitPath, gitArgs, mkOpts(gitOpts, opts)).catch((err) => {
+        if (shouldRetry(err)) {
+          retry(err)
+        } else {
+          throw err
+        }
+      })
+    }, opts.retry)
   })
 }
 
 module.exports._spawn = spawnGit
 function spawnGit (gitArgs, gitOpts, opts) {
   opts = optCheck(opts)
   return checkGit().then(gitPath => {
-    return cp.spawn(gitPath, gitArgs, mkOpts(gitOpts, opts))
+    return promiseRetry((retry, number) => {
+      if (number !== 1) {
+        opts.log.silly('pacote', 'Retrying git command: ' + gitArgs.join(' ') + ' attempt # ' + number)
+      }
+      const child = cp.spawn(gitPath, gitArgs, mkOpts(gitOpts, opts))
+
+      let stdout = ''
+      let stderr = ''
+      child.stdout.on('data', d => { stdout += d })
+      child.stderr.on('data', d => { stderr += d })
+
+      return finished(child).catch(err => {
+        if (shouldRetry(stderr)) {
+          retry(err)
+        } else {
+          err.stderr = stderr
+          throw err
+        }
+      }).then(() => {
+        return stdout
+      })
+    }, opts.retry)
   })
 }