Scenario: get archive

bin/record.js

@@ -66,10 +66,10 @@ scenarios.reduce(async (promise, scenarioPath) => {
     ids: {}
   }
 
-  const newNormalizedFixtures = newRawFixtures
+  const newNormalizedFixtures = await Promise.all(newRawFixtures
     .map(cloneDeep)
     .filter(hasntIgnoreHeader)
-    .map(normalize.bind(null, scenarioState))
+    .map(normalize.bind(null, scenarioState)))
 
   const fixturesDiffs = diff(newNormalizedFixtures, oldNormalizedFixtures)
   if (!fixturesDiffs) {

lib/fixturize-entity-id.js

@@ -14,6 +14,12 @@ function fixturizeEntityId (entityIdsMap, entityName, id) {
     return map[id]
   }
 
+  // Do nothing if passed id is a normalized one
+  const check = parseInt(id, 10)
+  if (Object.values(map).includes(check)) {
+    return check
+  }
+
   // Otherwise calculate the new id and set it on the passed state map.
   // IDs start at 1000 to differentiate from issue/PR numbers
   const counter = Object.keys(map).length + 1000

lib/normalize/archive.js

@@ -0,0 +1,62 @@
+module.exports = normalizeArchive
+
+const zlib = require('zlib')
+
+const intoStream = require('into-stream')
+const getStream = require('get-stream')
+const tar = require('tar-stream')
+const gunzip = require('gunzip-maybe')
+
+const temporaryRepository = require('../temporary-repository')
+
+async function normalizeArchive (scenarioState, response, fixture) {
+  fixture.headers['content-disposition'] = fixture.headers['content-disposition']
+    // normalize folder name in file name
+    .replace(temporaryRepository.regex, '$1')
+    // zerofy sha
+    .replace(/archive-\w{7}/, 'archive-0000000')
+
+  const extract = tar.extract()
+  const pack = tar.pack()
+  const readStream = intoStream(Buffer.from(response, 'hex'))
+
+  // The response is the Repository folder with the README.md file inside. The
+  // folder name is always different, based on the repository name when recorded.
+  // That's why we have to untar/zip the response, change the folder name and
+  // retar/zip it again.
+
+  extract.on('entry', function (header, stream, callback) {
+    header.name = header.name
+      // normalize folder name in path
+      .replace(temporaryRepository.regex, '$1')
+      // zerofy sha in path
+      .replace(/-(\w){7}\//, '-0000000/')
+
+    // normalize mtime
+    header.mtime = {
+      getTime: () => 1507651200000
+    }
+
+    // write the new entry to the pack stream
+    stream.pipe(pack.entry(header, callback))
+  })
+
+  extract.on('finish', function () {
+    // all entries done - lets finalize it
+    pack.finalize()
+  })
+
+  // pipe the old tarball to the extractor
+  readStream.pipe(gunzip()).pipe(extract)
+
+  // pipe the new tarball the another stream
+  const writeStream = pack.pipe(zlib.createGzip())
+
+  const result = await getStream.buffer(writeStream).catch(console.log)
+  fixture.response = result.toString('hex')
+
+  // normalize across operating systems / extra flags
+  // see http://www.zlib.org/rfc-gzip.html#header-trailer
+  const normalizedHeader = '1f8b0800000000000003'
+  fixture.response = normalizedHeader + fixture.response.substr(normalizedHeader.length)
+}

lib/normalize/index.js

@@ -8,7 +8,7 @@ const headers = require('../headers')
 const setIfExists = require('../set-if-exists')
 const toEntityName = require('../to-entity-name')
 
-function normalize (scenarioState, fixture) {
+async function normalize (scenarioState, fixture) {
   // fixture.rawHeaders is an array in the form of ['key1', 'value1', 'key2', 'value2']
   // But the order of these can change, e.g. between local tests and CI on Travis.
   // That’s why we turn them into an object before storing the fixtures and turn
@@ -23,6 +23,7 @@ function normalize (scenarioState, fixture) {
 
   // set all dates to Universe 2017 Keynote time
   setIfExists(fixture.headers, 'date', 'Tue, 10 Oct 2017 16:00:00 GMT')
+  setIfExists(fixture.headers, 'expires', 'Tue, 10 Oct 2017 16:00:00 GMT')
   setIfExists(fixture.headers, 'last-modified', 'Tue, 10 Oct 2017 16:00:00 GMT')
   setIfExists(fixture.headers, 'x-ratelimit-reset', '1507651200000')
 
@@ -64,19 +65,25 @@ function normalize (scenarioState, fixture) {
   }
 
   const responses = Array.isArray(fixture.response) ? fixture.response : [fixture.response]
-  responses.forEach(response => {
+  await Promise.all(responses.map(async (response) => {
     const entityName = toEntityName(response, fixture)
     if (entityName) {
-      require(`./${entityName}`)(scenarioState, response, fixture)
+      await require(`./${entityName}`)(scenarioState, response, fixture)
     }
-  })
+  }))
 
   // remove headers added by proxy
   // see https://github.com/octokit/fixtures/pull/20#issuecomment-331558385
   fixture.headers = omitBy(fixture.headers, (value, key) => /^x-(now|zeit)-/.test(key))
 
   // update content length
-  fixture.headers['content-length'] = String(calculateBodyLength(fixture.response))
+  if (/^application\/json/.test(fixture.headers['content-type'])) {
+    fixture.headers['content-length'] = String(calculateBodyLength(fixture.response))
+  }
+
+  if (fixture.responseIsBinary) {
+    fixture.headers['content-length'] = Buffer.from(fixture.response, 'hex').length
+  }
 
   // remove `Transfer-Encoding: chunked` headers:
   // https://github.com/octokit/fixtures/issues/97octokit/fixtures#97
@@ -90,7 +97,10 @@ function normalize (scenarioState, fixture) {
 
   // handle redirect response
   if (fixture.status > 300 && fixture.status < 400) {
-    fixture.response.url = fixturizePath(scenarioState, fixture.response.url)
+    fixture.headers.location = fixturizePath(scenarioState, fixture.headers.location)
+    if (fixture.response.url) {
+      fixture.response.url = fixture.headers.location
+    }
   }
 
   return fixture

lib/record-scenario.js

@@ -14,13 +14,21 @@ async function recordScenario ({request, scenario}) {
   if (Array.isArray(scenario)) {
     // if scenario is an array of request options, send requests sequentially
     await scenario.reduce(async (promise, step) => {
+      let response
+
       try {
-        await promise
+        response = await promise
       } catch (error) {
         // don’t fail on 4xx errors, they are valid fixtures
         if (error.response.status >= 500) {
           throw error
         }
+
+        response = error.response
+      }
+
+      if (typeof step === 'function') {
+        return request(step(response))
       }
 
       return request(step)

lib/to-entity-name.js

@@ -1,9 +1,15 @@
 module.exports = toEntityName
 
 function toEntityName (object, fixture) {
+  // object is binary response, so we check for it above the object check
+  if (/\/legacy\.(tar\.gz|zip)\/master$/.test(fixture.path)) {
+    return 'archive'
+  }
+
   if (typeof object !== 'object') {
     return
   }
+
   if (object.type === 'Organization') {
     return 'organization'
   }
@@ -52,6 +58,7 @@ function toEntityName (object, fixture) {
   if (/^\/search\/issues\?/.test(fixture.path)) {
     return 'search-issues'
   }
+
   if ('errors' in object) {
     return 'error'
   }

package.json

@@ -35,8 +35,8 @@
   },
   "homepage": "https://github.com/octokit/fixtures#readme",
   "dependencies": {
-    "lodash": "^4.17.4",
     "json-diff": "^0.5.2",
+    "lodash": "^4.17.4",
     "nock": "^9.1.6",
     "url-template": "^2.0.8"
   },
@@ -45,14 +45,18 @@
     "chalk": "^2.1.0",
     "coveralls": "^3.0.0",
     "envalid": "^4.0.1",
+    "get-stream": "^3.0.0",
     "glob": "^7.1.2",
+    "gunzip-maybe": "^1.4.1",
     "humanize-string": "^1.0.1",
+    "into-stream": "^3.1.0",
     "minimist": "^1.2.0",
     "mkdirp": "^0.5.1",
     "proxyquire": "^1.8.0",
     "semantic-release": "^13.1.3",
     "standard": "^10.0.3",
-    "tap": "^11.0.0"
+    "tap": "^11.0.0",
+    "tar-stream": "^1.5.5"
   },
   "nyc": {
     "exclude": [

scenarios/api.github.com/add-and-remove-repository-collaborator/normalized-fixture.json

@@ -327,8 +327,7 @@
       "x-ratelimit-remaining": "4999",
       "x-ratelimit-reset": "1507651200000",
       "x-runtime-rack": "0.000000",
-      "x-xss-protection": "1; mode=block",
-      "content-length": "0"
+      "x-xss-protection": "1; mode=block"
     }
   },
   {
@@ -448,8 +447,7 @@
       "x-ratelimit-remaining": "4999",
       "x-ratelimit-reset": "1507651200000",
       "x-runtime-rack": "0.000000",
-      "x-xss-protection": "1; mode=block",
-      "content-length": "0"
+      "x-xss-protection": "1; mode=block"
     }
   },
   {

scenarios/api.github.com/branch-protection/normalized-fixture.json

@@ -268,8 +268,7 @@
       "x-ratelimit-remaining": "4999",
       "x-ratelimit-reset": "1507651200000",
       "x-runtime-rack": "0.000000",
-      "x-xss-protection": "1; mode=block",
-      "content-length": "0"
+      "x-xss-protection": "1; mode=block"
     }
   }
 ]

scenarios/api.github.com/get-archive/normalized-fixture.json

@@ -0,0 +1,71 @@
+[
+  {
+    "scope": "https://api.github.com:443",
+    "method": "get",
+    "path": "/repos/octokit-fixture-org/get-archive/tarball/master",
+    "body": "",
+    "status": 302,
+    "response": "",
+    "reqheaders": {
+      "accept": "application/vnd.github.v3+json",
+      "host": "api.github.com"
+    },
+    "headers": {
+      "access-control-allow-origin": "*",
+      "access-control-expose-headers": "ETag, Link, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval",
+      "cache-control": "public, must-revalidate, max-age=0",
+      "connection": "close",
+      "content-length": "0",
+      "content-security-policy": "default-src 'none'",
+      "content-type": "text/html;charset=utf-8",
+      "date": "Tue, 10 Oct 2017 16:00:00 GMT",
+      "expires": "Tue, 10 Oct 2017 16:00:00 GMT",
+      "location": "https://codeload.github.com/octokit-fixture-org/get-archive/legacy.tar.gz/master",
+      "status": "302 Found",
+      "strict-transport-security": "max-age=31536000; includeSubdomains; preload",
+      "x-content-type-options": "nosniff",
+      "x-frame-options": "deny",
+      "x-github-request-id": "0000:00000:0000000:0000000:00000000",
+      "x-ratelimit-limit": "60",
+      "x-ratelimit-remaining": "59",
+      "x-ratelimit-reset": "1507651200000",
+      "x-runtime-rack": "0.000000",
+      "x-xss-protection": "1; mode=block"
+    },
+    "badheaders": [
+      "authorization"
+    ]
+  },
+  {
+    "scope": "https://codeload.github.com:443",
+    "method": "get",
+    "path": "/octokit-fixture-org/get-archive/legacy.tar.gz/master",
+    "body": "",
+    "status": 200,
+    "response": "1f8b0800000000000003cb4f2ec9cfce2cd14dcbac28292d4ad5cd2f4ad74d4f2dd14d2c4acec82c4bd53580007d060a0050bfb9b9a90203c428741ac2313436343307222320dbc010a8dc5c81c194124b8905a5c525894540a714e5e797e05347481edd734304e41319ff41ae8e2ebeae7ab92964d801d46f66668227fe0d4d51e3dfc8d0c8d808284f75df6201233cfe951590627ba01d330a46c1281805a3806e000024cb59d6000a0000",
+    "reqheaders": {
+      "accept": "application/vnd.github.v3+json",
+      "host": "codeload.github.com"
+    },
+    "responseIsBinary": true,
+    "headers": {
+      "access-control-allow-origin": "https://render.githubusercontent.com",
+      "connection": "close",
+      "content-disposition": "attachment; filename=octokit-fixture-org-get-archive-0000000.tar.gz",
+      "content-security-policy": "default-src 'none'; style-src 'unsafe-inline'; sandbox",
+      "content-type": "application/x-gzip",
+      "date": "Tue, 10 Oct 2017 16:00:00 GMT",
+      "etag": "\"00000000000000000000000000000000\"",
+      "strict-transport-security": "max-age=31536000",
+      "x-content-type-options": "nosniff",
+      "x-frame-options": "deny",
+      "x-geo-block-list": "",
+      "x-github-request-id": "0000:00000:0000000:0000000:00000000",
+      "x-xss-protection": "1; mode=block",
+      "content-length": 172
+    },
+    "badheaders": [
+      "authorization"
+    ]
+  }
+]