Headers (#32)

* headers and published versions * headers only in touch and import for now * add tests
clay · Sep 13, 2017 · 53bcc84 · 53bcc84
1 parent 1e60bbe
commit 53bcc84
Show file tree

Hide file tree

Showing 10 changed files with 187 additions and 21 deletions.
diff --git a/.gitignore b/.gitignore
@@ -35,3 +35,6 @@ jspm_packages
 
 # Optional REPL history
 .node_repl_history
+
+# ide
+.idea
diff --git a/lib/cmd/import.js b/lib/cmd/import.js
@@ -10,7 +10,8 @@ const _ = require('lodash'),
   urlUtil = require('../utils/urls'),
   rest = require('../utils/rest'),
   clayInput = require('../io/input-clay'),
-  files = require('../io/input-files');
+  files = require('../io/input-files'),
+  getYargHeaders = require('../utils/headers').getYargHeaders;
 
 /**
  * handle errors and exit when importer fails
@@ -100,15 +101,16 @@ function showCompleted(results) {
  * @return {Stream}
  */
 function importStream(prefix, argv) {
-  const key = config.getKey(argv.key);
+  const key = config.getKey(argv.key),
+    headers = getYargHeaders(argv);
 
   return h(byline(process.stdin)) // byline splits on newlines and removes empty lines
     .map(JSON.parse)
     .map(chunks.validate) // validate chunks coming in from stdin
     .stopOnError(fatalError)
     // map agnostic chunks to data we can save
     .map(mapChunksToData(prefix))
-    .flatMap((item) => _.includes(item.url, '/uris') ? rest.put(item, key, argv.concurrency, 'text') : rest.put(item, key, argv.concurrency))
+    .flatMap((item) => rest.put(item, key, argv.concurrency, _.includes(item.url, '/uris') ? 'text': 'json', headers))
     .map(showProgress(argv))
     .toArray(showCompleted);
 }
@@ -122,13 +124,14 @@ function importStream(prefix, argv) {
  */
 function importSingleUrl(url, prefix, argv) {
   const key = config.getKey(argv.key),
-    newUrl = urlUtil.uriToUrl(prefix, url);
+    newUrl = urlUtil.uriToUrl(prefix, url),
+    headers = getYargHeaders(argv);
 
   logger.info('Importing single URL:', `\n${url}\n↓ ↓ ↓\n${newUrl}`);
   return clayInput.importUrl(url, argv.concurrency)
     .flatMap(chunks.replacePrefixes(prefix))
     .stopOnError(fatalError) // exit early if there's a problem reaching the input clay instance
-    .flatMap((item) => rest.put(item, key, argv.concurrency))
+    .flatMap((item) => rest.put(item, key, argv.concurrency, 'json', headers))
     .map(showProgress(argv))
     .toArray(showCompleted);
 }
@@ -145,7 +148,8 @@ function importSite() {
  * @return {Stream}
  */
 function importFile(filepath, prefix, argv) {
-  const key = config.getKey(argv.key);
+  const key = config.getKey(argv.key),
+    headers = getYargHeaders(argv);
 
   logger.debug(`Attempting to import ${filepath}`);
   return files.get(filepath).stopOnError((error) => {
@@ -158,7 +162,7 @@ function importFile(filepath, prefix, argv) {
   .map(chunks.validate)
   // map agnostic chunks to data we can save
   .map(mapChunksToData(prefix))
-  .flatMap((item) => _.includes(item.url, '/uris') ? rest.put(item, key, argv.concurrency, 'text') : rest.put(item, key, argv.concurrency))
+  .flatMap((item) => rest.put(item, key, argv.concurrency, _.includes(item.url, '/uris') ? 'text' : 'json', headers))
   .map(showProgress(argv))
   .toArray(showCompleted);
 }
@@ -188,7 +192,8 @@ function builder(yargs) {
     .option('o', options.offset)
     .option('q', options.query)
     // other options
-    .option('k', options.key);
+    .option('k', options.key)
+    .option('headers', options.headers);
 }
 
 function handler(argv) {

diff --git a/lib/cmd/touch.js b/lib/cmd/touch.js
@@ -5,12 +5,21 @@ const _ = require('lodash'),
   config = require('../utils/config'),
   logger = require('../utils/logger'),
   clay = require('../io/input-clay'),
-  rest = require('../utils/rest');
+  rest = require('../utils/rest'),
+  getYargHeaders = require('../utils/headers').getYargHeaders;
 
+/**
+ *
+ * @param {string} prefix
+ * @param {string} name
+ * @param {object} argv
+ */
 function runLogic(prefix, name, argv) {
   const concurrency = argv.concurrency,
+    headers = getYargHeaders(argv),
+    published = argv.published,
     processSpinner = logger.startSpinner('Processing...'),
-    instancesStream = clay.getComponentInstances(prefix, name, concurrency).stopOnError((err) => {
+    instancesStream = clay.getComponentInstances(prefix, name, concurrency, headers, published).stopOnError((err) => {
       logger.stopSpinner(processSpinner);
       logger.error(`Error resolving instances of ${name}!`, err.message);
     });
@@ -31,7 +40,7 @@ function runLogic(prefix, name, argv) {
     instancesStream.flatMap((instances) => {
       const urls = _.map(instances, config.normalizeSite);
 
-      return rest.get(urls, concurrency).stopOnError((err) => {
+      return rest.get(urls, concurrency, 'json', headers).stopOnError((err) => {
         logger.stopSpinner(processSpinner);
         logger.error(`Error resolving instance of ${name}!`, err.message);
       });
@@ -42,15 +51,27 @@ function runLogic(prefix, name, argv) {
   }
 }
 
+/**
+ * add options and comments specific to `touch`
+ * @param {object} yargs
+ * @returns {object}
+ */
 function builder(yargs) {
   return yargs
     .usage('Usage: $0 touch <component>')
     .example('$0 touch domain.com/components/foo', 'GET all instances of foo')
     .example('$0 touch foo --site bar', 'GET all instances of foo on site bar')
     .option('s', options.site)
-    .option('n', options.dryRun);
+    .option('n', options.dryRun)
+    .option('headers', options.headers)
+    .option('published', options.published);
 }
 
+/**
+ *
+ * @param {object} argv
+ * @returns {*}
+ */
 function handler(argv) {
   if (clayUtils.isComponent(argv.component)) {
     // component url passed in, get the number of instances and tell the user

diff --git a/lib/io/input-clay.js b/lib/io/input-clay.js
@@ -39,10 +39,12 @@ function expandPropReferences(val) {
  * @param  {string} prefix of site
  * @param  {string} name of component
  * @param {number} concurrency
+ * @param {object} [headers]
+ * @param {boolean} [onlyPublishedInstances]
  * @return {Stream}
  */
-function getComponentInstances(prefix, name, concurrency) {
-  return rest.get(`${prefix}/components/${name}/instances`, concurrency);
+function getComponentInstances(prefix, name, concurrency, headers, onlyPublishedInstances) {
+  return rest.get(`${prefix}/components/${name}/instances${onlyPublishedInstances ? '/@published' : ''}`, concurrency, 'json', headers);
 }
 
 /**

diff --git a/lib/io/input-clay.test.js b/lib/io/input-clay.test.js
@@ -35,6 +35,15 @@ describe('input clay', () => {
         done(err);
       });
     });
+
+    it('fetches published component instances', (done) => {
+      rest.get.returns(h(['one']));
+      fn('domain.com', 'foo', 1, {headerKey: 'headerVal'}, true).collect().toCallback((err, data) => {
+        expect(data).to.eql(['one']);
+        expect(rest.get).to.have.been.calledWith('domain.com/components/foo/instances/@published', 1, 'json', {headerKey: 'headerVal'});
+        done(err);
+      });
+    });
   });
 
   describe('listComponentReferences', () => {

diff --git a/lib/utils/headers.js b/lib/utils/headers.js
@@ -0,0 +1,25 @@
+/**
+ *
+ * @param {object} argv
+ * @param {string|Array} [argv.headers] e.g. `key:val`
+ * @returns {object} headers object for request
+ * @throws on unknown header format
+ */
+function getYargHeaders(argv) {
+  var headers = argv.headers || [];
+
+  if (typeof headers === 'string') {
+    headers = [headers];
+  }
+  return headers.reduce((obj, header) => {
+    const split = header.split(':');
+
+    if (split.length !== 2) {
+      throw new Error(`Unknown header format: ${header}`);
+    }
+    obj[split[0].trim()] = split[1].trim();
+    return obj;
+  }, {});
+}
+
+module.exports.getYargHeaders = getYargHeaders;
diff --git a/lib/utils/headers.test.js b/lib/utils/headers.test.js
@@ -0,0 +1,38 @@
+const lib = require('./headers'),
+  singleHeaderMock = {
+    argv: {
+      headers: 'X-Forwarded-Host: some-site.com'
+    },
+    headersObj: {
+      'X-Forwarded-Host': 'some-site.com'
+    }
+  },
+  multipleHeadersMock = {
+    argv: {
+      headers: [singleHeaderMock.argv.headers].concat(['Custom-Header: some-value'])
+    },
+    headersObj: Object.assign({'Custom-Header': 'some-value'}, singleHeaderMock.headersObj)
+  };
+
+describe('headers', function () {
+
+  describe('getYargHeaders', function () {
+    const fn = lib[this.title];
+
+    it('returns empty object if no headers yarg', () =>
+      expect(fn({})).to.deep.equal({})
+    );
+
+    it('returns headers object for single headers yarg', () =>
+      expect(fn(singleHeaderMock.argv)).to.deep.equal(singleHeaderMock.headersObj)
+    );
+
+    it('returns headers object for multiple headers yargs', () =>
+      expect(fn(multipleHeadersMock.argv)).to.deep.equal(multipleHeadersMock.headersObj)
+    );
+
+    it('throws on unknown header format', () =>
+      expect(fn.bind(lib, {headers: 'weird-header=unknown-format'})).to.throw(Error)
+    );
+  });
+});
diff --git a/lib/utils/rest.js b/lib/utils/rest.js
@@ -62,14 +62,18 @@ function createStream(items) {
  * @param  {Stream|array|string} urls array of urls or single url
  * @param {number} [concurrency]
  * @param {string} type 'json' or 'text'
+ * @param {object} [headers]
  * @return {Stream}
  */
-function get(urls, concurrency, type) {
+function get(urls, concurrency, type, headers) {
   type = type || 'json';
   concurrency = concurrency || DEFAULT_CONCURRENCY;
   return createStream(urls).map((url) => {
     logger.debug(`GET ${url}`);
-    return h(send(url, { method: 'GET' })
+    return h(send(url, {
+        method: 'GET',
+        headers: headers
+      })
       .then((res) => res[type]())
       .catch((e) => {
         e.url = url; // capture the url every time we error
@@ -84,9 +88,10 @@ function get(urls, concurrency, type) {
  * @param {string} key
  * @param  {number} [concurrency]
  * @param {string} [type]
+ * @param {object} [headers]
  * @return {Stream}
  */
-function put(items, key, concurrency, type) {
+function put(items, key, concurrency, type, headers) {
   concurrency = concurrency || DEFAULT_CONCURRENCY;
   type = type || 'json';
   return createStream(items).map((item) => {
@@ -98,10 +103,13 @@ function put(items, key, concurrency, type) {
     return h(send(url, {
       method: 'PUT',
       body: data,
-      headers: {
-        [contentHeader]: type === 'json' ? contentJSON : contentText,
-        Authorization: `Token ${key}`
-      }
+      headers: Object.assign(
+        {
+          [contentHeader]: type === 'json' ? contentJSON : contentText,
+          Authorization: `Token ${key}`
+        },
+        headers
+      )
       // we don't care about the data returned from the put, but we do care it it worked or not
     }).then(() => ({ result: 'success', url })).catch((e) => ({ result: 'error', url, message: e.message })));
   }).mergeWithLimit(concurrency);

diff --git a/lib/utils/rest.test.js b/lib/utils/rest.test.js
@@ -89,6 +89,21 @@ describe('rest', () => {
         done(err);
       });
     });
+
+    it('gets with headers', (done) => {
+      const mockHeaders = {
+        headerKey: 'headerVal'
+      };
+
+      fetch.send.returns(Promise.resolve({ url, status: 200, json: () => ({ a: 'b' })}));
+      fn(url, 1, 'json', mockHeaders).collect().toCallback((err, data) => {
+        expect(fetch.send.getCall(0).calledWith(url, {
+          method: 'GET',
+          headers: mockHeaders
+        }));
+        done(err);
+      });
+    });
   });
 
   describe('put', () => {
@@ -130,5 +145,31 @@ describe('rest', () => {
         done(err);
       });
     });
+
+    it('puts with headers', (done) => {
+      const mockHeaders = {
+          headerKey: 'headerVal',
+          'Content-Type': 'overwriteContentType'
+        },
+        mockHeadersWithAuth = {
+          headerKey: 'headerVal',
+          'Content-Type': 'overwriteContentType',
+          Authorization: 'Token null'
+        };
+
+      fetch.send.returns(Promise.resolve({ url, status: 200 }));
+      fn({ url, data: 'hi'}, null, 10, 'text', mockHeaders).collect().toCallback((err, data) => {
+        expect(data).to.eql([{ result: 'success', url }]);
+        expect(fetch.send.getCall(0).args).to.deep.equal([
+          url,
+          {
+            method: 'PUT',
+            body: 'hi',
+            headers: mockHeadersWithAuth
+          }
+        ]);
+        done(err);
+      });
+    });
   });
 });
diff --git a/lib/utils/shared-options.js b/lib/utils/shared-options.js
@@ -82,5 +82,19 @@ module.exports = {
     describe: 'number of concurrent requests against clay',
     type: 'number',
     default: 10
+  },
+  headers: {
+    // no aliases
+    demand: false,
+    nargs: 1,
+    describe: 'headers that may be required for requests to clay, e.g. X-Forwarded-Host:clay-site.com',
+    type: 'string'
+  },
+  published: {
+    // no aliases
+    demand: false,
+    describe: 'only published instances, e.g. site.com/components/a/instances/b@published',
+    type: 'boolean',
+    default: false
   }
 };