Support for tarball

lib/document-downloader.js

@@ -40,15 +40,15 @@ DocumentDownloader.fetchAll = function (urls) {
 };
 
 DocumentDownloader.install = function (dest, content) {
+  var path = require('path').dirname(dest);
+
+  Mkdirp.sync(path);
   return Promise.denodeify(Fs.writeFile)(dest, content);
 };
 
 DocumentDownloader.installAll = function (destsContents) {
   // `e` is a tuple of [dest, content]
   return Promise.all(destsContents.toArray().map(function (e) {
-    var path = require('path').dirname(e[0]);
-
-    Mkdirp.sync(path);
     return DocumentDownloader.install(e[0], e[1]);
   }));
 };
@@ -62,6 +62,7 @@ DocumentDownloader.installAll = function (destsContents) {
 DocumentDownloader.isAllowed = function isAllowed(filename) {
   if (filename.toLowerCase().indexOf('.htaccess') !== -1) return false;
   else if (filename.toLowerCase().indexOf('.php') !== -1) return false;
+  else if (filename.indexOf('CVS') !== -1) return false;
   else if (filename.indexOf('../') !== -1) return false;
   else return true;
 };
@@ -76,35 +77,67 @@ DocumentDownloader.fetchAndInstall = function (url, dest) {
     if (!pathExists) return mkdir(dest);
   }).then(function () {
     return _.fetch(url).then(function (content) {
-      var contentUtf8 = content.toString('utf8');
-
-      if (contentUtf8.trim().charAt(0) !== '<') {
-        var filenames = _.getFilenames(contentUtf8).filter(_.isAllowed);
-
-        var dests = filenames
-          .set(0, 'Overview.html')
-          .map(function (filename) { return dest + '/' + filename; });
-
-        var urls = filenames.map(function (filename) {
-          // If an entry in the manifest had a space in it,
-          // we assume it needs to be built from the spec-generator.
-          // See https://github.com/w3c/spec-generator
-          var specGeneratorComp = filename.split(' ');
-          var absUrl = Url.resolve(url, specGeneratorComp[0]);
-
-          if (specGeneratorComp.length === 2) {
-            return global.SPEC_GENERATOR +
-              '?type=' + encodeURIComponent(specGeneratorComp[1]) +
-              '&url=' + encodeURIComponent(absUrl);
-          }
-          return absUrl;
-        });
-
-        return _.fetchAll(urls).then(function (contents) {
-          return _.installAll(dests.zip(contents));
+      var fileType = require('file-type');
+      var type = fileType(content);
+
+      if (type && type.mime === 'application/x-tar') {
+        return new Promise(function (resolve, reject) {
+          var tar = require('tar-stream');
+          var extract = tar.extract();
+          var hasOverview = false;
+
+          extract.on('entry', function (header, stream, callback) {
+            stream.on('data', function (data) {
+              if (_.isAllowed(header.name)) {
+                if (header.name === 'Overview.html') {
+                  hasOverview = true;
+                }
+                _.install(dest + '/' + header.name, data).then(function () {
+                  callback();
+                });
+              }
+            });
+          });
+          extract.on('finish', function () {
+            return (hasOverview) ? resolve() :
+              reject(
+                new Error('The tarball doesn\'t have an Overview.html file.')
+              );
+          });
+          extract.end(content);
         });
       }
-      else return _.install(dest + '/Overview.html', content);
+      else {
+        var contentUtf8 = content.toString('utf8');
+
+        if (contentUtf8.trim().charAt(0) !== '<') {
+          var filenames = _.getFilenames(contentUtf8).filter(_.isAllowed);
+
+          var dests = filenames
+            .set(0, 'Overview.html')
+            .map(function (filename) { return dest + '/' + filename; });
+
+          var urls = filenames.map(function (filename) {
+            // If an entry in the manifest had a space in it,
+            // we assume it needs to be built from the spec-generator.
+            // See https://github.com/w3c/spec-generator
+            var specGeneratorComp = filename.split(' ');
+            var absUrl = Url.resolve(url, specGeneratorComp[0]);
+
+            if (specGeneratorComp.length === 2) {
+              return global.SPEC_GENERATOR +
+                '?type=' + encodeURIComponent(specGeneratorComp[1]) +
+                '&url=' + encodeURIComponent(absUrl);
+            }
+            return absUrl;
+          });
+
+          return _.fetchAll(urls).then(function (contents) {
+            return _.installAll(dests.zip(contents));
+          });
+        }
+        else return _.install(dest + '/Overview.html', content);
+      }
     });
   });
 };

package.json

@@ -46,13 +46,15 @@
     "body-parser": "^1.13.2",
     "compression": "^1.5.1",
     "express": "^4.13.1",
+    "file-type": "^3.3.0",
     "immutable": "^3.7.4",
     "mkdirp": "^0.5.1",
     "moment": "^2.10.3",
     "node-uuid": "^1.4.3",
     "promise": "^7.0.3",
     "request": "^2.58.0",
     "specberus": "^1.3.0",
+    "tar-stream": "^1.3.1",
     "third-party-resources-checker": "^1.0.2"
   },
   "devDependencies": {

test/test.js

@@ -282,6 +282,19 @@ describe('DocumentDownloader', function () {
         Fs.rmdirSync('/tmp/testechidnaSpecGeneration');
       });
     });
+
+    it('should read a tarball and install its content', function () {
+      return DocumentDownloader.fetchAndInstall(
+        server.location() + '/drafts/frame-timing.tar',
+        '/tmp/testechidnaTarball'
+      ).then(function () {
+        expect(readFileSyncUtf8('/tmp/testechidnaTarball/Overview.html'))
+          .to.contain('Frame Timing');
+
+        Fs.unlinkSync('/tmp/testechidnaTarball/Overview.html');
+        Fs.rmdirSync('/tmp/testechidnaTarball');
+      });
+    });
   });
 
   describe('getFilenames(manifestContent)', function () {