Compare JS files against known JS libs (fix #383)

package.json

@@ -59,6 +59,7 @@
     "eslint": "1.10.3",
     "postcss": "5.0.12",
     "semver": "5.1.0",
+    "sha.js": "2.4.4",
     "source-map-support": "0.4.0",
     "xmldom": "0.1.19",
     "yargs": "3.30.0",

src/filehasher.js

@@ -0,0 +1,90 @@
+import fs from 'fs';
+import readline from 'readline';
+
+import createHash from 'sha.js';
+
+import log from 'logger';
+import { lstatPromise } from 'io/utils';
+
+
+export default class FileHasher {
+
+  constructor({hashes=null, path='./src/hashes.txt', lstat=lstatPromise} = {}) {
+    this._hashes = hashes;
+    this._lstatPromise = lstat;
+    this._pathToHashes = path;
+  }
+
+  getHashes() {
+    if (this._hashes) {
+      log.debug('Hashes already loaded; skipping file lookup.');
+      return Promise.resolve(this._hashes);
+    }
+
+    return this._loadHashesFromFile();
+  }
+
+  matchesJSLibrary(fileContents) {
+    return this.getHashes()
+      .then((hashes) => {
+        var hash = this._makeHash(fileContents);
+
+        if (Object.keys(hashes).includes(hash)) {
+          return {
+            matches: true,
+            name: hashes[hash],
+          };
+        } else {
+          return {
+            matches: false,
+            name: null,
+          };
+        }
+      });
+  }
+
+  _loadHashesFromFile(path=this._pathToHashes) {
+    var invalidMessage = new Error(
+      `Path "${path}" is not a file or does not exist.`);
+
+    log.debug(`Attempting to load hashes from ${path}.`);
+
+    return new Promise((resolve, reject) => {
+      this._lstatPromise(path)
+        .then((stats) => {
+          if (stats.isFile() !== true) {
+            return reject(invalidMessage);
+          }
+
+          var hashes = {};
+          var hashFile = readline.createInterface({
+            input: fs.createReadStream(path),
+          });
+
+          hashFile.on('line', (line) => {
+            var hash = line.split(' ')[0];
+            var name = line.split(' ')[1];
+            hashes[hash] = name;
+          });
+
+          hashFile.on('close', () => {
+            log.debug(`Loaded ${Object.keys(hashes).length} hashes.`);
+            this._hashes = hashes;
+
+            resolve(hashes);
+          });
+        })
+        .catch((err) => {
+          if (err.code !== 'ENOENT') {
+            reject(err);
+          } else {
+            reject(invalidMessage);
+          }
+        });
+    });
+  }
+
+  _makeHash(string) {
+    return createHash('sha256').update(string, 'utf8').digest('hex');
+  }
+}

src/validator.js

@@ -10,11 +10,11 @@ import { ARCH_DEFAULT, ARCH_JETPACK, CHROME_MANIFEST, INSTALL_RDF,
          MANIFEST_JSON } from 'const';
 import * as exceptions from 'exceptions';
 import * as messages from 'messages';
-import { checkMinNodeVersion,
-         gettext as _, singleLineString } from 'utils';
+import { checkMinNodeVersion, gettext as _, singleLineString } from 'utils';
 
 import log from 'logger';
 import Collector from 'collector';
+import FileHasher from 'filehasher';
 import InstallRdfParser from 'parsers/installrdf';
 import ManifestJSONParser from 'parsers/manifestjson';
 import ChromeManifestScanner from 'scanners/chromemanifest';
@@ -375,7 +375,7 @@ export default class Validator {
         return this.getAddonMetadata();
       })
       .then((addonMetadata) => {
-        return this._markEmptyFiles(addonMetadata);
+        return this.markSpecialFiles(addonMetadata);
       })
       .then((addonMetadata) => {
         log.info('Metadata option is set to %s', this.config.metadata);
@@ -460,6 +460,13 @@ export default class Validator {
     }
   }
 
+  markSpecialFiles(addonMetadata) {
+    return this._markEmptyFiles(addonMetadata)
+      .then((addonMetadata) => {
+        return this._markJSLibs(addonMetadata);
+      });
+  }
+
   _getAddonArchitecture(xpiMetadata) {
     // If we find a file named bootstrap.js this is assumed to be a
     // Jetpack add-on: https://github.com/mozilla/amo-validator/blob/7a8011a/validator/testcases/jetpack.py#L154
@@ -497,4 +504,31 @@ export default class Validator {
       });
   }
 
+  _markJSLibs(addonMetadata) {
+    var jsLibs = {};
+    var promises = [];
+    var hasher = new FileHasher();
+
+    return this.io.getFilesByExt('.js')
+      .then((files) => {
+        for (let filename of files) {
+          promises.push(this.io.getFile(filename)
+            .then((file) => {
+              return hasher.matchesJSLibrary(file);
+            })
+            .then((hashResult) => {
+              if (hashResult.matches === true) {
+                log.debug(`${hashResult.name} detected in ${filename}`);
+                jsLibs[filename] = hashResult.name;
+              }
+            }));
+        }
+
+        return Promise.all(promises);
+      }).then(() => {
+        addonMetadata.jsLibs = jsLibs;
+        return addonMetadata;
+      });
+  }
+
 }

tasks/eslint.js

@@ -2,6 +2,7 @@ module.exports = {
   js: {
     src: [
       'tests/**/*.js*',
+      '!tests/fixtures/jslibs/**.js',
       'src/**/*.js',
       'Gruntfile.js',
     ],

tasks/jscs.js

@@ -7,6 +7,7 @@ module.exports = {
       src: [
         'tasks/**/*.js*',
         'tests/**/*.js*',
+        '!tests/fixtures/jslibs/**.js',
         'src/**/*.js',
         'Gruntfile.js',
         'webpack.config.js',