Merge branch 'jrburke-596932-async-define'

packages/api-utils/lib/securable-module.js

@@ -48,6 +48,23 @@
    var systemPrincipal = Cc["@mozilla.org/systemprincipal;1"]
                          .createInstance(Ci.nsIPrincipal);
 
+   // Even though manifest.py does some dependency scanning, that
+   // scan is done as part of an evaluation of what the add-on needs
+   // for security purposes. The following regexps are used to scan for
+   // dependencies inside a simplified define() callback:
+   // define(function(require, exports, module){ var a = require('a'); });
+   // and are used at runtime ensure the dependencies needed by
+   // the define factory function are already evaluated and ready.
+   // Even though this loader is a sync loader, and could fetch the module
+   // as the require() call happens, it would differ in behavior as
+   // compared to the async browser case, which would make sure to execute
+   // the dependencies first before executing the define() factory function.
+   // So this dependency scanning and evaluation is kept to match the
+   // async behavior.
+   var commentRegExp = /(\/\*([\s\S]*?)\*\/|\/\/(.*)$)/mg;
+   var cjsRequireRegExp = /require\(["']([\w\!\-_\.\/]+)["']\)/g;
+   var cjsStandardDeps = ['require', 'exports', 'module'];
+
    function resolvePrincipal(principal, defaultPrincipal) {
      if (principal === undefined)
        return defaultPrincipal;
@@ -180,25 +197,33 @@
        );
      if ('modules' in options)
        throw new Error('options.modules is no longer supported');
+     // pathAccessed used to know if a module was accessed/required
+     // by another module, and in that case, assigning the module value
+     // via a define callback is not allowed.
+     if (options.pathAccessed === undefined)
+       options.pathAccessed = {};
      if (options.globals === undefined)
        options.globals = {};
 
      this.fs = options.fs;
      this.sandboxFactory = options.sandboxFactory;
      this.sandboxes = {};
      this.modules = {};
+     this.pathAccessed = options.pathAccessed;
      this.module_infos = {};
+     this.pathToModule = {};
+     this.defineUsed = {};
      this.globals = options.globals;
      this.getModuleExports = options.getModuleExports;
      this.modifyModuleSandbox = options.modifyModuleSandbox;
      this.securityPolicy = options.securityPolicy;
    };
 
    exports.Loader.prototype = {
-     _makeRequire: function _makeRequire(basePath) {
+     _makeApi: function _makeApi(basePath) {
        var self = this;
 
-       return function require(module) {
+       function syncRequire(module) {
          var exports;
 
          if (self.getModuleExports)
@@ -209,6 +234,16 @@
            var path = self.fs.resolveModule(basePath, module);
            if (!path)
              throw new Error('Module "' + module + '" not found');
+
+           // Track accesses to this module via its normalized path
+           if (!self.pathAccessed[path]) {
+             self.pathAccessed[path] = 0;
+           }
+           self.pathAccessed[path] += 1;
+
+           // Remember the name of the module as it maps to its path
+           self.pathToModule[path] = module;
+
            if (path in self.modules) {
              module_info = self.module_infos[path];
            } else {
@@ -229,7 +264,9 @@
              self.sandboxes[path] = sandbox;
              for (name in self.globals)
                sandbox.defineProperty(name, self.globals[name]);
-             sandbox.defineProperty('require', self._makeRequire(path));
+             var api = self._makeApi(path);
+             sandbox.defineProperty('require', api.require);
+             sandbox.defineProperty('define', api.define);
              self.module_infos[path] = module_info;
              if (self.modifyModuleSandbox)
                self.modifyModuleSandbox(sandbox, module_info);
@@ -247,6 +284,197 @@
 
          return exports;
        };
+
+       // START support Async module-style require and define calls.
+       // If the only argument to require is a string, then the module that
+       // is represented by that string is fetched for the appropriate context.
+       //
+       // If the first argument is an array, then it will be treated as an array
+       // of dependency string names to fetch. An optional function callback can
+       // be specified to execute when all of those dependencies are available.
+       function asyncRequire(deps, callback) {
+         if (typeof deps === "string" && !callback) {
+           // Just return the module wanted via sync require.
+           return syncRequire(deps);
+         } else {
+           asyncMain(null, basePath, null, deps, callback);
+           return undefined;
+         }
+       }
+
+       // The function that handles definitions of modules. Differs from
+       // require() in that a string for the module should be the first
+       // argument, and the function to execute after dependencies are loaded
+       // should return a value to define the module corresponding to the first
+       // argument's name.
+       function define (name, deps, callback) {
+
+         // Only allow one call to define per module/file.
+         if (self.defineUsed[basePath]) {
+           throw new Error("Only one call to define() allowed per file: " +
+                            basePath);
+         } else {
+           self.defineUsed[basePath] = true;
+         }
+
+         // For anonymous modules, the namePath is the basePath
+         var namePath = basePath,
+             exports = {}, exported;
+
+         // Adjust args if an anonymous module
+         if (typeof name !== 'string') {
+           callback = deps;
+           deps = name;
+           name = null;
+         }
+
+         // If just a define({}) call (no dependencies),
+         // adjust args accordingly.
+         if (!Array.isArray(deps)) {
+           callback = deps;
+           deps = null;
+         }
+
+         // Set up the path if we have a name
+         if (name) {
+           // Make sure that the name matches the expected name, otherwise
+           // throw an error.
+           namePath = self.fs.resolveModule(basePath, name);
+           if (self.pathToModule[namePath] !== name) {
+             throw new Error("Mismatched define(). Named module " + name +
+                             " does not match expected name of " +
+                             self.pathToModule[basePath] +
+                             " in " + basePath);
+           }
+         }
+
+         // If the callback is not an actual function, it means it already
+         // has the definition of the module as a literal value.
+         if (!deps && callback && typeof callback !== 'function') {
+           self.modules[namePath] = callback;
+           return;
+         }
+
+         // Set the exports value now in case other modules need a handle
+         // on it for cyclical cases.
+         self.modules[namePath] = exports;
+
+         // Load dependencies and call the module's definition function.
+         exported = asyncMain(name, namePath, exports, deps, callback);
+
+         // Assign output of function to name, if exports was not
+         // in play (which asyncMain already figured out).
+         if (exported !== undefined) {
+           if (self.pathAccessed[namePath] > 1) {
+             // Another module already accessed the exported value,
+             // need to throw to avoid nasty circular dependency weirdness
+             throw new Error('Module "' + (name || namePath) + '" cannot use ' +
+                             'return from define to define the module ' +
+                             'after another module has referenced its ' +
+                             'exported value.');
+           } else {
+             self.modules[namePath] = exported;
+           }
+         }
+       }
+
+       // The function that handles the main async module work, for both
+       // require([], function(){}) calls and define calls.
+       // It makes sure all the dependencies exist before calling the
+       // callback function. It will return the result of the callback
+       // function if "exports" is not a dependency.
+       function asyncMain (name, namePath, exports, deps, callback) {
+
+         if (typeof deps === 'function') {
+           callback = deps;
+           deps = null;
+         }
+
+         if (!deps) {
+           deps = [];
+           // The shortened form of the async wrapper for CommonJS modules:
+           // define(function (require, exports, module) {});
+           // require calls could be inside the function, so toString it
+           // and pull out the dependencies.
+
+           // Remove comments from the callback string,
+           // look for require calls, and pull them into the dependencies.
+           // The comment regexp is not very robust, but good enough to
+           // avoid commented out require calls and to find normal, sync
+           // require calls in the function.
+           callback
+               .toString()
+               .replace(commentRegExp, "")
+               .replace(cjsRequireRegExp, function (match, dep) {
+                 deps.push(dep);
+               });
+           // Prepend standard require, exports, and module dependencies
+           // (and in that *exact* order per spec), but only add as many as
+           // was asked for via the callback's function argument length.
+           // In particular, do *not* pass exports if it was not asked for.
+           // By asking for exports as a dependency the rest of this
+           // asyncRequire code assumes then that the return value from the
+           // function should not be used as the exported module value.
+           deps = cjsStandardDeps.slice(0, callback.length).concat(deps);
+         }
+
+         var depModules = [],
+             usesExports = false,
+             exported;
+
+         // Load all the dependencies, with the "require", "exports" and
+         // "module" ones getting special handling to match the traditional
+         // CommonJS sync module expectations.
+         deps.forEach(function (dep) {
+             if (dep === "require") {
+               depModules.push(asyncRequire);
+             } else if (dep === "module") {
+               depModules.push({
+                 id: name
+               });
+             } else if (dep === "exports") {
+               usesExports = true;
+               depModules.push(exports);
+             } else {
+               var overridden;
+               if (self.getModuleExports)
+                 overridden = self.getModuleExports(basePath, dep);
+               if (overridden) {
+                 depModules.push(overridden);
+                 return;
+               }
+
+               var depPath = self.fs.resolveModule(basePath, dep);
+
+               if (!self.modules[depPath]) {
+                 syncRequire(dep);
+               }
+               depModules.push(self.modules[depPath]);
+             }
+         });
+
+         // Execute the function.
+         if (callback) {
+           exported = callback.apply(null, depModules);
+         }
+
+         if (exported !== undefined) {
+           if (usesExports) {
+             throw new Error('Inside "' + namePath + '", cannot use exports ' +
+                             'and also return a value from a define ' +
+                             'definition function');
+           } else {
+             return exported;
+           }
+         }
+         return undefined;
+       };
+
+       return {
+         require: asyncRequire,
+         define: define
+       };
+       // END support for Async module-style
      },
 
      // This is only really used by unit tests and other
@@ -264,8 +492,8 @@
        return this.sandboxes[path];
      },
 
-     require: function require(module) {
-       return (this._makeRequire(null))(module);
+     require: function require(module, callback) {
+       return (this._makeApi(null).require)(module, callback);
      },
 
      runScript: function runScript(options, extraOutput) {
@@ -277,7 +505,9 @@
          extraOutput.sandbox = sandbox;
        for (name in this.globals)
          sandbox.defineProperty(name, this.globals[name]);
-       sandbox.defineProperty('require', this._makeRequire(null));
+       var api = this._makeApi(null);
+       sandbox.defineProperty('require', api.require);
+       sandbox.defineProperty('define', api.define);
        return sandbox.evaluate(options);
      }
    };

packages/api-utils/tests/modules/add.js

@@ -0,0 +1,5 @@
+define('modules/add', function () {
+  return function (a, b) {
+    return a + b;
+  };
+});

packages/api-utils/tests/modules/async1.js

@@ -0,0 +1,10 @@
+define(['./traditional2', './async2'], function () {
+  var traditional2 = require('./traditional2');
+  return {
+    name: 'async1',
+    traditional1Name: traditional2.traditional1Name,
+    traditional2Name: traditional2.name,
+    async2Name: require('./async2').name,
+    async2Traditional2Name: require('./async2').traditional2Name
+  };
+});

packages/api-utils/tests/modules/async2.js

@@ -0,0 +1,4 @@
+define(['./traditional2', 'exports'], function (traditional2, exports) {
+  exports.name = 'async2';
+  exports.traditional2Name = traditional2.name;
+});

packages/api-utils/tests/modules/badExportAndReturn.js

@@ -0,0 +1,6 @@
+// This is a bad module, it asks for exports but also returns a value from
+// the define defintion function.
+define(['exports'], function (exports) {    
+    return 'badExportAndReturn';
+});
+

packages/api-utils/tests/modules/badFirst.js

@@ -0,0 +1,5 @@
+define(['./badSecond'], function (badSecond) {
+  return {
+    name: 'badFirst'
+  };
+});

packages/api-utils/tests/modules/badSecond.js

@@ -0,0 +1,4 @@
+var first = require('./badFirst');
+
+exports.name = 'badSecond';
+exports.badFirstName = first.name;

packages/api-utils/tests/modules/blue.js

@@ -0,0 +1,5 @@
+define(function () {
+  return {
+      name: 'blue'
+  };
+});

packages/api-utils/tests/modules/castor.js

@@ -0,0 +1,6 @@
+define(['exports', './pollux'], function(exports, pollux) {
+  exports.name = 'castor';
+  exports.getPolluxName = function () {
+    return pollux.name;
+  };
+});

packages/api-utils/tests/modules/cheetah.js

@@ -0,0 +1,5 @@
+define(function () {
+  return function () {
+    return 'cheetah';
+  };
+});

packages/api-utils/tests/modules/color.js

@@ -0,0 +1,3 @@
+define({
+  type: 'color'
+});

packages/api-utils/tests/modules/dupe.js

@@ -0,0 +1,11 @@
+define({
+  name: 'dupe'
+});
+
+// This is wrong and should not be allowed. Only one call to
+// define per file.
+define([], function () {
+  return {
+    name: 'dupe2'
+  };
+});

packages/api-utils/tests/modules/dupeNested.js

@@ -0,0 +1,11 @@
+
+define(function () {
+  // This is wrong and should not be allowed.
+  define('dupeNested2', {
+      name: 'dupeNested2'
+  });
+
+  return {
+    name: 'dupeNested'
+  };
+});