Merge pull request mozilla#787 from ochameau/use-mc-modules

Bug 793925: final part - CFX should recognize modules shipped with platform r=@Gozala

app-extension/bootstrap.js

@@ -19,6 +19,13 @@ const resourceHandler = ioService.getProtocolHandler('resource').
 const systemPrincipal = CC('@mozilla.org/systemprincipal;1', 'nsIPrincipal')();
 const scriptLoader = Cc['@mozilla.org/moz/jssubscript-loader;1'].
                      getService(Ci.mozIJSSubScriptLoader);
+const prefService = Cc['@mozilla.org/preferences-service;1'].
+                    getService(Ci.nsIPrefService);
+const appInfo = Cc["@mozilla.org/xre/app-info;1"].
+                getService(Ci.nsIXULAppInfo);
+const vc = Cc["@mozilla.org/xpcom/version-comparator;1"].
+           getService(Ci.nsIVersionComparator);
+
 
 const REASON = [ 'unknown', 'startup', 'shutdown', 'enable', 'disable',
                  'install', 'uninstall', 'upgrade', 'downgrade' ];
@@ -120,19 +127,54 @@ function startup(data, reasonCode) {
     if (name == 'addon-sdk')
       paths['tests/'] = prefixURI + name + '/tests/';
 
-    // Maps sdk module folders to their resource folder
-    paths['sdk/'] = prefixURI + 'addon-sdk/lib/sdk/';
-    paths['toolkit/'] = prefixURI + 'addon-sdk/lib/toolkit/';
-    // test.js is usually found in root commonjs or SDK_ROOT/lib/ folder,
-    // so that it isn't shipped in the xpi. Keep a copy of it in sdk/ folder
-    // until we no longer support SDK modules in XPI:
-    paths['test'] = prefixURI + 'addon-sdk/lib/sdk/test.js';
+    // Starting with Firefox 21.0a1, we start using modules shipped into firefox
+    // Still allow using modules from the xpi if the manifest tell us to do so.
+    // And only try to look for sdk modules in xpi if the xpi actually ship them
+    if (options['is-sdk-bundled'] &&
+        (vc.compare(appInfo.version, '21.0a1') < 0 ||
+         options['force-use-bundled-sdk'])) {
+      // Maps sdk module folders to their resource folder
+      paths[''] = prefixURI + 'addon-sdk/lib/';
+      // test.js is usually found in root commonjs or SDK_ROOT/lib/ folder,
+      // so that it isn't shipped in the xpi. Keep a copy of it in sdk/ folder
+      // until we no longer support SDK modules in XPI:
+      paths['test'] = prefixURI + 'addon-sdk/lib/sdk/test.js';
+    }
+
+    // Retrieve list of module folder overloads based on preferences in order to
+    // eventually used a local modules instead of files shipped into Firefox.
+    let branch = prefService.getBranch('extensions.modules.' + id + '.path');
+    paths = branch.getChildList('', {}).reduce(function (result, name) {
+      // Allows overloading of any sub folder by replacing . by / in pref name
+      let path = name.substr(1).split('.').join('/');
+      // Only accept overloading folder by ensuring always ending with `/`
+      if (path) path += '/';
+      let fileURI = branch.getCharPref(name);
+
+      // Maps the given file:// URI to a resource:// in order to avoid various
+      // failure that happens with file:// URI and be close to production env
+      let resourcesURI = ioService.newURI(fileURI, null, null);
+      let resName = 'extensions.modules.' + domain + '.commonjs.path' + name;
+      resourceHandler.setSubstitution(resName, resourcesURI);
+
+      result[path] = 'resource://' + resName + '/';
+      return result;
+    }, paths);
 
     // Make version 2 of the manifest
     let manifest = options.manifest;
 
     // Import `cuddlefish.js` module using a Sandbox and bootstrap loader.
-    let cuddlefishURI = prefixURI + options.loader;
+    let cuddlefishPath = 'loader/cuddlefish.js';
+    let cuddlefishURI = 'resource://gre/modules/commonjs/sdk/' + cuddlefishPath;
+    if (paths['sdk/']) { // sdk folder has been overloaded
+                         // (from pref, or cuddlefish is still in the xpi)
+      cuddlefishURI = paths['sdk/'] + cuddlefishPath;
+    }
+    else if (paths['']) { // root modules folder has been overloaded
+      cuddlefishURI = paths[''] + 'sdk/' + cuddlefishPath;
+    }
+
     cuddlefishSandbox = loadSandbox(cuddlefishURI);
     let cuddlefish = cuddlefishSandbox.exports;
 

python-lib/cuddlefish/__init__.py

@@ -161,6 +161,28 @@
                                 default="firefox",
                                 cmds=['test', 'run', 'testex', 'testpkgs',
                                       'testall'])),
+        (("-o", "--overload-modules",), dict(dest="overload_modules",
+                                     help=("Overload JS modules integrated into"
+                                           " Firefox with the one from your SDK"
+                                           " repository"),
+                                     action="store_true",
+                                     default=False,
+                                     cmds=['run', 'test', 'testex', 'testpkgs',
+                                           'testall'])),
+        (("", "--strip-sdk",), dict(dest="bundle_sdk",
+                                    help=("Do not ship SDK modules in the xpi"),
+                                    action="store_false",
+                                    default=True,
+                                    cmds=['run', 'test', 'testex', 'testpkgs',
+                                          'testall', 'xpi'])),
+        (("", "--force-use-bundled-sdk",), dict(dest="force_use_bundled_sdk",
+                                    help=("When --strip-sdk isn't passed, "
+                                          "force using sdk modules shipped in "
+                                          "the xpi instead of firefox ones"),
+                                    action="store_true",
+                                    default=False,
+                                    cmds=['run', 'test', 'testex', 'testpkgs',
+                                          'testall', 'xpi'])),
         (("", "--no-run",), dict(dest="no_run",
                                      help=("Instead of launching the "
                                            "application, just show the command "
@@ -764,6 +786,14 @@ def run(arguments=sys.argv[1:], target_cfg=None, pkg_cfg=None,
 
     harness_options.update(build)
 
+    # When cfx is run from sdk root directory, we will strip sdk modules and
+    # override them with local modules.
+    # So that integration tools will continue to work and use local modules
+    if os.getcwd() == env_root:
+        options.bundle_sdk = True
+        options.force_use_bundled_sdk = False
+        options.overload_modules = True
+
     extra_environment = {}
     if command == "test":
         # This should be contained in the test runner package.
@@ -792,14 +822,32 @@ def run(arguments=sys.argv[1:], target_cfg=None, pkg_cfg=None,
         mydir = os.path.dirname(os.path.abspath(__file__))
         app_extension_dir = os.path.join(mydir, "../../app-extension")
 
-
     if target_cfg.get('preferences'):
         harness_options['preferences'] = target_cfg.get('preferences')
 
-    harness_options['manifest'] = manifest.get_harness_options_manifest()
-    harness_options['allTestModules'] = manifest.get_all_test_modules()
-    if len(harness_options['allTestModules']) == 0 and command == "test":
-        sys.exit(0)
+    harness_options['manifest'] = \
+        manifest.get_harness_options_manifest(options.bundle_sdk)
+
+    # Gives an hint to tell if sdk modules are bundled or not
+    harness_options['is-sdk-bundled'] = options.bundle_sdk
+
+    if options.force_use_bundled_sdk:
+        if not options.bundle_sdk:
+            print >>sys.stderr, ("--force-use-bundled-sdk and --strip-sdk "
+                                 "can't be used at the same time.")
+            sys.exit(1)
+        if options.overload_modules:
+            print >>sys.stderr, ("--force-use-bundled-sdk and --overload-modules "
+                                 "can't be used at the same time.")
+            sys.exit(1)
+        # Pass a flag in order to force using sdk modules shipped in the xpi
+        harness_options['force-use-bundled-sdk'] = True
+
+    # Pass the list of absolute path for all test modules
+    if command == "test":
+        harness_options['allTestModules'] = manifest.get_all_test_modules()
+        if len(harness_options['allTestModules']) == 0:
+            sys.exit(0)
 
     from cuddlefish.rdf import gen_manifest, RDFUpdate
 
@@ -825,7 +873,7 @@ def run(arguments=sys.argv[1:], target_cfg=None, pkg_cfg=None,
     # build_manifest earlier
     used_files = None
     if command == "xpi":
-      used_files = set(manifest.get_used_files())
+        used_files = set(manifest.get_used_files(options.bundle_sdk))
 
     if options.no_strip_xpi:
         used_files = None # disables the filter, includes all files
@@ -872,7 +920,11 @@ def run(arguments=sys.argv[1:], target_cfg=None, pkg_cfg=None,
                              norun=options.no_run,
                              used_files=used_files,
                              enable_mobile=options.enable_mobile,
-                             mobile_app_name=options.mobile_app_name)
+                             mobile_app_name=options.mobile_app_name,
+                             env_root=env_root,
+                             is_running_tests=(command == "test"),
+                             overload_modules=options.overload_modules,
+                             bundle_sdk=options.bundle_sdk)
         except ValueError, e:
             print ""
             print "A given cfx option has an inappropriate value:"

python-lib/cuddlefish/manifest.py

@@ -260,7 +260,7 @@ def get_used_packages(self):
             used.add(package)
         return sorted(used)
 
-    def get_used_files(self):
+    def get_used_files(self, bundle_sdk_modules):
         # returns all .js files that we reference, plus data/ files. You will
         # need to add the loader, off-manifest files that it needs, and
         # generated metadata.
@@ -269,16 +269,22 @@ def get_used_files(self):
                 yield absname
 
         for me in self.get_module_entries():
-            yield me.js_filename
+            # Do not add manifest entries for system modules,
+            # so that we won't ship SDK files.
+            if me.packageName != "addon-sdk" or bundle_sdk_modules:
+                yield me.js_filename
 
     def get_all_test_modules(self):
         return self.test_modules
 
-    def get_harness_options_manifest(self):
+    def get_harness_options_manifest(self, bundle_sdk_modules):
         manifest = {}
         for me in self.get_module_entries():
             path = me.get_path()
-            manifest[path] = me.get_entry_for_manifest()
+            # Do not add manifest entries for system modules,
+            # so that we won't ship SDK files.
+            if me.packageName != "addon-sdk" or bundle_sdk_modules:
+                manifest[path] = me.get_entry_for_manifest()
         return manifest
 
     def get_manifest_entry(self, package, section, module):

python-lib/cuddlefish/runner.py

@@ -380,7 +380,11 @@ def run_app(harness_root_dir, manifest_rdf, harness_options,
             logfile=None, addons=None, args=None, extra_environment={},
             norun=None,
             used_files=None, enable_mobile=False,
-            mobile_app_name=None):
+            mobile_app_name=None,
+            env_root=None,
+            is_running_tests=False,
+            overload_modules=False,
+            bundle_sdk=True):
     if binary:
         binary = os.path.expanduser(binary)
 
@@ -392,6 +396,22 @@ def run_app(harness_root_dir, manifest_rdf, harness_options,
     cmdargs = []
     preferences = dict(DEFAULT_COMMON_PREFS)
 
+    # Overload global commonjs path with lib/ folders
+    file_scheme = "file://"
+    # win32 file scheme needs 3 slashes
+    if not env_root.startswith("/"):
+      file_scheme = file_scheme + "/"
+    addon_id = harness_options["jetpackID"]
+    pref_prefix = "extensions.modules." + addon_id + ".path"
+    if overload_modules:
+        preferences[pref_prefix] = file_scheme + \
+            os.path.join(env_root, "lib").replace("\\", "/") + "/"
+
+    # Overload tests/ mapping with test/ folder, only when running test
+    if is_running_tests:
+        preferences[pref_prefix + ".tests"] = file_scheme + \
+            os.path.join(env_root, "test").replace("\\", "/") + "/"
+
     # For now, only allow running on Mobile with --force-mobile argument
     if app_type in ["fennec", "fennec-on-device"] and not enable_mobile:
         print """
@@ -483,7 +503,8 @@ def maybe_remove_logfile():
               manifest=manifest_rdf,
               xpi_path=xpi_path,
               harness_options=harness_options,
-              limit_to=used_files)
+              limit_to=used_files,
+              bundle_sdk=bundle_sdk)
     addons.append(xpi_path)
 
     starttime = last_output_time = time.time()

python-lib/cuddlefish/tests/test_linker.py

@@ -43,7 +43,7 @@ def test_manifest(self):
         # target_cfg.dependencies is not provided, so we'll search through
         # all known packages (everything in 'deps').
         m = manifest.build_manifest(target_cfg, pkg_cfg, deps, scan_tests=False)
-        m = m.get_harness_options_manifest()
+        m = m.get_harness_options_manifest(False)
 
         def assertReqIs(modname, reqname, path):
             reqs = m["one/%s" % modname]["requirements"]
@@ -72,7 +72,7 @@ def test_main_in_deps(self):
                                               [target_cfg.name, "addon-sdk"])
         self.failUnlessEqual(deps, ["addon-sdk", "three"])
         m = manifest.build_manifest(target_cfg, pkg_cfg, deps, scan_tests=False)
-        m = m.get_harness_options_manifest()
+        m = m.get_harness_options_manifest(False)
         def assertReqIs(modname, reqname, path):
             reqs = m["three/%s" % modname]["requirements"]
             self.failUnlessEqual(reqs[reqname], path)
@@ -90,7 +90,7 @@ def test_relative_main_in_top(self):
         self.failUnlessEqual(deps, ["addon-sdk", "five"])
         # all we care about is that this next call doesn't raise an exception
         m = manifest.build_manifest(target_cfg, pkg_cfg, deps, scan_tests=False)
-        m = m.get_harness_options_manifest()
+        m = m.get_harness_options_manifest(False)
         reqs = m["five/main"]["requirements"]
         self.failUnlessEqual(reqs, {});
 

python-lib/cuddlefish/tests/test_xpi.py

@@ -202,7 +202,7 @@ def test_contents(self):
                                               [target_cfg.name, "addon-sdk"])
         addon_sdk_dir = pkg_cfg.packages["addon-sdk"].lib[0]
         m = manifest.build_manifest(target_cfg, pkg_cfg, deps, scan_tests=False)
-        used_files = list(m.get_used_files())
+        used_files = list(m.get_used_files(True))
         here = up(os.path.abspath(__file__))
         def absify(*parts):
             fn = os.path.join(here, "linker-files", *parts)

python-lib/cuddlefish/xpi.py

@@ -22,7 +22,8 @@ def mkzipdir(zf, path):
     zf.writestr(dirinfo, "")
 
 def build_xpi(template_root_dir, manifest, xpi_path,
-              harness_options, limit_to=None, extra_harness_options={}):
+              harness_options, limit_to=None, extra_harness_options={},
+              bundle_sdk=True):
     zf = zipfile.ZipFile(xpi_path, "w", zipfile.ZIP_DEFLATED)
 
     open('.install.rdf', 'w').write(str(manifest))
@@ -84,6 +85,10 @@ def build_xpi(template_root_dir, manifest, xpi_path,
     # of all packages sections directories
     for packageName in harness_options['packages']:
       base_arcpath = ZIPSEP.join(['resources', packageName])
+      # Eventually strip sdk files. We need to do that in addition to the
+      # whilelist as the whitelist is only used for `cfx xpi`:
+      if not bundle_sdk and packageName == 'addon-sdk':
+          continue
       # Always write the top directory, even if it contains no files, since
       # the harness will try to access it.
       dirs_to_create.add(base_arcpath)