Bug 722230: make 'cfx test' respect --filter for filenames. r=mhammond

With this patch, the .allTestModules property of harness-options.json will
omit test modules that do not match the -f filename regexp. All test
files (everything in tests/) will still be included in the XPI, but the
runtime code will only execute the tests listed in .allTestModules .
Previously, the runtime code would load+execute all test modules, which
caused problems on mobile (where most tests still fail badly).
(cherry picked from commit 96dfad6)

python-lib/cuddlefish/__init__.py

@@ -639,8 +639,14 @@ def run(arguments=sys.argv[1:], target_cfg=None, pkg_cfg=None,
                                       "lib", "cuddlefish.js")
     loader_modules = [("api-utils", "lib", "cuddlefish", cuddlefish_js_path)]
     scan_tests = command == "test"
+    test_filter_re = None
+    if scan_tests and options.filter:
+        test_filter_re = options.filter
+        if ":" in options.filter:
+            test_filter_re = options.filter.split(":")[0]
     try:
-        manifest = build_manifest(target_cfg, pkg_cfg, deps, scan_tests,
+        manifest = build_manifest(target_cfg, pkg_cfg, deps,
+                                  scan_tests, test_filter_re,
                                   loader_modules)
     except ModuleNotFoundError, e:
         print str(e)

python-lib/cuddlefish/manifest.py

@@ -183,7 +183,7 @@ def __init__(self, target_cfg, pkg_cfg, deps, extra_modules,
         self.files = [] # maps manifest index to (absfn,absfn) js/docs pair
         self.test_modules = [] # for runtime
 
-    def build(self, scan_tests):
+    def build(self, scan_tests, test_filter_re):
         # process the top module, which recurses to process everything it
         # reaches
         if "main" in self.target_cfg:
@@ -208,7 +208,9 @@ def build(self, scan_tests):
                 for filename in os.listdir(d):
                     if filename.startswith("test-") and filename.endswith(".js"):
                         testname = filename[:-3] # require(testname)
-                        #re.search(r'^test-.*\.js$', filename):
+                        if test_filter_re:
+                            if not re.search(test_filter_re, testname):
+                                continue
                         tmi = ModuleInfo(self.target_cfg, "tests", testname,
                                          os.path.join(d, filename), None)
                         # scan the test's dependencies
@@ -575,7 +577,7 @@ def _find_module_in_package(self, pkgname, sections, name, looked_in):
         return None
 
 def build_manifest(target_cfg, pkg_cfg, deps, scan_tests,
-                   extra_modules=[]):
+                   test_filter_re=None, extra_modules=[]):
     """
     Perform recursive dependency analysis starting from entry_point,
     building up a manifest of modules that need to be included in the XPI.
@@ -602,7 +604,7 @@ def build_manifest(target_cfg, pkg_cfg, deps, scan_tests,
     """
 
     mxt = ManifestBuilder(target_cfg, pkg_cfg, deps, extra_modules)
-    mxt.build(scan_tests)
+    mxt.build(scan_tests, test_filter_re)
     return mxt
 
 

python-lib/cuddlefish/tests/linker-files/three/tests/nontest.js

@@ -0,0 +1 @@
+// dummy

python-lib/cuddlefish/tests/linker-files/three/tests/test-one.js

@@ -0,0 +1 @@
+// dummy

python-lib/cuddlefish/tests/linker-files/three/tests/test-two.js

@@ -0,0 +1 @@
+// dummy

python-lib/cuddlefish/tests/test_xpi.py

@@ -246,6 +246,89 @@ def absify(*parts):
             "uft8_value": "\u00e9"
           }'''))
 
+    def test_scantests(self):
+        target_cfg = self.get_pkg("three")
+        package_path = [self.get_linker_files_dir("three-deps")]
+        pkg_cfg = packaging.build_config(self.root, target_cfg,
+                                         packagepath=package_path)
+        deps = packaging.get_deps_for_targets(pkg_cfg,
+                                              [target_cfg.name, "addon-kit"])
+        m = manifest.build_manifest(target_cfg, pkg_cfg, deps, scan_tests=True)
+        self.failUnlessEqual(sorted(m.get_all_test_modules()),
+                             sorted(["test-one", "test-two"]))
+        # the current __init__.py code omits limit_to=used_files for 'cfx
+        # test', so all test files are included in the XPI. But the test
+        # runner will only execute the tests that m.get_all_test_modules()
+        # tells us about (which are put into the .allTestModules property of
+        # harness-options.json).
+        used_deps = m.get_used_packages()
+
+        build = packaging.generate_build_for_target(pkg_cfg, target_cfg.name,
+                                                    used_deps,
+                                                    include_tests=True)
+        options = {'main': target_cfg.main}
+        options.update(build)
+        basedir = self.make_basedir()
+        xpi_name = os.path.join(basedir, "contents.xpi")
+        xpi.build_xpi(template_root_dir=xpi_template_path,
+                      manifest=fake_manifest,
+                      xpi_path=xpi_name,
+                      harness_options=options,
+                      limit_to=None)
+        x = zipfile.ZipFile(xpi_name, "r")
+        names = x.namelist()
+        self.failUnless("resources/api-utils/lib/unit-test.js" in names, names)
+        self.failUnless("resources/api-utils/lib/unit-test-finder.js" in names, names)
+        self.failUnless("resources/test-harness/lib/harness.js" in names, names)
+        self.failUnless("resources/test-harness/lib/run-tests.js" in names, names)
+        # all files are copied into the XPI, even the things that don't look
+        # like tests.
+        self.failUnless("resources/three/tests/test-one.js" in names, names)
+        self.failUnless("resources/three/tests/test-two.js" in names, names)
+        self.failUnless("resources/three/tests/nontest.js" in names, names)
+
+    def test_scantests_filter(self):
+        target_cfg = self.get_pkg("three")
+        package_path = [self.get_linker_files_dir("three-deps")]
+        pkg_cfg = packaging.build_config(self.root, target_cfg,
+                                         packagepath=package_path)
+        deps = packaging.get_deps_for_targets(pkg_cfg,
+                                              [target_cfg.name, "addon-kit"])
+        FILTER = ".*one.*"
+        m = manifest.build_manifest(target_cfg, pkg_cfg, deps, scan_tests=True,
+                                    test_filter_re=FILTER)
+        self.failUnlessEqual(sorted(m.get_all_test_modules()),
+                             sorted(["test-one"]))
+        # the current __init__.py code omits limit_to=used_files for 'cfx
+        # test', so all test files are included in the XPI. But the test
+        # runner will only execute the tests that m.get_all_test_modules()
+        # tells us about (which are put into the .allTestModules property of
+        # harness-options.json).
+        used_deps = m.get_used_packages()
+
+        build = packaging.generate_build_for_target(pkg_cfg, target_cfg.name,
+                                                    used_deps,
+                                                    include_tests=True)
+        options = {'main': target_cfg.main}
+        options.update(build)
+        basedir = self.make_basedir()
+        xpi_name = os.path.join(basedir, "contents.xpi")
+        xpi.build_xpi(template_root_dir=xpi_template_path,
+                      manifest=fake_manifest,
+                      xpi_path=xpi_name,
+                      harness_options=options,
+                      limit_to=None)
+        x = zipfile.ZipFile(xpi_name, "r")
+        names = x.namelist()
+        self.failUnless("resources/api-utils/lib/unit-test.js" in names, names)
+        self.failUnless("resources/api-utils/lib/unit-test-finder.js" in names, names)
+        self.failUnless("resources/test-harness/lib/harness.js" in names, names)
+        self.failUnless("resources/test-harness/lib/run-tests.js" in names, names)
+        # get_all_test_modules() respects the filter. But all files are still
+        # copied into the XPI.
+        self.failUnless("resources/three/tests/test-one.js" in names, names)
+        self.failUnless("resources/three/tests/test-two.js" in names, names)
+        self.failUnless("resources/three/tests/nontest.js" in names, names)
 
 
 def document_dir(name):