Jetpack tests

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "jetpack/addon-sdk"]
+	path = jetpack/addon-sdk
+	url = git://github.com/mozilla/addon-sdk.git

MANIFEST.in

@@ -1 +1,4 @@
 include validator/testcases/hashes.txt
+include validator/app_versions.json
+include validator/testcases/jetpack_data.txt
+include validator/testcases/whitelist_hashes.txt

README.rst

@@ -32,6 +32,17 @@ You can install everything you need for running and testing with ::
 
 	pip install -r requirements.txt
 
+
+Submodules
+==========
+
+The validator may require some submodules to work. Make sure to run ::
+
+    git clone --recursive git://github.com/mozilla/amo-validator.git
+
+so that you get all of the goodies inside.
+
+
 Spidermonkey
 ============
 
@@ -375,3 +386,51 @@ This file should overwrite the standard nose coverage plugin at the appropriate
 	~/.virtualenvs/[virtual environment]/lib/pythonX.X/site-packages/nose/plugins/cover.py
 	/usr/lib/pythonX.X/site-packages/nose/plugins/cover.py
 
+
+----------
+ Updating
+----------
+
+Some regular maintenance needs to be performed on the validator in order to
+make sure that the results are accurate.
+
+App Versions
+============
+
+A list of Mozilla `<em:targetApplication>` values is stored in the
+`validator/app_versions.json` file. This must be updated to include the latest
+application versions. This information can be found on AMO:
+
+https://addons.mozilla.org/en-US/firefox/pages/appversions/
+
+
+JS Libraries
+============
+
+A list of JS library hashes is kept to allow for whitelisting. This must be
+regenerated with each new library version. To update: ::
+
+    cd extras
+    mkdir jslibs
+    python jslibfetcher.py
+    python build_whitelist.py
+    # We keep a special hash for testing
+    echo "e96461c6c19608f528b4a3c33a032b697b999b62" >> whitelist_hashes.txt
+    mv whitelist_hashes.txt ../validator/testcases/hashes.txt
+
+To add new libraries to the mix, edit `extras/jslibfetcher.py` and add the
+version number to the appropriate tuple.
+
+
+Jetpack
+=======
+
+In order to maintain Jetpack compatibility, the whitelist hashes need to be
+regenerated with each successive Jetpack version. To rebuild the hash library,
+simply run: ::
+
+    cd jetpack
+    ./generate_jp_whitelist.sh
+
+That's it!
+

jetpack/generate_jp_whitelist.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+rm ../validator/testcases/jetpack_data.txt
+
+cd addon-sdk
+git pull origin --tags
+for tagname in `git tag`;
+do
+    git checkout $tagname
+    for f in `find . -type f -name "*.js"`;
+    do
+        python ../make_hash.py $f $tagname >> ../../validator/testcases/jetpack_data.txt
+    done
+done

jetpack/make_hash.py

@@ -0,0 +1,6 @@
+import hashlib
+import os
+import sys
+
+hash = hashlib.sha256(open(sys.argv[1]).read()).hexdigest()
+print sys.argv[1], sys.argv[2], hash

tests/test_chromemanifest_testcases.py

@@ -22,6 +22,10 @@ def test_no_chromemanifest():
     assert tc_chromemanifest.test_categories(err) is None
     assert not err.failed()
 
+    err = ErrorBundle()
+    assert tc_chromemanifest.test_resourcemodules(err) is None
+    assert not err.failed()
+
 
 def test_js_categories_gecko2():
     """Test that JS categories raise problems for hyphenated values."""

tests/test_jetpack.py

@@ -0,0 +1,272 @@
+import hashlib
+import json
+import nose
+import validator.testcases.jetpack as jetpack
+from validator.errorbundler import ErrorBundle
+
+def _do_test(xpi_package):
+
+    err = ErrorBundle()
+    jetpack.inspect_jetpack(err, xpi_package)
+    return err
+
+class MockXPI:
+
+    def __init__(self, resources):
+        self.resources = resources
+
+    def read(self, name):
+        return self.resources[name]
+
+    def __iter__(self):
+        for name in self.resources.keys():
+            yield name
+
+    def __contains__(self, name):
+        return name in self.resources
+
+
+def test_not_jetpack():
+    """Test that add-ons which do not match the Jetpack pattern are ignored."""
+
+    err = _do_test(MockXPI({"foo": True, "bar": True}))
+    assert not err.errors
+    assert not err.warnings
+    assert not err.notices
+    assert not err.metadata
+
+
+def test_bad_harnessoptions():
+    """Test that a malformed harness-options.json file is warned against."""
+
+    err = _do_test(MockXPI({"bootstrap.js": True,
+                            "components/harness.js": True,
+                            "harness-options.json": "foo bar"}))
+    assert err.failed()
+    assert err.warnings
+    print err.warnings
+    assert err.warnings[0]["id"][-1] == "bad_harness-options.json"
+
+
+def test_pass_jetpack():
+    """Test that a minimalistic Jetpack setup will pass."""
+
+    harnessoptions = {"sdkVersion": "foo",
+                      "manifest": {}}
+
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/bootstrap.js") as bootstrap_file:
+        bootstrap = bootstrap_file.read()
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/components/harness.js") as harness_file:
+        harness = harness_file.read()
+
+    err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+                            "components/harness.js": harness,
+                            "harness-options.json":
+                                json.dumps(harnessoptions)}))
+    assert not err.failed()
+    assert "is_jetpack" in err.metadata and err.metadata["is_jetpack"]
+
+    # Test that all files are marked as pretested.
+    pretested_files = err.get_resource("pretested_files")
+    assert pretested_files
+    assert "bootstrap.js" in pretested_files
+    assert "components/harness.js" in pretested_files
+
+
+def test_missing_elements():
+    """Test that missing elements in harness-options will fail."""
+
+    harnessoptions = {"sdkVersion": "foo"}
+
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/bootstrap.js") as bootstrap_file:
+        bootstrap = bootstrap_file.read()
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/components/harness.js") as harness_file:
+        harness = harness_file.read()
+
+    err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+                            "components/harness.js": harness,
+                            "harness-options.json":
+                                json.dumps(harnessoptions)}))
+    assert err.failed()
+
+
+def test_skip_safe_files():
+    """Test that missing elements in harness-options will fail."""
+
+    harnessoptions = {"sdkVersion": "foo",
+                      "manifest": {}}
+
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/bootstrap.js") as bootstrap_file:
+        bootstrap = bootstrap_file.read()
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/components/harness.js") as harness_file:
+        harness = harness_file.read()
+
+    err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+                            "components/harness.js": harness,
+                            "harness-options.json":
+                                json.dumps(harnessoptions),
+                            "foo.png": True,
+                            "bar.JpG": True,
+                            "safe.GIF": True,
+                            "icon.ico": True,
+                            "foo/.DS_Store": True}))
+    assert not err.failed()
+
+
+def test_pass_manifest_elements():
+    """Test that proper elements in harness-options will pass."""
+
+    harnessoptions = {"sdkVersion": "foo",
+                      "manifest":
+                          {"resource://bootstrap.js":
+                               {"name": "bootstrap_test",
+                                "hash":
+            "fc074074112a10a0267beb7c8ac0a6fd4d7d308eb7e3fb50c1db3d3becf5c99b",
+                                "zipname": "bootstrap.js"}}}
+
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/bootstrap.js") as bootstrap_file:
+        bootstrap = bootstrap_file.read()
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/components/harness.js") as harness_file:
+        harness = harness_file.read()
+
+    err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+                            "components/harness.js": harness,
+                            "harness-options.json":
+                                json.dumps(harnessoptions)}))
+    print err.print_summary(verbose=True)
+    assert not err.failed()
+    assert "jetpack_loaded_modules" in err.metadata
+    nose.tools.eq_(err.metadata["jetpack_loaded_modules"],
+                   ["bootstrap_test"])
+    assert "jetpack_identified_files" in err.metadata
+    assert "bootstrap.js" in err.metadata["jetpack_identified_files"]
+
+    assert "jetpack_unknown_files" in err.metadata
+    assert not err.metadata["jetpack_unknown_files"]
+
+
+def test_bad_resource():
+    """Test for failure on non-resource:// modules."""
+
+    harnessoptions = {"sdkVersion": "foo",
+                      "manifest":
+                          {"bootstrap.js":
+                               {"name": "bootstrap_test",
+                                "hash":
+            "fc074074112a10a0267beb7c8ac0a6fd4d7d308eb7e3fb50c1db3d3becf5c99b",
+                                "zipname": "bootstrap.js"}}}
+
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/bootstrap.js") as bootstrap_file:
+        bootstrap = bootstrap_file.read()
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/components/harness.js") as harness_file:
+        harness = harness_file.read()
+
+    err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+                            "components/harness.js": harness,
+                            "harness-options.json":
+                                json.dumps(harnessoptions)}))
+    print err.print_summary(verbose=True)
+    assert err.failed()
+
+
+def test_missing_manifest_elements():
+    """Test that missing manifest elements in harness-options will fail."""
+
+    harnessoptions = {"sdkVersion": "foo",
+                      "manifest":
+                          {"resource://bootstrap.js":
+                               {"name": "bootstrap_test",
+                                "zipname": "bootstrap.js"}}}
+
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/bootstrap.js") as bootstrap_file:
+        bootstrap = bootstrap_file.read()
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/components/harness.js") as harness_file:
+        harness = harness_file.read()
+
+    err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+                            "components/harness.js": harness,
+                            "harness-options.json":
+                                json.dumps(harnessoptions)}))
+    print err.print_summary(verbose=True)
+    assert err.failed()
+
+
+def test_mismatched_hash():
+    """
+    Test that failure occurs when the actual file hash doesn't match the hash
+    provided by harness-options.js.
+    """
+
+    harnessoptions = {"sdkVersion": "foo",
+                      "manifest":
+                          {"resource://bootstrap.js":
+                               {"name": "bootstrap_test",
+                                "hash": "foobar",
+                                "zipname": "bootstrap.js"}}}
+
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/bootstrap.js") as bootstrap_file:
+        bootstrap = bootstrap_file.read()
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/components/harness.js") as harness_file:
+        harness = harness_file.read()
+
+    err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+                            "components/harness.js": harness,
+                            "harness-options.json":
+                                json.dumps(harnessoptions)}))
+    print err.print_summary(verbose=True)
+    assert err.failed()
+
+
+def test_mismatched_db_hash():
+    """
+    Test that failure occurs when the hash of a file doesn't exist in the
+    Jetpack known file database.
+    """
+
+    test_file = "This is the content of the test file."
+
+    harnessoptions = {"sdkVersion": "foo",
+                      "manifest":
+                          {"resource://test.js":
+                               {"name": "test file",
+                                "hash": hashlib.sha256(test_file).hexdigest(),
+                                "zipname": "test.js"}}}
+
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/bootstrap.js") as bootstrap_file:
+        bootstrap = bootstrap_file.read()
+    with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+                  "app-extension/components/harness.js") as harness_file:
+        harness = harness_file.read()
+
+    err = _do_test(MockXPI({"test.js": test_file,
+                            "bootstrap.js": bootstrap,
+                            "components/harness.js": harness,
+                            "harness-options.json":
+                                json.dumps(harnessoptions)}))
+    print err.print_summary(verbose=True)
+    assert err.failed()
+
+    assert "jetpack_loaded_modules" in err.metadata
+    assert not err.metadata["jetpack_loaded_modules"]
+    assert "jetpack_identified_files" in err.metadata
+
+    assert "jetpack_unknown_files" in err.metadata
+    nose.tools.eq_(err.metadata["jetpack_unknown_files"],
+                   ["test.js"])
+
+