Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Jetpack tests

  • Loading branch information...
commit c30caba2deeaf1e295db2d58f47ca1d61d8aec3d 1 parent bf5ab5e
@mattbasta mattbasta authored
View
3  .gitmodules
@@ -0,0 +1,3 @@
+[submodule "jetpack/addon-sdk"]
+ path = jetpack/addon-sdk
+ url = git://github.com/mozilla/addon-sdk.git
View
3  MANIFEST.in
@@ -1 +1,4 @@
include validator/testcases/hashes.txt
+include validator/app_versions.json
+include validator/testcases/jetpack_data.txt
+include validator/testcases/whitelist_hashes.txt
View
59 README.rst
@@ -32,6 +32,17 @@ You can install everything you need for running and testing with ::
pip install -r requirements.txt
+
+Submodules
+==========
+
+The validator may require some submodules to work. Make sure to run ::
+
+ git clone --recursive git://github.com/mozilla/amo-validator.git
+
+so that you get all of the goodies inside.
+
+
Spidermonkey
============
@@ -375,3 +386,51 @@ This file should overwrite the standard nose coverage plugin at the appropriate
~/.virtualenvs/[virtual environment]/lib/pythonX.X/site-packages/nose/plugins/cover.py
/usr/lib/pythonX.X/site-packages/nose/plugins/cover.py
+
+----------
+ Updating
+----------
+
+Some regular maintenance needs to be performed on the validator in order to
+make sure that the results are accurate.
+
+App Versions
+============
+
+A list of Mozilla `<em:targetApplication>` values is stored in the
+`validator/app_versions.json` file. This must be updated to include the latest
+application versions. This information can be found on AMO:
+
+https://addons.mozilla.org/en-US/firefox/pages/appversions/
+
+
+JS Libraries
+============
+
+A list of JS library hashes is kept to allow for whitelisting. This must be
+regenerated with each new library version. To update: ::
+
+ cd extras
+ mkdir jslibs
+ python jslibfetcher.py
+ python build_whitelist.py
+ # We keep a special hash for testing
+ echo "e96461c6c19608f528b4a3c33a032b697b999b62" >> whitelist_hashes.txt
+ mv whitelist_hashes.txt ../validator/testcases/hashes.txt
+
+To add new libraries to the mix, edit `extras/jslibfetcher.py` and add the
+version number to the appropriate tuple.
+
+
+Jetpack
+=======
+
+In order to maintain Jetpack compatibility, the whitelist hashes need to be
+regenerated with each successive Jetpack version. To rebuild the hash library,
+simply run: ::
+
+ cd jetpack
+ ./generate_jp_whitelist.sh
+
+That's it!
+
1  jetpack/addon-sdk
@@ -0,0 +1 @@
+Subproject commit 5a73e756c5c3dd8ac9609b915534c62402a31c44
View
14 jetpack/generate_jp_whitelist.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+rm ../validator/testcases/jetpack_data.txt
+
+cd addon-sdk
+git pull origin --tags
+for tagname in `git tag`;
+do
+ git checkout $tagname
+ for f in `find . -type f -name "*.js"`;
+ do
+ python ../make_hash.py $f $tagname >> ../../validator/testcases/jetpack_data.txt
+ done
+done
View
6 jetpack/make_hash.py
@@ -0,0 +1,6 @@
+import hashlib
+import os
+import sys
+
+hash = hashlib.sha256(open(sys.argv[1]).read()).hexdigest()
+print sys.argv[1], sys.argv[2], hash
View
4 tests/test_chromemanifest_testcases.py
@@ -22,6 +22,10 @@ def test_no_chromemanifest():
assert tc_chromemanifest.test_categories(err) is None
assert not err.failed()
+ err = ErrorBundle()
+ assert tc_chromemanifest.test_resourcemodules(err) is None
+ assert not err.failed()
+
def test_js_categories_gecko2():
"""Test that JS categories raise problems for hyphenated values."""
View
272 tests/test_jetpack.py
@@ -0,0 +1,272 @@
+import hashlib
+import json
+import nose
+import validator.testcases.jetpack as jetpack
+from validator.errorbundler import ErrorBundle
+
+def _do_test(xpi_package):
+
+ err = ErrorBundle()
+ jetpack.inspect_jetpack(err, xpi_package)
+ return err
+
+class MockXPI:
+
+ def __init__(self, resources):
+ self.resources = resources
+
+ def read(self, name):
+ return self.resources[name]
+
+ def __iter__(self):
+ for name in self.resources.keys():
+ yield name
+
+ def __contains__(self, name):
+ return name in self.resources
+
+
+def test_not_jetpack():
+ """Test that add-ons which do not match the Jetpack pattern are ignored."""
+
+ err = _do_test(MockXPI({"foo": True, "bar": True}))
+ assert not err.errors
+ assert not err.warnings
+ assert not err.notices
+ assert not err.metadata
+
+
+def test_bad_harnessoptions():
+ """Test that a malformed harness-options.json file is warned against."""
+
+ err = _do_test(MockXPI({"bootstrap.js": True,
+ "components/harness.js": True,
+ "harness-options.json": "foo bar"}))
+ assert err.failed()
+ assert err.warnings
+ print err.warnings
+ assert err.warnings[0]["id"][-1] == "bad_harness-options.json"
+
+
+def test_pass_jetpack():
+ """Test that a minimalistic Jetpack setup will pass."""
+
+ harnessoptions = {"sdkVersion": "foo",
+ "manifest": {}}
+
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/bootstrap.js") as bootstrap_file:
+ bootstrap = bootstrap_file.read()
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/components/harness.js") as harness_file:
+ harness = harness_file.read()
+
+ err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+ "components/harness.js": harness,
+ "harness-options.json":
+ json.dumps(harnessoptions)}))
+ assert not err.failed()
+ assert "is_jetpack" in err.metadata and err.metadata["is_jetpack"]
+
+ # Test that all files are marked as pretested.
+ pretested_files = err.get_resource("pretested_files")
+ assert pretested_files
+ assert "bootstrap.js" in pretested_files
+ assert "components/harness.js" in pretested_files
+
+
+def test_missing_elements():
+ """Test that missing elements in harness-options will fail."""
+
+ harnessoptions = {"sdkVersion": "foo"}
+
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/bootstrap.js") as bootstrap_file:
+ bootstrap = bootstrap_file.read()
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/components/harness.js") as harness_file:
+ harness = harness_file.read()
+
+ err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+ "components/harness.js": harness,
+ "harness-options.json":
+ json.dumps(harnessoptions)}))
+ assert err.failed()
+
+
+def test_skip_safe_files():
+ """Test that missing elements in harness-options will fail."""
+
+ harnessoptions = {"sdkVersion": "foo",
+ "manifest": {}}
+
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/bootstrap.js") as bootstrap_file:
+ bootstrap = bootstrap_file.read()
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/components/harness.js") as harness_file:
+ harness = harness_file.read()
+
+ err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+ "components/harness.js": harness,
+ "harness-options.json":
+ json.dumps(harnessoptions),
+ "foo.png": True,
+ "bar.JpG": True,
+ "safe.GIF": True,
+ "icon.ico": True,
+ "foo/.DS_Store": True}))
+ assert not err.failed()
+
+
+def test_pass_manifest_elements():
+ """Test that proper elements in harness-options will pass."""
+
+ harnessoptions = {"sdkVersion": "foo",
+ "manifest":
+ {"resource://bootstrap.js":
+ {"name": "bootstrap_test",
+ "hash":
+ "fc074074112a10a0267beb7c8ac0a6fd4d7d308eb7e3fb50c1db3d3becf5c99b",
+ "zipname": "bootstrap.js"}}}
+
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/bootstrap.js") as bootstrap_file:
+ bootstrap = bootstrap_file.read()
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/components/harness.js") as harness_file:
+ harness = harness_file.read()
+
+ err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+ "components/harness.js": harness,
+ "harness-options.json":
+ json.dumps(harnessoptions)}))
+ print err.print_summary(verbose=True)
+ assert not err.failed()
+ assert "jetpack_loaded_modules" in err.metadata
+ nose.tools.eq_(err.metadata["jetpack_loaded_modules"],
+ ["bootstrap_test"])
+ assert "jetpack_identified_files" in err.metadata
+ assert "bootstrap.js" in err.metadata["jetpack_identified_files"]
+
+ assert "jetpack_unknown_files" in err.metadata
+ assert not err.metadata["jetpack_unknown_files"]
+
+
+def test_bad_resource():
+ """Test for failure on non-resource:// modules."""
+
+ harnessoptions = {"sdkVersion": "foo",
+ "manifest":
+ {"bootstrap.js":
+ {"name": "bootstrap_test",
+ "hash":
+ "fc074074112a10a0267beb7c8ac0a6fd4d7d308eb7e3fb50c1db3d3becf5c99b",
+ "zipname": "bootstrap.js"}}}
+
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/bootstrap.js") as bootstrap_file:
+ bootstrap = bootstrap_file.read()
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/components/harness.js") as harness_file:
+ harness = harness_file.read()
+
+ err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+ "components/harness.js": harness,
+ "harness-options.json":
+ json.dumps(harnessoptions)}))
+ print err.print_summary(verbose=True)
+ assert err.failed()
+
+
+def test_missing_manifest_elements():
+ """Test that missing manifest elements in harness-options will fail."""
+
+ harnessoptions = {"sdkVersion": "foo",
+ "manifest":
+ {"resource://bootstrap.js":
+ {"name": "bootstrap_test",
+ "zipname": "bootstrap.js"}}}
+
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/bootstrap.js") as bootstrap_file:
+ bootstrap = bootstrap_file.read()
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/components/harness.js") as harness_file:
+ harness = harness_file.read()
+
+ err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+ "components/harness.js": harness,
+ "harness-options.json":
+ json.dumps(harnessoptions)}))
+ print err.print_summary(verbose=True)
+ assert err.failed()
+
+
+def test_mismatched_hash():
+ """
+ Test that failure occurs when the actual file hash doesn't match the hash
+ provided by harness-options.js.
+ """
+
+ harnessoptions = {"sdkVersion": "foo",
+ "manifest":
+ {"resource://bootstrap.js":
+ {"name": "bootstrap_test",
+ "hash": "foobar",
+ "zipname": "bootstrap.js"}}}
+
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/bootstrap.js") as bootstrap_file:
+ bootstrap = bootstrap_file.read()
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/components/harness.js") as harness_file:
+ harness = harness_file.read()
+
+ err = _do_test(MockXPI({"bootstrap.js": bootstrap,
+ "components/harness.js": harness,
+ "harness-options.json":
+ json.dumps(harnessoptions)}))
+ print err.print_summary(verbose=True)
+ assert err.failed()
+
+
+def test_mismatched_db_hash():
+ """
+ Test that failure occurs when the hash of a file doesn't exist in the
+ Jetpack known file database.
+ """
+
+ test_file = "This is the content of the test file."
+
+ harnessoptions = {"sdkVersion": "foo",
+ "manifest":
+ {"resource://test.js":
+ {"name": "test file",
+ "hash": hashlib.sha256(test_file).hexdigest(),
+ "zipname": "test.js"}}}
+
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/bootstrap.js") as bootstrap_file:
+ bootstrap = bootstrap_file.read()
+ with open("jetpack/addon-sdk/python-lib/cuddlefish/"
+ "app-extension/components/harness.js") as harness_file:
+ harness = harness_file.read()
+
+ err = _do_test(MockXPI({"test.js": test_file,
+ "bootstrap.js": bootstrap,
+ "components/harness.js": harness,
+ "harness-options.json":
+ json.dumps(harnessoptions)}))
+ print err.print_summary(verbose=True)
+ assert err.failed()
+
+ assert "jetpack_loaded_modules" in err.metadata
+ assert not err.metadata["jetpack_loaded_modules"]
+ assert "jetpack_identified_files" in err.metadata
+
+ assert "jetpack_unknown_files" in err.metadata
+ nose.tools.eq_(err.metadata["jetpack_unknown_files"],
+ ["test.js"])
+
+
View
13 validator/errorbundler.py
@@ -335,6 +335,19 @@ def print_summary(self, verbose=False, no_color=False):
message=notice,
verbose=verbose)
+ if "is_jetpack" in self.metadata and verbose:
+ self.handler.write("\n")
+ self.handler.write("<<GREEN>>Jetpack add-on detected.<<NORMAL>>\n"
+ "Identified files:")
+ for filename, data in self.metadata["jetpack_identified_files"].items():
+ self.handler.write((" %s\n" % filename) +
+ (" %s : %s" % data))
+
+ self.handler.write("Unknown files:")
+ for filename in self.metadata["jetpack_unknown_files"]:
+ self.handler.write(" %s" % filename)
+
+
self.handler.write("\n")
if self.unfinished:
self.handler.write("<<RED>>Validation terminated early")
View
1  validator/loader.py
@@ -3,6 +3,7 @@
import validator.testcases.conduit
import validator.testcases.content
import validator.testcases.installrdf
+import validator.testcases.jetpack
import validator.testcases.l10ncompleteness
import validator.testcases.langpack
import validator.testcases.packagelayout
View
3  validator/testcases/content.py
@@ -78,6 +78,9 @@ def test_packed_packages(err, xpi_package=None):
filename=name)
continue
+ if "pretested" in data and data["pretested"]:
+ continue
+
try:
file_data = xpi_package.read(name)
except KeyError: # pragma: no cover
View
191 validator/testcases/jetpack.py
@@ -0,0 +1,191 @@
+import hashlib
+import json
+import os
+import validator.decorator as decorator
+from validator.constants import PACKAGE_EXTENSION
+
+
+@decorator.register_test(tier=1, expected_type=PACKAGE_EXTENSION)
+def inspect_jetpack(err, xpi_package):
+ """
+ If the add-on is a Jetpack extension, its contents should be tested to
+ ensure that none of the Jetpack libraries have been tampered with.
+ """
+
+ jetpack_triggers = ("bootstrap.js",
+ "components/harness.js",
+ "harness-options.json")
+
+ # Make sure this is a Jetpack add-on.
+ if not all(trigger in xpi_package for trigger in jetpack_triggers):
+ return
+
+ try:
+ harnessoptions = json.loads(xpi_package.read("harness-options.json"))
+ except ValueError:
+ err.warning(
+ err_id=("testcases_jetpack",
+ "inspect_jetpack",
+ "bad_harness-options.json"),
+ warning="harness-options.json is not decodable JSON",
+ description="The harness-options.json file is not decodable as "
+ "valid JSON data.",
+ filename="harness-options.json")
+ return
+
+ err.metadata["is_jetpack"] = True
+
+ # Test the harness-options file for the mandatory values.
+ mandatory_elements = ("sdkVersion", "manifest")
+ missing_elements = []
+ for element in mandatory_elements:
+ if element not in harnessoptions:
+ missing_elements.append(element)
+
+ if missing_elements:
+ err.warning(
+ err_id=("testcases_jetpack",
+ "inspect_jetpack",
+ "harness-options_missing_elements"),
+ warning="Elements are missing from harness-options.json",
+ description=["The harness-options.json file seems to be missing "
+ "elements. It may have been tampered with or is "
+ "corrupt.",
+ "Missing elements: %s" % ", ".join(missing_elements)],
+ filename="harness-options.json")
+ return
+
+ # Save the SDK version information to the metadata.
+ err.metadata["jetpack_sdk_version"] = harnessoptions["sdkVersion"]
+ pretested_files = err.get_resource("pretested_files")
+ if not pretested_files:
+ err.save_resource("pretested_files", [])
+ pretested_files = []
+
+ # Read the jetpack data file in.
+ jetpack_data = open(os.path.join(os.path.dirname(__file__),
+ "jetpack_data.txt"))
+ # Parse the jetpack data into something useful.
+ jetpack_hash_table = {}
+ for line in map(lambda x: x.split(), jetpack_data):
+ jetpack_hash_table[line[-1]] = tuple(line[:-1])
+
+ loaded_modules = []
+ tested_files = {}
+ unknown_files = []
+ mandatory_module_elements = ("name", "hash", "zipname")
+
+ # Iterate each loaded module and perform a sha256 hash on the files.
+ for uri, module in harnessoptions["manifest"].items():
+
+ # Make sure the module is a resource:// URL
+ if not uri.startswith("resource://"):
+ err.warning(
+ err_id=("testcases_jetpack",
+ "inspect_jetpack",
+ "irregular_module_location"),
+ warning="Irregular Jetpack module location",
+ description="A Jetpack module is referenced with a non-"
+ "resource URI.",
+ filename="harness-options.json")
+ continue
+
+ # Make sure all of the mandatory elements are present.
+ if not all(el in module for el in mandatory_module_elements):
+ err.warning(
+ err_id=("testcases_jetpack",
+ "inspect_jetpack",
+ "irregular_module_elements"),
+ warning="Irregular Jetpack module elements",
+ description=["A Jetpack module in harness-options.json is "
+ "missing some of its required JSON elements.",
+ "Module: %s" % uri],
+ filename="harness-options.json")
+ continue
+
+ zip_path = module["zipname"]
+
+ # Make sure that the module actually exists in the add-on.
+ if zip_path not in xpi_package:
+ err.warning(
+ err_id=("testcases_jetpack",
+ "inspect_jetpack",
+ "missing_jetpack_module"),
+ warning="Missing Jetpack module",
+ description=["A Jetpack module listed in harness-options.json "
+ "could not be found in the add-on.",
+ "Path: %s" % module["zipname"]],
+ filename="harness-options.json")
+ continue
+
+ blob = xpi_package.read(zip_path)
+ blob_hash = hashlib.sha256(blob).hexdigest()
+
+ if blob_hash != module["hash"]:
+ # Warn that the hashes don't match up.
+ err.warning(
+ err_id=("testcases_jetpack",
+ "inspect_jetpack",
+ "mismatched_checksum"),
+ warning="Jetpack module hash mismatch",
+ description=["A file in the Jetpack add-on does not match the "
+ "corresponding hash listed in harness-options"
+ ".json.",
+ "Module: %s" % uri,
+ "Hashes: %s/%s" % (blob_hash, module["hash"])],
+ filename=zip_path)
+ continue
+
+ if blob_hash not in jetpack_hash_table:
+ # Warn that the hash isn't a valid jetpack hash.
+ err.warning(
+ err_id=("testcases_jetpack",
+ "inspect_jetpack",
+ "unknown_hash"),
+ warning="Jetpack module hash unknown",
+ description=["A file in the Jetpack add-on could not be "
+ "identified in the Jetpack SDK checksum "
+ "database. The file may have been tampered with.",
+ "Module: %s" % uri,
+ "Hash: %s" % blob_hash],
+ filename=zip_path)
+ continue
+
+ # Keep track of all of the valid modules that were loaded.
+ loaded_modules.append(module["name"])
+
+ # Save information on what was matched.
+ tested_files[zip_path] = jetpack_hash_table[blob_hash]
+ pretested_files.append(zip_path)
+
+ safe_files = (".jpg", ".ico", ".png", ".gif", ".ds_store")
+
+ # Iterate the rest of the files in the package for testing.
+ for filename in xpi_package:
+
+ # Skip files we've already identified.
+ if (filename in tested_files or
+ filename == "harness-options.json"):
+ continue
+
+ # Skip safe files.
+ if filename.lower().endswith(safe_files):
+ continue
+
+ blob = xpi_package.read(filename)
+ blob_hash = hashlib.sha256(blob).hexdigest()
+
+ if blob_hash not in jetpack_hash_table:
+ unknown_files.append(filename)
+ continue
+ else:
+ tested_files[filename] = jetpack_hash_table[blob_hash]
+ pretested_files.append(filename)
+
+ # Store the collected information in the output metadata.
+ err.metadata["jetpack_loaded_modules"] = loaded_modules
+ err.metadata["jetpack_identified_files"] = tested_files
+ err.metadata["jetpack_unknown_files"] = unknown_files
+
+ err.save_resource("pretested_files", pretested_files)
+
View
8,239 validator/testcases/jetpack_data.txt
8,239 additions, 0 deletions not shown
Please sign in to comment.
Something went wrong with that request. Please try again.