Merge pull request mozilla#341 from magopian/integrate-functional-tests

Integrate the functional tests from the test-amo-validator repository
kumar303 · Nov 5, 2015 · bc0ac75 · bc0ac75
2 parents 1096b1a + 53bbef5
commit bc0ac75
Show file tree

Hide file tree

Showing 31 changed files with 395 additions and 4 deletions.
diff --git a/README.md b/README.md
@@ -321,7 +321,23 @@ The same rule applies for the end of a file and for files with only one line.
 Unit tests can be run with
 
 ```bash
-nosetests
+nosetests tests/
+```
+
+Functional tests, which take longer, can be run with
+
+```bash
+nosetests --processes=2 functional_tests/
+```
+
+Then make a cup of tea while all of those tests run. It takes a while. If you
+have more than two cores on your machine or you don't mind pwnage, you can try
+to increase the number of parallel processes used for testing.
+
+If you just want to run all the tests at once, run
+
+```bash
+nosetests --processes=2
 ```
 
 

diff --git a/functional_tests/addons/add-on-20110113408.xpi b/functional_tests/addons/add-on-20110113408.xpi
diff --git a/functional_tests/addons/add-on20101228444 (1).jar b/functional_tests/addons/add-on20101228444 (1).jar
diff --git a/functional_tests/addons/add-on201101081038.xpi b/functional_tests/addons/add-on201101081038.xpi
diff --git a/functional_tests/addons/add-on201101101027.xpi b/functional_tests/addons/add-on201101101027.xpi
diff --git a/functional_tests/addons/add-on20110110322.xpi b/functional_tests/addons/add-on20110110322.xpi
diff --git a/functional_tests/addons/add-on20110110356.xpi b/functional_tests/addons/add-on20110110356.xpi
diff --git a/functional_tests/addons/add-ongoogle-201101121132.xpi b/functional_tests/addons/add-ongoogle-201101121132.xpi
diff --git a/functional_tests/addons/addon-12201-latest.xml b/functional_tests/addons/addon-12201-latest.xml
@@ -0,0 +1,15 @@
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/"
+                       xmlns:moz="http://www.mozilla.org/2006/browser/search/"><ShortName>(G)mailto:</ShortName>
+<ShortName>(G)mailto:</ShortName>
+<Description>g-mailto</Description>
+<InputEncoding>UTF-8</InputEncoding>
+<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACZUlEQVQ4jY2Ry08aURTGL666alK3TRPX3XZrADWlsKsLt121SXXZVfunmJjU7qyp1pqgiTpDOjIODAOIdgAZoFZEMYz1Na8749yvCxCN9nWSk3z35JzfeVxCCAmQa7vSfQeb26/tbfU0tRx/19/ff/8q3s0J3Mq/DrwYG3volEuLpiRRU0zC2JB8e6e8trSw8PhGceB2cYAQ0nf6o/7KyuX2TFGEmU7D1jSYsgxDFGEq2eY3QXg5MDBw71YdIW8mJh7RSvW9mUq5pijCKhTgnZ2BMYZLy4JdLncgkuTp+fyHt+PjD7rrEPJxcvK5kcuphpiEKUlwajX4ngvGGAAADGCXl3BbhzBlGaaYhLW1tadtbkcIIX3EWF/H+eoqjFQKtHUEjzrwXPeuUwp6cgKzUMDF6hoMQWjMTk8/IdVoDLVIBKfxOFz9uOv6DX0M91jvxnQY+RxqTyPYGR1FIy2rpDI8gmp4CBcJHv9jtqahGgqjGI2hLqyDVIZHoIXDOOevAb39/wDQQiGo0RjqgoDeBOcc98/ujDE4XUAxGkOtBwh1AIyxztc5DpzdXdiqCtrYg09pD2DfANyZwHddOAdNWEUVnt6G73lwj1qwSyU4+/vwXQ92pXJ3Ai0Uws+5TzCLKuhBE77n9e7AAPieB9o6hFUs4mJjA1rwN4CT+Tn4lP71gD6lMFIpVIJBqFcAbWgYWnAQzakptBUFbSXb8UxH6923nsuhrWRx+GUR2uAgSs+iqAtfQfKfF6DMziKzvASFTyCbSCDD8ZA5DjLPI8NxyHAcZJ6DwvOQV1aQnpmBMj+Pxvda/Bcam/UA7uZvlAAAAABJRU5ErkJggg==</Image>
+<moz:SearchForm>https://mail.google.com/mail/#compose</moz:SearchForm>
+<Url type="text/html" method="GET" template="https://mail.google.com/mail/">
+  <Param name="view" value="cm"/>
+  <Param name="fs" value="1"/>
+  <Param name="tf" value="1"/>
+  <Param name="source" value="mailto"/>
+  <Param name="to" value="{searchTerms}"/>
+</Url>
+</OpenSearchDescription>
diff --git a/functional_tests/addons/amazonassist-201103011128.xpi b/functional_tests/addons/amazonassist-201103011128.xpi
diff --git a/functional_tests/addons/babuji-20110124355.xpi b/functional_tests/addons/babuji-20110124355.xpi
diff --git a/functional_tests/addons/chromemanifest-traceback.jar b/functional_tests/addons/chromemanifest-traceback.jar
diff --git a/functional_tests/addons/cooliris-1.12.2.44172-fx-mac.xpi.xpi b/functional_tests/addons/cooliris-1.12.2.44172-fx-mac.xpi.xpi
diff --git a/functional_tests/addons/download_statusbar-0.9.7.2-fx (1).xpi b/functional_tests/addons/download_statusbar-0.9.7.2-fx (1).xpi
diff --git a/functional_tests/addons/enigmail-1.2-sm-windows.xpi b/functional_tests/addons/enigmail-1.2-sm-windows.xpi
diff --git a/functional_tests/addons/feedly-addon-201101111013.xpi b/functional_tests/addons/feedly-addon-201101111013.xpi
diff --git a/functional_tests/addons/gabbielsan_tools-1.01-ff.xpi b/functional_tests/addons/gabbielsan_tools-1.01-ff.xpi
diff --git a/functional_tests/addons/glee-20101227219.xpi b/functional_tests/addons/glee-20101227219.xpi
diff --git a/functional_tests/addons/invalid maximum version number.xpi b/functional_tests/addons/invalid maximum version number.xpi
diff --git a/functional_tests/addons/lavafox_test-theme-20101130538.xpi b/functional_tests/addons/lavafox_test-theme-20101130538.xpi
@@ -0,0 +1,9 @@
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
+<html><head>
+<title>404 Not Found</title>
+</head><body>
+<h1>Not Found</h1>
+<p>The requested URL /pub/mozilla.org/57532/lavafox_v1-green-1.2.3-fx-macosx.jar was not found on this server.</p>
+<hr>
+<address>Apache/2.2.8 Server at releases.mozilla.org Port 80</address>
+</body></html>
diff --git a/functional_tests/addons/lexisone_citation_search-20100116 .xml b/functional_tests/addons/lexisone_citation_search-20100116 .xml
diff --git a/functional_tests/addons/littlemonkey-1.8.56-sm.xpi b/functional_tests/addons/littlemonkey-1.8.56-sm.xpi
diff --git a/functional_tests/addons/non-ascii-html.xpi b/functional_tests/addons/non-ascii-html.xpi
diff --git a/functional_tests/addons/peerscape-3.1.5-fx.xpi b/functional_tests/addons/peerscape-3.1.5-fx.xpi
diff --git a/functional_tests/addons/sms_search-20110115 .xml b/functional_tests/addons/sms_search-20110115 .xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/"
+                       xmlns:moz="http://www.mozilla.org/2006/browser/search/">
+  <ShortName>SMS Search</ShortName>
+  <Description>Search the best SMS collection. Love, Romantic, Birthday, Valentine days sms and much more.</Description>
+  <InputEncoding>UTF-8</InputEncoding>
+  <Image width="16" height="16" type="image/x-icon">data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAABMLAAAT
+CwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4uXdOsDCwXZ7fHzOWlpa2FNTU9hZWVnY
+ZGRl2JGSkrnHyMlr297fQ+fq6wwAAAAAAAAAAAAAAADj5uwEb27Zcl5dxo4yMjfDAAAA
+/wAAAP8AAAD/AAAA/wMDA/8lJSX0cXJylsHDxR4AAAAAAAAAAAAAAAAAAAAAdHTafwAA
+z/8AANT/AABF/wAAAP8AAAD/AAAA/wAAAP8AAAD/BAQE/xISEv9BQkLmvL6/OAAAAAAA
+AAAAAAAAAD9H5cYHEuf/AgPt/wMCmP8CAgD/AAAA/wAAAP8AAAD/AAAA/wAAAP8GBgb/
+FhYW/09PUOTW2doUAAAAAMHAxgtCULfcM1L9/xsw/P8DBeX/AQBV/w8PD/8YGBj/FhYW
+/w8PD/8ICAj/AAAA/wcHB/8eHh3/jpCQfwAAAACjpKA6DA0h/z9Sy/8yTfz/Ex7m/yYk
+sP9ydHv/fn57/39+f/9wcHD/aWlp/0RERP8NDQ3/Dw8O/1paW8oAAAAAUFJSXgMCAP9M
+TFr/TVCC/x4fOP8dHhj/EhIQ/wgICP8KCgr/FBQU/y0tLf9ZWVn/aGho/yIiIv80NDTn
+AAAAAEVGR2A3ODj/WlpW/ywqIv8xMCv/Kisp/yEhIf8UFBT/BgYG/wAAAP8AAAD/AAAA
+/x8fH/9dXV3/Kioq6QAAAACRk5M/LS0t/0BAQP9LS0v/RUVF/zk5Of8zMzP/JiYm/xIS
+Ev8KCgr/MjIy/ygoKP8AAAD/KSkp/1hYWNMAAAAAyMrLDENERN9UVFT/WVlZ/3BwcP+Q
+kJD/UFBQ/zExMf8hISH/oKCg//Pz8//e3t7/XFxc/wAAAP90dXaOAAAAAAAAAACJioty
+TExM/2RkZP/j4+P/ra2t/7Kysv8sLCz/V1dX//39/f/5+fn//////8PDw/8QEBD1rrGy
+JQAAAAAAAAAA0tTWCFRUVLlLS0v/6enp//////+enp7/MTEx/11dXf+fn5//sbGx////
+//+xsbH/Pz9AtwAAAAAAAAAAAAAAAAAAAACam5wLYWJikEtMTPlsbGz/SUlJ/0hISP8m
+Jib/fX19/9vb2/++vr7/KCgo/5OVllcAAAAAAAAAAAAAAAAAAAAA1NbYBKGjpB97fH04
+U1NUiUVGRsAnJyf0EhIR/xEREeEnJyfhHBwcz3JzdGcAAAAAAAAAAAAAAAAAAAAAAAAA
+AODj5AJ4enonlpeYAQAAAAAAAAAAjY6QJU5PUJufoKI+nJ6fCp2foAScnZ4IxsjKEwAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADDxcYU5+nrBAAAAACr
+rK0EaGpqQ5ucnhgAAAAA8AEAAMAD8L/AAQAAwAAAAIAAAACAAAAAgAAAAIAAAACAAAAA
+gACDQMAAAADAAQAA4AH//+AD///jAZIH/5EAAA==
+</Image>
+  <Url type="text/html" method="GET" template="http://sms.smile4friends.com/search?cx=011753032400493950399%3Avmqrbptahly&amp;cof=FORID%3A11&amp;ie=UTF-8&amp;q={searchTerms}&amp;sa=Search#1355" />
+  <Url type="application/x-suggestions+json" template="http://suggestqueries.google.com/complete/search?output=firefox&amp;client=firefox&amp;hl=en&amp;q={searchTerms}"/>
+  <AdultContent>false</AdultContent>
+  <Url type="application/opensearchdescription+xml"
+     rel="self"
+     template="http://sms.smile4friends.com/opensearch.xml" />
+
+  <moz:SearchForm>http://sms.smile4friends.com/</moz:SearchForm>
+</OpenSearchDescription>
diff --git a/functional_tests/addons/stumbleupon-3.76-fx+sm+mz.xpi b/functional_tests/addons/stumbleupon-3.76-fx+sm+mz.xpi
diff --git a/functional_tests/addons/test (1).xpi b/functional_tests/addons/test (1).xpi
diff --git a/functional_tests/addons/test-theme-3004.jar b/functional_tests/addons/test-theme-3004.jar
diff --git a/functional_tests/addons/tmp.xpi b/functional_tests/addons/tmp.xpi
diff --git a/functional_tests/test_validator.py b/functional_tests/test_validator.py
@@ -0,0 +1,293 @@
+import json
+import os
+import sys
+import unittest
+from cStringIO import StringIO
+
+from nose.tools import eq_
+from validator.validate import validate
+
+
+FIREFOX_GUID = '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}'
+MOBILE_GUID = '{a23983c0-fd0e-11dc-95ff-0800200c9a66}'
+THUNDERBIRD_GUID = '{3550f703-e582-4d05-9a08-453d09bdfdc6}'
+
+
+def _validator(file_path, for_appversions=None, overrides=None):
+    # TODO(Kumar) This is currently copied from Zamboni because
+    # it's really hard to import from zamboni outside of itself.
+    # TODO(Kumar) remove this when validator is fixed, see bug 620503
+    from validator.testcases import scripting
+    import validator
+    import validator.constants
+    js = os.environ.get('SPIDERMONKEY_INSTALLATION', 'js')
+    scripting.SPIDERMONKEY_INSTALLATION = js
+    validator.constants.SPIDERMONKEY_INSTALLATION = js
+    apps = os.path.join(os.path.dirname(validator.__file__),
+                        'app_versions.json')
+    if not os.path.exists(apps):
+        raise EnvironmentError('Could not locate app_versions.json in git '
+                               'repo for validator. Tried: %s' % apps)
+    orig = sys.stderr
+    sys.stderr = StringIO()
+    try:
+        result = validate(file_path, format='json',
+                        # Test all tiers at once. This will make sure we see
+                        # all error messages.
+                        determined=True,
+                        approved_applications=apps,
+                        spidermonkey=js,
+                        for_appversions=for_appversions,
+                        timeout=60 * 3,  # seconds
+                        overrides=overrides)
+        sys.stdout.write(sys.stderr.getvalue())
+        if 'Traceback' in sys.stderr.getvalue():
+            # the validator catches and ignores certain errors in an attempt
+            # to remain versatile.  There should not be any exceptions
+            # while testing.
+            raise RuntimeError(
+                "An exception was raised during validation. Check stderr")
+    finally:
+        sys.stderr = orig
+    return result
+
+
+_cached_validation = {}
+
+
+class ValidatorTest(unittest.TestCase):
+
+    def setUp(self):
+        self.validation = None
+        self.messages = None
+        self.ids = None
+
+    def msg_set(self, d):
+        return sorted(set([m['message'] for m in d['messages']]))
+
+    def id_set(self, d):
+        return set([tuple(m['id']) for m in d['messages']])
+
+    def validate(self, xpi, **validate_kwargs):
+        self.validation = self._run_validation(xpi, **validate_kwargs)
+        self.messages = self.msg_set(self.validation)
+        self.ids = self.id_set(self.validation)
+        return self.validation
+
+    def _cache_key(self, *vals):
+        args = []
+        for v in vals:
+            self._flatten(args, v)
+        return tuple(sorted(args))
+
+    def _flatten(self, args, val):
+        if isinstance(val, dict):
+            for k, v in val.iteritems():
+                self._flatten(args, k)
+                self._flatten(args, v)
+        elif isinstance(val, list):
+            for v in val:
+                self._flatten(args, v)
+        else:
+            args.append(val)
+
+    def _run_validation(self, xpi, **validate_kwargs):
+        path = os.path.join(os.path.dirname(__file__), 'addons', xpi)
+        cache_key = self._cache_key(path, validate_kwargs)
+        if cache_key in _cached_validation:
+            return _cached_validation[cache_key]
+        v = json.loads(_validator(path, **validate_kwargs))
+        _cached_validation[cache_key] = v
+        return v
+
+    def assertPartialMsg(self, partial_msg):
+        found = False
+        for m in self.messages:
+            if m.startswith(partial_msg):
+                found = True
+        assert found, ('Unexpected: %r' % self.messages)
+
+    def expectMsg(self, msg):
+        assert msg in self.messages, (
+                    'Expected %r but only got %r' % (msg, self.messages))
+
+    def shouldNotGetMsg(self, msg):
+        assert msg not in self.messages, ('Did not expect %r' % (msg))
+
+    def expectId(self, id):
+        assert id in self.ids, (
+                    'Expected %r but only got %r' % (id, self.ids))
+
+
+class CompatValidatorTest(ValidatorTest):
+
+    def validate_for_appver(self, xpi, app_guid, app_ver):
+        overrides = {'targetapp_maxVersion': {app_guid: app_ver}}
+        return self.validate(xpi, overrides=overrides,
+                             for_appversions={app_guid: [app_ver]})
+
+
+class JavaScriptTests(ValidatorTest):
+
+    def test_createelement__used(self):
+        self.validate('glee-20101227219.xpi')
+        self.assertPartialMsg('createElement() used to create script tag')
+
+    def test_dangerous_global(self):
+        self.validate('feedly-addon-201101111013.xpi')
+        self.expectMsg(u"`setTimeout` called in potentially "
+                       u"dangerous manner")
+
+    def test_global_called(self):
+        self.validate('babuji-20110124355.xpi')
+        self.expectMsg(u"`setTimeout` called in potentially "
+                       u"dangerous manner")
+
+    def test_potentially_malicious(self):
+        self.validate('add-on201101101027.xpi')
+        self.expectMsg(u'DOM Mutation Events Prohibited')
+
+    def test_variable_element(self):
+        self.validate('glee-20101227219.xpi')
+        self.expectMsg(u'Variable element type being created')
+
+
+class GeneralTests(ValidatorTest):
+
+    def test_contains_jar_files(self):
+        self.validate('test-theme-3004.jar')
+        self.expectMsg(u'Add-on contains JAR files, no <em:unpack>')
+
+    def test_potentially_illegal_name(self):
+        self.validate('add-on20110110322.xpi')
+        self.expectMsg(u'Add-on has potentially illegal name.')
+
+    def test_banned_element(self):
+        self.validate('gabbielsan_tools-1.01-ff.xpi')
+        self.expectMsg(u'Banned element in install.rdf')
+
+    def test_blacklisted_file(self):
+        self.validate('babuji-20110124355.xpi')
+        self.expectMsg(u'Flagged file extensions found.')
+
+    def test_blacklisted_file_2(self):
+        self.validate('peerscape-3.1.5-fx.xpi')
+        self.expectMsg(u'Flagged file type found')
+
+    def test_em_type_not(self):
+        self.validate('babuji-20110124355.xpi')
+        self.expectMsg(u'No <em:type> element found in install.rdf')
+
+    def test_obsolete_element(self):
+        self.validate('gabbielsan_tools-1.01-ff.xpi')
+        self.expectMsg(u'Banned element in install.rdf')
+
+    def test_unknown_file(self):
+        self.validate('gabbielsan_tools-1.01-ff.xpi')
+        self.expectMsg(u'Unrecognized element in install.rdf')
+
+    def test_unrecognized_element(self):
+        self.validate('littlemonkey-1.8.56-sm.xpi')
+        self.expectMsg(u'Add-on missing install.rdf.')
+
+    def test_invalid_id(self):
+        self.validate('add-ongoogle-201101121132.xpi')
+        self.expectMsg(u'The value of <em:id> is invalid')
+
+    def test_xpi_cannot(self):
+        self.validate('lavafox_test-theme-20101130538.xpi')
+        self.expectMsg(u'Corrupt ZIP file')
+
+    def test_invalid_version(self):
+        self.validate('invalid maximum version number.xpi')
+        self.expectMsg(u'Invalid maximum version number')
+
+    def test_non_ascii_html_markup(self):
+        # should be no Unicode errors
+        self.validate('non-ascii-html.xpi')
+
+
+class LocalizationTests(ValidatorTest):
+
+    def test_translation(self):
+        self.validate('babuji-20110124355.xpi')
+        self.expectMsg(u'Unchanged translation entities')
+
+    def test_encodings(self):
+        self.validate('babuji-20110124355.xpi')
+        self.expectMsg(u'Unexpected encodings in locale files')
+
+    def test_missing_translation(self):
+        self.validate('download_statusbar-0.9.7.2-fx (1).xpi')
+        self.expectMsg(u'Missing translation entity')
+
+
+class SecurityTests(CompatValidatorTest):
+
+    def test_missing_comments(self):
+        self.validate('add-on-20110113408.xpi')
+        self.expectMsg(u'Global variable overwrite')
+
+    def test_typeless_iframes_browsers(self):
+        self.validate('add-on201101081038.xpi')
+        self.expectMsg(u'Typeless iframes/browsers must be local.')
+
+    def test_binary_files(self):
+        self.validate_for_appver('cooliris-1.12.2.44172-fx-mac.xpi.xpi',
+                                 FIREFOX_GUID, '5.0a2')
+        self.expectMsg(u"Flagged file extensions found.")
+        self.expectMsg(u"Flagged file type found")
+        self.expectId(('testcases_packagelayout',
+                       'test_compatibility_binary',
+                       'disallowed_file_type'))
+
+    def test_thunderbird_binary_files(self):
+        self.validate_for_appver('enigmail-1.2-sm-windows.xpi',
+                                 THUNDERBIRD_GUID, '6.0a1')
+        self.expectMsg(u"Flagged file extensions found.")
+        self.expectId(('testcases_packagelayout',
+                       'test_compatibility_binary',
+                       'disallowed_file_type'))
+
+
+class NoErrorsExpected(ValidatorTest):
+
+    def test_an_attempt(self):
+        d = self.validate('tmp.xpi')
+        eq_(d['errors'], 0)
+
+    def test_don_t_freak(self):
+        d = self.validate('test (1).xpi')
+        eq_(d['errors'], 0)
+
+    def test_don_t_freak_2(self):
+        d = self.validate('littlemonkey-1.8.56-sm.xpi')
+        msg = self.msg_set(d)
+        ok = True
+        for m in msg:
+            if 'install.js' in msg:
+                ok = False
+        assert ok, ('Unexpected: %r' % msg)
+
+    def test_unknown_file(self):
+        d = self.validate('add-on20101228444 (1).jar')
+        eq_(d['errors'], 0)
+
+    def test_chromemanifest_traceback(self):
+        d = self.validate('chromemanifest-traceback.jar')
+        eq_(d['errors'], 0)
+
+
+class SearchTools(ValidatorTest):
+
+    def test_opensearch_providers(self):
+        self.validate('sms_search-20110115 .xml')
+        self.expectMsg(u'OpenSearch: <Url> elements may not be rel=self')
+
+    def test_opensearch_shortname(self):
+        self.validate('lexisone_citation_search-20100116 .xml')
+        self.expectMsg(u'OpenSearch: <ShortName> element too long')
+
+    def test_too_many(self):
+        self.validate('addon-12201-latest.xml')
+        self.expectMsg(u'OpenSearch: Too many <ShortName> elements')
diff --git a/tox.ini b/tox.ini
@@ -8,9 +8,8 @@ deps =
 commands =
     git submodule sync -q
     git submodule update --init
-    nosetests --with-xunit
+    nosetests --with-xunit tests/
 
 [testenv:functional-tests]
 commands =
-    git clone https://github.com/mozilla/test-amo-validator
-    nosetests --with-xunit test-amo-validator
+    nosetests --with-xunit functional_tests/