Permalink
Browse files

Normalize URLs for manifests (bug 837945)

  • Loading branch information...
1 parent 7d3f492 commit 76a3d20785d77d41df9fb6a47c9f9276a2791d2b @mattbasta committed Feb 5, 2013
View
@@ -1,6 +1,8 @@
-import argparse
import sys
+import argparse
+import requests
+
from . import validate_app, validate_packaged_app
@@ -48,8 +50,20 @@ def main():
print "Invalid timeout. Integer expected."
sys.exit(1)
- error_bundle = validate_packaged_app(
- args.package, listed=not args.unlisted, format=None, timeout=timeout)
+ if "://" in args.package:
+ error_bundle = validate_app(
+ requests.get(args.package).content, listed=not args.unlisted,
+ format=None, url=args.package)
+
+ elif args.package.endswith(".webapp"):
+ with open(args.package) as f:
+ error_bundle = validate_app(
+ f.read(), listed=not args.unlisted, format=None)
+
+ else:
+ error_bundle = validate_packaged_app(
+ args.package, listed=not args.unlisted, format=None,
+ timeout=timeout)
# Print the output of the tests based on the requested format.
if args.output == "text":
@@ -64,5 +78,5 @@ def main():
sys.exit(0)
# Start up the testing and return the output.
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
View
@@ -110,15 +110,7 @@ def test_inner_package(err, package):
# Iterate through each test of our detected type.
for test in testcases._get_tests(tier):
-
- test_func = test["test"]
- if test["simple"]:
- test_func(err)
- else:
- # Pass in:
- # - Error Bundler
- # - A copy of the package itself
- test_func(err, package)
+ test(err, package)
# Return any errors at the end of the tier if undetermined.
if err.failed(fail_on_warnings=False) and not err.determined:
@@ -2,7 +2,7 @@
TEST_TIERS = {}
-def register_test(tier=1, simple=False):
+def register_test(tier=1):
"""Register tests for the validation flow."""
def wrap(function):
@@ -13,7 +13,7 @@ def wrap(function):
TEST_TIERS[tier] = []
# Add a test object to the test's tier
- TEST_TIERS[tier].append({"test": function, "simple": simple})
+ TEST_TIERS[tier].append(function)
# Return the function to be run
return function
@@ -19,7 +19,9 @@
@register_test(tier=2)
def test_packed_packages(err, package=None):
- "Tests XPI and JAR files for naughty content."
+
+ if not package:
+ return
processed_files = 0
pretested_files = err.get_resource("pretested_files") or []
@@ -22,6 +22,9 @@
def test_blacklisted_files(err, package=None):
"Detects blacklisted files and extensions."
+ if not package:
+ return
+
flagged_files = []
for name in package:
@@ -88,6 +91,9 @@ def test_blacklisted_files(err, package=None):
def test_layout_all(err, package):
"""Tests the well-formedness of extensions."""
+ if not package:
+ return
+
package_namelist = list(package.zf.namelist())
package_nameset = set(package_namelist)
if len(package_namelist) != len(package_nameset):
@@ -1,4 +1,5 @@
import base64
+import urlparse
from cStringIO import StringIO
import requests
@@ -12,6 +13,10 @@
@register_test(tier=1)
def test_app_manifest(err, package):
+ if not err.get_resource("packaged"):
+ # This is done by the validate_*() functions.
+ return
+
if "manifest.webapp" not in package:
return err.error(
err_id=("webappbase", "test_app_manifest", "missing_manifest"),
@@ -45,6 +50,13 @@ def try_get_data_uri(data_url):
return decoded
+def _normalize_url(err, url):
+ p_url = urlparse.urlparse(url)
+ p_defurl = urlparse.urlparse(err.get_resource("manifest_url"))
+
+ return urlparse.urlunparse(p_defurl[:2] + p_url[2:])
+
+
def try_get_resource(err, package, url, filename, resource_type="URL",
max_size=True):
@@ -62,20 +74,23 @@ def try_get_resource(err, package, url, filename, resource_type="URL",
return
# Pull in whatever packaged app resources are required.
- if err.get_resource("packaged") and "://" not in url:
- url = url.lstrip("/")
- try:
- return package.read(url)
- except Exception:
- err.error(
- err_id=("resources", "packaged", "not_found"),
- error="Resource in packaged app not found.",
- description=["A resource within a packaged app is "
- "referenced, but the path used does not "
- "point to a valid item in the package.",
- "Requested resource: %s" % url],
- filename=filename)
- return
+ if "://" not in url:
+ if err.get_resource("packaged"):
+ url = url.lstrip("/")
+ try:
+ return package.read(url)
+ except Exception:
+ err.error(
+ err_id=("resources", "packaged", "not_found"),
+ error="Resource in packaged app not found.",
+ description=["A resource within a packaged app is "
+ "referenced, but the path used does not "
+ "point to a valid item in the package.",
+ "Requested resource: %s" % url],
+ filename=filename)
+ return
+ else:
+ url = _normalize_url(err, url)
http_cache = err.get_or_create('http_cache', {})
if url in http_cache:
@@ -121,6 +136,18 @@ def generic_http_error():
return data
except requests.exceptions.MissingSchema:
+ if (not err.get_resource("packaged") and
+ not err.get_resource("manifest_url")):
+ err.warning(
+ err_id=("resources", "missing_schema"),
+ warning="Unable to fetch resource",
+ description=["A relative URL was encountered, but because "
+ "the full URL of the manifest is not known, it "
+ "is not possible to fetch the resources. You "
+ "can provide the URL to fix this issue.",
+ "URL: %s" % url],
+ filename=filename)
+ return
err.error(
err_id=("resources", "invalid_url", "schema"),
error="Invalid URL",
@@ -129,6 +156,15 @@ def generic_http_error():
"contain a schema.",
"URL: %s" % url],
filename=filename)
+ except requests.exceptions.InvalidSchema:
+ err.error(
+ err_id=("resources", "invalid_url", "bad_schema"),
+ error="Invalid URL Schema",
+ description=["While attempting to retrieve a remote resource, "
+ "an invalid URL was encountered. The URL uses an "
+ "invalid schema.",
+ "URL: %s" % url],
+ filename=filename)
except requests.exceptions.URLRequired:
err.error(
err_id=("resources", "invalid_url", "none"),
View
@@ -1,13 +1,14 @@
import json
import constants
-from errorbundle import ErrorBundle
import loader
import submain
import webapp
+from errorbundle import ErrorBundle
-def validate_app(data, listed=True, market_urls=None):
+def validate_app(data, listed=True, market_urls=None, url=None,
+ format="json"):
"""
A handy function for validating apps.
@@ -18,19 +19,24 @@ def validate_app(data, listed=True, market_urls=None):
`market_urls`:
A list of URLs to use when validating the `installs_allowed_from`
field of the manifest. Does not apply if `listed` is not set to `True`.
+ `url`:
+ The URL of the manifest. Used to resolve non-absolute URLs.
+ `format`:
+ The output format to return the results in.
Notes:
- App validation is always determined because there is only one tier.
- Spidermonkey paths are not accepted by this function because we don't
perform JavaScript validation on webapps.
"""
bundle = ErrorBundle(listed=listed)
-
- # Set the market URLs.
bundle.save_resource("market_urls", market_urls)
+ bundle.save_resource("manifest_url", url)
webapp.detect_webapp_string(bundle, data)
- return format_result(bundle, "json")
+ submain.test_inner_package(bundle, None)
+
+ return format_result(bundle, format)
def validate_packaged_app(path, listed=True, format="json", market_urls=None,
View
@@ -1,6 +1,7 @@
from functools import wraps
import sys
+import requests
from mock import MagicMock, Mock, patch
from appvalidator.zip import ZipPackage
@@ -45,13 +46,20 @@ def wrap(test_icon, *args, **kwargs):
"`requests.get` already mocked")
with patch("requests.get") as r_g:
- request = Mock()
- request.text = "foo bar"
- request.status_code = 200
- # The first bit is the return value. The second bit tells whatever
- # is requesting the data that there's no more data.
- request.raw.read.side_effect = [request.text, ""]
- r_g.return_value = request
+ def request_generator(*args, **kwargs):
+ url = kwargs.get("url", args[0])
+ if "://" not in url:
+ raise requests.exceptions.MissingSchema
+
+ request = Mock()
+ request.text = "foo bar"
+ request.status_code = 200
+ # The first bit is the return value. The second bit tells whatever
+ # is requesting the data that there's no more data.
+ request.raw.read.side_effect = [request.text, ""]
+ return request
+
+ r_g.side_effect = request_generator
return func(*args, **kwargs)
return wrap
View
@@ -12,7 +12,7 @@ def test_tiers():
"""
testcases.register_test(tier=1)(lambda: None)
testcases.register_test(tier=2)(lambda: None)
- testcases.register_test(tier=2, simple=True)(lambda: None)
+ testcases.register_test(tier=2)(lambda: None)
tiers = testcases._get_tiers()
print tiers
View
@@ -78,24 +78,15 @@ def _get_tests(self, tier):
if self.fail_tier is not None:
if tier == self.fail_tier:
print "> Fail Tier"
-
- yield {"test": lambda x, y: x.fail_tier(),
- "simple": False,
- "versions": None}
+ yield lambda x, y: x.fail_tier()
assert tier <= self.fail_tier or self.determined
self.last_tier = tier
for x in range(1,10): # Ten times because we care
print "Yielding Complex"
- yield {"test": lambda x, z: x.report(tier),
- "simple": False,
- "versions": None}
- print "Yielding Simple"
- yield {"test": lambda x, z=None: x.test_simple(z),
- "simple": True,
- "versions": None}
+ yield lambda x, z: x.report(tier)
def report_tier(self, tier):
"Checks to make sure the last test run is on the current tier."
@@ -139,10 +130,6 @@ def fail_tier(self):
self.has_failed = True
self.decorator.report_fail()
- def test_simple(self, z):
- "Makes sure that the second two params of a simple test are respected"
- assert z is None
-
def failed(self, fail_on_warnings=False):
"Simple accessor because the standard error handler has one"
return self.has_failed
Oops, something went wrong.

0 comments on commit 76a3d20

Please sign in to comment.