Merge branch 'release/0.4.0'

CHANGELOG

@@ -2,6 +2,11 @@
 ChangeLog
 *********
 
+0.4.0 (2016-09-20)
+==================
+- Update the URLValidator to support unicode domain names. (thanks, @caspinelli and @acshi!)
+- Get rid of the unused FlaskStoredObject, letting us remove Flask as a dependency.
+
 0.3.0 (2016-03-10)
 ==================
 - An Actual Release? Sure, why not!

dev-requirements.txt

@@ -2,5 +2,5 @@ nose
 mock
 tox
 wheel
-invoke
+invoke>=0.12.0,<0.13.0
 sphinx

docs/query_syntax.rst

@@ -13,12 +13,12 @@ Operators
 Equality
 ++++++++
 
-=========== =============== ====================================
+=========== =============== =====================================================
 Keyword     Operator        Description
-=========== =============== ====================================
-eq          equal to
+=========== =============== =====================================================
+eq          equal to        Match a single value, including inside a list field
 ne          not equal to
-=========== =============== ====================================
+=========== =============== =====================================================
 
 Comparison
 ++++++++++

modularodm/__init__.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"
 
 from .storedobject import StoredObject
-from .ext.odmflask import FlaskStoredObject
 
 from .query.querydialect import DefaultQueryDialect as Q

modularodm/validators/__init__.py

@@ -1,5 +1,6 @@
+# -*- coding: utf-8 -*-
 import six
-from six.moves.urllib_parse import urlsplit, urlunsplit
+from six.moves.urllib.parse import urlsplit, urlunsplit
 
 from modularodm.exceptions import (
     ValidationError,
@@ -72,35 +73,72 @@ def __init__(self, regex=None, flags=0):
 
     def __call__(self, value):
 
-        if not self.regex.search(value):
+        if not self.regex.findall(value):
             raise ValidationError(
-                'Value must match regex {0} and flags {1}; received value <{2}>'.format(
+                u'Value must match regex {0} and flags {1}; received value <{2}>'.format(
                     self.regex.pattern,
                     self.regex.flags,
                     value
                 )
             )
 
-# Adapted from Django URLValidator
+# Adapted from Django URLValidator v1.10
+# https://docs.djangoproject.com/en/1.10/_modules/django/core/validators/#URLValidator
 class URLValidator(RegexValidator):
+    ul = u'\u00a1-\uffff'  # unicode letters range, must be a unicode string
+
+    # IP patterns
+    ipv4_re = r'(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}'
+    ipv6_re = r'\[[0-9a-f:\.]+\]'  # (simple regex, validated later)
+
+    # Host patterns
+    hostname_re = r'[a-z' + ul + r'0-9](?:[a-z' + ul + r'0-9-]{0,61}[a-z' + ul + r'0-9])?'
+    # Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1
+    domain_re = r'(?:\.(?!-)[a-z' + ul + r'0-9-]{1,63}(?<!-))*'
+    tld_re = (
+        r'\.'                                # dot
+        r'(?!-)'                             # can't start with a dash
+        r'(?:xn--[a-z0-9]{1,59}'             # punycode labels (first for an eager regex match)
+        r'|[a-z' + ul + '-]{2,63})'          # or domain label
+        r'(?<!-)'                            # can't end with a dash
+        r'\.?'                               # may have a trailing dot
+    )
+    host_re = r'(' + hostname_re + domain_re + tld_re + r'|localhost)'
+
     regex = re.compile(
-        r'^(?:http|ftp)s?://'  # http:// or https://
-        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
-        r'localhost|'  # localhost...
-        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|'  # ...or ipv4
-        r'\[?[A-F0-9]*:[A-F0-9:]+\]?)'  # ...or ipv6
-        r'(?::\d+)?'  # optional port
-        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
-    # message = _('Enter a valid URL.')
+        r'^(?:[a-z0-9\.\-\+]*)://'  # scheme is validated separately
+        r'(?:\S+(?::\S*)?@)?'  # user:pass authentication
+        r'(?:' + ipv4_re + '|' + ipv6_re + '|' + host_re + ')'
+        r'(?::\d{2,5})?'  # port
+        r'(?:[/?#][^\s]*)?'  # resource path
+        r'\Z', re.IGNORECASE)
+    message = 'Invalid URL'
+    schemes = ['http', 'https', 'ftp', 'ftps']
+
+    def __init__(self, schemes=None, **kwargs):
+        super(URLValidator, self).__init__(**kwargs)
+        if schemes is not None:
+            self.schemes = schemes
 
     def __call__(self, value):
+        # Check first if the scheme is valid (if there is a scheme used)
+        if '://' in value:
+            scheme = value.split('://')[0].lower()
+            if scheme not in self.schemes:
+                raise ValidationError(self.message + ' ' + value)
+        else:
+            value = 'http://' + value # implicit scheme
+
+        # Then check full URL
         try:
             super(URLValidator, self).__call__(value)
         except ValidationError as e:
             # Trivial case failed. Try for possible IDN domain
             if value:
-                # value = force_text(value)
-                scheme, netloc, path, query, fragment = urlsplit(value)
+                try:
+                    scheme, netloc, path, query, fragment = urlsplit(value)
+                except ValueError:  # for example, "Invalid IPv6 URL"
+                    raise ValidationError(self.message + ' ' + value)
                 try:
                     netloc = netloc.encode('idna').decode('ascii')  # IDN -> ACE
                 except UnicodeError:  # invalid domain part
@@ -110,9 +148,22 @@ def __call__(self, value):
             else:
                 raise
         else:
-            pass
-            # url = value
-
+            # Now verify IPv6 in the netloc part
+            host_match = re.search(r'^\[(.+)\](?::\d{2,5})?$', urlsplit(value).netloc)
+            if host_match:
+                potential_ip = host_match.groups()[0]
+                try:
+                    validate_ipv6_address(potential_ip)
+                except ValidationError:
+                    raise ValidationError(self.message, code=self.code)
+            url = value
+
+        # The maximum length of a full host name is 253 characters per RFC 1034
+        # section 3.1. It's defined to be 255 bytes or less, but this includes
+        # one byte for the length of the name and one byte for the trailing dot
+        # that's used to indicate absolute names in DNS.
+        if len(urlsplit(value).netloc) > 253:
+            raise ValidationError(self.message, code=self.code)
 
 class BaseValidator(Validator):
 

requirements.txt

@@ -1,4 +1,4 @@
-Flask
 blinker
 pymongo
 python-dateutil
+Werkzeug==0.11.11

tests/validators/test_url_validation.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+import os.path
+import json
+
+from modularodm import StoredObject
+from modularodm.exceptions import ValidationError
+from modularodm.fields import StringField, IntegerField
+from modularodm.validators import URLValidator
+
+from tests.base import ModularOdmTestCase
+
+class UrlValueValidatorTestCase(ModularOdmTestCase):
+
+    def test_url(self):
+        basepath = os.path.dirname(__file__)
+        url_data_path = os.path.join(basepath, 'urlValidatorTest.json')
+        with open(url_data_path) as url_test_data:
+            data = json.load(url_test_data)
+
+        class Foo(StoredObject):
+            _id = IntegerField()
+            url_field = StringField(
+                list=False,
+                validate=[URLValidator()]
+            )
+
+        Foo.set_storage(self.make_storage())
+        test_object = Foo()
+
+        for urlTrue in data['testsPositive']:
+            test_object.url_field = urlTrue
+            test_object.save()
+
+        for urlFalse in data['testsNegative']:
+            test_object.url_field = urlFalse
+            try:
+                with self.assertRaises(ValidationError):
+                    test_object.save()
+            except AssertionError as e:
+                e.args += (' for ', urlFalse)
+                raise

tests/validators/urlValidatorTest.json

@@ -0,0 +1,71 @@
+{
+  "_comment": "These tests adapted from https://mathiasbynens.be/demo/url-regex",
+  "testsPositive": {
+    "definitelyawebsite.com":"should accept simple valid website",
+    "https://Definitelyawebsite.com":"should accept valid website with protocol",
+    "http://foo.com/blah_blah":"should accept valid website with path",
+    "http://foo.com/blah_blah:5000/pathhere":"should accept valid website with port and path",
+    "http://foo.com/blah_blah_(wikipedia)":"should accept valid website with parentheses in path",
+    "https://userid@example.com/":"should accept valid user website",
+    "http://userid@example.com:8080":"should accept valid user website with port",
+    "http://userid:password@example.com/":"should accept valid user website with password",
+    "http://userid:password@example.com:8080/":"should accept valid user and password website with path",
+    "http://142.42.1.1":"should accept valid ipv4 website test 1",
+    "http://10.1.1.0":"should accept valid ipv4 website test 2",
+    "http://10.1.1.255":"should accept valid ipv4 website test 3",
+    "http://224.1.1.1":"should accept valid ipv4 website test 4",
+    "http://142.42.1.1:8080/":"should accept valid ipv4 website with port",
+    "http://Bücher.de":"should accept valid website with unicode in domain",
+    "http://heynow.ws/䨹":"should accept valid website with unicode in path",
+    "http://localhost:5000/meetings":"should accept valid localhost website",
+    "http://⌘.ws":"should accept valid website with only unicode in path",
+    "http://⌘.ws/":"should accept valid website with only unicode in path and a / after domain",
+    "http://foo.com/blah_(wikipedia)#cite-1":"should accept valid website with hashtag following parentheses in path",
+    "http://foo.com/blah_(wikipedia)_blah#cite-1":"should accept valid website with hashtag in path",
+    "http://foo.com/unicode_(✪)_in_parens":"should accept valid website with unicode in parentheses in path",
+    "http://foo.com/(something)?after=parens":"should accept valid website with something after path",
+    "http://staging.damowmow.com/":"should accept valid website with sub-domain",
+    "http://☺.damowmow.com/":"should accept valid website with unicode in sub-domain",
+    "http://code.google.com/events/#&product=browser":"should accept valid website with variables",
+    "ftp://foo.bar/baz":"should acccept valid website with ftps",
+    "http://foo.bar/?q=Test%20URL-encoded%20stuff":"should accept valid website with encoded stuff in path",
+    "http://مثال.إختبار":"should accept valid unicode heavy website test 1",
+    "http://例子.测试":"should accept valid unicode heavy website test 2",
+    "http://उदाहरण.परीक्षा":"should accept valid unicode heavy website test 3",
+    "http://-.~_!$&()*+,;=:%40:80%2f::::::@example.com":"should accept valid website with user but crazy username",
+    "http://1337.net":"should accept valid website with just numbers in domain",
+    "definitelyawebsite.com?real=yes&page=definitely":"should accept valid website with query",
+    "http://a.b-c.de":"should accept valid website with dash",
+    "http://website.com:3000/?q=400":"should accept valid website with port and query",
+    "http://asd/asd@asd.com/":"should accept valid website with unusual username"
+  },
+  "testsNegative": {
+    "notevenclose": "should deny simple invalid website",
+    "http://": "should deny invalid website with only http://",
+    "http://.": "should deny invalid website with only http://.",
+    "http://..": "should deny invalid website with only http://..",
+    "http://../": "should deny invalid website with only http://../",
+    "http://?": "should deny invalid website with only http://?",
+    "http://??": "should deny invalid website with only http://??",
+    "http://??/": "should deny invalid website with only http://??/",
+    "http://#": "should deny invalid website with only http://#",
+    "http://##": "should deny invalid website with only http://##",
+    "http://##/": "should deny invalid website with only http://##/",
+    "http://foo.bar?q=Spaces should be encoded": "should deny invalid website with spaces in path",
+    "//": "should deny invalid website with only //",
+    "//a": "should deny invalid website with only //a",
+    "///a": "should deny invalid website with only ///a",
+    "///": "should deny invalid website with only ///",
+    "http:///a": "should deny invalid website with three / in protocol",
+    "rdar://1234": "should deny invalid website with invalid protocol",
+    "h://test": "should deny invalid website with missing letters from protocol",
+    "http:// shouldfail.com": "should deny invalid website with space in beginning of domain",
+    "http://should fail": "should deny invalid website with space in middle of domain",
+    "http://-error-.invalid/": "should deny invalid website with dash at beginning and end of domain",
+    "http://1.1.1.1.1": "should deny invalid ipv4 website with 5 numbers",
+    "http://567.100.100.100": "should deny invalid ipv4 website with a number out of range",
+    "http://-a.b.co": "should deny invalid website with dash at beginning of sub-domain",
+    "http://.www.foo.bar/": "should deny invalid website with dot before sub-domain",
+    "httsp://userid@example.com/": "should deny invalid website with username and invalid scheme"
+  }
+}