add UrlStr and urlstr types pydantic#236

HISTORY.rst

@@ -6,6 +6,7 @@ History
 v0.12.2 (XXXX-XX-XX)
 ....................
 * raise an exception if a field's name shadows an existing ``BaseModel`` attribute #242
+* add ``UrlStr`` and ``urlstr`` types #236
 
 v0.12.1 (2018-07-31)
 ....................

docs/examples/exotic.py

@@ -4,8 +4,8 @@
 from uuid import UUID
 
 from pydantic import (DSN, UUID1, UUID3, UUID4, UUID5, BaseModel, DirectoryPath, EmailStr, FilePath, NameEmail,
-                      NegativeFloat, NegativeInt, PositiveFloat, PositiveInt, PyObject, condecimal, confloat, conint,
-                      constr)
+                      NegativeFloat, NegativeInt, PositiveFloat, PositiveInt, PyObject, UrlStr, condecimal, confloat,
+                      conint, constr)
 
 
 class Model(BaseModel):
@@ -31,6 +31,8 @@ class Model(BaseModel):
     email_address: EmailStr = None
     email_and_name: NameEmail = None
 
+    url: UrlStr = None
+
     db_name = 'foobar'
     db_user = 'postgres'
     db_password: str = None
@@ -66,6 +68,7 @@ class Model(BaseModel):
     unit_interval=0.5,
     email_address='Samuel Colvin <s@muelcolvin.com >',
     email_and_name='Samuel Colvin <s@muelcolvin.com >',
+    url='http://example.com',
     decimal=Decimal('42.24'),
     decimal_positive=Decimal('21.12'),
     decimal_negative=Decimal('-21.12'),
@@ -95,6 +98,7 @@ class Model(BaseModel):
     'unit_interval': 0.5,
     'email_address': 's@muelcolvin.com',
     'email_and_name': <NameEmail("Samuel Colvin <s@muelcolvin.com>")>,
+    'url': 'http://example.com',
     ...
     'dsn': 'postgres://postgres@localhost:5432/foobar',
     'decimal': Decimal('42.24'),

pydantic/errors.py

@@ -59,6 +59,19 @@ class EmailError(PydanticValueError):
     msg_template = 'value is not a valid email address'
 
 
+class UrlSchemeError(PydanticValueError):
+    code = 'url.scheme'
+    msg_template = 'url scheme "{scheme}" is not allowed'
+
+    def __init__(self, *, scheme: str) -> None:
+        super().__init__(scheme=scheme)
+
+
+class UrlRegexError(PydanticValueError):
+    code = 'url.regex'
+    msg_template = 'url string does not match regex'
+
+
 class EnumError(PydanticTypeError):
     msg_template = 'value is not a valid enumeration member'
 

pydantic/types.py

@@ -2,11 +2,11 @@
 import re
 from decimal import Decimal
 from pathlib import Path
-from typing import Optional, Pattern, Type, Union
+from typing import Optional, Pattern, Set, Type, Union
 from uuid import UUID
 
 from . import errors
-from .utils import change_exception, import_string, make_dsn, validate_email
+from .utils import change_exception, import_string, make_dsn, url_regex_generator, validate_email
 from .validators import (anystr_length_validator, anystr_strip_whitespace, decimal_validator, float_validator,
                          int_validator, not_none_validator, number_size_validator, path_exists_validator,
                          path_validator, str_validator)
@@ -25,6 +25,8 @@
     'ConstrainedStr',
     'constr',
     'EmailStr',
+    'UrlStr',
+    'urlstr',
     'NameEmail',
     'PyObject',
     'DSN',
@@ -92,6 +94,18 @@ def validate(cls, value: str) -> str:
         return value
 
 
+def constr(*, strip_whitespace=False, min_length=0, max_length=2**16, curtail_length=None, regex=None) -> Type[str]:
+    # use kwargs then define conf in a dict to aid with IDE type hinting
+    namespace = dict(
+        strip_whitespace=strip_whitespace,
+        min_length=min_length,
+        max_length=max_length,
+        curtail_length=curtail_length,
+        regex=regex and re.compile(regex)
+    )
+    return type('ConstrainedStrValue', (ConstrainedStr,), namespace)
+
+
 class EmailStr(str):
     @classmethod
     def get_validators(cls):
@@ -107,6 +121,60 @@ def validate(cls, value):
         return validate_email(value)[1]
 
 
+class UrlStr(str):
+    strip_whitespace = True
+    min_length = 1
+    max_length = 2 ** 16
+    default_schemes = {'http', 'https', 'ftp', 'ftps'}
+    schemes: Optional[Set[str]] = None
+    relative = False  # whether to allow relative URLs
+    require_tld = True  # whether to reject non-FQDN hostnames
+
+    @classmethod
+    def get_validators(cls):
+        yield not_none_validator
+        yield str_validator
+        yield anystr_strip_whitespace
+        yield anystr_length_validator
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, value: str) -> str:
+        # Check first if the scheme is valid
+        schemes = cls.schemes or cls.default_schemes
+        if '://' in value:
+            scheme = value.split('://')[0].lower()
+            if scheme not in schemes:
+                raise errors.UrlSchemeError(scheme=scheme)
+
+        regex = url_regex_generator(relative=cls.relative, require_tld=cls.require_tld)
+        if not regex.match(value):
+            raise errors.UrlRegexError()
+
+        return value
+
+
+def urlstr(
+    *,
+    strip_whitespace=True,
+    min_length=1,
+    max_length=2**16,
+    relative=False,
+    require_tld=True,
+    schemes: Optional[Set[str]] = None
+) -> Type[str]:
+    # use kwargs then define conf in a dict to aid with IDE type hinting
+    namespace = dict(
+        strip_whitespace=strip_whitespace,
+        min_length=min_length,
+        max_length=max_length,
+        relative=relative,
+        require_tld=require_tld,
+        schemes=schemes,
+    )
+    return type('UrlStrValue', (UrlStr,), namespace)
+
+
 class NameEmail:
     __slots__ = 'name', 'email'
 
@@ -133,18 +201,6 @@ def __repr__(self):
         return f'<NameEmail("{self}")>'
 
 
-def constr(*, strip_whitespace=False, min_length=0, max_length=2**16, curtail_length=None, regex=None) -> Type[str]:
-    # use kwargs then define conf in a dict to aid with IDE type hinting
-    namespace = dict(
-        strip_whitespace=strip_whitespace,
-        min_length=min_length,
-        max_length=max_length,
-        curtail_length=curtail_length,
-        regex=regex and re.compile(regex)
-    )
-    return type('ConstrainedStrValue', (ConstrainedStr,), namespace)
-
-
 class PyObject:
     validate_always = True
 

pydantic/utils.py

@@ -2,9 +2,10 @@
 import re
 from contextlib import contextmanager
 from enum import Enum
+from functools import lru_cache
 from importlib import import_module
 from textwrap import dedent
-from typing import List, Tuple, Type
+from typing import List, Pattern, Tuple, Type
 
 from . import errors
 
@@ -165,3 +166,27 @@ def validate_field_name(bases: List[Type['BaseModel']], field_name: str) -> None
         if getattr(base, field_name, None):
             raise NameError(f'Field name "{field_name}" shadows a BaseModel attribute; '
                             f'use a different field name with "alias=\'{field_name}\'".')
+
+
+@lru_cache(maxsize=None)
+def url_regex_generator(*, relative: bool, require_tld: bool) -> Pattern:
+    return re.compile(
+        r''.join((
+            r'^',
+            r'(' if relative else r'',
+            r'(?:[a-z0-9\.\-\+]*)://',  # scheme is validated separately
+            r'(?:[^:@]+?:[^:@]*?@|)',  # basic auth
+            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+',
+            r'(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|',  # domain...
+            r'localhost|',  # localhost...
+            (
+                r'(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.?)|'
+                if not require_tld else r''
+            ),  # allow dotless hostnames
+            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|',  # ...or ipv4
+            r'\[[A-F0-9]*:[A-F0-9:]+\])',  # ...or ipv6
+            r'(?::\d+)?',  # optional port
+            r')?' if relative else r'',  # host is optional, allow for relative URLs
+            r'(?:/?|[/?]\S+)$',
+        )), re.IGNORECASE,
+    )