Skip to content
Permalink
Browse files

cython (#548)

* user cython for fields.py, parse.py and validators.py, fix #547

* fix coverage

* no cython on windows

* speedup error_wrappers, more cython

* conditional validators

* more tweaks to validators.py

* add compiled check

* fix mypy and tweak

* benchmark with cython

* simplify anystr_strip_whitespace

* build binaries on travis

* fix travis manylinux builds

* correct test stages

* cibuildwheel to dist

* fix manylinux build

* don't upgrade pip on wheel build

* try a fix for cibuildwheel

* speedup deploy stage

* revert file rearrangement, cythonize main.py

* tweak main.py

* update docs and history

* fix deploy stage of travis

* Cythonize more files (#553)

* Cythonize more files

* Tests pass

* Fixed ordering

* Some code cleanup

* Every last file cythonized

* cython coverage

* upgrade cython and tweak build setup

* different build stages
  • Loading branch information...
samuelcolvin committed May 30, 2019
1 parent af26f7f commit d473f4abc9d040c8c90e102017aacfc078f0f37d
@@ -18,3 +18,6 @@ docs/.tmp_schema_mappings.rst
.pytest_cache/
.vscode/
_build/
pydantic/*.c
pydantic/*.so
.auto-format
@@ -1,58 +1,112 @@
os: linux
dist: xenial
sudo: required
language: python

cache: pip

matrix:
include:
- python: '3.6'
- python: '3.7'
dist: xenial
sudo: required
- python: 3.8-dev
dist: xenial
sudo: required

allow_failures:
- python: 3.8-dev
python:
- '3.6'
- '3.7'
- '3.8-dev'

install:
- make install
- pip freeze

script:
- make lint

# test with and without ujson and email-validator then combine coverage
- make test && mv .coverage .coverage.extra
- pip uninstall -y ujson email-validator
- make test && mv .coverage .coverage.no-extra
- coverage combine
# test without cython but with ujson and email-validator
- python -c "import sys, pydantic; print('compiled:', pydantic.compiled); sys.exit(1 if pydantic.compiled else 0)"
- make test

- make lint
- make mypy
- make external-mypy
- make docs
- BENCHMARK_REPEATS=1 make benchmark-all
- ./tests/check_tag.py

after_success:
- ls -lha
- bash <(curl -s https://codecov.io/bash)

env:
- secure: "vpTd8bkwPBP0CV3EJBAwSMNMnNK3m/71dvTvBd1T4YGuefyJvYhtA7wauA5xRL9jpK2mu5QR5eo0owTUJhKi4DjpafMMd1bc4PnXlrdZFzkn3VsGmlKt74D/aJgiuiNyhd/Qvq4OxMHrMhf4f6lKWoMM1vh6yT0yp3+51SexSh2Me0Q+npxbjXwoxX5XUHRcoSLtFk4GbYI88a2I+08XWI6v+Awo/giQ5QurUJhjAklbosrrQVr1FCOkU0em5jeyZvEbZSLmaMtbX1JlRdKoJm6WMU+y9I7zj35w6ue/vgfcLz7b/HDZrBx7/L9g1LxRo80briueX/IbHvN7DOVFKvaXVmnEa6lIDdCeOLOyESpIbmjqmDKi8JeexdPNxKq4Tvo2VEA9dL2w2aw+aALNtU2OF5iEMfPTUQyosu/CNu2PKtiuZkSOdvpYbSy1WUNHJRvomdR4Olzg8ZIScNsxU3IIPdrlG/LUA8auXcE9juFeZfD6D2hQZATqWeEe/C2J7amNSD+mLLaTf6nMQw8oNtKYOvYK17M7xyvi7HXDy711Bi18U3x6Ye0xGx8CDbFwl0ICNzIk9rrSAh9hEHTvfdUUkk35pxifvO0Hrh4SArCA20ozcH/hHWBhyqGdxoIQ6KoDgNbIFIGQ6/vugxL/pt8z1sJwPfJnq8tRDAyWZvE="

deploy:
- provider: pypi
user: samuelcolvin
password:
secure: QbXFF2puEWjhFUpD0yu2R+wP4QI1IKIomBkMizsiCyMutlexERElranyYB8bsakvjPaJ+zU14ufffh2u7UA7Zhep/iE4skRHq4XWxnnRLHGu5nyGf3+zSM3F9MOzV32eZ4CDLJtFb6I0ensjTpodJH2EsIYHYxTgndIZn56Qbh6CStj7Xg1zm0Ujxdzm4ZLgcS28SOF/tpjsDW9+GXwc6L1mAZWYiS98gVgzL1vBd9tL9uFbbuFwGz9uhFMzFJko7vXSl8urWB4qeCspKXa9iKH7/AOYSwXTCwcg8U2hhC9UsOapnga2BubZKlU5HRfSs9fQcpnzcP2lwhSmkrEFa8VOw83hX6+bL564xK1Q4kanfGZ1fLU4FYge3iOnqjH7ajO7xEcUrcOEYUPfxM4EfdiDw0xnAzE1ITGH1/pZikF+wjlu+ez7RmmnejgK7quT1WU7keo7pSlRSfQtNgNl6xu818x0xZ1TScfN6e9npNy4TYyIooMOOeI4tMdfcR4JClkjGKhAtBk81DH7isZgPv3uwocGnKZ2S7La97CE3ADzU3MTA9xVIOSOjzwuvAe72uS2nwzqXkS9KATdATkC9QCvheJ9jIBB4UcqnHbD8L1gkqdmZwXZqHZldq8wcqNYZb+81lumy5EZ6xSoEzlLDpXHe80EjMUOBkb5fz3D44s=
distributions: sdist bdist_wheel
skip_upload_docs: true
on:
tags: true
jobs:
allow_failures:
- python: '3.8-dev'

include:
- stage: test
python: 3.6
- provider: script
script: make publish
on:
tags: true
name: 'Cython 3.6'
script:
# test with cython, ujson and email-validator
- make build-cython-trace
- python -c "import sys, pydantic; print('compiled:', pydantic.compiled); sys.exit(0 if pydantic.compiled else 1)"
- make test
- stage: test
python: 3.7
name: 'Cython 3.7'
script:
# test with cython, ujson and email-validator
- make build-cython-trace
- python -c "import sys, pydantic; print('compiled:', pydantic.compiled); sys.exit(0 if pydantic.compiled else 1)"
- make test

- stage: test
python: 3.6
name: 'Without Deps 3.6'
script:
# test without cython, ujson and email-validator
- pip uninstall -y ujson email-validator
- make test
- stage: test
python: 3.7
name: 'Without Deps 3.7'
script:
# test without cython, ujson and email-validator
- pip uninstall -y ujson email-validator cython
- make test

- stage: test
python: 3.7
name: 'Benchmarks'
script:
# default install skips cython compilation, need to compile for benchmarks
- make build-cython
- BENCHMARK_REPEATS=1 make benchmark-all
after_success: skip

- stage: build
name: 'PyPI Build and Upload'
if: type = push AND (branch = master OR tag IS present)
python: 3.7
services:
- docker
install: skip
script:
- ./tests/check_tag.py
- pip install -U cibuildwheel
- cibuildwheel --output-dir dist
- ls -lha dist
env:
- 'PIP=pip'
- 'CIBW_BUILD="cp36-manylinux1_x86_64 cp36-manylinux1_i686 cp37-manylinux1_x86_64 cp37-manylinux1_i686"'
- 'CIBW_BEFORE_BUILD="pip install -U cython"'
deploy:
provider: pypi
skip_cleanup: true
user: samuelcolvin
password:
secure: 'QbXFF2puEWjhFUpD0yu2R+wP4QI1IKIomBkMizsiCyMutlexERElranyYB8bsakvjPaJ+zU14ufffh2u7UA7Zhep/iE4skRHq4XWxnnRLHGu5nyGf3+zSM3F9MOzV32eZ4CDLJtFb6I0ensjTpodJH2EsIYHYxTgndIZn56Qbh6CStj7Xg1zm0Ujxdzm4ZLgcS28SOF/tpjsDW9+GXwc6L1mAZWYiS98gVgzL1vBd9tL9uFbbuFwGz9uhFMzFJko7vXSl8urWB4qeCspKXa9iKH7/AOYSwXTCwcg8U2hhC9UsOapnga2BubZKlU5HRfSs9fQcpnzcP2lwhSmkrEFa8VOw83hX6+bL564xK1Q4kanfGZ1fLU4FYge3iOnqjH7ajO7xEcUrcOEYUPfxM4EfdiDw0xnAzE1ITGH1/pZikF+wjlu+ez7RmmnejgK7quT1WU7keo7pSlRSfQtNgNl6xu818x0xZ1TScfN6e9npNy4TYyIooMOOeI4tMdfcR4JClkjGKhAtBk81DH7isZgPv3uwocGnKZ2S7La97CE3ADzU3MTA9xVIOSOjzwuvAe72uS2nwzqXkS9KATdATkC9QCvheJ9jIBB4UcqnHbD8L1gkqdmZwXZqHZldq8wcqNYZb+81lumy5EZ6xSoEzlLDpXHe80EjMUOBkb5fz3D44s='
on:
tags: true
all_branches: true

- stage: build
name: 'Docs Build and Upload'
if: type = push AND (branch = master OR tag IS present)
python: 3.7
script: make docs
env:
- secure: "vpTd8bkwPBP0CV3EJBAwSMNMnNK3m/71dvTvBd1T4YGuefyJvYhtA7wauA5xRL9jpK2mu5QR5eo0owTUJhKi4DjpafMMd1bc4PnXlrdZFzkn3VsGmlKt74D/aJgiuiNyhd/Qvq4OxMHrMhf4f6lKWoMM1vh6yT0yp3+51SexSh2Me0Q+npxbjXwoxX5XUHRcoSLtFk4GbYI88a2I+08XWI6v+Awo/giQ5QurUJhjAklbosrrQVr1FCOkU0em5jeyZvEbZSLmaMtbX1JlRdKoJm6WMU+y9I7zj35w6ue/vgfcLz7b/HDZrBx7/L9g1LxRo80briueX/IbHvN7DOVFKvaXVmnEa6lIDdCeOLOyESpIbmjqmDKi8JeexdPNxKq4Tvo2VEA9dL2w2aw+aALNtU2OF5iEMfPTUQyosu/CNu2PKtiuZkSOdvpYbSy1WUNHJRvomdR4Olzg8ZIScNsxU3IIPdrlG/LUA8auXcE9juFeZfD6D2hQZATqWeEe/C2J7amNSD+mLLaTf6nMQw8oNtKYOvYK17M7xyvi7HXDy711Bi18U3x6Ye0xGx8CDbFwl0ICNzIk9rrSAh9hEHTvfdUUkk35pxifvO0Hrh4SArCA20ozcH/hHWBhyqGdxoIQ6KoDgNbIFIGQ6/vugxL/pt8z1sJwPfJnq8tRDAyWZvE="
deploy:
provider: script
script: make publish
on:
tags: true
@@ -3,12 +3,18 @@
History
-------

v0.xx (xxxx-xx-xx)
v0.27 (unreleased)
..................
* fix JSON Schema for ``list``, ``tuple``, and ``set``, #540 by @tiangolo
* Change `_pydantic_post_init` to execute dataclass' original `__post_init__` before
* Change ``_pydantic_post_init`` to execute dataclass' original ``__post_init__`` before
validation, #560 by @HeavenVolkoff
* fix handling of generic types without specified parameters, #550 by @dmontagu
* **breaking change** (maybe): this is the first release compiled with **cython**, see the docs and please
submit an issue if you run into problems

v0.27.0a1 (2019-05-26)
......................
* fix JSON Schema for ``list``, ``tuple``, and ``set``, #540 by @tiangolo
* compiling with cython, ``manylinux`` binaries, some other performance improvements, #548 by @samuelcolvin

v0.26 (2019-05-22)
..................
@@ -6,7 +6,15 @@ black = black -S -l 120 --target-version py36 pydantic tests
install:
pip install -U setuptools pip
pip install -U -r requirements.txt
pip install -e .
SKIP_CYTHON=1 pip install -e .

.PHONY: build-cython-trace
build-cython-trace: clean
python setup.py build_ext --force --inplace --define CYTHON_TRACE

.PHONY: build-cython
build-cython: clean
python setup.py build_ext --inplace

.PHONY: format
format:
@@ -42,8 +50,12 @@ external-mypy:
(echo "mypy_test_fails2: mypy passed when it should have failed!"; exit 1)

.PHONY: testcov
testcov:
pytest --cov=pydantic
testcov: test
@echo "building coverage html"
@coverage html

.PHONY: testcov-compile
testcov-compile: build-cython-trace test
@echo "building coverage html"
@coverage html

@@ -72,6 +84,7 @@ clean:
rm -f .coverage
rm -f .coverage.*
rm -rf build
rm -f pydantic/*.c pydantic/*.so
python setup.py clean
make -C docs clean

@@ -80,8 +80,19 @@ Just::
*pydantic* has no required dependencies except python 3.6 or 3.7 (and the dataclasses package in python 3.6).
If you've got python 3.6 and ``pip`` installed - you're good to go.

*pydantic* can optionally be compiled with `cython <https://cython.org/>`_ which should give a 30-50% performance
improvement. ``manylinux`` binaries exist for python 3.6 and 3.7, so if you're installing from PyPI on linux, you
should get *pydantic* compiled with no extra work. If you're installing manually, install ``cython`` before installing
*pydantic* and you should get *pydandic* compiled. Compilation with cython is not tested on windows or mac.
`[issue] <https://github.com/samuelcolvin/pydantic/issues/555>`_

To test if *pydantic* is compiled run::

import pydantic
print('compiled:', pydantic.compiled)

If you want *pydantic* to parse json faster you can add `ujson <https://pypi.python.org/pypi/ujson>`_
as an optional dependency. Similarly if *pydantic's* email validation relies on
as an optional dependency. Similarly *pydantic's* email validation relies on
`email-validator <https://github.com/JoshData/python-email-validator>`_ ::

pip install pydantic[ujson]
@@ -5,7 +5,7 @@
from .error_wrappers import ValidationError
from .errors import *
from .fields import Required
from .main import BaseConfig, BaseModel, Extra, create_model, validate_model
from .main import BaseConfig, BaseModel, Extra, compiled, create_model, validate_model
from .parse import Protocol
from .schema import Schema
from .types import *
@@ -1,5 +1,4 @@
from collections import ChainMap
from dataclasses import dataclass
from functools import wraps
from inspect import Signature, signature
from itertools import chain
@@ -17,13 +16,13 @@
ValidatorCallable = Callable[[Optional[ModelOrDc], Any, Dict[str, Any], Field, Type[BaseConfig]], Any]


@dataclass
class Validator:
func: AnyCallable
pre: bool
whole: bool
always: bool
check_fields: bool
def __init__(self, func: AnyCallable, pre: bool, whole: bool, always: bool, check_fields: bool):
self.func = func
self.pre = pre
self.whole = whole
self.always = always
self.check_fields = check_fields


_FUNCS: Set[str] = set()
@@ -57,7 +56,10 @@ def dec(f: AnyCallable) -> classmethod:
raise ConfigError(f'duplicate validator function "{ref}"')
_FUNCS.add(ref)
f_cls = classmethod(f)
f_cls.__validator_config = fields, Validator(f, pre, whole, always, check_fields) # type: ignore
f_cls.__validator_config = ( # type: ignore
fields,
Validator(func=f, pre=pre, whole=whole, always=always, check_fields=check_fields),
)
return f_cls

return dec
@@ -10,7 +10,7 @@
import math
import re
from colorsys import hls_to_rgb, rgb_to_hls
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Tuple, Union, cast
from typing import TYPE_CHECKING, Any, Optional, Tuple, Union, cast

from pydantic.validators import not_none_validator

@@ -20,21 +20,28 @@
if TYPE_CHECKING: # pragma: no cover
from .types import CallableGenerator


ColorTuple = Union[Tuple[int, int, int], Tuple[int, int, int, float]]
ColorType = Union[ColorTuple, str]
HslColorTuple = Union[Tuple[float, float, float], Tuple[float, float, float, float]]


class RGBA(NamedTuple):
class RGBA:
"""
Internal use only as a representation of a color.
"""

r: float
g: float
b: float
alpha: Optional[float]
__slots__ = 'r', 'g', 'b', 'alpha', '_tuple'

def __init__(self, r: float, g: float, b: float, alpha: Optional[float]):
self.r = r
self.g = g
self.b = b
self.alpha = alpha

self._tuple: Tuple[float, float, float, Optional[float]] = (r, g, b, alpha)

def __getitem__(self, item: Any) -> Any:
return self._tuple[item]


r_hex_short = re.compile(r'\s*(?:#|0x)?([0-9a-f])([0-9a-f])([0-9a-f])([0-9a-f])?\s*')
@@ -1,6 +1,6 @@
import json
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Sequence, Tuple, Type, Union, cast
from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Sequence, Tuple, Type, Union

if TYPE_CHECKING: # pragma: no cover
from pydantic import BaseConfig # noqa: F401
@@ -9,13 +9,14 @@


class ErrorWrapper:
__slots__ = 'exc', 'loc', 'msg_template'
__slots__ = 'exc', 'type_', 'loc', 'msg_template'

def __init__(
self, exc: Exception, *, loc: Union[Tuple[str, ...], str], config: Optional[Type['BaseConfig']] = None
) -> None:
self.exc = exc
self.loc: Tuple[str, ...] = cast(Tuple[str, ...], loc if isinstance(loc, tuple) else (loc,))
self.type_ = get_exc_type(type(exc))
self.loc: Tuple[str, ...] = loc if isinstance(loc, tuple) else (loc,) # type: ignore
self.msg_template = config.error_msg_templates.get(self.type_) if config else None

@property
@@ -31,10 +32,6 @@ def msg(self) -> str:

return str(self.exc)

@property
def type_(self) -> str:
return get_exc_type(self.exc)

def dict(self, *, loc_prefix: Optional[Tuple[str, ...]] = None) -> Dict[str, Any]:
loc = self.loc if loc_prefix is None else loc_prefix + self.loc

@@ -107,8 +104,7 @@ def flatten_errors(


@lru_cache()
def get_exc_type(exc: Exception) -> str:
cls = type(exc)
def get_exc_type(cls: Type[Exception]) -> str:

base_name = 'type_error' if issubclass(cls, TypeError) else 'value_error'
if cls in (TypeError, ValueError):

0 comments on commit d473f4a

Please sign in to comment.
You can’t perform that action at this time.