-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1068 from open-contracting/995-pydantic
Use Pydantic
- Loading branch information
Showing
26 changed files
with
224 additions
and
303 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,37 +1,68 @@ | ||
from dataclasses import dataclass | ||
from typing import Any | ||
import enum | ||
import io | ||
import zipfile | ||
from typing import Any, TypedDict | ||
|
||
import pydantic | ||
import rarfile | ||
|
||
@dataclass | ||
class Item: | ||
file_name: str | ||
url: str | ||
Data = ( | ||
# https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractfile (DigiwhistBase) | ||
io.BufferedReader | ||
# https://rarfile.readthedocs.io/api.html#rarfile.RarFile.open (CompressedFileSpider) | ||
| rarfile.RarExtFile | ||
# https://docs.python.org/3/library/zipfile.html#zipfile.ZipFile.open (CompressedFileSpider) | ||
| zipfile.ZipExtFile | ||
| pydantic.conbytes(strict=True, min_length=1) | ||
# `dict` behaves better last. https://github.com/open-contracting/kingfisher-collect/issues/995 | ||
| dict | ||
) | ||
|
||
base_kwargs = {'validate_assignment': True} | ||
|
||
# data can be: bytes | dict | list | io.BufferedReader | rarfile.RarExtFile | zipfile.ZipExtFile | ||
@dataclass | ||
class File(Item): | ||
data_type: str | ||
data: Any | ||
|
||
class DataType(str, enum.Enum): | ||
record = "record" | ||
release = "release" | ||
record_package = "record_package" | ||
release_package = "release_package" | ||
|
||
|
||
class Errors(TypedDict): | ||
http_code: pydantic.conint(strict=True, ge=100, lt=600) | ||
|
||
|
||
class Resource(pydantic.BaseModel, **base_kwargs): | ||
file_name: pydantic.constr(strict=True, regex=r'^[^/\\]+$') # noqa: F722 pydantic/pydantic#2872 | ||
url: pydantic.HttpUrl | ||
|
||
|
||
class DataResource(Resource, arbitrary_types_allowed=True, use_enum_values=True): | ||
data_type: DataType | ||
data: Data | ||
# Added by the FilesStore extension, for the KingfisherProcessAPI2 extension to refer to the file. | ||
path: str = "" | ||
|
||
@pydantic.validator('data', pre=True) # `pre` is needed to prevent pydantic from type casting | ||
def check_data(cls, v): | ||
# pydantic has no `condict()` to set `strict=True` or `min_properties=1`. pydantic/pydantic#1277 | ||
assert isinstance(v, (Data, bytes)), f'{v.__class__.__name__} is not a valid type' | ||
assert v, 'ensure this value is non-empty' | ||
return v | ||
|
||
|
||
class File(DataResource): | ||
pass | ||
|
||
|
||
# This doesn't inherit from the File class, because we want isinstance(item, File) to be false for FileItem instances. | ||
@dataclass | ||
class FileItem(Item): | ||
data_type: str | ||
data: Any | ||
number: int | ||
# Added by the FilesStore extension, for the KingfisherProcessAPI2 extension to refer to the file. | ||
path: str = "" | ||
class FileItem(DataResource): | ||
number: pydantic.conint(strict=True, gt=0) | ||
|
||
|
||
@dataclass | ||
class FileError(Item): | ||
errors: dict | ||
class FileError(Resource): | ||
errors: Errors | ||
|
||
|
||
@dataclass | ||
class PluckedItem: | ||
class PluckedItem(pydantic.BaseModel, **base_kwargs): | ||
value: Any |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
7b422dc5cc62fb663ac45e841547b7983537cfc528d3c43ef39afe09b62cc7ca requirements.txt | ||
8e1eea5a0879c0002ed9878e2c470b3baaf54ae158179c7a7d260ff2e9219a94 requirements.txt |
Oops, something went wrong.