Skip to content

Commit

Permalink
Add advanced exclude and include support for dict, json and copy (#648)
Browse files Browse the repository at this point in the history 
* Add advanced exclude support for dict, json and copy

* Add advanced exclude support for dict, json and copy

Add new version section (v0.31)

* Add advanced include support, add more tests, improve code style
Rename ValueExclude to ValueItems and move it to utils
Use old logic to calculate keys, but still exclude it in _iter

* Add more tests for ValueItems

* Removed update arg check in _calculate_keys for return None
This will increase speed when no include or exclude given and skip_defaults is False

* Fix formatting, remove duplicate imports

* Add # pragma: no cover to 'if TYPE_CHECKING:' block

* tweaks and coverage

* fix history

* Add docs

* tweak docs
  • Loading branch information
@Bobronium @samuelcolvin
Bobronium authored and samuelcolvin committed Jul 24, 2019
1 parent bc60014 commit 74768c1
Show file tree
Hide file tree
Showing 9 changed files with 527 additions and 34 deletions.
1 change: 1 addition & 0 deletions HISTORY.rst
View file
Expand Up @@ -7,6 +7,7 @@ v0.31 (unreleased)
..................
* better support for floating point `multiple_of` values, #652 by @justindujardin
* fix schema generation for ``NewType`` and ``Literal``, #649 by @dmontagu
* add advanced exclude support for ``dict``, ``json`` and ``copy``, #648 by @MrMrRobat
* add documentation for Literal type, #651 by @dmontagu

v0.30.1 (2019-07-15)
Expand Down
32 changes: 32 additions & 0 deletions docs/examples/advanced_exclude1.py
View file
@@ -0,0 +1,32 @@
from pydantic import BaseModel, SecretStr

class User(BaseModel):
id: int
username: str
password: SecretStr

class Transaction(BaseModel):
id: str
user: User
value: int

transaction = Transaction(
id="1234567890",
user=User(
id=42,
username="JohnDoe",
password="hashedpassword"
),
value=9876543210
)

# using a set:
print(transaction.dict(exclude={'user', 'value'}))
#> {'id': '1234567890'}

# using a dict:
print(transaction.dict(exclude={'user': {'username', 'password'}, 'value': ...}))
#> {'id': '1234567890', 'user': {'id': 42}}

print(transaction.dict(include={'id': ..., 'user': {'id'}}))
#> {'id': '1234567890', 'user': {'id': 42}}
73 changes: 73 additions & 0 deletions docs/examples/advanced_exclude2.py
View file
@@ -0,0 +1,73 @@
import datetime
from typing import List

from pydantic import BaseModel, SecretStr

class Country(BaseModel):
name: str
phone_code: int

class Address(BaseModel):
post_code: int
country: Country

class CardDetails(BaseModel):
number: SecretStr
expires: datetime.date

class Hobby(BaseModel):
name: str
info: str

class User(BaseModel):
first_name: str
second_name: str
address: Address
card_details: CardDetails
hobbies: List[Hobby]

user = User(
first_name='John',
second_name='Doe',
address=Address(
post_code=123456,
country=Country(
name='USA',
phone_code=1
)
),
card_details=CardDetails(
number=4212934504460000,
expires=datetime.date(2020, 5, 1)
),
hobbies=[
Hobby(name='Programming', info='Writing code and stuff'),
Hobby(name='Gaming', info='Hell Yeah!!!')
]

)

exclude_keys = {
'second_name': ...,
'address': {'post_code': ..., 'country': {'phone_code'}},
'card_details': ...,
'hobbies': {-1: {'info'}}, # You can exclude values from tuples and lists by indexes
}

include_keys = {
'first_name': ...,
'address': {'country': {'name'}},
'hobbies': {0: ..., -1: {'name'}}
}

print(
user.dict(include=include_keys) == user.dict(exclude=exclude_keys) == {
'first_name': 'John',
'address': {'country': {'name': 'USA'}},
'hobbies': [
{'name': 'Programming', 'info': 'Writing code and stuff'},
{'name': 'Gaming'}
]
}
)
# True
16 changes: 16 additions & 0 deletions docs/index.rst
View file
Expand Up @@ -922,6 +922,22 @@ are not often changed.

.. literalinclude:: examples/copy_dict.py

Advanced include and exclude
............................

The ``dict``, ``json`` and ``copy`` methods support ``include`` and ``exclude`` arguments which can either be
sets or dictionaries, allowing nested selection of which fields to export:

.. literalinclude:: examples/advanced_exclude1.py

The ``...`` value indicates that we want to exclude or include entire key, just as if we included it in a set.

Of course same can be done on any depth level:

.. literalinclude:: examples/advanced_exclude2.py

Same goes for ``json`` and ``copy`` methods.

Serialisation
.............

Expand Down
163 changes: 129 additions & 34 deletions pydantic/main.py
View file
Expand Up @@ -37,6 +37,7 @@
AnyType,
ForwardRef,
GetterDict,
ValueItems,
change_exception,
is_classvar,
resolve_annotations,
Expand All @@ -58,7 +59,9 @@
SetStr = Set[str]
ListStr = List[str]
Model = TypeVar('Model', bound='BaseModel')

IntStr = Union[int, str]
SetIntStr = Set[IntStr]
DictIntStrAny = Dict[IntStr, Any]

try:
import cython # type: ignore
Expand Down Expand Up @@ -304,21 +307,30 @@ def __setstate__(self, state: 'DictAny') -> None:
object.__setattr__(self, '__fields_set__', state['__fields_set__'])

def dict(
self, *, include: 'SetStr' = None, exclude: 'SetStr' = None, by_alias: bool = False, skip_defaults: bool = False
self,
*,
include: Union['SetIntStr', 'DictIntStrAny'] = None,
exclude: Union['SetIntStr', 'DictIntStrAny'] = None,
by_alias: bool = False,
skip_defaults: bool = False,
) -> 'DictStrAny':
"""
Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.
"""
get_key = self._get_key_factory(by_alias)
get_key = partial(get_key, self.fields)

return_keys = self._calculate_keys(include=include, exclude=exclude, skip_defaults=skip_defaults)
if return_keys is None:
return {get_key(k): v for k, v in self._iter(by_alias=by_alias, skip_defaults=skip_defaults)}
else:
return {
get_key(k): v for k, v in self._iter(by_alias=by_alias, skip_defaults=skip_defaults) if k in return_keys
}
allowed_keys = self._calculate_keys(include=include, exclude=exclude, skip_defaults=skip_defaults)
return {
get_key(k): v
for k, v in self._iter(
by_alias=by_alias,
allowed_keys=allowed_keys,
include=include,
exclude=exclude,
skip_defaults=skip_defaults,
)
}

def _get_key_factory(self, by_alias: bool) -> Callable[..., str]:
if by_alias:
Expand All @@ -329,8 +341,8 @@ def _get_key_factory(self, by_alias: bool) -> Callable[..., str]:
def json(
self,
*,
include: 'SetStr' = None,
exclude: 'SetStr' = None,
include: Union['SetIntStr', 'DictIntStrAny'] = None,
exclude: Union['SetIntStr', 'DictIntStrAny'] = None,
by_alias: bool = False,
skip_defaults: bool = False,
encoder: Optional[Callable[[Any], Any]] = None,
Expand Down Expand Up @@ -417,8 +429,8 @@ def construct(cls: Type['Model'], values: 'DictAny', fields_set: 'SetStr') -> 'M
def copy(
self: 'Model',
*,
include: 'SetStr' = None,
exclude: 'SetStr' = None,
include: Union['SetIntStr', 'DictIntStrAny'] = None,
exclude: Union['SetIntStr', 'DictIntStrAny'] = None,
update: 'DictStrAny' = None,
deep: bool = False,
) -> 'Model':
Expand All @@ -436,11 +448,23 @@ def copy(
# skip constructing values if no arguments are passed
v = self.__values__
else:
return_keys = self._calculate_keys(include=include, exclude=exclude, skip_defaults=False)
if return_keys:
v = {**{k: v for k, v in self.__values__.items() if k in return_keys}, **(update or {})}
else:
allowed_keys = self._calculate_keys(include=include, exclude=exclude, skip_defaults=False, update=update)
if allowed_keys is None:
v = {**self.__values__, **(update or {})}
else:
v = {
**dict(
self._iter(
to_dict=False,
by_alias=False,
include=include,
exclude=exclude,
skip_defaults=False,
allowed_keys=allowed_keys,
)
),
**(update or {}),
}

if deep:
v = deepcopy(v)
Expand Down Expand Up @@ -487,17 +511,56 @@ def _decompose_class(cls: Type['Model'], obj: Any) -> GetterDict:
return GetterDict(obj)

@classmethod
def _get_value(cls, v: Any, by_alias: bool, skip_defaults: bool) -> Any:
@no_type_check
def _get_value(
cls,
v: Any,
to_dict: bool,
by_alias: bool,
include: Optional[Union['SetIntStr', 'DictIntStrAny']],
exclude: Optional[Union['SetIntStr', 'DictIntStrAny']],
skip_defaults: bool,
) -> Any:

if isinstance(v, BaseModel):
return v.dict(by_alias=by_alias, skip_defaults=skip_defaults)
elif isinstance(v, list):
return [cls._get_value(v_, by_alias=by_alias, skip_defaults=skip_defaults) for v_ in v]
elif isinstance(v, dict):
return {k_: cls._get_value(v_, by_alias=by_alias, skip_defaults=skip_defaults) for k_, v_ in v.items()}
elif isinstance(v, set):
return {cls._get_value(v_, by_alias=by_alias, skip_defaults=skip_defaults) for v_ in v}
elif isinstance(v, tuple):
return tuple(cls._get_value(v_, by_alias=by_alias, skip_defaults=skip_defaults) for v_ in v)
if to_dict:
return v.dict(by_alias=by_alias, skip_defaults=skip_defaults, include=include, exclude=exclude)
else:
return v.copy(include=include, exclude=exclude)

value_exclude = ValueItems(v, exclude) if exclude else None
value_include = ValueItems(v, include) if include else None

if isinstance(v, dict):
return {
k_: cls._get_value(
v_,
to_dict=to_dict,
by_alias=by_alias,
skip_defaults=skip_defaults,
include=value_include and value_include.for_element(k_),
exclude=value_exclude and value_exclude.for_element(k_),
)
for k_, v_ in v.items()
if (not value_exclude or not value_exclude.is_excluded(k_))
and (not value_include or value_include.is_included(k_))
}

elif isinstance(v, (list, set, tuple)):
return type(v)(
cls._get_value(
v_,
to_dict=to_dict,
by_alias=by_alias,
skip_defaults=skip_defaults,
include=value_include and value_include.for_element(i),
exclude=value_exclude and value_exclude.for_element(i),
)
for i, v_ in enumerate(v)
if (not value_exclude or not value_exclude.is_excluded(i))
and (not value_include or value_include.is_included(i))
)

else:
return v

Expand All @@ -517,14 +580,37 @@ def __iter__(self) -> 'AnyGenerator':
"""
yield from self._iter()

def _iter(self, by_alias: bool = False, skip_defaults: bool = False) -> 'TupleGenerator':
def _iter(
self,
to_dict: bool = True,
by_alias: bool = False,
allowed_keys: Optional['SetStr'] = None,
include: Union['SetIntStr', 'DictIntStrAny'] = None,
exclude: Union['SetIntStr', 'DictIntStrAny'] = None,
skip_defaults: bool = False,
) -> 'TupleGenerator':

value_exclude = ValueItems(self, exclude) if exclude else None
value_include = ValueItems(self, include) if include else None

for k, v in self.__values__.items():
yield k, self._get_value(v, by_alias=by_alias, skip_defaults=skip_defaults)
if allowed_keys is None or k in allowed_keys:
yield k, self._get_value(
v,
to_dict=to_dict,
by_alias=by_alias,
include=value_include and value_include.for_element(k),
exclude=value_exclude and value_exclude.for_element(k),
skip_defaults=skip_defaults,
)

def _calculate_keys(
self, include: 'SetStr' = None, exclude: Optional['SetStr'] = None, skip_defaults: bool = False
self,
include: Optional[Union['SetIntStr', 'DictIntStrAny']],
exclude: Optional[Union['SetIntStr', 'DictIntStrAny']],
skip_defaults: bool,
update: Optional['DictStrAny'] = None,
) -> Optional['SetStr']:

if include is None and exclude is None and skip_defaults is False:
return None

Expand All @@ -533,11 +619,20 @@ def _calculate_keys(
else:
keys = set(self.__values__.keys())

if include:
keys &= include
if include is not None:
if isinstance(include, dict):
keys &= include.keys()
else:
keys &= include

if update:
keys -= update.keys()

if exclude:
keys -= exclude
if isinstance(exclude, dict):
keys -= {k for k, v in exclude.items() if v is ...}
else:
keys -= exclude

return keys

Expand Down

0 comments on commit 74768c1

Please sign in to comment.