Excludes for aliased nested fields are skipped when using model.dict() with by_alias=True

[AlexECX]

Bug

Output of python -c "import pydantic.utils; print(pydantic.utils.version_info())":

             pydantic version: 1.5a1
            pydantic compiled: False
                 install path: /home/acox/.local/share/virtualenvs/google-people-python-client-H9fvGxOj/src/pydantic/pydantic
               python version: 3.7.5 (default, Feb 28 2020, 23:28:26)  [GCC 7.4.0]
                     platform: Linux-4.4.0-18362-Microsoft-x86_64-with-debian-buster-sid
     optional deps. installed: ['typing-extensions']

Exemple demonstrating the problem:

from pydantic import BaseModel, Field


class NestedModel(BaseModel):
    to_exclude: str


class MyModel(BaseModel):
    aliased: NestedModel = Field(None, alias="an_alias")
    not_aliased: NestedModel = None


data = {
    "an_alias": {"to_exclude": "hello"},
    "not_aliased": {"to_exclude": "hello"}
}

model = MyModel(**data)
assert model.aliased
assert model.not_aliased

# Case 1 using a top-level Ellipsis with a nested aliased field, all is well.

excludes = {
    "aliased": ...,
    "not_aliased": {"to_exclude"},
}

model_dict = MyModel(**data).dict(exclude=excludes, by_alias=True)
assert "to_exclude" not in model_dict["not_aliased"]
assert "an_alias" not in model_dict  # <-- This works fine

# Case 2 using a Set or Dict with a nested aliased field, problem.

excludes = {
    "aliased": {"to_exclude"},
    "not_aliased": {"to_exclude"},
}

model_dict = MyModel(**data).dict(exclude=excludes, by_alias=True)
assert "to_exclude" not in model_dict["an_alias"]
assert "to_exclude" not in model_dict["not_aliased"]  # <-- This will fail

Code where the problem could be solved:

Before (for context)

# pydantic/main.py:671 in BaseModel._iter()
        allowed_keys = self._calculate_keys(include=include, exclude=exclude, exclude_unset=exclude_unset)
        if allowed_keys is None and not (to_dict or by_alias or exclude_unset or exclude_defaults or exclude_none):
            # huge boost for plain _iter()
            yield from self.__dict__.items()
            return

        value_exclude = ValueItems(self, exclude) if exclude else None
        value_include = ValueItems(self, include) if include else None

        for k, v in self.__dict__.items():
            if (
                (allowed_keys is not None and k not in allowed_keys)
                or (exclude_none and v is None)
                or (exclude_defaults and self.__field_defaults__.get(k, _missing) == v)
            ):
                continue
            if by_alias and k in self.__fields__:
                k = self.__fields__[k].alias
            if to_dict or value_include or value_exclude:
                v = self._get_value(
                    v,
                    to_dict=to_dict,
                    by_alias=by_alias,
                    include=value_include and value_include.for_element(k),
                    exclude=value_exclude and value_exclude.for_element(k),
                    exclude_unset=exclude_unset,
                    exclude_defaults=exclude_defaults,
                    exclude_none=exclude_none,
                )
            yield k, v

The problem seems to be that allowed_keys needs the model's field real names, so exclude must contain real field names, while value_exclude.for_element(k) is passed either an alias when by_alias=True, or a real field name when by_alias=False. This makes value_exclude.for_element() fetch the wrong name when by_alias=True.

This could be fixed by always passing the real field name to value_exclude.for_element(), regardless of by_alias.

Changes

# pydantic/main.py:680 in BaseModel._iter()
        for field_key, v in self.__dict__.items():
            if (
                (allowed_keys is not None and field_key not in allowed_keys)
                or (exclude_none and v is None)
                or (exclude_defaults and self.__field_defaults__.get(field_key, _missing) == v)
            ):
                continue
            if by_alias and field_key in self.__fields__:
                dict_key = self.__fields__[field_key].alias
            else:
                dict_key = field_key
            if to_dict or value_include or value_exclude:
                v = self._get_value(
                    v,
                    to_dict=to_dict,
                    by_alias=by_alias,
                    include=value_include and value_include.for_element(field_key),
                    exclude=value_exclude and value_exclude.for_element(field_key),
                    exclude_unset=exclude_unset,
                    exclude_defaults=exclude_defaults,
                    exclude_none=exclude_none,
                )
            yield dict_key, v

I'll try and make a proper PR later today, hopefully this can be fixed before the 1.5 release.

[samuelcolvin]

Thanks for reporting, PR welcome.

Don't worry too much about including this in v1.5, I will try and release the next version sooner so it's not the end of the world if this needs to wait until v1.6.