Generate pydantic model from ormar.Model doesn't work for ormar.JSON fields

[shepilov-vladislav]

Describe the bug
If I try to use get_pydantic for model with ormar.JSON field - it's failed

To Reproduce

import uuid
from typing import List

import databases
import pydantic
import pytest
import sqlalchemy
from fastapi import FastAPI
from starlette.testclient import TestClient

import ormar

app = FastAPI()

DATABASE_URL = 'sqlite:///example.db'
database = databases.Database(DATABASE_URL)
metadata = sqlalchemy.MetaData()
app.state.database = database


@app.on_event("startup")
async def startup() -> None:
    database_ = app.state.database
    if not database_.is_connected:
        await database_.connect()


@app.on_event("shutdown")
async def shutdown() -> None:
    database_ = app.state.database
    if database_.is_connected:
        await database_.disconnect()


class BaseMeta(ormar.ModelMeta):
    metadata = metadata
    database = database


class Thing(ormar.Model):
    class Meta(BaseMeta):
        tablename = "things"

    id: uuid.UUID = ormar.UUID(primary_key=True, default=uuid.uuid4)
    name: str = ormar.Text(default="")
    json_field: pydantic.Json = ormar.JSON()


@app.get("/things", response_model=List[Thing])
async def read_things():
    return await Thing.objects.order_by("name").all()


ResponseThing = Thing.get_pydantic(exclude={"id": ...})


@app.get("/things/{id}", response_model=ResponseThing)
async def read_things(id: uuid.UUID):
    return await Thing.objects.get(pk=id)


@pytest.fixture(autouse=True, scope="module")
def create_test_database():
    engine = sqlalchemy.create_engine(DATABASE_URL)
    metadata.create_all(engine)
    yield
    metadata.drop_all(engine)


@pytest.mark.asyncio
async def test_read_main():
    client = TestClient(app)
    with client as client:
        thing1 = await Thing.objects.create(name="thing1", json_field={"1": "1", "2": 2})

        response = client.get("/things")
        assert response.status_code == 200
        assert response.json() == [{'id': str(thing1.id), 'name': 'thing1', 'json_field': {'1': '1', '2': 2}}]

        response = client.get(f"/things/{thing1.id}")  # Failed here
        #     async def serialize_response(
        #         *,
        #         field: Optional[ModelField] = None,
        #         response_content: Any,
        #         include: Optional[Union[SetIntStr, DictIntStrAny]] = None,
        #         exclude: Optional[Union[SetIntStr, DictIntStrAny]] = None,
        #         by_alias: bool = True,
        #         exclude_unset: bool = False,
        #         exclude_defaults: bool = False,
        #         exclude_none: bool = False,
        #         is_coroutine: bool = True,
        #     ) -> Any:
        #         if field:
        #             errors = []
        #             response_content = _prepare_response_content(
        #                 response_content,
        #                 exclude_unset=exclude_unset,
        #                 exclude_defaults=exclude_defaults,
        #                 exclude_none=exclude_none,
        #             )
        #             if is_coroutine:
        #                 value, errors_ = field.validate(response_content, {}, loc=("response",))
        #             else:
        #                 value, errors_ = await run_in_threadpool(
        #                     field.validate, response_content, {}, loc=("response",)
        #                 )
        #             if isinstance(errors_, ErrorWrapper):
        #                 errors.append(errors_)
        #             elif isinstance(errors_, list):
        #                 errors.extend(errors_)
        #             if errors:
        # >               raise ValidationError(errors, field.type_)
        # E               pydantic.error_wrappers.ValidationError: 1 validation error for Thing_OSI
        # E               response -> json_field
        # E                 JSON object must be str, bytes or bytearray (type=type_error.json)
        assert response.status_code == 200
        assert response.json() == {'name': 'thing1', 'json_field': {'1': '1', '2': 2}}

Versions

pyproject.toml

[tool.poetry]
name = "ormar_issue"
version = "0.1.0"
description = ""
authors = ["Vladislav Shepilov <shepilov.v@protonmail.com>"]

[tool.poetry.dependencies]
python = "^3.10"
fastapi = "^0.70.0"
pydantic = "^1.8.2"
ormar = "^0.10.22"
uvicorn = "^0.15.0"
requests = "^2.26.0"
databases = "^0.5.3"

[tool.poetry.dev-dependencies]
pytest = "^6.2.5"
pytest-asyncio = "^0.16.0"

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

[shepilov-vladislav]

@collerek Seems like related to #95

[collerek]

And what is the actual error message or what happens?

[shepilov-vladislav]

            async def serialize_response(
                *,
                field: Optional[ModelField] = None,
                response_content: Any,
                include: Optional[Union[SetIntStr, DictIntStrAny]] = None,
                exclude: Optional[Union[SetIntStr, DictIntStrAny]] = None,
                by_alias: bool = True,
                exclude_unset: bool = False,
                exclude_defaults: bool = False,
                exclude_none: bool = False,
                is_coroutine: bool = True,
            ) -> Any:
                if field:
                    errors = []
                    response_content = _prepare_response_content(
                        response_content,
                        exclude_unset=exclude_unset,
                        exclude_defaults=exclude_defaults,
                        exclude_none=exclude_none,
                    )
                    if is_coroutine:
                        value, errors_ = field.validate(response_content, {}, loc=("response",))
                    else:
                        value, errors_ = await run_in_threadpool(
                            field.validate, response_content, {}, loc=("response",)
                        )
                    if isinstance(errors_, ErrorWrapper):
                        errors.append(errors_)
                    elif isinstance(errors_, list):
                        errors.extend(errors_)
                    if errors:
        >               raise ValidationError(errors, field.type_)
        E               pydantic.error_wrappers.ValidationError: 1 validation error for Thing_OSI
        E                 JSON object must be str, bytes or bytearray (type=type_error.json)
        E               response -> json_field

[shepilov-vladislav]

@collerek I could look at the code and try to write a pull request. Tell me which place should I pay attention to?

[collerek]

OK, I checked again and I forgot that this is actually how Json field works in pydantic :)

That's ormar that gives you the extra feature of converting dicts to strings before it passes to pydantic, but when you generate a pydantic model out of ormar you are back into pydantic-only world.

To verify (will fail):

class AA(pydantic.BaseModel):
    test: pydantic.Json

def test1():
    aa = AA(test={'1': '1', '2': 2})

which is also verified by pydantic own tests:

def test_valid_simple_json():
    class JsonModel(BaseModel):
        json_obj: Json

    obj = '{"a": 1, "b": [2, 3]}' # <- note the string
    assert JsonModel(json_obj=obj).dict() == {'json_obj': {'a': 1, 'b': [2, 3]}}

https://github.com/samuelcolvin/pydantic/blob/d36bb74e83f5c8f1fd073d39d935f34876af6da4/tests/test_types.py#L2316