Skip to content

Commit

Permalink
SQLModel support
Browse files Browse the repository at this point in the history
  • Loading branch information
bkalashnikov authored and bogdandm committed Jan 2, 2023
1 parent 202fbf8 commit 88e196c
Show file tree
Hide file tree
Showing 6 changed files with 285 additions and 11 deletions.
16 changes: 9 additions & 7 deletions README.md
Expand Up @@ -8,16 +8,17 @@
![Example](/etc/convert.png)

json2python-models is a [Python](https://www.python.org/) tool that can generate Python models classes
([pydantic](https://github.com/samuelcolvin/pydantic), dataclasses, [attrs](https://github.com/python-attrs/attrs))
([pydantic](https://docs.pydantic.dev/), ([sqlmodel](https://sqlmodel.tiangolo.com/),
dataclasses, [attrs](https://www.attrs.org/en/stable/))
from JSON datasets.

## Features

* Full **`typing` module** support
* **Types merging** - if some field contains data of different types this will be represent as `Union` type
* **Types merging** - if some field contains data of different types this will be represented as `Union` type
* Fields and models **names** generation (unicode support included)
* Similar **models generalization**
* Handling **recursive data** structures (i.e family tree)
* Handling **recursive data** structures (i.e. family tree)
* Detecting **string serializable types** (i.e. datetime or just stringify numbers)
* Detecting fields containing string constants (`Literal['foo', 'bar']`)
* Generation models as **list** (flat models structure) or **tree** (nested models)
Expand Down Expand Up @@ -157,7 +158,7 @@ class Constructor(BaseModel):

It requires a bit of tweaking:
* Some fields store routes/models specs as dicts
* There are a lot of optinal fields so we reduce merging threshold
* There are a lot of optional fields, so we reduce merging threshold
* Disable string literals

```
Expand Down Expand Up @@ -495,9 +496,10 @@ Arguments:

* `-f`, `--framework` - Model framework for which python code is generated.
`base` (default) mean no framework so code will be generated without any decorators and additional meta-data.
* **Format**: `-f {base, pydantic, attrs, dataclasses, custom}`
* **Format**: `-f {base, pydantic, sqlmodel, attrs, dataclasses, custom}`
* **Example**: `-f pydantic`
* **Default**: `-f base`
* **Warning**: SQLModel generator does not support Relationships and Foreign keys, they have to be added manually

* `-s`, `--structure` - Models composition style.
* **Format**: `-s {flat, nested}`
Expand Down Expand Up @@ -546,7 +548,7 @@ Arguments:
this dict will be marked as dict field but not nested model.
* **Format**: `--dkr RegEx [RegEx ...]`
* **Example**: `--dkr node_\d+ \d+_\d+_\d+`
* **Note**: `^` and `$` (string borders) tokens will be added automatically but you
* **Note**: `^` and `$` (string borders) tokens will be added automatically, but you
have to escape other special characters manually.
* **Optional**

Expand Down Expand Up @@ -588,7 +590,7 @@ cd json2python-models
python setup.py test -a '<pytest additional arguments>'
```

Also I would recommend you to install `pytest-sugar` for pretty printing test results
Also, I would recommend you to install `pytest-sugar` for pretty printing test results

### Test examples

Expand Down
6 changes: 5 additions & 1 deletion json_to_models/cli.py
Expand Up @@ -11,6 +11,8 @@
from pathlib import Path
from typing import Any, Callable, Dict, Generator, Iterable, List, Tuple, Type, Union

from .models.sqlmodel import SqlModelCodeGenerator

try:
import ruamel.yaml as yaml
except ImportError:
Expand Down Expand Up @@ -55,6 +57,7 @@ class Cli:
"dataclasses": convert_args(DataclassModelCodeGenerator, meta=bool_js_style,
post_init_converters=bool_js_style),
"pydantic": convert_args(PydanticModelCodeGenerator),
"sqlmodel": convert_args(SqlModelCodeGenerator),
}

def __init__(self):
Expand Down Expand Up @@ -122,7 +125,8 @@ def run(self):
structure,
self.model_generator,
class_generator_kwargs=self.model_generator_kwargs,
preamble=self.preamble)
preamble=self.preamble
)
if self.output_file:
with open(self.output_file, "w", encoding="utf-8") as f:
f.write(output)
Expand Down
1 change: 1 addition & 0 deletions json_to_models/models/__init__.py
Expand Up @@ -14,3 +14,4 @@ class ClassType(Enum):
Dataclass = "dataclass"
Attrs = "attrs"
Pydantic = "pydantic"
SqlModel = "sqlmodel"
10 changes: 7 additions & 3 deletions json_to_models/models/pydantic.py
Expand Up @@ -70,7 +70,6 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP
"""
imports, data = super().field_data(name, meta, optional)
default: Optional[str] = None
body_kwargs = {}
if optional:
meta: DOptional
if isinstance(meta.type, DList):
Expand All @@ -80,8 +79,7 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP
else:
default = "None"

if name != data["name"]:
body_kwargs["alias"] = f'"{name}"'
body_kwargs = self._get_field_kwargs(name, meta, optional, data)
if body_kwargs:
data["body"] = self.PYDANTIC_FIELD.render(
default=default or '...',
Expand All @@ -90,3 +88,9 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP
elif default is not None:
data["body"] = default
return imports, data

def _get_field_kwargs(self, name: str, meta: MetaData, optional: bool, data: dict):
body_kwargs = {}
if name != data["name"]:
body_kwargs["alias"] = f'"{name}"'
return body_kwargs
33 changes: 33 additions & 0 deletions json_to_models/models/sqlmodel.py
@@ -0,0 +1,33 @@
from typing import List, Tuple

from json_to_models.dynamic_typing import ImportPathList, MetaData
from json_to_models.models.base import GenericModelCodeGenerator
from json_to_models.models.pydantic import PydanticModelCodeGenerator


class SqlModelCodeGenerator(PydanticModelCodeGenerator):
def generate(self, nested_classes: List[str] = None, extra: str = "", **kwargs) \
-> Tuple[ImportPathList, str]:
imports, body = GenericModelCodeGenerator.generate(
self,
bases='SQLModel, table=True',
nested_classes=nested_classes,
extra=extra
)
imports.append(('sqlmodel', ['SQLModel', 'Field']))
body = """
# Warn! This generated code does not respect SQLModel Relationship and foreign_key, please add them manually.
""".strip() + '\n' + body
return imports, body

def convert_field_name(self, name):
if name in ('id', 'pk'):
return name
return super().convert_field_name(name)

def _get_field_kwargs(self, name: str, meta: MetaData, optional: bool, data: dict):
kwargs = super()._get_field_kwargs(name, meta, optional, data)
# Detect primary key
if data['name'] in ('id', 'pk') and meta is int:
kwargs['primary_key'] = True
return kwargs
230 changes: 230 additions & 0 deletions test/test_code_generation/test_sqlmodel_generation.py
@@ -0,0 +1,230 @@
from typing import Dict, List

import pytest

from json_to_models.dynamic_typing import (
DDict,
DList,
DOptional,
DUnion,
FloatString,
IntString,
ModelMeta,
compile_imports,
)
from json_to_models.models.base import generate_code
from json_to_models.models.sqlmodel import SqlModelCodeGenerator
from json_to_models.models.structure import sort_fields
from test.test_code_generation.test_models_code_generator import model_factory, trim

# Data structure:
# pytest.param id -> {
# "model" -> (model_name, model_metadata),
# test_name -> expected, ...
# }
test_data = {
"base": {
"model": ("Test", {
"foo": int,
"Bar": int,
"baz": float
}),
"fields_data": {
"foo": {
"name": "foo",
"type": "int"
},
"Bar": {
"name": "bar",
"type": "int",
"body": 'Field(..., alias="Bar")'
},
"baz": {
"name": "baz",
"type": "float"
}
},
"fields": {
"imports": "",
"fields": [
f"foo: int",
f'bar: int = Field(..., alias="Bar")',
f"baz: float",
]
},
"generated": trim(f"""
from sqlmodel import Field, SQLModel
# Warn! This generated code does not respect SQLModel Relationship and foreign_key, please add them manually.
class Test(SQLModel, table=True):
foo: int
bar: int = Field(..., alias="Bar")
baz: float
""")
},
"complex": {
"model": ("Test", {
"foo": int,
"baz": DOptional(DList(DList(str))),
"bar": DOptional(IntString),
"qwerty": FloatString,
"asdfg": DOptional(int),
"dict": DDict(int),
"not": bool,
"1day": int,
"день_недели": str,
}),
"fields_data": {
"foo": {
"name": "foo",
"type": "int"
},
"baz": {
"name": "baz",
"type": "Optional[List[List[str]]]",
"body": "[]"
},
"bar": {
"name": "bar",
"type": "Optional[int]",
"body": "None"
},
"qwerty": {
"name": "qwerty",
"type": "float"
},
"asdfg": {
"name": "asdfg",
"type": "Optional[int]",
"body": "None"
},
"dict": {
"name": "dict_",
"type": "Dict[str, int]",
"body": 'Field(..., alias="dict")'
},
"not": {
"name": "not_",
"type": "bool",
"body": 'Field(..., alias="not")'
},
"1day": {
"name": "one_day",
"type": "int",
"body": 'Field(..., alias="1day")'
},
"день_недели": {
"name": "den_nedeli",
"type": "str",
"body": 'Field(..., alias="день_недели")'
}
},
"generated": trim(f"""
from sqlmodel import Field, SQLModel
from typing import Dict, List, Optional
# Warn! This generated code does not respect SQLModel Relationship and foreign_key, please add them manually.
class Test(SQLModel, table=True):
foo: int
qwerty: float
dict_: Dict[str, int] = Field(..., alias="dict")
not_: bool = Field(..., alias="not")
one_day: int = Field(..., alias="1day")
den_nedeli: str = Field(..., alias="день_недели")
baz: Optional[List[List[str]]] = []
bar: Optional[int] = None
asdfg: Optional[int] = None
""")
},
"converters": {
"model": ("Test", {
"a": int,
"b": IntString,
"c": DOptional(FloatString),
"d": DList(DList(DList(IntString))),
"e": DDict(IntString),
"u": DUnion(DDict(IntString), DList(DList(IntString))),
}),
"generated": trim("""
from sqlmodel import Field, SQLModel
from typing import Dict, List, Optional, Union
# Warn! This generated code does not respect SQLModel Relationship and foreign_key, please add them manually.
class Test(SQLModel, table=True):
a: int
b: int
d: List[List[List[int]]]
e: Dict[str, int]
u: Union[Dict[str, int], List[List[int]]]
c: Optional[float] = None
""")
},
"sql_models": {
"model": ("Test", {
"id": int,
"name": str,
"x": DList(int)
}),
"generated": trim("""
from sqlmodel import Field, SQLModel
from typing import List
# Warn! This generated code does not respect SQLModel Relationship and foreign_key, please add them manually.
class Test(SQLModel, table=True):
id: int = Field(..., primary_key=True)
name: str
x: List[int]
""")
}
}

test_data_unzip = {
test: [
pytest.param(
model_factory(*data["model"]),
data[test],
id=id
)
for id, data in test_data.items()
if test in data
]
for test in ("fields_data", "fields", "generated")
}


@pytest.mark.parametrize("value,expected", test_data_unzip["fields_data"])
def test_fields_data_attr(value: ModelMeta, expected: Dict[str, dict]):
gen = SqlModelCodeGenerator(value)
required, optional = sort_fields(value)
for is_optional, fields in enumerate((required, optional)):
for field in fields:
field_imports, data = gen.field_data(field, value.type[field], bool(is_optional))
assert data == expected[field]


@pytest.mark.parametrize("value,expected", test_data_unzip["fields"])
def test_fields_attr(value: ModelMeta, expected: dict):
expected_imports: str = expected["imports"]
expected_fields: List[str] = expected["fields"]
gen = SqlModelCodeGenerator(value)
imports, fields = gen.fields
imports = compile_imports(imports)
assert imports == expected_imports
assert fields == expected_fields


@pytest.mark.parametrize("value,expected", test_data_unzip["generated"])
def test_generated_attr(value: ModelMeta, expected: str):
generated = generate_code(
(
[{"model": value, "nested": []}],
{}
),
SqlModelCodeGenerator,
class_generator_kwargs={}
)
assert generated.rstrip() == expected, generated

0 comments on commit 88e196c

Please sign in to comment.