From 982bd2edefc11cf4cacdc71b2c990482f9b5f898 Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Tue, 13 Jul 2021 19:59:28 +0300 Subject: [PATCH] Add docs; Change default yaml parser; Handle non-str keys in dicts --- README.md | 162 +++++++++++++++++++++--- json_to_models/cli.py | 6 +- json_to_models/generator.py | 5 + setup.py | 2 +- test/test_generator/test_detect_type.py | 2 +- 5 files changed, 157 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 133a90c..9dd8154 100644 --- a/README.md +++ b/README.md @@ -277,15 +277,138 @@ class Response: @dataclass class Definition_Schema: - type_: str - required: Optional[List[str]] = field(default_factory=list) - properties: Optional[Dict[str, Union['Property', 'Property_2E']]] = field(default_factory=dict) - ref: Optional[str] = None + type_: str + required: Optional[List[str]] = field(default_factory=list) + properties: Optional[Dict[str, Union['Property', 'Property_2E']]] = field(default_factory=dict) + ref: Optional[str] = None ```

+### Github-actions config files + +
----- Show ----- +

+ +Github-actions model based on files from [starter-workflows](https://github.com/actions/starter-workflows/tree/main/ci) + +``` +json2models -m Actions "./starter-workflows/ci/*.yml" -s flat -f pydantic -i yaml --dkf env with jobs +``` + +```python +r""" +generated by json2python-models v0.2.3 at Tue Jul 13 19:52:43 2021 +command: /opt/projects/json2python-models/venv/bin/json2models -m Actions ./starter-workflows/ci/*.yml -s flat -f pydantic -i yaml --dkf env with jobs +""" +from pydantic import BaseModel, Field +from typing import Dict, List, Optional, Union +from typing_extensions import Literal + + +class Actions(BaseModel): + on: Union['On', List[Literal["push"]]] + jobs: Dict[str, 'Job'] + name: Optional[str] = None + env: Optional[Dict[str, Union[int, str]]] = {} + + +class On(BaseModel): + push: Optional['Push'] = None + pull_request: Optional['PullRequest'] = None + release: Optional['Release'] = None + schedule: Optional[List['Schedule']] = [] + workflow_dispatch: Optional[None] = None + + +class Push(BaseModel): + branches: List[Literal["$default-branch"]] + tags: Optional[List[Literal["v*.*.*"]]] = [] + + +class PullRequest(BaseModel): + branches: List[Literal["$default-branch"]] + + +class Release(BaseModel): + types: List[Literal["created", "published"]] + + +class Schedule(BaseModel): + cron: Literal["$cron-daily"] + + +class Job(BaseModel): + runson: Literal[ + "${{ matrix.os }}", "macOS-latest", "macos-latest", "ubuntu-18.04", "ubuntu-latest", "windows-latest"] = Field( + ..., alias="runs-on") + steps: List['Step'] + name: Optional[str] = None + environment: Optional[Literal["production"]] = None + outputs: Optional['Output'] = None + container: Optional['Container'] = None + needs: Optional[Literal["build"]] = None + permissions: Optional['Permission'] = None + strategy: Optional['Strategy'] = None + defaults: Optional['Default'] = None + env: Optional[Dict[str, str]] = {} + + +class Step(BaseModel): + uses: Optional[str] = None + name: Optional[str] = None + with_: Optional[Dict[str, Union[bool, float, str]]] = Field({}, alias="with") + run: Optional[str] = None + env: Optional[Dict[str, str]] = {} + workingdirectory: Optional[str] = Field(None, alias="working-directory") + id_: Optional[Literal[ + "build-image", "composer-cache", "deploy-and-expose", "image-build", "login-ecr", "meta", "push-to-registry", "task-def"]] = Field( + None, alias="id") + if_: Optional[str] = Field(None, alias="if") + shell: Optional[Literal["Rscript {0}"]] = None + + +class Output(BaseModel): + route: str = Field(..., alias="ROUTE") + selector: str = Field(..., alias="SELECTOR") + + +class Container(BaseModel): + image: Literal["crystallang/crystal", "erlang:22.0.7"] + + +class Permission(BaseModel): + contents: Literal["read"] + packages: Literal["write"] + + +class Strategy(BaseModel): + matrix: Optional['Matrix'] = None + maxparallel: Optional[int] = Field(None, alias="max-parallel") + failfast: Optional[bool] = Field(None, alias="fail-fast") + + +class Matrix(BaseModel): + rversion: Optional[List[float]] = Field([], alias="r-version") + pythonversion: Optional[List[float]] = Field([], alias="python-version") + deno: Optional[List[Literal["canary", "v1.x"]]] = [] + os: Optional[List[Literal["macOS-latest", "ubuntu-latest", "windows-latest"]]] = [] + rubyversion: Optional[List[float]] = Field([], alias="ruby-version") + nodeversion: Optional[List[Literal["12.x", "14.x", "16.x"]]] = Field([], alias="node-version") + configuration: Optional[List[Literal["Debug", "Release"]]] = [] + + +class Default(BaseModel): + run: 'Run' + + +class Run(BaseModel): + shell: Literal["bash"] +``` + +

+ ## Installation | **Be ware**: this project supports only `python3.7` and higher. | @@ -315,24 +438,33 @@ json2models -m Car car_*.json -f attrs > car.py Arguments: * `-h`, `--help` - Show help message and exit - -* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern. `*`, `**` or `?` patterns symbols are supported. + +* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern. `*`, `**` or `?` patterns symbols + are supported. * **Format**: `-m [ ...]` * **Example**: `-m Car audi.json reno.json` or `-m Car audi.json -m Car reno.json` (results will be the same) - -* `-l`, `--list` - Like `-m` but given json file should contain list of model data (dataset). - If this file contains dict with nested list than you can pass `` to lookup. - Deep lookups are supported by dot-separated path. If no lookup needed pass `-` as ``. + +* `-l`, `--list` - Like `-m` but given json file should contain list of model data (dataset). If this file contains dict + with nested list than you can pass `` to lookup. Deep lookups are supported by dot-separated path. If no + lookup needed pass `-` as ``. * **Format**: `-l ` * **Example**: `-l Car - cars.json -l Person fetch_results.items.persons result.json` - * **Note**: Models names under this arguments should be unique. - + * **Note**: Models names under these arguments should be unique. + +* `-i`, `--input-format` - Input file format (parser). Default is JSON parser. Yaml parser requires PyYaml or + ruamel.yaml to be installed. Ini parser uses + builtin [configparser](https://docs.python.org/3/library/configparser.html). To implement new one - add new method + to `cli.FileLoaders` (and create pull request :) ) + * **Format**: `-i {json, yaml, ini}` + * **Example**: `-i yaml` + * **Default**: `-i json` + * `-o`, `--output` - Output file * **Format**: `-o ` * **Example**: `-o car_model.py` - -* `-f`, `--framework` - Model framework for which python code is generated. - `base` (default) mean no framework so code will be generated without any decorators and additional meta-data. + +* `-f`, `--framework` - Model framework for which python code is generated. + `base` (default) mean no framework so code will be generated without any decorators and additional meta-data. * **Format**: `-f {base, pydantic, attrs, dataclasses, custom}` * **Example**: `-f pydantic` * **Default**: `-f base` diff --git a/json_to_models/cli.py b/json_to_models/cli.py index 573ea98..20222c1 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -12,10 +12,10 @@ from typing import Any, Callable, Dict, Generator, Iterable, List, Tuple, Type, Union try: - import yaml + import ruamel.yaml as yaml except ImportError: try: - import ruamel.yaml as yaml + import yaml except ImportError: yaml = None @@ -268,7 +268,7 @@ def _create_argparser(cls) -> argparse.ArgumentParser: ) parser.add_argument( "-i", "--input-format", - metavar="FORMAT", default="json", + default="json", choices=['json', 'yaml', 'ini'], help="Input files parser ('PyYaml' is required to parse yaml files)\n\n" ) diff --git a/json_to_models/generator.py b/json_to_models/generator.py index 603fc5c..5184737 100644 --- a/json_to_models/generator.py +++ b/json_to_models/generator.py @@ -56,6 +56,11 @@ def _convert(self, data: dict): """ fields = dict() for key, value in data.items(): + if not isinstance(key, str): + raise TypeError(f'You probably using some not JSON-compatible parser and have some {type(key)} as dict key. ' + f'This is not supported.\n' + f'Context: {data}\n' + f'(If you parsing yaml try to replace PyYaml with ruamel.yaml)') convert_dict = key not in self.dict_keys_fields fields[key] = self._detect_type(value, convert_dict) return fields diff --git a/setup.py b/setup.py index af89a0d..6362a1f 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,6 @@ def run_tests(self): }, install_requires=required, cmdclass={"test": PyTest}, - tests_require=["pytest>=4.4.0", "pytest-xdist", "requests", "attrs", "pydantic>=1.3", "PyYaml"], + tests_require=["pytest>=4.4.0", "pytest-xdist", "requests", "attrs", "pydantic>=1.3", "ruamel.yaml"], data_files=[('', ['requirements.txt', 'pytest.ini', '.coveragerc', 'LICENSE', 'README.md', 'CHANGELOG.md'])] ) diff --git a/test/test_generator/test_detect_type.py b/test/test_generator/test_detect_type.py index b2398fe..2f11c86 100644 --- a/test/test_generator/test_detect_type.py +++ b/test/test_generator/test_detect_type.py @@ -28,7 +28,7 @@ pytest.param("1.0", FloatString, id="float_str"), pytest.param("true", BooleanString, id="bool_str"), pytest.param({"test_dict_field_a": 1, "test_dict_field_b": "a"}, DDict(DUnion(int, StringLiteral({"a"}))), id="simple_dict"), - pytest.param({}, DDict(Unknown)) + pytest.param({}, DDict(Unknown), id="empty_dict") ] test_dict = {param.id: param.values[0] for param in test_data}