Merge bbcbe35 into 74750f1

README.md

@@ -103,13 +103,13 @@ driver_standings.json
 ```
 
 ```
-json2models -f pydantic -l DriverStandings - driver_standings.json
+json2models -f pydantic -m DriverStandings driver_standings.json
 ```
 
 ```python
 r"""
 generated by json2python-models v0.2.0 at Mon May  4 17:46:30 2020
-command: /opt/projects/json2python-models/venv/bin/json2models -f pydantic -s flat -l DriverStandings - driver_standings.json
+command: /opt/projects/json2python-models/venv/bin/json2models -f pydantic -s flat -m DriverStandings driver_standings.json
 """
 from pydantic import BaseModel, Field
 from typing import List
@@ -469,27 +469,25 @@ json2models -m Car car_*.json -f attrs > car.py
 ```
 
 Arguments:
-* `-h`, `--help` - Show help message and exit
 
-* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern. `*`,  `**` or `?` patterns symbols
-  are supported.
-    * **Format**: `-m <Model name> [<JSON files> ...]`
-    * **Example**: `-m Car audi.json reno.json` or `-m Car audi.json -m Car reno.json` (results will be the same)
+* `-h`, `--help` - Show help message and exit
 
-* `-l`, `--list` - Like `-m` but given json file should contain list of model data (dataset). If this file contains dict
-  with nested list than you can pass `<JSON key>` to lookup. Deep lookups are supported by dot-separated path. If no
-  lookup needed pass `-` as `<JSON key>`.
-    * **Format**: `-l <Model name> <JSON key> <JSON file>`
-    * **Example**: `-l Car - cars.json -l Person fetch_results.items.persons result.json`
-    * **Note**: Models names under these arguments should be unique.
+* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern.
+  `*`,  `**` or `?` patterns symbols are supported.
+  JSON data could be an array of models or single model.
+  If this file contains dict with nested list than you can pass
+  <JSON lookup>. Deep lookups are supported by dot-separated path.
+  If no lookup needed pass '-' as <JSON lookup> (default)
+  * **Format**: `-m <Model name> [<JSON lookup>] <File path or pattern>`
+  * **Example**: `-m Car audi.json -m Car results reno.json`
 
 * `-i`, `--input-format` - Input file format (parser). Default is JSON parser. Yaml parser requires PyYaml or
   ruamel.yaml to be installed. Ini parser uses
   builtin [configparser](https://docs.python.org/3/library/configparser.html). To implement new one - add new method
   to `cli.FileLoaders` (and create pull request :) )
-    * **Format**: `-i {json, yaml, ini}`
-    * **Example**: `-i yaml`
-    * **Default**: `-i json`
+  * **Format**: `-i {json, yaml, ini}`
+  * **Example**: `-i yaml`
+  * **Default**: `-i json`
 
 * `-o`, `--output` - Output file
     * **Format**: `-o <FILE>`

json_to_models/cli.py

@@ -81,14 +81,6 @@ def parse_args(self, args: List[str] = None):
         namespace = parser.parse_args(args)
 
         # Extract args
-        models: List[Tuple[str, Iterable[Path]]] = [
-            (model_name, itertools.chain(*map(_process_path, paths)))
-            for model_name, *paths in namespace.model or ()
-        ]
-        models_lists: List[Tuple[str, Tuple[str, Path]]] = [
-            (model_name, (lookup, Path(path)))
-            for model_name, lookup, path in namespace.list or ()
-        ]
         parser = getattr(FileLoaders, namespace.input_format)
         self.output_file = namespace.output
         self.enable_datetime = namespace.datetime
@@ -104,8 +96,8 @@ def parse_args(self, args: List[str] = None):
         dict_keys_fields: List[str] = namespace.dict_keys_fields
         preamble: str = namespace.preamble
 
-        self.validate(models_lists, merge_policy, framework, code_generator)
-        self.setup_models_data(models, models_lists, parser)
+        self.setup_models_data(namespace.model or (), namespace.list or (), parser)
+        self.validate(merge_policy, framework, code_generator)
         self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs_raw,
                       dict_keys_regex, dict_keys_fields, disable_unicode_conversion, preamble)
 
@@ -144,20 +136,15 @@ def version_string(self):
             '"""\n'
         )
 
-    def validate(self, models_list, merge_policy, framework, code_generator):
+    def validate(self, merge_policy, framework, code_generator):
         """
         Validate parsed args
 
-        :param models_list: List of pairs (model name, list of lookup expr and filesystem path)
         :param merge_policy: List of merge policies. Each merge policy is either string or string and policy arguments
         :param framework: Framework name (predefined code generator)
         :param code_generator: Code generator import string
         :return:
         """
-        names = {name for name, _ in models_list}
-        if len(names) != len(models_list):
-            raise ValueError("Model names under -l flag should be unique")
-
         for m in merge_policy:
             if isinstance(m, list):
                 if m[0] not in self.MODEL_CMP_MAPPING:
@@ -172,23 +159,33 @@ def validate(self, models_list, merge_policy, framework, code_generator):
 
     def setup_models_data(
             self,
-            models: Iterable[Tuple[str, Iterable[Path]]],
-            models_lists: Iterable[Tuple[str, Tuple[str, Path]]],
+            models: Iterable[Union[
+                Tuple[str, str],
+                Tuple[str, str, str],
+            ]],
+            models_lists: Iterable[Tuple[str, str, str]],
             parser: 'FileLoaders.T'
     ):
         """
         Initialize lazy loaders for models data
         """
-        models_dict: Dict[str, List[Iterable[dict]]] = defaultdict(list)
-        for model_name, paths in models:
-            models_dict[model_name].append(parser(path) for path in paths)
-        for model_name, (lookup, path) in models_lists:
-            models_dict[model_name].append(iter_json_file(parser(path), lookup))
+        models_dict: Dict[str, List[dict]] = defaultdict(list)
+
+        models = list(models) + list(models_lists)
+        for model_tuple in models:
+            if len(model_tuple) == 2:
+                model_name, path_raw = model_tuple
+                lookup = '-'
+            elif len(model_tuple) == 3:
+                model_name, lookup, path_raw = model_tuple
+            else:
+                raise RuntimeError('`--model` argument should contain exactly 2 or 3 strings')
 
-        self.models_data = {
-            model_name: itertools.chain(*list_of_gen)
-            for model_name, list_of_gen in models_dict.items()
-        }
+            for real_path in process_path(path_raw):
+                iterator = iter_json_file(parser(real_path), lookup)
+                models_dict[model_name].extend(iterator)
+
+        self.models_data = models_dict
 
     def set_args(
             self,
@@ -257,20 +254,13 @@ def _create_argparser(cls) -> argparse.ArgumentParser:
 
         parser.add_argument(
             "-m", "--model",
-            nargs="+", action="append", metavar=("<Model name>", "<JSON files>"),
+            nargs="+", action="append", metavar=("<Model name> [<JSON lookup>] <File path or pattern>", ""),
             help="Model name and its JSON data as path or unix-like path pattern.\n"
                  "'*',  '**' or '?' patterns symbols are supported.\n\n"
-        )
-        parser.add_argument(
-            "-l", "--list",
-            nargs=3, action="append", metavar=("<Model name>", "<JSON key>", "<JSON file>"),
-            help="Like -m but given json file should contain list of model data.\n"
+                 "JSON data could be array of models or single model\n\n"
                  "If this file contains dict with nested list than you can pass\n"
-                 "<JSON key> to lookup. Deep lookups are supported by dot-separated path.\n"
-                 "If no lookup needed pass '-' as <JSON key>\n\n"
-
-                 "I.e. for file that contains dict {\"a\": {\"b\": [model_data, ...]}} you should\n"
-                 "pass 'a.b' as <JSON key>.\n\n"
+                 "<JSON lookup>. Deep lookups are supported by dot-separated path.\n"
+                 "If no lookup needed pass '-' as <JSON lookup> (default)\n\n"
         )
         parser.add_argument(
             "-i", "--input-format",
@@ -377,6 +367,11 @@ def _create_argparser(cls) -> argparse.ArgumentParser:
             type=str,
             help="Code to insert into the generated file after the imports and before the list of classes\n\n"
         )
+        parser.add_argument(
+            "-l", "--list",
+            nargs=3, action="append", metavar=("<Model name>", "<JSON lookup>", "<JSON file>"),
+            help="DEPRECATED, use --model argument instead"
+        )
 
         return parser
 
@@ -395,27 +390,6 @@ def main():
     print(cli.run())
 
 
-def path_split(path: str) -> List[str]:
-    """
-    Split path into list of components
-
-    :param path: string path
-    :return: List of files/patterns
-    """
-    folders = []
-    while True:
-        path, folder = os.path.split(path)
-
-        if folder:
-            folders.append(folder)
-        else:
-            if path:
-                folders.append(path)
-            break
-    folders.reverse()
-    return folders
-
-
 class FileLoaders:
     T = Callable[[Path], Union[dict, list]]
 
@@ -442,7 +416,7 @@ def ini(path: Path) -> dict:
 
 def dict_lookup(d: Union[dict, list], lookup: str) -> Union[dict, list]:
     """
-    Extract nested dictionary value from key path.
+    Extract nested value from key path.
     If lookup is "-" returns dict as is.
 
     :param d: Nested dict
@@ -460,25 +434,26 @@ def dict_lookup(d: Union[dict, list], lookup: str) -> Union[dict, list]:
 
 def iter_json_file(data: Union[dict, list], lookup: str) -> Generator[Union[dict, list], Any, None]:
     """
-    Loads given 'path' file, perform lookup and return generator over json list.
+    Perform lookup and return generator over json list.
     Does not open file until iteration is started.
 
-    :param path: File Path instance
+    :param data: JSON data
     :param lookup: Dot separated lookup path
-    :return:
+    :return: Generator of the model data
     """
-    l = dict_lookup(data, lookup)
-    assert isinstance(l, list), f"Dict lookup return {type(l)} but list is expected, check your lookup path"
-    yield from l
+    item = dict_lookup(data, lookup)
+    if isinstance(item, list):
+        yield from item
+    elif isinstance(item, dict):
+        yield item
+    else:
+        raise TypeError(f'dict or list is expected at {lookup if lookup != "-" else "JSON root"}, not {type(item)}')
 
 
-def _process_path(path: str) -> Iterable[Path]:
+def process_path(path: str) -> Iterable[Path]:
     """
     Convert path pattern into path iterable.
     If non-pattern path is given return tuple of one element: (path,)
-
-    :param path:
-    :return:
     """
     split_path = path_split(path)
     clean_path = list(itertools.takewhile(
@@ -502,3 +477,24 @@ def _process_path(path: str) -> Iterable[Path]:
         return path.glob(pattern_path)
     else:
         return path,
+
+
+def path_split(path: str) -> List[str]:
+    """
+    Split path into list of components
+
+    :param path: string path
+    :return: List of files/patterns
+    """
+    folders = []
+    while True:
+        path, folder = os.path.split(path)
+
+        if folder:
+            folders.append(folder)
+        else:
+            if path:
+                folders.append(path)
+            break
+    folders.reverse()
+    return folders

json_to_models/generator.py

@@ -15,7 +15,7 @@
     StringSerializable,
     StringSerializableRegistry,
     Unknown,
-    registry
+    registry,
 )
 
 _static_types = {float, bool, int}
@@ -46,8 +46,6 @@ def generate(self, *data_variants: dict) -> dict:
         """
         Convert given list of data variants to metadata dict
         """
-        if isinstance(data_variants[0], list):
-            data_variants = [item for sublist in data_variants for item in sublist]
         fields_sets = [self._convert(data) for data in data_variants]
         fields = self.merge_field_sets(fields_sets)
         return self.optimize_type(fields)

test/test_cli/test_script.py

@@ -52,16 +52,20 @@ def test_help():
 
 
 test_commands = [
-    pytest.param(f"""{executable} -l Photo items "{test_data_path / 'photos.json'}" """, id="list1"),
-    pytest.param(f"""{executable} -l User - "{test_data_path / 'users.json'}" """, id="list2"),
+    pytest.param(f"""{executable} -m Photo items "{test_data_path / 'photos.json'}" """, id="list1"),
+    pytest.param(f"""{executable} -l Photo items "{test_data_path / 'photos.json'}" """, id="list1_legacy"),
+    pytest.param(f"""{executable} -m User "{test_data_path / 'users.json'}" """, id="list2"),
+    pytest.param(f"""{executable} -l User - "{test_data_path / 'users.json'}" """, id="list2_legacy"),
     pytest.param(f"""{executable} -m Photos "{test_data_path / 'photos.json'}" """, id="model1"),
+    pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" \
+                              -m Model - "{test_data_path / 'users.json'}" """, id="duplicate_name"),
 
-    pytest.param(f"""{executable} -l Photo items "{test_data_path / 'photos.json'}" \
+    pytest.param(f"""{executable} -m Photo items "{test_data_path / 'photos.json'}" \
                                   -m Photos "{test_data_path / 'photos.json'}" """,
                  id="list1_model1"),
 
-    pytest.param(f"""{executable} -l Photo items "{test_data_path / 'photos.json'}" \
-                                  -l User - "{test_data_path / 'users.json'}" """,
+    pytest.param(f"""{executable} -m Photo items "{test_data_path / 'photos.json'}" \
+                                  -m User "{test_data_path / 'users.json'}" """,
                  id="list1_list2"),
 
     pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files""",
@@ -75,7 +79,7 @@ def test_help():
     pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --datetime --strings-converters""",
                  id="gists_strings_converters"),
 
-    pytest.param(f"""{executable} -l User - "{test_data_path / 'users.json'}" --strings-converters""",
+    pytest.param(f"""{executable} -m User "{test_data_path / 'users.json'}" --strings-converters""",
                  id="users_strings_converters"),
     pytest.param(f"""{executable} -m SomeUnicode "{test_data_path / 'unicode.json'}" """,
                  id="convert_unicode"),
@@ -216,20 +220,22 @@ def trim_header(line_string):
 
 
 wrong_arguments_commands = [
-    pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" \
-                                  -l Model - "{test_data_path / 'users.json'}" """, id="duplicate_name"),
-    pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" --merge unknown""",
+    pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --merge unknown""",
                  id="wrong_merge_policy"),
-    pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" --merge unknown_10""",
+    pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --merge unknown_10""",
                  id="wrong_merge_policy"),
-    pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" -f custom""",
+    pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" -f custom""",
                  id="custom_model_generator_without_class_link"),
-    pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" --code-generator test""",
+    pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --code-generator test""",
                  id="class_link_without_custom_model_generator_enabled"),
+    pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" another_arg --code-generator test""",
+                 id="4_args_model"),
+    pytest.param(f"""{executable} -m Model total "{test_data_path / 'photos.json'}" --code-generator test""",
+                 id="non_dict_or_list_data"),
 ]
 
 
-@pytest.mark.xfail
+@pytest.mark.xfail(strict=True)
 @pytest.mark.parametrize("command", wrong_arguments_commands)
 def test_wrong_arguments(command):
     print("Command:", command)

test/test_cli/test_utils.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from json_to_models.cli import _process_path, dict_lookup, iter_json_file, path_split
+from json_to_models.cli import dict_lookup, iter_json_file, path_split, process_path
 from json_to_models.utils import convert_args
 
 echo = lambda *args, **kwargs: (args, kwargs)
@@ -158,5 +158,5 @@ def test_iter_json_file(value, expected):
 
 @pytest.mark.parametrize("value,expected", test_process_path_data)
 def test_process_path(value, expected):
-    result = set(str(p).replace("\\", "/") for p in _process_path(value))
+    result = set(str(p).replace("\\", "/") for p in process_path(value))
     assert result == expected, f"(in value: {value})"