From e69b3b445e17b776a0c501766223aad4e2f0524b Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 14 Jul 2024 00:50:18 -0400 Subject: [PATCH 1/7] feat: add command line interface Signed-off-by: Jinzhe Zeng --- dargs/__main__.py | 6 +++ dargs/_test.py | 21 +++++++++++ dargs/check.py | 32 ++++++++++++++++ dargs/cli.py | 78 +++++++++++++++++++++++++++++++++++++++ dargs/sphinx.py | 14 ------- docs/cli.rst | 9 +++++ docs/index.rst | 1 + docs/requirements.txt | 1 + docs/sphinx.rst | 14 +++---- pyproject.toml | 3 ++ tests/test_arguments.json | 4 ++ tests/test_cli.py | 12 ++++++ 12 files changed, 174 insertions(+), 21 deletions(-) create mode 100644 dargs/__main__.py create mode 100644 dargs/_test.py create mode 100644 dargs/check.py create mode 100644 dargs/cli.py create mode 100644 docs/cli.rst create mode 100644 tests/test_arguments.json create mode 100644 tests/test_cli.py diff --git a/dargs/__main__.py b/dargs/__main__.py new file mode 100644 index 0000000..83ac704 --- /dev/null +++ b/dargs/__main__.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +from dargs.cli import main + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/dargs/_test.py b/dargs/_test.py new file mode 100644 index 0000000..8ae7c4a --- /dev/null +++ b/dargs/_test.py @@ -0,0 +1,21 @@ +from typing import List +from dargs.dargs import Argument + + +def test_arguments() -> list[Argument]: + """Returns a list of arguments.""" + return [ + Argument(name="test1", dtype=int, doc="Argument 1"), + Argument(name="test2", dtype=[float, None], doc="Argument 2"), + Argument( + name="test3", + dtype=List[str], + default=["test"], + optional=True, + doc="Argument 3", + ), + ] + +__all__ = [ + "test_arguments", +] diff --git a/dargs/check.py b/dargs/check.py new file mode 100644 index 0000000..d6cc35d --- /dev/null +++ b/dargs/check.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from dargs.dargs import Argument + + +def check( + arginfo: Argument | list[Argument] | tuple[Argument, ...], + data: dict, + strict: bool = True, +) -> None: + """Check and normalize input data. + + Parameters + ---------- + arginfo : Union[Argument, List[Argument], Tuple[Argument, ...]] + Argument object + data : dict + data to check + strict : bool, optional + If True, raise an error if the key is not pre-defined, by default True + + Returns + ------- + dict + normalized data + """ + if isinstance(arginfo, (list, tuple)): + arginfo = Argument("base", dtype=dict, sub_fields=arginfo) + + data = arginfo.normalize_value(data, trim_pattern="_*") + arginfo.check_value(data, strict=strict) + return data diff --git a/dargs/cli.py b/dargs/cli.py new file mode 100644 index 0000000..6ac6fbd --- /dev/null +++ b/dargs/cli.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import argparse +import json +import sys + +from dargs._version import __version__ +from dargs.check import check + + +def main_parser() -> argparse.ArgumentParser: + """Create the main parser for the command line interface. + + Returns + ------- + argparse.ArgumentParser + The main parser + """ + parser = argparse.ArgumentParser(description='dargs: Argument checking for Python programs') + subparsers = parser.add_subparsers(help="Sub-commands") + parser_check = subparsers.add_parser("check", help="Check a JSON file against an Argument") + parser_check.add_argument("func", type=str, help="Function that returns an Argument object. E.g., `dargs._test.test_arguments`") + parser_check.add_argument("jdata", type=argparse.FileType('r'), + default=sys.stdin, + help="Path to the JSON file. If not given, read from stdin.", ) + parser_check.add_argument("--no-strict", action="store_false", dest="strict", help="Do not raise an error if the key is not pre-defined") + parser_check.set_defaults(entrypoint=check_cli) + + # --version + parser.add_argument("--version", action="version", version=__version__) + return parser + +def main(): + """Main entry point for the command line interface.""" + parser = main_parser() + args = parser.parse_args() + + args.entrypoint(**vars(args)) + + +def check_cli(*, + func: str, + jdata: argparse.FileType, + strict: bool, + **kwargs, +) -> None: + """Normalize and check input data. + + Parameters + ---------- + func : str + Function that returns an Argument object. E.g., `dargs._test.test_arguments` + jdata : argparse.FileType + File object that contains the JSON data + strict : bool + If True, raise an error if the key is not pre-defined + + Returns + ------- + dict + normalized data + """ + module_name, attr_name = func.rsplit(".", 1) + try: + mod = __import__(module_name, globals(), locals(), [attr_name]) + except ImportError: + raise RuntimeError( + f'Failed to import "{attr_name}" from "{module_name}".\n{sys.exc_info()[1]}' + ) + + if not hasattr(mod, attr_name): + raise RuntimeError( + f'Module "{module_name}" has no attribute "{attr_name}"' + ) + func = getattr(mod, attr_name) + arginfo = func() + data = json.load(jdata) + return check(arginfo, data, strict=strict) diff --git a/dargs/sphinx.py b/dargs/sphinx.py index ac86f3e..928dac0 100644 --- a/dargs/sphinx.py +++ b/dargs/sphinx.py @@ -218,17 +218,3 @@ def _test_argument() -> Argument: ], ) - -def _test_arguments() -> list[Argument]: - """Returns a list of arguments.""" - return [ - Argument(name="test1", dtype=int, doc="Argument 1"), - Argument(name="test2", dtype=[float, None], doc="Argument 2"), - Argument( - name="test3", - dtype=List[str], - default=["test"], - optional=True, - doc="Argument 3", - ), - ] diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 0000000..dab9742 --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,9 @@ +.. _cli: + +Command line interface +====================== + +.. argparse:: + :module: dargs.cli + :func: main_parser + :prog: dargs diff --git a/docs/index.rst b/docs/index.rst index 83b72a3..cdf7bd9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ Welcome to dargs's documentation! :caption: Contents: intro + cli sphinx dpgui nb diff --git a/docs/requirements.txt b/docs/requirements.txt index d2c7bc5..63c515c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,3 +3,4 @@ numpydoc deepmodeling_sphinx>=0.1.1 myst-nb sphinx_rtd_theme +sphinx-argparse diff --git a/docs/sphinx.rst b/docs/sphinx.rst index ee16d89..2521beb 100644 --- a/docs/sphinx.rst +++ b/docs/sphinx.rst @@ -15,20 +15,20 @@ Then `dargs` directive will be enabled: .. code-block:: rst .. dargs:: - :module: dargs.sphinx - :func: _test_argument + :module: dargs._test + :func: test_argument -where `_test_argument` returns an :class:`Argument `. The documentation will be rendered as: +where `test_argument` returns an :class:`Argument `. The documentation will be rendered as: .. dargs:: - :module: dargs.sphinx - :func: _test_argument + :module: dargs._test + :func: test_argument A :class:`list` of :class:`Argument ` is also accepted. .. dargs:: - :module: dargs.sphinx - :func: _test_arguments + :module: dargs._test + :func: test_arguments Cross-referencing Arguments --------------------------- diff --git a/pyproject.toml b/pyproject.toml index c4b0900..5834458 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ typecheck = [ "ipython", ] +[project.scripts] +dargs = "dargs.cli:main" + [tool.setuptools.packages.find] include = ["dargs*"] diff --git a/tests/test_arguments.json b/tests/test_arguments.json new file mode 100644 index 0000000..9c6e237 --- /dev/null +++ b/tests/test_arguments.json @@ -0,0 +1,4 @@ +{ + "test1": 1, + "test2": 2 +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..1acfbde --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,12 @@ +import subprocess +import sys +import unittest + +from pathlib import Path + +this_directory = Path(__file__).parent + +class TestCli(unittest.TestCase): + def test_check(self): + subprocess.check_call(["dargs", "check", "dargs._test.test_arguments", str(this_directory / "test_arguments.json")]) + subprocess.check_call([sys.executable, "-m", "dargs", "check", "dargs._test.test_arguments", str(this_directory / "test_arguments.json")]) \ No newline at end of file From a4df1e19b255b3a261093d2e2911dc82ad06bc3d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 04:52:32 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dargs/__main__.py | 2 +- dargs/_test.py | 4 ++++ dargs/cli.py | 44 ++++++++++++++++++++++++++------------- dargs/sphinx.py | 1 - tests/test_arguments.json | 2 +- tests/test_cli.py | 24 ++++++++++++++++++--- 6 files changed, 57 insertions(+), 20 deletions(-) diff --git a/dargs/__main__.py b/dargs/__main__.py index 83ac704..f6a9c52 100644 --- a/dargs/__main__.py +++ b/dargs/__main__.py @@ -3,4 +3,4 @@ from dargs.cli import main if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/dargs/_test.py b/dargs/_test.py index 8ae7c4a..9378270 100644 --- a/dargs/_test.py +++ b/dargs/_test.py @@ -1,4 +1,7 @@ +from __future__ import annotations + from typing import List + from dargs.dargs import Argument @@ -16,6 +19,7 @@ def test_arguments() -> list[Argument]: ), ] + __all__ = [ "test_arguments", ] diff --git a/dargs/cli.py b/dargs/cli.py index 6ac6fbd..0225765 100644 --- a/dargs/cli.py +++ b/dargs/cli.py @@ -10,35 +10,53 @@ def main_parser() -> argparse.ArgumentParser: """Create the main parser for the command line interface. - + Returns ------- argparse.ArgumentParser The main parser """ - parser = argparse.ArgumentParser(description='dargs: Argument checking for Python programs') + parser = argparse.ArgumentParser( + description="dargs: Argument checking for Python programs" + ) subparsers = parser.add_subparsers(help="Sub-commands") - parser_check = subparsers.add_parser("check", help="Check a JSON file against an Argument") - parser_check.add_argument("func", type=str, help="Function that returns an Argument object. E.g., `dargs._test.test_arguments`") - parser_check.add_argument("jdata", type=argparse.FileType('r'), - default=sys.stdin, - help="Path to the JSON file. If not given, read from stdin.", ) - parser_check.add_argument("--no-strict", action="store_false", dest="strict", help="Do not raise an error if the key is not pre-defined") + parser_check = subparsers.add_parser( + "check", help="Check a JSON file against an Argument" + ) + parser_check.add_argument( + "func", + type=str, + help="Function that returns an Argument object. E.g., `dargs._test.test_arguments`", + ) + parser_check.add_argument( + "jdata", + type=argparse.FileType("r"), + default=sys.stdin, + help="Path to the JSON file. If not given, read from stdin.", + ) + parser_check.add_argument( + "--no-strict", + action="store_false", + dest="strict", + help="Do not raise an error if the key is not pre-defined", + ) parser_check.set_defaults(entrypoint=check_cli) - + # --version parser.add_argument("--version", action="version", version=__version__) return parser + def main(): """Main entry point for the command line interface.""" parser = main_parser() args = parser.parse_args() - + args.entrypoint(**vars(args)) -def check_cli(*, +def check_cli( + *, func: str, jdata: argparse.FileType, strict: bool, @@ -69,9 +87,7 @@ def check_cli(*, ) if not hasattr(mod, attr_name): - raise RuntimeError( - f'Module "{module_name}" has no attribute "{attr_name}"' - ) + raise RuntimeError(f'Module "{module_name}" has no attribute "{attr_name}"') func = getattr(mod, attr_name) arginfo = func() data = json.load(jdata) diff --git a/dargs/sphinx.py b/dargs/sphinx.py index 928dac0..379b5cb 100644 --- a/dargs/sphinx.py +++ b/dargs/sphinx.py @@ -217,4 +217,3 @@ def _test_argument() -> Argument: ), ], ) - diff --git a/tests/test_arguments.json b/tests/test_arguments.json index 9c6e237..a3291ee 100644 --- a/tests/test_arguments.json +++ b/tests/test_arguments.json @@ -1,4 +1,4 @@ { "test1": 1, "test2": 2 -} \ No newline at end of file +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 1acfbde..8abb50a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,12 +1,30 @@ +from __future__ import annotations + import subprocess import sys import unittest - from pathlib import Path this_directory = Path(__file__).parent + class TestCli(unittest.TestCase): def test_check(self): - subprocess.check_call(["dargs", "check", "dargs._test.test_arguments", str(this_directory / "test_arguments.json")]) - subprocess.check_call([sys.executable, "-m", "dargs", "check", "dargs._test.test_arguments", str(this_directory / "test_arguments.json")]) \ No newline at end of file + subprocess.check_call( + [ + "dargs", + "check", + "dargs._test.test_arguments", + str(this_directory / "test_arguments.json"), + ] + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "dargs", + "check", + "dargs._test.test_arguments", + str(this_directory / "test_arguments.json"), + ] + ) From 6d7e06849289dda39331cbd04ac7dae3bd3607ee Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 14 Jul 2024 00:52:37 -0400 Subject: [PATCH 3/7] fix typing Signed-off-by: Jinzhe Zeng --- dargs/check.py | 2 +- dargs/cli.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dargs/check.py b/dargs/check.py index d6cc35d..c9809dc 100644 --- a/dargs/check.py +++ b/dargs/check.py @@ -7,7 +7,7 @@ def check( arginfo: Argument | list[Argument] | tuple[Argument, ...], data: dict, strict: bool = True, -) -> None: +) -> dict: """Check and normalize input data. Parameters diff --git a/dargs/cli.py b/dargs/cli.py index 0225765..82043f4 100644 --- a/dargs/cli.py +++ b/dargs/cli.py @@ -3,6 +3,7 @@ import argparse import json import sys +from typing import IO from dargs._version import __version__ from dargs.check import check @@ -58,7 +59,7 @@ def main(): def check_cli( *, func: str, - jdata: argparse.FileType, + jdata: IO, strict: bool, **kwargs, ) -> None: @@ -68,7 +69,7 @@ def check_cli( ---------- func : str Function that returns an Argument object. E.g., `dargs._test.test_arguments` - jdata : argparse.FileType + jdata : IO File object that contains the JSON data strict : bool If True, raise an error if the key is not pre-defined @@ -88,7 +89,7 @@ def check_cli( if not hasattr(mod, attr_name): raise RuntimeError(f'Module "{module_name}" has no attribute "{attr_name}"') - func = getattr(mod, attr_name) - arginfo = func() + func_obj = getattr(mod, attr_name) + arginfo = func_obj() data = json.load(jdata) - return check(arginfo, data, strict=strict) + check(arginfo, data, strict=strict) From 914aa043da64a990ffa2ab3cff715b30187f0615 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 14 Jul 2024 01:01:28 -0400 Subject: [PATCH 4/7] add trim_pattern option Signed-off-by: Jinzhe Zeng --- dargs/check.py | 5 ++++- dargs/cli.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dargs/check.py b/dargs/check.py index c9809dc..2711927 100644 --- a/dargs/check.py +++ b/dargs/check.py @@ -7,6 +7,7 @@ def check( arginfo: Argument | list[Argument] | tuple[Argument, ...], data: dict, strict: bool = True, + trim_pattern: str = "_*", ) -> dict: """Check and normalize input data. @@ -18,6 +19,8 @@ def check( data to check strict : bool, optional If True, raise an error if the key is not pre-defined, by default True + trim_pattern : str, optional + Pattern to trim the key, by default "_*" Returns ------- @@ -27,6 +30,6 @@ def check( if isinstance(arginfo, (list, tuple)): arginfo = Argument("base", dtype=dict, sub_fields=arginfo) - data = arginfo.normalize_value(data, trim_pattern="_*") + data = arginfo.normalize_value(data, trim_pattern=trim_pattern) arginfo.check_value(data, strict=strict) return data diff --git a/dargs/cli.py b/dargs/cli.py index 82043f4..b721704 100644 --- a/dargs/cli.py +++ b/dargs/cli.py @@ -22,7 +22,9 @@ def main_parser() -> argparse.ArgumentParser: ) subparsers = parser.add_subparsers(help="Sub-commands") parser_check = subparsers.add_parser( - "check", help="Check a JSON file against an Argument" + "check", + help="Check a JSON file against an Argument", + epilog="Example: dargs check dargs._test.test_arguments test_arguments.json", ) parser_check.add_argument( "func", @@ -41,6 +43,12 @@ def main_parser() -> argparse.ArgumentParser: dest="strict", help="Do not raise an error if the key is not pre-defined", ) + parser_check.add_argument( + "--trim-pattern", + type=str, + default="_*", + help="Pattern to trim the key", + ) parser_check.set_defaults(entrypoint=check_cli) # --version From 71e1a3f83ec171e0d2fbd679b0f5914869fdd2b3 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 14 Jul 2024 03:16:16 -0400 Subject: [PATCH 5/7] B904 Signed-off-by: Jinzhe Zeng --- dargs/cli.py | 4 ++-- dargs/sphinx.py | 4 ++-- pyproject.toml | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dargs/cli.py b/dargs/cli.py index b721704..e2f3b5f 100644 --- a/dargs/cli.py +++ b/dargs/cli.py @@ -90,10 +90,10 @@ def check_cli( module_name, attr_name = func.rsplit(".", 1) try: mod = __import__(module_name, globals(), locals(), [attr_name]) - except ImportError: + except ImportError as e: raise RuntimeError( f'Failed to import "{attr_name}" from "{module_name}".\n{sys.exc_info()[1]}' - ) + ) from e if not hasattr(mod, attr_name): raise RuntimeError(f'Module "{module_name}" has no attribute "{attr_name}"') diff --git a/dargs/sphinx.py b/dargs/sphinx.py index 379b5cb..8eb0b9e 100644 --- a/dargs/sphinx.py +++ b/dargs/sphinx.py @@ -57,10 +57,10 @@ def run(self): try: mod = __import__(module_name, globals(), locals(), [attr_name]) - except ImportError: + except ImportError as e: raise self.error( f'Failed to import "{attr_name}" from "{module_name}".\n{sys.exc_info()[1]}' - ) + ) from e if not hasattr(mod, attr_name): raise self.error( diff --git a/pyproject.toml b/pyproject.toml index 5834458..1370c44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ select = [ "RUF", # ruff "I", # isort "TCH", # flake8-type-checking + "B904", # raise-without-from-inside-except ] ignore = [ From a8d538cd3eae065bd33665f53feb9c3d81588cf4 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 14 Jul 2024 03:21:55 -0400 Subject: [PATCH 6/7] add `sphinxarg.ext` to sphinx `extensions` Signed-off-by: Jinzhe Zeng --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 34270de..d3588bd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,6 +49,7 @@ "numpydoc", "myst_nb", "dargs.sphinx", + "sphinxarg.ext", ] # Add any paths that contain templates here, relative to this directory. From 4eda21a32c1773cdf3aa2a890f8d3720f4f7ea1e Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 14 Jul 2024 05:34:03 -0400 Subject: [PATCH 7/7] support multiple inputs Signed-off-by: Jinzhe Zeng --- dargs/cli.py | 16 ++++++++++------ tests/test_cli.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/dargs/cli.py b/dargs/cli.py index e2f3b5f..045e87b 100644 --- a/dargs/cli.py +++ b/dargs/cli.py @@ -24,17 +24,20 @@ def main_parser() -> argparse.ArgumentParser: parser_check = subparsers.add_parser( "check", help="Check a JSON file against an Argument", - epilog="Example: dargs check dargs._test.test_arguments test_arguments.json", + epilog="Example: dargs check -f dargs._test.test_arguments test_arguments.json", ) parser_check.add_argument( - "func", + "-f", + "--func", type=str, help="Function that returns an Argument object. E.g., `dargs._test.test_arguments`", + required=True, ) parser_check.add_argument( "jdata", type=argparse.FileType("r"), - default=sys.stdin, + default=[sys.stdin], + nargs="*", help="Path to the JSON file. If not given, read from stdin.", ) parser_check.add_argument( @@ -67,7 +70,7 @@ def main(): def check_cli( *, func: str, - jdata: IO, + jdata: list[IO], strict: bool, **kwargs, ) -> None: @@ -99,5 +102,6 @@ def check_cli( raise RuntimeError(f'Module "{module_name}" has no attribute "{attr_name}"') func_obj = getattr(mod, attr_name) arginfo = func_obj() - data = json.load(jdata) - check(arginfo, data, strict=strict) + for jj in jdata: + data = json.load(jj) + check(arginfo, data, strict=strict) diff --git a/tests/test_cli.py b/tests/test_cli.py index 8abb50a..a3a76f0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,8 +14,10 @@ def test_check(self): [ "dargs", "check", + "-f", "dargs._test.test_arguments", str(this_directory / "test_arguments.json"), + str(this_directory / "test_arguments.json"), ] ) subprocess.check_call( @@ -24,7 +26,19 @@ def test_check(self): "-m", "dargs", "check", + "-f", "dargs._test.test_arguments", str(this_directory / "test_arguments.json"), + str(this_directory / "test_arguments.json"), ] ) + with (this_directory / "test_arguments.json").open() as f: + subprocess.check_call( + [ + "dargs", + "check", + "-f", + "dargs._test.test_arguments", + ], + stdin=f, + )