From 494aecda5e017fecd40c750686aaee54c0dd7371 Mon Sep 17 00:00:00 2001 From: Eric Hutton Date: Sat, 22 Aug 2020 17:34:46 -0600 Subject: [PATCH] feat: standardize command line interface (#16) * add tomlkit to requirements * add compact generate command, toml input file * test cli on Travis * fix tests for new cli * update test data files for new cli * fix mypy errors * remove lint * add tests for 100% coverage --- .travis.yml | 6 + compaction/cli.py | 209 +++++++++++------- requirements.txt | 1 + tests/test_cli.py | 142 +++--------- tests/test_cli/compact.toml | 2 + tests/test_cli/config.yaml | 1 - .../{porosity_profile.txt => porosity.csv} | 0 tests/test_compaction.py | 67 +++--- 8 files changed, 217 insertions(+), 211 deletions(-) create mode 100644 tests/test_cli/compact.toml delete mode 100644 tests/test_cli/config.yaml rename tests/test_cli/{porosity_profile.txt => porosity.csv} (100%) diff --git a/.travis.yml b/.travis.yml index 866d6a3..94b2b2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,4 +53,10 @@ script: - pip install -r requirements-testing.txt - pip install pytest pytest-cov pytest-datadir pytest-benchmark coveralls - pytest -vvv --cov=compaction --cov-report=xml:$(pwd)/coverage.xml --mypy +- compact --help +- compact --version +- compact generate compact.toml +- compact generate porosity.csv +- mkdir example +- compact --cd=example setup run after_success: coveralls diff --git a/compaction/cli.py b/compaction/cli.py index 7ea312c..8fdb3bd 100644 --- a/compaction/cli.py +++ b/compaction/cli.py @@ -1,5 +1,7 @@ +import os import pathlib import sys +import warnings from functools import partial from io import StringIO from typing import Optional, TextIO @@ -7,7 +9,7 @@ import click import numpy as np # type: ignore import pandas # type: ignore -import yaml +import tomlkit as toml # type: ignore from .compaction import compact as _compact @@ -16,12 +18,82 @@ err = partial(click.secho, fg="red", err=True) -def load_config(file: Optional[TextIO] = None): +def _tomlkit_to_popo(d): + """Convert a tomlkit doc to plain-old-python objects. + + Examples + -------- + >>> import tomlkit + >>> from compaction.cli import _tomlkit_to_popo + + >>> contents = \"\"\" + ... [[test]] + ... int_value = 3 + ... float_value = 3.14 + ... str_value = "pi" + ... bool_value = true + ... \"\"\" + + >>> doc = tomlkit.parse(contents) + >>> doc + {'test': [{'int_value': 3, 'float_value': 3.14, 'str_value': 'pi', 'bool_value': True}]} + + >>> isinstance(doc["test"][0]["int_value"], tomlkit.items.Item) + True + >>> isinstance(doc["test"][0]["float_value"], tomlkit.items.Item) + True + >>> isinstance(doc["test"][0]["str_value"], tomlkit.items.Item) + True + + >>> popo = _tomlkit_to_popo(doc) + >>> popo + {'test': [{'int_value': 3, 'float_value': 3.14, 'str_value': 'pi', 'bool_value': True}]} + + >>> isinstance(popo["test"][0]["int_value"], tomlkit.items.Item) + False + >>> isinstance(popo["test"][0]["float_value"], tomlkit.items.Item) + False + >>> isinstance(popo["test"][0]["str_value"], tomlkit.items.Item) + False + >>> isinstance(popo["test"][0]["bool_value"], tomlkit.items.Item) + False + """ + try: + result = getattr(d, "value") + except AttributeError: + result = d + + if isinstance(result, list): + result = [_tomlkit_to_popo(x) for x in result] + elif isinstance(result, dict): + result = { + _tomlkit_to_popo(key): _tomlkit_to_popo(val) for key, val in result.items() + } + elif isinstance(result, toml.items.Integer): + result = int(result) + elif isinstance(result, toml.items.Float): + result = float(result) + elif isinstance(result, (toml.items.String, str)): + result = str(result) + elif isinstance(result, (toml.items.Bool, bool)): + result = bool(result) + else: + if not isinstance(result, (int, float, str, bool)): + warnings.warn( # pragma: no cover + "unexpected type ({0!r}) encountered when converting toml to a dict".format( + result.__class__.__name__ + ) + ) + + return result + + +def load_config(stream: Optional[TextIO] = None): """Load compaction config file. Parameters ---------- - fname : file-like, optional + stream : file-like, optional Opened config file or ``None``. If ``None``, return default values. @@ -31,15 +103,30 @@ def load_config(file: Optional[TextIO] = None): Config parameters. """ conf = { - "c": 5e-8, - "porosity_min": 0.0, - "porosity_max": 0.5, - "rho_grain": 2650.0, - "rho_void": 1000.0, + "compact" : { + "constants": { + "c": 5e-8, + "porosity_min": 0.0, + "porosity_max": 0.5, + "rho_grain": 2650.0, + "rho_void": 1000.0, + } + } } - if file is not None: - conf.update(yaml.safe_load(file)) - return conf + if stream is not None: + try: + local_params = toml.parse(stream.read())["compact"] + except KeyError: + local_params = {"constants": {}} + + try: + local_constants = local_params["constants"] + except KeyError: + local_constants = {} + + conf["compact"]["constants"].update(local_constants) + + return _tomlkit_to_popo(conf).pop("compact") def _contents_of_input_file(infile: str) -> str: @@ -52,8 +139,8 @@ def as_csv(data, header=None): return contents contents = { - "config": yaml.dump(params, default_flow_style=False), - "porosity": as_csv( + "compact.toml": toml.dumps(dict(compact=params)), + "porosity.csv": as_csv( [[100.0, 0.5], [100.0, 0.5], [100.0, 0.5]], header="Layer Thickness [m], Porosity [-]", ), @@ -62,12 +149,7 @@ def as_csv(data, header=None): return contents[infile] -def run_compaction( - src: Optional[TextIO] = None, dest: Optional[TextIO] = None, **kwds -) -> None: - src = src or sys.stdin - dest = dest or sys.stdout - +def run_compaction(src: str, dest: str, **kwds) -> None: init = pandas.read_csv(src, names=("dz", "porosity"), dtype=float, comment="#") dz_new = np.empty_like(init.dz) @@ -75,14 +157,22 @@ def run_compaction( init.dz.values, init.porosity.values, return_dz=dz_new, **kwds ) - out = pandas.DataFrame.from_dict({"dz": dz_new, "porosity": porosity_new}) - print("# Layer Thickness [m], Porosity [-]", file=dest) - out.to_csv(dest, index=False, header=False) + result = pandas.DataFrame.from_dict({"dz": dz_new, "porosity": porosity_new}) + with open(dest, "w") as fp: + print("# Layer Thickness [m], Porosity [-]", file=fp) + result.to_csv(fp, index=False, header=False) -@click.group() + +@click.group(chain=True) @click.version_option() -def compact() -> None: +@click.option( + "--cd", + default=".", + type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), + help="chage to directory, then execute", +) +def compact(cd) -> None: """Compact layers of sediment. \b @@ -90,93 +180,62 @@ def compact() -> None: Create a folder with example input files, - $ compact setup compact-example + $ mkdir compaction-example && cd compaction-example + $ compact setup Run a simulation using the examples input files, - $ compact run compact-example/porosity.csv + $ compact run """ - pass # pragma: no cover + os.chdir(cd) # pragma: no cover @compact.command() @click.version_option() @click.option("-v", "--verbose", is_flag=True, help="Emit status messages to stderr.") @click.option("--dry-run", is_flag=True, help="Do not actually run the model") -@click.option( - "--config", - type=click.Path( - exists=False, file_okay=True, dir_okay=False, readable=True, allow_dash=False - ), - default="config.yaml", - is_eager=True, - help="Read configuration from PATH.", -) -@click.argument("src", type=click.File(mode="r")) -@click.argument("dest", default="-", type=click.File(mode="w")) -def run(src: TextIO, dest: TextIO, config: str, dry_run: bool, verbose: bool) -> None: +def run(dry_run: bool, verbose: bool) -> None: """Run a simulation.""" - try: - from_stdin = src.name == "" - except AttributeError: - from_stdin = True - try: - from_stdout = dest.name == "" - except AttributeError: - from_stdout = True - - config_path = pathlib.Path(config) - if from_stdin: - rundir = pathlib.Path(".") - else: - rundir = pathlib.Path(src.name).parent.resolve() - - with open(rundir / config_path, "r") as fp: + with open("compact.toml", "r") as fp: params = load_config(fp) if verbose: - out(yaml.dump(params, default_flow_style=False)) + out(toml.dumps(params)) if dry_run: out("Nothing to do. 😴") else: - run_compaction(src, dest, **params) + run_compaction("porosity.csv", "porosity-out.csv", **params["constants"]) out("💥 Finished! 💥") - out("Output written to {0}".format("" if from_stdout else dest.name)) - - sys.exit(0) + out("Output written to {0}".format("porosity-out.csv")) @compact.command() @click.argument( - "infile", type=click.Choice(["config", "porosity"]), + "infile", type=click.Choice(["compact.toml", "porosity.csv"]), ) -def show(infile: str) -> None: +def generate(infile: str) -> None: """Show example input files.""" print(_contents_of_input_file(infile)) @compact.command() -@click.argument( - "dest", type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), -) -def setup(dest: str) -> None: +def setup() -> None: """Setup a folder of input files for a simulation.""" - folder = pathlib.Path(dest) - - files = [pathlib.Path(fname) for fname in ["porosity.csv", "config.yaml"]] + files = [pathlib.Path(fname) for fname in ["porosity.csv", "compact.toml"]] - existing_files = [folder / name for name in files if (folder / name).exists()] + existing_files = [str(file_) for file_ in files if file_.exists()] if existing_files: for name in existing_files: err( f"{name}: File exists. Either remove and then rerun or choose a different destination folder", ) else: - for fname in files: - with open(folder / fname, "w") as fp: - print(_contents_of_input_file(fname.stem), file=fp) - print(str(folder / "porosity.csv")) + for file_ in files: + with open(file_, "w") as fp: + print(_contents_of_input_file(file_.name), file=fp) + print(pathlib.Path.cwd()) - sys.exit(len(existing_files)) + if existing_files: + sys.exit(len(existing_files)) diff --git a/requirements.txt b/requirements.txt index 23eb43d..3860a29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ landlab numpy pandas pyyaml +tomlkit scipy diff --git a/tests/test_cli.py b/tests/test_cli.py index 5c38817..706aef7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -import filecmp -import os +import shutil import numpy as np # type: ignore import pandas # type: ignore import pytest # type: ignore -import yaml +import tomlkit # type: ignore from click.testing import CliRunner from numpy.testing import assert_array_almost_equal # type: ignore @@ -29,135 +28,64 @@ def test_command_line_interface(): def test_dry_run(tmpdir, datadir): with tmpdir.as_cwd(): + shutil.copy(datadir / "compact.toml", ".") + shutil.copy(datadir / "porosity.csv", ".") + runner = CliRunner() result = runner.invoke( - cli.run, - [ - "--dry-run", - "--config={0}".format(datadir / "config.yaml"), - str(datadir / "porosity_profile.txt"), - "out.txt", - ], + cli.run, ["--dry-run"], ) assert result.exit_code == 0 assert "Nothing to do" in result.output - assert not (tmpdir / "out.txt").exists() + assert not (tmpdir / "porosity-out.csv").exists() def test_verbose(tmpdir, datadir): with tmpdir.as_cwd(): + shutil.copy(datadir / "compact.toml", ".") + shutil.copy(datadir / "porosity.csv", ".") + runner = CliRunner() result = runner.invoke( - cli.run, - [ - "--verbose", - "--config={0}".format(datadir / "config.yaml"), - str(datadir / "porosity_profile.txt"), - "out.txt", - ], + cli.run, ["--verbose"], ) assert result.exit_code == 0 - assert (tmpdir / "out.txt").exists() - - conf = yaml.safe_load(os.linesep.join(result.output.splitlines()[:5])) - assert isinstance(conf, dict) + assert (tmpdir / "porosity-out.csv").exists() def test_constant_porosity(tmpdir, datadir): data = pandas.read_csv( - datadir / "porosity_profile.txt", names=("dz", "porosity"), dtype=float + datadir / "porosity.csv", names=("dz", "porosity"), dtype=float ) phi_expected = compact(data["dz"], data["porosity"], porosity_max=0.6) with tmpdir.as_cwd(): + shutil.copy(datadir / "compact.toml", ".") + shutil.copy(datadir / "porosity.csv", ".") + runner = CliRunner(mix_stderr=False) - result = runner.invoke( - cli.run, - [ - "--config={0}".format(datadir / "config.yaml"), - str(datadir / "porosity_profile.txt"), - "out.txt", - ], - ) + result = runner.invoke(cli.run) assert result.exit_code == 0 - assert "Output written to out.txt" in result.stderr + assert "Output written to porosity-out.csv" in result.stderr phi_actual = pandas.read_csv( - "out.txt", names=("dz", "porosity"), dtype=float, comment="#" + "porosity-out.csv", names=("dz", "porosity"), dtype=float, comment="#" ) assert_array_almost_equal(phi_actual["porosity"], phi_expected) -def test_run_from_stdin(tmpdir, datadir): - path_to_porosity = str(datadir / "porosity_profile.txt") - with open(path_to_porosity) as fp: - porosity_profile = fp.read() - - runner = CliRunner(mix_stderr=False) - - with tmpdir.as_cwd(): - result = runner.invoke( - cli.run, - [ - "--config={0}".format(datadir / "config.yaml"), - path_to_porosity, - "expected.txt", - ], - ) - assert result.exit_code == 0 - - result = runner.invoke( - cli.run, - ["--config={0}".format(datadir / "config.yaml"), "-", "actual.txt"], - input=porosity_profile, - ) - assert result.exit_code == 0 - - assert filecmp.cmp("actual.txt", "expected.txt") - - -def test_run_to_stdout(tmpdir, datadir): - path_to_porosity = str(datadir / "porosity_profile.txt") - - runner = CliRunner(mix_stderr=False) - - with tmpdir.as_cwd(): - result = runner.invoke( - cli.run, - [ - "--config={0}".format(datadir / "config.yaml"), - path_to_porosity, - "expected.txt", - ], - ) - assert result.exit_code == 0 - with open("expected.txt") as fp: - expected = fp.read() - expected_lines = [line.strip() for line in expected.splitlines() if line] - - runner = CliRunner(mix_stderr=False) - result = runner.invoke( - cli.run, - ["--config={0}".format(datadir / "config.yaml"), path_to_porosity], - ) - assert result.exit_code == 0 - actual_lines = [line.strip() for line in result.stdout.splitlines() if line] - - assert actual_lines == expected_lines - - def test_setup(tmpdir): with tmpdir.as_cwd(): runner = CliRunner() - result = runner.invoke(cli.setup, ["."]) + result = runner.invoke(cli.setup) assert result.exit_code == 0 - assert (tmpdir / "config.yaml").exists() + assert (tmpdir / "compact.toml").exists() assert (tmpdir / "porosity.csv").exists() @pytest.mark.parametrize( - "files", (("config.yaml",), ("porosity.csv",), ("config.yaml", "porosity.csv")) + "files", (("compact.toml",), ("porosity.csv",), ("compact.toml", "porosity.csv")) ) def test_setup_with_existing_files(tmpdir, files): runner = CliRunner(mix_stderr=False) @@ -166,7 +94,7 @@ def test_setup_with_existing_files(tmpdir, files): with open(name, "w"): pass - result = runner.invoke(cli.setup, ["."]) + result = runner.invoke(cli.setup) assert result.exit_code == len(files) assert result.stdout == "" @@ -174,39 +102,37 @@ def test_setup_with_existing_files(tmpdir, files): assert name in result.stderr -def test_show(tmpdir): +def test_generate(tmpdir): with tmpdir.as_cwd(): - result = CliRunner(mix_stderr=False).invoke(cli.show, ["config"]) + result = CliRunner(mix_stderr=False).invoke(cli.generate, ["compact.toml"]) assert result.exit_code == 0 - with open("config.yaml", "w") as fp: + with open("compact.toml", "w") as fp: fp.write(result.stdout) - result = CliRunner(mix_stderr=False).invoke(cli.show, ["porosity"]) + result = CliRunner(mix_stderr=False).invoke(cli.generate, ["porosity.csv"]) assert result.exit_code == 0 with open("porosity.csv", "w") as fp: fp.write(result.stdout) - result = CliRunner(mix_stderr=False).invoke( - cli.run, ["porosity.csv", "out.csv"] - ) + result = CliRunner(mix_stderr=False).invoke(cli.run) - assert (tmpdir / "out.csv").exists() + assert (tmpdir / "porosity-out.csv").exists() assert result.exit_code == 0 -def test_show_config(tmpdir): +def test_generate_toml(tmpdir): runner = CliRunner(mix_stderr=False) - result = runner.invoke(cli.show, ["config"]) + result = runner.invoke(cli.generate, ["compact.toml"]) assert result.exit_code == 0 - params = yaml.safe_load(result.stdout) + params = tomlkit.parse(result.stdout) assert isinstance(params, dict) -def test_show_porosity(tmpdir): +def test_generate_porosity(tmpdir): runner = CliRunner(mix_stderr=False) - result = runner.invoke(cli.show, ["porosity"]) + result = runner.invoke(cli.generate, ["porosity.csv"]) assert result.exit_code == 0 diff --git a/tests/test_cli/compact.toml b/tests/test_cli/compact.toml new file mode 100644 index 0000000..77d1f0b --- /dev/null +++ b/tests/test_cli/compact.toml @@ -0,0 +1,2 @@ +[compact.constants] +porosity_max = 0.6 diff --git a/tests/test_cli/config.yaml b/tests/test_cli/config.yaml deleted file mode 100644 index 4551b5a..0000000 --- a/tests/test_cli/config.yaml +++ /dev/null @@ -1 +0,0 @@ -porosity_max: 0.6 diff --git a/tests/test_cli/porosity_profile.txt b/tests/test_cli/porosity.csv similarity index 100% rename from tests/test_cli/porosity_profile.txt rename to tests/test_cli/porosity.csv diff --git a/tests/test_compaction.py b/tests/test_compaction.py index 21a7314..fbb0da6 100644 --- a/tests/test_compaction.py +++ b/tests/test_compaction.py @@ -1,9 +1,9 @@ """Unit tests for compaction.""" +from io import StringIO + import numpy as np # type: ignore import pandas # type: ignore -import yaml from pytest import approx, mark, raises # type: ignore -from six import StringIO from compaction import compact from compaction.cli import load_config, run_compaction @@ -191,48 +191,61 @@ def test_load_config_defaults() -> None: """Test load_config without file name.""" config = load_config() defaults = { - "c": 5e-8, - "porosity_min": 0.0, - "porosity_max": 0.5, - "rho_grain": 2650.0, - "rho_void": 1000.0, + "constants": { + "c": 5e-8, + "porosity_min": 0.0, + "porosity_max": 0.5, + "rho_grain": 2650.0, + "rho_void": 1000.0, + } } assert config == defaults + config = load_config(StringIO("")) + assert config == defaults + + config = load_config(StringIO("[another_group]")) + assert config == defaults + + config = load_config(StringIO("[compact]")) + assert config == defaults + + config = load_config(StringIO("[compact]")) + assert config == defaults + def test_load_config_from_file() -> None: """Test config vars from a file.""" - file_like = StringIO() - yaml.dump(dict(c=3.14), file_like) - file_like.seek(0) + file_like = StringIO( + """[compact.constants] + c = 3.14 + """ + ) config = load_config(file_like) expected = { - "c": 3.14, - "porosity_min": 0.0, - "porosity_max": 0.5, - "rho_grain": 2650.0, - "rho_void": 1000.0, + "constants": { + "c": 3.14, + "porosity_min": 0.0, + "porosity_max": 0.5, + "rho_grain": 2650.0, + "rho_void": 1000.0, + } } assert config == expected -def test_run() -> None: - """Test running compaction with file-like objects.""" +def test_run(tmpdir) -> None: dz_0 = np.full(100, 1.0) phi_0 = np.full(100, 0.5) phi_1 = compact(dz_0, phi_0, porosity_max=0.5) - src = StringIO() - dest = StringIO() - - df = pandas.DataFrame.from_dict({"dz": dz_0, "porosity": phi_0}) - df.to_csv(src, index=False, header=False) + with tmpdir.as_cwd(): + df = pandas.DataFrame.from_dict({"dz": dz_0, "porosity": phi_0}) + df.to_csv("porosity.csv", index=False, header=False) - src.seek(0) - run_compaction(src=src, dest=dest, porosity_max=0.5) - dest.seek(0) + run_compaction("porosity.csv", "porosity-out.csv", porosity_max=0.5) - data = pandas.read_csv(dest, names=("dz", "porosity"), dtype=float, comment="#") + data = pandas.read_csv("porosity-out.csv", names=("dz", "porosity"), dtype=float, comment="#") - assert np.all(data.porosity.values == approx(phi_1)) + assert np.all(data.porosity.values == approx(phi_1))