From 9af50211e9f131bb6b7e2245d0714623be7748bd Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Thu, 13 Feb 2020 11:47:16 +0100 Subject: [PATCH 01/19] Bump to 0.0.3.dev0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1f40d22b..497056eb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = openpathsampling-cli -version = 0.0.2 +version = 0.0.3.dev0 # version should end in .dev0 if this isn't to be released description = Command line tool for OpenPathSampling long_description = file: README.md From d638dcf4874c459adbed8bc68beb8be7c3002ec3 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Sun, 16 Feb 2020 23:44:05 +0100 Subject: [PATCH 02/19] make strip-snapshots stuff more reusable --- paths_cli/commands/strip_snapshots.py | 80 +++++++++++++++++---------- 1 file changed, 51 insertions(+), 29 deletions(-) diff --git a/paths_cli/commands/strip_snapshots.py b/paths_cli/commands/strip_snapshots.py index 83d3e285..7fd635ec 100644 --- a/paths_cli/commands/strip_snapshots.py +++ b/paths_cli/commands/strip_snapshots.py @@ -21,26 +21,37 @@ def strip_snapshots(input_file, output_file, cv, blocksize): By giving the --cv option (once for each CV), you can select to only save certain CVs. If you do not give that option, all CVs will be saved. """ + input_storage = INPUT_FILE.get(input_file) return strip_snapshots_main( - input_storage=INPUT_FILE.get(input_file), + input_storage=input_storage, output_storage=OUTPUT_FILE.get(output_file), - cvs=CVS.get(cv), + cvs=CVS.get(input_storage, cv), blocksize=blocksize ) -def block_precompute_cvs(cvs, proxy_snaps, blocksize): - n_snaps = len(proxy_snaps) - n_blocks = (n_snaps // blocksize) +1 +def make_blocks(listlike, blocksize): + n_objs = len(listlike) + n_blocks = (n_objs // blocksize) + 1 + minval = lambda i: i * blocksize + maxval = lambda i: min((i + 1) * blocksize, n_objs) + blocks = [listlike[minval(i):maxval(i)] for i in range(n_blocks)] + return blocks - blocks = [proxy_snaps[i*blocksize: min((i+1)*blocksize, n_snaps)] - for i in range(n_blocks)] - for block_num in tqdm(range(n_blocks), leave=False): - block = blocks[block_num] - for cv in cvs: - cv.enable_diskcache() - _ = cv(block) +def precompute_cvs(cvs, block): + for cv in cvs: + cv.enable_diskcache() + _ = cv(block) + + +def rewrite_file(stage_names, stage_mapping): + stages = tqdm(stage_names) + for stage in stages: + stages.set_description("%s" % stage) + store_func, inputs = stage_mapping[stage] + for obj in tqdm(inputs): + store_func(obj) def strip_snapshots_main(input_storage, output_storage, cvs, blocksize): @@ -50,24 +61,35 @@ def strip_snapshots_main(input_storage, output_storage, cvs, blocksize): # save template output_storage.save(input_storage.snapshots[0]) + precompute_func = lambda inps: precompute_cvs(cvs, inps) snapshot_proxies = input_storage.snapshots.all().as_proxies() - stages = tqdm(['precompute', 'cvs', 'snapshots', 'trajectories', - 'steps']) - for stage in stages: - stages.set_description("%s" % stage) - if stage == 'precompute': - block_precompute_cvs(cvs, snapshot_proxies, blocksize) - else: - store_func, inputs = { - 'cvs': (output_storage.cvs.save, input_storage.cvs), - 'snapshots': (output_storage.snapshots.mention, - snapshot_proxies), - 'trajectories': (output_storage.trajectories.mention, - input_storage.trajectories), - 'steps': (output_storage.steps.save, input_storage.steps) - }[stage] - for obj in tqdm(inputs): - store_func(obj) + snapshot_blocks = make_blocks(snapshot_proxies, blocksize) + stages = ['precompute', 'cvs', 'snapshots', 'trajectories', 'steps'] + stage_mapping = { + 'precompute': (precompute_func, snapshot_blocks), + 'cvs': (output_storage.cvs.save, input_storage.cvs), + 'snapshots': (output_storage.snapshots.mention, + snapshot_proxies), + 'trajectories': (output_storage.trajectories.mention, + input_storage.trajectories), + 'steps': (output_storage.steps.save, input_storage.steps), + } + return rewrite_file(stages, stage_mapping) + # for stage in stages: + # stages.set_description("%s" % stage) + # if stage == 'precompute': + # block_precompute_cvs(cvs, snapshot_proxies, blocksize) + # else: + # store_func, inputs = { + # 'cvs': (output_storage.cvs.save, input_storage.cvs), + # 'snapshots': (output_storage.snapshots.mention, + # snapshot_proxies), + # 'trajectories': (output_storage.trajectories.mention, + # input_storage.trajectories), + # 'steps': (output_storage.steps.save, input_storage.steps) + # }[stage] + # for obj in tqdm(inputs): + # store_func(obj) CLI = strip_snapshots From a44578fdf2751c07ac51250b782379e733cca7ff Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 25 Feb 2020 13:36:48 +0100 Subject: [PATCH 03/19] (temporarily) remove strip_snapshots I'm not sure it's doing what it was supposed to be doing.... --- paths_cli/commands/strip_snapshots.py | 97 --------------------------- 1 file changed, 97 deletions(-) delete mode 100644 paths_cli/commands/strip_snapshots.py diff --git a/paths_cli/commands/strip_snapshots.py b/paths_cli/commands/strip_snapshots.py deleted file mode 100644 index 7fd635ec..00000000 --- a/paths_cli/commands/strip_snapshots.py +++ /dev/null @@ -1,97 +0,0 @@ -import click -from paths_cli.parameters import ( - INPUT_FILE, OUTPUT_FILE, CVS -) - -from tqdm.auto import tqdm - -@click.command( - 'strip-snapshots', - short_help="Remove coordinates/velocities from an OPS storage" -) -@INPUT_FILE.clicked(required=True) -@OUTPUT_FILE.clicked(required=True) -@CVS.clicked(required=False) -@click.option('--blocksize', type=int, default=10, - help="block size for precomputing CVs") -def strip_snapshots(input_file, output_file, cv, blocksize): - """ - Remove snapshot information (coordinates, velocities) from INPUT_FILE. - - By giving the --cv option (once for each CV), you can select to only - save certain CVs. If you do not give that option, all CVs will be saved. - """ - input_storage = INPUT_FILE.get(input_file) - return strip_snapshots_main( - input_storage=input_storage, - output_storage=OUTPUT_FILE.get(output_file), - cvs=CVS.get(input_storage, cv), - blocksize=blocksize - ) - - -def make_blocks(listlike, blocksize): - n_objs = len(listlike) - n_blocks = (n_objs // blocksize) + 1 - minval = lambda i: i * blocksize - maxval = lambda i: min((i + 1) * blocksize, n_objs) - blocks = [listlike[minval(i):maxval(i)] for i in range(n_blocks)] - return blocks - - -def precompute_cvs(cvs, block): - for cv in cvs: - cv.enable_diskcache() - _ = cv(block) - - -def rewrite_file(stage_names, stage_mapping): - stages = tqdm(stage_names) - for stage in stages: - stages.set_description("%s" % stage) - store_func, inputs = stage_mapping[stage] - for obj in tqdm(inputs): - store_func(obj) - - -def strip_snapshots_main(input_storage, output_storage, cvs, blocksize): - if not cvs: - cvs = list(input_storage.cvs) - - # save template - output_storage.save(input_storage.snapshots[0]) - - precompute_func = lambda inps: precompute_cvs(cvs, inps) - snapshot_proxies = input_storage.snapshots.all().as_proxies() - snapshot_blocks = make_blocks(snapshot_proxies, blocksize) - stages = ['precompute', 'cvs', 'snapshots', 'trajectories', 'steps'] - stage_mapping = { - 'precompute': (precompute_func, snapshot_blocks), - 'cvs': (output_storage.cvs.save, input_storage.cvs), - 'snapshots': (output_storage.snapshots.mention, - snapshot_proxies), - 'trajectories': (output_storage.trajectories.mention, - input_storage.trajectories), - 'steps': (output_storage.steps.save, input_storage.steps), - } - return rewrite_file(stages, stage_mapping) - # for stage in stages: - # stages.set_description("%s" % stage) - # if stage == 'precompute': - # block_precompute_cvs(cvs, snapshot_proxies, blocksize) - # else: - # store_func, inputs = { - # 'cvs': (output_storage.cvs.save, input_storage.cvs), - # 'snapshots': (output_storage.snapshots.mention, - # snapshot_proxies), - # 'trajectories': (output_storage.trajectories.mention, - # input_storage.trajectories), - # 'steps': (output_storage.steps.save, input_storage.steps) - # }[stage] - # for obj in tqdm(inputs): - # store_func(obj) - - -CLI = strip_snapshots -SECTION = "Miscellaneous" -REQUIRES_OPS = (1, 0) From 0dc3fa46be6ba86fb9dc16d0cf91c4c68d9a9126 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 25 Feb 2020 15:51:17 +0100 Subject: [PATCH 04/19] Add --log (logging conf) option to main command --- paths_cli/cli.py | 46 ++++++++++++++++++-------- paths_cli/tests/null_command.py | 22 +++++++++++++ paths_cli/tests/test_cli.py | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 paths_cli/tests/null_command.py create mode 100644 paths_cli/tests/test_cli.py diff --git a/paths_cli/cli.py b/paths_cli/cli.py index 90edc42a..de4b3851 100644 --- a/paths_cli/cli.py +++ b/paths_cli/cli.py @@ -4,6 +4,8 @@ """ # builds off the example of MultiCommand in click's docs import collections +import logging +import logging.config import os import click @@ -36,16 +38,30 @@ def __init__(self, *args, **kwargs): self.plugin_folders.append(folder) plugin_files = self._list_plugin_files(self.plugin_folders) - self.plugins = self._load_plugin_files(plugin_files) + plugins = self._load_plugin_files(plugin_files) self._get_command = {} self._sections = collections.defaultdict(list) - for plugin in self.plugins: - self._get_command[plugin.name] = plugin.func - self._sections[plugin.section].append(plugin.name) + self.plugins = [] + for plugin in plugins: + self._register_plugin(plugin) super(OpenPathSamplingCLI, self).__init__(*args, **kwargs) + def _register_plugin(self, plugin): + self.plugins.append(plugin) + self._get_command[plugin.name] = plugin.func + self._sections[plugin.section].append(plugin.name) + + def _deregister_plugin(self, plugin): + # mainly used in testing + self.plugins.remove(plugin) + del self._get_command[plugin.name] + self._sections[plugin.section].remove(plugin.name) + + def plugin_for_command(self, command_name): + return {p.name: p for p in self.plugins}[name] + @staticmethod def _list_plugin_files(plugin_folders): def is_plugin(filename): @@ -118,16 +134,18 @@ def format_commands(self, ctx, formatter): openpathsampling strip-snapshots --help """ -OPS_CLI = OpenPathSamplingCLI( - name="openpathsampling", - help=_MAIN_HELP, - context_settings=CONTEXT_SETTINGS -) - -def main(): # no-cov - OPS_CLI() +@click.command(cls=OpenPathSamplingCLI, name="openpathsampling", + help=_MAIN_HELP, context_settings=CONTEXT_SETTINGS) +@click.option('--log', type=click.Path(exists=True, readable=True), + help="logging configuration file") +def main(log): + if log: + logging.config.fileConfig(log, disable_existing_loggers=False) + # TODO: if log not given, check for logging.conf in .openpathsampling/ + logger = logging.getLogger(__name__) + logger.debug("About to run command") # TODO: maybe log invocation? + pass if __name__ == '__main__': # no-cov - main() - # print("list commands:", cli.list_commands()) + cli() diff --git a/paths_cli/tests/null_command.py b/paths_cli/tests/null_command.py new file mode 100644 index 00000000..85c7010a --- /dev/null +++ b/paths_cli/tests/null_command.py @@ -0,0 +1,22 @@ +import click +@click.command( + 'null-command', + short_help="Do nothing (testing)" +) +def null_command(): + print("Running null command") + +CLI = null_command +SECTION = "Workflow" + +class NullCommandContext(object): + """Context that registers/deregisters the null command (for tests)""" + def __init__(self, cli): + self.plugin = cli._load_plugin_files([__file__])[0] + self.cli = cli + + def __enter__(self): + self.cli._register_plugin(self.plugin) + + def __exit__(self, type, value, traceback): + self.cli._deregister_plugin(self.plugin) diff --git a/paths_cli/tests/test_cli.py b/paths_cli/tests/test_cli.py new file mode 100644 index 00000000..fa4e1da9 --- /dev/null +++ b/paths_cli/tests/test_cli.py @@ -0,0 +1,58 @@ +import pytest +from click.testing import CliRunner + +import logging + +from paths_cli.cli import * +from .null_command import NullCommandContext + +class TestOpenPathSamplingCLI(object): + def setup(self): + # TODO: patch out the directory to fake the plugins + self.cli = OpenPathSamplingCLI() + + def test_plugins(self): + pytest.skip() + pass + + def test_get_command(self): + # test renamings + pytest.skip() + pass + + def test_format_commands(self): + pytest.skip() + # use a mock to get the formatter + # test that it skips a section if it is empty + pass + +@pytest.mark.parametrize('with_log', [True, False]) +def test_main_log(with_log): + logged_stdout = "About to run command\n" + cmd_stdout = "Running null command\n" + logfile_text = "\n".join([ + "[loggers]", "keys=root", "", + "[handlers]", "keys=std", "", + "[formatters]", "keys=default", "", + "[formatter_default]", "format=%(message)s", "", + "[handler_std]", "class=StreamHandler", "level=NOTSET", + "formatter=default", "args=(sys.stdout,)", "" + "[logger_root]", "level=DEBUG", "handlers=std" + ]) + runner = CliRunner() + invocation = { + True: ['--log', 'logging.conf', 'null-command'], + False: ['null-command'] + }[with_log] + expected = { + True: logged_stdout + cmd_stdout, + False: cmd_stdout + }[with_log] + with runner.isolated_filesystem(): + with open("logging.conf", mode='w') as log_conf: + log_conf.write(logfile_text) + + with NullCommandContext(main): + result = runner.invoke(main, invocation) + found = result.stdout_bytes + assert found.decode('utf-8') == expected From 107b1ac1f182790dfc939ee49836ddaa42e3331b Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 25 Feb 2020 16:03:24 +0100 Subject: [PATCH 05/19] test getting logs from inside the command --- paths_cli/cli.py | 1 - paths_cli/tests/null_command.py | 4 +++- paths_cli/tests/test_cli.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/paths_cli/cli.py b/paths_cli/cli.py index de4b3851..670d82d8 100644 --- a/paths_cli/cli.py +++ b/paths_cli/cli.py @@ -145,7 +145,6 @@ def main(log): logger = logging.getLogger(__name__) logger.debug("About to run command") # TODO: maybe log invocation? - pass if __name__ == '__main__': # no-cov cli() diff --git a/paths_cli/tests/null_command.py b/paths_cli/tests/null_command.py index 85c7010a..6d1044eb 100644 --- a/paths_cli/tests/null_command.py +++ b/paths_cli/tests/null_command.py @@ -1,10 +1,12 @@ +import logging import click @click.command( 'null-command', short_help="Do nothing (testing)" ) def null_command(): - print("Running null command") + logger = logging.getLogger(__name__) + logger.info("Running null command") CLI = null_command SECTION = "Workflow" diff --git a/paths_cli/tests/test_cli.py b/paths_cli/tests/test_cli.py index fa4e1da9..677de73c 100644 --- a/paths_cli/tests/test_cli.py +++ b/paths_cli/tests/test_cli.py @@ -46,7 +46,7 @@ def test_main_log(with_log): }[with_log] expected = { True: logged_stdout + cmd_stdout, - False: cmd_stdout + False: "" }[with_log] with runner.isolated_filesystem(): with open("logging.conf", mode='w') as log_conf: From 748705b12c737575e9cdfcf0ad480bb974b2a5c9 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 28 Feb 2020 20:28:50 +0100 Subject: [PATCH 06/19] add file_copying and start to its tests --- paths_cli/file_copying.py | 103 +++++++++++++++++++++++++++ paths_cli/tests/test_file_copying.py | 33 +++++++++ 2 files changed, 136 insertions(+) create mode 100644 paths_cli/file_copying.py create mode 100644 paths_cli/tests/test_file_copying.py diff --git a/paths_cli/file_copying.py b/paths_cli/file_copying.py new file mode 100644 index 00000000..4050ed25 --- /dev/null +++ b/paths_cli/file_copying.py @@ -0,0 +1,103 @@ +"""Tools to facilitate copying files. + +This is mainly aimed at cases where a file is being copied with some sort of +modification, or where CVs need to be disk-cached. +""" + +import click +from tqdm.auto import tqdm +from paths_cli.parameters import ( + Option, Argument, HELP_MULTIPLE, StorageLoader, OPSStorageLoadNames +) + +INPUT_APPEND_FILE = StorageLoader( + param=Argument('append_file', + type=click.Path(writable=True, readable=True)), + mode='a' +) + +class PrecomputeLoadNames(OPSStorageLoadNames): + def get(self, storage, name): + if len(name) == 0: + return list(getattr(storage, self.store)) + elif len(name) == 1 and name[0] == '--': + return [] + + return super(PrecomputeLoadNames, self).get(storage, name) + +PRECOMPUTE_CVS = PrecomputeLoadNames( + param=Option('--cv', type=str, multiple=True, + help=('name of CV to precompute; if not specified all will' + + ' be used' + HELP_MULTIPLE + + ' (use `--cv --` to disable precomputing)')), + store='cvs' +) + + +def make_blocks(listlike, blocksize): + """Make blocks out of a listlike object. + + Parameters + ---------- + listlike : Iterable + must be an iterable that supports slicing + blocksize : int + number of objects per block + + + Returns + ------- + List[List[Any]] : + the input iterable chunked into blocks + """ + n_objs = len(listlike) + partial_block = 1 if n_objs % blocksize else 0 + n_blocks = (n_objs // blocksize) + partial_block + minval = lambda i: i * blocksize + maxval = lambda i: min((i + 1) * blocksize, n_objs) + blocks = [listlike[minval(i):maxval(i)] for i in range(n_blocks)] + return blocks + + +def precompute_cvs(cvs, block): + """Calculate a CV for a a given block. + + Parameters + ---------- + cvs : List[:class:`openpathsampling.CollectiveVariable`] + CVs to precompute + block : List[Any] + b + """ + for cv in cvs: + cv.enable_diskcache() + _ = cv(block) + + +def precompute_cvs_func_and_inputs(input_storage, cvs, blocksize): + """ + Parameters + ---------- + input_storage : :class:`openpathsampling.Storage` + storage file to read from + cvs : List[:class:`openpathsampling.CollectiveVariable`] + list of CVs to precompute; if None, use all CVs in ``input_storage`` + blocksize : int + number of snapshots per block to precompute + """ + if cvs is None: + cvs = list(input_storage.cvs) + + precompute_func = lambda inps: precompute_cvs(cvs, inps) + snapshot_proxies = input_storage.snapshots.all().as_proxies() + snapshot_blocks = make_blocks(snapshot_proxies, blocksize) + return precompute_func, snapshot_blocks + + +def rewrite_file(stage_names, stage_mapping): + stages = tqdm(stage_names, desc="All stages") + for stage in stages: + store_func, inputs = stage_mapping[stage] + desc = "This stage: {}".format(stage) + for obj in tqdm(inputs, desc=desc, leave=False): + store_func(obj) diff --git a/paths_cli/tests/test_file_copying.py b/paths_cli/tests/test_file_copying.py new file mode 100644 index 00000000..9d304652 --- /dev/null +++ b/paths_cli/tests/test_file_copying.py @@ -0,0 +1,33 @@ +import os + +import pytest + +from paths_cli.file_copying import * + +class Test_PRECOMPUTE_CVS(object): + pass + + +@pytest.mark.parametrize('blocksize', [2, 3, 5, 10, 12]) +def test_make_blocks(blocksize): + expected_lengths = {2: [2, 2, 2, 2, 2], + 3: [3, 3, 3, 1], + 5: [5, 5], + 10: [10], + 12: [10]}[blocksize] + ll = list(range(10)) + blocks = make_blocks(ll, blocksize) + assert [len(block) for block in blocks] == expected_lengths + assert sum(blocks, []) == ll + + +class TestPrecompute(object): + def test_precompute_cvs(self): + pytest.skip() + + def test_precompute_cvs_and_inputs(self): + pytest.skip() + + +def test_rewrite_file(): + pytest.skip() From 7033c0da0b444eb5cf5b2fd71a525ae0e361950e Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Mon, 2 Mar 2020 15:25:50 +0100 Subject: [PATCH 07/19] tests for precomputing CVs --- paths_cli/tests/test_file_copying.py | 56 ++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/paths_cli/tests/test_file_copying.py b/paths_cli/tests/test_file_copying.py index 9d304652..a2adf085 100644 --- a/paths_cli/tests/test_file_copying.py +++ b/paths_cli/tests/test_file_copying.py @@ -1,11 +1,41 @@ import os - +import tempfile +from unittest.mock import MagicMock, patch import pytest + +import openpathsampling as paths +from openpathsampling.tests.test_helpers import make_1d_traj + from paths_cli.file_copying import * class Test_PRECOMPUTE_CVS(object): - pass + def setup(self): + self.tmpdir = tempfile.mkdtemp() + self.storage_filename = os.path.join(self.tmpdir, "test.nc") + self.storage = paths.Storage(self.storage_filename, mode='w') + snap = make_1d_traj([1])[0] + self.storage.save(snap) + self.cv_x = paths.CoordinateFunctionCV("x", lambda s: s.xyz[0][0]) + self.cv_y = paths.CoordinateFunctionCV("y", lambda s: s.xyz[0][1]) + self.storage.save([self.cv_x, self.cv_y]) + + def teardown(self): + self.storage.close() + + for filename in os.listdir(self.tmpdir): + os.remove(os.path.join(self.tmpdir, filename)) + os.rmdir(self.tmpdir) + + @pytest.mark.parametrize('getter', ['x', None, '--']) + def test_get(self, getter): + expected = {'x': [self.cv_x], + None: [self.cv_x, self.cv_y], + '--': []}[getter] + getter = [] if getter is None else [getter] # CLI gives a list + cvs = PRECOMPUTE_CVS.get(self.storage, getter) + assert len(cvs) == len(expected) + assert set(cvs) == set(expected) @pytest.mark.parametrize('blocksize', [2, 3, 5, 10, 12]) @@ -22,8 +52,28 @@ def test_make_blocks(blocksize): class TestPrecompute(object): + def setup(self): + class RunOnceFunction(object): + def __init__(self): + self.previously_seen = set([]) + + def __call__(self, snap): + if snap in self.previously_seen: + raise AssertionError("Second CV eval for " + str(snap)) + self.previously_seen.update({snap}) + return snap.xyz[0][0] + + self.cv = paths.FunctionCV("test", RunOnceFunction()) + traj = paths.tests.test_helpers.make_1d_traj([2, 1]) + self.snap = traj[0] + self.other_snap = traj[1] + def test_precompute_cvs(self): - pytest.skip() + precompute_cvs([self.cv], [self.snap]) + assert self.cv.f.previously_seen == {self.snap} + recalced = self.cv(self.snap) # AssertionError if func called + assert recalced == 2 + assert self.cv.diskcache_enabled is True def test_precompute_cvs_and_inputs(self): pytest.skip() From 3ae0ce165daf76e40cfd72bcb0e78304f4776478 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Mon, 2 Mar 2020 17:07:31 +0100 Subject: [PATCH 08/19] remaining tests for file_copying --- paths_cli/tests/commands/test_append.py | 1 - paths_cli/tests/test_file_copying.py | 56 ++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/paths_cli/tests/commands/test_append.py b/paths_cli/tests/commands/test_append.py index c8931f06..40d50994 100644 --- a/paths_cli/tests/commands/test_append.py +++ b/paths_cli/tests/commands/test_append.py @@ -88,7 +88,6 @@ def test_append_remove_tag(tps_network_and_traj): result = runner.invoke(append, [in_file, '-a', "output.nc", "--tag", 'template', '--save-tag', '']) - print(result.output) assert result.exception is None assert result.exit_code == 0 diff --git a/paths_cli/tests/test_file_copying.py b/paths_cli/tests/test_file_copying.py index a2adf085..0a59cbc6 100644 --- a/paths_cli/tests/test_file_copying.py +++ b/paths_cli/tests/test_file_copying.py @@ -1,9 +1,10 @@ +import collections +import functools import os import tempfile from unittest.mock import MagicMock, patch import pytest - import openpathsampling as paths from openpathsampling.tests.test_helpers import make_1d_traj @@ -64,7 +65,7 @@ def __call__(self, snap): return snap.xyz[0][0] self.cv = paths.FunctionCV("test", RunOnceFunction()) - traj = paths.tests.test_helpers.make_1d_traj([2, 1]) + traj = make_1d_traj([2, 1]) self.snap = traj[0] self.other_snap = traj[1] @@ -75,9 +76,54 @@ def test_precompute_cvs(self): assert recalced == 2 assert self.cv.diskcache_enabled is True - def test_precompute_cvs_and_inputs(self): - pytest.skip() + @pytest.mark.parametrize('cvs', [['test'], None]) + def test_precompute_cvs_and_inputs(self, cvs): + with tempfile.TemporaryDirectory() as tmpdir: + storage = paths.Storage(os.path.join(tmpdir, "test.nc"), + mode='w') + traj = make_1d_traj(list(range(10))) + cv = paths.FunctionCV("test", lambda s: s.xyz[0][0]) + storage.save(traj) + storage.save(cv) + + if cvs is not None: + cvs = [storage.cvs[cv] for cv in cvs] + + precompute_func, blocks = precompute_cvs_func_and_inputs( + input_storage=storage, + cvs=cvs, + blocksize=2 + ) + assert len(blocks) == 5 + for block in blocks: + assert len(block) == 2 + + # smoke test: only effect should be caching results + precompute_func(blocks[0]) def test_rewrite_file(): - pytest.skip() + # making a mock for storage instead of actually testing integration + class FakeStore(object): + def __init__(self): + self._stores = collections.defaultdict(list) + + def store(self, obj, store_name): + self._stores[store_name].append(obj) + + stage_names = ['foo', 'bar'] + storage = FakeStore() + store_funcs = { + name: functools.partial(storage.store, store_name=name) + for name in stage_names + } + stage_mapping = { + 'foo': (store_funcs['foo'], [0, 1, 2]), + 'bar': (store_funcs['bar'], [[3], [4], [5]]) + } + silent_tqdm = lambda x, desc=None, leave=True: x + with patch('paths_cli.file_copying.tqdm', silent_tqdm): + rewrite_file(stage_names, stage_mapping) + + assert storage._stores['foo'] == [0, 1, 2] + assert storage._stores['bar'] == [[3], [4], [5]] From 222bd2fb7560a34261edd89b555555f275878b40 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Mon, 2 Mar 2020 17:17:03 +0100 Subject: [PATCH 09/19] remove references to strip-snapshots --- README.md | 1 - docs/for_core/cli.rst | 7 +++++-- paths_cli/cli.py | 5 ++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 31bd2a13..4de58ae9 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,6 @@ miscellaneous operations on OPS output files. **Miscellaneous Commands:** * `contents`: List named objects from an OPS .nc file -* `strip-snapshots`: Remove coordinates/velocities from an OPS storage * `append`: add objects from INPUT_FILE to another file Full documentation is at https://openpathsampling-cli.readthedocs.io/; a brief diff --git a/docs/for_core/cli.rst b/docs/for_core/cli.rst index ae9161f5..2277e4ee 100644 --- a/docs/for_core/cli.rst +++ b/docs/for_core/cli.rst @@ -79,10 +79,13 @@ foregoing the CLI tools to run simulations, some of the "miscellaneous" commands are likely to be quite useful. Here are some that are available in the CLI: -* ``nclist``: list all the named objects in an OPS storage, organized by +* ``contents``: list all the named objects in an OPS storage, organized by store (type); this is extremely useful to get the name of an object to use as command-line input to one of the simulation scripts -* ``strip-snapshots``: create a copy of the input storage file with the +.. * ``strip-snapshots``: create a copy of the input storage file with the details (coordinates/velocities) of all snapshots removed; this allows you to make a much smaller copy (with results of CVs) to copy back to a local computer for analysis +* ``append`` : add an object from once OPS storage into another one; this is + useful for getting everything into a single file before running a + simulation diff --git a/paths_cli/cli.py b/paths_cli/cli.py index 90edc42a..73d60a80 100644 --- a/paths_cli/cli.py +++ b/paths_cli/cli.py @@ -112,10 +112,9 @@ def format_commands(self, ctx, formatter): OpenPathSampling is a Python library for path sampling simulations. This command line tool facilitates common tasks when working with OpenPathSampling. To use it, use one of the subcommands below. For example, -you can get more information about the strip-snapshots (filesize reduction) -tool with: +you can get more information about the pathsampling tool with: - openpathsampling strip-snapshots --help + openpathsampling pathsampling --help """ OPS_CLI = OpenPathSamplingCLI( From 2093b0dde810dc90017a18af56c00cb1d4f784b3 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 3 Mar 2020 09:11:53 +0100 Subject: [PATCH 10/19] start to tests for OpenPathSamplingCLI class --- paths_cli/cli.py | 2 +- paths_cli/tests/test_cli.py | 43 ++++++++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/paths_cli/cli.py b/paths_cli/cli.py index 91e3ea71..56ef8b77 100644 --- a/paths_cli/cli.py +++ b/paths_cli/cli.py @@ -60,7 +60,7 @@ def _deregister_plugin(self, plugin): self._sections[plugin.section].remove(plugin.name) def plugin_for_command(self, command_name): - return {p.name: p for p in self.plugins}[name] + return {p.name: p for p in self.plugins}[command_name] @staticmethod def _list_plugin_files(plugin_folders): diff --git a/paths_cli/tests/test_cli.py b/paths_cli/tests/test_cli.py index 677de73c..dabb8500 100644 --- a/paths_cli/tests/test_cli.py +++ b/paths_cli/tests/test_cli.py @@ -1,4 +1,5 @@ import pytest +from unittest.mock import patch, MagicMock from click.testing import CliRunner import logging @@ -7,18 +8,43 @@ from .null_command import NullCommandContext class TestOpenPathSamplingCLI(object): + # TODO: more thorough testing of private methods to find/register + # plugins might be nice; so far we mainly focus on testing the API. + # (Still have smoke tests covering everything, though.) def setup(self): - # TODO: patch out the directory to fake the plugins - self.cli = OpenPathSamplingCLI() + self.plugin_dict = { + 'foo': OPSPlugin(name='foo', + filename='foo.py', + func=lambda: 'foo', + section='Simulation'), + 'foo-bar': OPSPlugin(name='foo-bar', + filename='foo_bar.py', + func=lambda: 'foobar', + section='Miscellaneous') + } + self.fake_plugins = list(self.plugin_dict.values()) + mock_plugins = MagicMock(return_value=self.fake_plugins) + with patch.object(OpenPathSamplingCLI, '_load_plugin_files', + mock_plugins): + self.cli = OpenPathSamplingCLI() def test_plugins(self): - pytest.skip() - pass + assert self.cli.plugins == self.fake_plugins + assert self.cli._sections['Simulation'] == ['foo'] + assert self.cli._sections['Miscellaneous'] == ['foo-bar'] - def test_get_command(self): - # test renamings - pytest.skip() - pass + @pytest.mark.parametrize('name', ['foo', 'foo-bar']) + def test_plugin_for_command(self, name): + assert self.cli.plugin_for_command(name) == self.plugin_dict[name] + + def test_list_commands(self): + assert self.cli.list_commands(ctx=None) == ['foo', 'foo-bar'] + + @pytest.mark.parametrize('command', ['foo-bar', 'foo_bar']) + def test_get_command(self, command): + # this tests that renamings work + cmd = self.cli.get_command(ctx=None, name=command) + assert cmd() == 'foobar' def test_format_commands(self): pytest.skip() @@ -26,6 +52,7 @@ def test_format_commands(self): # test that it skips a section if it is empty pass + @pytest.mark.parametrize('with_log', [True, False]) def test_main_log(with_log): logged_stdout = "About to run command\n" From 359350d11bd5c3b0ba90eb2f0f557b4691f58339 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 3 Mar 2020 10:21:55 +0100 Subject: [PATCH 11/19] tests for cli.format_commands --- paths_cli/tests/test_cli.py | 40 +++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/paths_cli/tests/test_cli.py b/paths_cli/tests/test_cli.py index dabb8500..042182ac 100644 --- a/paths_cli/tests/test_cli.py +++ b/paths_cli/tests/test_cli.py @@ -2,24 +2,31 @@ from unittest.mock import patch, MagicMock from click.testing import CliRunner -import logging - from paths_cli.cli import * from .null_command import NullCommandContext + class TestOpenPathSamplingCLI(object): # TODO: more thorough testing of private methods to find/register # plugins might be nice; so far we mainly focus on testing the API. # (Still have smoke tests covering everything, though.) def setup(self): + def make_mock(name, helpless=False): + mock = MagicMock(return_value=name) + if helpless: + mock.short_help = None + else: + mock.short_help = name + " help" + return mock + self.plugin_dict = { 'foo': OPSPlugin(name='foo', filename='foo.py', - func=lambda: 'foo', + func=make_mock('foo'), section='Simulation'), 'foo-bar': OPSPlugin(name='foo-bar', filename='foo_bar.py', - func=lambda: 'foobar', + func=make_mock('foobar', helpless=True), section='Miscellaneous') } self.fake_plugins = list(self.plugin_dict.values()) @@ -47,10 +54,27 @@ def test_get_command(self, command): assert cmd() == 'foobar' def test_format_commands(self): - pytest.skip() - # use a mock to get the formatter - # test that it skips a section if it is empty - pass + class MockFormatter(object): + def __init__(self): + self.title = None + self.contents = {} + + def section(self, title): + self.title = title + return MagicMock() + + def write_dl(self, rows): + self.contents[self.title] = rows + + formatter = MockFormatter() + # add a non-existent command; tests when get_command is None + self.cli._sections['Workflow'] = ['baz'] + self.cli.format_commands(ctx=None, formatter=formatter) + foo_row = ('foo', 'foo help') + foobar_row = ('foo-bar', '') + assert formatter.contents['Simulation Commands'] == [foo_row] + assert formatter.contents['Miscellaneous Commands'] == [foobar_row] + assert len(formatter.contents) == 2 @pytest.mark.parametrize('with_log', [True, False]) From 142db2d9658a3d5003b03517983e04d68e5d850d Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 3 Mar 2020 12:50:40 +0100 Subject: [PATCH 12/19] updates to docs for core --- docs/for_core/cli.rst | 91 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/docs/for_core/cli.rst b/docs/for_core/cli.rst index 2277e4ee..0fc7f249 100644 --- a/docs/for_core/cli.rst +++ b/docs/for_core/cli.rst @@ -28,8 +28,8 @@ to develop plugins for the CLI, see its documentation. The CLI subcommands are defined through a plugin system, which makes it very easy for developers to create new subcommands. -* CLI documentation: -* CLI code repository: +* CLI documentation: https://openpathsampling-cli.readthedocs.io/ +* CLI code repository: https://github.com/openpathsampling/openpathsampling-cli/ Workflow with the CLI --------------------- @@ -41,6 +41,76 @@ files. To use it, you'll want to first set up + +Finding your way around the CLI +------------------------------- + +Like many command line tools, the OPS CLI has the options ``-h`` or +``--help`` to get help. If you run ``openpathsampling --help`` you should +see something like this:: + + Usage: openpathsampling [OPTIONS] COMMAND [ARGS]... + + OpenPathSampling is a Python library for path sampling simulations. This + command line tool facilitates common tasks when working with + OpenPathSampling. To use it, use one of the subcommands below. For + example, you can get more information about the pathsampling tool with: + + openpathsampling pathsampling --help + + Options: + --log PATH logging configuration file + -h, --help Show this message and exit. + + Simulation Commands: + visit-all Run MD to generate initial trajectories + equilibrate Run equilibration for path sampling + pathsampling Run any path sampling simulation, including TIS variants + + Miscellaneous Commands: + contents list named objects from an OPS .nc file + append add objects from INPUT_FILE to another file + +The ``--log`` option takes a logging configuration file (e.g., `logging.conf +<>`_, and sets that logging behavior. If you use it, it must come before the +subcommand name. + +You can find out more about each subcommand by putting ``--help`` *after* +the subcommand name, e.g., ``openpathsampling pathsampling --help``, which +returns:: + + Usage: openpathsampling pathsampling [OPTIONS] INPUT_FILE + + General path sampling, using setup in INPUT_FILE + + Options: + -o, --output-file PATH output ncfile [required] + -m, --scheme TEXT identifier for the move scheme + -t, --init-conds TEXT identifier for initial conditions (sample set or + trajectory) + -n, --nsteps INTEGER number of Monte Carlo trials to run + -h, --help Show this message and exit. + +Here you see the list of the options for the running a path sampling +simulation. In general, path sampling requires an output +file, a move scheme and initial conditions from some input file, and the +number of steps to run. Note that only the output file is technically +required: the CLI will default to running 0 steps (essentially, testing the +validity of your setup), and it can try to guess the move scheme and initial +conditions. In general, the way it guesses follows the following path: + +1. If there is only one object of the suitable type in the INPUT_FILE, use + that. +2. If there are multiple objects of the correct type, but only one has a + name, use the named object. +3. In special cases it looks for specific names, such as + ``initial_conditions``, and will use those. + +Full details on how various CLI parameters search the storage can be seen in +the `Parameter Interpretation +`_ +section of the CLI docs. + Simulation Commands ------------------- @@ -68,9 +138,6 @@ Here are some of the simulation commands implemented in the OPS CLI: have been visited (works for MSTIS or any 2-state system); must provide states, engine, and initial snapshot on command line -.. TODO figure showing how these all work -- what is needed for each, what - is implicit - Miscellaneous Commands ---------------------- @@ -89,3 +156,17 @@ the CLI: * ``append`` : add an object from once OPS storage into another one; this is useful for getting everything into a single file before running a simulation + +Customizing the CLI +------------------- + +The OPS CLI uses a flexible plugin system to enable users to easily add +custom functionality. This way, you can create and distribute custom +plugins, giving more functionality to other users who would benefit from it, +without adding everything to the core package and thus overwhelming new +users. + +Installing a plugin is easy: just create the directory +``$HOME/.openpathsampling/cli-plugins/``, and copy the plugin Python script +into there. For details on how to write a CLI plugin, see the `CLI +development docs `_. From f0a41747bd98ce392829893a36d7cd1bb4f548c3 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 3 Mar 2020 14:06:20 +0100 Subject: [PATCH 13/19] Refactor parameters --- paths_cli/parameters.py | 187 ++++++++++++++++++++++------------------ 1 file changed, 103 insertions(+), 84 deletions(-) diff --git a/paths_cli/parameters.py b/paths_cli/parameters.py index 94bbf60d..e2afbe63 100644 --- a/paths_cli/parameters.py +++ b/paths_cli/parameters.py @@ -1,6 +1,9 @@ import click import os -# import openpathsampling as paths + +_UNNAMED_STORES = ['snapshots', 'trajectories', 'samples', 'sample_sets', + 'steps'] + class AbstractParameter(object): def __init__(self, *args, **kwargs): @@ -12,11 +15,10 @@ def __init__(self, *args, **kwargs): def clicked(self, required=False): raise NotImplementedError() -# we'll use tests of the -h option in the .travis.yml to ensure that the -# .clicked methods work - HELP_MULTIPLE = "; may be used more than once" +# we'll use tests of the -h option in the .travis.yml to ensure that the +# .clicked methods work class Option(AbstractParameter): def clicked(self, required=False): # no-cov return click.option(*self.args, **self.kwargs, required=required) @@ -77,119 +79,136 @@ def get(self, storage, names): for name in int_corrected] +class Getter(object): + def __init__(self, store_name): + self.store_name = store_name + + def _get(self, storage, name): + store = getattr(storage, self.store_name) + try: + return store[name] + except: + return None + +class GetByName(Getter): + def __call__(self, storage, name): + return self._get(storage, name) + +class GetByNumber(Getter): + def __call__(self, storage, name): + try: + num = int(name) + except: + return None + + return self._get(storage, num) + +class GetPredefinedName(Getter): + def __init__(self, store_name, name): + super().__init__(store_name=store_name) + self.name = name + + def __call__(self, storage): + return self._get(storage, self.name) + +class GetOnly(Getter): + def __call__(self, storage): + store = getattr(storage, self.store_name) + if len(store) == 1: + return store[0] + +class GetOnlyNamed(Getter): + def __call__(self, storage): + store = getattr(storage, self.store_name) + named_things = [o for o in store if o.is_named] + if len(named_things) == 1: + return named_things[0] + +class GetOnlySnapshot(Getter): + def __init__(self, store_name="snapshots"): + super().__init__(store_name) + + def __call__(self, storage): + store = getattr(storage, self.store_name) + if len(store) == 2: + # this is really only 1 snapshot; reversed copy gets saved + return store[0] + + +def _try_strategies(strategies, storage, **kwargs): + result = None + for strategy in strategies: + result = strategy(storage, **kwargs) + if result is not None: + return result + + class OPSStorageLoadSingle(AbstractLoader): - def __init__(self, param, store, fallback=None, num_store=None): + """Objects that expect to load a single object. + + These can sometimes include guesswork to figure out which object is + desired. + """ + def __init__(self, param, store, value_strategies=None, + none_strategies=None): super(OPSStorageLoadSingle, self).__init__(param) self.store = store - self.fallback = fallback - if num_store is None: - num_store = store - self.num_store = num_store + if value_strategies is None: + value_strategies = [GetByName(self.store), + GetByNumber(self.store)] + self.value_strategies = value_strategies + + if none_strategies is None: + none_strategies = [GetOnly(self.store), + GetOnlyNamed(self.store)] + self.none_strategies = none_strategies def get(self, storage, name): store = getattr(storage, self.store) - num_store = getattr(storage, self.num_store) + # num_store = getattr(storage, self.num_store) - result = None - # if the we can get by name/number, do it if name is not None: - try: - result = store[name] - except: - # on any error, we try everything else - pass - - if result is None: - try: - num = int(name) - except ValueError: - pass - else: - result = num_store[num] - - if result is not None: - return result - - # if only one is named, take it - if self.store != 'tags' and name is None: - # if there's only one of them, take that - if len(store) == 1: - return store[0] - named_things = [o for o in store if o.is_named] - if len(named_things) == 1: - return named_things[0] - - if len(num_store) == 1 and name is None: - return num_store[0] - - if self.fallback: - result = self.fallback(self, storage, name) + result = _try_strategies(self.value_strategies, storage, + name=name) + else: + result = _try_strategies(self.none_strategies, storage) if result is None: raise RuntimeError("Couldn't find %s", name) return result - -def init_traj_fallback(parameter, storage, name): - result = None - - if isinstance(name, int): - return storage.trajectories[name] - - if name is None: - # fallback to final_conditions, initial_conditions, only trajectory - # the "get" here may need to be changed for new storage - for tag in ['final_conditions', 'initial_conditions']: - result = storage.tags[tag] - if result: - return result - - # already tried storage.samplesets - if len(storage.trajectories) == 1: - return storage.trajectories[0] - - -def init_snap_fallback(parameter, storage, name): - # this is structured so that other things can be added to it later - result = None - - if name is None: - result = storage.tags['initial_snapshot'] - if result: - return result - - if len(storage.snapshots) == 2: - # this is really only 1 snapshot; reversed copy gets saved - return storage.snapshots[0] - ENGINE = OPSStorageLoadSingle( param=Option('-e', '--engine', help="identifer for the engine"), store='engines', - fallback=None # for now... I'll add more tricks later + # fallback=None # for now... I'll add more tricks later ) SCHEME = OPSStorageLoadSingle( param=Option('-m', '--scheme', help="identifier for the move scheme"), store='schemes', - fallback=None + # fallback=None ) INIT_CONDS = OPSStorageLoadSingle( param=Option('-t', '--init-conds', help=("identifier for initial conditions " + "(sample set or trajectory)")), - store='tags', - num_store='samplesets', - fallback=init_traj_fallback + store='samplesets', + value_strategies=[GetByName('tags'), GetByNumber('samplesets'), + GetByNumber('trajectories')], + none_strategies=[GetOnly('samplesets'), GetOnly('trajectories'), + GetPredefinedName('tags', 'final_conditions'), + GetPredefinedName('tags', 'initial_conditions')] ) INIT_SNAP = OPSStorageLoadSingle( param=Option('-f', '--init-frame', help="identifier for initial snapshot"), - store='tags', - num_store='snapshots', - fallback=init_snap_fallback + store='snapshots', + value_strategies=[GetByName('tags'), GetByNumber('snapshots')], + none_strategies=[GetOnlySnapshot(), + GetPredefinedName('tags', 'initial_snapshot')] ) CVS = OPSStorageLoadNames( From 5522bbecdd954953dbb6eedfc3b4f64d29e75325 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 3 Mar 2020 14:52:56 +0100 Subject: [PATCH 14/19] docstrings --- paths_cli/parameters.py | 115 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 8 deletions(-) diff --git a/paths_cli/parameters.py b/paths_cli/parameters.py index e2afbe63..9832bccc 100644 --- a/paths_cli/parameters.py +++ b/paths_cli/parameters.py @@ -1,11 +1,20 @@ import click import os -_UNNAMED_STORES = ['snapshots', 'trajectories', 'samples', 'sample_sets', - 'steps'] - class AbstractParameter(object): + """Abstract wrapper for click parameters. + + Forbids use setting ``required``, because we do that for each command + individually. + + Parameters + ---------- + args : + args to pass to click parameters + kwargs : + kwargs to pass to click parameters + """ def __init__(self, *args, **kwargs): self.args = args if 'required' in kwargs: @@ -15,32 +24,57 @@ def __init__(self, *args, **kwargs): def clicked(self, required=False): raise NotImplementedError() + HELP_MULTIPLE = "; may be used more than once" + # we'll use tests of the -h option in the .travis.yml to ensure that the # .clicked methods work class Option(AbstractParameter): + """Wrapper for click.option decorators""" def clicked(self, required=False): # no-cov + """Create the click decorator""" return click.option(*self.args, **self.kwargs, required=required) + class Argument(AbstractParameter): + """Wrapper for click.argument decorators""" def clicked(self, required=False): # no-cov + """Create the click decorator""" return click.argument(*self.args, **self.kwargs, required=required) class AbstractLoader(object): + """Abstract object for getting relevant OPS object from the CLI. + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + """ def __init__(self, param): self.param = param @property def clicked(self): # no-cov + """Create the click decorator""" return self.param.clicked def get(self, *args, **kwargs): + """Get the desired OPS object, based on the CLI input""" raise NotImplementedError() class StorageLoader(AbstractLoader): + """Open an OPS storage file + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + mode : 'r', 'w', or 'a' + the mode for the file + """ def __init__(self, param, mode): super(StorageLoader, self).__init__(param) self.mode = mode @@ -61,12 +95,34 @@ def get(self, name): class OPSStorageLoadNames(AbstractLoader): """Simple loader that expects its input to be a name or index. + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + store : Str + the name of the store to search """ def __init__(self, param, store): super(OPSStorageLoadNames, self).__init__(param) self.store = store def get(self, storage, names): + """Get the names from the storage + + Parameters + ---------- + storage : :class:`openpathsampling.Storage` + storage file to search in + names : List[Str] + names or numbers (as string) to use as keys to load from + storage + + Returns + ------- + List[Any] : + the desired objects + """ int_corrected = [] for name in names: try: @@ -80,6 +136,13 @@ def get(self, storage, names): class Getter(object): + """Abstract strategy for getting things from storage + + Parameters + ---------- + store_name : Str + the name of the storage to search + """ def __init__(self, store_name): self.store_name = store_name @@ -90,11 +153,15 @@ def _get(self, storage, name): except: return None + class GetByName(Getter): + """Strategy using the CLI input as name for a stored item""" def __call__(self, storage, name): return self._get(storage, name) + class GetByNumber(Getter): + """Strategy using the CLI input as numeric index of the stored item""" def __call__(self, storage, name): try: num = int(name) @@ -103,7 +170,9 @@ def __call__(self, storage, name): return self._get(storage, num) + class GetPredefinedName(Getter): + """Strategy predefining name and store, allow default names""" def __init__(self, store_name, name): super().__init__(store_name=store_name) self.name = name @@ -111,20 +180,26 @@ def __init__(self, store_name, name): def __call__(self, storage): return self._get(storage, self.name) + class GetOnly(Getter): + """Strategy getting item from store if it is the only one""" def __call__(self, storage): store = getattr(storage, self.store_name) if len(store) == 1: return store[0] + class GetOnlyNamed(Getter): + """Strategy selecting item from store if it is the only named item""" def __call__(self, storage): store = getattr(storage, self.store_name) named_things = [o for o in store if o.is_named] if len(named_things) == 1: return named_things[0] + class GetOnlySnapshot(Getter): + """Strategy selecting only snapshot from a snapshot store""" def __init__(self, store_name="snapshots"): super().__init__(store_name) @@ -147,7 +222,24 @@ class OPSStorageLoadSingle(AbstractLoader): """Objects that expect to load a single object. These can sometimes include guesswork to figure out which object is - desired. + desired. The details of how that guesswork is performed is determined + by the strategy lists that are given. + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + store : Str + the name of the store to search + value_strategies : List[Callable[(:class:`.Storage`, Str), Any]] + The strategies to be used when the CLI provides a value for this + parameter. Each should be a callable taking a storage and the string + input from the CLI, and should return the desired object or None if + it cannot be found. + none_strategies : List[Callable[:class:`openpathsampling.Storage, Any]] + The strategies to be used when the CLI does not provide a value for + this parameter. Each should be a callable taking a storage, and + returning the desired object or None if it cannot be found. """ def __init__(self, param, store, value_strategies=None, none_strategies=None): @@ -164,9 +256,16 @@ def __init__(self, param, store, value_strategies=None, self.none_strategies = none_strategies def get(self, storage, name): - store = getattr(storage, self.store) - # num_store = getattr(storage, self.num_store) - + """Load desired object from storage. + + Parameters + ---------- + storage : openpathsampling.Storage + the input storage to search + name : Str or None + string from CLI providing the identifier (name or index) for + this object; None if not provided + """ if name is not None: result = _try_strategies(self.value_strategies, storage, name=name) @@ -178,6 +277,7 @@ def get(self, storage, name): return result + ENGINE = OPSStorageLoadSingle( param=Option('-e', '--engine', help="identifer for the engine"), store='engines', @@ -187,7 +287,6 @@ def get(self, storage, name): SCHEME = OPSStorageLoadSingle( param=Option('-m', '--scheme', help="identifier for the move scheme"), store='schemes', - # fallback=None ) INIT_CONDS = OPSStorageLoadSingle( From f1bb7a667d0fa055ffa00ce1eeaf92d4bc5f074d Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 3 Mar 2020 15:32:24 +0100 Subject: [PATCH 15/19] Split off param_core from parameters --- paths_cli/file_copying.py | 5 +- paths_cli/param_core.py | 275 +++++++++++++++++++++++++++++++++++++ paths_cli/parameters.py | 281 +------------------------------------- 3 files changed, 283 insertions(+), 278 deletions(-) create mode 100644 paths_cli/param_core.py diff --git a/paths_cli/file_copying.py b/paths_cli/file_copying.py index 4050ed25..825aafb6 100644 --- a/paths_cli/file_copying.py +++ b/paths_cli/file_copying.py @@ -6,9 +6,10 @@ import click from tqdm.auto import tqdm -from paths_cli.parameters import ( - Option, Argument, HELP_MULTIPLE, StorageLoader, OPSStorageLoadNames +from paths_cli.param_core import ( + Option, Argument, StorageLoader, OPSStorageLoadNames ) +from paths_cli.parameters import HELP_MULTIPLE INPUT_APPEND_FILE = StorageLoader( param=Argument('append_file', diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py new file mode 100644 index 00000000..3d379b58 --- /dev/null +++ b/paths_cli/param_core.py @@ -0,0 +1,275 @@ +import click +import os + + +class AbstractParameter(object): + """Abstract wrapper for click parameters. + + Forbids use setting ``required``, because we do that for each command + individually. + + Parameters + ---------- + args : + args to pass to click parameters + kwargs : + kwargs to pass to click parameters + """ + def __init__(self, *args, **kwargs): + self.args = args + if 'required' in kwargs: + raise ValueError("Can't set required status now") + self.kwargs = kwargs + + def clicked(self, required=False): + raise NotImplementedError() + + +# we'll use tests of the -h option in the .travis.yml to ensure that the +# .clicked methods work +class Option(AbstractParameter): + """Wrapper for click.option decorators""" + def clicked(self, required=False): # no-cov + """Create the click decorator""" + return click.option(*self.args, **self.kwargs, required=required) + + +class Argument(AbstractParameter): + """Wrapper for click.argument decorators""" + def clicked(self, required=False): # no-cov + """Create the click decorator""" + return click.argument(*self.args, **self.kwargs, required=required) + + +class AbstractLoader(object): + """Abstract object for getting relevant OPS object from the CLI. + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + """ + def __init__(self, param): + self.param = param + + @property + def clicked(self): # no-cov + """Create the click decorator""" + return self.param.clicked + + def get(self, *args, **kwargs): + """Get the desired OPS object, based on the CLI input""" + raise NotImplementedError() + + +class StorageLoader(AbstractLoader): + """Open an OPS storage file + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + mode : 'r', 'w', or 'a' + the mode for the file + """ + def __init__(self, param, mode): + super(StorageLoader, self).__init__(param) + self.mode = mode + + def _workaround(self, name): + # this is messed up... for some reason, storage doesn't create a new + # file in append mode. That may be a bug + import openpathsampling as paths + if self.mode == 'a' and not os.path.exists(name): + st = paths.Storage(name, mode='w') + st.close() + + def get(self, name): + import openpathsampling as paths + self._workaround(name) + return paths.Storage(name, mode=self.mode) + + +class OPSStorageLoadNames(AbstractLoader): + """Simple loader that expects its input to be a name or index. + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + store : Str + the name of the store to search + """ + def __init__(self, param, store): + super(OPSStorageLoadNames, self).__init__(param) + self.store = store + + def get(self, storage, names): + """Get the names from the storage + + Parameters + ---------- + storage : :class:`openpathsampling.Storage` + storage file to search in + names : List[Str] + names or numbers (as string) to use as keys to load from + storage + + Returns + ------- + List[Any] : + the desired objects + """ + int_corrected = [] + for name in names: + try: + name = int(name) + except ValueError: + pass + int_corrected.append(name) + + return [getattr(storage, self.store)[name] + for name in int_corrected] + + +class Getter(object): + """Abstract strategy for getting things from storage + + Parameters + ---------- + store_name : Str + the name of the storage to search + """ + def __init__(self, store_name): + self.store_name = store_name + + def _get(self, storage, name): + store = getattr(storage, self.store_name) + try: + return store[name] + except: + return None + + +class GetByName(Getter): + """Strategy using the CLI input as name for a stored item""" + def __call__(self, storage, name): + return self._get(storage, name) + + +class GetByNumber(Getter): + """Strategy using the CLI input as numeric index of the stored item""" + def __call__(self, storage, name): + try: + num = int(name) + except: + return None + + return self._get(storage, num) + + +class GetPredefinedName(Getter): + """Strategy predefining name and store, allow default names""" + def __init__(self, store_name, name): + super().__init__(store_name=store_name) + self.name = name + + def __call__(self, storage): + return self._get(storage, self.name) + + +class GetOnly(Getter): + """Strategy getting item from store if it is the only one""" + def __call__(self, storage): + store = getattr(storage, self.store_name) + if len(store) == 1: + return store[0] + + +class GetOnlyNamed(Getter): + """Strategy selecting item from store if it is the only named item""" + def __call__(self, storage): + store = getattr(storage, self.store_name) + named_things = [o for o in store if o.is_named] + if len(named_things) == 1: + return named_things[0] + + +class GetOnlySnapshot(Getter): + """Strategy selecting only snapshot from a snapshot store""" + def __init__(self, store_name="snapshots"): + super().__init__(store_name) + + def __call__(self, storage): + store = getattr(storage, self.store_name) + if len(store) == 2: + # this is really only 1 snapshot; reversed copy gets saved + return store[0] + + +def _try_strategies(strategies, storage, **kwargs): + result = None + for strategy in strategies: + result = strategy(storage, **kwargs) + if result is not None: + return result + + +class OPSStorageLoadSingle(AbstractLoader): + """Objects that expect to load a single object. + + These can sometimes include guesswork to figure out which object is + desired. The details of how that guesswork is performed is determined + by the strategy lists that are given. + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + store : Str + the name of the store to search + value_strategies : List[Callable[(:class:`.Storage`, Str), Any]] + The strategies to be used when the CLI provides a value for this + parameter. Each should be a callable taking a storage and the string + input from the CLI, and should return the desired object or None if + it cannot be found. + none_strategies : List[Callable[:class:`openpathsampling.Storage, Any]] + The strategies to be used when the CLI does not provide a value for + this parameter. Each should be a callable taking a storage, and + returning the desired object or None if it cannot be found. + """ + def __init__(self, param, store, value_strategies=None, + none_strategies=None): + super(OPSStorageLoadSingle, self).__init__(param) + self.store = store + if value_strategies is None: + value_strategies = [GetByName(self.store), + GetByNumber(self.store)] + self.value_strategies = value_strategies + + if none_strategies is None: + none_strategies = [GetOnly(self.store), + GetOnlyNamed(self.store)] + self.none_strategies = none_strategies + + def get(self, storage, name): + """Load desired object from storage. + + Parameters + ---------- + storage : openpathsampling.Storage + the input storage to search + name : Str or None + string from CLI providing the identifier (name or index) for + this object; None if not provided + """ + if name is not None: + result = _try_strategies(self.value_strategies, storage, + name=name) + else: + result = _try_strategies(self.none_strategies, storage) + + if result is None: + raise RuntimeError("Couldn't find %s", name) + + return result diff --git a/paths_cli/parameters.py b/paths_cli/parameters.py index 9832bccc..70b8ec4c 100644 --- a/paths_cli/parameters.py +++ b/paths_cli/parameters.py @@ -1,287 +1,16 @@ import click -import os - - -class AbstractParameter(object): - """Abstract wrapper for click parameters. - - Forbids use setting ``required``, because we do that for each command - individually. - - Parameters - ---------- - args : - args to pass to click parameters - kwargs : - kwargs to pass to click parameters - """ - def __init__(self, *args, **kwargs): - self.args = args - if 'required' in kwargs: - raise ValueError("Can't set required status now") - self.kwargs = kwargs - - def clicked(self, required=False): - raise NotImplementedError() +from paths_cli.param_core import ( + Option, Argument, OPSStorageLoadSingle, OPSStorageLoadNames, + StorageLoader, GetByName, GetByNumber, GetOnly, GetOnlySnapshot, + GetPredefinedName +) HELP_MULTIPLE = "; may be used more than once" - -# we'll use tests of the -h option in the .travis.yml to ensure that the -# .clicked methods work -class Option(AbstractParameter): - """Wrapper for click.option decorators""" - def clicked(self, required=False): # no-cov - """Create the click decorator""" - return click.option(*self.args, **self.kwargs, required=required) - - -class Argument(AbstractParameter): - """Wrapper for click.argument decorators""" - def clicked(self, required=False): # no-cov - """Create the click decorator""" - return click.argument(*self.args, **self.kwargs, required=required) - - -class AbstractLoader(object): - """Abstract object for getting relevant OPS object from the CLI. - - Parameters - ---------- - param : :class:`.AbstractParameter` - the Option or Argument wrapping a click decorator - """ - def __init__(self, param): - self.param = param - - @property - def clicked(self): # no-cov - """Create the click decorator""" - return self.param.clicked - - def get(self, *args, **kwargs): - """Get the desired OPS object, based on the CLI input""" - raise NotImplementedError() - - -class StorageLoader(AbstractLoader): - """Open an OPS storage file - - Parameters - ---------- - param : :class:`.AbstractParameter` - the Option or Argument wrapping a click decorator - mode : 'r', 'w', or 'a' - the mode for the file - """ - def __init__(self, param, mode): - super(StorageLoader, self).__init__(param) - self.mode = mode - - def _workaround(self, name): - # this is messed up... for some reason, storage doesn't create a new - # file in append mode. That may be a bug - import openpathsampling as paths - if self.mode == 'a' and not os.path.exists(name): - st = paths.Storage(name, mode='w') - st.close() - - def get(self, name): - import openpathsampling as paths - self._workaround(name) - return paths.Storage(name, mode=self.mode) - - -class OPSStorageLoadNames(AbstractLoader): - """Simple loader that expects its input to be a name or index. - - Parameters - ---------- - param : :class:`.AbstractParameter` - the Option or Argument wrapping a click decorator - store : Str - the name of the store to search - """ - def __init__(self, param, store): - super(OPSStorageLoadNames, self).__init__(param) - self.store = store - - def get(self, storage, names): - """Get the names from the storage - - Parameters - ---------- - storage : :class:`openpathsampling.Storage` - storage file to search in - names : List[Str] - names or numbers (as string) to use as keys to load from - storage - - Returns - ------- - List[Any] : - the desired objects - """ - int_corrected = [] - for name in names: - try: - name = int(name) - except ValueError: - pass - int_corrected.append(name) - - return [getattr(storage, self.store)[name] - for name in int_corrected] - - -class Getter(object): - """Abstract strategy for getting things from storage - - Parameters - ---------- - store_name : Str - the name of the storage to search - """ - def __init__(self, store_name): - self.store_name = store_name - - def _get(self, storage, name): - store = getattr(storage, self.store_name) - try: - return store[name] - except: - return None - - -class GetByName(Getter): - """Strategy using the CLI input as name for a stored item""" - def __call__(self, storage, name): - return self._get(storage, name) - - -class GetByNumber(Getter): - """Strategy using the CLI input as numeric index of the stored item""" - def __call__(self, storage, name): - try: - num = int(name) - except: - return None - - return self._get(storage, num) - - -class GetPredefinedName(Getter): - """Strategy predefining name and store, allow default names""" - def __init__(self, store_name, name): - super().__init__(store_name=store_name) - self.name = name - - def __call__(self, storage): - return self._get(storage, self.name) - - -class GetOnly(Getter): - """Strategy getting item from store if it is the only one""" - def __call__(self, storage): - store = getattr(storage, self.store_name) - if len(store) == 1: - return store[0] - - -class GetOnlyNamed(Getter): - """Strategy selecting item from store if it is the only named item""" - def __call__(self, storage): - store = getattr(storage, self.store_name) - named_things = [o for o in store if o.is_named] - if len(named_things) == 1: - return named_things[0] - - -class GetOnlySnapshot(Getter): - """Strategy selecting only snapshot from a snapshot store""" - def __init__(self, store_name="snapshots"): - super().__init__(store_name) - - def __call__(self, storage): - store = getattr(storage, self.store_name) - if len(store) == 2: - # this is really only 1 snapshot; reversed copy gets saved - return store[0] - - -def _try_strategies(strategies, storage, **kwargs): - result = None - for strategy in strategies: - result = strategy(storage, **kwargs) - if result is not None: - return result - - -class OPSStorageLoadSingle(AbstractLoader): - """Objects that expect to load a single object. - - These can sometimes include guesswork to figure out which object is - desired. The details of how that guesswork is performed is determined - by the strategy lists that are given. - - Parameters - ---------- - param : :class:`.AbstractParameter` - the Option or Argument wrapping a click decorator - store : Str - the name of the store to search - value_strategies : List[Callable[(:class:`.Storage`, Str), Any]] - The strategies to be used when the CLI provides a value for this - parameter. Each should be a callable taking a storage and the string - input from the CLI, and should return the desired object or None if - it cannot be found. - none_strategies : List[Callable[:class:`openpathsampling.Storage, Any]] - The strategies to be used when the CLI does not provide a value for - this parameter. Each should be a callable taking a storage, and - returning the desired object or None if it cannot be found. - """ - def __init__(self, param, store, value_strategies=None, - none_strategies=None): - super(OPSStorageLoadSingle, self).__init__(param) - self.store = store - if value_strategies is None: - value_strategies = [GetByName(self.store), - GetByNumber(self.store)] - self.value_strategies = value_strategies - - if none_strategies is None: - none_strategies = [GetOnly(self.store), - GetOnlyNamed(self.store)] - self.none_strategies = none_strategies - - def get(self, storage, name): - """Load desired object from storage. - - Parameters - ---------- - storage : openpathsampling.Storage - the input storage to search - name : Str or None - string from CLI providing the identifier (name or index) for - this object; None if not provided - """ - if name is not None: - result = _try_strategies(self.value_strategies, storage, - name=name) - else: - result = _try_strategies(self.none_strategies, storage) - - if result is None: - raise RuntimeError("Couldn't find %s", name) - - return result - - ENGINE = OPSStorageLoadSingle( param=Option('-e', '--engine', help="identifer for the engine"), store='engines', - # fallback=None # for now... I'll add more tricks later ) SCHEME = OPSStorageLoadSingle( From 48bd295626ec64bfbfdd4fd4105c18569a75e645 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Thu, 12 Mar 2020 14:37:13 +0100 Subject: [PATCH 16/19] Fix up docs for things that have been moved --- docs/for_core/cli.rst | 4 +++- docs/full_cli.rst | 2 +- docs/sphinx-helpers/make_param_table.py | 11 ++++++----- paths_cli/cli.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/for_core/cli.rst b/docs/for_core/cli.rst index 2277e4ee..bf0d0c27 100644 --- a/docs/for_core/cli.rst +++ b/docs/for_core/cli.rst @@ -81,11 +81,13 @@ the CLI: * ``contents``: list all the named objects in an OPS storage, organized by store (type); this is extremely useful to get the name of an object to use - as command-line input to one of the simulation scripts + + .. * ``strip-snapshots``: create a copy of the input storage file with the details (coordinates/velocities) of all snapshots removed; this allows you to make a much smaller copy (with results of CVs) to copy back to a local computer for analysis + * ``append`` : add an object from once OPS storage into another one; this is useful for getting everything into a single file before running a simulation diff --git a/docs/full_cli.rst b/docs/full_cli.rst index c5327259..181613e2 100644 --- a/docs/full_cli.rst +++ b/docs/full_cli.rst @@ -9,6 +9,6 @@ currently gives all commands alphabetically, without regard to section. In general, this is not yet well-organized; contributions to improve that would be appreciated. -.. click:: paths_cli.cli:OPS_CLI +.. click:: paths_cli.cli:main :prog: openpathsampling :show-nested: diff --git a/docs/sphinx-helpers/make_param_table.py b/docs/sphinx-helpers/make_param_table.py index d31b5ecc..f9dd5bd8 100644 --- a/docs/sphinx-helpers/make_param_table.py +++ b/docs/sphinx-helpers/make_param_table.py @@ -12,6 +12,7 @@ import click from paths_cli import parameters +from paths_cli import param_core TableEntry = collections.namedtuple("TableEntry", "param flags get_args help") @@ -20,7 +21,7 @@ def is_click_decorator(thing): return getattr(thing, '__module__', None) == 'click.decorators' def is_parameter(thing): - return (isinstance(thing, parameters.AbstractLoader) + return (isinstance(thing, param_core.AbstractLoader) or is_click_decorator(thing)) def rst_wrap(code): @@ -33,10 +34,10 @@ def flags_help(click_decorator): return flags, help_ def get_args(parameter): - if isinstance(parameter, parameters.StorageLoader): + if isinstance(parameter, param_core.StorageLoader): return "``name``" - elif isinstance(parameter, (parameters.OPSStorageLoadNames, - parameters.OPSStorageLoadSingle)): + elif isinstance(parameter, (param_core.OPSStorageLoadNames, + param_core.OPSStorageLoadSingle)): return "``storage``, ``name``" elif is_click_decorator(parameter): return "No ``get`` function" @@ -44,7 +45,7 @@ def get_args(parameter): return "Unknown" def get_click_decorator(thing): - if isinstance(thing, parameters.AbstractLoader): + if isinstance(thing, param_core.AbstractLoader): return thing.param.clicked() elif is_click_decorator(thing): return thing diff --git a/paths_cli/cli.py b/paths_cli/cli.py index 56ef8b77..7d3097f0 100644 --- a/paths_cli/cli.py +++ b/paths_cli/cli.py @@ -146,4 +146,4 @@ def main(log): logger.debug("About to run command") # TODO: maybe log invocation? if __name__ == '__main__': # no-cov - cli() + main() From e27a1b29dbf2e9f626ab54e2a23c67add17c8302 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 8 Apr 2020 01:42:31 +0200 Subject: [PATCH 17/19] Release 0.0.3 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 497056eb..2fbfc3a2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = openpathsampling-cli -version = 0.0.3.dev0 +version = 0.0.3 # version should end in .dev0 if this isn't to be released description = Command line tool for OpenPathSampling long_description = file: README.md From 9d4847e06b799a2db426712e43ca59df0b7a2ac6 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 8 Apr 2020 02:30:31 +0200 Subject: [PATCH 18/19] small pylint fixes --- paths_cli/file_copying.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paths_cli/file_copying.py b/paths_cli/file_copying.py index 825aafb6..aa3e801d 100644 --- a/paths_cli/file_copying.py +++ b/paths_cli/file_copying.py @@ -17,6 +17,7 @@ mode='a' ) + class PrecomputeLoadNames(OPSStorageLoadNames): def get(self, storage, name): if len(name) == 0: @@ -26,6 +27,7 @@ def get(self, storage, name): return super(PrecomputeLoadNames, self).get(storage, name) + PRECOMPUTE_CVS = PrecomputeLoadNames( param=Option('--cv', type=str, multiple=True, help=('name of CV to precompute; if not specified all will' From 600edbd984ee70f9ab679594da7ecc5bdf59112a Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 8 Apr 2020 12:44:05 +0200 Subject: [PATCH 19/19] Add redirect follow to curl --- devtools/miniconda_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/miniconda_install.sh b/devtools/miniconda_install.sh index 28566ac1..96c4ef09 100644 --- a/devtools/miniconda_install.sh +++ b/devtools/miniconda_install.sh @@ -22,7 +22,7 @@ conda_version="latest" #conda_version="4.4.10" # can pin a miniconda version like this, if needed MINICONDA=Miniconda${pyV}-${conda_version}-${OS_ARCH}.sh -MINICONDA_MD5=$(curl -s https://repo.continuum.io/miniconda/ | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *\(.*\)<\/td> */\1/p') +MINICONDA_MD5=$(curl -sL https://repo.continuum.io/miniconda/ | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *\(.*\)<\/td> */\1/p') wget https://repo.continuum.io/miniconda/$MINICONDA SCRIPT_MD5=`eval "$MD5_CMD $MD5_OPT $MINICONDA" | cut -d ' ' -f 1`