diff --git a/.travis.yml b/.travis.yml index 1778659..a47826d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,19 @@ language: python python: - '3.6' +before_install: +- openssl aes-256-cbc -K $encrypted_04ab0de0cbc0_key -iv $encrypted_04ab0de0cbc0_iv + -in test-files/DLHub_Client_tokens.json.enc -out test-files/DLHub_Client_tokens.json + -d +- mkdir -p ~/.dlhub/credentials +- cp test-files/DLHub_Client_tokens.json ~/.dlhub/credentials install: - pip install -e . - pip install --only-binary=numpy,scipy -r requirements.txt -- pip install coveralls +- pip install coveralls flake8 - pip install pytest pytest-cov --upgrade script: +- flake8 - pytest --show-capture stdout after_success: - coveralls diff --git a/dlhub_cli/__init__.py b/dlhub_cli/__init__.py index 9e1be4d..ffca8e9 100644 --- a/dlhub_cli/__init__.py +++ b/dlhub_cli/__init__.py @@ -1,4 +1,3 @@ from dlhub_cli.main import cli_root - -__all__ = ['cli_root'] \ No newline at end of file +__all__ = ['cli_root'] diff --git a/dlhub_cli/commands/__init__.py b/dlhub_cli/commands/__init__.py index 1ef8241..61c210d 100644 --- a/dlhub_cli/commands/__init__.py +++ b/dlhub_cli/commands/__init__.py @@ -1,15 +1,17 @@ +from dlhub_cli.commands.describe import describe_cmd from dlhub_cli.commands.init import init_cmd -from dlhub_cli.commands.publish import publish_cmd -from dlhub_cli.commands.update import update_cmd -from dlhub_cli.commands.run import run_cmd -from dlhub_cli.commands.ls import ls_cmd -from dlhub_cli.commands.status import status_cmd from dlhub_cli.commands.login import login_cmd from dlhub_cli.commands.logout import logout_cmd +from dlhub_cli.commands.ls import ls_cmd +from dlhub_cli.commands.methods import methods_cmd +from dlhub_cli.commands.publish import publish_cmd +from dlhub_cli.commands.run import run_cmd +from dlhub_cli.commands.search import search_cmd from dlhub_cli.commands.servables import servables_cmd -from dlhub_cli.commands.describe import describe_cmd +from dlhub_cli.commands.status import status_cmd +from dlhub_cli.commands.whoami import whoami_cmd __all__ = ['init_cmd', 'publish_cmd', 'servables_cmd', - 'update_cmd', 'run_cmd', 'ls_cmd', + 'run_cmd', 'ls_cmd', 'methods_cmd', 'status_cmd', 'login_cmd', 'logout_cmd', - 'describe_cmd'] \ No newline at end of file + 'describe_cmd', 'whoami_cmd', 'search_cmd'] diff --git a/dlhub_cli/commands/describe.py b/dlhub_cli/commands/describe.py index 1c823d8..47e3590 100644 --- a/dlhub_cli/commands/describe.py +++ b/dlhub_cli/commands/describe.py @@ -1,48 +1,89 @@ +from datetime import datetime import click +import yaml from dlhub_cli.config import get_dlhub_client from dlhub_cli.printing import format_output from dlhub_cli.parsing import dlhub_cmd -HELP_STR = """\ +_unwanted_fields = [ + ('dlhub', 'build_location'), + ('dlhub', 'ecr_arn'), + ('dlhub', 'ecr_uri'), + ('dlhub', 'id'), + ('dlhub', 'transfer_method'), + ('dlhub', 'user_id') +] - Describe a servable. -\b -Options: - --id TEXT The UUID of a servable. - --name TEXT The name of a servable. - -h, --help Show this message and exit. - -v, --version Show the version and exit.""" +def _remove_field(metadata, field): + """Remove a certain field from the metadata -@dlhub_cmd('describe', help='Describe a servable.') -@click.option('--id', - show_default=True, required=False, - help='The UUID of a servable.') -@click.option('--name', - show_default=True, required=False, - help='The name of a servable.') -def describe_cmd(id, name): + Args: + metadata (dict): Metadata to be pruned + field ([string]): Coordinates of fields to be removed + """ + + if len(field) == 1: + if field[0] in metadata: + del metadata[field[0]] + else: + if field[0] in metadata: + subset = metadata[field[0]] + return _remove_field(subset, field[1:]) + + +def _preprocess_metadata(metadata): + """Clean up a metadata record to make it more useful to humans + + Args: + metadata (metadata): Metadata record to be cleaned + """ + # Prune internal-only fields + for field in _unwanted_fields: + _remove_field(metadata, field) + + # Turn Timestamp (epoch time in ms) into a String + metadata['dlhub']['publication_date'] = \ + datetime.fromtimestamp(int(metadata['dlhub']['publication_date']) / 1000) \ + .strftime('%Y-%m-%d %H:%M') + + +@dlhub_cmd('describe', short_help="Get the description of a servable", + help="""Get the description of a servable + + OWNER is the username of the owner of the servable, and NAME is the name of the servable. + + You can optionally specify both owner and servable name as a single argument using + a "owner_name/servable_name" format + """) +@click.argument('owner', default=None) +@click.argument('name', default="") +def describe_cmd(owner, name): """Use DLHub to get a description of the servable. Args: - id (string): The uuid of the servable + owner (string): The owner of the servable name (string): The name of the servable Returns: (dict) a set of information regarding the servable """ - if not any([id, name]): - format_output(HELP_STR) - return + # Check if owner contains both + if name == "": + temp = owner.split("/") + if len(temp) != 2: + raise click.BadArgumentUsage('Model name missing') + owner, name = temp + # Retrieve the metadata client = get_dlhub_client() - res = "Unable to describe a servable (name: {0}, id: {1})".format(name, id) - if id: - res = client.describe_servable(servable_id=id) - elif name: - res = client.describe_servable(servable_name=name) + res = client.describe_servable(owner, name) + + # Clean up the metadata fields + _preprocess_metadata(res) - format_output("{0}".format(res)) + # Print it to screen + format_output(yaml.dump(res)) return res diff --git a/dlhub_cli/commands/init.py b/dlhub_cli/commands/init.py index afcb9c0..7e34316 100644 --- a/dlhub_cli/commands/init.py +++ b/dlhub_cli/commands/init.py @@ -66,7 +66,8 @@ def init_cmd(force, filename, author, title, name, skip_run): # Check if a file would be overwritten if os.path.isfile(filename) and not force: - format_output("There is already a file named '{}'. Use --force to overwrite".format(filename)) + format_output("There is already a file named '{}'." + " Use --force to overwrite".format(filename)) return 1 # Prepare a dict of objects to be replaced diff --git a/dlhub_cli/commands/login.py b/dlhub_cli/commands/login.py index 3a78cf5..be73ccd 100644 --- a/dlhub_cli/commands/login.py +++ b/dlhub_cli/commands/login.py @@ -1,6 +1,7 @@ import click from dlhub_cli.config import get_dlhub_client + @click.command('login', short_help=('Log into Globus to get credentials for ' 'the DLHub CLI'), @@ -16,4 +17,4 @@ def login_cmd(force): force (bool): Whether or not to force the login. """ - client = get_dlhub_client(force) + get_dlhub_client(force) diff --git a/dlhub_cli/commands/logout.py b/dlhub_cli/commands/logout.py index 4bfdcf7..c3a0b7d 100644 --- a/dlhub_cli/commands/logout.py +++ b/dlhub_cli/commands/logout.py @@ -2,6 +2,7 @@ from dlhub_cli.config import get_dlhub_client + @click.command('logout', short_help='Logout of the DLHub CLI', help=('Logout of the DLHub CLI. ' @@ -17,4 +18,4 @@ def logout_cmd(): """ client = get_dlhub_client() - client.logout() \ No newline at end of file + client.logout() diff --git a/dlhub_cli/commands/ls.py b/dlhub_cli/commands/ls.py index 29122b2..4e79093 100644 --- a/dlhub_cli/commands/ls.py +++ b/dlhub_cli/commands/ls.py @@ -1,8 +1,7 @@ import os -import json from dlhub_cli.printing import format_output -from dlhub_cli.parsing import dlhub_cmd, index_argument +from dlhub_cli.parsing import dlhub_cmd @dlhub_cmd('ls', help='List servables in this directory') @@ -19,4 +18,4 @@ def ls_cmd(): for serv in servables: res = serv.split(".json")[0] - format_output(res) \ No newline at end of file + format_output(res) diff --git a/dlhub_cli/commands/methods.py b/dlhub_cli/commands/methods.py new file mode 100644 index 0000000..43c1b02 --- /dev/null +++ b/dlhub_cli/commands/methods.py @@ -0,0 +1,59 @@ +import yaml +import click + +from dlhub_cli.parsing import dlhub_cmd +from dlhub_cli.printing import format_output +from dlhub_cli.config import get_dlhub_client + + +@dlhub_cmd('methods', short_help='Print method information', + help='''Print methods for a certain servable + + OWNER is the username of the servable owner, NAME is the name of the servable. + METHOD is optional. If provided, this command will only print the information + for the method with that name. Otherwise, it will print the information for all + methods implemented by the servable. + + You can optionally specify the servable name is owner_name/model_name format + ''') +@click.argument('owner', default=None) +@click.argument('name', default='') +@click.argument('method', default='') +def methods_cmd(owner, name, method): + """Print out the methods of a servable + + Args: + owner (str): Name of the servable's owner + name (str): Name of the servable + method (str): Name of the method (optional) + """ + + # Check if the owner sent model information in owner/model format + if '/' in owner: + # Get the owner and model name + temp = owner.split('/') + if len(temp) != 2: + raise click.BadArgumentUsage('Expected owner_name/model_name format') + + # If "name" is provided, it is actually the method name + if name != '': + method = name + + owner, name = temp + + # Make sure the name was provided + if name == '': + raise click.BadArgumentUsage('Model name not provided') + + # If the method name is blank, make it None (for compatibility with client) + if method == '': + method = None + + # Get the DLHub client + client = get_dlhub_client() + + # Get the method details + method_details = client.describe_methods(owner, name, method) + + # Print them in YAML format + format_output(yaml.dump(method_details, default_flow_style=False)) diff --git a/dlhub_cli/commands/publish.py b/dlhub_cli/commands/publish.py index 076b09e..fb4bdfb 100644 --- a/dlhub_cli/commands/publish.py +++ b/dlhub_cli/commands/publish.py @@ -1,10 +1,8 @@ -import os import json import click -import pickle as pkl from dlhub_cli.config import get_dlhub_client -from dlhub_cli.printing import format_output, safeprint +from dlhub_cli.printing import format_output from dlhub_cli.parsing import dlhub_cmd from dlhub_sdk.utils import unserialize_object @@ -29,14 +27,16 @@ default=None, show_default=True, help='The repository to publish.') def publish_cmd(local, repository): - """Publish a model to DLHub. Either a metadata description file for a local servable or a remote github address. + """Publish a model to DLHub. Either a metadata description file for a + local servable or a remote github address. The servable's metadata will be sent to the DLHub service. If using a local servable the files described in the - metadata's 'files' field will be zipped into a tmp archive and staged to S3. An ingestion pipeline will then - download the data from S3, build the servable, and publish the servable to DLHub. + metadata's 'files' field will be zipped into a tmp archive + and then sent to DLHub via HTTPS - When using a repository the github url is passed to a specific publish_repo endpoint on DLHub. This will + When using a repository the github url is passed to a + specific publish_repo endpoint on DLHub. This will use repo2docker to build the servable and publish it. Args: @@ -50,8 +50,6 @@ def publish_cmd(local, repository): format_output(HELP_STR) return - loaded_servable = None - client = get_dlhub_client() res = None if local: @@ -78,7 +76,3 @@ def publish_cmd(local, repository): res = client.publish_repository(repository) format_output("Task_id: {}".format(res)) return res - - - - diff --git a/dlhub_cli/commands/run.py b/dlhub_cli/commands/run.py index ad47a46..ed950e5 100644 --- a/dlhub_cli/commands/run.py +++ b/dlhub_cli/commands/run.py @@ -1,8 +1,8 @@ import json import click -from dlhub_cli.printing import format_output, safeprint -from dlhub_cli.config import (get_dlhub_client) +from dlhub_cli.config import get_dlhub_client +from dlhub_cli.printing import format_output from dlhub_cli.parsing import dlhub_cmd # @click.option('--servable-uuid', @@ -30,7 +30,8 @@ default=None, show_default=True, required=False, help='Input to pass to the servable.') def run_cmd(servable, input): - """Invoke a servable. The input data will be read with json.loads(input) and passed to the servable. + """Invoke a servable. The input data will be read with + json.loads(input) and passed to the servable. Args: servable (string): The servable to invoke, e.g., "ryan_globusid/noop" @@ -53,4 +54,4 @@ def run_cmd(servable, input): res = client.run(namespace, model, data) format_output(res) - return res \ No newline at end of file + return res diff --git a/dlhub_cli/commands/search.py b/dlhub_cli/commands/search.py new file mode 100644 index 0000000..ec3e075 --- /dev/null +++ b/dlhub_cli/commands/search.py @@ -0,0 +1,74 @@ +"""Commands for basic searches from the commandline""" + +import click +import pandas as pd +from datetime import datetime +from tabulate import tabulate + +from dlhub_sdk.utils.search import DLHubSearchHelper, filter_latest + +from dlhub_cli.parsing import dlhub_cmd +from dlhub_cli.config import get_dlhub_client + + +@dlhub_cmd('search', + help='Search the servable index') +@click.option('--owner', help='Name of owner of the servable') +@click.option('--name', help='Name of the servable') +@click.option('--all', 'all_versions', help='Get all versions of each model', + is_flag=True) +@click.option('--author', help='Search by author name. Must be in form "Last, First"', + multiple=True) +@click.option('--domain', help='Domain of the servable (e.g., chemistry)', + multiple=True) +@click.option('--doi', help='DOI of an associated publication') +@click.argument('query', nargs=-1) +def search_cmd(owner, name, all_versions, author, domain, doi, query): + """Search command + + See above for argument details + """ + + # Get the client + client = get_dlhub_client() + + # Start the query object + query = DLHubSearchHelper(client._search_client, q="(" + " ".join(query), advanced=True) + + # Add the filters + query.match_owner(owner) + query.match_servable(servable_name=name) + query.match_authors(author) + query.match_domains(domain) + query.match_doi(doi) + + # If no query strings are given, return an error + if not query.initialized: + click.echo('Error: No query specified. For options, call: dlhub search --help') + click.get_current_context().exit(1) + + # Perform the query + results = query.search() + + # If no results, return nothing + if len(results) == 0: + click.echo('No results') + return + + # If desired, filter the entries + if not all_versions: + results = filter_latest(results) + + # Get only a subset of the data and print it as a table + results_df = pd.DataFrame([{ + 'Owner': r['dlhub']['owner'], + 'Model Name': r['dlhub']['name'], + 'Publication Date': datetime.fromtimestamp(int(r['dlhub']['publication_date']) / + 1000).strftime('%Y-%m-%d %H:%M'), + 'Type': r['servable']['type'] + } for r in results]) + + results_df.sort_values(['Owner', 'Model Name', 'Publication Date'], + ascending=[True, True, False], inplace=True) + + click.echo(tabulate(results_df.values, headers=results_df.columns)) diff --git a/dlhub_cli/commands/servables.py b/dlhub_cli/commands/servables.py index b71e133..d312097 100644 --- a/dlhub_cli/commands/servables.py +++ b/dlhub_cli/commands/servables.py @@ -1,5 +1,3 @@ -import click - from dlhub_cli.config import get_dlhub_client from dlhub_cli.printing import format_output from dlhub_cli.parsing import dlhub_cmd @@ -17,4 +15,4 @@ def servables_cmd(): res = client.list_servables() format_output("{0}".format(res)) - return res \ No newline at end of file + return res diff --git a/dlhub_cli/commands/tests/test_describe.py b/dlhub_cli/commands/tests/test_describe.py new file mode 100644 index 0000000..bc8f442 --- /dev/null +++ b/dlhub_cli/commands/tests/test_describe.py @@ -0,0 +1,31 @@ +import pytest +from click.testing import CliRunner +from dlhub_cli.commands.describe import describe_cmd + + +@pytest.fixture +def runner(): + return CliRunner() + + +def test_noargs(runner: CliRunner): + result = runner.invoke(describe_cmd) + assert result.exit_code > 0 + assert 'Missing' in result.output + + +def test_print(runner: CliRunner): + result = runner.invoke(describe_cmd, ['blaiszik_globusid', 'cherukara_phase']) + assert result.output.startswith('datacite') + assert 'Cherukara' in result.output + + +def test_single_arg(runner: CliRunner): + result = runner.invoke(describe_cmd, ['blaiszik_globusid/cherukara_phase']) + assert result.output.startswith('datacite') + assert 'Cherukara' in result.output + + # Test the model name being omitted + result = runner.invoke(describe_cmd, ['blaiszik_globusid']) + assert result.exit_code > 0 + assert 'Model name missing' in result.output diff --git a/dlhub_cli/commands/tests/test_methods.py b/dlhub_cli/commands/tests/test_methods.py new file mode 100644 index 0000000..e910f22 --- /dev/null +++ b/dlhub_cli/commands/tests/test_methods.py @@ -0,0 +1,55 @@ +import pytest + +from click.testing import CliRunner + +from dlhub_cli.commands.methods import methods_cmd + + +@pytest.fixture() +def runner(): + return CliRunner() + + +def test_long_args(runner: CliRunner): + # Leave off the model name + result = runner.invoke(methods_cmd, ['dlhub.test_gmail']) + assert result.exit_code > 0 + assert 'Model name not provided' in result.output + + # Leave off the method name, should give all methods + result = runner.invoke(methods_cmd, ['dlhub.test_gmail', '1d_norm']) + assert result.exit_code == 0 + assert result.output.startswith('run') + + # Provide the method name + result = runner.invoke(methods_cmd, ['dlhub.test_gmail', '1d_norm', 'run']) + assert result.exit_code == 0 + assert result.output.startswith('input') + + # Provide a non-existant method name + result = runner.invoke(methods_cmd, ['dlhub.test_gmail', '1d_norm', 'notamethod']) + assert result.exit_code != 0 + assert 'No such method' in str(result.exception) + + +def test_short_args(runner: CliRunner): + """Using the owner/model syntax""" + # Leave off the model name + result = runner.invoke(methods_cmd, ['dlhub.test_gmail/']) + assert result.exit_code > 0 + assert 'Model name not provided' in result.output + + # Leave off the method name, should give all methods + result = runner.invoke(methods_cmd, ['dlhub.test_gmail/1d_norm']) + assert result.exit_code == 0 + assert result.output.startswith('run') + + # Provide the method name + result = runner.invoke(methods_cmd, ['dlhub.test_gmail/1d_norm', 'run']) + assert result.exit_code == 0 + assert result.output.startswith('input') + + # Provide a non-existant method name + result = runner.invoke(methods_cmd, ['dlhub.test_gmail/1d_norm', 'notamethod']) + assert result.exit_code != 0 + assert 'No such method' in str(result.exception) diff --git a/dlhub_cli/commands/tests/test_search.py b/dlhub_cli/commands/tests/test_search.py new file mode 100644 index 0000000..9c7ffb1 --- /dev/null +++ b/dlhub_cli/commands/tests/test_search.py @@ -0,0 +1,47 @@ +import pytest +from click.testing import CliRunner +from dlhub_cli.commands.search import search_cmd + + +@pytest.fixture() +def runner(): + return CliRunner() + + +def test_noargs(runner: CliRunner): + result = runner.invoke(search_cmd) + assert result.exit_code == 1 + assert 'No query' in result.output + + +def test_noresults(runner: CliRunner): + result = runner.invoke(search_cmd, ["--owner", "totallynotauser_uchicago"]) + assert result.exit_code == 0 + assert "No results" in result.output + + +def test_all_models(runner: CliRunner): + # Get only 1 version + result = runner.invoke(search_cmd, ["--owner", "dlhub.test_gmail", "--name", "1d_norm"]) + assert result.exit_code == 0 + assert result.output.count('dlhub.test_gmail') == 1 + + # Get all versions + result = runner.invoke(search_cmd, ["--owner", "dlhub.test_gmail", + "--name", "1d_norm", "--all"]) + assert result.exit_code == 0 + assert result.output.count('dlhub.test_gmail') > 1 + + +def test_manual_query(runner: CliRunner): + result = runner.invoke(search_cmd, ["dlhub.name:1d_norm", 'AND', + "dlhub.owner:dlhub.test_gmail"]) + assert result.exit_code == 0 + assert result.output.count('1d_norm') == 1 + + # Also add in a query term + result = runner.invoke(search_cmd, + ["dlhub.name:1d_norm", 'AND', "dlhub.owner:dlhub.test_gmail", + "--domain", "chemistry"]) + assert result.exit_code == 0 + assert 'No results' in result.output diff --git a/dlhub_cli/commands/tests/test_whoami.py b/dlhub_cli/commands/tests/test_whoami.py new file mode 100644 index 0000000..d8d5616 --- /dev/null +++ b/dlhub_cli/commands/tests/test_whoami.py @@ -0,0 +1,16 @@ +import os +import pytest +from click.testing import CliRunner +from dlhub_cli.commands.whoami import whoami_cmd + +# Check if we are on travis +# See: https://blog.travis-ci.com/august-2012-upcoming-ci-environment-updates +is_travis = 'HAS_JOSH_K_SEAL_OF_APPROVAL' in os.environ + + +@pytest.mark.skipif(not is_travis, reason='Only runs with credentials on Travis CI') +def test_whoami(): + runner = CliRunner() + res = runner.invoke(whoami_cmd) + assert res.exit_code == 0 + assert 'dlhub.test_gmail' in res.output diff --git a/dlhub_cli/commands/update.py b/dlhub_cli/commands/update.py deleted file mode 100644 index a21d878..0000000 --- a/dlhub_cli/commands/update.py +++ /dev/null @@ -1,21 +0,0 @@ -import click - -from dlhub_cli.printing import format_output - -from dlhub_cli.parsing import dlhub_cmd, index_argument - - -@dlhub_cmd('update', help='Update a servables metadata') -@click.option('--servable', - default=None, show_default=True, - help='The servable to update.') -def update_cmd(servable): - """Update the servable - - Args: - servable (string): The uuid of the servable to update - Returns: - (none) None. - """ - format_output("Update {}".format(servable)) - pass \ No newline at end of file diff --git a/dlhub_cli/commands/whoami.py b/dlhub_cli/commands/whoami.py new file mode 100644 index 0000000..34520e9 --- /dev/null +++ b/dlhub_cli/commands/whoami.py @@ -0,0 +1,11 @@ +import click + +from dlhub_cli.parsing import dlhub_cmd +from dlhub_cli.config import get_dlhub_client + + +@dlhub_cmd('whoami', + help='Get the username of logged in user') +def whoami_cmd(): + client = get_dlhub_client() + click.echo(client.get_username()) diff --git a/dlhub_cli/config.py b/dlhub_cli/config.py index 300b0e9..5dea182 100644 --- a/dlhub_cli/config.py +++ b/dlhub_cli/config.py @@ -10,14 +10,12 @@ CONF_SECTION_NAME = 'dlhub-cli' -DLHUB_CLIENT = DLHubClient - def get_dlhub_client(force=False): """Get the DLHub client Returns: - (DLHubClient) the dlhub client. + DLHubClient: DLHubClient with the proper credentials """ - return DLHUB_CLIENT.login(force=force) + return DLHubClient(force=force) diff --git a/dlhub_cli/main.py b/dlhub_cli/main.py index 90421ad..8560409 100644 --- a/dlhub_cli/main.py +++ b/dlhub_cli/main.py @@ -1,9 +1,10 @@ from dlhub_cli.parsing import main_func from dlhub_cli.commands import ( - init_cmd, publish_cmd, update_cmd, - run_cmd, ls_cmd, status_cmd, login_cmd, + init_cmd, publish_cmd, whoami_cmd, methods_cmd, + run_cmd, search_cmd, status_cmd, login_cmd, logout_cmd, servables_cmd, describe_cmd) + @main_func def cli_root(): """Root to add everything to. @@ -20,7 +21,8 @@ def cli_root(): cli_root.add_command(status_cmd) cli_root.add_command(login_cmd) cli_root.add_command(logout_cmd) +cli_root.add_command(methods_cmd) cli_root.add_command(servables_cmd) cli_root.add_command(describe_cmd) -# cli_root.add_command(update_cmd) -# cli_root.add_command(ls_cmd) \ No newline at end of file +cli_root.add_command(whoami_cmd) +cli_root.add_command(search_cmd) diff --git a/dlhub_cli/parsing/__init__.py b/dlhub_cli/parsing/__init__.py index d0f8b57..f300b27 100644 --- a/dlhub_cli/parsing/__init__.py +++ b/dlhub_cli/parsing/__init__.py @@ -1,8 +1,7 @@ from dlhub_cli.parsing.main import main_func -from dlhub_cli.parsing.click_wrappers import ( - dlhub_group, dlhub_cmd, index_argument) +from dlhub_cli.parsing.click_wrappers import dlhub_group, dlhub_cmd, index_argument __all__ = ( 'index_argument', 'dlhub_group', 'dlhub_cmd', 'main_func' -) \ No newline at end of file +) diff --git a/dlhub_cli/parsing/click_wrappers.py b/dlhub_cli/parsing/click_wrappers.py index bc915d6..06b33d6 100644 --- a/dlhub_cli/parsing/click_wrappers.py +++ b/dlhub_cli/parsing/click_wrappers.py @@ -135,6 +135,9 @@ def invoke(self, ctx): ctx.exit() try: return super(DLHubCommandGroup, self).invoke(ctx) + except click.ClickException as err: + click.echo(err.show()) + click.get_current_context().exit(1) except Exception as err: click.echo(err, err=True) click.get_current_context().exit(1) diff --git a/dlhub_cli/parsing/main.py b/dlhub_cli/parsing/main.py index 9b01229..2fe3d4f 100644 --- a/dlhub_cli/parsing/main.py +++ b/dlhub_cli/parsing/main.py @@ -9,5 +9,5 @@ def main_func(f): :return: """ f = dlhub_group('dlhub-client', - help='CLI Client to the DLHub API')(f) - return f \ No newline at end of file + help='CLI Client to the DLHub API')(f) + return f diff --git a/dlhub_cli/printing.py b/dlhub_cli/printing.py index cd7d95c..450d9fb 100644 --- a/dlhub_cli/printing.py +++ b/dlhub_cli/printing.py @@ -1,3 +1,5 @@ +"""Functions related to cleaner printing of the outputs""" + import six import sys import json @@ -8,8 +10,6 @@ def safeprint(s): Args: s (string): String to print. - Returns: - (none) none. """ try: print(s) @@ -23,10 +23,8 @@ def format_output(dataobject): Args: dataobject (string): String to print. - Returns: - (none) none. """ if isinstance(dataobject, six.string_types): safeprint(dataobject) else: - safeprint(json.dumps(dataobject, indent=2, separators=(',', ': '))) \ No newline at end of file + safeprint(json.dumps(dataobject, indent=2, separators=(',', ': '))) diff --git a/dlhub_cli/version.py b/dlhub_cli/version.py index 734be06..c27f74a 100644 --- a/dlhub_cli/version.py +++ b/dlhub_cli/version.py @@ -1,6 +1,6 @@ # single source of truth for package version, # see https://packaging.python.org/en/latest/single_source_version/ -__version__ = "0.4.2" +__version__ = "0.5.0" # app name to send as part of SDK requests app_name = "DLHub CLI v{}".format(__version__) diff --git a/docs/conf.py b/docs/conf.py index 1254dd7..dded6d8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -182,8 +182,5 @@ 'Miscellaneous'), ] - - - # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/source/dlhub_cli.commands.rst b/docs/source/dlhub_cli.commands.rst index 80d4155..4556173 100644 --- a/docs/source/dlhub_cli.commands.rst +++ b/docs/source/dlhub_cli.commands.rst @@ -1,74 +1,6 @@ dlhub\_cli\.commands package ============================ -Submodules ----------- - -dlhub\_cli\.commands\.init module ---------------------------------- - -.. automodule:: dlhub_cli.commands.init - :members: - :undoc-members: - :show-inheritance: - -dlhub\_cli\.commands\.login module ----------------------------------- - -.. automodule:: dlhub_cli.commands.login - :members: - :undoc-members: - :show-inheritance: - -dlhub\_cli\.commands\.logout module ------------------------------------ - -.. automodule:: dlhub_cli.commands.logout - :members: - :undoc-members: - :show-inheritance: - -dlhub\_cli\.commands\.ls module -------------------------------- - -.. automodule:: dlhub_cli.commands.ls - :members: - :undoc-members: - :show-inheritance: - -dlhub\_cli\.commands\.publish module ------------------------------------- - -.. automodule:: dlhub_cli.commands.publish - :members: - :undoc-members: - :show-inheritance: - -dlhub\_cli\.commands\.run module --------------------------------- - -.. automodule:: dlhub_cli.commands.run - :members: - :undoc-members: - :show-inheritance: - -dlhub\_cli\.commands\.status module ------------------------------------ - -.. automodule:: dlhub_cli.commands.status - :members: - :undoc-members: - :show-inheritance: - -dlhub\_cli\.commands\.update module ------------------------------------ - -.. automodule:: dlhub_cli.commands.update - :members: - :undoc-members: - :show-inheritance: - - Module contents --------------- diff --git a/docs/tutorial.rst b/docs/tutorial.rst index eee89cf..9c0a443 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -17,15 +17,17 @@ Once installed, the CLI can be accessed via the command 'dlhub':: -v, --version Show the version and exit. Commands: - describe Describe a servable. - init Initialize a DLHub servable + describe Get the description of a servable + init Initialize a DLHub servable. login Log into Globus to get credentials for the DLHub CLI logout Logout of the DLHub CLI + methods Print method information publish Publish a servable to DLHub. run Invoke a servable + search Search the servable index servables List the available servables. status Check the status of a DLHub task. - update Update a servables metadata + whoami Get the username of logged in user Authentication @@ -47,42 +49,76 @@ Logging in can also be explicitly invoked with the 'login' command:: $ dlhub login --force - (the --force flag will initiate a login flow even when the user is already logged in.) - +The ``--force`` flag will initiate a login flow even when the user is already logged in. Logging Out ----------- -The logout process is initiated using the logout command. This will invalidate and remove the credentials stored in the -~/.globus.cfg file and will force subsequent commands to initiate a new login flow.:: +The logout process is initiated using the logout command. This will invalidate and remove the credentials stored in +the ``~/.dlhub/credentials/`` directory and will force subsequent commands to initiate a new login flow.:: $ dlhub logout Finding Servables ^^^^^^^^^^^^^^^^^ -The CLI can be used to list the available servables accessible to the user. This command will provide a list of servables in (id, name) pairs:: +The CLI can be used to list the available servables accessible to the user. This command will provide a list of +servables that are currently available in DLHub:: $ dlhub servables +You can also search the index of DLHub servables with the 'search' command. +The 'search' command supports tags for common options, such as the owner:: + + $ dlhub search --owner blaiszik_globusid + + Model Name Owner Publication Date Type + --------------- ----------------- ------------------ ----------- + cherukara_phase blaiszik_globusid 2019-02-19 15:21 Keras Model + +Call ``dlhub search --help`` for a full listing of search tags. + +.. TODO: Add a link to DLHub search docs when available + +You can also provide a full query string using the DLHub query syntax:: + + $ dlhub search dlhub.owner:blaiszik_globusid AND servable.type:"Keras Model" + + Model Name Owner Publication Date Type + --------------- ----------------- ------------------ ----------- + cherukara_phase blaiszik_globusid 2019-02-19 15:21 Keras Model + Describing Servables ^^^^^^^^^^^^^^^^^^^^ -The 'describe' command queries the service for additional information about a servable. This information includes details on how to invoke the servable, required inputs, and expected outputs. -The describe command can be passed either the servables text name or identifier:: +The 'describe' command queries the service for additional information about a servable. +This information includes details on how to invoke the servable, required inputs, and expected outputs. +The describe command requires the owner name and user name of the servable:: - $ dlhub describe --id 5e21f7da-7788-406a-a0ac-805aa17811ff + $ dlhub describe dlhub.test_gmail 1d_norm -or:: +Use the 'methods' to return only information about the methods implemented by a servable:: - $ dlhub describe --name mnist + $ dlhub methods dlhub.test_gmail 1d_norm + + run: + input: + description: Array to be normed + shape: + - None + type: ndarray + output: + description: Norm of the array + type: number Running Servables ^^^^^^^^^^^^^^^^^ -Servables can be invoked through the CLI using the run command. The run command accepts flags to specify the servable -and input parameters. The servable flag requires the identifier of the servable. Input parameters should be well formated JSON strings. -The run command first attempts to json.loads() the input before using the DLHub SDK to invoke the servable. Output will be returned -as well formatted JSON documents.:: +Servables can be invoked through the CLI using the run command. +The run command accepts flags to specify the servable and input parameters. +The servable flag requires the identifier of the servable. +Input parameters should be correctly-formatted JSON strings. +The run command first attempts to json.loads() the input before using the DLHub SDK to invoke the servable. +Output will be returned as well formatted JSON documents.:: $ dlhub run --servable 50358d8c-be7a-41bf-af76-a460223907fe --input '[{"composition": "Al"}]' @@ -96,7 +132,7 @@ as well formatted JSON documents.:: Publishing Servables ^^^^^^^^^^^^^^^^^^^^ -Publishing a servable can be achieved by issuing a publish command using either a github repository or a local servable definition file. +Publishing a servable can be achieved by issuing a publish command using either a GitHub repository or a local servable definition file. Description Files ----------------- @@ -104,14 +140,16 @@ Description Files Publishing a servable relies on a compatible metadata document. The publication process uses the metadata document to determine which shim to use when loading and interacting the servable. -Details on this process can be found `here `_. +A guide for describing servables can be found in the +`DLHub SDK documentation `_. Publishing a Repository ----------------------- -Publishing a model can also be achieved by specifying a compliant github repository. The repository will need to include the -dlhub.json file already. The publication flow relies on repo2docker to construct an initial container image before using -the image as the basis for another container that includes the necessary DLHub modules (dlhub_sdk and parsl). +Publishing a model can also be achieved by specifying a compliant GitHub repository. +The repository will need to include the ``dlhub.json`` file already. +The publication flow relies on `repo2docker `_ +to construct a container with all of the required dependencies. An example repository can be found here: https://github.com/ryanchard/dlhub_publish_example @@ -124,15 +162,15 @@ The publication command will return a task identifier that can subsequently be u Publishing a Local Servable --------------------------- -Publishing a local servable requires the servable have been initalized and the dlhub.json file already exist locally. -Once that file has been generated you can use the --local flag to initiate a publication for the local model. -Files mentioned within the dlhub.json document will be packaged into a temporary zip file then transmitted to the DLHub service -using the boto3 library, staging the data through an S3 bucket.:: +Publishing a local servable requires first generating the ``dlhub.json`` file and storing it on your system. +Once that file has been generated you can use the ``--local`` flag to initiate a publication for the local model. +Files mentioned within the dlhub.json document will be packaged into a temporary zip file +then transmitted to the DLHub service using HTTP:: $ dlhub publish --local Checking Publication Status --------------- +--------------------------- The status of a publication task can be queried using the status command. The status command requires the task id and will return a JSON status document.:: diff --git a/requirements.txt b/requirements.txt index 2b80d24..7e61abd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ click==6.7 requests==2.21.0 six==1.11.0 boto3==1.5.36 -dlhub_sdk>=0.4.0 +dlhub_sdk>=0.6.0 configobj==5.0.6 nbsphinx==0.4.1 +pyyaml==3.12 diff --git a/setup.py b/setup.py index 8268984..d58222d 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,9 @@ install_requires=[ 'click>=6.6,<7.0', 'requests>=2.0.0,<3.0.0', - 'dlhub_sdk' + 'dlhub_sdk>=0.6.0', + 'pyyaml', + 'tabulate' ], entry_points={ 'console_scripts': ['dlhub = dlhub_cli:cli_root'] @@ -43,4 +45,4 @@ "Operating System :: POSIX", "Programming Language :: Python", ], -) \ No newline at end of file +) diff --git a/test-files/DLHub_Client_tokens.json.enc b/test-files/DLHub_Client_tokens.json.enc new file mode 100644 index 0000000..1dc7358 Binary files /dev/null and b/test-files/DLHub_Client_tokens.json.enc differ