Merge pull request #7 from DLHub-Argonne/search_functions

Adding Search Capabilities to CLI

.travis.yml

@@ -1,12 +1,19 @@
 language: python
 python:
 - '3.6'
+before_install:
+- openssl aes-256-cbc -K $encrypted_04ab0de0cbc0_key -iv $encrypted_04ab0de0cbc0_iv
+  -in test-files/DLHub_Client_tokens.json.enc -out test-files/DLHub_Client_tokens.json
+  -d
+- mkdir -p ~/.dlhub/credentials
+- cp test-files/DLHub_Client_tokens.json ~/.dlhub/credentials
 install:
 - pip install -e .
 - pip install --only-binary=numpy,scipy -r requirements.txt
-- pip install coveralls
+- pip install coveralls flake8
 - pip install pytest pytest-cov --upgrade
 script:
+- flake8
 - pytest --show-capture stdout
 after_success:
 - coveralls

dlhub_cli/__init__.py

@@ -1,4 +1,3 @@
 from dlhub_cli.main import cli_root
 
-
-__all__ = ['cli_root']
+__all__ = ['cli_root']

dlhub_cli/commands/__init__.py

@@ -1,15 +1,17 @@
+from dlhub_cli.commands.describe import describe_cmd
 from dlhub_cli.commands.init import init_cmd
-from dlhub_cli.commands.publish import publish_cmd
-from dlhub_cli.commands.update import update_cmd
-from dlhub_cli.commands.run import run_cmd
-from dlhub_cli.commands.ls import ls_cmd
-from dlhub_cli.commands.status import status_cmd
 from dlhub_cli.commands.login import login_cmd
 from dlhub_cli.commands.logout import logout_cmd
+from dlhub_cli.commands.ls import ls_cmd
+from dlhub_cli.commands.methods import methods_cmd
+from dlhub_cli.commands.publish import publish_cmd
+from dlhub_cli.commands.run import run_cmd
+from dlhub_cli.commands.search import search_cmd
 from dlhub_cli.commands.servables import servables_cmd
-from dlhub_cli.commands.describe import describe_cmd
+from dlhub_cli.commands.status import status_cmd
+from dlhub_cli.commands.whoami import whoami_cmd
 
 __all__ = ['init_cmd', 'publish_cmd', 'servables_cmd',
-           'update_cmd', 'run_cmd', 'ls_cmd',
+           'run_cmd', 'ls_cmd', 'methods_cmd',
            'status_cmd', 'login_cmd', 'logout_cmd',
-           'describe_cmd']
+           'describe_cmd', 'whoami_cmd', 'search_cmd']

dlhub_cli/commands/describe.py

@@ -1,48 +1,89 @@
+from datetime import datetime
 import click
+import yaml
 
 from dlhub_cli.config import get_dlhub_client
 from dlhub_cli.printing import format_output
 from dlhub_cli.parsing import dlhub_cmd
 
 
-HELP_STR = """\
+_unwanted_fields = [
+    ('dlhub', 'build_location'),
+    ('dlhub', 'ecr_arn'),
+    ('dlhub', 'ecr_uri'),
+    ('dlhub', 'id'),
+    ('dlhub', 'transfer_method'),
+    ('dlhub', 'user_id')
+]
 
-  Describe a servable.
-\b
-Options:
-  --id TEXT      The UUID of a servable.
-  --name TEXT    The name of a servable.
-  -h, --help     Show this message and exit.
-  -v, --version  Show the version and exit."""
 
+def _remove_field(metadata, field):
+    """Remove a certain field from the metadata
 
-@dlhub_cmd('describe', help='Describe a servable.')
-@click.option('--id',
-              show_default=True, required=False,
-              help='The UUID of a servable.')
-@click.option('--name',
-              show_default=True, required=False,
-              help='The name of a servable.')
-def describe_cmd(id, name):
+    Args:
+        metadata (dict): Metadata to be pruned
+        field ([string]): Coordinates of fields to be removed
+    """
+
+    if len(field) == 1:
+        if field[0] in metadata:
+            del metadata[field[0]]
+    else:
+        if field[0] in metadata:
+            subset = metadata[field[0]]
+            return _remove_field(subset, field[1:])
+
+
+def _preprocess_metadata(metadata):
+    """Clean up a metadata record to make it more useful to humans
+
+    Args:
+         metadata (metadata): Metadata record to be cleaned
+    """
+    # Prune internal-only fields
+    for field in _unwanted_fields:
+        _remove_field(metadata, field)
+
+    # Turn Timestamp (epoch time in ms) into a String
+    metadata['dlhub']['publication_date'] = \
+        datetime.fromtimestamp(int(metadata['dlhub']['publication_date']) / 1000) \
+        .strftime('%Y-%m-%d %H:%M')
+
+
+@dlhub_cmd('describe', short_help="Get the description of a servable",
+           help="""Get the description of a servable
+
+           OWNER is the username of the owner of the servable, and NAME is the name of the servable.
+
+           You can optionally specify both owner and servable name as a single argument using
+           a "owner_name/servable_name" format
+           """)
+@click.argument('owner', default=None)
+@click.argument('name', default="")
+def describe_cmd(owner, name):
     """Use DLHub to get a description of the servable.
 
     Args:
-        id (string): The uuid of the servable
+        owner (string): The owner of the servable
         name (string): The name of the servable
     Returns:
         (dict) a set of information regarding the servable
     """
 
-    if not any([id, name]):
-        format_output(HELP_STR)
-        return
+    # Check if owner contains both
+    if name == "":
+        temp = owner.split("/")
+        if len(temp) != 2:
+            raise click.BadArgumentUsage('Model name missing')
+        owner, name = temp
 
+    # Retrieve the metadata
     client = get_dlhub_client()
-    res = "Unable to describe a servable (name: {0}, id: {1})".format(name, id)
-    if id:
-        res = client.describe_servable(servable_id=id)
-    elif name:
-        res = client.describe_servable(servable_name=name)
+    res = client.describe_servable(owner, name)
+
+    # Clean up the metadata fields
+    _preprocess_metadata(res)
 
-    format_output("{0}".format(res))
+    # Print it to screen
+    format_output(yaml.dump(res))
     return res

dlhub_cli/commands/init.py

@@ -66,7 +66,8 @@ def init_cmd(force, filename, author, title, name, skip_run):
 
     # Check if a file would be overwritten
     if os.path.isfile(filename) and not force:
-        format_output("There is already a file named '{}'. Use --force to overwrite".format(filename))
+        format_output("There is already a file named '{}'."
+                      " Use --force to overwrite".format(filename))
         return 1
 
     # Prepare a dict of objects to be replaced

dlhub_cli/commands/login.py

@@ -1,6 +1,7 @@
 import click
 from dlhub_cli.config import get_dlhub_client
 
+
 @click.command('login',
                short_help=('Log into Globus to get credentials for '
                            'the DLHub CLI'),
@@ -16,4 +17,4 @@ def login_cmd(force):
         force (bool): Whether or not to force the login.
     """
 
-    client = get_dlhub_client(force)
+    get_dlhub_client(force)

dlhub_cli/commands/logout.py

@@ -2,6 +2,7 @@
 
 from dlhub_cli.config import get_dlhub_client
 
+
 @click.command('logout',
                short_help='Logout of the DLHub CLI',
                help=('Logout of the DLHub CLI. '
@@ -17,4 +18,4 @@ def logout_cmd():
     """
     client = get_dlhub_client()
 
-    client.logout()
+    client.logout()

dlhub_cli/commands/ls.py

@@ -1,8 +1,7 @@
 import os
-import json
 
 from dlhub_cli.printing import format_output
-from dlhub_cli.parsing import dlhub_cmd, index_argument
+from dlhub_cli.parsing import dlhub_cmd
 
 
 @dlhub_cmd('ls', help='List servables in this directory')
@@ -19,4 +18,4 @@ def ls_cmd():
 
     for serv in servables:
         res = serv.split(".json")[0]
-        format_output(res)
+        format_output(res)

dlhub_cli/commands/methods.py

@@ -0,0 +1,59 @@
+import yaml
+import click
+
+from dlhub_cli.parsing import dlhub_cmd
+from dlhub_cli.printing import format_output
+from dlhub_cli.config import get_dlhub_client
+
+
+@dlhub_cmd('methods', short_help='Print method information',
+           help='''Print methods for a certain servable
+
+           OWNER is the username of the servable owner, NAME is the name of the servable.
+           METHOD is optional. If provided, this command will only print the information
+           for the method with that name. Otherwise, it will print the information for all
+           methods implemented by the servable.
+
+           You can optionally specify the servable name is owner_name/model_name format
+           ''')
+@click.argument('owner', default=None)
+@click.argument('name', default='')
+@click.argument('method', default='')
+def methods_cmd(owner, name, method):
+    """Print out the methods of a servable
+
+    Args:
+        owner (str): Name of the servable's owner
+        name (str): Name of the servable
+        method (str): Name of the method (optional)
+    """
+
+    # Check if the owner sent model information in owner/model format
+    if '/' in owner:
+        # Get the owner and model name
+        temp = owner.split('/')
+        if len(temp) != 2:
+            raise click.BadArgumentUsage('Expected owner_name/model_name format')
+
+        # If "name" is provided, it is actually the method name
+        if name != '':
+            method = name
+
+        owner, name = temp
+
+    # Make sure the name was provided
+    if name == '':
+        raise click.BadArgumentUsage('Model name not provided')
+
+    # If the method name is blank, make it None (for compatibility with client)
+    if method == '':
+        method = None
+
+    # Get the DLHub client
+    client = get_dlhub_client()
+
+    # Get the method details
+    method_details = client.describe_methods(owner, name, method)
+
+    # Print them in YAML format
+    format_output(yaml.dump(method_details, default_flow_style=False))

dlhub_cli/commands/publish.py

@@ -1,10 +1,8 @@
-import os
 import json
 import click
-import pickle as pkl
 
 from dlhub_cli.config import get_dlhub_client
-from dlhub_cli.printing import format_output, safeprint
+from dlhub_cli.printing import format_output
 from dlhub_cli.parsing import dlhub_cmd
 
 from dlhub_sdk.utils import unserialize_object
@@ -29,14 +27,16 @@
               default=None, show_default=True,
               help='The repository to publish.')
 def publish_cmd(local, repository):
-    """Publish a model to DLHub. Either a metadata description file for a local servable or a remote github address.
+    """Publish a model to DLHub. Either a metadata description file for a
+    local servable or a remote github address.
     The servable's metadata will be sent to the DLHub service.
 
     If using a local servable the files described in the
-    metadata's 'files' field will be zipped into a tmp archive and staged to S3. An ingestion pipeline will then
-    download the data from S3, build the servable, and publish the servable to DLHub.
+    metadata's 'files' field will be zipped into a tmp archive
+    and then sent to DLHub via HTTPS
 
-    When using a repository the github url is passed to a specific publish_repo endpoint on DLHub. This will
+    When using a repository the github url is passed to a
+    specific publish_repo endpoint on DLHub. This will
     use repo2docker to build the servable and publish it.
 
     Args:
@@ -50,8 +50,6 @@ def publish_cmd(local, repository):
         format_output(HELP_STR)
         return
 
-    loaded_servable = None
-
     client = get_dlhub_client()
     res = None
     if local:
@@ -78,7 +76,3 @@ def publish_cmd(local, repository):
         res = client.publish_repository(repository)
         format_output("Task_id: {}".format(res))
     return res
-
-
-
-

dlhub_cli/commands/run.py

@@ -1,8 +1,8 @@
 import json
 import click
 
-from dlhub_cli.printing import format_output, safeprint
-from dlhub_cli.config import (get_dlhub_client)
+from dlhub_cli.config import get_dlhub_client
+from dlhub_cli.printing import format_output
 from dlhub_cli.parsing import dlhub_cmd
 
 # @click.option('--servable-uuid',
@@ -30,7 +30,8 @@
               default=None, show_default=True, required=False,
               help='Input to pass to the servable.')
 def run_cmd(servable, input):
-    """Invoke a servable. The input data will be read with json.loads(input) and passed to the servable.
+    """Invoke a servable. The input data will be read with
+    json.loads(input) and passed to the servable.
 
     Args:
         servable (string): The servable to invoke, e.g., "ryan_globusid/noop"
@@ -53,4 +54,4 @@ def run_cmd(servable, input):
     res = client.run(namespace, model, data)
 
     format_output(res)
-    return res
+    return res