Merge pull request #186 from datmo/issue-159

Resolved #159, merged in changes from master, moved to smaller environments (#130)

.travis.yml

@@ -12,7 +12,7 @@ install:
 - pip install coveralls
 - python setup.py install
 script:
-- pytest --cov-config .coveragerc --cov=datmo
+- pytest --cov-config .coveragerc --cov=datmo -s -v
 - echo $TRAVIS_COMMIT_RANGE
 - bash devtools/travis-ci/test_format_code.sh
 after_success:

appveyor.yml

@@ -49,4 +49,4 @@ install:
   - "%PYTHON%/python.exe setup.py install"
 test_script:
   - "%PYTHON%/Scripts/pip.exe --version"
-  - "%PYTHON%/Scripts/pytest "
+  - "%PYTHON%/Scripts/pytest -s -v"

circle.yml

@@ -5,7 +5,7 @@ machine:
 
   environment:
     LOGGING_LEVEL: DEBUG
-    TEST_PACKAGE: python -m pytest
+    TEST_PACKAGE: python -m pytest -s -v
 
   pre:
     - echo "Creating python environments"

datmo/cli/command/environment.py

@@ -1,12 +1,13 @@
 from __future__ import print_function
 
-import prettytable
+import os
+from datetime import datetime
 
 from datmo.core.util.i18n import get as __
 from datmo.core.controller.environment.environment import EnvironmentController
-from datmo.core.util.misc_functions import printable_string
 from datmo.cli.command.project import ProjectCommand
 from datmo.core.util.exceptions import EnvironmentDoesNotExist
+from datmo.core.util.misc_functions import printable_string
 
 
 class EnvironmentCommand(ProjectCommand):
@@ -61,22 +62,42 @@ def delete(self, **kwargs):
                 __("info", "cli.environment.delete.success", environment_id))
             return True
 
-    def ls(self):
-        environments = self.environment_controller.list()
+    def ls(self, **kwargs):
+        print_format = kwargs.get('format', "table")
+        download = kwargs.get('download', None)
+        download_path = kwargs.get('download_path', None)
+        environment_objs = self.environment_controller.list()
         header_list = ["id", "created at", "name", "description"]
-        t = prettytable.PrettyTable(header_list)
-        environment_ids = []
-        for environment_obj in environments:
-            environment_ids.append(environment_obj.id)
-            environment_created_at = printable_string(
+        item_dict_list = []
+        for environment_obj in environment_objs:
+            environment_obj_created_at = printable_string(
                 environment_obj.created_at.strftime("%Y-%m-%d %H:%M:%S"))
-            environment_name = printable_string(environment_obj.name) \
+            environment_obj_name = printable_string(environment_obj.name) \
                 if environment_obj.name is not None else ""
-            environment_description = printable_string(environment_obj.description) \
+            environment_obj_description = printable_string(environment_obj.description) \
                 if environment_obj.description is not None else ""
-            t.add_row([
-                environment_obj.id, environment_created_at, environment_name,
-                environment_description
-            ])
-        self.cli_helper.echo(t)
-        return environments
+            item_dict_list.append({
+                "id": environment_obj.id,
+                "created at": environment_obj_created_at,
+                "name": environment_obj_name,
+                "description": environment_obj_description
+            })
+        if download:
+            if not download_path:
+                # download to current working directory with timestamp
+                current_time = datetime.utcnow()
+                epoch_time = datetime.utcfromtimestamp(0)
+                current_time_unix_time_ms = (
+                    current_time - epoch_time).total_seconds() * 1000.0
+                download_path = os.path.join(
+                    os.getcwd(),
+                    "environment_ls_" + str(current_time_unix_time_ms))
+            self.cli_helper.print_items(
+                header_list,
+                item_dict_list,
+                print_format=print_format,
+                output_path=download_path)
+            return environment_objs
+        self.cli_helper.print_items(
+            header_list, item_dict_list, print_format=print_format)
+        return environment_objs

datmo/cli/command/session.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
-import prettytable
+import os
+from datetime import datetime
 
 from datmo.core.util.i18n import get as __
 from datmo.core.controller.session import SessionController
@@ -20,9 +21,9 @@ def session(self):
 
     def create(self, **kwargs):
         name = kwargs.get('name')
-        self.session_controller.create(kwargs)
+        session_obj = self.session_controller.create(kwargs)
         self.cli_helper.echo(__("info", "cli.session.create", name))
-        return True
+        return session_obj
 
     def delete(self, **kwargs):
         name = kwargs.get('name')
@@ -36,23 +37,46 @@ def select(self, **kwargs):
         return self.session_controller.select(name)
 
     def ls(self, **kwargs):
+        print_format = kwargs.get('format', "table")
+        download = kwargs.get('download', None)
+        download_path = kwargs.get('download_path', None)
         sessions = self.session_controller.list(
             sort_key="created_at", sort_order="descending")
         header_list = ["name", "selected", "tasks", "snapshots"]
-        t = prettytable.PrettyTable(header_list)
-        for sess in sessions:
+        item_dict_list = []
+        for session_obj in sessions:
             snapshot_count = len(
                 self.session_controller.dal.snapshot.query({
-                    "session_id": sess.id,
+                    "session_id": session_obj.id,
                     "model_id": self.session_controller.model.id
                 }))
             task_count = len(
                 self.session_controller.dal.task.query({
-                    "session_id": sess.id,
+                    "session_id": session_obj.id,
                     "model_id": self.session_controller.model.id
                 }))
-            t.add_row([sess.name, sess.current, task_count, snapshot_count])
-
-        self.cli_helper.echo(t)
-
-        return True
+            item_dict_list.append({
+                "name": session_obj.name,
+                "selected": str(session_obj.current),
+                "tasks": str(task_count),
+                "snapshots": str(snapshot_count)
+            })
+        if download:
+            if not download_path:
+                # download to current working directory with timestamp
+                current_time = datetime.utcnow()
+                epoch_time = datetime.utcfromtimestamp(0)
+                current_time_unix_time_ms = (
+                    current_time - epoch_time).total_seconds() * 1000.0
+                download_path = os.path.join(
+                    os.getcwd(),
+                    "session_ls_" + str(current_time_unix_time_ms))
+            self.cli_helper.print_items(
+                header_list,
+                item_dict_list,
+                print_format=print_format,
+                output_path=download_path)
+            return sessions
+        self.cli_helper.print_items(
+            header_list, item_dict_list, print_format=print_format)
+        return sessions

datmo/cli/command/snapshot.py

@@ -1,7 +1,7 @@
 from __future__ import print_function
 
-import prettytable
-import datetime
+import os
+from datetime import datetime
 
 from datmo.core.util.i18n import get as __
 from datmo.core.util.misc_functions import mutually_exclusive, printable_string, prettify_datetime, parse_cli_key_value, format_table
@@ -46,7 +46,7 @@ def create(self, **kwargs):
                 message, task_id, label=label)
             self.cli_helper.echo(
                 "Created snapshot id: %s" % snapshot_task_obj.id)
-            return snapshot_task_obj.id
+            return snapshot_task_obj
         else:
             # creating snapshot without task id
             snapshot_dict = {"visible": True}
@@ -107,7 +107,7 @@ def create(self, **kwargs):
             snapshot_obj = self.snapshot_controller.create(snapshot_dict)
             self.cli_helper.echo(
                 __("info", "cli.snapshot.create.success", snapshot_obj.id))
-            return snapshot_obj.id
+            return snapshot_obj
 
     def delete(self, **kwargs):
         self.cli_helper.echo(__("info", "cli.snapshot.delete"))
@@ -163,58 +163,77 @@ def update(self, **kwargs):
     def ls(self, **kwargs):
         session_id = kwargs.get('session_id',
                                 self.snapshot_controller.current_session.id)
-        # Get all snapshot meta information
         detailed_info = kwargs.get('details', None)
-        # List of ids shown
-        listed_snapshot_ids = []
+        print_format = kwargs.get('format', "table")
+        download = kwargs.get('download', None)
+        download_path = kwargs.get('download_path', None)
         snapshot_objs = self.snapshot_controller.list(
             session_id=session_id,
             visible=True,
-            sort_key='created_at',
-            sort_order='descending')
+            sort_key="created_at",
+            sort_order="descending")
+        item_dict_list = []
         if detailed_info:
             header_list = [
                 "id", "created at", "config", "stats", "message", "label",
                 "code id", "environment id", "file collection id"
             ]
-            t = prettytable.PrettyTable(header_list)
             for snapshot_obj in snapshot_objs:
                 snapshot_config_printable = printable_string(
                     str(snapshot_obj.config))
                 snapshot_stats_printable = printable_string(
                     str(snapshot_obj.stats))
                 snapshot_message = printable_string(snapshot_obj.message)
-                t.add_row([
-                    snapshot_obj.id,
-                    prettify_datetime(snapshot_obj.created_at),
-                    snapshot_config_printable, snapshot_stats_printable,
-                    snapshot_message, snapshot_obj.label, snapshot_obj.code_id,
-                    snapshot_obj.environment_id,
-                    snapshot_obj.file_collection_id
-                ])
-                listed_snapshot_ids.append(snapshot_obj.id)
+                snapshot_label = printable_string(snapshot_obj.label)
+                item_dict_list.append({
+                    "id": snapshot_obj.id,
+                    "created at ": prettify_datetime(snapshot_obj.created_at),
+                    "config": snapshot_config_printable,
+                    "stats": snapshot_stats_printable,
+                    "message": snapshot_message,
+                    "label": snapshot_label,
+                    "code id": snapshot_obj.code_id,
+                    "environment id": snapshot_obj.environment_id,
+                    "file collection id": snapshot_obj.file_collection_id
+                })
         else:
             header_list = [
                 "id", "created at", "config", "stats", "message", "label"
             ]
-            t = prettytable.PrettyTable(header_list)
             for snapshot_obj in snapshot_objs:
                 snapshot_config_printable = printable_string(
                     str(snapshot_obj.config))
                 snapshot_stats_printable = printable_string(
                     str(snapshot_obj.stats))
                 snapshot_message = printable_string(snapshot_obj.message)
-                snapshot_created_at = printable_string(
-                    snapshot_obj.created_at.strftime("%Y-%m-%d %H:%M:%S"))
-                t.add_row([
-                    snapshot_obj.id, snapshot_created_at,
-                    snapshot_config_printable, snapshot_stats_printable,
-                    snapshot_message, snapshot_obj.label
-                ])
-                listed_snapshot_ids.append(snapshot_obj.id)
-
-        self.cli_helper.echo(t)
-        return listed_snapshot_ids
+                snapshot_label = printable_string(snapshot_obj.label)
+                item_dict_list.append({
+                    "id": snapshot_obj.id,
+                    "created at ": prettify_datetime(snapshot_obj.created_at),
+                    "config": snapshot_config_printable,
+                    "stats": snapshot_stats_printable,
+                    "message": snapshot_message,
+                    "label": snapshot_label,
+                })
+        if download:
+            if not download_path:
+                # download to current working directory with timestamp
+                current_time = datetime.utcnow()
+                epoch_time = datetime.utcfromtimestamp(0)
+                current_time_unix_time_ms = (
+                    current_time - epoch_time).total_seconds() * 1000.0
+                download_path = os.path.join(
+                    os.getcwd(),
+                    "snapshot_ls_" + str(current_time_unix_time_ms))
+            self.cli_helper.print_items(
+                header_list,
+                item_dict_list,
+                print_format=print_format,
+                output_path=download_path)
+            return snapshot_objs
+        self.cli_helper.print_items(
+            header_list, item_dict_list, print_format=print_format)
+        return snapshot_objs
 
     def checkout(self, **kwargs):
         snapshot_id = kwargs.get("id", None)
@@ -240,16 +259,18 @@ def diff(self, **kwargs):
                 snapshot_obj_1, attribute) else "N/A"
             value_2 = getattr(snapshot_obj_2, attribute) if getattr(
                 snapshot_obj_2, attribute) else "N/A"
-            if isinstance(value_1, datetime.datetime):
+            if isinstance(value_1, datetime):
                 value_1 = prettify_datetime(value_1)
-            if isinstance(value_2, datetime.datetime):
+            if isinstance(value_2, datetime):
                 value_2 = prettify_datetime(value_2)
             table_data.append([attribute, value_1, "->", value_2])
-        self.cli_helper.echo(format_table(table_data))
-        return True
+        output = format_table(table_data)
+        self.cli_helper.echo(output)
+        return output
 
     def inspect(self, **kwargs):
         snapshot_id = kwargs.get("id", None)
         snapshot_obj = self.snapshot_controller.get(snapshot_id)
-        self.cli_helper.echo(str(snapshot_obj))
-        return True
+        output = str(snapshot_obj)
+        self.cli_helper.echo(output)
+        return output

datmo/cli/command/task.py

@@ -1,8 +1,9 @@
 from __future__ import print_function
 
+import os
+from datetime import datetime
 import shlex
 import platform
-import prettytable
 # https://stackoverflow.com/questions/11301138/how-to-check-if-variable-is-string-with-python-2-and-3-compatibility/11301392?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa
 try:
     basestring
@@ -65,21 +66,41 @@ def run(self, **kwargs):
     def ls(self, **kwargs):
         session_id = kwargs.get('session_id',
                                 self.task_controller.current_session.id)
-        # Get all snapshot meta information
-        header_list = ["id", "command", "status", "results", "created at"]
-        t = prettytable.PrettyTable(header_list)
+        print_format = kwargs.get('format', "table")
+        download = kwargs.get('download', None)
+        download_path = kwargs.get('download_path', None)
+        # Get all task meta information
         task_objs = self.task_controller.list(
             session_id, sort_key='created_at', sort_order='descending')
+        header_list = ["id", "command", "status", "results", "created at"]
+        item_dict_list = []
         for task_obj in task_objs:
             task_results_printable = printable_string(str(task_obj.results))
-            t.add_row([
-                task_obj.id, task_obj.command, task_obj.status,
-                task_results_printable,
-                prettify_datetime(task_obj.created_at)
-            ])
-        self.cli_helper.echo(t)
-
-        return True
+            item_dict_list.append({
+                "id": task_obj.id,
+                "command": task_obj.command,
+                "status": task_obj.status,
+                "results": task_results_printable,
+                "created at": prettify_datetime(task_obj.created_at)
+            })
+        if download:
+            if not download_path:
+                # download to current working directory with timestamp
+                current_time = datetime.utcnow()
+                epoch_time = datetime.utcfromtimestamp(0)
+                current_time_unix_time_ms = (
+                    current_time - epoch_time).total_seconds() * 1000.0
+                download_path = os.path.join(
+                    os.getcwd(), "task_ls_" + str(current_time_unix_time_ms))
+            self.cli_helper.print_items(
+                header_list,
+                item_dict_list,
+                print_format=print_format,
+                output_path=download_path)
+            return task_objs
+        self.cli_helper.print_items(
+            header_list, item_dict_list, print_format=print_format)
+        return task_objs
 
     def stop(self, **kwargs):
         input_dict = {}