diff --git a/gradient/api_sdk/clients/notebook_client.py b/gradient/api_sdk/clients/notebook_client.py index 5365c9e2..7aa67a49 100644 --- a/gradient/api_sdk/clients/notebook_client.py +++ b/gradient/api_sdk/clients/notebook_client.py @@ -233,3 +233,55 @@ def artifacts_list(self, notebook_id, files=None, size=False, links=True): repository = self.build_repository(repositories.ListNotebookArtifacts) artifacts = repository.list(notebook_id=notebook_id, files=files, links=links, size=size) return artifacts + + def logs(self, notebook_id, line=1, limit=10000): + """ + Method to retrieve notebook logs. + + .. code-block:: python + :linenos: + :emphasize-lines: 2 + + notebook_logs = notebook_client.logs( + notebook_id='Your_job_id_here', + line=100, + limit=100 + ) + + :param str notebook_id: id of notebook that we want to retrieve logs + :param int line: from what line you want to retrieve logs. Default 0 + :param int limit: how much lines you want to retrieve logs. Default 10000 + + :returns: list of formatted logs lines + :rtype: list + """ + notebook = self.get(notebook_id) + repository = self.build_repository(repositories.ListNotebookLogs) + logs = repository.list(job_id=notebook.job_handle, notebook_id=notebook_id, line=line, limit=limit) + return logs + + def yield_logs(self, notebook_id, line=1, limit=10000): + """Get log generator. Polls the API for new logs + + .. code-block:: python + :linenos: + :emphasize-lines: 2 + + notebook_logs_generator = notebook_client.yield_logs( + notebook_id='Your_job_id_here', + line=100, + limit=100 + ) + + :param str notebook_id: + :param int line: line number at which logs starts to display on screen + :param int limit: maximum lines displayed on screen, default set to 10 000 + + :returns: generator yielding LogRow instances + :rtype: Iterator[models.LogRow] + """ + + notebook = self.get(notebook_id) + repository = self.build_repository(repositories.ListNotebookLogs) + logs = repository.yield_logs(job_id=notebook.job_handle, notebook_id=notebook_id, line=line, limit=limit) + return logs diff --git a/gradient/api_sdk/repositories/__init__.py b/gradient/api_sdk/repositories/__init__.py index 31592af8..48ef3ae0 100644 --- a/gradient/api_sdk/repositories/__init__.py +++ b/gradient/api_sdk/repositories/__init__.py @@ -13,7 +13,7 @@ RestartMachine, GetMachine, UpdateMachine, GetMachineUtilization from .models import DeleteModel, ListModels, UploadModel, GetModel, ListModelFiles from .notebooks import CreateNotebook, DeleteNotebook, GetNotebook, ListNotebooks, GetNotebookMetrics, \ - StreamNotebookMetrics, StopNotebook, StartNotebook, ForkNotebook, ListNotebookArtifacts + StreamNotebookMetrics, StopNotebook, StartNotebook, ForkNotebook, ListNotebookArtifacts, ListNotebookLogs from .projects import CreateProject, ListProjects, DeleteProject, GetProject from .secrets import ListSecrets, SetSecret, DeleteSecret from .tensorboards import CreateTensorboard, GetTensorboard, ListTensorboards, UpdateTensorboard, DeleteTensorboard diff --git a/gradient/api_sdk/repositories/notebooks.py b/gradient/api_sdk/repositories/notebooks.py index 0d07412c..fde00b0c 100644 --- a/gradient/api_sdk/repositories/notebooks.py +++ b/gradient/api_sdk/repositories/notebooks.py @@ -3,7 +3,7 @@ from ..clients import http_client from ..sdk_exceptions import ResourceCreatingError from .common import CreateResource, DeleteResource, ListResources, GetResource, \ - StopResource, GetMetrics, StreamMetrics, BaseRepository + StopResource, GetMetrics, StreamMetrics, BaseRepository, ListLogs from .. import config from .. import serializers, sdk_exceptions @@ -211,3 +211,13 @@ def _get_request_params(self, kwargs): return params +class ListNotebookLogs(ListLogs): + def _get_request_params(self, kwargs): + params = { + "jobId": kwargs["job_id"], + "notebookId": kwargs["notebook_id"], + "line": kwargs["line"], + "limit": kwargs["limit"] + } + return params + diff --git a/gradient/api_sdk/utils.py b/gradient/api_sdk/utils.py index 0928186b..11bc5304 100644 --- a/gradient/api_sdk/utils.py +++ b/gradient/api_sdk/utils.py @@ -48,6 +48,9 @@ def get_error_messages(self, data, add_prefix=False): if isinstance(data, six.string_types): yield data + if six.PY3 and isinstance(data, bytes): + yield data.decode('utf-8') + def print_dict_recursive(input_dict, logger, indent=0, tabulator=" "): for key, val in input_dict.items(): diff --git a/gradient/cli/notebooks.py b/gradient/cli/notebooks.py index dc8448c2..6897dd70 100644 --- a/gradient/cli/notebooks.py +++ b/gradient/cli/notebooks.py @@ -459,3 +459,36 @@ def list_artifacts(notebook_id, size, links, files, options_file, api_key=None): command = notebooks.ArtifactsListCommand(api_key=api_key) command.execute(notebook_id=notebook_id, size=size, links=links, files=files) +@notebooks_group.command("logs", help="List notebook logs") +@click.option( + "--id", + "notebook_id", + required=True, + cls=common.GradientOption, +) +@click.option( + "--line", + "line", + required=False, + default=0, + cls=common.GradientOption, +) +@click.option( + "--limit", + "limit", + required=False, + default=10000, + cls=common.GradientOption, +) +@click.option( + "--follow", + "follow", + required=False, + default=False, + cls=common.GradientOption, +) +@api_key_option +@common.options_file +def list_logs(notebook_id, line, limit, follow, options_file, api_key=None): + command = notebooks.NotebookLogsCommand(api_key=api_key) + command.execute(notebook_id, line, limit, follow) diff --git a/gradient/commands/common.py b/gradient/commands/common.py index 8669237c..7eb597a8 100644 --- a/gradient/commands/common.py +++ b/gradient/commands/common.py @@ -6,6 +6,7 @@ import click import six import terminaltables +from click import style from halo import halo from gradient.clilogger import CliLogger @@ -179,14 +180,6 @@ def _get_table_data(self, objects): class LogsCommandMixin(object): - @abc.abstractmethod - def _make_table(self, logs, id): - pass - - @abc.abstractmethod - def _get_log_row_string(self, id, log): - pass - def execute(self, id, line, limit, follow): if follow: self.logger.log("Awaiting logs...") @@ -214,3 +207,22 @@ def _log_logs_continuously(self, id, line, limit): def _get_logs_generator(self, id, line, limit): logs_gen = self.client.yield_logs(id, line, limit) return logs_gen + + def _make_table(self, logs, id): + table_title = "%s %s logs" % (self.ENTITY, id) + table_data = [("LINE", "MESSAGE")] + table = terminaltables.AsciiTable(table_data, title=table_title) + + for log in logs: + table_data.append(self._format_row(log)) + + return table.table + + def _get_log_row_string(self, id, log): + log_msg = "{}\t{}".format(*self._format_row(log)) + return log_msg + + @staticmethod + def _format_row(log_row): + return (style(fg="red", text=str(log_row.line)), + log_row.message) diff --git a/gradient/commands/jobs.py b/gradient/commands/jobs.py index 905d2837..faece7c3 100644 --- a/gradient/commands/jobs.py +++ b/gradient/commands/jobs.py @@ -4,7 +4,6 @@ import six import terminaltables -from click import style from halo import halo from gradient import api_sdk, exceptions, Job, JobArtifactsDownloader, cli_constants @@ -157,24 +156,7 @@ def _make_table(table_data): class JobLogsCommand(LogsCommandMixin, BaseJobCommand): - def _make_table(self, logs, id): - table_title = "Job %s logs" % id - table_data = [("LINE", "MESSAGE")] - table = terminaltables.AsciiTable(table_data, title=table_title) - - for log in logs: - table_data.append(self._format_row(log)) - - return table.table - - def _get_log_row_string(self, id, log): - log_msg = "{}\t{}".format(*self._format_row(log)) - return log_msg - - @staticmethod - def _format_row(log_row): - return (style(fg="red", text=str(log_row.line)), - log_row.message) + ENTITY = "Job" class CreateJobCommand(BaseCreateJobCommandMixin, BaseJobCommand): diff --git a/gradient/commands/notebooks.py b/gradient/commands/notebooks.py index cf0d0bee..5d7b597a 100644 --- a/gradient/commands/notebooks.py +++ b/gradient/commands/notebooks.py @@ -11,7 +11,8 @@ from gradient.api_sdk import sdk_exceptions from gradient.cli_constants import CLI_PS_CLIENT_NAME from gradient.cliutils import get_terminal_lines -from gradient.commands.common import BaseCommand, ListCommandMixin, DetailsCommandMixin, StreamMetricsCommand +from gradient.commands.common import BaseCommand, ListCommandMixin, DetailsCommandMixin, StreamMetricsCommand, \ + LogsCommandMixin @six.add_metaclass(abc.ABCMeta) @@ -156,49 +157,6 @@ class StreamNotebookMetricsCommand(StreamMetricsCommand, BaseNotebookCommand): pass -class NotebookLogsCommand(BaseNotebookCommand): - - def execute(self, notebook_id, line, limit, follow): - if follow: - self.logger.log("Awaiting logs...") - self._log_logs_continuously(notebook_id, line, limit) - else: - self._log_table_of_logs(notebook_id, line, limit) - - def _log_table_of_logs(self, notebook_id, line, limit): - logs = self.client.logs(notebook_id, line, limit) - if not logs: - self.logger.log("No logs found") - return - - table_str = self._make_table(logs, notebook_id) - if len(table_str.splitlines()) > get_terminal_lines(): - pydoc.pager(table_str) - else: - self.logger.log(table_str) - - def _log_logs_continuously(self, notebook_id, line, limit): - logs_gen = self.client.yield_logs(notebook_id, line, limit) - for log in logs_gen: - log_msg = "{}\t{}".format(*self._format_row(log)) - self.logger.log(log_msg) - - @staticmethod - def _format_row(log_row): - return (style(fg="red", text=str(log_row.line)), - log_row.message) - - def _make_table(self, logs, notebook_id): - table_title = "Notebook %s logs" % notebook_id - table_data = [("LINE", "MESSAGE")] - table = terminaltables.AsciiTable(table_data, title=table_title) - - for log in logs: - table_data.append(self._format_row(log)) - - return table.table - - class ArtifactsListCommand(BaseNotebookCommand): WAITING_FOR_RESPONSE_MESSAGE = "Waiting for data..." @@ -249,3 +207,7 @@ def _make_table(table_data): ascii_table = terminaltables.AsciiTable(table_data) table_string = ascii_table.table return table_string + + +class NotebookLogsCommand(LogsCommandMixin, BaseNotebookCommand): + ENTITY = "Notebook" \ No newline at end of file