From 3f0c8f9bfc34de66f57a7d4fd984673915c1c8eb Mon Sep 17 00:00:00 2001 From: Jacob Freck Date: Mon, 4 Jun 2018 17:39:24 -0700 Subject: [PATCH] Fix: set logger to stdout (#588) * set logger to stdout * typo * add log.print level --- aztk_cli/logger.py | 23 +++++++++++++++---- .../endpoints/cluster/cluster_app_logs.py | 4 ++-- .../spark/endpoints/cluster/cluster_copy.py | 12 +++++----- .../spark/endpoints/cluster/cluster_run.py | 2 +- aztk_cli/spark/endpoints/job/get_app_logs.py | 4 ++-- aztk_cli/spark/endpoints/job/stop.py | 4 ++-- aztk_cli/utils.py | 5 ++-- 7 files changed, 34 insertions(+), 20 deletions(-) diff --git a/aztk_cli/logger.py b/aztk_cli/logger.py index 95606110..9b7b5ff5 100644 --- a/aztk_cli/logger.py +++ b/aztk_cli/logger.py @@ -1,10 +1,11 @@ import logging import platform +import sys root = logging.getLogger("aztk") DEFAULT_FORMAT = '%(message)s' -VERBOSE_FORMAT = '%(asctime)s: %(levelname)s: %(message)s' +VERBOSE_FORMAT = '[%(asctime)s] [%(filename)s:%(module)s:%(funcName)s:%(lineno)d] %(levelname)s - %(message)s' def add_coloring_to_emit_windows(fn): @@ -61,6 +62,8 @@ def new(*args): color = FOREGROUND_YELLOW | FOREGROUND_INTENSITY elif levelno >= 20: color = FOREGROUND_GREEN + elif levelno >= 19: + color = FOREGROUND_WHITE elif levelno >= 10: color = FOREGROUND_MAGENTA else: @@ -86,6 +89,8 @@ def new(*args): color = '\x1b[33m' # yellow elif levelno >= 20: color = '\x1b[32m' # green + elif levelno >= 19: + color = '\x1b[0m' # normal elif levelno >= 10: color = '\x1b[35m' # pink else: @@ -104,10 +109,20 @@ def new(*args): logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) +logging.PRINT = 19 +logging.addLevelName(logging.PRINT, "PRINT") + +def print_level(self, message, *args, **kwargs): + self._log(logging.PRINT, message, args, **kwargs) + + def setup_logging(verbose=False): if verbose: root.setLevel(logging.DEBUG) - logging.basicConfig(format=VERBOSE_FORMAT, datefmt="%Y-%m-%d %H:%M:%S") + logging.basicConfig(format=VERBOSE_FORMAT, datefmt="%Y-%m-%d %H:%M:%S", stream=sys.stdout) else: - root.setLevel(logging.INFO) - logging.basicConfig(format=DEFAULT_FORMAT) + root.setLevel(logging.PRINT) + logging.basicConfig(format=DEFAULT_FORMAT, stream=sys.stdout) + + # add custom levels + logging.Logger.print = print_level diff --git a/aztk_cli/spark/endpoints/cluster/cluster_app_logs.py b/aztk_cli/spark/endpoints/cluster/cluster_app_logs.py index 4a27be31..27ffa748 100644 --- a/aztk_cli/spark/endpoints/cluster/cluster_app_logs.py +++ b/aztk_cli/spark/endpoints/cluster/cluster_app_logs.py @@ -3,7 +3,7 @@ import typing import aztk -from aztk_cli import config, utils +from aztk_cli import config, utils, log def setup_parser(parser: argparse.ArgumentParser): @@ -36,4 +36,4 @@ def execute(args: typing.NamedTuple): with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(app_log.log) else: - print(app_log.log) + log.print(app_log.log) diff --git a/aztk_cli/spark/endpoints/cluster/cluster_copy.py b/aztk_cli/spark/endpoints/cluster/cluster_copy.py index 8a861d82..ffcb6262 100644 --- a/aztk_cli/spark/endpoints/cluster/cluster_copy.py +++ b/aztk_cli/spark/endpoints/cluster/cluster_copy.py @@ -3,7 +3,7 @@ import typing import aztk.spark -from aztk_cli import config, utils +from aztk_cli import config, log, utils def setup_parser(parser: argparse.ArgumentParser): @@ -35,10 +35,10 @@ def execute(args: typing.NamedTuple): def print_copy_result(node_id, success, err): - print("-" * (len(node_id) + 6)) - print("| ", node_id, " |") - print("-" * (len(node_id) + 6)) + log.print("-" * (len(node_id) + 6)) + log.print("| %s |", node_id) + log.print("-" * (len(node_id) + 6)) if success: - print("Copy successful") + log.print("Copy successful") else: - print(err) + log.print(err) diff --git a/aztk_cli/spark/endpoints/cluster/cluster_run.py b/aztk_cli/spark/endpoints/cluster/cluster_run.py index ef612b80..2ab910da 100644 --- a/aztk_cli/spark/endpoints/cluster/cluster_run.py +++ b/aztk_cli/spark/endpoints/cluster/cluster_run.py @@ -2,7 +2,7 @@ import typing import aztk.spark -from aztk_cli import config, utils +from aztk_cli import config, log, utils def setup_parser(parser: argparse.ArgumentParser): diff --git a/aztk_cli/spark/endpoints/job/get_app_logs.py b/aztk_cli/spark/endpoints/job/get_app_logs.py index d6c5e0cf..3981f4f2 100644 --- a/aztk_cli/spark/endpoints/job/get_app_logs.py +++ b/aztk_cli/spark/endpoints/job/get_app_logs.py @@ -3,7 +3,7 @@ import typing import aztk.spark -from aztk_cli import config, utils +from aztk_cli import config, log, utils def setup_parser(parser: argparse.ArgumentParser): @@ -28,4 +28,4 @@ def execute(args: typing.NamedTuple): with open(os.path.abspath(os.path.expanduser(args.output)), "w", encoding="UTF-8") as f: f.write(app_log.log) else: - print(app_log.log) + log.print(app_log.log) diff --git a/aztk_cli/spark/endpoints/job/stop.py b/aztk_cli/spark/endpoints/job/stop.py index d261d747..9232d106 100644 --- a/aztk_cli/spark/endpoints/job/stop.py +++ b/aztk_cli/spark/endpoints/job/stop.py @@ -3,7 +3,7 @@ import typing import aztk.spark -from aztk_cli import config, utils +from aztk_cli import config, log, utils def setup_parser(parser: argparse.ArgumentParser): @@ -16,4 +16,4 @@ def setup_parser(parser: argparse.ArgumentParser): def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) spark_client.stop_job(args.job_id) - print("Stopped Job {0}".format(args.job_id)) + log.print("Stopped Job {0}".format(args.job_id)) diff --git a/aztk_cli/utils.py b/aztk_cli/utils.py index 4af3d337..9d6f65fd 100644 --- a/aztk_cli/utils.py +++ b/aztk_cli/utils.py @@ -4,7 +4,6 @@ import sys import threading import time -import yaml from subprocess import call from typing import List @@ -131,7 +130,7 @@ def stream_logs(client, cluster_id, application_name): application_name=application_name, tail=True, current_bytes=current_bytes) - print(app_logs.log, end="") + log.print(app_logs.log) if app_logs.application_state == 'completed': return app_logs.exit_code current_bytes = app_logs.total_bytes @@ -466,4 +465,4 @@ def log_execute_result(node_id, result): log.info("%s\n", result) else: for line in result: - print(line) + log.print(line)