From 743430529f106c609be1a90db57c2dfa551510f5 Mon Sep 17 00:00:00 2001 From: Ilya Siamionau Date: Mon, 11 Sep 2023 21:49:26 +0200 Subject: [PATCH] CM-27145 - Add setting of logger level for all loggers; fix stream of loggers; fix setting of logger level from env --- cycode/cli/code_scanner.py | 6 ++--- cycode/cli/consts.py | 2 -- cycode/cli/main.py | 9 +++---- cycode/cyclient/config.py | 54 ++++++++++++++++++++++++-------------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/cycode/cli/code_scanner.py b/cycode/cli/code_scanner.py index f63ebf08..128e8a6f 100644 --- a/cycode/cli/code_scanner.py +++ b/cycode/cli/code_scanner.py @@ -33,13 +33,13 @@ load_json, ) from cycode.cli.utils.progress_bar import ProgressBarSection -from cycode.cli.utils.progress_bar import logger as progress_bar_logger from cycode.cli.utils.scan_batch import run_parallel_batched_scan from cycode.cli.utils.scan_utils import set_issue_detected from cycode.cli.utils.string_utils import get_content_size, is_binary_content from cycode.cli.utils.task_timer import TimeoutAfter from cycode.cli.zip_file import InMemoryZip from cycode.cyclient import logger +from cycode.cyclient.config import set_logging_level from cycode.cyclient.models import Detection, DetectionSchema, DetectionsPerFile, ZippedFileScanResult if TYPE_CHECKING: @@ -1399,9 +1399,7 @@ def perform_post_pre_receive_scan_actions(context: click.Context) -> None: def enable_verbose_mode(context: click.Context) -> None: context.obj['verbose'] = True - # TODO(MarshalX): rework setting the log level for loggers - logger.setLevel(logging.DEBUG) - progress_bar_logger.setLevel(logging.DEBUG) + set_logging_level(logging.DEBUG) def is_verbose_mode_requested_in_pre_receive_scan() -> bool: diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 43046f7f..23b7471a 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -112,8 +112,6 @@ TIMEOUT_ENV_VAR_NAME = 'TIMEOUT' CYCODE_CLI_REQUEST_TIMEOUT_ENV_VAR_NAME = 'CYCODE_CLI_REQUEST_TIMEOUT' LOGGING_LEVEL_ENV_VAR_NAME = 'LOGGING_LEVEL' -# use only for dev envs locally -BATCH_SIZE_ENV_VAR_NAME = 'BATCH_SIZE' VERBOSE_ENV_VAR_NAME = 'CYCODE_CLI_VERBOSE' CYCODE_CONFIGURATION_DIRECTORY: str = '.cycode' diff --git a/cycode/cli/main.py b/cycode/cli/main.py index 23c211c3..94f3ff29 100644 --- a/cycode/cli/main.py +++ b/cycode/cli/main.py @@ -22,8 +22,7 @@ from cycode.cli.user_settings.user_settings_commands import add_exclusions, set_credentials from cycode.cli.utils import scan_utils from cycode.cli.utils.progress_bar import get_progress_bar -from cycode.cli.utils.progress_bar import logger as progress_bar_logger -from cycode.cyclient import logger +from cycode.cyclient.config import set_logging_level from cycode.cyclient.cycode_client_base import CycodeClientBase from cycode.cyclient.models import UserAgentOptionScheme from cycode.cyclient.scan_config.scan_config_creator import create_scan_client @@ -228,10 +227,8 @@ def main_cli( verbose = verbose or configuration_manager.get_verbose_flag() context.obj['verbose'] = verbose - # TODO(MarshalX): rework setting the log level for loggers - log_level = logging.DEBUG if verbose else logging.INFO - logger.setLevel(log_level) - progress_bar_logger.setLevel(log_level) + if verbose: + set_logging_level(logging.DEBUG) context.obj['output'] = output if output == 'json': diff --git a/cycode/cyclient/config.py b/cycode/cyclient/config.py index 2c2f3c00..da3c6a18 100644 --- a/cycode/cyclient/config.py +++ b/cycode/cyclient/config.py @@ -6,17 +6,21 @@ from cycode.cli import consts from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cyclient.config_dev import DEV_MODE_ENV_VAR_NAME, DEV_TENANT_ID_ENV_VAR_NAME -# set io encoding (for windows) -from .config_dev import DEV_MODE_ENV_VAR_NAME, DEV_TENANT_ID_ENV_VAR_NAME -sys.stdout.reconfigure(encoding='UTF-8') -sys.stderr.reconfigure(encoding='UTF-8') +def _set_io_encodings() -> None: + # set io encoding (for Windows) + sys.stdout.reconfigure(encoding='UTF-8') + sys.stderr.reconfigure(encoding='UTF-8') + + +_set_io_encodings() # logs logging.basicConfig( - stream=sys.stdout, - level=logging.DEBUG, + stream=sys.stderr, + level=logging.INFO, format='%(asctime)s [%(name)s] %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', ) @@ -34,19 +38,28 @@ consts.TIMEOUT_ENV_VAR_NAME: 300, consts.LOGGING_LEVEL_ENV_VAR_NAME: logging.INFO, DEV_MODE_ENV_VAR_NAME: 'False', - consts.BATCH_SIZE_ENV_VAR_NAME: 20, } configuration = dict(DEFAULT_CONFIGURATION, **os.environ) +_CREATED_LOGGERS = set() + def get_logger(logger_name: Optional[str] = None) -> logging.Logger: - logger = logging.getLogger(logger_name) - level = _get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME) - level = level if level in logging._nameToLevel else int(level) - logger.setLevel(level) + config_level = _get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME) + level = logging.getLevelName(config_level) + + new_logger = logging.getLogger(logger_name) + new_logger.setLevel(level) + + _CREATED_LOGGERS.add(new_logger) - return logger + return new_logger + + +def set_logging_level(level: int) -> None: + for created_logger in _CREATED_LOGGERS: + created_logger.setLevel(level) def _get_val_as_string(key: str) -> str: @@ -66,15 +79,20 @@ def _get_val_as_int(key: str) -> Optional[int]: return None -logger = get_logger('cycode cli') +def _is_valid_url(url: str) -> bool: + try: + urlparse(url) + return True + except ValueError as e: + logger.warning(f'Invalid cycode api url: {url}, using default value', e) + return False + +logger = get_logger('cycode cli') configuration_manager = ConfigurationManager() cycode_api_url = configuration_manager.get_cycode_api_url() -try: - urlparse(cycode_api_url) -except ValueError as e: - logger.warning(f'Invalid cycode api url: {cycode_api_url}, using default value', e) +if not _is_valid_url(cycode_api_url): cycode_api_url = consts.DEFAULT_CYCODE_API_URL timeout = _get_val_as_int(consts.CYCODE_CLI_REQUEST_TIMEOUT_ENV_VAR_NAME) @@ -83,5 +101,3 @@ def _get_val_as_int(key: str) -> Optional[int]: dev_mode = _get_val_as_bool(DEV_MODE_ENV_VAR_NAME) dev_tenant_id = _get_val_as_string(DEV_TENANT_ID_ENV_VAR_NAME) -batch_size = _get_val_as_int(consts.BATCH_SIZE_ENV_VAR_NAME) -verbose = _get_val_as_bool(consts.VERBOSE_ENV_VAR_NAME)