Set logger error

kapitan/__init__.py

@@ -7,32 +7,35 @@
 
 import os
 import sys
-import logging
 
-
-def setup_logging(name=None, level=logging.INFO, force=False):
-    "setup logging and deal with logging behaviours in MacOS python 3.8 and below"
-    # default opts
-    kwopts = {"format": "%(message)s", "level": level}
-
-    if level == logging.DEBUG:
-        kwopts["format"] = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"
-
-    if sys.version_info >= (3, 8) and force:
-        kwopts["force"] = True
-
-    logging.basicConfig(**kwopts)
-
-    if sys.version_info < (3, 8) and force:
-        logging.getLogger(name).setLevel(level)
-
-
-# XXX in MacOS, updating logging level in __main__ doesn't work for python3.8+
-# XXX this is a hack that seems to work
-if "-v" in sys.argv or "--verbose" in sys.argv:
-    setup_logging(level=logging.DEBUG)
-else:
-    setup_logging()
+# this dict is used to confgiure the python logging in various places, such as setup spawned processes
+logging_config = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "brief": {"format": "%(message)s"},
+        "extended": {"format": "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"},
+    },
+    "handlers": {
+        "brief": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "brief",
+            "stream": "ext://sys.stdout",
+        },
+        "extended": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "extended",
+            "stream": "ext://sys.stdout",
+        },
+    },
+    "loggers": {
+        "kapitan": {"level": "INFO", "propagate": True},
+        "reclass": {"level": "INFO", "propagate": True},
+    },
+    "root": {"level": "ERROR", "handlers": ["brief"]},
+}
 
 # Adding reclass to PYTHONPATH
 sys.path.insert(0, os.path.dirname(__file__) + "/reclass")

kapitan/cli.py

@@ -11,14 +11,14 @@
 
 import argparse
 import json
-import logging
+import logging.config
 import multiprocessing
 import os
 import sys
 
 import yaml
 
-from kapitan import cached, defaults, setup_logging
+from kapitan import cached, defaults, logging_config
 from kapitan.initialiser import initialise_skeleton
 from kapitan.inputs.jsonnet import jsonnet_file
 from kapitan.lint import start_lint
@@ -305,6 +305,7 @@ def build_parser():
         action="store_true",
         help="dumps all none-type entries as empty, default is dumping as 'null'",
     )
+    add_logging_argument("compile", compile_parser)
 
     compile_selector_parser = compile_parser.add_mutually_exclusive_group()
     compile_selector_parser.add_argument(
@@ -379,6 +380,7 @@ def build_parser():
         default=from_dot_kapitan("inventory", "multiline-string-style", "double-quotes"),
         help="set multiline string style to STYLE, default is 'double-quotes'",
     )
+    add_logging_argument("inventory", inventory_parser)
 
     searchvar_parser = subparser.add_parser(
         "searchvar", aliases=["sv"], help="show all inventory files where var is declared"
@@ -622,6 +624,57 @@ def build_parser():
     return parser
 
 
+def add_logging_argument(command, parser):
+    """Adds logging argument for command to a parser.
+
+    Args:
+        command (str): the command like compile, inventory
+        parser (ArgumentParser): the argument parser to which to add the argument
+    """
+    parser.add_argument(
+        "--logging-config",
+        metavar="FILE",
+        default=from_dot_kapitan(command, "logging", ""),
+        help="python logging configuration",
+    )
+
+
+def setup_logging(args=None):
+    """Setups logging using the args
+    The default configuration is:
+        kapitan and reclass on INFO as they are for regular output to users
+
+    The logging-config option takes precedence over all. It reads a yaml file with based
+    on the https://docs.python.org/3/library/logging.config.html#logging-config-dictschema.
+
+    The verbose option option takes precedence over the quiet one when they are used.
+    """
+    if hasattr(args, "logging_config") and args.logging_config != "":
+        with open(args.logging_config, "r", encoding="ascii") as f:
+            logging_config.update(yaml.safe_load(f))
+    else:
+        if hasattr(args, "verbose") and args.verbose:
+            set_kapitan_loggers("DEBUG", "extended")
+        elif hasattr(args, "quiet") and args.quiet:
+            set_kapitan_loggers("CRITICAL", "extended")
+
+    logging.config.dictConfig(logging_config)
+
+
+def set_kapitan_loggers(level, stream):
+    """Set logging level of kapitan and reclass loggers.
+
+    Args:
+        level (str): the log level to use
+        stream (str): the stream to use
+    """
+    logging_config["loggers"] = {
+        "kapitan": {"level": level, "propagate": True},
+        "reclass": {"level": level, "propagate": True},
+    }
+    logging_config["root"] = {"level": level, "handlers": [stream]}
+
+
 def main():
     """main function for command line usage"""
     try:
@@ -649,10 +702,7 @@ def main():
     assert "name" in args, "All cli commands must have provided default name"
     cached.args[args.name] = args
 
-    if hasattr(args, "verbose") and args.verbose:
-        setup_logging(level=logging.DEBUG, force=True)
-    elif hasattr(args, "quiet") and args.quiet:
-        setup_logging(level=logging.CRITICAL, force=True)
+    setup_logging(args)
 
     # call chosen command
     args.func(args)

kapitan/dependency_manager/base.py

@@ -4,6 +4,7 @@
 
 import hashlib
 import logging
+import logging.config
 import multiprocessing
 import os
 from collections import defaultdict, namedtuple
@@ -14,6 +15,7 @@
 
 from git import GitCommandError
 from git import Repo
+from kapitan import logging_config
 from kapitan.errors import GitSubdirNotFoundError, GitFetchingError, HelmFetchingError
 from kapitan.helm_cli import helm_cli
 from kapitan.utils import (
@@ -82,20 +84,37 @@ def fetch_dependencies(output_path, target_objs, save_dir, force, pool):
             )
             continue
 
-    git_worker = partial(fetch_git_dependency, save_dir=save_dir, force=force)
-    http_worker = partial(fetch_http_dependency, save_dir=save_dir, force=force)
-    helm_worker = partial(fetch_helm_chart, save_dir=save_dir, force=force)
+    git_worker = partial(
+        fetch_git_dependency,
+        save_dir=save_dir,
+        force=force,
+        logging_config_dict=logging_config,
+    )
+    http_worker = partial(
+        fetch_http_dependency,
+        save_dir=save_dir,
+        force=force,
+        logging_config_dict=logging_config,
+    )
+    helm_worker = partial(
+        fetch_helm_chart,
+        save_dir=save_dir,
+        force=force,
+        logging_config_dict=logging_config,
+    )
     [p.get() for p in pool.imap_unordered(http_worker, http_deps.items()) if p]
     [p.get() for p in pool.imap_unordered(git_worker, git_deps.items()) if p]
     [p.get() for p in pool.imap_unordered(helm_worker, helm_deps.items()) if p]
 
 
-def fetch_git_dependency(dep_mapping, save_dir, force, item_type="Dependency"):
+def fetch_git_dependency(dep_mapping, save_dir, force, item_type="Dependency", logging_config_dict=None):
     """
     fetches a git repository at source into save_dir, and copy the repository into
     output_path stored in dep_mapping. ref is used to checkout if exists, fetches master branch by default.
     only subdir is copied into output_path if specified.
     """
+    if logging_config_dict is not None:
+        logging.config.dictConfig(logging_config_dict)
     source, deps = dep_mapping
     # to avoid collisions between basename(source)
     path_hash = hashlib.sha256(os.path.dirname(source).encode()).hexdigest()[:8]
@@ -152,11 +171,13 @@ def fetch_git_source(source, save_dir, item_type):
         raise GitFetchingError("{} {}: fetching unsuccessful\n{}".format(item_type, source, e.stderr))
 
 
-def fetch_http_dependency(dep_mapping, save_dir, force, item_type="Dependency"):
+def fetch_http_dependency(dep_mapping, save_dir, force, item_type="Dependency", logging_config_dict=None):
     """
     fetches a http[s] file at source and saves into save_dir, after which it is copied into
     the output_path stored in dep_mapping
     """
+    if logging_config_dict is not None:
+        logging.config.dictConfig(logging_config_dict)
     source, deps = dep_mapping
     # to avoid collisions between basename(source)
     path_hash = hashlib.sha256(os.path.dirname(source).encode()).hexdigest()[:8]
@@ -221,10 +242,12 @@ def fetch_http_source(source, save_path, item_type):
     return None
 
 
-def fetch_helm_chart(dep_mapping, save_dir, force):
+def fetch_helm_chart(dep_mapping, save_dir, force, logging_config_dict=None):
     """
     downloads a helm chart and its subcharts from source then untars and moves it to save_dir
     """
+    if logging_config_dict is not None:
+        logging.config.dictConfig(logging_config_dict)
     source, deps = dep_mapping
 
     # to avoid collisions between source.chart_name/source.version

kapitan/remoteinventory/fetch.py

@@ -4,7 +4,7 @@
 from collections import defaultdict
 from functools import partial
 
-from kapitan import cached
+from kapitan import cached, logging_config
 from kapitan.dependency_manager.base import fetch_git_dependency, fetch_http_dependency
 from kapitan.utils import normalise_join_path
 
@@ -69,7 +69,13 @@ def fetch_inventories(inventory_path, target_objs, save_dir, force, pool):
             logger.debug("Target object %s has no inventory key", target_obj["vars"]["target"])
             continue
 
-    git_worker = partial(fetch_git_dependency, save_dir=save_dir, force=force, item_type="Inventory")
+    git_worker = partial(
+        fetch_git_dependency,
+        save_dir=save_dir,
+        force=force,
+        item_type="Inventory",
+        logging_config_dict=logging_config,
+    )
     http_worker = partial(fetch_http_dependency, save_dir=save_dir, force=force, item_type="Inventory")
     [p.get() for p in pool.imap_unordered(git_worker, git_inventories.items()) if p]
     [p.get() for p in pool.imap_unordered(http_worker, http_inventories.items()) if p]

kapitan/targets.py

@@ -8,6 +8,7 @@
 "kapitan targets"
 import json
 import logging
+import logging.config
 import multiprocessing
 import os
 import shutil
@@ -21,7 +22,7 @@
 import yaml
 from reclass.errors import NotFoundError, ReclassException
 
-from kapitan import cached, defaults
+from kapitan import cached, defaults, logging_config
 from kapitan.dependency_manager.base import fetch_dependencies
 from kapitan.errors import CompileError, InventoryError, KapitanError
 from kapitan.inputs.copy import Copy
@@ -158,6 +159,7 @@ def compile_targets(
             ref_controller=ref_controller,
             inventory_path=inventory_path,
             globals_cached=cached.as_dict(),
+            logging_config_dict=logging_config,
             **kwargs,
         )
 
@@ -187,10 +189,13 @@ def compile_targets(
 
         # validate the compiled outputs
         if kwargs.get("validate", False):
-            validate_map = create_validate_mapping(target_objs, compile_path)
+            validate_map = create_validate_mapping(
+                target_objs, compile_path, logging_config_dict=logging_config
+            )
             worker = partial(
                 schema_validate_kubernetes_output,
                 cache_dir=kwargs.get("schemas_path", "./schemas"),
+                logging_config_dict=logging_config,
             )
             [p.get() for p in pool.imap_unordered(worker, validate_map.items()) if p]
 
@@ -455,8 +460,19 @@ def search_targets(inventory_path, targets, labels):
     return targets_found
 
 
-def compile_target(target_obj, search_paths, compile_path, ref_controller, globals_cached=None, **kwargs):
+def compile_target(
+    target_obj,
+    search_paths,
+    compile_path,
+    ref_controller,
+    globals_cached=None,
+    logging_config_dict=None,
+    **kwargs,
+):
     """Compiles target_obj and writes to compile_path"""
+
+    if logging_config_dict is not None:
+        logging.config.dictConfig(logging_config_dict)
     start = time.time()
     compile_objs = target_obj["compile"]
     ext_vars = target_obj["vars"]
@@ -776,12 +792,18 @@ def schema_validate_compiled(args):
         os.makedirs(args.schemas_path)
         logger.info("created schema-cache-path at %s", args.schemas_path)
 
-    worker = partial(schema_validate_kubernetes_output, cache_dir=args.schemas_path)
+    worker = partial(
+        schema_validate_kubernetes_output,
+        cache_dir=args.schemas_path,
+        logging_config_dict=logging_config,
+    )
     pool = multiprocessing.Pool(args.parallelism)
 
     try:
         target_objs = load_target_inventory(args.inventory_path, args.targets)
-        validate_map = create_validate_mapping(target_objs, args.compiled_path)
+        validate_map = create_validate_mapping(
+            target_objs, args.compiled_path, logging_config_dict=logging_config
+        )
 
         [p.get() for p in pool.imap_unordered(worker, validate_map.items()) if p]
         pool.close()
@@ -807,11 +829,13 @@ def schema_validate_compiled(args):
         pool.join()
 
 
-def create_validate_mapping(target_objs, compiled_path):
+def create_validate_mapping(target_objs, compiled_path, logging_config_dict=None):
     """
     creates mapping of (kind, version) tuple to output_paths across different targets
     this is required to avoid redundant schema fetch when multiple targets use the same schema for validation
     """
+    if logging_config_dict is not None:
+        logging.config.dictConfig(logging_config_dict)
     validate_files_map = defaultdict(list)
     for target_obj in target_objs:
         target_name = target_obj["vars"]["target"]
@@ -843,11 +867,13 @@ def create_validate_mapping(target_objs, compiled_path):
     return validate_files_map
 
 
-def schema_validate_kubernetes_output(validate_data, cache_dir):
+def schema_validate_kubernetes_output(validate_data, cache_dir, logging_config_dict=None):
     """
     validates given files according to kubernetes manifest schemas
     schemas are cached from/to cache_dir
     validate_data must be of structure ((kind, version), validate_files)
     """
     (kind, version), validate_files = validate_data
+    if logging_config_dict is not None:
+        logging.config.dictConfig(logging_config_dict)
     KubernetesManifestValidator(cache_dir).validate(validate_files, kind=kind, version=version)