Ensure logging.yml is optional and tidy logging config (#1543)

features/load_context.feature

@@ -26,13 +26,13 @@ Feature: Custom Kedro project
         # we have more clarity on the necessity of these logging messages.
         # And I should get a message including "Registered hooks from 1 installed plugin(s): test-plugin-0.1"
 
-     Scenario: Hooks from installed plugins are automatically registered and work with the parallel runner
+    Scenario: Hooks from installed plugins are automatically registered and work with the parallel runner
         Given I have installed the test plugin
         When I execute the kedro command "run --runner=ParallelRunner"
         Then I should get a successful exit code
         And I should get a message including "Reached after_catalog_created hook"
-         # See explanation in test above.
-         # And I should get a message including "Registered hooks from 1 installed plugin(s): test-plugin-0.1"
+        # See explanation in test above.
+        # And I should get a message including "Registered hooks from 1 installed plugin(s): test-plugin-0.1"
 
     Scenario: Disable automatically registered plugin hooks
         Given I have installed the test plugin

features/run.feature

@@ -9,17 +9,17 @@ Feature: Run Project
     And I have run a non-interactive kedro new with starter
     When I execute the kedro command "run"
     Then I should get a successful exit code
-    And the console log should show that 4 nodes were run
+    And the logs should show that 4 nodes were run
 
   Scenario: Run parallel runner with default python entry point with example code
     Given I have prepared a config file
     And I have run a non-interactive kedro new with starter
     When I execute the kedro command "run --runner=ParallelRunner"
     Then I should get a successful exit code
-    And the console log should show that "split_data" was run
-    And the console log should show that "train_model" was run
-    And the console log should show that "predict" was run
-    And the console log should show that "report_accuracy" was run
+    And the logs should show that "split_data" was run
+    And the logs should show that "train_model" was run
+    And the logs should show that "predict" was run
+    And the logs should show that "report_accuracy" was run
 
   Scenario: Run default python entry point without example code
     Given I have prepared a config file
@@ -34,19 +34,19 @@ Feature: Run Project
     And I have prepared a run_config file with config options
     When I execute the kedro command "run --config run_config.yml"
     Then I should get a successful exit code
-    And the console log should show that 1 nodes were run
+    And the logs should show that 1 nodes were run
 
   Scenario: Run kedro run with config file and override option
     Given I have prepared a config file
     And I have run a non-interactive kedro new with starter
     And I have prepared a run_config file with config options
     When I execute the kedro command "run --config run_config.yml --pipeline __default__"
     Then I should get a successful exit code
-    And the console log should show that 4 nodes were run
+    And the logs should show that 4 nodes were run
 
   Scenario: Run kedro run with extra parameters
     Given I have prepared a config file
     And I have run a non-interactive kedro new with starter
     When I execute the kedro command "run --params extra1:1,extra2:value2"
     Then I should get a successful exit code
-    And the console log should show that 4 nodes were run
+    And the logs should show that 4 nodes were run

features/steps/cli_steps.py

@@ -451,20 +451,24 @@ def check_pipeline_not_empty(context):
     assert "pipeline = pipeline([])" not in pipeline_file.read_text("utf-8")
 
 
-@then("the console log should show that {number} nodes were run")
+@then("the logs should show that {number} nodes were run")
 def check_one_node_run(context, number):
     expected_log_line = f"Completed {number} out of {number} tasks"
+    info_log = context.root_project_dir / "logs" / "info.log"
     assert expected_log_line in context.result.stdout
+    assert expected_log_line in info_log.read_text()
 
 
-@then('the console log should show that "{node}" was run')
+@then('the logs should show that "{node}" was run')
 def check_correct_nodes_run(context, node):
     expected_log_line = f"Running node: {node}"
+    info_log = context.root_project_dir / "logs" / "info.log"
     stdout = context.result.stdout
     assert expected_log_line in stdout, (
         "Expected the following message segment to be printed on stdout: "
         f"{expected_log_line},\nbut got {stdout}"
     )
+    assert expected_log_line in info_log.read_text()
 
 
 @then("I should get a successful exit code")

features/steps/test_plugin/plugin.py

@@ -3,13 +3,16 @@
 
 from kedro.framework.hooks import hook_impl
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
 
 class MyPluginHook:
     @hook_impl
     def after_catalog_created(
         self, catalog
     ):  # pylint: disable=unused-argument,no-self-use
-        logging.info("Reached after_catalog_created hook")
+        logger.info("Reached after_catalog_created hook")
 
 
 hooks = MyPluginHook()

features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/base/logging.yml

@@ -1,55 +1,44 @@
 version: 1
+
 disable_existing_loggers: False
+
 formatters:
-    simple:
-        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    json_formatter:
-        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-        class: pythonjsonlogger.jsonlogger.JsonFormatter
+  simple:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
 
 handlers:
-    console:
-        class: logging.StreamHandler
-        level: INFO
-        formatter: simple
-        stream: ext://sys.stdout
-
-    info_file_handler:
-        class: logging.handlers.RotatingFileHandler
-        level: INFO
-        formatter: simple
-        filename: logs/info.log
-        maxBytes: 10485760 # 10MB
-        backupCount: 20
-        encoding: utf8
-        delay: True
-
-    error_file_handler:
-        class: logging.handlers.RotatingFileHandler
-        level: ERROR
-        formatter: simple
-        filename: logs/errors.log
-        maxBytes: 10485760 # 10MB
-        backupCount: 20
-        encoding: utf8
-        delay: True
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    stream: ext://sys.stdout
 
-loggers:
-    anyconfig:
-        level: WARNING
-        handlers: [console, info_file_handler, error_file_handler]
-        propagate: no
+  info_file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: INFO
+    formatter: simple
+    filename: logs/info.log
+    maxBytes: 10485760 # 10MB
+    backupCount: 20
+    encoding: utf8
+    delay: True
 
-    kedro.io:
-        level: INFO
-        handlers: [console, info_file_handler, error_file_handler]
-        propagate: no
+  error_file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: ERROR
+    formatter: simple
+    filename: logs/errors.log
+    maxBytes: 10485760 # 10MB
+    backupCount: 20
+    encoding: utf8
+    delay: True
 
-    kedro.pipeline:
-        level: INFO
-        handlers: [console, info_file_handler, error_file_handler]
-        propagate: no
+loggers:
+  kedro:
+    level: INFO
 
-root:
+  {{ cookiecutter.python_package }}:
     level: INFO
-    handlers: [console, info_file_handler, error_file_handler]
+
+root:
+  handlers: [console, info_file_handler, error_file_handler]

features/steps/util.py

@@ -3,38 +3,11 @@
 
 import os
 import re
-import tempfile
 from contextlib import contextmanager
 from pathlib import Path
 from time import sleep, time
 from typing import Any, Callable, Iterator, List
 
-import pandas as pd
-
-
-def get_sample_csv_content():
-    return """col1, col2, col3
-    1, 2, 3
-    4, 5, 6
-    """
-
-
-def get_sample_data_frame():
-    data = {"col1": [1, 2], "col2": [4, 5], "col3": [5, 6]}
-    return pd.DataFrame(data)
-
-
-def create_temp_csv():
-    _, csv_file_path = tempfile.mkstemp(suffix=".csv")
-    return csv_file_path
-
-
-def create_sample_csv():
-    csv_file_path = create_temp_csv()
-    with open(csv_file_path, mode="w", encoding="utf-8") as output_file:
-        output_file.write(get_sample_csv_content())
-    return csv_file_path
-
 
 @contextmanager
 def chdir(path: Path) -> Iterator:
@@ -99,55 +72,6 @@ def wait_for(
     )
 
 
-def get_logline_count(logfile: str) -> int:
-    """Get line count in logfile
-
-    Note: If logfile doesn't exist will return 0
-
-    Args:
-        logfile: path to logfile
-
-    Returns:
-        line count of logfile
-    """
-    try:
-        with open(logfile, encoding="utf-8") as file_handle:
-            return sum(1 for i in file_handle)
-    except FileNotFoundError:
-        return 0
-
-
-def get_last_logline(logfile: str) -> str:
-    """Get last line of logfile
-
-    Args:
-        logfile: path to logfile
-
-    Returns:
-        last line of logfile
-    """
-    line = ""
-    with open(logfile, encoding="utf-8") as file_handle:
-        for line in file_handle:
-            pass
-
-    return line
-
-
-def get_logfile_path(proj_dir: Path) -> str:
-    """
-    Helper function to fet full path of `pipeline.log` inside project
-
-    Args:
-        proj_dir: path to proj_dir
-
-    Returns:
-        path to `pipeline.log`
-    """
-    log_file = (proj_dir / "logs" / "visualization" / "pipeline.log").absolute()
-    return str(log_file)
-
-
 def parse_csv(text: str) -> List[str]:
     """Parse comma separated **double quoted** strings in behave steps
 

kedro/config/logging.yml

@@ -1,44 +1,21 @@
 version: 1
+
 disable_existing_loggers: False
+
 formatters:
-    simple:
-        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+  simple:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
 
 handlers:
-    console:
-        class: logging.StreamHandler
-        level: INFO
-        formatter: simple
-        stream: ext://sys.stdout
-
-    info_file_handler:
-        class: logging.handlers.RotatingFileHandler
-        level: INFO
-        formatter: simple
-        filename: info.log
-        maxBytes: 10485760 # 10MB
-        backupCount: 20
-        encoding: utf8
-        delay: True
-
-    error_file_handler:
-        class: logging.handlers.RotatingFileHandler
-        level: ERROR
-        formatter: simple
-        filename: errors.log
-        maxBytes: 10485760 # 10MB
-        backupCount: 20
-        encoding: utf8
-        delay: True
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    stream: ext://sys.stdout
 
 loggers:
-    kedro.framework.cli.hooks.manager:
-        level: INFO
-        handlers: [console]
-        propagate: no
-
-    kedro:
-        level: INFO
+  kedro:
+    level: INFO
 
 root:
-    handlers: [console, info_file_handler, error_file_handler]
+  handlers: [console]

kedro/framework/session/session.py

@@ -13,7 +13,7 @@
 import click
 
 from kedro import __version__ as kedro_version
-from kedro.config import ConfigLoader
+from kedro.config import ConfigLoader, MissingConfigException
 from kedro.framework.context import KedroContext
 from kedro.framework.context.context import _convert_paths_to_absolute_posix
 from kedro.framework.hooks import _create_hook_manager
@@ -185,8 +185,15 @@ def _get_logging_config(self) -> Dict[str, Any]:
 
     def _setup_logging(self) -> None:
         """Register logging specified in logging directory."""
-        conf_logging = self._get_logging_config()
-        configure_logging(conf_logging)
+        try:
+            conf_logging = self._get_logging_config()
+        except MissingConfigException:
+            self._logger.debug(
+                "No project logging configuration loaded; "
+                "Kedro's default logging configuration will be used."
+            )
+        else:
+            configure_logging(conf_logging)
 
     def _init_store(self) -> BaseSessionStore:
         store_class = settings.SESSION_STORE_CLASS