feat: Added a --log-format option as a shorcut to quickly change th…

…e format of logs

docs/docs/reference/command-line-interface.md

@@ -20,6 +20,7 @@ The following CLI options are available for the top-level `meltano` command:
 ### Log Configurations
 
 - [`--log-config`](/reference/settings#clilog_config) - Path to a logging configuration file. See [Logging](/guide/logging) for more information.
+- [`--log-format`](/reference/settings#clilog_format) - Shortcut for setting the log format instead of using `--log-config`. See the CLI output for available options.
 - [`--log-level`](/reference/settings#clilog_level) - Set the log level for the command. Valid values are `debug`, `info`, `warning`, `error`, and `critical`.
 
 ### No Color

docs/docs/reference/settings.mdx

@@ -518,6 +518,41 @@ meltano --log-level=debug ...
   </TabItem>
 </Tabs>
 
+### `cli.log_format`
+
+- [Environment variable](/guide/configuration#configuring-settings): `MELTANO_CLI_LOG_FORMAT`.
+- `meltano` CLI option: `--log-format`
+- Options: `colored`, `uncolored`, `json`, `key_value`
+- Default: `colored`
+
+A shortcut for setting the format of the log output. Ignored if a local logging config is found.
+
+#### How to use
+
+<Tabs className="meltano-tabs" queryString="meltano-tabs">
+  <TabItem className="meltano-tab-content" value="meltano config" label="meltano config" default>
+
+```bash
+meltano config meltano set cli log_format json
+```
+
+  </TabItem>
+  <TabItem className="meltano-tab-content" value="env" label="env" default>
+
+```bash
+export MELTANO_CLI_LOG_FORMAT=json
+```
+
+  </TabItem>
+  <TabItem className="meltano-tab-content" value="command" label="command" default>
+
+```bash
+meltano --log-format=json ...
+```
+
+  </TabItem>
+</Tabs>
+
 ### `cli.log_config`
 
 - [Environment variable](/guide/configuration#configuring-settings): `MELTANO_CLI_LOG_CONFIG`.

src/meltano/cli/cli.py

@@ -17,7 +17,7 @@
 from meltano.cli.utils import InstrumentedGroup
 from meltano.core.behavior.versioned import IncompatibleVersionError
 from meltano.core.error import EmptyMeltanoFileException, ProjectNotFound
-from meltano.core.logging import LEVELS, setup_logging
+from meltano.core.logging import LEVELS, LogFormat, setup_logging
 from meltano.core.project import PROJECT_ENVIRONMENT_ENV, Project
 from meltano.core.project_settings_service import ProjectSettingsService
 from meltano.core.tracking import Tracker
@@ -57,6 +57,12 @@ def main(self, *args, **kwargs) -> t.Any:
     context_settings={"token_normalize_func": lambda x: x.replace("_", "-")},
 )
 @click.option("--log-level", type=click.Choice(tuple(LEVELS)))
+@click.option(
+    "--log-format",
+    type=click.Choice(tuple(LogFormat)),
+    default=LogFormat.colored.value,
+    help="A shortcut for setting the format of the log output.",
+)
 @click.option(
     "--log-config",
     type=str,
@@ -79,6 +85,7 @@ def main(self, *args, **kwargs) -> t.Any:
 def cli(  # noqa: C901,WPS231
     ctx: click.Context,
     log_level: str,
+    log_format: str,
     log_config: str,
     environment: str,
     no_environment: bool,
@@ -97,6 +104,9 @@ def cli(  # noqa: C901,WPS231
     if log_config:
         ProjectSettingsService.config_override["cli.log_config"] = log_config
 
+    if log_format:
+        ProjectSettingsService.config_override["cli.log_format"] = log_format
+
     ctx.obj["explicit_no_environment"] = no_environment
     no_color = get_no_color_flag()
     if no_color:

src/meltano/core/bundle/settings.yml

@@ -90,6 +90,18 @@ settings:
   env_specific: true
 - name: cli.log_config
   value: logging.yaml
+- name: cli.log_format
+  kind: options
+  options:
+  - label: Colored
+    value: colored
+  - label: Uncolored
+    value: uncolored
+  - label: JSON
+    value: json
+  - label: Key-Value
+    value: key_value
+  value: colored
 
 # Snowplow Tracking
 - name: snowplow.collector_endpoints

src/meltano/core/logging/__init__.py

@@ -9,10 +9,17 @@
     SizeThresholdJobLogException,
 )
 from .output_logger import OutputLogger
-from .utils import DEFAULT_LEVEL, LEVELS, capture_subprocess_output, setup_logging
+from .utils import (
+    DEFAULT_LEVEL,
+    LEVELS,
+    LogFormat,
+    capture_subprocess_output,
+    setup_logging,
+)
 
 __all__ = [
     "JobLoggingService",
+    "LogFormat",
     "MissingJobLogException",
     "SizeThresholdJobLogException",
     "OutputLogger",

src/meltano/core/logging/utils.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import enum
 import logging
 import os
 import typing as t
@@ -34,6 +35,32 @@
 )
 
 
+# TODO: Use StrEnum on Python 3.9+
+class LogFormat(str, enum.Enum):
+    """Log format options."""
+
+    colored = "colored"
+    uncolored = "uncolored"
+    json = "json"
+    key_value = "key_value"
+
+    def __str__(self) -> str:
+        """Return the str() of the enum member.
+
+        Returns:
+            Value of the enum as a string.
+        """
+        return self.value
+
+    def __repr__(self) -> str:
+        """Return the repr() of the enum member.
+
+        Returns:
+            Value of the enum as a string.
+        """
+        return self.value
+
+
 def parse_log_level(log_level: str) -> int:
     """Parse a level descriptor into an logging level.
 
@@ -62,40 +89,71 @@ def read_config(config_file: os.PathLike | None = None) -> dict | None:
         return None
 
 
-def default_config(log_level: str) -> dict:
+def default_config(
+    log_level: str,
+    *,
+    log_format: LogFormat = LogFormat.colored,
+) -> dict:
     """Generate a default logging config.
 
     Args:
         log_level: set log levels to provided level.
+        log_format: set log format to provided format.
 
     Returns:
          A logging config suitable for use with `logging.config.dictConfig`.
     """
-    no_color = get_no_color_flag()
-
-    if no_color:
-        formatter = rich_exception_formatter_factory(no_color=True)
-    else:
-        formatter = rich_exception_formatter_factory(color_system="truecolor")
+    if log_format == LogFormat.colored:
+        no_color = get_no_color_flag()
+
+        if no_color:
+            formatter = rich_exception_formatter_factory(no_color=True)
+        else:
+            formatter = rich_exception_formatter_factory(color_system="truecolor")
+        formatter_config = {
+            "()": structlog.stdlib.ProcessorFormatter,
+            "processor": structlog.dev.ConsoleRenderer(
+                colors=not no_color,
+                exception_formatter=formatter,
+            ),
+            "foreign_pre_chain": LEVELED_TIMESTAMPED_PRE_CHAIN,
+        }
+
+    elif log_format == LogFormat.json:
+        formatter_config = {
+            "()": structlog.stdlib.ProcessorFormatter,
+            "processor": structlog.processors.JSONRenderer(),
+            "foreign_pre_chain": LEVELED_TIMESTAMPED_PRE_CHAIN,
+        }
+    elif log_format == LogFormat.key_value:
+        formatter_config = {
+            "()": structlog.stdlib.ProcessorFormatter,
+            "processor": structlog.processors.KeyValueRenderer(
+                key_order=["timestamp", "level", "event", "logger"]
+            ),
+            "foreign_pre_chain": LEVELED_TIMESTAMPED_PRE_CHAIN,
+        }
+    elif log_format == LogFormat.uncolored:
+        formatter_config = {
+            "()": structlog.stdlib.ProcessorFormatter,
+            "processor": structlog.dev.ConsoleRenderer(
+                colors=False,
+                exception_formatter=rich_exception_formatter_factory(no_color=True),
+            ),
+            "foreign_pre_chain": LEVELED_TIMESTAMPED_PRE_CHAIN,
+        }
 
     return {
         "version": 1,
         "disable_existing_loggers": False,
         "formatters": {
-            "colored": {
-                "()": structlog.stdlib.ProcessorFormatter,
-                "processor": structlog.dev.ConsoleRenderer(
-                    colors=not no_color,
-                    exception_formatter=formatter,
-                ),
-                "foreign_pre_chain": LEVELED_TIMESTAMPED_PRE_CHAIN,
-            },
+            log_format: formatter_config,
         },
         "handlers": {
             "console": {
                 "class": "logging.StreamHandler",
                 "level": log_level.upper(),
-                "formatter": "colored",
+                "formatter": log_format,
                 "stream": "ext://sys.stderr",
             },
         },
@@ -127,22 +185,25 @@ def setup_logging(  # noqa: WPS210
     project: Project | None = None,
     log_level: str = DEFAULT_LEVEL,
     log_config: os.PathLike | None = None,
+    log_format: LogFormat = LogFormat.colored,
 ) -> None:
     """Configure logging for a meltano project.
 
     Args:
         project: the meltano project
         log_level: set log levels to provided level.
         log_config: a logging config suitable for use with `logging.config.dictConfig`.
+        log_format: set log format to provided format.
     """
     logging.basicConfig(force=True)
     log_level = log_level.upper()
 
     if project:
         log_config = log_config or project.settings.get("cli.log_config")
-        log_level = project.settings.get("cli.log_level")
+        log_level = str(project.settings.get("cli.log_level"))
+        log_format = LogFormat(project.settings.get("cli.log_format"))
 
-    config = read_config(log_config) or default_config(log_level)
+    config = read_config(log_config) or default_config(log_level, log_format=log_format)
     logging_config.dictConfig(config)
     structlog.configure(
         processors=[

tests/meltano/core/logging/test_logging_utils.py

@@ -1,10 +1,16 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 
+import freezegun
 import pytest
 
-from meltano.core.logging.utils import capture_subprocess_output
+from meltano.core.logging.utils import (
+    LogFormat,
+    capture_subprocess_output,
+    default_config,
+)
 
 
 class AsyncReader(asyncio.StreamReader):
@@ -31,3 +37,48 @@ def writeline(self, line: str):
 
     await capture_subprocess_output(reader, LineWriter())
     assert ["LINE\n", "LINE 2\n", "�\n"] == output_lines
+
+
+@pytest.mark.parametrize(
+    ("log_format", "expected"),
+    (
+        (
+            LogFormat.colored,
+            "\x1b[2m2021-01-01T00:00:00Z\x1b[0m [\x1b[32m\x1b[1minfo     \x1b[0m] \x1b[1mtest                          \x1b[0m",  # noqa: E501
+        ),
+        (LogFormat.uncolored, "2021-01-01T00:00:00Z [info     ] test"),
+        (
+            LogFormat.json,
+            '{"event": "test", "level": "info", "timestamp": "2021-01-01T00:00:00Z"}',
+        ),
+        (
+            LogFormat.key_value,
+            "timestamp='2021-01-01T00:00:00Z' level='info' event='test' logger=None",
+        ),
+    ),
+)
+def test_default_logging_config_format(log_format: LogFormat, expected: str):
+    config = default_config("info", log_format=log_format)
+    assert log_format in config["formatters"]
+    assert config["handlers"]["console"]["formatter"] == log_format
+    assert config["loggers"][""]["level"] == "INFO"
+
+    # Create a logger that uses the config
+    formatter_config = config["formatters"][log_format]
+    formatter_class = formatter_config.pop("()")
+    formatter = formatter_class(**formatter_config)
+
+    with freezegun.freeze_time("2021-01-01T00:00:00Z"):
+        record = logging.LogRecord(
+            name="test_logger",
+            level=logging.INFO,
+            pathname="",
+            lineno=0,
+            msg="test",
+            args=(),
+            exc_info=None,
+        )
+
+        # Test the formatted message
+        formatted = formatter.format(record)
+        assert formatted == expected