Merge branch 'main' into 2089-just-in-time-install

.github/workflows/dependency-review.yml

@@ -25,4 +25,4 @@ jobs:
 
       - name: GitHub dependency vulnerability check
         if: ${{ github.event_name == 'pull_request_target' }}
-        uses: actions/dependency-review-action@V4.3.0
+        uses: actions/dependency-review-action@v4.3.2

.github/workflows/test.yml

@@ -292,7 +292,7 @@ jobs:
         fail_ci_if_error: true
         files: ./coverage.xml
         token: ${{ secrets.CODECOV_TOKEN }}
-      uses: codecov/codecov-action@v4.3.0
+      uses: codecov/codecov-action@v4.3.1
 
   mypy:
     name: "Static type checking"

.pre-commit-config.yaml

@@ -74,7 +74,7 @@ repos:
           - "prettier@3.0.0"
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.2
+    rev: v0.4.3
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix, --show-fixes]

CHANGELOG.md

@@ -3,6 +3,25 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Changelog](http://keepachangelog.com/).
 
+## v3.4.1b1 (2024-05-06)
+
+### 🐛 Fixes
+
+- [#8509](https://github.com/meltano/meltano/issues/8509) Mapper configuration is no longer ignored and is passed to direct mapper invocations as well as mappings in pipelines
+- [#8527](https://github.com/meltano/meltano/issues/8527) Made the output of `meltano select ... list` consistent between different Python versions
+- [#8213](https://github.com/meltano/meltano/issues/8213) `meltano config <plugin> test` false-negative on Windows -- _**Thanks @ReubenFrankel!**_
+- [#8508](https://github.com/meltano/meltano/issues/8508) Cron regex pattern on meltano.schema.json -- _**Thanks @sabino!**_
+
+### ⚙️ Under the Hood
+
+- [#8470](https://github.com/meltano/meltano/issues/8470) Started enforcing usage of `structlog.stdlib.get_logger` over `logging.getLogger` and banned use of the root logger
+- [#8510](https://github.com/meltano/meltano/issues/8510) Plugin installation status is now logged instead of printed
+
+### 📚 Documentation Improvements
+
+- [#8524](https://github.com/meltano/meltano/issues/8524) Changed references of Airflow `orchestrator` to `utility`
+- [#8506](https://github.com/meltano/meltano/issues/8506) Document `venv.backend` usage to configure `uv`
+
 ## v3.4.0 (2024-04-18)
 
 ### ✨ New

docs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "docs.meltano.com",
   "license": "MIT",
-  "version": "3.4.0",
+  "version": "3.4.1",
   "description": "https://meltano.com",
   "private": true,
   "scripts": {

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "meltano"
-version = "3.4.0"
+version = "3.4.1"
 description = "Meltano is your CLI for ELT+: Open Source, Flexible, and Scalable. Move, transform, and test your data with confidence using a streamlined data engineering workflow you’ll love."
 authors = ["Meltano <hello@meltano.com>"]
 license = "MIT"
@@ -47,7 +47,7 @@ azure-common = {version = "^1.1.28", optional = true}
 azure-core = {version = "^1.30.1", optional = true}
 azure-identity = {version = "^1.16.0", optional = true}
 azure-storage-blob = {version = "^12.19.1", optional = true}
-boto3 = {version = "^1.34.93", optional = true}
+boto3 = {version = "^1.34.98", optional = true}
 check-jsonschema = "^0.28.2"
 click = "^8.1"
 click-default-group = "^1.2.4"
@@ -57,8 +57,8 @@ fasteners = "^0.19"
 flatten-dict = "^0"
 google-cloud-storage = {version = ">=1.31.0", optional = true}
 importlib-resources = "^6.4.0"
-jinja2 = "^3.1.3"
-jsonschema = "^4.21"
+jinja2 = "^3.1.4"
+jsonschema = "^4.22"
 packaging = "^24.0"
 platformdirs = "^4.2.1"
 psutil = "^5.9.8"
@@ -79,15 +79,15 @@ rich = "^13.7.1"
 setuptools = {version = "^69.5.1", python = ">=3.12"}
 smart-open = "^7.0.4"
 snowplow-tracker = "^1.0.2"
-sqlalchemy = "^2.0.29"
+sqlalchemy = "^2.0.30"
 structlog = "^24.1.0"
 tabulate = "^0.9.0"
 typing-extensions = "^4.11.0"
 tzlocal = "^5.2"
 # Compatibility issues with boto: https://github.com/boto/botocore/pull/3034
 urllib3 = "<2"
 uv = { version = ">=0.1.24,<0.2", optional = true }
-virtualenv = "^20.26.0"
+virtualenv = "^20.26.1"
 yaspin = "^2.3.0"
 
 [tool.poetry.extras]
@@ -102,12 +102,12 @@ uv = ["uv"]
 [tool.poetry.group.dev.dependencies]
 backoff = "^2.1.2"
 black = "^24.4.2"
-boto3-stubs = {extras = ["essential"], version = "1.34.93"}
+boto3-stubs = {extras = ["essential"], version = "1.34.98"}
 bumpversion = "^0.6.0"
 colorama = "^0.4.4"
 coverage = {extras = ["toml"], version = ">=7.3.2,!=7.3.3"}
 freezegun = "^1.5.0"
-hypothesis = "^6.100.2"
+hypothesis = "^6.100.4"
 mock = "^5.0.2"
 moto = "^5.0.6"
 mypy = "^1.10.0"
@@ -125,7 +125,7 @@ pytest-xdist = "^3.6"
 requests-mock = "^1.12.1"
 setproctitle = "^1.3" # Used by pytest-xdist to aid with handling resource intensive processes.
 types-croniter = "^2.0.0"
-types-jsonschema = "^4.21.0.20240331"
+types-jsonschema = "^4.22.0.20240501"
 types-psutil = "^5.9.5.20240423"
 types-python-dateutil = "^2.9.0.20240316"
 types-python-slugify = "^8.0.2.20240310"
@@ -474,7 +474,7 @@ name = "cz_version_bump"
 prerelease_offset = 1
 tag_format = "v$major.$minor.$patch$prerelease"
 changelog_merge_prerelease = true
-version = "3.4.0"
+version = "3.4.1"
 version_files = [
   "pyproject.toml:^version =",
   "src/meltano/__init__.py:^__version__ =",
@@ -647,5 +647,15 @@ required-imports = ["from __future__ import annotations"]
 convention = "google"
 
 [tool.ruff.lint.flake8-tidy-imports.banned-api]
+"logging.getLogger".msg = "Use `structlog.stdlib.get_logger` instead"
+"logging.critical".msg = "Create a logger with `structlog.stdlib.get_logger`"
+"logging.fatal".msg = "Create a logger with `structlog.stdlib.get_logger`"
+"logging.error".msg = "Create a logger with `structlog.stdlib.get_logger`"
+"logging.exception".msg = "Create a logger with `structlog.stdlib.get_logger`"
+"logging.warning".msg = "Create a logger with `structlog.stdlib.get_logger`"
+"logging.warn".msg = "Create a logger with `structlog.stdlib.get_logger`"
+"logging.info".msg = "Create a logger with `structlog.stdlib.get_logger`"
+"logging.debug".msg = "Create a logger with `structlog.stdlib.get_logger`"
+"logging.log".msg = "Create a logger with `structlog.stdlib.get_logger`"
 "meltano.cli.cli.command".msg = "Use `click.command` and `meltano.cli.cli.add_command` instead"
 "meltano.cli.cli.group".msg = "Use `click.group` and `meltano.cli.cli.add_command` instead"

src/meltano/__init__.py

@@ -3,4 +3,4 @@
 from __future__ import annotations
 
 # Managed by commitizen
-__version__ = "3.4.0"
+__version__ = "3.4.1"

src/meltano/cli/__init__.py

@@ -2,11 +2,12 @@
 
 from __future__ import annotations
 
-import logging
 import os
 import sys
 import typing as t
 
+import structlog
+
 from meltano.cli import (  # noqa: WPS235
     add,
     config,
@@ -71,7 +72,7 @@
 
 setup_logging()
 
-logger = logging.getLogger(__name__)
+logger = structlog.stdlib.get_logger(__name__)
 
 troubleshooting_message = """\
 Need help fixing this problem? Visit http://melta.no/ for troubleshooting steps, or to

src/meltano/cli/cli.py

@@ -2,14 +2,14 @@
 
 from __future__ import annotations
 
-import logging
 import os
 import platform
 import sys
 import typing as t
 from pathlib import Path
 
 import click
+import structlog
 
 from meltano import (
     __version__,
@@ -24,7 +24,7 @@
 from meltano.core.tracking.contexts import CliContext
 from meltano.core.utils import get_no_color_flag
 
-logger = logging.getLogger(__name__)
+logger = structlog.stdlib.get_logger(__name__)
 
 
 class NoWindowsGlobbingGroup(InstrumentedGroup):

src/meltano/cli/initialize.py

@@ -2,10 +2,10 @@
 
 from __future__ import annotations
 
-import logging
 from pathlib import Path
 
 import click
+import structlog
 
 from meltano.cli.params import database_uri_option
 from meltano.cli.utils import InstrumentedCmd
@@ -18,7 +18,7 @@
 LOADERS = "loaders"
 ALL = "all"
 
-logger = logging.getLogger(__name__)
+logger = structlog.stdlib.get_logger(__name__)
 path_type = click.Path(file_okay=False, path_type=Path)
 
 
@@ -52,7 +52,7 @@ def init(ctx, project_directory: Path, no_usage_stats: bool, force: bool):
 
     if ctx.obj["project"]:
         root = ctx.obj["project"].root
-        logging.warning(f"Found meltano project at: {root}")  # noqa: G004
+        logger.warning(f"Found meltano project at: {root}")  # noqa: G004
 
     if no_usage_stats:
         ProjectSettingsService.config_override["send_anonymous_usage_stats"] = False

src/meltano/cli/utils.py

@@ -10,6 +10,7 @@
 from enum import Enum, auto
 
 import click
+import structlog
 from click_default_group import DefaultGroup
 from click_didyoumean import DYMGroup
 
@@ -40,7 +41,7 @@
 
 setup_logging()
 
-logger = logging.getLogger(__name__)
+logger = structlog.stdlib.get_logger(__name__)
 
 
 class CliError(Exception):

src/meltano/core/behavior/hookable.py

@@ -7,11 +7,12 @@
 
 from __future__ import annotations
 
-import logging
 from collections import OrderedDict
 from contextlib import asynccontextmanager
 
-logger = logging.getLogger(__name__)
+import structlog
+
+logger = structlog.stdlib.get_logger(__name__)
 
 
 class hook:  # noqa: N801

src/meltano/core/block/future_utils.py

@@ -3,12 +3,15 @@
 from __future__ import annotations
 
 import asyncio
-import logging
 from asyncio import Task
 
+import structlog
+
 from meltano.core.runner import RunnerError
 from meltano.core.utils import human_size
 
+logger = structlog.stdlib.get_logger(__name__)
+
 
 def all_done(tasks: list[Task], done: set[Task]) -> bool:
     """Iterate through a task list checking if ALL tasks are in the done set."""
@@ -61,17 +64,17 @@ def handle_producer_line_length_limit_error(
     if not isinstance(exception, asyncio.LimitOverrunError):
         return
 
-    logging.error(
+    logger.error(
         "The extractor generated a message exceeding the message size limit "  # noqa: G004
         f"of {human_size(line_length_limit)} (half the buffer size "
         f"of {human_size(stream_buffer_size)}).",
     )
-    logging.error(
+    logger.error(
         "To let this message be processed, increase the 'elt.buffer_size' "
         "setting to at least double the size of the largest expected message, "
         "and try again.",
     )
-    logging.error(
+    logger.error(
         "To learn more, visit "
         "https://docs.meltano.com/reference/settings#eltbuffer_size",
     )

src/meltano/core/config_service.py

@@ -2,12 +2,12 @@
 
 from __future__ import annotations
 
-import logging
 import os
 import typing as t
 from contextlib import contextmanager
 from functools import cached_property
 
+import structlog
 import yaml
 
 from meltano.core import bundle
@@ -17,7 +17,7 @@
     from meltano.core.meltano_file import MeltanoFile
     from meltano.core.project import Project
 
-logger = logging.getLogger(__name__)
+logger = structlog.stdlib.get_logger(__name__)
 
 
 class ConfigService:

src/meltano/core/db.py

@@ -2,11 +2,11 @@
 
 from __future__ import annotations
 
-import logging
 import time
 import typing as t
 from urllib.parse import urlparse
 
+import structlog
 from sqlalchemy import create_engine
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm import sessionmaker
@@ -23,6 +23,7 @@
 # Keep a Project → Engine mapping to serve
 # the same engine for the same Project
 _engines = {}
+logger = structlog.stdlib.get_logger(__name__)
 
 
 class MeltanoDatabaseCompatibilityError(MeltanoError):
@@ -89,7 +90,7 @@ def project_engine(
         )
         + (parsed_db_uri.hostname or ""),
     ).geturl()
-    logging.debug(
+    logger.debug(
         f"Creating DB engine for project at {str(project.root)!r} "  # noqa: G004
         f"with DB URI {sanitized_db_uri!r}",
     )
@@ -142,13 +143,13 @@ def connect(
             return engine.connect()
         except OperationalError:
             if attempt >= max_retries:
-                logging.error(
+                logger.error(
                     f"Could not connect to the database after {attempt} "  # noqa: G004
                     "attempts. Max retries exceeded.",
                 )
                 raise
             attempt += 1
-            logging.info(
+            logger.info(
                 f"DB connection failed. Will retry after {retry_timeout}s. "  # noqa: G004
                 f"Attempt {attempt}/{max_retries}",
             )
@@ -210,9 +211,9 @@ def ensure_schema_exists(
             conn.execute(grant_select_schema)
             conn.execute(grant_usage_schema)
 
-    logging.info(f"Schema {schema_name} has been created successfully.")  # noqa: G004
+    logger.info(f"Schema {schema_name} has been created successfully.")  # noqa: G004
     for role in grant_roles:
-        logging.info(f"Usage has been granted for role: {role}.")  # noqa: G004
+        logger.info(f"Usage has been granted for role: {role}.")  # noqa: G004
 
 
 def check_database_compatibility(engine: Engine) -> None: