Add --backend=custom option for external databases in Breeze

Dockerfile.ci

@@ -1146,6 +1146,8 @@ function environment_initialization() {
         echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} MySQL: ${MYSQL_VERSION}"
     elif [[ ${BACKEND=} == "sqlite" ]]; then
         echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Sqlite"
+    elif [[ ${BACKEND=} == "custom" ]]; then
+        echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Custom"
     fi
     echo
 

dev/breeze/src/airflow_breeze/commands/common_options.py

@@ -116,9 +116,21 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
     help="Database backend to use. Default is 'sqlite'. "
     "If 'none' is chosen, Breeze will start with an invalid database configuration — "
     "no database will be available, and any attempt to run Airflow will fail. "
-    "Use 'none' only for specific non-DB test cases.",
+    "Use 'none' only for specific non-DB test cases. "
+    "If 'custom' is chosen, no database container will be started and you must provide "
+    "your own database connection via AIRFLOW__DATABASE__SQL_ALCHEMY_CONN environment variable. "
+    "Only officially supported backends (postgres, mysql, sqlite) are tested.",
     envvar="BACKEND",
 )
+option_custom_db_url = click.option(
+    "--custom-db-url",
+    type=str,
+    default=None,
+    help="SQLAlchemy connection URL for the custom database backend. "
+    "Only used when --backend=custom is selected. "
+    "Falls back to the AIRFLOW__DATABASE__SQL_ALCHEMY_CONN environment variable if not provided.",
+    envvar="AIRFLOW__DATABASE__SQL_ALCHEMY_CONN",
+)
 option_builder = click.option(
     "--builder",
     help="Buildx builder used to perform `docker buildx build` commands.",

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -41,6 +41,7 @@
     option_backend,
     option_builder,
     option_clean_airflow_installation,
+    option_custom_db_url,
     option_db_reset,
     option_debug_components,
     option_debugger,
@@ -305,6 +306,7 @@ def run(self):
 @option_answer
 @option_auth_manager
 @option_backend
+@option_custom_db_url
 @option_builder
 @option_celery_broker
 @option_celery_flower
@@ -368,6 +370,7 @@ def shell(
     celery_broker: str,
     celery_flower: bool,
     clean_airflow_installation: bool,
+    custom_db_url: str | None,
     db_reset: bool,
     downgrade_sqlalchemy: bool,
     downgrade_pendulum: bool,
@@ -444,6 +447,7 @@ def shell(
         celery_broker=celery_broker,
         celery_flower=celery_flower,
         clean_airflow_installation=clean_airflow_installation,
+        custom_db_url=custom_db_url or "",
         db_reset=db_reset,
         downgrade_sqlalchemy=downgrade_sqlalchemy,
         downgrade_pendulum=downgrade_pendulum,
@@ -538,6 +542,7 @@ def shell(
 @option_auth_manager
 @option_answer
 @option_backend
+@option_custom_db_url
 @option_builder
 @option_clean_airflow_installation
 @option_celery_broker
@@ -590,6 +595,7 @@ def start_airflow(
     celery_broker: str,
     celery_flower: bool,
     clean_airflow_installation: bool,
+    custom_db_url: str | None,
     db_reset: bool,
     debug_components: tuple[str, ...],
     debugger: str,
@@ -683,6 +689,7 @@ def start_airflow(
         celery_broker=celery_broker,
         celery_flower=celery_flower,
         clean_airflow_installation=clean_airflow_installation,
+        custom_db_url=custom_db_url or "",
         debug_components=debug_components,
         debugger=debugger,
         db_reset=db_reset,
@@ -1063,6 +1070,7 @@ def doctor(ctx):
 @click.argument("command_args", nargs=-1, type=click.UNPROCESSED)
 @option_answer
 @option_backend
+@option_custom_db_url
 @option_builder
 @option_docker_host
 @option_dry_run
@@ -1085,6 +1093,7 @@ def run(
     command_args: tuple,
     backend: str,
     builder: str,
+    custom_db_url: str | None,
     docker_host: str | None,
     force_build: bool,
     forward_credentials: bool,
@@ -1149,6 +1158,7 @@ def run(
     shell_params = ShellParams(
         backend=backend,
         builder=builder,
+        custom_db_url=custom_db_url or "",
         docker_host=docker_host,
         force_build=force_build,
         forward_credentials=forward_credentials,

dev/breeze/src/airflow_breeze/commands/developer_commands_config.py

@@ -53,6 +53,7 @@
             "name": "Database",
             "options": [
                 "--backend",
+                "--custom-db-url",
                 "--postgres-version",
                 "--mysql-version",
                 "--db-reset",
@@ -110,6 +111,7 @@
             "name": "Database",
             "options": [
                 "--backend",
+                "--custom-db-url",
                 "--postgres-version",
                 "--mysql-version",
                 "--db-reset",
@@ -219,6 +221,7 @@
             "name": "Database",
             "options": [
                 "--backend",
+                "--custom-db-url",
                 "--postgres-version",
                 "--mysql-version",
                 "--db-reset",
@@ -294,6 +297,7 @@
             "options": [
                 "--python",
                 "--backend",
+                "--custom-db-url",
                 "--postgres-version",
                 "--mysql-version",
                 "--forward-ports",

dev/breeze/src/airflow_breeze/commands/main_command.py

@@ -28,6 +28,7 @@
     option_auth_manager,
     option_backend,
     option_builder,
+    option_custom_db_url,
     option_db_reset,
     option_docker_host,
     option_dry_run,
@@ -102,6 +103,7 @@ def get_command(self, ctx: Context, cmd_name: str):
 @option_answer
 @option_auth_manager
 @option_backend
+@option_custom_db_url
 @option_builder
 @option_db_reset
 @option_docker_host

dev/breeze/src/airflow_breeze/commands/testing_commands.py

@@ -36,6 +36,7 @@
     option_backend,
     option_browser,
     option_clean_airflow_installation,
+    option_custom_db_url,
     option_core_integration,
     option_db_reset,
     option_debug_e2e,
@@ -623,6 +624,7 @@ def _verify_parallelism_parameters(
 )
 @option_airflow_constraints_reference
 @option_backend
+@option_custom_db_url
 @option_collect_only
 @option_clean_airflow_installation
 @option_db_reset
@@ -685,6 +687,7 @@ def core_tests(**kwargs):
 )
 @option_airflow_constraints_reference
 @option_backend
+@option_custom_db_url
 @option_collect_only
 @option_clean_airflow_installation
 @option_db_reset
@@ -974,6 +977,7 @@ def airflow_ctl_tests(python: str, parallelism: int, extra_pytest_args: tuple):
     ),
 )
 @option_backend
+@option_custom_db_url
 @option_collect_only
 @option_db_reset
 @option_dry_run
@@ -995,6 +999,7 @@ def airflow_ctl_tests(python: str, parallelism: int, extra_pytest_args: tuple):
 def core_integration_tests(
     backend: str,
     collect_only: bool,
+    custom_db_url: str | None,
     db_reset: bool,
     enable_coverage: bool,
     extra_pytest_args: tuple,
@@ -1015,6 +1020,7 @@ def core_integration_tests(
         test_group=GroupOfTests.INTEGRATION_CORE,
         backend=backend,
         collect_only=collect_only,
+        custom_db_url=custom_db_url or "",
         enable_coverage=enable_coverage,
         forward_credentials=forward_credentials,
         forward_ports=False,
@@ -1055,6 +1061,7 @@ def core_integration_tests(
     ),
 )
 @option_backend
+@option_custom_db_url
 @option_collect_only
 @option_db_reset
 @option_dry_run
@@ -1076,6 +1083,7 @@ def core_integration_tests(
 def integration_providers_tests(
     backend: str,
     collect_only: bool,
+    custom_db_url: str | None,
     db_reset: bool,
     enable_coverage: bool,
     extra_pytest_args: tuple,
@@ -1096,6 +1104,7 @@ def integration_providers_tests(
         test_group=GroupOfTests.INTEGRATION_PROVIDERS,
         backend=backend,
         collect_only=collect_only,
+        custom_db_url=custom_db_url or "",
         enable_coverage=enable_coverage,
         forward_credentials=forward_credentials,
         forward_ports=False,
@@ -1136,6 +1145,7 @@ def integration_providers_tests(
     ),
 )
 @option_backend
+@option_custom_db_url
 @option_collect_only
 @option_db_reset
 @option_dry_run
@@ -1164,6 +1174,7 @@ def integration_providers_tests(
 def system_tests(
     backend: str,
     collect_only: bool,
+    custom_db_url: str | None,
     db_reset: bool,
     enable_coverage: bool,
     extra_pytest_args: tuple,
@@ -1191,6 +1202,7 @@ def system_tests(
         test_group=GroupOfTests.SYSTEM,
         backend=backend,
         collect_only=collect_only,
+        custom_db_url=custom_db_url or "",
         enable_coverage=enable_coverage,
         forward_credentials=forward_credentials,
         forward_ports=True,
@@ -1297,6 +1309,7 @@ def helm_tests(
     ),
 )
 @option_backend
+@option_custom_db_url
 @option_collect_only
 @option_db_reset
 @option_no_db_cleanup
@@ -1316,6 +1329,7 @@ def helm_tests(
 def python_api_client_tests(
     backend: str,
     collect_only: bool,
+    custom_db_url: str | None,
     db_reset: bool,
     no_db_cleanup: bool,
     enable_coverage: bool,
@@ -1334,6 +1348,7 @@ def python_api_client_tests(
         test_group=GroupOfTests.PYTHON_API_CLIENT,
         backend=backend,
         collect_only=collect_only,
+        custom_db_url=custom_db_url or "",
         enable_coverage=enable_coverage,
         forward_credentials=forward_credentials,
         forward_ports=False,
@@ -1700,6 +1715,7 @@ def _run_test_command(
     allow_pre_releases: bool,
     backend: str,
     collect_only: bool,
+    custom_db_url: str | None = None,
     clean_airflow_installation: bool,
     db_reset: bool,
     debug_resources: bool,
@@ -1755,6 +1771,7 @@ def _run_test_command(
         backend=backend,
         collect_only=collect_only,
         clean_airflow_installation=clean_airflow_installation,
+        custom_db_url=custom_db_url or "",
         downgrade_sqlalchemy=downgrade_sqlalchemy,
         downgrade_pendulum=downgrade_pendulum,
         enable_coverage=enable_coverage,

dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

@@ -39,6 +39,7 @@
     "name": "Test environment",
     "options": [
         "--backend",
+        "--custom-db-url",
         "--no-db-cleanup",
         "--python",
         "--postgres-version",

dev/breeze/src/airflow_breeze/global_constants.py

@@ -62,11 +62,13 @@
 ALLOWED_ARCHITECTURES = [Architecture.X86_64, Architecture.ARM]
 # Database Backends used when starting Breeze. The "none" value means that the configuration is invalid.
 # No database will be started - access to a database will fail.
+# The "custom" value allows providing an external database via AIRFLOW__DATABASE__SQL_ALCHEMY_CONN.
 SQLITE_BACKEND = "sqlite"
 MYSQL_BACKEND = "mysql"
 POSTGRES_BACKEND = "postgres"
 NONE_BACKEND = "none"
-ALLOWED_BACKENDS = [SQLITE_BACKEND, MYSQL_BACKEND, POSTGRES_BACKEND, NONE_BACKEND]
+CUSTOM_BACKEND = "custom"
+ALLOWED_BACKENDS = [SQLITE_BACKEND, MYSQL_BACKEND, POSTGRES_BACKEND, NONE_BACKEND, CUSTOM_BACKEND]
 ALLOWED_PROD_BACKENDS = [MYSQL_BACKEND, POSTGRES_BACKEND]
 DEFAULT_BACKEND = ALLOWED_BACKENDS[0]
 TESTABLE_CORE_INTEGRATIONS = ["kerberos", "redis"]

dev/breeze/src/airflow_breeze/params/shell_params.py

@@ -168,6 +168,7 @@ class ShellParams:
     auth_manager: str = ALLOWED_AUTH_MANAGERS[0]
     backend: str = ALLOWED_BACKENDS[0]
     base_branch: str = "main"
+    custom_db_url: str = ""
     builder: str = "autodetect"
     celery_broker: str = DEFAULT_CELERY_BROKER
     celery_flower: bool = False
@@ -348,7 +349,7 @@ def get_backend_compose_files(self, backend: str) -> list[Path]:
             backend_docker_compose_file = SCRIPTS_CI_DOCKER_COMPOSE_PATH / f"backend-{backend}-no-volume.yml"
         else:
             backend_docker_compose_file = SCRIPTS_CI_DOCKER_COMPOSE_PATH / f"backend-{backend}.yml"
-        if backend in ("sqlite", "none") or not self.forward_ports:
+        if backend in ("sqlite", "none", "custom") or not self.forward_ports:
             return [backend_docker_compose_file]
         if self.project_name == "prek":
             # do not forward ports for prek - to not clash with running containers from breeze
@@ -604,6 +605,8 @@ def env_variables_for_docker_commands(self) -> dict[str, str]:
         _set_var(_env, "ANSWER", get_forced_answer() or "")
         _set_var(_env, "ALLOW_PRE_RELEASES", self.allow_pre_releases)
         _set_var(_env, "BACKEND", self.backend)
+        if self.backend == "custom":
+            _set_var(_env, "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN", self.custom_db_url or None)
         _set_var(_env, "BASE_BRANCH", self.base_branch, "main")
         _set_var(_env, "BREEZE", "true")
         _set_var(_env, "BREEZE_INIT_COMMAND", None, "")

dev/breeze/tests/test_cache.py

@@ -34,8 +34,8 @@
 @pytest.mark.parametrize(
     ("parameter", "value", "result", "exception"),
     [
-        ("backend", "mysql", (True, ["sqlite", "mysql", "postgres", "none"]), None),
-        ("backend", "xxx", (False, ["sqlite", "mysql", "postgres", "none"]), None),
+        ("backend", "mysql", (True, ["sqlite", "mysql", "postgres", "none", "custom"]), None),
+        ("backend", "xxx", (False, ["sqlite", "mysql", "postgres", "none", "custom"]), None),
         ("python_major_minor_version", "3.10", (True, ["3.10", "3.11", "3.12", "3.13"]), None),
         ("missing", "value", None, AttributeError),
     ],

scripts/ci/docker-compose/backend-custom.yml

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+services:
+  airflow:
+    environment:
+      - BACKEND=custom
+      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN

scripts/docker/entrypoint_ci.sh

@@ -121,6 +121,9 @@ function environment_initialization() {
         echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} MySQL: ${MYSQL_VERSION}"
     elif [[ ${BACKEND=} == "sqlite" ]]; then
         echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Sqlite"
+    elif [[ ${BACKEND=} == "custom" ]]; then
+        MASKED_URL=$(echo "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:=}" | sed -E 's|://([^:]+):([^@]+)@|://\1:***@|')
+        echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Custom (${MASKED_URL})"
     fi
     echo