[Feature] CLI Parameter for packages-install-path

.changes/unreleased/Features-20240413-093017.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add cli parameter for packages-install-path
+time: 2024-04-13T09:30:17.83824+01:00
+custom:
+  Author: stevenayers
+  Issue: "9932"

core/dbt/cli/flags.py

@@ -28,6 +28,7 @@
 FLAGS_DEFAULTS = {
     "INDIRECT_SELECTION": "eager",
     "TARGET_PATH": None,
+    "PACKAGES_INSTALL_PATH": None,
     "DEFER_STATE": None,  # necessary because of retry construction of flags
     "WARN_ERROR": None,
     # Cli args without project_flags or env var option.

core/dbt/cli/main.py

@@ -183,6 +183,7 @@ def cli(ctx, **kwargs):
 @p.show
 @p.store_failures
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -214,6 +215,7 @@ def build(ctx, **kwargs):
 @p.profiles_dir
 @p.project_dir
 @p.target_path
+@p.packages_install_path
 @p.vars
 @requires.postflight
 @requires.preflight
@@ -251,6 +253,7 @@ def docs(ctx, **kwargs):
 @p.empty_catalog
 @p.static
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -283,6 +286,7 @@ def docs_generate(ctx, **kwargs):
 @p.profiles_dir
 @p.project_dir
 @p.target_path
+@p.packages_install_path
 @p.vars
 @requires.postflight
 @requires.preflight
@@ -319,6 +323,7 @@ def docs_serve(ctx, **kwargs):
 @p.inline
 @p.compile_inject_ephemeral_ctes
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -358,6 +363,7 @@ def compile(ctx, **kwargs):
 @p.selector
 @p.inline
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -417,6 +423,7 @@ def debug(ctx, **kwargs):
 @p.lock
 @p.upgrade
 @p.add_package
+@p.packages_install_path
 @requires.postflight
 @requires.preflight
 @requires.unset_profile
@@ -485,6 +492,7 @@ def init(ctx, **kwargs):
 @p.raw_select
 @p.selector
 @p.target_path
+@p.packages_install_path
 @p.vars
 @requires.postflight
 @requires.preflight
@@ -520,6 +528,7 @@ def list(ctx, **kwargs):
 @p.profiles_dir
 @p.project_dir
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -546,6 +555,7 @@ def parse(ctx, **kwargs):
 @p.select
 @p.selector
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -577,6 +587,7 @@ def run(ctx, **kwargs):
 @p.profiles_dir
 @p.vars
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.full_refresh
 @requires.postflight
@@ -612,6 +623,7 @@ def retry(ctx, **kwargs):
 @p.select
 @p.selector
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.preflight
@@ -644,6 +656,7 @@ def clone(ctx, **kwargs):
 @p.profiles_dir
 @p.project_dir
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -679,6 +692,7 @@ def run_operation(ctx, **kwargs):
 @p.selector
 @p.show
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -711,6 +725,7 @@ def seed(ctx, **kwargs):
 @p.select
 @p.selector
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -753,6 +768,7 @@ def source(ctx, **kwargs):
 @p.select
 @p.selector
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight
@@ -793,6 +809,7 @@ def freshness(ctx, **kwargs):
 @p.selector
 @p.store_failures
 @p.target_path
+@p.packages_install_path
 @p.threads
 @p.vars
 @requires.postflight

core/dbt/cli/params.py

@@ -614,6 +614,13 @@
     type=click.Path(),
 )
 
+packages_install_path = click.option(
+    "--packages-install-path",
+    envvar="DBT_PACKAGES_INSTALL_PATH",
+    help="Configure the 'packages-install-path'. Only applies this setting for the current run. Overrides the 'DBT_PACKAGES_INSTALL_PATH' if it is set.",
+    type=click.Path(),
+)
+
 upgrade = click.option(
     "--upgrade",
     envvar=None,

core/dbt/config/project.py

@@ -430,7 +430,13 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         log_path: str = str(global_flags.LOG_PATH)
 
         clean_targets: List[str] = value_or(cfg.clean_targets, [target_path])
-        packages_install_path: str = value_or(cfg.packages_install_path, "dbt_packages")
+
+        flag_packages_install_path = (
+            str(global_flags.PACKAGES_INSTALL_PATH) if global_flags.PACKAGES_INSTALL_PATH else None
+        )
+        packages_install_path: str = flag_or(
+            flag_packages_install_path, cfg.packages_install_path, "dbt_packages"
+        )
         # in the default case we'll populate this once we know the adapter type
         # It would be nice to just pass along a Quoting here, but that would
         # break many things

core/dbt/flags.py

@@ -84,6 +84,7 @@ def get_flag_dict():
         "cache_selected_only",
         "introspect",
         "target_path",
+        "packages_install_path",
         "log_path",
         "invocation_command",
     }

tests/functional/custom_packages_install_path/test_custom_packages_install_path.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+
+import pytest
+
+from dbt.tests.util import run_dbt
+
+
+class TestPackagesInstallPathConfig:
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return {
+            "packages": [
+                {
+                    "package": "fivetran/fivetran_utils",
+                    "version": "0.4.7",
+                },
+            ]
+        }
+
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {"config-version": 2, "packages-install-path": "project_dbt_packages"}
+
+    def test_packages_install_path(self, project):
+        run_dbt(["deps"])
+        assert Path("project_dbt_packages").is_dir()
+        assert not Path("dbt_packages").is_dir()
+
+
+class TestPackagesInstallPathEnvVar:
+    def test_packages_install_path(self, project, monkeypatch):
+        monkeypatch.setenv("DBT_PACKAGES_INSTALL_PATH", "env_dbt_packages")
+        run_dbt(["deps"])
+        assert Path("env_dbt_packages").is_dir()
+        assert not Path("project_dbt_packages").is_dir()
+        assert not Path("dbt_packages").is_dir()
+
+
+class TestPackagesInstallPathCliArg:
+    def test_packages_install_path(self, project, monkeypatch):
+        monkeypatch.setenv("DBT_PACKAGES_INSTALL_PATH", "env_dbt_packages")
+        run_dbt(["deps", "--packages-install-path", "cli_dbt_packages"])
+        assert Path("cli_dbt_packages").is_dir()
+        assert not Path("env_dbt_packages").is_dir()
+        assert not Path("project_dbt_packages").is_dir()
+        assert not Path("dbt_packages").is_dir()