[Backport] deprecate materialization overrides from imported packages (…

…#9998)

.changes/unreleased/Features-20231218-195854.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Move flags from UserConfig in profiles.yml to flags in dbt_project.yml
+time: 2023-12-18T19:58:54.075811-05:00
+custom:
+  Author: gshank
+  Issue: "9183"

.changes/unreleased/Features-20240422-173703.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add require_explicit_package_overrides_for_builtin_materializations to dbt_project.yml flags, which can be used to opt-out of overriding built-in materializations from packages
+time: 2024-04-22T17:37:03.892268-04:00
+custom:
+  Author: michelleark
+  Issue: "10007"

.changes/unreleased/Under the Hood-20240418-172528.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Raise deprecation warning if installed package overrides built-in materialization
+time: 2024-04-18T17:25:28.37886-04:00
+custom:
+  Author: michelleark
+  Issue: "9971"

core/dbt/cli/flags.py

@@ -3,6 +3,7 @@
 from dataclasses import dataclass
 from importlib import import_module
 from multiprocessing import get_context
+from pathlib import Path
 from pprint import pformat as pf
 from typing import Any, Callable, Dict, List, Optional, Set, Union
 
@@ -11,8 +12,8 @@
 from dbt.cli.exceptions import DbtUsageException
 from dbt.cli.resolvers import default_log_path, default_project_dir
 from dbt.cli.types import Command as CliCommand
-from dbt.config.profile import read_user_config
-from dbt.contracts.project import UserConfig
+from dbt.config.project import read_project_flags
+from dbt.contracts.project import ProjectFlags
 from dbt.exceptions import DbtInternalError
 from dbt.deprecations import renamed_env_var
 from dbt.helper_types import WarnErrorOptions
@@ -25,7 +26,7 @@
     "INDIRECT_SELECTION": "eager",
     "TARGET_PATH": None,
     "WARN_ERROR": None,
-    # Cli args without user_config or env var option.
+    # Cli args without project_flags or env var option.
     "FULL_REFRESH": False,
     "STRICT_MODE": False,
     "STORE_FAILURES": False,
@@ -77,7 +78,7 @@ class Flags:
     """Primary configuration artifact for running dbt"""
 
     def __init__(
-        self, ctx: Optional[Context] = None, user_config: Optional[UserConfig] = None
+        self, ctx: Optional[Context] = None, project_flags: Optional[ProjectFlags] = None
     ) -> None:
         # Set the default flags.
         for key, value in FLAGS_DEFAULTS.items():
@@ -200,27 +201,40 @@ def _assign_params(
                 invoked_subcommand_ctx, params_assigned_from_default, deprecated_env_vars
             )
 
-        if not user_config:
+        if not project_flags:
+            project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir()))
             profiles_dir = getattr(self, "PROFILES_DIR", None)
-            user_config = read_user_config(profiles_dir) if profiles_dir else None
+            if profiles_dir and project_dir:
+                project_flags = read_project_flags(project_dir, profiles_dir)
+            else:
+                project_flags = None
 
         # Add entire invocation command to flags
         object.__setattr__(self, "INVOCATION_COMMAND", "dbt " + " ".join(sys.argv[1:]))
 
-        # Overwrite default assignments with user config if available.
-        if user_config:
+        if project_flags:
+            # Overwrite default assignments with project flags if available.
             param_assigned_from_default_copy = params_assigned_from_default.copy()
             for param_assigned_from_default in params_assigned_from_default:
-                user_config_param_value = getattr(user_config, param_assigned_from_default, None)
-                if user_config_param_value is not None:
+                project_flags_param_value = getattr(
+                    project_flags, param_assigned_from_default, None
+                )
+                if project_flags_param_value is not None:
                     object.__setattr__(
                         self,
                         param_assigned_from_default.upper(),
-                        convert_config(param_assigned_from_default, user_config_param_value),
+                        convert_config(param_assigned_from_default, project_flags_param_value),
                     )
                     param_assigned_from_default_copy.remove(param_assigned_from_default)
             params_assigned_from_default = param_assigned_from_default_copy
 
+            # Add project-level flags that are not available as CLI options / env vars
+            for (
+                project_level_flag_name,
+                project_level_flag_value,
+            ) in project_flags.project_only_flags.items():
+                object.__setattr__(self, project_level_flag_name.upper(), project_level_flag_value)
+
         # Set hard coded flags.
         object.__setattr__(self, "WHICH", invoked_subcommand_name or ctx.info_name)
         object.__setattr__(self, "MP_CONTEXT", get_context("spawn"))
@@ -234,9 +248,11 @@ def _assign_params(
         # Starting in v1.5, if `log-path` is set in `dbt_project.yml`, it will raise a deprecation warning,
         # with the possibility of removing it in a future release.
         if getattr(self, "LOG_PATH", None) is None:
-            project_dir = getattr(self, "PROJECT_DIR", default_project_dir())
+            project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir()))
             version_check = getattr(self, "VERSION_CHECK", True)
-            object.__setattr__(self, "LOG_PATH", default_log_path(project_dir, version_check))
+            object.__setattr__(
+                self, "LOG_PATH", default_log_path(Path(project_dir), version_check)
+            )
 
         # Support console DO NOT TRACK initiative.
         if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes"):

core/dbt/config/__init__.py

@@ -1,4 +1,4 @@
 # all these are just exports, they need "noqa" so flake8 will not complain.
-from .profile import Profile, read_user_config  # noqa
+from .profile import Profile  # noqa
 from .project import Project, IsFQNResource, PartialProject  # noqa
 from .runtime import RuntimeConfig  # noqa

core/dbt/config/profile.py

@@ -8,7 +8,7 @@
 from dbt.clients.system import load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.contracts.connection import Credentials, HasCredentials
-from dbt.contracts.project import ProfileConfig, UserConfig
+from dbt.contracts.project import ProfileConfig
 from dbt.exceptions import (
     CompilationError,
     DbtProfileError,
@@ -19,7 +19,6 @@
 )
 from dbt.events.types import MissingProfileTarget
 from dbt.events.functions import fire_event
-from dbt.utils import coerce_dict_str
 
 from .renderer import ProfileRenderer
 
@@ -51,27 +50,13 @@ def read_profile(profiles_dir: str) -> Dict[str, Any]:
     return {}
 
 
-def read_user_config(directory: str) -> UserConfig:
-    try:
-        profile = read_profile(directory)
-        if profile:
-            user_config = coerce_dict_str(profile.get("config", {}))
-            if user_config is not None:
-                UserConfig.validate(user_config)
-                return UserConfig.from_dict(user_config)
-    except (DbtRuntimeError, ValidationError):
-        pass
-    return UserConfig()
-
-
 # The Profile class is included in RuntimeConfig, so any attribute
 # additions must also be set where the RuntimeConfig class is created
 # `init=False` is a workaround for https://bugs.python.org/issue45081
 @dataclass(init=False)
 class Profile(HasCredentials):
     profile_name: str
     target_name: str
-    user_config: UserConfig
     threads: int
     credentials: Credentials
     profile_env_vars: Dict[str, Any]
@@ -80,7 +65,6 @@ def __init__(
         self,
         profile_name: str,
         target_name: str,
-        user_config: UserConfig,
         threads: int,
         credentials: Credentials,
     ) -> None:
@@ -89,7 +73,6 @@ def __init__(
         """
         self.profile_name = profile_name
         self.target_name = target_name
-        self.user_config = user_config
         self.threads = threads
         self.credentials = credentials
         self.profile_env_vars = {}  # never available on init
@@ -106,12 +89,10 @@ def to_profile_info(self, serialize_credentials: bool = False) -> Dict[str, Any]
         result = {
             "profile_name": self.profile_name,
             "target_name": self.target_name,
-            "user_config": self.user_config,
             "threads": self.threads,
             "credentials": self.credentials,
         }
         if serialize_credentials:
-            result["user_config"] = self.user_config.to_dict(omit_none=True)
             result["credentials"] = self.credentials.to_dict(omit_none=True)
         return result
 
@@ -124,7 +105,6 @@ def to_target_dict(self) -> Dict[str, Any]:
                 "name": self.target_name,
                 "target_name": self.target_name,
                 "profile_name": self.profile_name,
-                "config": self.user_config.to_dict(omit_none=True),
             }
         )
         return target
@@ -246,7 +226,6 @@ def from_credentials(
         threads: int,
         profile_name: str,
         target_name: str,
-        user_config: Optional[Dict[str, Any]] = None,
     ) -> "Profile":
         """Create a profile from an existing set of Credentials and the
         remaining information.
@@ -255,20 +234,13 @@ def from_credentials(
         :param threads: The number of threads to use for connections.
         :param profile_name: The profile name used for this profile.
         :param target_name: The target name used for this profile.
-        :param user_config: The user-level config block from the
-            raw profiles, if specified.
         :raises DbtProfileError: If the profile is invalid.
         :returns: The new Profile object.
         """
-        if user_config is None:
-            user_config = {}
-        UserConfig.validate(user_config)
-        user_config_obj: UserConfig = UserConfig.from_dict(user_config)
 
         profile = cls(
             profile_name=profile_name,
             target_name=target_name,
-            user_config=user_config_obj,
             threads=threads,
             credentials=credentials,
         )
@@ -316,7 +288,6 @@ def from_raw_profile_info(
         raw_profile: Dict[str, Any],
         profile_name: str,
         renderer: ProfileRenderer,
-        user_config: Optional[Dict[str, Any]] = None,
         target_override: Optional[str] = None,
         threads_override: Optional[int] = None,
     ) -> "Profile":
@@ -328,8 +299,6 @@ def from_raw_profile_info(
             disk as yaml and its values rendered with jinja.
         :param profile_name: The profile name used.
         :param renderer: The config renderer.
-        :param user_config: The global config for the user, if it
-            was present.
         :param target_override: The target to use, if provided on
             the command line.
         :param threads_override: The thread count to use, if
@@ -338,9 +307,6 @@ def from_raw_profile_info(
             target could not be found
         :returns: The new Profile object.
         """
-        # user_config is not rendered.
-        if user_config is None:
-            user_config = raw_profile.get("config")
         # TODO: should it be, and the values coerced to bool?
         target_name, profile_data = cls.render_profile(
             raw_profile, profile_name, target_override, renderer
@@ -361,7 +327,6 @@ def from_raw_profile_info(
             profile_name=profile_name,
             target_name=target_name,
             threads=threads,
-            user_config=user_config,
         )
 
     @classmethod
@@ -396,13 +361,11 @@ def from_raw_profiles(
         if not raw_profile:
             msg = f"Profile {profile_name} in profiles.yml is empty"
             raise DbtProfileError(INVALID_PROFILE_MESSAGE.format(error_string=msg))
-        user_config = raw_profiles.get("config")
 
         return cls.from_raw_profile_info(
             raw_profile=raw_profile,
             profile_name=profile_name,
             renderer=renderer,
-            user_config=user_config,
             target_override=target_override,
             threads_override=threads_override,
         )