diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0167cfb1a..251f933255c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ Contributors: - Wait for postgres docker container to be ready in `setup_db.sh`. ([#3876](https://github.com/dbt-labs/dbt-core/issues/3876), [#3908](https://github.com/dbt-labs/dbt-core/pull/3908)) - Prefer macros defined in the project over the ones in a package by default ([#4106](https://github.com/dbt-labs/dbt-core/issues/4106), [#4114](https://github.com/dbt-labs/dbt-core/pull/4114)) - Dependency updates ([#4079](https://github.com/dbt-labs/dbt-core/pull/4079)), ([#3532](https://github.com/dbt-labs/dbt-core/pull/3532) +- Schedule partial parsing for SQL files with env_var changes ([#3885](https://github.com/dbt-labs/dbt-core/issues/3885), [#4101](https://github.com/dbt-labs/dbt-core/pull/4101)) Contributors: - [@sungchun12](https://github.com/sungchun12) ([#4017](https://github.com/dbt-labs/dbt/pull/4017)) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 254d66f4d27..732b2e59914 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -968,10 +968,10 @@ def execute_macro( 'dbt could not find a macro with the name "{}" in {}' .format(macro_name, package_name) ) - # This causes a reference cycle, as generate_runtime_macro() + # This causes a reference cycle, as generate_runtime_macro_context() # ends up calling get_adapter, so the import has to be here. - from dbt.context.providers import generate_runtime_macro - macro_context = generate_runtime_macro( + from dbt.context.providers import generate_runtime_macro_context + macro_context = generate_runtime_macro_context( macro=macro, config=self.config, manifest=manifest, diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 2618d542558..cc6d0ea8d08 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -9,7 +9,7 @@ from dbt.adapters.factory import get_adapter from dbt.clients import jinja from dbt.clients.system import make_directory -from dbt.context.providers import generate_runtime_model +from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.manifest import Manifest, UniqueID from dbt.contracts.graph.compiled import ( COMPILED_TYPES, @@ -178,7 +178,7 @@ def _create_node_context( extra_context: Dict[str, Any], ) -> Dict[str, Any]: - context = generate_runtime_model( + context = generate_runtime_model_context( node, self.config, manifest ) context.update(extra_context) diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index f8f5d895233..bd86d7df084 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -253,7 +253,7 @@ def _get_v2_config_paths( ) -> PathSet: for key, value in config.items(): if isinstance(value, dict) and not key.startswith('+'): - self._get_v2_config_paths(value, path + (key,), paths) + self._get_config_paths(value, path + (key,), paths) else: paths.add(path) return frozenset(paths) diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index 6360a5e7f2e..2d1dbca1c88 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -160,9 +160,12 @@ def __call__(self, var_name, default=_VAR_NOTSET): class BaseContext(metaclass=ContextMeta): + # subclass is TargetContext def __init__(self, cli_vars): self._ctx = {} self.cli_vars = cli_vars + # Save the env_vars encountered using this + self.env_vars = {} def generate_builtins(self): builtins: Dict[str, Any] = {} @@ -271,17 +274,21 @@ def var(self) -> Var: return Var(self._ctx, self.cli_vars) @contextmember - @staticmethod - def env_var(var: str, default: Optional[str] = None) -> str: + def env_var(self, var: str, default: Optional[str] = None) -> str: """The env_var() function. Return the environment variable named 'var'. If there is no such environment variable set, return the default. If the default is None, raise an exception for an undefined variable. """ + return_value = None if var in os.environ: - return os.environ[var] + return_value = os.environ[var] elif default is not None: - return default + return_value = default + + if return_value is not None: + self.env_vars[var] = return_value + return return_value else: msg = f"Env var required but not provided: '{var}'" undefined_error(msg) diff --git a/core/dbt/context/configured.py b/core/dbt/context/configured.py index 1de3e761296..21095f786c9 100644 --- a/core/dbt/context/configured.py +++ b/core/dbt/context/configured.py @@ -9,6 +9,7 @@ class ConfiguredContext(TargetContext): + # subclasses are SchemaYamlContext, MacroResolvingContext, ManifestContext config: AdapterRequiredConfig def __init__( @@ -64,6 +65,7 @@ def __call__(self, var_name, default=Var._VAR_NOTSET): class SchemaYamlContext(ConfiguredContext): + # subclass is DocsRuntimeContext def __init__(self, config, project_name: str): super().__init__(config) self._project_name = project_name @@ -86,7 +88,7 @@ def var(self) -> ConfiguredVar: ) -def generate_schema_yml( +def generate_schema_yml_context( config: AdapterRequiredConfig, project_name: str ) -> Dict[str, Any]: ctx = SchemaYamlContext(config, project_name) diff --git a/core/dbt/context/docs.py b/core/dbt/context/docs.py index f36ddaf7d09..5d6092f7a05 100644 --- a/core/dbt/context/docs.py +++ b/core/dbt/context/docs.py @@ -75,7 +75,7 @@ def doc(self, *args: str) -> str: return target_doc.block_contents -def generate_runtime_docs( +def generate_runtime_docs_context( config: RuntimeConfig, target: Any, manifest: Manifest, diff --git a/core/dbt/context/manifest.py b/core/dbt/context/manifest.py index fae47c12166..c07a9cb9ed2 100644 --- a/core/dbt/context/manifest.py +++ b/core/dbt/context/manifest.py @@ -17,6 +17,7 @@ class ManifestContext(ConfiguredContext): The given macros can override any previous context values, which will be available as if they were accessed relative to the package name. """ + # subclasses are QueryHeaderContext and ProviderContext def __init__( self, config: AdapterRequiredConfig, diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 74334dad36f..a117f2aefb5 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -11,7 +11,7 @@ get_adapter, get_adapter_package_names, get_adapter_type_names ) from dbt.clients import agate_helper -from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack +from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack, undefined_error from dbt.config import RuntimeConfig, Project from .base import contextmember, contextproperty, Var from .configured import FQNLookup @@ -49,7 +49,7 @@ wrapped_exports, ) from dbt.config import IsFQNResource -from dbt.logger import GLOBAL_LOGGER as logger # noqa +from dbt.logger import GLOBAL_LOGGER as logger, SECRET_ENV_PREFIX # noqa from dbt.node_types import NodeType from dbt.utils import ( @@ -636,6 +636,7 @@ class OperationProvider(RuntimeProvider): # Base context collection, used for parsing configs. class ProviderContext(ManifestContext): + # subclasses are MacroContext, ModelContext, TestContext def __init__( self, model, @@ -1161,6 +1162,30 @@ def adapter_macro(self, name: str, *args, **kwargs): ) raise CompilationException(msg) + @contextmember + def env_var(self, var: str, default: Optional[str] = None) -> str: + """The env_var() function. Return the environment variable named 'var'. + If there is no such environment variable set, return the default. + + If the default is None, raise an exception for an undefined variable. + """ + return_value = None + if var in os.environ: + return_value = os.environ[var] + elif default is not None: + return_value = default + + if return_value is not None: + # Save the env_var value in the manifest and the var name in the source_file + if not var.startswith(SECRET_ENV_PREFIX) and self.model: + self.manifest.env_vars[var] = return_value + source_file = self.manifest.files[self.model.file_id] + source_file.env_vars.append(var) + return return_value + else: + msg = f"Env var required but not provided: '{var}'" + undefined_error(msg) + class MacroContext(ProviderContext): """Internally, macros can be executed like nodes, with some restrictions: @@ -1262,7 +1287,7 @@ def this(self) -> Optional[RelationProxy]: # This is called by '_context_for', used in 'render_with_context' -def generate_parser_model( +def generate_parser_model_context( model: ManifestNode, config: RuntimeConfig, manifest: Manifest, @@ -1279,7 +1304,7 @@ def generate_parser_model( return ctx.to_dict() -def generate_generate_component_name_macro( +def generate_generate_name_macro_context( macro: ParsedMacro, config: RuntimeConfig, manifest: Manifest, @@ -1290,7 +1315,7 @@ def generate_generate_component_name_macro( return ctx.to_dict() -def generate_runtime_model( +def generate_runtime_model_context( model: ManifestNode, config: RuntimeConfig, manifest: Manifest, @@ -1301,7 +1326,7 @@ def generate_runtime_model( return ctx.to_dict() -def generate_runtime_macro( +def generate_runtime_macro_context( macro: ParsedMacro, config: RuntimeConfig, manifest: Manifest, diff --git a/core/dbt/context/target.py b/core/dbt/context/target.py index c875bc6e9be..1a311d7c092 100644 --- a/core/dbt/context/target.py +++ b/core/dbt/context/target.py @@ -8,6 +8,7 @@ class TargetContext(BaseContext): + # subclass is ConfiguredContext def __init__(self, config: HasCredentials, cli_vars: Dict[str, Any]): super().__init__(cli_vars=cli_vars) self.config = config diff --git a/core/dbt/contracts/files.py b/core/dbt/contracts/files.py index 125485bba28..64134c4cecf 100644 --- a/core/dbt/contracts/files.py +++ b/core/dbt/contracts/files.py @@ -190,6 +190,7 @@ class SourceFile(BaseSourceFile): nodes: List[str] = field(default_factory=list) docs: List[str] = field(default_factory=list) macros: List[str] = field(default_factory=list) + env_vars: List[str] = field(default_factory=list) @classmethod def big_seed(cls, path: FilePath) -> 'SourceFile': @@ -230,6 +231,7 @@ class SchemaSourceFile(BaseSourceFile): # Patches are only against external sources. Sources can be # created too, but those are in 'sources' sop: List[SourceKey] = field(default_factory=list) + env_vars: Dict[str, Any] = field(default_factory=dict) pp_dict: Optional[Dict[str, Any]] = None pp_test_index: Optional[Dict[str, Any]] = None @@ -252,7 +254,7 @@ def source_patches(self): def __post_serialize__(self, dct): dct = super().__post_serialize__(dct) # Remove partial parsing specific data - for key in ('pp_files', 'pp_test_index', 'pp_dict'): + for key in ('pp_test_index', 'pp_dict'): if key in dct: del dct[key] return dct diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 04c7876b543..72300e9b525 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -569,6 +569,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin): state_check: ManifestStateCheck = field(default_factory=ManifestStateCheck) source_patches: MutableMapping[SourceKey, SourcePatch] = field(default_factory=dict) disabled: MutableMapping[str, List[CompileResultNode]] = field(default_factory=dict) + env_vars: MutableMapping[str, str] = field(default_factory=dict) _doc_lookup: Optional[DocLookup] = field( default=None, metadata={'serialize': lambda x: None, 'deserialize': lambda x: None} @@ -1048,6 +1049,7 @@ def __reduce_ex__(self, protocol): self.state_check, self.source_patches, self.disabled, + self.env_vars, self._doc_lookup, self._source_lookup, self._ref_lookup, diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 91e58c1d33f..8afea16775a 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -151,7 +151,7 @@ def patch(self, patch: 'ParsedNodePatch'): # Note: config should already be updated self.patch_path: Optional[str] = patch.file_id # update created_at so process_docs will run in partial parsing - self.created_at = int(time.time()) + self.created_at = time.time() self.description = patch.description self.columns = patch.columns self.meta = patch.meta @@ -193,7 +193,7 @@ class ParsedNodeDefaults(ParsedNodeMandatory): build_path: Optional[str] = None deferred: bool = False unrendered_config: Dict[str, Any] = field(default_factory=dict) - created_at: int = field(default_factory=lambda: int(time.time())) + created_at: float = field(default_factory=lambda: time.time()) config_call_dict: Dict[str, Any] = field(default_factory=dict) def write_node(self, target_path: str, subdirectory: str, payload: str): @@ -493,12 +493,12 @@ class ParsedMacro(UnparsedBaseNode, HasUniqueID): docs: Docs = field(default_factory=Docs) patch_path: Optional[str] = None arguments: List[MacroArgument] = field(default_factory=list) - created_at: int = field(default_factory=lambda: int(time.time())) + created_at: float = field(default_factory=lambda: time.time()) def patch(self, patch: ParsedMacroPatch): self.patch_path: Optional[str] = patch.file_id self.description = patch.description - self.created_at = int(time.time()) + self.created_at = time.time() self.meta = patch.meta self.docs = patch.docs self.arguments = patch.arguments @@ -614,7 +614,7 @@ class ParsedSourceDefinition( patch_path: Optional[Path] = None unrendered_config: Dict[str, Any] = field(default_factory=dict) relation_name: Optional[str] = None - created_at: int = field(default_factory=lambda: int(time.time())) + created_at: float = field(default_factory=lambda: time.time()) def same_database_representation( self, other: 'ParsedSourceDefinition' @@ -725,7 +725,7 @@ class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn): depends_on: DependsOn = field(default_factory=DependsOn) refs: List[List[str]] = field(default_factory=list) sources: List[List[str]] = field(default_factory=list) - created_at: int = field(default_factory=lambda: int(time.time())) + created_at: float = field(default_factory=lambda: time.time()) @property def depends_on_nodes(self): diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index c1e29f9bd1e..469d6aac5ad 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -10,8 +10,8 @@ from dbt import utils from dbt.clients.jinja import MacroGenerator from dbt.context.providers import ( - generate_parser_model, - generate_generate_component_name_macro, + generate_parser_model_context, + generate_generate_name_macro_context, ) from dbt.adapters.factory import get_adapter # noqa: F401 from dbt.clients.jinja import get_rendered @@ -101,7 +101,7 @@ def __init__( f'No macro with name generate_{component}_name found' ) - root_context = generate_generate_component_name_macro( + root_context = generate_generate_name_macro_context( macro, config, manifest ) self.updater = MacroGenerator(macro, root_context) @@ -252,7 +252,7 @@ def _create_parsetime_node( def _context_for( self, parsed_node: IntermediateNode, config: ContextConfig ) -> Dict[str, Any]: - return generate_parser_model( + return generate_parser_model_context( parsed_node, self.root_project, self.manifest, config ) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 729771c263c..f62ad02dfd8 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -25,7 +25,7 @@ from dbt.clients.jinja_static import statically_extract_macro_calls from dbt.clients.system import make_directory from dbt.config import Project, RuntimeConfig -from dbt.context.docs import generate_runtime_docs +from dbt.context.docs import generate_runtime_docs_context from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace from dbt.context.configured import generate_macro_context from dbt.context.providers import ParseProvider @@ -136,7 +136,7 @@ def __init__( self.new_manifest = self.manifest self.manifest.metadata = root_project.get_metadata() self.macro_resolver = None # built after macros are loaded - self.started_at = int(time.time()) + self.started_at = time.time() # This is a MacroQueryStringSetter callable, which is called # later after we set the MacroManifest in the adapter. It sets # up the query headers. @@ -772,7 +772,7 @@ def process_docs(self, config: RuntimeConfig): for node in self.manifest.nodes.values(): if node.created_at < self.started_at: continue - ctx = generate_runtime_docs( + ctx = generate_runtime_docs_context( config, node, self.manifest, @@ -782,7 +782,7 @@ def process_docs(self, config: RuntimeConfig): for source in self.manifest.sources.values(): if source.created_at < self.started_at: continue - ctx = generate_runtime_docs( + ctx = generate_runtime_docs_context( config, source, self.manifest, @@ -792,7 +792,7 @@ def process_docs(self, config: RuntimeConfig): for macro in self.manifest.macros.values(): if macro.created_at < self.started_at: continue - ctx = generate_runtime_docs( + ctx = generate_runtime_docs_context( config, macro, self.manifest, @@ -802,7 +802,7 @@ def process_docs(self, config: RuntimeConfig): for exposure in self.manifest.exposures.values(): if exposure.created_at < self.started_at: continue - ctx = generate_runtime_docs( + ctx = generate_runtime_docs_context( config, exposure, self.manifest, @@ -1144,7 +1144,7 @@ def _process_sources_for_node( def process_macro( config: RuntimeConfig, manifest: Manifest, macro: ParsedMacro ) -> None: - ctx = generate_runtime_docs( + ctx = generate_runtime_docs_context( config, macro, manifest, @@ -1163,5 +1163,5 @@ def process_node( manifest, config.project_name, node ) _process_refs_for_node(manifest, config.project_name, node) - ctx = generate_runtime_docs(config, node, manifest, config.project_name) + ctx = generate_runtime_docs_context(config, node, manifest, config.project_name) _process_docs_for_node(ctx, node) diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index 4b04ced4f3f..5e833e40efb 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -1,5 +1,7 @@ +import os from copy import deepcopy from typing import MutableMapping, Dict, List +from itertools import chain from dbt.contracts.graph.manifest import Manifest from dbt.contracts.files import ( AnySourceFile, ParseFileType, parse_file_type_to_parser, @@ -62,6 +64,7 @@ def __init__(self, saved_manifest: Manifest, new_files: MutableMapping[str, AnyS self.project_parser_files = {} self.deleted_manifest = Manifest() self.macro_child_map: Dict[str, List[str]] = {} + self.env_vars_to_source_files = self.build_env_vars_to_source_files() self.build_file_diff() self.processing_file = None self.deleted_special_override_macro = False @@ -114,6 +117,11 @@ def build_file_diff(self): if self.saved_files[file_id].parse_file_type in mg_files: changed_or_deleted_macro_file = True changed.append(file_id) + # handled changed env_vars for non-schema-files + for file_id in list(chain.from_iterable(self.env_vars_to_source_files.values())): + if file_id in deleted or file_id in changed: + continue + changed.append(file_id) file_diff = { "deleted": deleted, "deleted_schema_files": deleted_schema_files, @@ -795,3 +803,39 @@ def remove_source_override_target(self, source_dict): self.remove_tests(orig_file, 'sources', orig_source['name']) self.merge_patch(orig_file, 'sources', orig_source) self.add_to_pp_files(orig_file) + + # This builds a dictionary of files that need to be scheduled for parsing + # because the env var has changed. + def build_env_vars_to_source_files(self): + env_vars_to_source_files = {} + # The SourceFiles contain a list of env_vars that were used in the file. + # The SchemaSourceFiles contain a dictionary of env_vars to schema file blocks. + # Create a combined dictionary of env_vars to files that contain them. + for source_file in self.saved_files.values(): + if source_file.parse_file_type == ParseFileType.Schema: + continue + for env_var in source_file.env_vars: + if env_var not in env_vars_to_source_files: + env_vars_to_source_files[env_var] = [] + env_vars_to_source_files[env_var].append(source_file.file_id) + + # Check whether the env_var has changed. If it hasn't, remove the env_var + # from env_vars_to_source_files so that we can use it as dictionary of + # which files need to be scheduled for parsing. + delete_unchanged_vars = [] + for env_var in env_vars_to_source_files.keys(): + prev_value = None + if env_var in self.saved_manifest.env_vars: + prev_value = self.saved_manifest.env_vars[env_var] + current_value = os.getenv(env_var) + if prev_value == current_value: + delete_unchanged_vars.append(env_var) + if current_value is None: + # env_var no longer set, remove from manifest + del self.saved_manifest.env_vars[env_var] + + # Actually remove the vars that haven't changed + for env_var in delete_unchanged_vars: + del env_vars_to_source_files[env_var] + + return env_vars_to_source_files diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 39bad039387..4f770a806e5 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -17,8 +17,7 @@ from dbt.context.context_config import ( ContextConfig, ) -from dbt.context.configured import generate_schema_yml -from dbt.context.target import generate_target_context +from dbt.context.configured import generate_schema_yml_context from dbt.context.providers import ( generate_parse_exposure, generate_test_context ) @@ -168,18 +167,10 @@ def __init__( self, project, manifest, root_project, ) -> None: super().__init__(project, manifest, root_project) - all_v_2 = ( - self.root_project.config_version == 2 and - self.project.config_version == 2 + + self.render_ctx = generate_schema_yml_context( + self.root_project, self.project.project_name ) - if all_v_2: - self.render_ctx = generate_schema_yml( - self.root_project, self.project.project_name - ) - else: - self.render_ctx = generate_target_context( - self.root_project, self.root_project.cli_vars - ) self.raw_renderer = SchemaYamlRenderer(self.render_ctx) diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index bb3223fc39d..d04a108b242 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -20,7 +20,7 @@ from dbt import utils from dbt.adapters.base import BaseRelation from dbt.clients.jinja import MacroGenerator -from dbt.context.providers import generate_runtime_model +from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.compiled import CompileResultNode from dbt.contracts.graph.manifest import WritableManifest from dbt.contracts.graph.model_config import Hook @@ -225,7 +225,7 @@ def _materialization_relations( raise CompilationException(msg, node=model) def execute(self, model, manifest): - context = generate_runtime_model( + context = generate_runtime_model_context( model, self.config, manifest ) diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index e36ceb8d4fa..c172492dc34 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -17,7 +17,7 @@ ) from dbt.contracts.graph.manifest import Manifest from dbt.contracts.results import TestStatus, PrimitiveDict, RunResult -from dbt.context.providers import generate_runtime_model +from dbt.context.providers import generate_runtime_model_context from dbt.clients.jinja import MacroGenerator from dbt.exceptions import ( InternalException, @@ -75,7 +75,7 @@ def execute_test( test: Union[CompiledSingularTestNode, CompiledGenericTestNode], manifest: Manifest ) -> TestResultData: - context = generate_runtime_model( + context = generate_runtime_model_context( test, self.config, manifest ) diff --git a/test/integration/013_context_var_tests/test_context_vars.py b/test/integration/013_context_var_tests/test_context_vars.py index cf54c742fff..4707245540b 100644 --- a/test/integration/013_context_var_tests/test_context_vars.py +++ b/test/integration/013_context_var_tests/test_context_vars.py @@ -1,5 +1,5 @@ from dbt.logger import SECRET_ENV_PREFIX -from test.integration.base import DBTIntegrationTest, use_profile +from test.integration.base import DBTIntegrationTest, use_profile, get_manifest import os @@ -96,6 +96,10 @@ def test_postgres_env_vars_dev(self): self.assertEqual(len(results), 1) ctx = self.get_ctx_vars() + manifest = get_manifest() + expected = {'DBT_TEST_013_ENV_VAR': '1', 'DBT_TEST_013_NOT_SECRET': 'regular_variable'} + self.assertEqual(manifest.env_vars, expected) + this = '"{}"."{}"."context"'.format(self.default_database, self.unique_schema()) self.assertEqual(ctx['this'], this) diff --git a/test/integration/068_partial_parsing_tests/test-files/env_var_model.sql b/test/integration/068_partial_parsing_tests/test-files/env_var_model.sql new file mode 100644 index 00000000000..a926d16d9d8 --- /dev/null +++ b/test/integration/068_partial_parsing_tests/test-files/env_var_model.sql @@ -0,0 +1 @@ +select '{{ env_var('ENV_VAR_TEST') }}' as vartest diff --git a/test/integration/068_partial_parsing_tests/test_pp_vars.py b/test/integration/068_partial_parsing_tests/test_pp_vars.py new file mode 100644 index 00000000000..6adbe116a20 --- /dev/null +++ b/test/integration/068_partial_parsing_tests/test_pp_vars.py @@ -0,0 +1,88 @@ +from dbt.exceptions import CompilationException, UndefinedMacroException +from dbt.contracts.graph.manifest import Manifest +from dbt.contracts.files import ParseFileType +from dbt.contracts.results import TestStatus +from dbt.parser.partial import special_override_macros +from test.integration.base import DBTIntegrationTest, use_profile, normalize, get_manifest +import shutil +import os + + +# Note: every test case needs to have separate directories, otherwise +# they will interfere with each other when tests are multi-threaded + +class BasePPTest(DBTIntegrationTest): + + @property + def schema(self): + return "test_068A" + + @property + def models(self): + return "models" + + @property + def project_config(self): + return { + 'config-version': 2, + 'data-paths': ['seeds'], + 'test-paths': ['tests'], + 'macro-paths': ['macros'], + 'analysis-paths': ['analyses'], + 'snapshot-paths': ['snapshots'], + 'seeds': { + 'quote_columns': False, + }, + } + + def setup_directories(self): + # Create the directories for the test in the `self.test_root_dir` + # directory after everything else is symlinked. We can copy to and + # delete files in this directory without tests interfering with each other. + os.mkdir(os.path.join(self.test_root_dir, 'models')) + os.mkdir(os.path.join(self.test_root_dir, 'tests')) + os.mkdir(os.path.join(self.test_root_dir, 'seeds')) + os.mkdir(os.path.join(self.test_root_dir, 'macros')) + os.mkdir(os.path.join(self.test_root_dir, 'analyses')) + os.mkdir(os.path.join(self.test_root_dir, 'snapshots')) + + + +class EnvVarTest(BasePPTest): + + @use_profile('postgres') + def test_postgres_env_vars_models(self): + self.setup_directories() + self.copy_file('test-files/model_one.sql', 'models/model_one.sql') + # initial run + self.run_dbt(['clean']) + results = self.run_dbt(["run"]) + self.assertEqual(len(results), 1) + + # copy a file with an env_var call without an env_var + self.copy_file('test-files/env_var_model.sql', 'models/env_var_model.sql') + with self.assertRaises(UndefinedMacroException): + results = self.run_dbt(["--partial-parse", "run"]) + + # set the env var + os.environ['ENV_VAR_TEST'] = 'TestingEnvVars' + results = self.run_dbt(["--partial-parse", "run"]) + self.assertEqual(len(results), 2) + manifest = get_manifest() + expected_env_vars = {"ENV_VAR_TEST": "TestingEnvVars"} + self.assertEqual(expected_env_vars, manifest.env_vars) + model_id = 'model.test.env_var_model' + model = manifest.nodes[model_id] + model_created_at = model.created_at + + # change the env var + os.environ['ENV_VAR_TEST'] = 'second' + results = self.run_dbt(["--partial-parse", "run"]) + self.assertEqual(len(results), 2) + manifest = get_manifest() + expected_env_vars = {"ENV_VAR_TEST": "second"} + self.assertEqual(expected_env_vars, manifest.env_vars) + self.assertNotEqual(model_created_at, manifest.nodes[model_id].created_at) + + # delete the env var to cleanup + del os.environ['ENV_VAR_TEST'] diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py index 75cbf1e7607..46ce72ee807 100644 --- a/test/unit/test_compiler.py +++ b/test/unit/test_compiler.py @@ -60,13 +60,13 @@ def setUp(self): self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) - self._generate_runtime_model_patch = patch.object( - dbt.compilation, 'generate_runtime_model') - self.mock_generate_runtime_model = self._generate_runtime_model_patch.start() + self._generate_runtime_model_context_patch = patch.object( + dbt.compilation, 'generate_runtime_model_context') + self.mock_generate_runtime_model_context = self._generate_runtime_model_context_patch.start() inject_adapter(Plugin.adapter(self.config), Plugin) - def mock_generate_runtime_model_context(model, config, manifest): + def mock_generate_runtime_model_context_meth(model, config, manifest): def ref(name): result = f'__dbt__cte__{name}' unique_id = f'model.root.{name}' @@ -74,10 +74,10 @@ def ref(name): return result return {'ref': ref} - self.mock_generate_runtime_model.side_effect = mock_generate_runtime_model_context + self.mock_generate_runtime_model_context.side_effect = mock_generate_runtime_model_context_meth def tearDown(self): - self._generate_runtime_model_patch.stop() + self._generate_runtime_model_context_patch.stop() clear_plugin(Plugin) def test__prepend_ctes__already_has_cte(self): diff --git a/test/unit/test_context.py b/test/unit/test_context.py index 577840acf37..360c9284e9b 100644 --- a/test/unit/test_context.py +++ b/test/unit/test_context.py @@ -385,7 +385,7 @@ def test_query_header_context(config_postgres, manifest_fx): def test_macro_runtime_context(config_postgres, manifest_fx, get_adapter, get_include_paths): - ctx = providers.generate_runtime_macro( + ctx = providers.generate_runtime_macro_context( macro=manifest_fx.macros['macro.root.macro_a'], config=config_postgres, manifest=manifest_fx, @@ -395,7 +395,7 @@ def test_macro_runtime_context(config_postgres, manifest_fx, get_adapter, get_in def test_model_parse_context(config_postgres, manifest_fx, get_adapter, get_include_paths): - ctx = providers.generate_parser_model( + ctx = providers.generate_parser_model_context( model=mock_model(), config=config_postgres, manifest=manifest_fx, @@ -405,7 +405,7 @@ def test_model_parse_context(config_postgres, manifest_fx, get_adapter, get_incl def test_model_runtime_context(config_postgres, manifest_fx, get_adapter, get_include_paths): - ctx = providers.generate_runtime_model( + ctx = providers.generate_runtime_model_context( model=mock_model(), config=config_postgres, manifest=manifest_fx, @@ -414,7 +414,7 @@ def test_model_runtime_context(config_postgres, manifest_fx, get_adapter, get_in def test_docs_runtime_context(config_postgres): - ctx = docs.generate_runtime_docs(config_postgres, mock_model(), [], 'root') + ctx = docs.generate_runtime_docs_context(config_postgres, mock_model(), [], 'root') assert_has_keys(REQUIRED_DOCS_KEYS, MAYBE_KEYS, ctx) diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index 5d37cc7078b..63b3be78a08 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -122,7 +122,7 @@ def base_parsed_model_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Model), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -182,6 +182,7 @@ def basic_parsed_model_object(): config=NodeConfig(), meta={}, checksum=FileHash.from_contents(''), + created_at=1.0, ) @@ -190,7 +191,7 @@ def minimal_parsed_model_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Model), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -211,7 +212,7 @@ def complex_parsed_model_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Model), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -404,7 +405,7 @@ def basic_parsed_seed_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Seed), 'path': '/root/seeds/seed.csv', 'original_file_path': 'seeds/seed.csv', @@ -477,7 +478,7 @@ def minimal_parsed_seed_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Seed), 'path': '/root/seeds/seed.csv', 'original_file_path': 'seeds/seed.csv', @@ -497,7 +498,7 @@ def complex_parsed_seed_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Seed), 'path': '/root/seeds/seed.csv', 'original_file_path': 'seeds/seed.csv', @@ -718,6 +719,8 @@ def patched_model_object(): def test_patch_parsed_model(basic_parsed_model_object, basic_parsed_model_patch_object, patched_model_object): pre_patch = basic_parsed_model_object pre_patch.patch(basic_parsed_model_patch_object) + pre_patch.created_at = 1.0 + patched_model_object.created_at = 1.0 assert patched_model_object == pre_patch @@ -745,7 +748,7 @@ def base_parsed_hook_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Operation), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -815,7 +818,7 @@ def complex_parsed_hook_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Operation), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -933,7 +936,7 @@ def minimal_parsed_schema_test_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Test), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -958,7 +961,7 @@ def basic_parsed_schema_test_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Test), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -1030,7 +1033,7 @@ def complex_parsed_schema_test_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Test), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -1372,7 +1375,7 @@ def basic_timestamp_snapshot_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Snapshot), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -1504,7 +1507,7 @@ def basic_check_snapshot_dict(): return { 'name': 'foo', 'root_path': '/root/', - 'created_at': 1, + 'created_at': 1.0, 'resource_type': str(NodeType.Snapshot), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', @@ -1716,7 +1719,7 @@ def _ok_dict(self): 'name': 'foo', 'path': '/root/path.sql', 'original_file_path': '/root/path.sql', - 'created_at': 1, + 'created_at': 1.0, 'package_name': 'test', 'macro_sql': '{% macro foo() %}select 1 as id{% endmacro %}', 'root_path': '/root/', @@ -1807,7 +1810,7 @@ def minimum_parsed_source_definition_dict(): 'root_path': '/root', 'path': '/root/models/sources.yml', 'original_file_path': '/root/models/sources.yml', - 'created_at': 1, + 'created_at': 1.0, 'database': 'some_db', 'schema': 'some_schema', 'fqn': ['test', 'source', 'my_source', 'my_source_table'], @@ -1828,7 +1831,7 @@ def basic_parsed_source_definition_dict(): 'root_path': '/root', 'path': '/root/models/sources.yml', 'original_file_path': '/root/models/sources.yml', - 'created_at': 1, + 'created_at': 1.0, 'database': 'some_db', 'schema': 'some_schema', 'fqn': ['test', 'source', 'my_source', 'my_source_table'], @@ -1884,7 +1887,7 @@ def complex_parsed_source_definition_dict(): 'root_path': '/root', 'path': '/root/models/sources.yml', 'original_file_path': '/root/models/sources.yml', - 'created_at': 1, + 'created_at': 1.0, 'database': 'some_db', 'schema': 'some_schema', 'fqn': ['test', 'source', 'my_source', 'my_source_table'], @@ -2039,7 +2042,7 @@ def minimal_parsed_exposure_dict(): 'root_path': '/usr/src/app', 'original_file_path': 'models/something.yml', 'description': '', - 'created_at': 1, + 'created_at': 1.0, } @@ -2067,7 +2070,7 @@ def basic_parsed_exposure_dict(): 'description': '', 'meta': {}, 'tags': [], - 'created_at': 1, + 'created_at': 1.0, } @@ -2094,7 +2097,7 @@ def complex_parsed_exposure_dict(): return { 'name': 'my_exposure', 'type': 'analysis', - 'created_at': 1, + 'created_at': 1.0, 'owner': { 'email': 'test@example.com', 'name': 'A Name', diff --git a/test/unit/test_partial_parsing.py b/test/unit/test_partial_parsing.py index 9cae6f4fc44..93223de172b 100644 --- a/test/unit/test_partial_parsing.py +++ b/test/unit/test_partial_parsing.py @@ -23,6 +23,7 @@ def setUp(self): project_name=project_name, parse_file_type=ParseFileType.Model, nodes=['model.my_test.my_model'], + env_vars=[], ) schema_file = SchemaSourceFile( path=FilePath(project_root=project_root, searched_path='models', relative_path='schema.yml', modification_time=time.time()), @@ -31,6 +32,7 @@ def setUp(self): parse_file_type=ParseFileType.Schema, dfy={'version': 2, 'models': [{'name': 'my_model', 'description': 'Test model'}]}, ndp=['model.my_test.my_model'], + env_vars={}, ) self.saved_files = {model_file.file_id: model_file, schema_file.file_id: schema_file} model_node = self.get_model('my_model')