diff --git a/core/dbt/parser/models.py b/core/dbt/parser/models.py index 59d17f851ab..adbd1b6b1d9 100644 --- a/core/dbt/parser/models.py +++ b/core/dbt/parser/models.py @@ -87,9 +87,7 @@ def render_update( model_parser_copy.populate( exp_sample_node, exp_sample_config, - experimental_sample['refs'], - experimental_sample['sources'], - dict(experimental_sample['configs']) + experimental_sample ) # use the experimental parser exclusively if the flag is on if flags.USE_EXPERIMENTAL_PARSER: @@ -117,17 +115,12 @@ def render_update( super(ModelParser, model_parser_copy) \ .render_update(jinja_sample_node, jinja_sample_config) - # manually fit configs in - config._config_call_dict = _get_config_call_dict(statically_parsed) - # update the unrendered config with values from the static parser. # values from yaml files are in there already self.populate( node, config, - statically_parsed['refs'], - statically_parsed['sources'], - dict(statically_parsed['configs']) + statically_parsed ) # if we took a jinja sample, compare now that the base node has been populated @@ -268,28 +261,29 @@ def _has_banned_macro( False ) - # this method updates the model note rendered and unrendered config as well + # this method updates the model node rendered and unrendered config as well # as the node object. Used to populate these values when circumventing jinja # rendering like the static parser. def populate( self, node: ParsedModelNode, config: ContextConfig, - refs: List[List[str]], - sources: List[List[str]], - configs: Dict[str, Any] + statically_parsed: Dict[str, Any] ): + # manually fit configs in + config._config_call_dict = _get_config_call_dict(statically_parsed) + # if there are hooks present this, it WILL render jinja. Will need to change # when the experimental parser supports hooks self.update_parsed_node_config(node, config) # update the unrendered config with values from the file. # values from yaml files are in there already - node.unrendered_config.update(configs) + node.unrendered_config.update(dict(statically_parsed['configs'])) # set refs and sources on the node object - node.refs += refs - node.sources += sources + node.refs += statically_parsed['refs'] + node.sources += statically_parsed['sources'] # configs don't need to be merged into the node because they # are read from config._config_call_dict @@ -305,7 +299,7 @@ def partial_deepcopy(self): # pure function. safe to use elsewhere, but unlikely to be useful outside this file. def _get_config_call_dict( - static_parser_result: Dict[str, List[Any]] + static_parser_result: Dict[str, Any] ) -> Dict[str, Any]: config_call_dict: Dict[str, Any] = {} @@ -371,8 +365,8 @@ def _get_sample_result( ) -> List[Tuple[int, str]]: result: List[Tuple[int, str]] = [] # look for false positive configs - for k in config._config_call_dict: - if k not in config._config_call_dict: + for k in sample_config._config_call_dict.keys(): + if k not in config._config_call_dict.keys(): result += [(2, "false_positive_config_value")] break diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index 27d118a95d6..85d301ee13e 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -5,9 +5,11 @@ import os import yaml +from copy import deepcopy import dbt.flags import dbt.parser from dbt import tracking +from dbt.context.context_config import ContextConfig from dbt.exceptions import CompilationException from dbt.parser import ( ModelParser, MacroParser, SingularTestParser, GenericTestParser, @@ -32,6 +34,9 @@ ParsedAnalysisNode, UnpatchedSourceDefinition ) from dbt.contracts.graph.unparsed import Docs +from dbt.parser.models import ( + _get_config_call_dict, _shift_sources, _get_exp_sample_result, _get_stable_sample_result, _get_sample_result +) import itertools from .utils import config_from_parts_or_dicts, normalize, generate_name_macros, MockNode, MockSource, MockDocumentation @@ -571,6 +576,167 @@ def test_built_in_macro_override_detection(self): assert(self.parser._has_banned_macro(node)) +# TODO +class StaticModelParserUnitTest(BaseParserTest): + # _get_config_call_dict + # _shift_sources + # _get_exp_sample_result + # _get_stable_sample_result + # _get_sample_result + + def setUp(self): + super().setUp() + self.parser = ModelParser( + project=self.snowplow_project_config, + manifest=self.manifest, + root_project=self.root_project_config, + ) + self.example_node = ParsedModelNode( + alias='model_1', + name='model_1', + database='test', + schema='analytics', + resource_type=NodeType.Model, + unique_id='model.snowplow.model_1', + fqn=['snowplow', 'nested', 'model_1'], + package_name='snowplow', + original_file_path=normalize('models/nested/model_1.sql'), + root_path=get_abs_os_path('./dbt_packages/snowplow'), + config=NodeConfig(materialized='table'), + path=normalize('nested/model_1.sql'), + raw_sql='{{ config(materialized="table") }}select 1 as id', + checksum=None, + unrendered_config={'materialized': 'table'}, + ) + self.example_config = ContextConfig( + self.root_project_config, + self.example_node.fqn, + self.example_node.resource_type, + self.snowplow_project_config, + ) + + def file_block_for(self, data, filename): + return super().file_block_for(data, filename, 'models') + + # tests that configs get extracted properly. the function should respect merge behavior, + # but becuase it's only reading from one dictionary it won't matter except in edge cases + # like this example with tags changing type to a list. + def test_config_shifting(self): + static_parser_result = { + 'configs': [ + ('hello', 'world'), + ('flag', True), + ('tags', 'tag1'), + ('tags', 'tag2') + ] + } + expected = { + 'hello': 'world', + 'flag': True, + 'tags': ['tag1', 'tag2'] + } + got = _get_config_call_dict(static_parser_result) + self.assertEqual(expected, got) + + def test_source_shifting(self): + static_parser_result = { + 'sources': [('abc', 'def'), ('x', 'y')] + } + expected = { + 'sources': [['abc', 'def'], ['x', 'y']] + } + got = _shift_sources(static_parser_result) + self.assertEqual(expected, got) + + def test_sample_results(self): + # --- missed ref --- # + node = deepcopy(self.example_node) + config = deepcopy(self.example_config) + sample_node = deepcopy(self.example_node) + sample_config = deepcopy(self.example_config) + + sample_node.refs = [] + node.refs = ['myref'] + + result = _get_sample_result(sample_node, sample_config, node, config) + self.assertEqual([(7, "missed_ref_value")], result) + + # --- false positive ref --- # + node = deepcopy(self.example_node) + config = deepcopy(self.example_config) + sample_node = deepcopy(self.example_node) + sample_config = deepcopy(self.example_config) + + sample_node.refs = ['myref'] + node.refs = [] + + result = _get_sample_result(sample_node, sample_config, node, config) + self.assertEqual([(6, "false_positive_ref_value")], result) + + # --- missed source --- # + node = deepcopy(self.example_node) + config = deepcopy(self.example_config) + sample_node = deepcopy(self.example_node) + sample_config = deepcopy(self.example_config) + + sample_node.sources = [] + node.sources = [['abc', 'def']] + + result = _get_sample_result(sample_node, sample_config, node, config) + self.assertEqual([(5, 'missed_source_value')], result) + + # --- false positive source --- # + node = deepcopy(self.example_node) + config = deepcopy(self.example_config) + sample_node = deepcopy(self.example_node) + sample_config = deepcopy(self.example_config) + + sample_node.sources = [['abc', 'def']] + node.sources = [] + + result = _get_sample_result(sample_node, sample_config, node, config) + self.assertEqual([(4, 'false_positive_source_value')], result) + + # --- missed config --- # + node = deepcopy(self.example_node) + config = deepcopy(self.example_config) + sample_node = deepcopy(self.example_node) + sample_config = deepcopy(self.example_config) + + sample_config._config_call_dict = {} + config._config_call_dict = {'key': 'value'} + + result = _get_sample_result(sample_node, sample_config, node, config) + self.assertEqual([(3, 'missed_config_value')], result) + + # --- false positive config --- # + node = deepcopy(self.example_node) + config = deepcopy(self.example_config) + sample_node = deepcopy(self.example_node) + sample_config = deepcopy(self.example_config) + + sample_config._config_call_dict = {'key': 'value'} + config._config_call_dict = {} + + result = _get_sample_result(sample_node, sample_config, node, config) + self.assertEqual([(2, "false_positive_config_value")], result) + + def test_exp_sample_results(self): + node = deepcopy(self.example_node) + config = deepcopy(self.example_config) + sample_node = deepcopy(self.example_node) + sample_config = deepcopy(self.example_config) + result = _get_exp_sample_result(sample_node, sample_config, node, config) + self.assertEqual(["00_experimental_exact_match"], result) + + def test_stable_sample_results(self): + node = deepcopy(self.example_node) + config = deepcopy(self.example_config) + sample_node = deepcopy(self.example_node) + sample_config = deepcopy(self.example_config) + result = _get_stable_sample_result(sample_node, sample_config, node, config) + self.assertEqual(["80_stable_exact_match"], result) + class SnapshotParserTest(BaseParserTest): def setUp(self):