Merge pull request #1665 from edx/musama/DPMC-165

feat: include functionality to optionally run parents model tests

dataeng/jobs/analytics/WarehouseTransforms.groovy

@@ -31,6 +31,7 @@ class WarehouseTransforms{
                     stringParam('TEST_SOURCES_FIRST', env_config.get('TEST_SOURCES_FIRST', 'true'), 'Set to \'true\' to perform source testing first (if SKIP_TESTS is false). All other values test sources post-run.')
                     booleanParam('CAUTIOUS_INDIRECT_SELECTION', env_config.get('CAUTIOUS_INDIRECT_SELECTION', false), 'Check this box if you want a test to run if and only if ALL the models associated with that test have been selected (see dbt docs for --indirect-selection=cautious).')
                     stringParam('PUSH_ARTIFACTS_TO_SNOWFLAKE', env_config.get('PUSH_ARTIFACTS_TO_SNOWFLAKE', 'false'), 'Set to \'true\' to push the run results file to Snowflake for telemetry. Avoid this on frequently-running jobs.')
+                    stringParam('TEST_PARENT_MODELS_FIRST', env_config.get('TEST_PARENT_MODELS_FIRST', 'false'), 'Set to \'true\' to run the upstream models tests first.')
                     stringParam('NOTIFY', env_config.get('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY')), 'Space separated list of emails to send notifications to.')
                     booleanParam('FULL_REFRESH_INCREMENTALS', false, '[DANGEROUS] Supply the --full-refresh flag to the `dbt run` command, and use a larger warehouse. Use when you need to re-compute an incremental table from scratch.  Applies to ALL incrementals in this run.')
                 }

dataeng/resources/warehouse-transforms.sh

@@ -66,6 +66,30 @@ then
     postCommandChecks "source_test" $ret ;
 fi
 
+# Parent models tests *before* model-building can be enabled/disabled with this envvar.
+if [ "$TEST_PARENT_MODELS_FIRST" = 'true' ]
+then
+    # Copy the value of MODEL_SELECTOR to MODEL_SELECTOR_WITH_PARENTS and EXCLUDE_MODELS.
+    MODEL_SELECTOR_WITH_PARENTS="$MODEL_SELECTOR"
+    EXCLUDE_MODELS="$MODEL_SELECTOR"
+
+    # Check if MODEL_SELECTOR_WITH_PARENTS doesn't start with '+', then add '+' at the beginning.
+    if [ ${MODEL_SELECTOR_WITH_PARENTS:0:1} != "+" ]
+    then
+      MODEL_SELECTOR_WITH_PARENTS="+$MODEL_SELECTOR_WITH_PARENTS"
+    fi
+
+    # Check if EXCLUDE_MODELS starts with '+', then remove the '+' at the beginning
+    if [ ${EXCLUDE_MODELS:0:1} = "+" ]
+    then
+      EXCLUDE_MODELS="${EXCLUDE_MODELS:1}"
+    fi
+
+    # This will only runs parents tests without running current models tests.
+    dbt test --models $MODEL_SELECTOR_WITH_PARENTS --exclude $EXCLUDE_MODELS --profile $DBT_PROFILE --target $DBT_TARGET --profiles-dir $WORKSPACE/analytics-secure/warehouse-transforms/ ; ret=$?;
+    postCommandChecks "parent_models_tests" $ret ;
+fi
+
 # Compile/build all models with this tag.
 dbt $DBT_COMMAND $FULL_REFRESH_ARG --models $MODEL_SELECTOR --profile $DBT_PROFILE --target $DBT_TARGET --profiles-dir $WORKSPACE/analytics-secure/warehouse-transforms/ ; ret=$?;
 postCommandChecks "run" $ret ;