empty unit test csv fixture values default to null (#10117) (#10118)

.changes/unreleased/Fixes-20240509-091411.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Unit test fixture (csv) returns null for empty value
+time: 2024-05-09T09:14:11.772709-04:00
+custom:
+  Author: michelleark
+  Issue: "9881"

core/dbt/events/types.py

@@ -1815,7 +1815,7 @@ def code(self) -> str:
         return "Z026"
 
     def message(self) -> str:
-        return f"  compiled Code at {self.path}"
+        return f"  compiled code at {self.path}"
 
 
 class CheckNodeTestFailure(InfoLevel):

core/dbt/parser/unit_tests.py

@@ -366,11 +366,17 @@ def _validate_and_normalize_rows(self, ut_fixture, unit_test_definition, fixture
                 )
 
             if ut_fixture.fixture:
-                ut_fixture.rows = self.get_fixture_file_rows(
+                csv_rows = self.get_fixture_file_rows(
                     ut_fixture.fixture, self.project.project_name, unit_test_definition.unique_id
                 )
             else:
-                ut_fixture.rows = self._convert_csv_to_list_of_dicts(ut_fixture.rows)
+                csv_rows = self._convert_csv_to_list_of_dicts(ut_fixture.rows)
+
+            # Empty values (e.g. ,,) in a csv fixture should default to null, not ""
+            ut_fixture.rows = [
+                {k: (None if v == "" else v) for k, v in row.items()} for row in csv_rows
+            ]
+
         elif ut_fixture.format == UnitTestFormat.SQL:
             if not (isinstance(ut_fixture.rows, str) or isinstance(ut_fixture.fixture, str)):
                 raise ParsingError(

tests/functional/unit_testing/fixtures.py

@@ -41,6 +41,15 @@
 'b' as string_b
 """
 
+my_model_check_null_sql = """
+SELECT
+CASE
+  WHEN a IS null THEN True
+  ELSE False
+END a_is_null
+FROM {{ ref('my_model_a') }}
+"""
+
 test_my_model_yml = """
 unit_tests:
   - name: test_my_model
@@ -507,6 +516,11 @@
 1,a
 """
 
+test_my_model_a_with_null_fixture_csv = """id,a
+1,
+2,3
+"""
+
 test_my_model_a_empty_fixture_csv = """
 """
 
@@ -1060,3 +1074,39 @@ def external_package():
       rows:
         - {id: 2}
 """
+
+
+test_my_model_csv_null_yml = """
+unit_tests:
+  - name: test_my_model_check_null
+    model: my_model_check_null
+    given:
+      - input: ref('my_model_a')
+        format: csv
+        rows: |
+          id,a
+          1,
+          2,3
+    expect:
+      format: csv
+      rows: |
+        a_is_null
+        True
+        False
+"""
+
+test_my_model_file_csv_null_yml = """
+unit_tests:
+  - name: test_my_model_check_null
+    model: my_model_check_null
+    given:
+      - input: ref('my_model_a')
+        format: csv
+        fixture: test_my_model_a_with_null_fixture
+    expect:
+      format: csv
+      rows: |
+        a_is_null
+        True
+        False
+"""

tests/functional/unit_testing/test_csv_fixtures.py

@@ -2,24 +2,28 @@
 from dbt.exceptions import ParsingError, YamlParseDictError, DuplicateResourceNameError
 from dbt.tests.util import run_dbt, write_file, rm_file
 from fixtures import (
-    my_model_sql,
-    my_model_a_sql,
-    my_model_b_sql,
-    test_my_model_csv_yml,
     datetime_test,
-    datetime_test_invalid_format_key,
     datetime_test_invalid_csv_values,
-    test_my_model_file_csv_yml,
-    test_my_model_fixture_csv,
+    datetime_test_invalid_format_key,
+    my_model_a_sql,
+    my_model_b_sql,
+    my_model_check_null_sql,
+    my_model_sql,
+    test_my_model_a_empty_fixture_csv,
     test_my_model_a_fixture_csv,
+    test_my_model_a_numeric_fixture_csv,
+    test_my_model_a_with_null_fixture_csv,
     test_my_model_b_fixture_csv,
     test_my_model_basic_fixture_csv,
-    test_my_model_a_numeric_fixture_csv,
-    test_my_model_a_empty_fixture_csv,
     test_my_model_concat_fixture_csv,
-    test_my_model_mixed_csv_yml,
-    test_my_model_missing_csv_yml,
+    test_my_model_csv_null_yml,
+    test_my_model_csv_yml,
     test_my_model_duplicate_csv_yml,
+    test_my_model_file_csv_null_yml,
+    test_my_model_file_csv_yml,
+    test_my_model_fixture_csv,
+    test_my_model_missing_csv_yml,
+    test_my_model_mixed_csv_yml,
 )
 
 
@@ -207,6 +211,50 @@ def test_unit_test(self, project):
             results = run_dbt(["test", "--select", "my_model"], expect_pass=False)
 
 
+class TestUnitTestsInlineCSVEmptyValueIsNull:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_model_a.sql": my_model_a_sql,
+            "my_model_check_null.sql": my_model_check_null_sql,
+            "test_my_model_csv_null.yml": test_my_model_csv_null_yml,
+        }
+
+    def test_unit_test(self, project):
+        results = run_dbt(["run"])
+        assert len(results) == 2
+
+        # Select by model name
+        results = run_dbt(["test", "--select", "my_model_check_null"], expect_pass=True)
+        assert len(results) == 1
+
+
+class TestUnitTestsFileCSVEmptyValueIsNull:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_model_a.sql": my_model_a_sql,
+            "my_model_check_null.sql": my_model_check_null_sql,
+            "test_my_model_file_csv_null.yml": test_my_model_file_csv_null_yml,
+        }
+
+    @pytest.fixture(scope="class")
+    def tests(self):
+        return {
+            "fixtures": {
+                "test_my_model_a_with_null_fixture.csv": test_my_model_a_with_null_fixture_csv,
+            }
+        }
+
+    def test_unit_test(self, project):
+        results = run_dbt(["run"])
+        assert len(results) == 2
+
+        # Select by model name
+        results = run_dbt(["test", "--select", "my_model_check_null"], expect_pass=True)
+        assert len(results) == 1
+
+
 class TestUnitTestsMissingCSVFile:
     @pytest.fixture(scope="class")
     def models(self):