null_out_values tests added

tests/test_remove_anomalies.py

@@ -0,0 +1,67 @@
+import pytest
+import pandas as pd
+
+from splink_data_standardisation.remove_anomalies import null_out_values
+from pyspark.sql import Row
+
+
+def test_null_out_vals_0(spark):
+
+    data_list = [
+        {"id": 1, "mycol": "A"},
+        {"id": 2, "mycol": "B"},
+        {"id": 3, "mycol": "B"},
+        {"id": 4, "mycol": "C"},
+        {"id": 5, "mycol": "C"},
+    ]
+
+    garbagevals = []
+
+    df = spark.createDataFrame(Row(**x) for x in data_list)
+
+    df = null_out_values(df, "mycol", garbagevals)
+
+    df_result = df.toPandas()
+
+    df_expected = [
+        {"id": 1, "mycol": "A"},
+        {"id": 2, "mycol": "B"},
+        {"id": 3, "mycol": "B"},
+        {"id": 4, "mycol": "C"},
+        {"id": 5, "mycol": "C"},
+    ]
+
+    df_expected = pd.DataFrame(df_expected)
+
+    pd.testing.assert_frame_equal(df_result, df_expected)
+
+
+def test_null_out_vals_1(spark):
+
+    data_list = [
+        {"id": 1, "mycol": "A"},
+        {"id": 2, "mycol": "B"},
+        {"id": 3, "mycol": "B"},
+        {"id": 4, "mycol": "C"},
+        {"id": 5, "mycol": "C"},
+    ]
+
+    garbagevals = ["C"]
+
+    df = spark.createDataFrame(Row(**x) for x in data_list)
+
+    df = null_out_values(df, "mycol", garbagevals)
+
+    df_result = df.toPandas()
+
+    df_expected = [
+        {"id": 1, "mycol": "A"},
+        {"id": 2, "mycol": "B"},
+        {"id": 3, "mycol": "B"},
+        {"id": 4, "mycol": None},
+        {"id": 5, "mycol": None},
+    ]
+
+    df_expected = pd.DataFrame(df_expected)
+
+    pd.testing.assert_frame_equal(df_result, df_expected)