Update and adjust daft to version 0.2.23 & above (#238)

* - Updated `daft_validation`
- Updated related tests
- Updated `pyproject.toml`

* - fixed has entropy

* - made `are_complete` backward compatible

pyproject.toml

@@ -65,7 +65,7 @@ pdf = [
   "fpdf2==2.7.8"
 ]
 daft = [
-  "getdaft == 0.2.19"
+  "getdaft == 0.2.24"
 ]
 all = [
   "cuallee[dev,pyspark,pyspark_connect,snowpark,pandas,bigquery,duckdb,polars,test,dagster,cloud,pdf,daft]"

test/unit/daft/test_are_complete.py

@@ -32,6 +32,8 @@ def test_coverage(check: Check):
     check.are_complete(("id", "id2"), 0.75)
     df = daft.from_pydict({"id": [10, None], "id2": [300, 500]})
     result = check.validate(df)
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 0.75).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.75).to_pandas().pass_rate.all()
     )

test/unit/daft/test_are_unique.py

@@ -33,6 +33,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [10, None], "id2": [300, 500]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 0.75).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.75).to_pandas().pass_rate.all()
     )

test/unit/daft/test_has_pattern.py

@@ -47,6 +47,7 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": ["Herminio", "Hola", "Villain", "Heroe"]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 0.75).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.75).to_pandas().pass_rate.all()
     )

test/unit/daft/test_has_workflow.py

@@ -30,6 +30,8 @@ def test_coverage(check: Check):
     )
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 4 / 6).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 4/6).to_pandas().pass_rate.all()
     )

test/unit/daft/test_is_between.py

@@ -45,6 +45,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": np.arange(20)})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 0.55).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.55).to_pandas().pass_rate.all()
     )

test/unit/daft/test_is_complete.py

@@ -21,4 +21,7 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [10, None], "id2": [300, 500]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_contained_in.py

@@ -33,4 +33,7 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": np.arange(10)})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_daily.py

@@ -34,4 +34,7 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.6).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.60).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_equal_than.py

@@ -32,6 +32,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [1, 1, 1, 0]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 0.75).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.75).to_pandas().pass_rate.all()
     )
+

test/unit/daft/test_is_greater_or_equal_than.py

@@ -32,4 +32,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.6).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.60).to_pandas().pass_rate.all()
+    )
+

test/unit/daft/test_is_greater_than.py

@@ -32,4 +32,7 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_in_billions.py

@@ -22,4 +22,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [1.0, 1e9]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_in_millions.py

@@ -22,4 +22,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [1.0, 1e6]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_inside_interquartile_range.py

@@ -180,4 +180,7 @@ def test_coverage(check: Check):
     check.is_inside_interquartile_range("id", pct=0.5)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_less_or_equal_than.py

@@ -32,4 +32,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.6).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.60).to_pandas().pass_rate.all()
+    )
+

test/unit/daft/test_is_less_than.py

@@ -32,4 +32,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.6).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.60).to_pandas().pass_rate.all()
+    )
+

test/unit/daft/test_is_negative.py

@@ -33,4 +33,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [1, 2, -1, -2]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_on_friday.py

@@ -36,6 +36,9 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 1 / 7).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 1 / 7).to_pandas().pass_rate.all()
     )
+

test/unit/daft/test_is_on_monday.py

@@ -36,6 +36,9 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 1 / 7).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 1/7).to_pandas().pass_rate.all()
     )
+

test/unit/daft/test_is_on_saturday.py

@@ -36,6 +36,8 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 1 / 7).to_pandas().pass_rate.all()
-    )
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 1 / 7).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_on_schedule.py

@@ -75,6 +75,8 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 7 / 8).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 7 / 8).to_pandas().pass_rate.all()
     )

test/unit/daft/test_is_on_sunday.py

@@ -36,6 +36,8 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 1 / 7).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 1 / 7).to_pandas().pass_rate.all()
     )

test/unit/daft/test_is_on_thursday.py

@@ -36,6 +36,8 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 1 / 7).to_pandas().pass_rate.all()
-    )
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 1 / 7).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_on_tuesday.py

@@ -36,6 +36,8 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 1 / 7).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 1 / 7).to_pandas().pass_rate.all()
     )

test/unit/daft/test_is_on_wednesday.py

@@ -36,6 +36,8 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 1 / 7).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 1 / 7).to_pandas().pass_rate.all()
     )

test/unit/daft/test_is_on_weekday.py

@@ -36,6 +36,8 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 5 / 7).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 5 / 7).to_pandas().pass_rate.all()
     )

test/unit/daft/test_is_on_weekend.py

@@ -36,6 +36,8 @@ def test_coverage(check: Check):
     df = daft.from_pandas(pd_df)
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 2 / 7).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 2 / 7).to_pandas().pass_rate.all()
     )

test/unit/daft/test_is_positive.py

@@ -34,4 +34,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [1, 2, -1, -2]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_is_unique.py

@@ -35,6 +35,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [10, 20, 30, 10]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
+
+    col_pass_rate = daft.col("pass_rate")
     assert (
-        result.select(daft.col("pass_rate").max() == 0.75).to_pandas().pass_rate.all()
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.75).to_pandas().pass_rate.all()
     )

test/unit/daft/test_not_contained_in.py

@@ -33,4 +33,8 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": np.arange(10)})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )

test/unit/daft/test_satisfies.py

@@ -24,4 +24,7 @@ def test_coverage(check: Check):
     df = daft.from_pydict({"id": [10, -10], "id2": [300, 500]})
     result = check.validate(df)
     assert result.select(daft.col("status").str.match("PASS")).to_pandas().status.all()
-    assert result.select(daft.col("pass_rate").max() == 0.5).to_pandas().pass_rate.all()
+    col_pass_rate = daft.col("pass_rate")
+    assert (
+        result.agg(col_pass_rate.max()).select(col_pass_rate == 0.50).to_pandas().pass_rate.all()
+    )