WIP: enable liquid clustering for delta lakes

smart-on-fhir · May 30, 2024 · b55a263 · b55a263
1 parent fa64d81
commit b55a263
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/cumulus_etl/formats/deltalake.py b/cumulus_etl/formats/deltalake.py
@@ -98,6 +98,7 @@ def update_delta_table(self, updates: pyspark.sql.DataFrame, groups: set[str]) -
         table = (
             delta.DeltaTable.createIfNotExists(self.spark)
             .addColumns(updates.schema)
+            .clusterBy(*self.uniqueness_fields)
             .location(self._table_path(self.dbname))
             .execute()
         )

diff --git a/pyproject.toml b/pyproject.toml
@@ -13,7 +13,7 @@ requires-python = ">= 3.10"
 dependencies = [
     "ctakesclient >= 5.1, < 6",
     "cumulus-fhir-support >= 1, < 2",
-    "delta-spark >= 3, < 4",
+    "delta-spark >= 3.2, < 4",
     "httpx < 1",
     "inscriptis < 3",
     "jwcrypto < 2",