feat: Enable Arrow-based columnar data transfers (#3996)

feat: Enable Arrow-based columnar data transfers Signed-off-by: tanlocnguyen <tanlocnguyen296@gmail.com> Co-authored-by: tanlocnguyen <tanlocnguyen296@gmail.com>
feast-dev · Mar 21, 2024 · d8d7567 · d8d7567
1 parent 6d9156b
commit d8d7567
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/docs/reference/offline-stores/spark.md b/docs/reference/offline-stores/spark.md
@@ -30,6 +30,8 @@ offline_store:
         spark.sql.catalogImplementation: "hive"
         spark.sql.parser.quotedRegexColumnNames: "true"
         spark.sql.session.timeZone: "UTC"
+        spark.sql.execution.arrow.fallback.enabled: "true"
+        spark.sql.execution.arrow.pyspark.enabled: "true"
 online_store:
     path: data/online_store.db
 ```

diff --git a/sdk/python/feast/templates/spark/feature_repo/feature_store.yaml b/sdk/python/feast/templates/spark/feature_repo/feature_store.yaml
@@ -12,6 +12,8 @@ offline_store:
     spark.sql.catalogImplementation: "hive"
     spark.sql.parser.quotedRegexColumnNames: "true"
     spark.sql.session.timeZone: "UTC"
+    spark.sql.execution.arrow.fallback.enabled: "true"
+    spark.sql.execution.arrow.pyspark.enabled: "true"
 online_store:
   path: data/online_store.db
 entity_key_serialization_version: 2