Remove spark bigquery package download at run time (#96)

Signed-off-by: Khor Shu Heng <khor.heng@go-jek.com>
feast-dev · Sep 25, 2021 · 8de7d33 · 8de7d33
1 parent 0e905ae
commit 8de7d33
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/python/feast_spark/pyspark/launchers/k8s/k8s.py b/python/feast_spark/pyspark/launchers/k8s/k8s.py
@@ -13,7 +13,6 @@
 
 from feast.staging.storage_client import AbstractStagingClient
 from feast_spark.pyspark.abc import (
-    BQ_SPARK_PACKAGE,
     BatchIngestionJob,
     BatchIngestionJobParameters,
     JobLauncher,
@@ -347,7 +346,7 @@ def offline_to_online_ingestion(
             job_type=OFFLINE_TO_ONLINE_JOB_TYPE,
             main_application_file=jar_s3_path,
             main_class=ingestion_job_params.get_class_name(),
-            packages=[BQ_SPARK_PACKAGE],
+            packages=[],
             jars=[],
             extra_metadata={},
             azure_credentials=self._get_azure_credentials(),
@@ -399,7 +398,7 @@ def schedule_offline_to_online_ingestion(
             job_type=OFFLINE_TO_ONLINE_JOB_TYPE,
             main_application_file=jar_s3_path,
             main_class=ingestion_job_params.get_class_name(),
-            packages=[BQ_SPARK_PACKAGE],
+            packages=[],
             jars=[],
             extra_metadata={},
             azure_credentials=self._get_azure_credentials(),
@@ -460,7 +459,7 @@ def start_stream_to_online_ingestion(
             job_type=STREAM_TO_ONLINE_JOB_TYPE,
             main_application_file=jar_s3_path,
             main_class=ingestion_job_params.get_class_name(),
-            packages=[BQ_SPARK_PACKAGE],
+            packages=[],
             jars=extra_jar_paths,
             extra_metadata={METADATA_JOBHASH: job_hash},
             azure_credentials=self._get_azure_credentials(),

diff --git a/python/feast_spark/pyspark/launchers/k8s/k8s_utils.py b/python/feast_spark/pyspark/launchers/k8s/k8s_utils.py
@@ -147,7 +147,8 @@ def _prepare_job_resource(
     _add_keys(job, ("spec", "sparkConf"), extra_metadata)
     _add_keys(job, ("spec", "sparkConf"), azure_credentials)
 
-    _append_items(job, ("spec", "deps", "packages"), packages)
+    if len(packages) > 1:
+        _append_items(job, ("spec", "deps", "packages"), packages)
     _append_items(job, ("spec", "deps", "jars"), jars)
 
     return job