From 281027af2463c29b00016775e3770ed6e313ee1d Mon Sep 17 00:00:00 2001 From: niklasvm Date: Thu, 8 Sep 2022 08:31:58 +0200 Subject: [PATCH 1/3] fix: Local staging location provision Signed-off-by: niklasvm --- .../infra/offline_stores/contrib/spark_offline_store/spark.py | 2 +- .../contrib/spark_offline_store/tests/data_source.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index 0a4ec05c23d..21d89a37d9e 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -368,7 +368,7 @@ def to_remote_storage(self) -> List[str]: sdf: pyspark.sql.DataFrame = self.to_spark_df() - if self._config.offline_store.staging_location.startswith("file://"): + if self._config.offline_store.staging_location.startswith("/"): local_file_staging_location = os.path.abspath( self._config.offline_store.staging_location ) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py index 64a2a01cee4..71c07b20c27 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py @@ -58,8 +58,8 @@ def create_offline_store_config(self): self.spark_offline_store_config = SparkOfflineStoreConfig() self.spark_offline_store_config.type = "spark" self.spark_offline_store_config.spark_conf = self.spark_conf - self.spark_offline_store_config.staging_location = "file://" + str( - tempfile.TemporaryDirectory() + self.spark_offline_store_config.staging_location = ( + tempfile.TemporaryDirectory().name ) self.spark_offline_store_config.region = "eu-west-1" return self.spark_offline_store_config From cfcb7c0dab48c82af0e4a4df7e5ceef042cca269 Mon Sep 17 00:00:00 2001 From: niklasvm Date: Thu, 8 Sep 2022 08:35:35 +0200 Subject: [PATCH 2/3] fix import Signed-off-by: niklasvm --- .../infra/offline_stores/contrib/spark_offline_store/spark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index 21d89a37d9e..4a50995ee43 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -15,7 +15,7 @@ from pyspark import SparkConf from pyspark.sql import SparkSession from pytz import utc -from sdk.python.feast.infra.utils import aws_utils +from feast.infra.utils import aws_utils from feast import FeatureView, OnDemandFeatureView from feast.data_source import DataSource From db6ac041cb05732e5a87e4b663f5a754a7d0b68d Mon Sep 17 00:00:00 2001 From: niklasvm Date: Thu, 8 Sep 2022 08:37:02 +0200 Subject: [PATCH 3/3] linting fix Signed-off-by: niklasvm --- .../infra/offline_stores/contrib/spark_offline_store/spark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index 4a50995ee43..44b573d66b0 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -15,7 +15,6 @@ from pyspark import SparkConf from pyspark.sql import SparkSession from pytz import utc -from feast.infra.utils import aws_utils from feast import FeatureView, OnDemandFeatureView from feast.data_source import DataSource @@ -32,6 +31,7 @@ RetrievalMetadata, ) from feast.infra.registry.registry import Registry +from feast.infra.utils import aws_utils from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage from feast.type_map import spark_schema_to_np_dtypes