From 530b7cdc4820edcb90da57d251cfd0cbfa2b5828 Mon Sep 17 00:00:00 2001 From: Gal Topper Date: Wed, 20 Mar 2024 17:10:08 +0800 Subject: [PATCH] [Datastore] Fix Spark read path in `CSVSource` (#5307) [ML-5896](https://iguazio.atlassian.net/browse/ML-5896) Introduced in #5158, first appearing in v1.7.0-rc4. --- mlrun/datastore/sources.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mlrun/datastore/sources.py b/mlrun/datastore/sources.py index e4eb74f891b..64b092723f5 100644 --- a/mlrun/datastore/sources.py +++ b/mlrun/datastore/sources.py @@ -204,11 +204,11 @@ def to_step(self, key_field=None, time_field=None, context=None): ) def get_spark_options(self): - store, path, url = mlrun.store_manager.get_or_create_store(self.path) + store, path, _ = mlrun.store_manager.get_or_create_store(self.path) spark_options = store.get_spark_options() spark_options.update( { - "path": url, + "path": store.spark_url + path, "format": "csv", "header": "true", "inferSchema": "true", @@ -357,7 +357,7 @@ def to_step( ) def get_spark_options(self): - store, path, url = mlrun.store_manager.get_or_create_store(self.path) + store, path, _ = mlrun.store_manager.get_or_create_store(self.path) spark_options = store.get_spark_options() spark_options.update( {