[Feature Store] Block dataframe as source when running remotely #1517
Conversation
mlrun/feature_store/api.py
Outdated
| @@ -386,6 +394,10 @@ def ingest( | |||
| "featureset.spec.engine must be set to 'spark' to ingest with spark" | |||
| ) | |||
| if featureset.spec.engine == "spark": | |||
| if isinstance(source, pd.DataFrame): | |||
There was a problem hiding this comment.
Why? What about simple ingest with spark?
https://docs.mlrun.org/en/latest/feature-store/transformations.html#using-spark-execution-engine
There was a problem hiding this comment.
with a df as a source it would fail (and not with a clear error)
mlrun/feature_store/api.py
Outdated
| @@ -393,7 +393,7 @@ def ingest( | |||
| raise mlrun.errors.MLRunInvalidArgumentError( | |||
| "featureset.spec.engine must be set to 'spark' to ingest with spark" | |||
| ) | |||
| if featureset.spec.engine == "spark": | |||
| if featureset.spec.engine == "spark" and run_config is not None: | |||
There was a problem hiding this comment.
But now it'll just continue to a non-spark ingest.
It has to be added in the if isinstance in the next line.
mlrun/feature_store/api.py
Outdated
| @@ -311,6 +315,10 @@ def ingest( | |||
|
|
|||
| if mlrun_context: | |||
| # extract ingestion parameters from mlrun context | |||
| if isinstance(source, pd.DataFrame): | |||
| raise mlrun.errors.MLRunInvalidArgumentError( | |||
| "DataFrame source is illegal when running ingest with mlrun context" | |||
There was a problem hiding this comment.
"with mlrun context" won't mean anything to a user, explain it in their words
There was a problem hiding this comment.
mlrun context is a parameter that the user passes to ingest function
| try: | ||
| df = fs.ingest( | ||
| stocks_set, | ||
| stocks, | ||
| infer_options=fs.InferOptions.default(), | ||
| run_config=fs.RunConfig(local=True), | ||
| ) | ||
| assert False | ||
| except mlrun.errors.MLRunInvalidArgumentError: | ||
| pass |
There was a problem hiding this comment.
The way to assert something is failed is with pytest.raises(... we use it a lot, search the code base
| "measurements", entities=[fs.Entity("name")], engine="spark", | ||
| ) | ||
|
|
||
| try: |
Hedingber
changed the title
https://jira.iguazeng.com/browse/ML-1346