Support list as pipeline parameter expression

examples/pipeline-with-parameters-example.yaml

@@ -15,6 +15,12 @@
         type: string
       - name: param0-flag
         type: flag
+      - name: listings
+        default: [all_harbors, all_uploads, all_uploads2]
+        type: string
+        multiple: separate
+        multiple-separator: ","
+        optional: false
 
 - pipeline:
     name: Example Pipeline with Parameters
@@ -45,6 +51,11 @@
         targets:
           - train.parameter.param0-flag
         default: false
+      - name: listings
+        targets:
+          - train.parameter.listings
+          - train_parallel.parameter.listings
+        default: [abc, def, ghi]
     nodes:
       - name: train
         step: train_step

tests/__snapshots__/test_validation.ambr

@@ -104,10 +104,10 @@
 # name: test_bad_examples_cli[step-missing-required-properties.yaml]
   '''
   >>> step-missing-required-properties.yaml
-  error:   Oneof validation on step.command.oneOf: 8 is not valid under any of the given schemas (1.step.command)
-  ------------------------------------------------------------
   error:   Type validation on step.image.type: True is not of type 'string' (0.step.image)
   ------------------------------------------------------------
+  error:   Oneof validation on step.command.oneOf: 8 is not valid under any of the given schemas (1.step.command)
+  ------------------------------------------------------------
   error:   Required validation on step.required: 'command' is a required property (0.step)
   ------------------------------------------------------------
   error:   Required validation on step.required: 'name' is a required property (0.step)

tests/test_pipeline_conversion.py

@@ -73,5 +73,38 @@ def test_pipeline_parameter_conversion(pipeline_with_parameters_config):
             # When pipeline parameter has no default value, the expression should be empty
             parameter_config = next(param for param in pipe.parameters if param.name == parameter_name)
             expression_value = parameter_config.default if parameter_config.default is not None else ""
-            assert parameter["expression"] == expression_value
-            assert type(parameter["expression"]) == type(expression_value)
+            # When pipeline parameter is list it should be converted to a variant parameter
+            if isinstance(expression_value, list):
+                assert parameter["expression"] == {
+                    "style": "single",
+                    "rules": {"value": expression_value},
+                }
+            else:
+                assert parameter["expression"] == expression_value
+                assert type(parameter["expression"]) is type(expression_value)
+
+
+def test_pipeline_parameter_conversion_with_args(pipeline_with_parameters_config):
+    pipeline = pipeline_with_parameters_config.pipelines["Example Pipeline with Parameters"]
+    args = {
+        "param0-float": 1.5,
+        "param0-int": 3,
+        "param0-string": "iceberg",
+        "param0-flag": True,
+        "listings": ["msg", "pfa"],
+    }
+    result = PipelineConverter(
+        config=pipeline_with_parameters_config,
+        commit_identifier="latest",
+        parameter_arguments=args,
+    ).convert_pipeline(pipeline)
+    params = result["parameters"]
+
+    assert params["param0-float"]["expression"] == 1.5
+    assert params["param0-int"]["expression"] == 3
+    assert params["param0-string"]["expression"] == "iceberg"
+    assert params["param0-flag"]["expression"]
+    assert params["listings"]["expression"] == {
+        "style": "single",
+        "rules": {"value": ["msg", "pfa"]},
+    }

valohai_yaml/objs/pipelines/pipeline_parameter.py

@@ -19,7 +19,7 @@ def __init__(
         name: str,
         targets: Optional[Union[List[str], str]] = None,
         value: Optional[str] = None,
-        default: Optional[str] = None,
+        default: Optional[Union[List[str], str]] = None,
         category: Optional[str] = None,
         description: Optional[str] = None,
     ) -> None:

valohai_yaml/pipelines/conversion.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, TypedDict, Union
+from typing import Any, Dict, List, Optional, TypedDict, Union
 
 from valohai_yaml.objs import (
     Config,
@@ -14,6 +14,16 @@
 ConvertedObject = Dict[str, Any]
 
 
+class VariantExpression(TypedDict):
+    """Variant expression template."""
+
+    style: str
+    rules: Dict[str, Any]
+
+
+ExpressionValue = Union[str, int, bool, float, VariantExpression]
+
+
 class ConvertedPipeline(TypedDict):
     """TypedDict for converted Pipeline object."""
 
@@ -30,9 +40,11 @@ def __init__(
         *,
         config: Config,
         commit_identifier: str,
+        parameter_arguments: Optional[Dict[str, Union[str, list]]] = None,
     ) -> None:
         self.config = config
         self.commit_identifier = commit_identifier
+        self.parameter_arguments = parameter_arguments or {}
 
     def convert_pipeline(self, pipeline: Pipeline) -> ConvertedPipeline:
         return {
@@ -43,9 +55,17 @@ def convert_pipeline(self, pipeline: Pipeline) -> ConvertedPipeline:
 
     def convert_parameter(self, parameter: PipelineParameter) -> ConvertedObject:
         """Convert a pipeline parameter to a config-expression payload."""
+        param_value: Union[ExpressionValue, List[str]]
+        if parameter.name in self.parameter_arguments:
+            param_value = self.parameter_arguments[parameter.name]
+        elif parameter.default is not None:
+            param_value = parameter.default
+        else:
+            param_value = ""
+
         return {
             "config": {**parameter.serialize()},
-            "expression": parameter.default if parameter.default is not None else "",
+            "expression": self.convert_expression(param_value),
         }
 
     def convert_node(self, node: Node) -> ConvertedObject:
@@ -102,3 +122,11 @@ def convert_executionlike_node(
         }
 
         return node_data
+
+    def convert_expression(self, expression: Union[ExpressionValue, list]) -> ExpressionValue:
+        if isinstance(expression, list):
+            return VariantExpression(
+                style="single",
+                rules={"value": expression},
+            )
+        return expression