[cpp] Add an option to exclude some scenarios in system-tests.yml

.github/workflows/compute-workflow-parameters.yml

@@ -15,6 +15,10 @@ on:
         description: "Comma-separated list of scenarios groups to run"
         type: string
         default: ""
+      excluded_scenarios:
+        description: "Comma-separated list of scenarios not to run"
+        type: string
+        default: ""
       parametric_job_count:
         description: "*EXPERIMENTAL* : How many jobs should be used to run PARAMETRIC scenario"
         default: 1
@@ -110,9 +114,10 @@ jobs:
       run: |
         source venv/bin/activate
         python utils/scripts/compute-workflow-parameters.py ${{ inputs.library }} \
+          -t ${{ inputs.desired_execution_time }} \
           -s "${{ inputs.scenarios }}" \
           -g "${{ inputs.scenarios_groups }}" \
-          -t ${{ inputs.desired_execution_time }} \
+          --excluded-scenarios "${{ inputs.excluded_scenarios }}" \
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
@@ -121,9 +126,10 @@ jobs:
       run: |
         source venv/bin/activate
         python utils/scripts/compute-workflow-parameters.py ${{ inputs.library }} \
+          -t ${{ inputs.desired_execution_time }} \
           -s "${{ inputs.scenarios }}" \
           -g "${{ inputs.scenarios_groups }}" \
-          -t ${{ inputs.desired_execution_time }} \
+          --excluded-scenarios "${{ inputs.excluded_scenarios }}" \
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \

.github/workflows/system-tests.yml

@@ -16,6 +16,11 @@ on:
         default: ''
         required: false
         type: string
+      excluded_scenarios:
+        description: "Comma-separated list of scenarios not to run"
+        type: string
+        default: ""
+        required: false
       binaries_artifact:
         description: "Artifact name containing the binaries to test"
         default: ''
@@ -90,6 +95,7 @@ jobs:
       library: ${{ inputs.library }}
       scenarios: ${{ inputs.scenarios }}
       scenarios_groups: ${{ inputs.scenarios_groups }}
+      excluded_scenarios: ${{ inputs.excluded_scenarios }}
       parametric_job_count: ${{ inputs.parametric_job_count }}
       desired_execution_time: ${{ inputs.desired_execution_time }}
       binaries_artifact: ${{ inputs.binaries_artifact }}

utils/scripts/compute-workflow-parameters.py

@@ -28,6 +28,7 @@ def __init__(
         library: str,
         scenarios: str,
         groups: str,
+        excluded_scenarios: str,
         parametric_job_count: int,
         desired_execution_time: int,
         explicit_binaries_artifact: str,
@@ -53,7 +54,11 @@ def __init__(
 
         self.data["miscs"]["ci_environment"] = self.ci_environment
 
-        scenario_map = self._get_workflow_map(scenarios.split(","), groups.split(","))
+        scenario_map = self._get_workflow_map(
+            scenario_names=scenarios.split(","),
+            scenario_group_names=groups.split(","),
+            excluded_scenario_names=excluded_scenarios.split(","),
+        )
 
         self.data |= get_endtoend_definitions(
             library, scenario_map, self.ci_environment, desired_execution_time, maximum_parallel_jobs=256
@@ -140,16 +145,20 @@ def _export(data: str, output: str) -> None:
             print(data)
 
     @staticmethod
-    def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str]) -> dict:
+    def _get_workflow_map(
+        *, scenario_names: list[str], excluded_scenario_names: list[str], scenario_group_names: list[str]
+    ) -> dict:
         """Returns a dict where:
         * the key is the workflow identifier
         * the value is a list of scenarios to run, associated to the workflow
         """
 
         result: dict[str, list[str]] = {}
 
-        scenarios_groups = [group.strip() for group in scenario_group_names if group.strip()]
+        # using a set to check that the user input is valid
         scenarios = {scenario.strip(): False for scenario in scenario_names if scenario.strip()}
+        scenarios_groups = [group.strip() for group in scenario_group_names if group.strip()]
+        excluded_scenarios = [scenario.strip() for scenario in excluded_scenario_names if scenario.strip()]
 
         for group in scenarios_groups:
             try:
@@ -158,23 +167,29 @@ def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str]
                 raise ValueError(f"Valid groups are: {[item.value for item in ScenarioGroup]}") from e
 
         for scenario in get_all_scenarios():
+            # set the value to true to save that the value exists
+            # a final loop will look for false values and report an error if any
+            scenarios[scenario.name] = True
+
             # TODO change the variable "github_workflow" to "ci_workflow" in the scenario object
             if not scenario.github_workflow:
-                scenarios[scenario.name] = True  # won't be executed, but it exists
                 continue
 
             if scenario.github_workflow not in result:
                 result[scenario.github_workflow] = []
 
+            if scenario.name in excluded_scenarios:
+                continue
+
             if scenario.name in scenarios:
                 result[scenario.github_workflow].append(scenario.name)
-                scenarios[scenario.name] = True
-
-            for group in scenarios_groups:
-                if ScenarioGroup(group) in scenario.scenario_groups:
-                    result[scenario.github_workflow].append(scenario.name)
-                    break
+            else:
+                for group in scenarios_groups:
+                    if ScenarioGroup(group) in scenario.scenario_groups:
+                        result[scenario.github_workflow].append(scenario.name)
+                        break
 
+        # check that all scenarios provided by the user are valid
         for scenario_name, found in scenarios.items():
             if not found:
                 raise ValueError(f"Scenario {scenario_name} does not exists")
@@ -216,6 +231,7 @@ def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str]
 
     parser.add_argument("--scenarios", "-s", type=str, help="Scenarios to run", default="")
     parser.add_argument("--groups", "-g", type=str, help="Scenario groups to run", default="")
+    parser.add_argument("--excluded-scenarios", type=str, help="Scenarios to excluded", default="")
 
     # how long the workflow is expected to run
     parser.add_argument(
@@ -257,6 +273,7 @@ def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str]
         library=args.library,
         scenarios=args.scenarios,
         groups=args.groups,
+        excluded_scenarios=args.excluded_scenarios,
         parametric_job_count=args.parametric_job_count,
         desired_execution_time=args.desired_execution_time,
         explicit_binaries_artifact=args.explicit_binaries_artifact,