diff --git a/.github/workflows/compute-workflow-parameters.yml b/.github/workflows/compute-workflow-parameters.yml index 45abe477829..cd9cbbc83aa 100644 --- a/.github/workflows/compute-workflow-parameters.yml +++ b/.github/workflows/compute-workflow-parameters.yml @@ -15,6 +15,10 @@ on: description: "Comma-separated list of scenarios groups to run" type: string default: "" + excluded_scenarios: + description: "Comma-separated list of scenarios not to run" + type: string + default: "" parametric_job_count: description: "*EXPERIMENTAL* : How many jobs should be used to run PARAMETRIC scenario" default: 1 @@ -110,9 +114,10 @@ jobs: run: | source venv/bin/activate python utils/scripts/compute-workflow-parameters.py ${{ inputs.library }} \ + -t ${{ inputs.desired_execution_time }} \ -s "${{ inputs.scenarios }}" \ -g "${{ inputs.scenarios_groups }}" \ - -t ${{ inputs.desired_execution_time }} \ + --excluded-scenarios "${{ inputs.excluded_scenarios }}" \ --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ @@ -121,9 +126,10 @@ jobs: run: | source venv/bin/activate python utils/scripts/compute-workflow-parameters.py ${{ inputs.library }} \ + -t ${{ inputs.desired_execution_time }} \ -s "${{ inputs.scenarios }}" \ -g "${{ inputs.scenarios_groups }}" \ - -t ${{ inputs.desired_execution_time }} \ + --excluded-scenarios "${{ inputs.excluded_scenarios }}" \ --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index ab4e659a4ad..09830ab7264 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -16,6 +16,11 @@ on: default: '' required: false type: string + excluded_scenarios: + description: "Comma-separated list of scenarios not to run" + type: string + default: "" + required: false binaries_artifact: description: "Artifact name containing the binaries to test" default: '' @@ -90,6 +95,7 @@ jobs: library: ${{ inputs.library }} scenarios: ${{ inputs.scenarios }} scenarios_groups: ${{ inputs.scenarios_groups }} + excluded_scenarios: ${{ inputs.excluded_scenarios }} parametric_job_count: ${{ inputs.parametric_job_count }} desired_execution_time: ${{ inputs.desired_execution_time }} binaries_artifact: ${{ inputs.binaries_artifact }} diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index 7b12e4572ac..0912026601c 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -28,6 +28,7 @@ def __init__( library: str, scenarios: str, groups: str, + excluded_scenarios: str, parametric_job_count: int, desired_execution_time: int, explicit_binaries_artifact: str, @@ -53,7 +54,11 @@ def __init__( self.data["miscs"]["ci_environment"] = self.ci_environment - scenario_map = self._get_workflow_map(scenarios.split(","), groups.split(",")) + scenario_map = self._get_workflow_map( + scenario_names=scenarios.split(","), + scenario_group_names=groups.split(","), + excluded_scenario_names=excluded_scenarios.split(","), + ) self.data |= get_endtoend_definitions( library, scenario_map, self.ci_environment, desired_execution_time, maximum_parallel_jobs=256 @@ -140,7 +145,9 @@ def _export(data: str, output: str) -> None: print(data) @staticmethod - def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str]) -> dict: + def _get_workflow_map( + *, scenario_names: list[str], excluded_scenario_names: list[str], scenario_group_names: list[str] + ) -> dict: """Returns a dict where: * the key is the workflow identifier * the value is a list of scenarios to run, associated to the workflow @@ -148,8 +155,10 @@ def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str] result: dict[str, list[str]] = {} - scenarios_groups = [group.strip() for group in scenario_group_names if group.strip()] + # using a set to check that the user input is valid scenarios = {scenario.strip(): False for scenario in scenario_names if scenario.strip()} + scenarios_groups = [group.strip() for group in scenario_group_names if group.strip()] + excluded_scenarios = [scenario.strip() for scenario in excluded_scenario_names if scenario.strip()] for group in scenarios_groups: try: @@ -158,23 +167,29 @@ def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str] raise ValueError(f"Valid groups are: {[item.value for item in ScenarioGroup]}") from e for scenario in get_all_scenarios(): + # set the value to true to save that the value exists + # a final loop will look for false values and report an error if any + scenarios[scenario.name] = True + # TODO change the variable "github_workflow" to "ci_workflow" in the scenario object if not scenario.github_workflow: - scenarios[scenario.name] = True # won't be executed, but it exists continue if scenario.github_workflow not in result: result[scenario.github_workflow] = [] + if scenario.name in excluded_scenarios: + continue + if scenario.name in scenarios: result[scenario.github_workflow].append(scenario.name) - scenarios[scenario.name] = True - - for group in scenarios_groups: - if ScenarioGroup(group) in scenario.scenario_groups: - result[scenario.github_workflow].append(scenario.name) - break + else: + for group in scenarios_groups: + if ScenarioGroup(group) in scenario.scenario_groups: + result[scenario.github_workflow].append(scenario.name) + break + # check that all scenarios provided by the user are valid for scenario_name, found in scenarios.items(): if not found: raise ValueError(f"Scenario {scenario_name} does not exists") @@ -216,6 +231,7 @@ def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str] parser.add_argument("--scenarios", "-s", type=str, help="Scenarios to run", default="") parser.add_argument("--groups", "-g", type=str, help="Scenario groups to run", default="") + parser.add_argument("--excluded-scenarios", type=str, help="Scenarios to excluded", default="") # how long the workflow is expected to run parser.add_argument( @@ -257,6 +273,7 @@ def _get_workflow_map(scenario_names: list[str], scenario_group_names: list[str] library=args.library, scenarios=args.scenarios, groups=args.groups, + excluded_scenarios=args.excluded_scenarios, parametric_job_count=args.parametric_job_count, desired_execution_time=args.desired_execution_time, explicit_binaries_artifact=args.explicit_binaries_artifact,