Skip to content

Commit

Permalink
Support full combinatorics in gitlab-ci:mappings
Browse files Browse the repository at this point in the history
  • Loading branch information
@blue42u
blue42u committed Sep 13, 2022
1 parent 937b576 commit 212d894
Show file tree
Hide file tree
Showing 15 changed files with 432 additions and 401 deletions.
20 changes: 14 additions & 6 deletions lib/spack/spack/ci.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,14 @@ def _spec_matches(spec, match_string):
return spec.satisfies(match_string)


def _remove_attributes(src_dict, dest_dict):
if "tags" in src_dict and "tags" in dest_dict:
# For 'tags', we remove any tags that are listed for removal
for tag in src_dict["tags"]:
while tag in dest_dict["tags"]:
dest_dict["tags"].remove(tag)


def _copy_attributes(attrs_list, src_dict, dest_dict):
for runner_attr in attrs_list:
if runner_attr in src_dict:
Expand Down Expand Up @@ -429,19 +437,19 @@ def _find_matching_config(spec, gitlab_ci):

_copy_attributes(overridable_attrs, gitlab_ci, runner_attributes)

ci_mappings = gitlab_ci["mappings"]
for ci_mapping in ci_mappings:
matched = False
for ci_mapping in gitlab_ci["mappings"]:
for match_string in ci_mapping["match"]:
if _spec_matches(spec, match_string):
matched = True
if "remove-attributes" in ci_mapping:
_remove_attributes(ci_mapping["remove-attributes"], runner_attributes)
if "runner-attributes" in ci_mapping:
_copy_attributes(
overridable_attrs, ci_mapping["runner-attributes"], runner_attributes
)
return runner_attributes
else:
return None

return runner_attributes
return runner_attributes if matched else None


def _pkg_name_from_spec_label(spec_label):
Expand Down
10 changes: 10 additions & 0 deletions lib/spack/spack/schema/gitlab_ci.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,15 @@
"properties": runner_attributes_schema_items,
}

remove_attributes_schema = {
"type": "object",
"additionalProperties": False,
"required": ["tags"],
"properties": {
"tags": {"type": "array", "items": {"type": "string"}},
},
}


core_shared_properties = union_dicts(
runner_attributes_schema_items,
Expand Down Expand Up @@ -93,6 +102,7 @@
"type": "string",
},
},
"remove-attributes": remove_attributes_schema,
"runner-attributes": runner_selector_schema,
},
},
Expand Down
9 changes: 9 additions & 0 deletions lib/spack/spack/test/cmd/ci.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1404,6 +1404,14 @@ def test_ci_generate_override_runner_attrs(
- dependency-install
- match:
- a
runner-attributes:
tags:
- bad-a
- match:
- a
remove-attributes:
tags:
- bad-a
runner-attributes:
tags:
- specific-a
Expand Down Expand Up @@ -1456,6 +1464,7 @@ def test_ci_generate_override_runner_attrs(
assert len(the_elt["tags"]) == 2
assert "specific-a" in the_elt["tags"]
assert "toplevel" in the_elt["tags"]
assert "bad-a" not in the_elt["tags"]
assert len(the_elt["before_script"]) == 1
assert the_elt["before_script"][0] == "custom pre step one"
assert len(the_elt["script"]) == 1
Expand Down
25 changes: 13 additions & 12 deletions share/spack/gitlab/cloud_pipelines/stacks/aws-ahug-aarch64/spack.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,16 +253,11 @@ spack:

image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
mappings:
- match:
- llvm
- llvm-amdgpu
- paraview
- match: ['os=amzn2']
runner-attributes:
tags: [ "spack", "huge", "aarch64" ]
tags: ["spack", "aarch64"]
variables:
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
KUBERNETES_MEMORY_REQUEST: 42G
CI_JOB_SIZE: "default"


- match:
Expand Down Expand Up @@ -293,17 +288,23 @@ spack:
- vtk-m
- warpx
runner-attributes:
tags: [ "spack", "large", "aarch64" ]
tags: [ "large" ]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G

- match: ['os=amzn2']

- match:
- llvm
- llvm-amdgpu
- paraview
runner-attributes:
tags: ["spack", "aarch64"]
tags: [ "huge" ]
variables:
CI_JOB_SIZE: "default"
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
KUBERNETES_MEMORY_REQUEST: 42G

broken-specs-url: "s3://spack-binaries/broken-specs"

Expand Down
25 changes: 13 additions & 12 deletions share/spack/gitlab/cloud_pipelines/stacks/aws-ahug/spack.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,16 +254,11 @@ spack:

image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
mappings:
- match:
- llvm
- llvm-amdgpu
- paraview
- match: ['os=amzn2']
runner-attributes:
tags: [ "spack", "huge", "x86_64_v4" ]
tags: ["spack", "x86_64_v4"]
variables:
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
KUBERNETES_MEMORY_REQUEST: 42G
CI_JOB_SIZE: "default"


- match:
Expand Down Expand Up @@ -294,17 +289,23 @@ spack:
- vtk-m
- warpx
runner-attributes:
tags: [ "spack", "large", "x86_64_v4" ]
tags: [ "large" ]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G

- match: ['os=amzn2']

- match:
- llvm
- llvm-amdgpu
- paraview
runner-attributes:
tags: ["spack", "x86_64_v4"]
tags: [ "huge" ]
variables:
CI_JOB_SIZE: "default"
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
KUBERNETES_MEMORY_REQUEST: 42G

broken-specs-url: "s3://spack-binaries/broken-specs"

Expand Down
24 changes: 13 additions & 11 deletions share/spack/gitlab/cloud_pipelines/stacks/aws-isc-aarch64/spack.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,16 +171,11 @@ spack:

image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
mappings:
- match:
- llvm
- llvm-amdgpu
- paraview
- match: ['os=amzn2']
runner-attributes:
tags: [ "spack", "huge", "aarch64" ]
tags: ["spack", "aarch64"]
variables:
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 15000m
KUBERNETES_MEMORY_REQUEST: 62G
CI_JOB_SIZE: "default"


- match:
Expand Down Expand Up @@ -226,11 +221,18 @@ spack:
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G

- match: ['os=amzn2']

- match:
- llvm
- llvm-amdgpu
- paraview
runner-attributes:
tags: ["spack", "aarch64"]
tags: [ "spack", "huge", "aarch64" ]
variables:
CI_JOB_SIZE: "default"
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 15000m
KUBERNETES_MEMORY_REQUEST: 62G


broken-specs-url: "s3://spack-binaries/broken-specs"

Expand Down
27 changes: 14 additions & 13 deletions share/spack/gitlab/cloud_pipelines/stacks/aws-isc/spack.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,17 +172,11 @@ spack:

image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2022-03-21", "entrypoint": [""] }
mappings:
- match:
- llvm
- llvm-amdgpu
- pango
- paraview
- match: ['os=amzn2']
runner-attributes:
tags: [ "spack", "huge", "x86_64_v4" ]
tags: ["spack", "x86_64_v4"]
variables:
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
KUBERNETES_MEMORY_REQUEST: 42G
CI_JOB_SIZE: "default"


- match:
Expand Down Expand Up @@ -222,17 +216,24 @@ spack:
- wrf
- wxwidgets
runner-attributes:
tags: [ "spack", "large", "x86_64_v4" ]
tags: [ "large" ]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G

- match: ['os=amzn2']

- match:
- llvm
- llvm-amdgpu
- pango
- paraview
runner-attributes:
tags: ["spack", "x86_64_v4"]
tags: [ "huge" ]
variables:
CI_JOB_SIZE: "default"
CI_JOB_SIZE: huge
KUBERNETES_CPU_REQUEST: 11000m
KUBERNETES_MEMORY_REQUEST: 42G

broken-specs-url: "s3://spack-binaries/broken-specs"

Expand Down
44 changes: 22 additions & 22 deletions share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,26 +48,11 @@ spack:

mappings:
- match:
- cmake
runner-attributes:
tags: [ "spack", "large", "x86_64"]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G

- match:
- curl
- gettext
- mpich
- openjpeg
- sqlite
- 'os=ubuntu18.04'
runner-attributes:
tags: [ "spack", "medium", "x86_64" ]
tags: ["spack", "x86_64"]
variables:
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_MEMORY_REQUEST: "4G"
CI_JOB_SIZE: "default"

- match:
- bzip2
Expand All @@ -87,18 +72,33 @@ spack:
- xz
- zlib
runner-attributes:
tags: [ "spack", "medium", "x86_64" ]
tags: [ "medium" ]
variables:
CI_JOB_SIZE: "small"
KUBERNETES_CPU_REQUEST: "500m"
KUBERNETES_MEMORY_REQUEST: "500M"

- match:
- 'os=ubuntu18.04'
- curl
- gettext
- mpich
- openjpeg
- sqlite
runner-attributes:
tags: ["spack", "x86_64"]
tags: [ "medium" ]
variables:
CI_JOB_SIZE: "default"
CI_JOB_SIZE: "medium"
KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_MEMORY_REQUEST: "4G"

- match:
- cmake
runner-attributes:
tags: [ "large" ]
variables:
CI_JOB_SIZE: large
KUBERNETES_CPU_REQUEST: 8000m
KUBERNETES_MEMORY_REQUEST: 12G

broken-specs-url: "s3://spack-binaries/broken-specs"

Expand Down

0 comments on commit 212d894

Please sign in to comment.