From 5e3d6fe12bcf99da45ec4260fe93da45b2410acc Mon Sep 17 00:00:00 2001 From: blue42u Date: Tue, 23 Aug 2022 07:38:29 -0500 Subject: [PATCH] Add a custom HPCToolkit stack --- .../gitlab/cloud_pipelines/.gitlab-ci.yml | 300 ++++++++++++++++++ .../stacks/hpctoolkit/spack.yaml | 291 +++++++++++++++++ 2 files changed, 591 insertions(+) create mode 100644 share/spack/gitlab/cloud_pipelines/stacks/hpctoolkit/spack.yaml diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 8098d6f5febc0a..3f202f25c48c79 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -59,18 +59,28 @@ default: extends: [ ".generate" ] tags: ["spack", "aws", "public", "medium", "aarch64"] +.generate-ppc64le: + extends: .generate + tags: ["spack", "public", "medium", "ppc64le"] + .pr-generate: extends: [ ".pr", ".generate" ] .pr-generate-aarch64: extends: [ ".pr", ".generate-aarch64" ] +.pr-generate-ppc64le: + extends: [ ".pr", ".generate-ppc64le" ] + .protected-generate: extends: [ ".protected", ".generate" ] .protected-generate-aarch64: extends: [ ".protected", ".generate-aarch64" ] +.protected-generate-ppc64le: + extends: [ ".protected", ".generate-ppc64le" ] + .build: stage: build @@ -872,3 +882,293 @@ ml-rocm-protected-build: needs: - artifacts: True job: ml-rocm-protected-generate + +####################################### +# HPCToolkit Multi-OS and Multi-Arch +####################################### +.hpctoolkit: + variables: + SPACK_CI_STACK_NAME: hpctoolkit +.hpctoolkit-centos7: + extends: .hpctoolkit + image: ghcr.io/spack/e4s-centos-7@sha256:e9c5b1d3dd6bc0c202cc8d8df69c4bb0da4ec93d060a8f213a1c4f541f09e6ae +.hpctoolkit-fedora36: + extends: .hpctoolkit + image: ghcr.io/spack/e4s-fedora-36@sha256:a70ca93bdf4795259ed7373cc507a9fcca0b8d92f068904705ecb3e8d93d11e4 +.hpctoolkit-leap15: + extends: .hpctoolkit + image: ghcr.io/spack/e4s-leap-15@sha256:4fe84fa2bbfc46ce451367a5201dfd48cfa289afce2b1875129a31016eef3d69 +.hpctoolkit-almalinux8: + extends: .hpctoolkit + image: ghcr.io/spack/e4s-almalinux-8@sha256:780286ddcccdd6a69d8a8d3aefddfec49b1d04c6f58d77fb340e0af803928b70 +.hpctoolkit-ubuntu20.04: + extends: .hpctoolkit + image: ghcr.io/spack/e4s-ubuntu-20.04@sha256:7cabb40c05e3bb6bbe709aad49429179e812187f0697cbb54b939dcb667c9f5f + +hpctoolkit-centos7-pr-generate: + extends: [ ".hpctoolkit-centos7", ".pr-generate"] +hpctoolkit-fedora36-pr-generate: + extends: [ ".hpctoolkit-fedora36", ".pr-generate"] +hpctoolkit-fedora36-pr-generate-aarch64: + extends: [ ".hpctoolkit-fedora36", ".pr-generate-aarch64"] +# hpctoolkit-fedora36-pr-generate-ppc64le: +# extends: [ ".hpctoolkit-fedora36", ".pr-generate-ppc64le"] +hpctoolkit-leap15-pr-generate: + extends: [ ".hpctoolkit-leap15", ".pr-generate"] +hpctoolkit-almalinux8-pr-generate: + extends: [ ".hpctoolkit-almalinux8", ".pr-generate"] +hpctoolkit-almalinux8-pr-generate-aarch64: + extends: [ ".hpctoolkit-almalinux8", ".pr-generate-aarch64"] +hpctoolkit-almalinux8-pr-generate-ppc64le: + extends: [ ".hpctoolkit-almalinux8", ".pr-generate-ppc64le"] +hpctoolkit-ubuntu20.04-pr-generate: + extends: [ ".hpctoolkit-ubuntu20.04", ".pr-generate"] +hpctoolkit-ubuntu20.04-pr-generate-aarch64: + extends: [ ".hpctoolkit-ubuntu20.04", ".pr-generate-aarch64"] +hpctoolkit-ubuntu20.04-pr-generate-ppc64le: + extends: [ ".hpctoolkit-ubuntu20.04", ".pr-generate-ppc64le"] + +hpctoolkit-centos7-protected-generate: + extends: [ ".hpctoolkit-centos7", ".protected-generate"] +hpctoolkit-fedora36-protected-generate: + extends: [ ".hpctoolkit-fedora36", ".protected-generate"] +hpctoolkit-fedora36-protected-generate-aarch64: + extends: [ ".hpctoolkit-fedora36", ".protected-generate-aarch64"] +# hpctoolkit-fedora36-protected-generate-ppc64le: +# extends: [ ".hpctoolkit-fedora36", ".protected-generate-ppc64le"] +hpctoolkit-leap15-protected-generate: + extends: [ ".hpctoolkit-leap15", ".protected-generate"] +hpctoolkit-almalinux8-protected-generate: + extends: [ ".hpctoolkit-almalinux8", ".protected-generate"] +hpctoolkit-almalinux8-protected-generate-aarch64: + extends: [ ".hpctoolkit-almalinux8", ".protected-generate-aarch64"] +hpctoolkit-almalinux8-protected-generate-ppc64le: + extends: [ ".hpctoolkit-almalinux8", ".protected-generate-ppc64le"] +hpctoolkit-ubuntu20.04-protected-generate: + extends: [ ".hpctoolkit-ubuntu20.04", ".protected-generate"] +hpctoolkit-ubuntu20.04-protected-generate-aarch64: + extends: [ ".hpctoolkit-ubuntu20.04", ".protected-generate-aarch64"] +hpctoolkit-ubuntu20.04-protected-generate-ppc64le: + extends: [ ".hpctoolkit-ubuntu20.04", ".protected-generate-ppc64le"] + +hpctoolkit-centos7-pr-build: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-centos7-pr-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-centos7-pr-generate +hpctoolkit-fedora36-pr-build: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-fedora36-pr-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-fedora36-pr-generate +hpctoolkit-fedora36-pr-build-aarch64: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-fedora36-pr-generate-aarch64 + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-fedora36-pr-generate-aarch64 +# hpctoolkit-fedora36-pr-build-ppc64le: +# extends: [ ".hpctoolkit", ".pr-build" ] +# trigger: +# include: +# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml +# job: hpctoolkit-fedora36-pr-generate-ppc64le +# strategy: depend +# needs: +# - artifacts: true +# job: hpctoolkit-fedora36-pr-generate-ppc64le +hpctoolkit-leap15-pr-build: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-leap15-pr-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-leap15-pr-generate +hpctoolkit-almalinux8-pr-build: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-almalinux8-pr-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-almalinux8-pr-generate +hpctoolkit-almalinux8-pr-build-aarch64: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-almalinux8-pr-generate-aarch64 + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-almalinux8-pr-generate-aarch64 +hpctoolkit-almalinux8-pr-build-ppc64le: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-almalinux8-pr-generate-ppc64le + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-almalinux8-pr-generate-ppc64le +hpctoolkit-ubuntu20.04-pr-build: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-ubuntu20.04-pr-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-ubuntu20.04-pr-generate +hpctoolkit-ubuntu20.04-pr-build-aarch64: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-ubuntu20.04-pr-generate-aarch64 + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-ubuntu20.04-pr-generate-aarch64 +hpctoolkit-ubuntu20.04-pr-build-ppc64le: + extends: [ ".hpctoolkit", ".pr-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-ubuntu20.04-pr-generate-ppc64le + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-ubuntu20.04-pr-generate-ppc64le + +hpctoolkit-centos7-protected-build: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-centos7-protected-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-centos7-protected-generate +hpctoolkit-fedora36-protected-build: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-fedora36-protected-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-fedora36-protected-generate +hpctoolkit-fedora36-protected-build-aarch64: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-fedora36-protected-generate-aarch64 + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-fedora36-protected-generate-aarch64 +# hpctoolkit-fedora36-protected-build-ppc64le: +# extends: [ ".hpctoolkit", ".protected-build" ] +# trigger: +# include: +# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml +# job: hpctoolkit-fedora36-protected-generate-ppc64le +# strategy: depend +# needs: +# - artifacts: true +# job: hpctoolkit-fedora36-protected-generate-ppc64le +hpctoolkit-leap15-protected-build: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-leap15-protected-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-leap15-protected-generate +hpctoolkit-almalinux8-protected-build: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-almalinux8-protected-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-almalinux8-protected-generate +hpctoolkit-almalinux8-protected-build-aarch64: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-almalinux8-protected-generate-aarch64 + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-almalinux8-protected-generate-aarch64 +hpctoolkit-almalinux8-protected-build-ppc64le: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-almalinux8-protected-generate-ppc64le + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-almalinux8-protected-generate-ppc64le +hpctoolkit-ubuntu20.04-protected-build: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-ubuntu20.04-protected-generate + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-ubuntu20.04-protected-generate +hpctoolkit-ubuntu20.04-protected-build-aarch64: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-ubuntu20.04-protected-generate-aarch64 + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-ubuntu20.04-protected-generate-aarch64 +hpctoolkit-ubuntu20.04-protected-build-ppc64le: + extends: [ ".hpctoolkit", ".protected-build" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: hpctoolkit-ubuntu20.04-protected-generate-ppc64le + strategy: depend + needs: + - artifacts: true + job: hpctoolkit-ubuntu20.04-protected-generate-ppc64le diff --git a/share/spack/gitlab/cloud_pipelines/stacks/hpctoolkit/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/hpctoolkit/spack.yaml new file mode 100644 index 00000000000000..3c79623469f456 --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/hpctoolkit/spack.yaml @@ -0,0 +1,291 @@ +spack: + view: false + + concretizer: + reuse: false + unify: false + + config: + concretizer: clingo + install_tree: + root: /tmp/home/software/spack + padded_length: 512 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + + definitions: + - when: arch.satisfies('target=x86_64_v3:') + target: [target=x86_64_v3] + - when: arch.satisfies('target=aarch64:') + target: [target=aarch64] + - when: arch.satisfies('target=ppc64le:') + target: [target=ppc64le] + + - gpus: + # No GPU support + - ~cuda ~rocm ~level_zero + # CUDA (Nvidia) support + - +cuda ~rocm ~level_zero + ^cuda@11 + # Level Zero (Intel) support + # GTPin support relies on external packages and so is disabled + - +level_zero ~gtpin ~cuda ~rocm + - when: not arch.satisfies('target=aarch64:') + gpus: + # CUDA (Nvidia) support + # CUDA on ARM is only supported in CUDA 11 and following + - +cuda ~rocm ~level_zero + ^cuda@10.2 + - when: arch.satisfies('target=x86_64:') + gpus: + # ROCm (AMD) support + # ROCm currently only supports x86 + - +rocm ~cuda ~level_zero + ^hip@5.2 ^hsa-rocr-dev@5.2 ^rocprofiler-dev@5.2 ^roctracer-dev@5.2 + + specs: + - matrix: + - &core ['hpctoolkit @2022.10.01: ~mpi +viewer ~opencl +papi'] + - [$gpus] + - [$target] + + mirrors: + mirror: s3://spack-binaries/develop/hpctoolkit + + gitlab-ci: + script: + - . "./share/spack/setup-env.sh" + - spack --version + - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi + - spack -d ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + + variables: + CI_JOB_SIZE: default + + broken-specs-url: "s3://spack-binaries/broken-specs" + + service-job-attributes: + before_script: + - . "./share/spack/setup-env.sh" + - spack --version + image: { "name": "ghcr.io/spack/ubuntu-jammy:latest", "entrypoint": [""] } + tags: ["spack", "public", "x86_64_v3"] + + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + + match_behavior: merge + mappings: + - match: ['os=centos7'] + runner-attributes: + image: ghcr.io/spack/e4s-centos-7@sha256:e9c5b1d3dd6bc0c202cc8d8df69c4bb0da4ec93d060a8f213a1c4f541f09e6ae + tags: [] + - match: ['os=fedora36'] + runner-attributes: + image: ghcr.io/spack/e4s-fedora-36@sha256:a70ca93bdf4795259ed7373cc507a9fcca0b8d92f068904705ecb3e8d93d11e4 + tags: [] + - match: ['os=opensuse_leap15'] + runner-attributes: + image: ghcr.io/spack/e4s-leap-15@sha256:4fe84fa2bbfc46ce451367a5201dfd48cfa289afce2b1875129a31016eef3d69 + tags: [] + - match: ['os=almalinux8'] + runner-attributes: + image: ghcr.io/spack/e4s-almalinux-8@sha256:780286ddcccdd6a69d8a8d3aefddfec49b1d04c6f58d77fb340e0af803928b70 + tags: [] + - match: ['os=ubuntu20.04'] + runner-attributes: + image: ghcr.io/spack/e4s-ubuntu-20.04@sha256:7cabb40c05e3bb6bbe709aad49429179e812187f0697cbb54b939dcb667c9f5f + tags: [] + + - match: ['target=:x86_64_v3'] + runner-attributes: + tags: [x86_64_v3] + - match: ['target=:graviton'] + runner-attributes: + tags: [graviton] + - match: ['target=:ppc64le'] + runner-attributes: + tags: [ppc64le] + + - match: + - alsa-lib + - ant + - antlr + - argobots + - automake + - berkeley-db + - bison + - blt + - cmake + - curl + - darshan-util + - diffutils + - exmcutils + - expat + - flit + - freetype + - gdbm + - gotcha + - hpcviewer + - jansson + - json-c + - libbsd + - libevent + - libjpeg-turbo + - libnrm + - libpng + - libunistring + - lua-luaposix + - m4 + - mpfr + - ncurses + - openblas + - openjdk + - papi + - parallel-netcdf + - pcre2 + - perl-data-dumper + - pkgconf + - py-alembic + - py-idna + - py-testpath + - qhull + - snappy + - swig + - tar + - tcl + - texinfo + - unzip + - util-linux-uuid + - util-macros + - yaml-cpp + - zlib + - zstd + remove-attributes: + tags: &all_sizes [default, small, medium, large, huge] + runner-attributes: + tags: [small] + variables: + CI_JOB_SIZE: small + KUBERNETES_CPU_REQUEST: 500m + KUBERNETES_MEMORY_REQUEST: 500M + + - match: + - adios2 + - amrex + - archer + - ascent + - axom + - binutils + - blaspp + - boost + - butterflypack + - cabana + - caliper + - camp + - chai + - conduit + - datatransferkit + - faodel + - ffmpeg + - fftw + - fortrilinos + - gperftools + - gptune + - hdf5 + - heffte + - hpctoolkit + - hwloc + - hypre + - kokkos + - lammps + - lapackpp + - legion + - libzmq + - llvm-openmp-ompt + - mbedtls + - netlib-scalapack + - omega-h + - openmpi + - openpmd-api + - pagmo2 + - papyrus + - parsec + - pdt + - petsc + - pumi + - py-ipython-genutils + - py-petsc4py + - py-scipy + - py-statsmodels + - py-warlock + - py-warpx + - pygmo + - slepc + - slurm + - strumpack + - sundials + - superlu-dist + - tasmanian + - tau + - upcxx + - vtk-h + - zfp + remove-attributes: + tags: *all_sizes + runner-attributes: + tags: [medium] + variables: + CI_JOB_SIZE: medium + KUBERNETES_CPU_REQUEST: 2000m + KUBERNETES_MEMORY_REQUEST: 4G + + - match: + - cuda + - dyninst + - ginkgo + - hpx + - kokkos-kernels + - kokkos-nvcc-wrapper + - magma + - mfem + - mpich + - openturns + - precice + - raja + - rust + - slate + - trilinos + - vtk-m + - warpx + remove-attributes: + tags: *all_sizes + runner-attributes: + tags: [large] + variables: + CI_JOB_SIZE: large + KUBERNETES_CPU_REQUEST: 8000m + KUBERNETES_MEMORY_REQUEST: 12G + + - match: + - hipblas + - llvm + - llvm-amdgpu + - rocblas + remove-attributes: + tags: *all_sizes + runner-attributes: + tags: [huge] + variables: + CI_JOB_SIZE: huge + KUBERNETES_CPU_REQUEST: 11000m + KUBERNETES_MEMORY_REQUEST: 42G