From d55f3d94f05c6e49fba5ea4d7dec50020ae2f623 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Sat, 10 Sep 2022 00:50:13 +0000 Subject: [PATCH] CI: major refactor in images-v2 --- CTestCustom.cmake.in | 1 + .../ci/images-v2/Dockerfile.ci-spack-el8-base | 7 +- .../Dockerfile.ci-spack-el8-gcc8-base | 3 +- .../ci/images-v2/Dockerfile.ci-spack-el8-leaf | 72 ++++++++----------- scripts/ci/images-v2/build-cuda.sh | 33 ++------- scripts/ci/images-v2/build-gcc.sh | 9 ++- scripts/ci/images-v2/build-mpich-image.sh | 9 --- scripts/ci/images-v2/build-mpich.sh | 20 ++++++ scripts/ci/images-v2/modules.yaml | 17 ++--- scripts/ci/images-v2/packages.yaml.base | 10 --- scripts/ci/images-v2/packages.yaml.in | 16 +++++ .../images-v2/spack-external-cuda-addon.bash | 8 --- scripts/ci/images-v2/specs.yaml.in | 13 ++++ 13 files changed, 108 insertions(+), 110 deletions(-) delete mode 100755 scripts/ci/images-v2/build-mpich-image.sh create mode 100755 scripts/ci/images-v2/build-mpich.sh delete mode 100644 scripts/ci/images-v2/packages.yaml.base create mode 100644 scripts/ci/images-v2/packages.yaml.in delete mode 100644 scripts/ci/images-v2/spack-external-cuda-addon.bash create mode 100644 scripts/ci/images-v2/specs.yaml.in diff --git a/CTestCustom.cmake.in b/CTestCustom.cmake.in index 9adb03ef8c..a1fdb706dc 100644 --- a/CTestCustom.cmake.in +++ b/CTestCustom.cmake.in @@ -20,6 +20,7 @@ list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION "warning generated" "warnings generated" "warning: template parameter ... is not used in declaring the parameter types of function template" + "warning: command-line option '.*' is valid for Fortran but not for C" ) list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE ".*/thirdparty/.*" diff --git a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-base b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-base index 73c754827d..acbaec7685 100644 --- a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-base +++ b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-base @@ -8,12 +8,11 @@ RUN dnf install -y dnf-plugins-core && \ dnf install -y epel-release RUN dnf install -y \ bzip2 curl file gcc-c++ gcc git gnupg2 gzip \ - make patch tcl unzip which xz patchelf diffutils tar Lmod \ + make patch tcl unzip which xz patchelf diffutils findutils tar Lmod \ python3 python3-pip RUN dnf clean all -RUN git clone -b v0.18.0 https://github.com/spack/spack.git /opt/spack -COPY packages.yaml.base /etc/spack/packages.yaml +RUN git clone -b v0.18.1 https://github.com/spack/spack.git /opt/spack COPY modules.yaml /etc/spack/modules.yaml COPY 0001-spack-rhel-fix.patch /root/0001-spack-rhel-fix.patch @@ -26,5 +25,5 @@ RUN . /opt/spack/share/spack/setup-env.sh && \ spack compiler find --scope system && \ spack clean -a && \ spack mirror add E4S https://cache.e4s.io && \ - spack buildcache keys -it && \ + spack buildcache keys --install --trust && \ spack spec zlib diff --git a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base index 5b74cfacdc..9607e82a63 100644 --- a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base +++ b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-gcc8-base @@ -9,5 +9,4 @@ RUN dnf install -y gcc-gfortran && \ RUN . /opt/spack/share/spack/setup-env.sh && \ spack compiler rm --scope system gcc && \ spack compiler add --scope system && \ - spack config --scope system add "packages:all:compiler:[gcc]" && \ - spack external find --scope system --not-buildable cuda + spack config --scope system add "packages:all:compiler:[gcc]" diff --git a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf index 641b0c99cd..4cc1c3ef61 100644 --- a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf +++ b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf @@ -1,22 +1,27 @@ +# vim: ft=dockerfile ARG COMPILER_IMG_BASE -FROM ornladios/adios2:ci-spack-el8-${COMPILER_IMG_BASE}-base +FROM almalinux/almalinux:8 AS builder -ARG MPI_FLAVOR=openmpi -RUN sed "s|^packages:$|&\n mpich:\n version: [3.3.2]\n variants: device=ch3 netmod=tcp|" \ - -i /etc/spack/packages.yaml +WORKDIR /root + +RUN dnf update -y && \ + dnf install -y python3-pip && \ + pip3 install pyexpander -ARG EXTRA_VARIANTS -ARG CUDA_VARIANT -RUN sed \ - -e "s|packages:|packages:\n zfp:\n variants: ${CUDA_VARIANT} +shared|" \ - -e "s|variants: +blosc|variants: ${EXTRA_VARIANTS} +blosc|" \ - -e "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \ - -i /etc/spack/packages.yaml +COPY packages.yaml.in packages.yaml.in +COPY specs.yaml.in specs.yaml.in +ARG DEVICE_FLAVOR=cpu +ARG MPI_VARIANT=~mpi ARG MPI_FLAVOR=openmpi -RUN sed "s|^\(\s\+\)all:.*|&\n\1 providers:\n\1 mpi: [${MPI_FLAVOR}]|" \ - -i /etc/spack/packages.yaml && \ - cat /etc/spack/packages.yaml + +RUN expander.py --eval "device_flavor=\"${DEVICE_FLAVOR}\"" specs.yaml.in > specs.yaml && \ + expander.py --eval "device_flavor=\"${DEVICE_FLAVOR}\";mpi_variant=\"${MPI_VARIANT}\";mpi_flavor=\"${MPI_FLAVOR}\"" packages.yaml.in > packages.yaml + +FROM ornladios/adios2:ci-spack-el8-${COMPILER_IMG_BASE}-base + +COPY --from=builder /root/packages.yaml /etc/spack/packages.yaml +COPY --from=builder /root/specs.yaml /tmp/specs.yaml # Build dependencies ARG COMPILER_SPACK_ID @@ -25,40 +30,25 @@ RUN . /etc/profile.d/modules.sh && \ spack spec --reuse adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \ spack install \ --no-checksum \ - --reuse \ --fail-fast \ -v \ - -j$(grep -c '^processor' /proc/cpuinfo) \ + -j$((2 * $(grep -c '^processor' /proc/cpuinfo))) \ --only dependencies \ adios2%${COMPILER_SPACK_ID} ^rhash%gcc && \ - spack spec --reuse libsodium%${COMPILER_SPACK_ID} && \ - spack install \ - --no-checksum \ - --reuse \ - --fail-fast \ - -v \ - -j$(grep -c '^processor' /proc/cpuinfo) \ - libsodium%${COMPILER_SPACK_ID} && \ spack clean -a # Setup modules RUN . /opt/spack/share/spack/setup-env.sh && \ - spack env create --without-view adios2-ci && \ - spack -e adios2-ci config add concretizer:unify:true && \ - spack -e adios2-ci add $(spack find --format "/{hash}") && \ - spack -e adios2-ci rm cuda && \ - spack -e adios2-ci install && \ - spack env activate adios2-ci && \ - spack env deactivate && \ - rm -rf /root/.spack && \ - spack -e adios2-ci env loads + spack env create adios2-ci /tmp/specs.yaml && \ + spack -e adios2-ci add $(spack find --implicit --format "/{hash}") && \ + spack -e adios2-ci concretize --force --fresh && \ + spack -e adios2-ci install \ + --no-checksum \ + --fail-fast \ + -v \ + -j$((2 * $(grep -c '^processor' /proc/cpuinfo))) # Setup default login environment -RUN . /opt/spack/share/spack/setup-env.sh && \ - echo 'source /opt/spack/share/spack/setup-env.sh' >> /etc/profile.d/zz-adios2-ci-env.sh && \ - echo "module use ${SPACK_ROOT}/share/spack/modules/$(spack debug report | grep -Po 'Platform:\*\*\s\K.*')" >> /etc/profile.d/zz-adios2-ci-env.sh && \ - echo 'source ${SPACK_ROOT}/var/spack/environments/adios2-ci/loads' >> /etc/profile.d/zz-adios2-ci-env.sh - -# Enable CUDA in Spack if enabled -COPY spack-external-cuda-addon.bash /root/spack-external-cuda-addon.bash -RUN test x"${CUDA_VARIANT}" == x"+cuda" && . /root/spack-external-cuda-addon.bash +RUN . /etc/profile.d/modules.sh && \ + echo "source /opt/spack/share/spack/setup-env.sh" >> /etc/profile.d/zz-adios2-ci-env.sh && \ + echo "spack env activate adios2-ci" >> /etc/profile.d/zz-adios2-ci-env.sh diff --git a/scripts/ci/images-v2/build-cuda.sh b/scripts/ci/images-v2/build-cuda.sh index 62b184f300..ab9699f867 100755 --- a/scripts/ci/images-v2/build-cuda.sh +++ b/scripts/ci/images-v2/build-cuda.sh @@ -1,38 +1,19 @@ #!/bin/bash -set +e +set -ex BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE})) source "${BASE_DIR}/build-functions.sh" -message1 "Building ci-spack-el8 base image" -if ! build_squash \ - nvidia/cuda:11.7.0-devel-rockylinux8 \ - ornladios/adios2:ci-spack-el8-cuda-base \ - Dockerfile.ci-spack-el8-base \ - "--build-arg=BASE_IMAGE=nvidia/cuda:11.7.0-devel-rockylinux8" -then - echo "Error: Failed to build ci-spack-el8 base image" - exit 3 -fi - -message1 "Building ci-spack-el8 gcc8 base image" -if ! build_squash \ - ornladios/adios2:ci-spack-el8-cuda-base \ - ornladios/adios2:ci-spack-el8-cuda-gcc8-base \ - Dockerfile.ci-spack-el8-gcc8-base \ - "--build-arg=BASE_IMAGE=ornladios/adios2:ci-spack-el8-cuda-base" -then - echo "Error: Failed to build ci-spack-el8 leaf image" - exit 3 -fi - message1 "Building ci-spack-el8 leaf image" -build_conf=cuda-gcc8 +build_conf=gcc8 if ! build_squash \ - ornladios/adios2:ci-spack-el8-cuda-gcc8-base \ + ornladios/adios2:ci-spack-el8-gcc8-base \ ornladios/adios2:ci-spack-el8-cuda-serial \ Dockerfile.ci-spack-el8-leaf \ - "--build-arg=COMPILER_IMG_BASE=${build_conf} --build-arg=COMPILER_SPACK_ID=gcc --build-arg=CUDA_VARIANT=+cuda" + "--build-arg=COMPILER_IMG_BASE=${build_conf} + --build-arg=COMPILER_SPACK_ID=gcc + --build-arg=DEVICE_FLAVOR=CUDA + --build-arg=MPI_VARIANT=~mpi" then echo "Error: Failed to build ci-spack-el8 leaf image" exit 3 diff --git a/scripts/ci/images-v2/build-gcc.sh b/scripts/ci/images-v2/build-gcc.sh index 86fa959cb1..a483addeac 100755 --- a/scripts/ci/images-v2/build-gcc.sh +++ b/scripts/ci/images-v2/build-gcc.sh @@ -1,5 +1,6 @@ #!/bin/bash set +e +set -x BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE})) source "${BASE_DIR}/build-functions.sh" @@ -11,7 +12,9 @@ do ornladios/adios2:ci-spack-el8-gcc${ver}-base \ ornladios/adios2:ci-spack-el8-gcc${ver}-serial \ Dockerfile.ci-spack-el8-leaf \ - "--build-arg COMPILER=gcc${ver} --build-arg EXTRA_VARIANTS=~mpi" + "--build-arg COMPILER_IMG_BASE=gcc${ver} + --build-arg COMPILER_SPACK_ID=gcc + --build-arg=MPI_VARIANT=~mpi" then echo "Error: Failed to build gcc${ver}-serial image" exit 3 @@ -22,7 +25,9 @@ do ornladios/adios2:ci-spack-el8-gcc${ver}-base \ ornladios/adios2:ci-spack-el8-gcc${ver}-mpi \ Dockerfile.ci-spack-el8-leaf \ - "--build-arg COMPILER=gcc${ver} --build-arg EXTRA_VARIANTS=+mpi" + "--build-arg COMPILER_IMG_BASE=gcc${ver} + --build-arg COMPILER_SPACK_ID=gcc + --build-arg=MPI_VARIANT=+mpi" then echo "Error: Failed to build gcc${ver}-mpi image" exit 3 diff --git a/scripts/ci/images-v2/build-mpich-image.sh b/scripts/ci/images-v2/build-mpich-image.sh deleted file mode 100755 index 134e71b2a2..0000000000 --- a/scripts/ci/images-v2/build-mpich-image.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -docker build -t "ornladios/adios2:ci-spack-el8-gcc10-mpich" \ - -f Dockerfile.ci-spack-el8-leaf \ - --build-arg COMPILER_IMG_BASE=gcc10 \ - --build-arg EXTRA_VARIANTS=+mpi \ - --build-arg MPI_FLAVOR=mpich \ - --build-arg COMPILER_SPACK_ID=gcc \ - . diff --git a/scripts/ci/images-v2/build-mpich.sh b/scripts/ci/images-v2/build-mpich.sh new file mode 100755 index 0000000000..a82db528f8 --- /dev/null +++ b/scripts/ci/images-v2/build-mpich.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -ex + +BASE_DIR=$(dirname $(readlink -f ${BASH_SOURCE})) +source "${BASE_DIR}/build-functions.sh" + +message1 "Building ci-spack-el8 leaf image" +build_conf=gcc10 +if ! build_squash \ + ornladios/adios2:ci-spack-el8-gcc10-base \ + ornladios/adios2:ci-spack-el8-gcc10-mpich \ + Dockerfile.ci-spack-el8-leaf \ + "--build-arg=COMPILER_IMG_BASE=${build_conf} + --build-arg=COMPILER_SPACK_ID=gcc + --build-arg=MPI_FLAVOR=mpich + --build-arg=MPI_VARIANT=+mpi" +then + echo "Error: Failed to build ci-spack-el8 leaf image" + exit 3 +fi diff --git a/scripts/ci/images-v2/modules.yaml b/scripts/ci/images-v2/modules.yaml index b908a05880..583ff26fbe 100644 --- a/scripts/ci/images-v2/modules.yaml +++ b/scripts/ci/images-v2/modules.yaml @@ -1,9 +1,10 @@ modules: - enable: - - tcl - tcl: - hash_length: 0 - all: - autoload: 'direct' - projections: - all: '{name}/{version}' + default: + enable: + - tcl + tcl: + hash_length: 0 + all: + autoload: 'direct' + projections: + all: '{name}/{version}' diff --git a/scripts/ci/images-v2/packages.yaml.base b/scripts/ci/images-v2/packages.yaml.base deleted file mode 100644 index 0ba9f4c067..0000000000 --- a/scripts/ci/images-v2/packages.yaml.base +++ /dev/null @@ -1,10 +0,0 @@ -packages: - all: - target: [x86_64] - cmake: - variants: ~ownlibs - adios2: - variants: +blosc +bzip2 +zfp +sz +png +sst +dataman +ssc +hdf5 +python +fortran - hdf5: - variants: api=default - version: ['1.13.0:'] diff --git a/scripts/ci/images-v2/packages.yaml.in b/scripts/ci/images-v2/packages.yaml.in new file mode 100644 index 0000000000..bff25fc3e6 --- /dev/null +++ b/scripts/ci/images-v2/packages.yaml.in @@ -0,0 +1,16 @@ +packages: + all: + target: [haswell] + providers: + mpi: [$(mpi_flavor)] + adios2: + variants: $(mpi_variant)+blosc+bzip2+zfp+sz+png+sst+dataman+ssc+hdf5+python+fortran + cmake: + version: [3.22.4] + mpich: + version: [3.3.2] + variants: device=ch3 netmod=tcp +$if(device_flavor == "CUDA")\ + zfp: + variants: +cuda+shared +$endif\ diff --git a/scripts/ci/images-v2/spack-external-cuda-addon.bash b/scripts/ci/images-v2/spack-external-cuda-addon.bash deleted file mode 100644 index 47f26c6a61..0000000000 --- a/scripts/ci/images-v2/spack-external-cuda-addon.bash +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -source /opt/spack/share/spack/setup-env.sh - -# External CUDA is erroneously treated as a module, in no CUDA builds this is no-op -find $SPACK_ROOT/share/spack/modules -type f -exec sed -i '/module load cuda/d' {} \; - -echo 'export LD_LIBRARY_PATH=/usr/local/cuda/compat/:$LD_LIBRARY_PATH' >> /etc/profile.d/zz-adios2-ci-env.sh diff --git a/scripts/ci/images-v2/specs.yaml.in b/scripts/ci/images-v2/specs.yaml.in new file mode 100644 index 0000000000..8a5a83dbf0 --- /dev/null +++ b/scripts/ci/images-v2/specs.yaml.in @@ -0,0 +1,13 @@ +$# Run this with pyexpander --eval "flavor=xxyzz" +$default(compiler_spack_id="gcc",device_flavor="BASE")\ +spack: + specs: + - libsodium%$(compiler_spack_id) +$if(device_flavor == "CUDA")\ + - kokkos@3.6.01%gcc+cuda+wrapper cuda_arch=52 +$endif\ + view: true + config: + checksum: false + concretizer: + unify: false