Merge pull request #1091 from LLNL/task/white238/toss4

Quartz: The Return
LLNL · May 19, 2023 · 26c57ff · 26c57ff
2 parents 7059a36 + 44b5b60
commit 26c57ff
Show file tree

Hide file tree

Showing 18 changed files with 771 additions and 869 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -56,6 +56,6 @@ stages:
 
 # This is where jobs are included
 include:
-  - local: .gitlab/build_ruby.yml
+  - local: .gitlab/build_quartz.yml
   - local: .gitlab/build_lassen.yml
   - local: .gitlab/build_tioga.yml
diff --git a/.gitlab/build_quartz.yml b/.gitlab/build_quartz.yml
@@ -0,0 +1,128 @@
+# Copyright (c) 2017-2023, Lawrence Livermore National Security, LLC and
+# other Axom Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+
+####
+# This is the shared configuration of jobs for quartz
+.on_quartz:
+  tags:
+    - shell
+    - quartz
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_RUBY == "OFF"' #run except if ...
+      when: never
+    - if: '$CI_JOB_NAME =~ /quartz_release/'
+      when: always
+    - when: on_success
+  before_script:
+    - module load cmake/3.19.2
+
+####
+# In pre-build phase, allocate a node for builds
+quartz_allocate:
+  variables:
+    GIT_STRATEGY: none
+  extends: [.on_quartz, .src_workflow]
+  stage: allocate
+  script:
+    # Use when ellastic ci is on quartz or we go back to ruby
+    #- salloc --reservation=ci --qos=ci_ruby -N 1 -c 36 -t 60 --no-shell --job-name=${PROJECT_ALLOC_NAME}
+    - salloc -N 1 -c 36 -t 60 --no-shell --job-name=${PROJECT_ALLOC_NAME}
+  needs: []
+
+####
+# In post-build phase, deallocate resources
+# Note : make sure this is run even on build phase failure
+quartz_release:
+  variables:
+    GIT_STRATEGY: none
+  extends: [.on_quartz, .src_workflow]
+  stage: release
+  script:
+    - export JOBID=$(squeue -h --name=${PROJECT_ALLOC_NAME} --format=%A)
+    - if [[ -n "${JOBID}" ]]; then scancel ${JOBID}; fi
+
+####
+# Template
+.src_build_on_quartz:
+  stage: build
+  variables:
+    # Use when ellastic ci is on quartz or we go back to ruby
+    # ALLOC_COMMAND: "srun --reservation=ci --qos=ci_ruby -t 60 -N 1 "
+    ALLOC_COMMAND: "srun -t 60 -N 1 "
+  extends: [.src_build_script, .on_quartz, .src_workflow]
+  needs: [quartz_allocate]
+
+.full_build_on_quartz:
+  stage: build
+  variables:
+    # Use when ellastic ci is on quartz or we go back to ruby
+    # ALLOC_COMMAND: "srun --reservation=ci --qos=ci_ruby -t 60 -N 1 "
+    ALLOC_COMMAND: "srun -t 60 -N 1 "
+  extends: [.full_build_script, .on_quartz, .full_workflow]
+  needs: []
+
+####
+# PR Build jobs
+quartz-clang_14_0_6-debug-src:
+  variables:
+    COMPILER: "clang@14.0.6"
+    HOST_CONFIG: "quartz-toss_4_x86_64_ib-${COMPILER}.cmake"
+  extends: .src_build_on_quartz
+
+quartz-clang_14_0_6-release-src:
+  variables:
+    COMPILER: "clang@14.0.6"
+    HOST_CONFIG: "quartz-toss_4_x86_64_ib-${COMPILER}.cmake"
+    BUILD_TYPE: "Release"
+    EXTRA_CMAKE_OPTIONS: "-DAXOM_QUEST_ENABLE_EXTRA_REGRESSION_TESTS:BOOL=ON"
+  extends: .src_build_on_quartz
+
+quartz-gcc_10_3_1-src:
+  variables:
+    COMPILER: "gcc@10.3.1"
+    HOST_CONFIG: "quartz-toss_4_x86_64_ib-${COMPILER}.cmake"
+  extends: .src_build_on_quartz
+
+# TODO: turn back on
+#quartz-gcc_10_3_1_no_fortran-src:
+#  variables:
+#    COMPILER: "gcc@10.3.1"
+#    HOST_CONFIG: "quartz-toss_4_x86_64_ib-${COMPILER}_nofortran.cmake"
+#  extends: .src_build_on_quartz
+
+# disabled due to not actually having a host-config yet
+# quartz-intel_19_0_4-src:
+#   variables:
+#     COMPILER: "intel@19.0.4"
+#     HOST_CONFIG: "quartz-toss_4_x86_64_ib-${COMPILER}.cmake"
+#   extends: .src_build_on_quartz
+
+
+####
+# Full Build jobs
+quartz-clang_14_0_6-full:
+  variables:
+    COMPILER: "clang@14.0.6"
+    SPEC: "%${COMPILER}+mfem"
+  extends: .full_build_on_quartz
+
+quartz-gcc_10_3_1-full:
+  variables:
+    COMPILER: "gcc@8.3.1"
+    SPEC: "%${COMPILER}+mfem"
+  extends: .full_build_on_quartz
+
+quartz-gcc_10_3_1_no_fortran-full:
+  variables:
+    COMPILER: "gcc@10.3.1"
+    SPEC: "%${COMPILER}~fortran+mfem"
+  extends: .full_build_on_quartz
+
+# disabled due to not actually having a host-config yet
+#quartz-intel_19_0_4-full:
+#  variables:
+#    COMPILER: "intel@19.0.4"
+#    SPEC: "%${COMPILER}+mfem~cpp14"
+#  extends: .full_build_on_quartz
diff --git a/.gitlab/build_ruby.yml b/.gitlab/build_ruby.yml
diff --git a/.uberenv_config.json b/.uberenv_config.json
@@ -4,7 +4,7 @@
 "package_final_phase": "initconfig",
 "package_source_dir": "../..",
 "spack_url": "https://github.com/spack/spack.git",
-"spack_commit": "9ea7937f6de3fc0ba66cbd399f57ad976341d9b5",
+"spack_commit": "af0f0942920ffb1fd15e5458e9796f4bd7c8a9d1",
 "spack_configs_path": "scripts/spack/configs",
 "spack_packages_path": "scripts/spack/packages",
 "spack_concretizer": "clingo",

diff --git a/host-configs/quartz-toss_4_x86_64_ib-clang@14.0.6.cmake b/host-configs/quartz-toss_4_x86_64_ib-clang@14.0.6.cmake
@@ -0,0 +1,106 @@
+#------------------------------------------------------------------------------
+# !!!! This is a generated file, edit at own risk !!!!
+#------------------------------------------------------------------------------
+# CMake executable path: /usr/tce/packages/cmake/cmake-3.19.2/bin/cmake
+#------------------------------------------------------------------------------
+
+#------------------------------------------------------------------------------
+# Compilers
+#------------------------------------------------------------------------------
+# Compiler Spec: clang@=14.0.6
+#------------------------------------------------------------------------------
+if(DEFINED ENV{SPACK_CC})
+
+  set(CMAKE_C_COMPILER "/usr/WS1/axom/libs/toss_4_x86_64_ib/2023_05_18_12_46_06/spack/lib/spack/env/clang/clang" CACHE PATH "")
+
+  set(CMAKE_CXX_COMPILER "/usr/WS1/axom/libs/toss_4_x86_64_ib/2023_05_18_12_46_06/spack/lib/spack/env/clang/clang++" CACHE PATH "")
+
+  set(CMAKE_Fortran_COMPILER "/usr/WS1/axom/libs/toss_4_x86_64_ib/2023_05_18_12_46_06/spack/lib/spack/env/clang/gfortran" CACHE PATH "")
+
+else()
+
+  set(CMAKE_C_COMPILER "/usr/tce/packages/clang/clang-14.0.6/bin/clang" CACHE PATH "")
+
+  set(CMAKE_CXX_COMPILER "/usr/tce/packages/clang/clang-14.0.6/bin/clang++" CACHE PATH "")
+
+  set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran" CACHE PATH "")
+
+endif()
+
+set(ENABLE_FORTRAN ON CACHE BOOL "")
+
+set(BLT_EXE_LINKER_FLAGS " -Wl,-rpath,/usr/tce/packages/clang/clang-14.0.6/lib" CACHE STRING "Adds a missing libstdc++ rpath")
+
+#------------------------------------------------------------------------------
+# MPI
+#------------------------------------------------------------------------------
+
+set(MPI_C_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.6-clang-14.0.6/bin/mpicc" CACHE PATH "")
+
+set(MPI_CXX_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.6-clang-14.0.6/bin/mpicxx" CACHE PATH "")
+
+set(MPI_Fortran_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.6-clang-14.0.6/bin/mpif90" CACHE PATH "")
+
+set(MPIEXEC_EXECUTABLE "/usr/bin/srun" CACHE PATH "")
+
+set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")
+
+set(ENABLE_MPI ON CACHE BOOL "")
+
+#------------------------------------------------------------------------------
+# Hardware
+#------------------------------------------------------------------------------
+
+#------------------------------------------------
+# Hardware Specifics
+#------------------------------------------------
+
+set(ENABLE_OPENMP ON CACHE BOOL "")
+
+set(ENABLE_GTEST_DEATH_TESTS ON CACHE BOOL "")
+
+#------------------------------------------------------------------------------
+# TPLs
+#------------------------------------------------------------------------------
+
+set(TPL_ROOT "/usr/WS1/axom/libs/toss_4_x86_64_ib/2023_05_18_12_46_06/clang-14.0.6" CACHE PATH "")
+
+set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.8.6-sthcqefj6i6hfx6jxqsqat6kydip4bh5" CACHE PATH "")
+
+set(C2C_DIR "${TPL_ROOT}/c2c-1.3.0-suwgwulvsi2a4ghsejbkzhw6nvcavlwm" CACHE PATH "")
+
+set(MFEM_DIR "${TPL_ROOT}/mfem-4.5.0-znv6samadahgjaimxs7qzx67oejhjtfy" CACHE PATH "")
+
+set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.22-xsjkllrmkjvoxo6bdj4kca76ogweoopr" CACHE PATH "")
+
+set(LUA_DIR "/usr" CACHE PATH "")
+
+set(RAJA_DIR "${TPL_ROOT}/raja-2022.03.0-jjcsjmscrhxcksd7m26q462vhhgicylf" CACHE PATH "")
+
+set(UMPIRE_DIR "${TPL_ROOT}/umpire-2022.03.1-eri46kk5td3v23xgppxxwxovlusgm4zn" CACHE PATH "")
+
+set(CAMP_DIR "${TPL_ROOT}/camp-2022.10.1-ug3vynwpuw3zbkvpscygelocijatm3fu" CACHE PATH "")
+
+# scr not built
+
+#------------------------------------------------------------------------------
+# Devtools
+#------------------------------------------------------------------------------
+
+set(DEVTOOLS_ROOT "/collab/usr/gapps/axom/devtools/toss_4_x86_64_ib/2023_05_18_11_52_05/._view/btoxy5ovdbouub2brzxcmjwzdhvzatlc" CACHE PATH "")
+
+set(CLANGFORMAT_EXECUTABLE "/collab/usr/gapps/axom/devtools/toss_4_x86_64_ib/latest/llvm-10.0.0/bin/clang-format" CACHE PATH "")
+
+set(PYTHON_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.10.10/bin/python3.10" CACHE PATH "")
+
+set(ENABLE_DOCS ON CACHE BOOL "")
+
+set(SPHINX_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.10.10/bin/sphinx-build" CACHE PATH "")
+
+set(SHROUD_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.10.10/bin/shroud" CACHE PATH "")
+
+set(CPPCHECK_EXECUTABLE "${DEVTOOLS_ROOT}/cppcheck-2.9/bin/cppcheck" CACHE PATH "")
+
+set(DOXYGEN_EXECUTABLE "${DEVTOOLS_ROOT}/doxygen-1.8.14/bin/doxygen" CACHE PATH "")
+
+