Update Uberenv and introduce Gitlab CI [feature/uberenv_ci]

.gitlab-ci.yml

@@ -0,0 +1,82 @@
+##############################################################################
+# Copyright (c) 2014-20, Lawrence Livermore National Security, LLC and Conduit
+# project contributors. See the COPYRIGHT file for details.
+##############################################################################
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+  CONDUIT_ALLOC_NAME: conduit_${CI_PIPELINE_ID}
+  BUILD_ROOT: ${CI_BUILDS_DIR}/conduit/${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  INSTALL_ROOT: ${CI_BUILDS_DIR}/conduit/${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}/uberenv_libs
+  SPACK_UPSTREAM: /usr/workspace/radiuss/spack-chain-parent/opt/spack
+
+stages:
+  - allocate_resources
+  - deps
+  - install
+  - release_resources
+  - clean
+
+#########
+# SCRIPTS
+#########
+
+# Whether and how to update uberenv
+.run_update_uberenv: &run_update_uberenv |
+  [[ -n "${UPDATE_UBERENV}" ]] && ./scripts/ci/update-uberenv.sh "${UPDATE_UBERENV}"
+
+# How to build uberenv command
+.def_uberenv_cmd: &def_uberenv_cmd |
+  INSTALL=""
+  [[ "${UBERENV_INSTALL}" == "ON" ]] && INSTALL="--install --run_tests"
+  mkdir -p ${INSTALL_ROOT}_${SYS_TYPE}_${TOOLCHAIN}
+  PREFIX="--prefix=${INSTALL_ROOT}_${SYS_TYPE}_${TOOLCHAIN}"
+  UPSTREAM="--upstream=${SPACK_UPSTREAM}"
+  SPEC="--spec=${PKG_SPEC}"
+  UBERENV_CMD="scripts/uberenv/uberenv.py ${INSTALL} ${SPEC} ${PREFIX} ${UPSTREAM}"
+
+# Allocation sequence
+.def_srun_prefix: &def_srun_prefix |
+  JOB_ID=$(squeue -h --name=${CONDUIT_ALLOC_NAME} --format=%A)
+  [[ -n "${JOB_ID}" ]] && JOB_ID="--jobid=${JOB_ID}"
+  RESOURCES="-N 1 -n 1 -c 12"
+  export SRUN_PREFIX="srun ${JOB_ID} ${RESOURCES}"
+
+# Define the generic script for quartz (template)
+.srun_build_script:
+  script:
+    - *run_update_uberenv
+    - *def_srun_prefix
+    - *def_uberenv_cmd
+    - set -x; ${SRUN_PREFIX} ${UBERENV_CMD}; set +x
+
+# Define the generic script for lassen (template)
+.lsf_build_script:
+  script:
+    - *run_update_uberenv
+    - *def_uberenv_cmd
+    - set -x; lalloc 1 ${UBERENV_CMD}; set +x
+
+# Build modes templates:
+# A job will have one on this template to pilot the behavior of def_uberenv_cmd
+.mode_deps:
+  stage: deps
+  variables:
+    UBERENV_INSTALL: "OFF"
+
+.mode_install:
+  stage: install
+  variables:
+    UBERENV_INSTALL: "ON"
+
+##########
+# INCLUDES
+##########
+
+# This is where templates and jobs are included
+include:
+  - local: .gitlab/toolchains_templates.yml
+  - local: .gitlab/quartz_templates.yml
+  - local: .gitlab/quartz_jobs.yml
+  - local: .gitlab/lassen_templates.yml
+  - local: .gitlab/lassen_jobs.yml

.gitlab/lassen_jobs.yml

@@ -0,0 +1,21 @@
+##############################################################################
+# Copyright (c) 2016-2020, Lawrence Livermore National Security, LLC and
+# Conduit project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# Here are all lassen build jobs
+clang (build deps on lassen):
+  extends: [.build_deps_on_lassen, .with_clang]
+
+clang (install on lassen):
+  extends: [.install_on_lassen, .with_clang]
+  needs: [clang (build deps on lassen)]
+
+xl (build deps on lassen):
+  extends: [.build_deps_on_lassen, .with_xl]
+
+xl (install on lassen):
+  extends: [.install_on_lassen, .with_xl]
+  needs: [xl (build deps on lassen)]

.gitlab/lassen_templates.yml

@@ -0,0 +1,28 @@
+##############################################################################
+# Copyright (c) 2014-20, Lawrence Livermore National Security, LLC and Conduit
+# project contributors. See the COPYRIGHT file for details.
+##############################################################################
+
+##################
+# LASSEN TEMPLATES
+##################
+
+# Shared configuration of jobs for lassen
+.on_lassen:
+  variables:
+  tags:
+    - shell
+    - lassen
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~/_lnone/ || $CONDUIT_CI_LASSEN == "OFF"'
+      when: never
+    - when: on_success
+  allow_failure: true
+
+####
+# Generic lassen jobs
+.build_deps_on_lassen:
+  extends: [.lsf_build_script, .mode_deps, .on_lassen]
+
+.install_on_lassen:
+  extends: [.lsf_build_script, .mode_install, .on_lassen]

.gitlab/quartz_jobs.yml

@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2016-2020, Lawrence Livermore National Security, LLC and
+# Conduit project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+## In pre-build phase, allocate a node for builds
+#allocate_resources_build_quartz:
+#  extends: .quartz_common
+#  stage: allocate_resources
+#  script:
+#    - salloc -N 1 -c 36 --no-shell --job-name=${CONDUIT_ALLOC_NAME}
+#
+## In post-build phase, deallocate resources
+## Note : make sure this is run even on build phase failure
+#release_resources_build_quartz:
+#  extends: .quartz_common
+#  stage: release_resources
+#  script:
+#    - export JOBID=$(squeue -h --name=${CONDUIT_ALLOC_NAME} --format=%A)
+#    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+#  when: always
+
+# Here are all quartz build jobs
+gcc (build deps on quartz):
+  extends: [.build_deps_on_quartz, .with_gcc]
+
+gcc (install on quartz):
+  extends: [.install_on_quartz, .with_gcc]
+  needs: [gcc (build deps on quartz)]
+
+gcc_static (build deps on quartz):
+  extends: [.build_deps_on_quartz, .with_gcc_static]
+
+gcc_static (install on quartz):
+  extends: [.install_on_quartz, .with_gcc_static]
+  needs: [gcc_static (build deps on quartz)]

.gitlab/quartz_templates.yml

@@ -0,0 +1,28 @@
+##############################################################################
+# Copyright (c) 2014-20, Lawrence Livermore National Security, LLC and Conduit
+# project contributors. See the COPYRIGHT file for details.
+##############################################################################
+
+##################
+# QUARTZ TEMPLATES
+##################
+
+# Shared configuration of jobs for quartz
+.on_quartz:
+  tags:
+    - shell
+    - quartz
+  timeout: 3h
+  except:
+    refs:
+      - /_qnone/
+    variables:
+      - $CONDUIT_CI_QUARTZ == "OFF"
+
+####
+# Generic qwartz jobs
+.build_deps_on_quartz:
+  extends: [.srun_build_script, .mode_deps, .on_quartz]
+
+.install_on_quartz:
+  extends: [.srun_build_script, .mode_install, .on_quartz]

.gitlab/toolchains_templates.yml

@@ -0,0 +1,28 @@
+##############################################################################
+# Copyright (c) 2014-20, Lawrence Livermore National Security, LLC and Conduit
+# project contributors. See the COPYRIGHT file for details.
+##############################################################################
+
+############
+# TOOLCHAINS
+############
+
+.with_gcc:
+  variables:
+    TOOLCHAIN: "gcc"
+    PKG_SPEC: ""
+
+.with_gcc_static:
+  variables:
+    TOOLCHAIN: "gcc_static"
+    PKG_SPEC: "%gcc~shared"
+
+.with_clang:
+  variables:
+    TOOLCHAIN: "clang"
+    PKG_SPEC: "%clang@coral~python~fortran"
+
+.with_xl:
+  variables:
+    TOOLCHAIN: "xl"
+    PKG_SPEC: "%xl@coral~shared~python~fortran"

scripts/ci/update-uberenv.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [[ ! ${1} ]]
+then
+    echo "ERROR: expecting reference for uberenv repo" >&2
+else
+    uberenv_ref="${1}"
+fi
+
+uberenv_file="scripts/uberenv/uberenv.py"
+uberenv_master="https://raw.githubusercontent.com/LLNL/uberenv/${uberenv_ref}/uberenv.py"
+
+curl --fail --output ${uberenv_file} ${uberenv_master}

scripts/uberenv/project.json

@@ -1,10 +1,12 @@
 {
 "package_name" : "conduit",
-"uberenv_package_name" : "uberenv-conduit",
-"spack_url": "https://github.com/alpine-DAV/spack",
-"spack_branch": "task/2019_10_update_conduit",
-"spack_activate" : {"py-numpy" : ["+python"], 
-                    "py-sphinx": ["+python","+doc"], 
+"package_version" : "master",
+"package_final_phase" : "configure",
+"package_source_dir" : "../..",
+"spack_url": "https://github.com/spack/spack",
+"spack_branch": "develop",
+"spack_activate" : {"py-numpy" : ["+python"],
+                    "py-sphinx": ["+python","+doc"],
                     "py-mpi4py" : ["+python", "+mpi"],
                     "py-sphinx-rtd-theme": ["+python","+doc"] }
 }