diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..c48608a76 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,144 @@ +############################################################################## +# Copyright (c) 2014-20, Lawrence Livermore National Security, LLC and Conduit +# project contributors. See the COPYRIGHT file for details. +############################################################################## + +variables: + GIT_SUBMODULE_STRATEGY: recursive + CONDUIT_ALLOC_NAME: conduit_${CI_PIPELINE_ID} + BUILD_ROOT: ${CI_BUILDS_DIR}/conduit_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID} + INSTALL_ROOT: ${CI_BUILDS_DIR}/spack_uberenv + SPACK_UPSTREAM: /usr/workspace/radiuss/spack-chain-parent/opt/spack + +stages: + - allocate_resources + - deps + - install + - release_resources + - clean + +# Some scripts +.run_update_uberenv: &run_update_uberenv | + [[ -n "${UPDATE_UBERENV}" ]] && ./scripts/ci/update-uberenv.sh "${UPDATE_UBERENV}" + +.def_uberenv_cmd: &def_uberenv_cmd | + INSTALL="" + [[ "${UBERENV_INSTALL}" == "ON" ]] && INSTALL="--install --run_tests" + PREFIX="--prefix=${INSTALL_ROOT}_${SYS_TYPE}_${TOOLCHAIN}" + UPSTREAM="--upstream=${SPACK_UPSTREAM}" + SPEC="--spec=${PKG_SPEC}" + UBERENV_CMD="scripts/uberenv/uberenv.py ${INSTALL} ${SPEC} ${PREFIX} ${UPSTREAM}" + +# Build modes +.mode_deps: + stage: deps + variables: + UBERENV_INSTALL: "OFF" + +.mode_install: + stage: install + variables: + UBERENV_INSTALL: "ON" + +################## +# QUARTZ TEMPLATES +################## + +# Shared configuration of jobs for quartz +.on_quartz: + tags: + - shell + - quartz + timeout: 3h + except: + refs: + - /_qnone/ + variables: + - $CONDUIT_CI_QUARTZ == "OFF" + +# Allocation sequence +.def_srun_prefix: &def_srun_prefix | + JOB_ID=$(squeue -h --name=${CONDUIT_ALLOC_NAME} --format=%A) + [[ -n "${JOB_ID}" ]] && JOB_ID="--jobid=${JOB_ID}" + RESOURCES="-N 1 -n 1 -c 12" + export SRUN_PREFIX="srun ${JOB_ID} ${RESOURCES}" + +# Template defining the generic script for quartz +.srun_build_script: + script: + - *run_update_uberenv + - *def_srun_prefix + - *def_uberenv_cmd + - set -x; ${SRUN_PREFIX} ${UBERENV_CMD}; set +x + +#### +# Generic qwartz jobs +.build_deps_on_quartz: + extends: [.srun_build_script, .mode_deps, .on_quartz] + +.install_on_quartz: + extends: [.srun_build_script, .mode_install, .on_quartz] + +################## +# LASSEN TEMPLATES +################## + +# Shared configuration of jobs for lassen +.on_lassen: + variables: + tags: + - shell + - lassen + rules: + - if: '$CI_COMMIT_BRANCH =~/_lnone/ || $CONDUIT_CI_LASSEN == "OFF"' + when: never + - when: on_success + allow_failure: true + +# Template defining the generic script for lassen +.lsf_build_script: + script: + - *run_update_uberenv + - *def_uberenv_cmd + - set -x; lalloc 1 ${UBERENV_CMD}; set +x + +#### +# Generic lassen jobs +.build_deps_on_lassen: + extends: [.lsf_build_script, .mode_deps, .on_lassen] + +.install_on_lassen: + extends: [.lsf_build_script, .mode_install, .on_lassen] + +############ +# TOOLCHAINS +############ + +.with_gcc: + variables: + TOOLCHAIN: "gcc" + PKG_SPEC: "" + +.with_gcc_static: + variables: + TOOLCHAIN: "gcc_static" + PKG_SPEC: "%gcc~shared" + +.with_clang: + variables: + TOOLCHAIN: "clang" + PKG_SPEC: "%clang@coral~python~fortran" + +.with_xl: + variables: + TOOLCHAIN: "xl" + PKG_SPEC: "%xl@coral~shared~python~fortran" + +################ +# JOBS INCLUSION +################ + +# This is where jobs are included +include: + - local: .gitlab/ci/build_quartz.yml + - local: .gitlab/ci/build_lassen.yml diff --git a/.gitlab/ci/build_lassen.yml b/.gitlab/ci/build_lassen.yml new file mode 100644 index 000000000..727fb2ef3 --- /dev/null +++ b/.gitlab/ci/build_lassen.yml @@ -0,0 +1,21 @@ +############################################################################## +# Copyright (c) 2016-2020, Lawrence Livermore National Security, LLC and +# Conduit project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################## + +# Here are all lassen build jobs +L_build_deps_clang: + extends: [.build_deps_on_lassen, .with_clang] + +L_install_clang: + extends: [.install_on_lassen, .with_clang] + needs: [L_build_deps_clang] + +L_build_deps_xl: + extends: [.build_deps_on_lassen, .with_xl] + +L_install_xl: + extends: [.install_on_lassen, .with_xl] + needs: [L_build_deps_xl] diff --git a/.gitlab/ci/build_quartz.yml b/.gitlab/ci/build_quartz.yml new file mode 100644 index 000000000..3130414e3 --- /dev/null +++ b/.gitlab/ci/build_quartz.yml @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2016-2020, Lawrence Livermore National Security, LLC and +# Conduit project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################## + +## In pre-build phase, allocate a node for builds +#allocate_resources_build_quartz: +# extends: .quartz_common +# stage: allocate_resources +# script: +# - salloc -N 1 -c 36 --no-shell --job-name=${CONDUIT_ALLOC_NAME} +# +## In post-build phase, deallocate resources +## Note : make sure this is run even on build phase failure +#release_resources_build_quartz: +# extends: .quartz_common +# stage: release_resources +# script: +# - export JOBID=$(squeue -h --name=${CONDUIT_ALLOC_NAME} --format=%A) +# - ([[ -n "${JOBID}" ]] && scancel ${JOBID}) +# when: always + +# Here are all quartz build jobs +Q_build_deps_gcc: + extends: [.build_deps_on_quartz, .with_gcc] + +Q_install_gcc: + extends: [.install_on_quartz, .with_gcc] + needs: [Q_build_deps_gcc] + +Q_build_deps_gcc_static: + extends: [.build_deps_on_quartz, .with_gcc_static] + +Q_install_gcc_static: + extends: [.install_on_quartz, .with_gcc_static] + needs: [Q_build_deps_gcc_static] diff --git a/scripts/ci/update-uberenv.sh b/scripts/ci/update-uberenv.sh new file mode 100755 index 000000000..f927465e1 --- /dev/null +++ b/scripts/ci/update-uberenv.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [[ ! ${1} ]] +then + echo "ERROR: expecting reference for uberenv repo" >&2 +else + uberenv_ref="${1}" +fi + +uberenv_file="scripts/uberenv/uberenv.py" +uberenv_master="https://raw.githubusercontent.com/LLNL/uberenv/${uberenv_ref}/uberenv.py" + +curl --fail --output ${uberenv_file} ${uberenv_master} diff --git a/scripts/uberenv/packages/uberenv-conduit/package.py b/scripts/uberenv/packages/uberenv-conduit/package.py deleted file mode 100644 index 044ecf0d0..000000000 --- a/scripts/uberenv/packages/uberenv-conduit/package.py +++ /dev/null @@ -1,87 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -import socket -import os - -from os.path import join as pjoin -from os import environ as env - -from .conduit import Conduit - -class UberenvConduit(Conduit): - """Conduit is an open source project from Lawrence Livermore National - Laboratory that provides an intuitive model for describing hierarchical - scientific data in C++, C, Fortran, and Python. It is used for data - coupling between packages in-core, serialization, and I/O tasks.""" - - version('0.0.0', 'c8b277080a00041cfc4f64619e31f6d6',preferred=True) - # default to building docs when using uberenv - variant("doc", - default=True, - description="Build deps needed to create Conduit's Docs") - variant("adios", default=False, description="Build Conduit ADIOS support") - - - # stick with cmake 3.8 or 3.9 until we use MPIEXEC_EXECUTABLE for 3.10+ - # in upstream spack package - depends_on("cmake@3.8.2:3.9.999", when="+cmake") - - # Try some basic ADIOS configurations. NOTE: these are more extensively - # covered in the Conduit Spack base class. These seem necessary here too. - # note: Conduit always depends on hdf5 by default. We have a problem with - # HDF5. The serial parts of Conduit *cannot* use a parallel version. We - # must therefore build ADIOS without HDF5 since it cannot use a serial - # version. This should make it possible for Conduit+ADIOS to have - # serial HDF5 for relay while still having a parallel ADIOS for relay::mpi::io. - depends_on("adios+mpi~hdf5", when="+adios+mpi") - depends_on("adios~mpi~hdf5", when="+adios~mpi") - - # build phases used by this package - phases = ["configure"] - - def cmake_args(self): - args = super(UberenvConduit, self).cmake_args() - return [] - - def url_for_version(self, version): - dummy_tar_path = os.path.abspath(pjoin(os.path.split(__file__)[0])) - dummy_tar_path = pjoin(dummy_tar_path,"uberenv-conduit.tar.gz") - url = "file://" + dummy_tar_path - return url - - def configure(self, spec, prefix): - """ - Create a host config for use in conduit - """ - print("UberenvConduit.configure") - with working_dir('spack-build', create=True): - host_cfg_fname = self.create_host_config(spec, prefix) - # place a copy in the spack install dir for the uberenv-conduit package - mkdirp(prefix) - install(host_cfg_fname,prefix) - install(host_cfg_fname,env["SPACK_DEBUG_LOG_DIR"]) - diff --git a/scripts/uberenv/packages/uberenv-conduit/uberenv-conduit.tar.gz b/scripts/uberenv/packages/uberenv-conduit/uberenv-conduit.tar.gz deleted file mode 100644 index b0ab276d4..000000000 Binary files a/scripts/uberenv/packages/uberenv-conduit/uberenv-conduit.tar.gz and /dev/null differ diff --git a/scripts/uberenv/project.json b/scripts/uberenv/project.json index 72e35ecc6..3fd8003ff 100644 --- a/scripts/uberenv/project.json +++ b/scripts/uberenv/project.json @@ -1,9 +1,11 @@ { "package_name" : "conduit", -"uberenv_package_name" : "uberenv-conduit", -"spack_url": "https://github.com/alpine-DAV/spack", -"spack_branch": "task/2019_10_update_conduit", -"spack_activate" : {"py-numpy" : ["+python"], - "py-sphinx": ["+python","+doc"], +"package_version" : "master", +"package_final_phase" : "configure", +"package_source_dir" : "../..", +"spack_url": "https://github.com/spack/spack", +"spack_commit": "6d2e6e1f4", +"spack_activate" : {"py-numpy" : ["+python"], + "py-sphinx": ["+python","+doc"], "py-sphinx-rtd-theme": ["+python","+doc"] } } diff --git a/scripts/uberenv/uberenv.py b/scripts/uberenv/uberenv.py index b1e713035..d292cb365 100755 --- a/scripts/uberenv/uberenv.py +++ b/scripts/uberenv/uberenv.py @@ -67,7 +67,7 @@ from os.path import join as pjoin -def sexe(cmd,ret_output=False,echo = False): +def sexe(cmd, ret_output=False, print_output=True, echo=False): """ Helper for executing shell commands. """ if echo: print("[exe: {}]".format(cmd)) @@ -75,10 +75,22 @@ def sexe(cmd,ret_output=False,echo = False): p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - res = p.communicate()[0] - res = res.decode('utf8') - return p.returncode,res + stderr=subprocess.STDOUT, + universal_newlines=True) + full_output = "" + if print_output: + while True: + output = p.stdout.readline() + full_output += output + if output == '' and p.poll() is not None: + break + if output: + print(output.strip('\n')) + rc = p.poll() + else: + full_output = p.communicate()[0] + rc = p.returncode + return rc, full_output else: return subprocess.call(cmd,shell=True) @@ -90,28 +102,39 @@ def parse_args(): action="store_true", dest="install", default=False, - help="Install `package_name` instead of `uberenv_package_name`.") + help="Install `package_name`, not just its dependencies.") + # where to install parser.add_option("--prefix", dest="prefix", default="uberenv_libs", help="destination directory") + # what compiler to use parser.add_option("--spec", dest="spec", default=None, help="spack compiler spec") + # optional location of spack mirror parser.add_option("--mirror", dest="mirror", default=None, help="spack mirror directory") + # flag to create mirror parser.add_option("--create-mirror", action="store_true", dest="create_mirror", default=False, help="Create spack mirror") + + # optional location of spack upstream + parser.add_option("--upstream", + dest="upstream", + default=None, + help="add an external spack instance as upstream") + # this option allows a user to explicitly to select a # group of spack settings files (compilers.yaml , packages.yaml) parser.add_option("--spack-config-dir", @@ -119,6 +142,24 @@ def parse_args(): default=None, help="dir with spack settings files (compilers.yaml, packages.yaml, etc)") + # overrides package_name + parser.add_option("--package-name", + dest="package_name", + default=None, + help="override the default package name") + + # controls after which package phase spack should stop + parser.add_option("--package-final-phase", + dest="package_final_phase", + default=None, + help="override the default phase after which spack should stop") + + # controls source_dir spack should use to build the package + parser.add_option("--package-source-dir", + dest="package_source_dir", + default=None, + help="override the default source dir spack should use") + # a file that holds settings for a specific project # using uberenv.py parser.add_option("--project-json", @@ -189,127 +230,444 @@ def load_json_file(json_file): # reads json file return json.load(open(json_file)) -def uberenv_detect_platform(): - # find supported sets of compilers.yaml, packages,yaml - res = None - if "darwin" in platform.system().lower(): - res = "darwin" - elif "SYS_TYPE" in os.environ.keys(): - sys_type = os.environ["SYS_TYPE"].lower() - res = sys_type - return res +def is_darwin(): + return "darwin" in platform.system().lower() -def uberenv_spack_config_dir(opts, uberenv_dir): - # path to compilers.yaml, which we will for compiler setup for spack - spack_config_dir = opts["spack_config_dir"] - if spack_config_dir is None: - uberenv_plat = uberenv_detect_platform() - if not uberenv_plat is None: - spack_config_dir = os.path.abspath(pjoin(uberenv_dir,"spack_configs",uberenv_plat)) - return spack_config_dir - - -def disable_spack_config_scopes(spack_dir): - # disables all config scopes except "default", which we will - # force our settings into - spack_lib_config = pjoin(spack_dir,"lib","spack","spack","config.py") - print("[disabling config scope (except default) in: {}]".format(spack_lib_config)) - cfg_script = open(spack_lib_config).read() - for cfg_scope_stmt in ["('system', os.path.join(spack.paths.system_etc_path, 'spack')),", - "('site', os.path.join(spack.paths.etc_path, 'spack')),", - "('user', spack.paths.user_config_path)"]: - cfg_script = cfg_script.replace(cfg_scope_stmt, - "#DISABLED BY UBERENV: " + cfg_scope_stmt) - open(spack_lib_config,"w").write(cfg_script) - - - -def patch_spack(spack_dir,uberenv_dir,cfg_dir,pkgs): - # force spack to use only default config scope - disable_spack_config_scopes(spack_dir) - spack_etc_defaults_dir = pjoin(spack_dir,"etc","spack","defaults") - # copy in default config.yaml - config_yaml = os.path.abspath(pjoin(uberenv_dir,"spack_configs","config.yaml")) - sexe("cp {} {}/".format(config_yaml, spack_etc_defaults_dir ), echo=True) - # copy in other settings per platform - if not cfg_dir is None: - print("[copying uberenv compiler and packages settings from {0}]".format(cfg_dir)) - - config_yaml = pjoin(cfg_dir,"config.yaml") - compilers_yaml = pjoin(cfg_dir,"compilers.yaml") - packages_yaml = pjoin(cfg_dir,"packages.yaml") - - if os.path.isfile(config_yaml): - sexe("cp {} {}/".format(config_yaml , spack_etc_defaults_dir ), echo=True) - - if os.path.isfile(compilers_yaml): - sexe("cp {} {}/".format(compilers_yaml, spack_etc_defaults_dir ), echo=True) - - if os.path.isfile(packages_yaml): - sexe("cp {} {}/".format(packages_yaml, spack_etc_defaults_dir ), echo=True) - else: - # let spack try to auto find compilers - sexe("spack/bin/spack compiler find", echo=True) - dest_spack_pkgs = pjoin(spack_dir,"var","spack","repos","builtin","packages") - # hot-copy our packages into spack - sexe("cp -Rf %s %s" % (pkgs,dest_spack_pkgs)) +def is_windows(): + return "windows" in platform.system().lower() +class UberEnv(): + """ Base class for package manager """ -def create_spack_mirror(mirror_path,pkg_name,ignore_ssl_errors=False): - """ - Creates a spack mirror for pkg_name at mirror_path. - """ - if not mirror_path: - print("[--create-mirror requires a mirror directory]") + def __init__(self, opts, extra_opts): + self.opts = opts + self.extra_opts = extra_opts + + # load project settings + self.project_opts = load_json_file(opts["project_json"]) + print("[uberenv project settings: {}]".format(str(self.project_opts))) + print("[uberenv options: {}]".format(str(self.opts))) + + def setup_paths_and_dirs(self): + self.uberenv_path = os.path.dirname(os.path.realpath(__file__)) + + def set_from_args_or_json(self,setting): + try: + setting_value = self.project_opts[setting] + except (KeyError): + print("ERROR: {} must at least be defined in project.json".format(setting)) + raise + else: + if self.opts[setting]: + setting_value = self.opts[setting] + return setting_value + + def set_from_json(self,setting): + try: + setting_value = self.project_opts[setting] + except (KeyError): + print("ERROR: {} must at least be defined in project.json".format(setting)) + raise + return setting_value + + def detect_platform(self): + # find supported sets of compilers.yaml, packages,yaml + res = None + if is_darwin(): + res = "darwin" + elif "SYS_TYPE" in os.environ.keys(): + sys_type = os.environ["SYS_TYPE"].lower() + res = sys_type + return res + + +class SpackEnv(UberEnv): + """ Helper to clone spack and install libraries on MacOS an Linux """ + + def __init__(self, opts, extra_opts): + UberEnv.__init__(self,opts,extra_opts) + + self.pkg_name = self.set_from_args_or_json("package_name") + self.pkg_version = self.set_from_json("package_version") + self.pkg_final_phase = self.set_from_args_or_json("package_final_phase") + self.pkg_src_dir = self.set_from_args_or_json("package_source_dir") + + # Some additional setup for macos + if is_darwin(): + if opts["macos_sdk_env_setup"]: + # setup osx deployment target and sdk settings + setup_osx_sdk_env_vars() + else: + print("[skipping MACOSX env var setup]") + + # setup default spec + if opts["spec"] is None: + if is_darwin(): + opts["spec"] = "%clang" + else: + opts["spec"] = "%gcc" + self.opts["spec"] = "@{}{}".format(self.pkg_version,opts["spec"]) + elif not opts["spec"].startswith("@"): + self.opts["spec"] = "@{}{}".format(self.pkg_version,opts["spec"]) + else: + self.opts["spec"] = "{}".format(opts["spec"]) + + print("[spack spec: {}]".format(self.opts["spec"])) + + def setup_paths_and_dirs(self): + # get the current working path, and the glob used to identify the + # package files we want to hot-copy to spack + + UberEnv.setup_paths_and_dirs(self) + + self.pkgs = pjoin(self.uberenv_path, "packages","*") + + # setup destination paths + self.dest_dir = os.path.abspath(self.opts["prefix"]) + self.dest_spack = pjoin(self.dest_dir,"spack") + print("[installing to: {0}]".format(self.dest_dir)) + + # print a warning if the dest path already exists + if not os.path.isdir(self.dest_dir): + os.mkdir(self.dest_dir) + else: + print("[info: destination '{}' already exists]".format(self.dest_dir)) + + if os.path.isdir(self.dest_spack): + print("[info: destination '{}' already exists]".format(self.dest_spack)) + + self.pkg_src_dir = os.path.join(self.uberenv_path,self.pkg_src_dir) + if not os.path.isdir(self.pkg_src_dir): + print("[ERROR: package_source_dir '{}' does not exist]".format(self.pkg_src_dir)) + sys.exit(-1) + + + def find_spack_pkg_path(self,pkg_name,spec): + r,rout = sexe("spack/bin/spack find -p " + pkg_name + spec,ret_output = True) + for l in rout.split("\n"): + # TODO: at least print a warning when several choices exist. This will + # pick the first in the list. + if l.startswith(pkg_name): + return {"name": pkg_name, "path": l.split()[-1]} + print("[ERROR: failed to find package named '{}']".format(pkg_name)) sys.exit(-1) - mirror_path = os.path.abspath(mirror_path) - mirror_cmd = "spack/bin/spack " - if ignore_ssl_errors: - mirror_cmd += "-k " - mirror_cmd += "mirror create -d {} --dependencies {}".format(mirror_path, - pkg_name) - return sexe(mirror_cmd, echo=True) + def read_spack_full_spec(self,pkg_name,spec): + rv, res = sexe("spack/bin/spack spec " + pkg_name + " " + spec, ret_output=True, print_output=False) + for l in res.split("\n"): + if l.startswith(pkg_name) and l.count("@") > 0 and l.count("arch=") > 0: + return l.strip() + + def clone_repo(self): + if not os.path.isdir(self.dest_spack): + + # compose clone command for the dest path, spack url and branch + print("[info: cloning spack develop branch from github]") + + os.chdir(self.dest_dir) + + clone_opts = ("-c http.sslVerify=false " + if self.opts["ignore_ssl_errors"] else "") + + spack_branch = self.project_opts.get("spack_branch", "develop") + spack_url = self.project_opts.get("spack_url", "https://github.com/spack/spack.git") + + clone_cmd = "git {} clone -b {} {}".format(clone_opts, spack_branch,spack_url) + sexe(clone_cmd, echo=True) + + # optionally, check out a specific commit + if "spack_commit" in self.project_opts: + sha1 = self.project_opts["spack_commit"] + print("[info: using spack commit {}]".format(sha1)) + os.chdir(pjoin(self.dest_dir,"spack")) + sexe("git checkout {}".format(sha1),echo=True) + + if self.opts["spack_pull"]: + # do a pull to make sure we have the latest + os.chdir(pjoin(self.dest_dir,"spack")) + sexe("git stash", echo=True) + sexe("git pull", echo=True) + + def config_dir(self): + """ path to compilers.yaml, which we will use for spack's compiler setup""" + spack_config_dir = self.opts["spack_config_dir"] + if spack_config_dir is None: + uberenv_plat = self.detect_platform() + if not uberenv_plat is None: + spack_config_dir = os.path.abspath(pjoin(self.uberenv_path,"spack_configs",uberenv_plat)) + return spack_config_dir + + + def disable_spack_config_scopes(self,spack_dir): + # disables all config scopes except "defaults", which we will + # force our settings into + spack_lib_config = pjoin(spack_dir,"lib","spack","spack","config.py") + print("[disabling config scope (except defaults) in: {}]".format(spack_lib_config)) + cfg_script = open(spack_lib_config).read() + for cfg_scope_stmt in ["('system', os.path.join(spack.paths.system_etc_path, 'spack')),", + "('site', os.path.join(spack.paths.etc_path, 'spack')),", + "('user', spack.paths.user_config_path)"]: + cfg_script = cfg_script.replace(cfg_scope_stmt, + "#DISABLED BY UBERENV: " + cfg_scope_stmt) + open(spack_lib_config,"w").write(cfg_script) -def find_spack_mirror(spack_dir, mirror_name): - """ - Returns the path of a site scoped spack mirror with the - given name, or None if no mirror exists. - """ - rv, res = sexe("spack/bin/spack mirror list", ret_output=True) - mirror_path = None - for mirror in res.split('\n'): - if mirror: - parts = mirror.split() - if parts[0] == mirror_name: - mirror_path = parts[1] - return mirror_path - - -def use_spack_mirror(spack_dir, - mirror_name, - mirror_path): - """ - Configures spack to use mirror at a given path. - """ - mirror_path = os.path.abspath(mirror_path) - existing_mirror_path = find_spack_mirror(spack_dir, mirror_name) - if existing_mirror_path and mirror_path != existing_mirror_path: - # Existing mirror has different URL, error out - print("[removing existing spack mirror `{}` @ {}]".format(mirror_name, - existing_mirror_path)) - # - # Note: In this case, spack says it removes the mirror, but we still - # get errors when we try to add a new one, sounds like a bug - # - sexe("spack/bin/spack mirror remove --scope=site {} ".format(mirror_name), - echo=True) - existing_mirror_path = None - if not existing_mirror_path: - # Add if not already there - sexe("spack/bin/spack mirror add --scope=site {} {}".format( - mirror_name, mirror_path), echo=True) - print("[using mirror {}]".format(mirror_path)) + + def patch(self): + + cfg_dir = self.config_dir() + spack_dir = self.dest_spack + + # force spack to use only "defaults" config scope + self.disable_spack_config_scopes(spack_dir) + spack_etc_defaults_dir = pjoin(spack_dir,"etc","spack","defaults") + + # copy in "defaults" config.yaml + config_yaml = os.path.abspath(pjoin(self.uberenv_path,"spack_configs","config.yaml")) + sexe("cp {} {}/".format(config_yaml, spack_etc_defaults_dir ), echo=True) + + # copy in other settings per platform + if not cfg_dir is None: + print("[copying uberenv compiler and packages settings from {0}]".format(cfg_dir)) + + config_yaml = pjoin(cfg_dir,"config.yaml") + compilers_yaml = pjoin(cfg_dir,"compilers.yaml") + packages_yaml = pjoin(cfg_dir,"packages.yaml") + + if os.path.isfile(config_yaml): + sexe("cp {} {}/".format(config_yaml , spack_etc_defaults_dir ), echo=True) + + if os.path.isfile(compilers_yaml): + sexe("cp {} {}/".format(compilers_yaml, spack_etc_defaults_dir ), echo=True) + + if os.path.isfile(packages_yaml): + sexe("cp {} {}/".format(packages_yaml, spack_etc_defaults_dir ), echo=True) + else: + # let spack try to auto find compilers + sexe("spack/bin/spack compiler find", echo=True) + + if not self.pkgs: + # hot-copy our packages into spack + dest_spack_pkgs = pjoin(spack_dir,"var","spack","repos","builtin","packages") + sexe("cp -Rf {} {}".format(self.pkgs,dest_spack_pkgs)) + + + def clean_build(self): + # clean out any temporary spack build stages + cln_cmd = "spack/bin/spack clean " + res = sexe(cln_cmd, echo=True) + + # clean out any spack cached stuff + cln_cmd = "spack/bin/spack clean --all" + res = sexe(cln_cmd, echo=True) + + # check if we need to force uninstall of selected packages + if self.opts["spack_clean"]: + if self.project_opts.has_key("spack_clean_packages"): + for cln_pkg in self.project_opts["spack_clean_packages"]: + if not self.find_spack_pkg_path(cln_pkg, self.opts["spec"]) is None: + unist_cmd = "spack/bin/spack uninstall -f -y --all --dependents " + cln_pkg + res = sexe(unist_cmd, echo=True) + + def show_info(self): + spec_cmd = "spack/bin/spack spec " + self.pkg_name + self.opts["spec"] + return sexe(spec_cmd, echo=True) + + def install(self): + # use the uberenv package to trigger the right builds + # and build an host-config.cmake file + install_cmd = "spack/bin/spack " + if self.opts["ignore_ssl_errors"]: + install_cmd += "-k " + if not self.opts["install"]: + install_cmd += "dev-build --quiet -d {} -u {} ".format(self.pkg_src_dir,self.pkg_final_phase) + else: + install_cmd += "install " + if self.opts["run_tests"]: + install_cmd += "--test=root " + install_cmd += self.pkg_name + self.opts["spec"] + res, out = sexe(install_cmd, ret_output=True, echo=True) + if res != 0: + error_key="==> Error: Already installed in " + install_path="" + for line in out.split("\n"): + if line.startswith(error_key): + install_path=line.replace(error_key,"") + + if install_path and os.path.isdir(install_path): + print("[Warning: {} has already been install in {}]".format(self.pkg_name,install_path)) + print("[Warning: Uberenv will proceed using this directory]".format(self.pkg_name)) + self.opts["use_install"] = True + elif not install_path: + return res + else: + print("[ERROR: not a directory {}".format(install_path)) + return res + + if "spack_activate" in self.project_opts: + print("[activating dependent packages]") + # get the full spack spec for our project + full_spec = self.read_spack_full_spec(self.pkg_name,self.opts["spec"]) + pkg_names = self.project_opts["spack_activate"].keys() + for pkg_name in pkg_names: + pkg_spec_requirements = self.project_opts["spack_activate"][pkg_name] + activate=True + for req in pkg_spec_requirements: + if req not in full_spec: + activate=False + break + if activate: + activate_cmd = "spack/bin/spack activate " + pkg_name + sexe(activate_cmd, echo=True) + # note: this assumes package extends python when +python + # this may fail general cases + if self.opts["install"] and "+python" in full_spec: + activate_cmd = "spack/bin/spack activate " + self.pkg_name + self.opts["spec"] + sexe(activate_cmd, echo=True) + # if user opt'd for an install, we want to symlink the final + # install to an easy place: + if self.opts["install"] or "use_install" in self.opts: + pkg_path = self.find_spack_pkg_path(self.pkg_name,self.opts["spec"]) + if self.pkg_name != pkg_path["name"]: + print("[ERROR: Could not find install of {}]".format(self.pkg_name)) + return -1 + else: + # Symlink host-config file + hcfg_glob = glob.glob(pjoin(pkg_path["path"],"*.cmake")) + if len(hcfg_glob) > 0: + hcfg_path = hcfg_glob[0] + hcfg_fname = os.path.split(hcfg_path)[1] + if os.path.islink(hcfg_fname): + os.unlink(hcfg_fname) + elif os.path.isfile(hcfg_fname): + sexe("rm -f {}".format(hcfg_fname)) + print("[symlinking host config file to {}]".format(pjoin(self.dest_dir,hcfg_fname))) + os.symlink(hcfg_path,hcfg_fname) + + # Symlink install directory + if self.opts["install"]: + pkg_lnk_dir = "{}-install".format(self.pkg_name) + if os.path.islink(pkg_lnk_dir): + os.unlink(pkg_lnk_dir) + print("") + print("[symlinking install to {}]".format(pjoin(self.dest_dir,pkg_lnk_dir))) + os.symlink(pkg_path["path"],os.path.abspath(pkg_lnk_dir)) + print("") + print("[install complete!]") + else: + pattern = "*{}.cmake".format(self.pkg_name) + build_dir = pjoin(self.pkg_src_dir,"spack-build") + hcfg_glob = glob.glob(pjoin(build_dir,pattern)) + if len(hcfg_glob) > 0: + hcfg_path = hcfg_glob[0] + hcfg_fname = os.path.split(hcfg_path)[1] + if os.path.islink(hcfg_fname): + os.unlink(hcfg_fname) + print("[copying host config file to {}]".format(pjoin(self.dest_dir,hcfg_fname))) + sexe("cp {} {}".format(hcfg_path,hcfg_fname)) + print("[removing project build directory {}]".format(pjoin(build_dir))) + sexe("rm -rf {}".format(build_dir)) + + def get_mirror_path(self): + mirror_path = self.opts["mirror"] + if not mirror_path: + print("[--create-mirror requires a mirror directory]") + sys.exit(-1) + return os.path.abspath(mirror_path) + + def create_mirror(self): + """ + Creates a spack mirror for pkg_name at mirror_path. + """ + + mirror_path = self.get_mirror_path() + + mirror_cmd = "spack/bin/spack " + if self.opts["ignore_ssl_errors"]: + mirror_cmd += "-k " + mirror_cmd += "mirror create -d {} --dependencies {}".format(mirror_path, + self.pkg_name) + return sexe(mirror_cmd, echo=True) + + def find_spack_mirror(self, mirror_name): + """ + Returns the path of a defaults scoped spack mirror with the + given name, or None if no mirror exists. + """ + rv, res = sexe("spack/bin/spack mirror list", ret_output=True) + mirror_path = None + for mirror in res.split('\n'): + if mirror: + parts = mirror.split() + if parts[0] == mirror_name: + mirror_path = parts[1] + return mirror_path + + def use_mirror(self): + """ + Configures spack to use mirror at a given path. + """ + mirror_name = self.pkg_name + mirror_path = self.get_mirror_path() + existing_mirror_path = self.find_spack_mirror(mirror_name) + + if existing_mirror_path and mirror_path != existing_mirror_path: + # Existing mirror has different URL, error out + print("[removing existing spack mirror `{}` @ {}]".format(mirror_name, + existing_mirror_path)) + # + # Note: In this case, spack says it removes the mirror, but we still + # get errors when we try to add a new one, sounds like a bug + # + sexe("spack/bin/spack mirror remove --scope=defaults {} ".format(mirror_name), + echo=True) + existing_mirror_path = None + if not existing_mirror_path: + # Add if not already there + sexe("spack/bin/spack mirror add --scope=defaults {} {}".format( + mirror_name, mirror_path), echo=True) + print("[using mirror {}]".format(mirror_path)) + + def find_spack_upstream(self, upstream_name): + """ + Returns the path of a defaults scoped spack upstream with the + given name, or None if no upstream exists. + """ + upstream_path = None + + rv, res = sexe('spack/bin/spack config get upstreams', ret_output=True) + if (not res) and ("upstreams:" in res): + res = res.replace(' ', '') + res = res.replace('install_tree:', '') + res = res.replace(':', '') + res = res.splitlines() + res = res[1:] + upstreams = dict(zip(res[::2], res[1::2])) + + for name in upstreams.keys(): + if name == upstream_name: + upstream_path = upstreams[name] + + return upstream_path + + def use_spack_upstream(self): + """ + Configures spack to use upstream at a given path. + """ + upstream_path = self.opts["upstream"] + if not upstream_path: + print("[--create-upstream requires a upstream directory]") + sys.exit(-1) + upstream_path = os.path.abspath(upstream_path) + upstream_name = self.pkg_name + existing_upstream_path = self.find_spack_upstream(upstream_name) + if (not existing_upstream_path) or (upstream_path != os.path.abspath(existing_upstream_path)): + # Existing upstream has different URL, error out + print("[removing existing spack upstream configuration file]") + sexe("rm spack/etc/spack/defaults/upstreams.yaml") + with open('spack/etc/spack/defaults/upstreams.yaml','w+') as upstreams_cfg_file: + upstreams_cfg_file.write("upstreams:\n") + upstreams_cfg_file.write(" {}:\n".format(upstream_name)) + upstreams_cfg_file.write(" install_tree: {}\n".format(upstream_path)) def find_osx_sdks(): @@ -353,119 +711,36 @@ def setup_osx_sdk_env_vars(): print("[setting SDKROOT to {}]".format(env[ "SDKROOT"])) -def find_spack_pkg_path(pkg_name): - r,rout = sexe("spack/bin/spack find -p " + pkg_name,ret_output = True) - for l in rout.split("\n"): - lstrip = l.strip() - if not lstrip == "" and \ - not lstrip.startswith("==>") and \ - not lstrip.startswith("--"): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - -def read_spack_full_spec(pkg_name,spec): - rv, res = sexe("spack/bin/spack spec " + pkg_name + " " + spec, ret_output=True) - for l in res.split("\n"): - if l.startswith(pkg_name) and l.count("@") > 0 and l.count("arch=") > 0: - return l.strip() + def main(): """ - clones and runs spack to setup our third_party libs and - creates a host-config.cmake file that can be used by - our project. + Clones and runs a package manager to setup third_party libs. + Also creates a host-config.cmake file that can be used by our project. """ + # parse args from command line - opts, extras = parse_args() + opts, extra_opts = parse_args() + + # Initialize the environment + env = SpackEnv(opts, extra_opts) + + # Setup the necessary paths and directories + env.setup_paths_and_dirs() + + # Clone the package manager + env.clone_repo() + + os.chdir(env.dest_dir) + + # Patch the package manager, as necessary + env.patch() + + # Clean the build + env.clean_build() + + # Show the spec for what will be built + env.show_info() - # load project settings - project_opts = load_json_file(opts["project_json"]) - if opts["install"]: - uberenv_pkg_name = project_opts["package_name"] - else: - uberenv_pkg_name = project_opts["uberenv_package_name"] - print("[uberenv project settings: {}]".format(str(project_opts))) - print("[uberenv options: {}]".format(str(opts))) - if "darwin" in platform.system().lower(): - if opts["macos_sdk_env_setup"]: - # setup osx deployment target and sdk settings - setup_osx_sdk_env_vars() - else: - print("[skipping MACOSX env var setup]") - # setup default spec - if opts["spec"] is None: - if "darwin" in platform.system().lower(): - opts["spec"] = "%clang" - else: - opts["spec"] = "%gcc" - print("[spack spec: {}]".format(opts["spec"])) - # get the current working path, and the glob used to identify the - # package files we want to hot-copy to spack - uberenv_path = os.path.split(os.path.abspath(__file__))[0] - pkgs = pjoin(uberenv_path, "packages","*") - # setup destination paths - dest_dir = os.path.abspath(opts["prefix"]) - dest_spack = pjoin(dest_dir,"spack") - print("[installing to: {0}]".format(dest_dir)) - # print a warning if the dest path already exists - if not os.path.isdir(dest_dir): - os.mkdir(dest_dir) - else: - print("[info: destination '{}' already exists]".format(dest_dir)) - if os.path.isdir(dest_spack): - print("[info: destination '{}' already exists]".format(dest_spack)) - - if not os.path.isdir(dest_spack): - print("[info: cloning spack develop branch from github]") - os.chdir(dest_dir) - # clone spack into the dest path - clone_cmd ="git " - if opts["ignore_ssl_errors"]: - clone_cmd +="-c http.sslVerify=false " - spack_url = "https://github.com/spack/spack.git" - spack_branch = "develop" - if "spack_url" in project_opts: - spack_url = project_opts["spack_url"] - if "spack_branch" in project_opts: - spack_branch = project_opts["spack_branch"] - clone_cmd += "clone -b %s %s" % (spack_branch,spack_url) - sexe(clone_cmd, echo=True) - if "spack_commit" in project_opts: - sha1 = project_opts["spack_commit"] - print("[info: using spack commit {}]".format(sha1)) - os.chdir(pjoin(dest_dir,"spack")) - sexe("git checkout %s" % sha1,echo=True) - - if opts["spack_pull"]: - # do a pull to make sure we have the latest - os.chdir(pjoin(dest_dir,"spack")) - sexe("git stash", echo=True) - sexe("git pull", echo=True) - - os.chdir(dest_dir) - # twist spack's arms - cfg_dir = uberenv_spack_config_dir(opts, uberenv_path) - patch_spack(dest_spack, uberenv_path, cfg_dir, pkgs) - - # show the spec for what will be built - spec_cmd = "spack/bin/spack spec " + uberenv_pkg_name + opts["spec"] - res = sexe(spec_cmd, echo=True) - - # clean out any temporary spack build stages - cln_cmd = "spack/bin/spack clean " - res = sexe(cln_cmd, echo=True) - - # clean out any spack cached stuff - cln_cmd = "spack/bin/spack clean --all" - res = sexe(cln_cmd, echo=True) - - # check if we need to force uninstall of selected packages - if opts["spack_clean"]: - if project_opts.has_key("spack_clean_packages"): - for cln_pkg in project_opts["spack_clean_packages"]: - if not find_spack_pkg_path(cln_pkg) is None: - unist_cmd = "spack/bin/spack uninstall -f -y --all --dependents " + cln_pkg - res = sexe(unist_cmd, echo=True) ########################################################## # we now have an instance of spack configured how we @@ -478,70 +753,16 @@ def main(): # ########################################################## if opts["create_mirror"]: - return create_spack_mirror(opts["mirror"], - uberenv_pkg_name, - opts["ignore_ssl_errors"]) + return env.create_mirror() else: if not opts["mirror"] is None: - use_spack_mirror(dest_spack, - uberenv_pkg_name, - opts["mirror"]) - # use the uberenv package to trigger the right builds - # and build an host-config.cmake file - install_cmd = "spack/bin/spack " - if opts["ignore_ssl_errors"]: - install_cmd += "-k " - install_cmd += "install " - if opts["run_tests"]: - install_cmd += "--test=root " - install_cmd += uberenv_pkg_name + opts["spec"] - res = sexe(install_cmd, echo=True) - if res != 0: - return res - if "spack_activate" in project_opts: - print("[activating dependent packages]") - # get the full spack spec for our project - full_spec = read_spack_full_spec(uberenv_pkg_name,opts["spec"]) - pkg_names = project_opts["spack_activate"].keys() - for pkg_name in pkg_names: - pkg_spec_requirements = project_opts["spack_activate"][pkg_name] - activate=True - for req in pkg_spec_requirements: - if req not in full_spec: - activate=False - break - if activate: - activate_cmd = "spack/bin/spack activate " + pkg_name - sexe(activate_cmd, echo=True) - # note: this assumes package extends python when +python - # this may fail general cases - if opts["install"] and "+python" in full_spec: - activate_cmd = "spack/bin/spack activate " + uberenv_pkg_name - sexe(activate_cmd, echo=True) - # if user opt'd for an install, we want to symlink the final ascent - # install to an easy place: - if opts["install"]: - pkg_path = find_spack_pkg_path(uberenv_pkg_name) - if uberenv_pkg_name != pkg_path["name"]: - print("[ERROR: Could not find install of {}]".format(uberenv_pkg_name)) - return -1 - else: - pkg_lnk_dir = "{}-install".format(uberenv_pkg_name) - if os.path.islink(pkg_lnk_dir): - os.unlink(pkg_lnk_dir) - print("") - print("[symlinking install to {}]".format(pjoin(dest_dir,pkg_lnk_dir))) - os.symlink(pkg_path["path"],os.path.abspath(pkg_lnk_dir)) - hcfg_glob = glob.glob(pjoin(pkg_lnk_dir,"*.cmake")) - if len(hcfg_glob) > 0: - hcfg_path = hcfg_glob[0] - hcfg_fname = os.path.split(hcfg_path)[1] - if os.path.islink(hcfg_fname): - os.unlink(hcfg_fname) - print("[symlinking host config file to {}]".format(pjoin(dest_dir,hcfg_fname))) - os.symlink(hcfg_path,hcfg_fname) - print("") - print("[install complete!]") + env.use_mirror() + + if not opts["upstream"] is None: + env.use_spack_upstream() + + res = env.install() + return res if __name__ == "__main__": diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 07a759fa9..57d6bae15 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -72,8 +72,8 @@ option(ENABLE_FORTRAN "Build Fortran Support" OFF) option(ENABLE_MPI "Build MPI Support" OFF) -# Add another option that provides extra -# control over conduit tests for cases where +# Add another option that provides extra +# control over conduit tests for cases where # conduit is brought in as a submodule option(CONDUIT_ENABLE_TESTS "Build conduit tests" ON) @@ -91,7 +91,7 @@ set(ENABLE_ALL_WARNINGS OFF CACHE BOOL "") ################################ # Init BLT ################################ -# This also includes +# This also includes # Conduit's BLT defaults include(cmake/SetupBLT.cmake) diff --git a/src/cmake/SetupBLT.cmake b/src/cmake/SetupBLT.cmake index 52eb57ccb..38870be58 100644 --- a/src/cmake/SetupBLT.cmake +++ b/src/cmake/SetupBLT.cmake @@ -1,45 +1,45 @@ ############################################################################### # Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -# +# # Produced at the Lawrence Livermore National Laboratory -# +# # LLNL-CODE-666778 -# +# # All rights reserved. -# -# This file is part of Conduit. -# +# +# This file is part of Conduit. +# # For details, see: http://software.llnl.gov/conduit/. -# +# # Please also read conduit/LICENSE -# -# Redistribution and use in source and binary forms, with or without +# +# Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, +# +# * Redistributions of source code must retain the above copyright notice, # this list of conditions and the disclaimer below. -# +# # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the disclaimer (as noted below) in the # documentation and/or other materials provided with the distribution. -# +# # * Neither the name of the LLNS/LLNL nor the names of its contributors may # be used to endorse or promote products derived from this software without # specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# +# ############################################################################### ################################################################