helper to gen uberenv mirror, use mirror for ci

.travis.yml

@@ -178,7 +178,7 @@ script:
   - while sleep 540; do echo "=====[ $SECONDS seconds still running ]====="; done &
   # build deps using uberenv
   # use -k to avoid suprises related to certs
-  - python scripts/uberenv/uberenv.py -k --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv/spack_configs/travis/ || travis_terminate 1;
+  - python scripts/uberenv/uberenv.py -k --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv/spack_configs/travis/ --mirror=https://www.ascent-dav.org/mirror/conduit/latest/ || travis_terminate 1;
   # todo:
   #- export SPACK_PYTHON_BIN_DIR=`ls -d ${TRAVIS_BUILD_DIR}/uberenv_libs/spack/opt/spack/*/*/python*/bin`
   #- pip install cpp-coveralls

azure-pipelines.yml

@@ -185,7 +185,7 @@ stages:
            # show final spec
            echo $SPACK_SPEC
            # run uber to build tpls
-           python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv/spack_configs/ci/ubuntu_16/
+           python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv/spack_configs/ci/ubuntu_16/ --mirror=https://www.ascent-dav.org/mirror/conduit/latest/
         displayName: 'Spack Build Tpls'
 
       - script: |
@@ -407,7 +407,7 @@ stages:
            # show final spec
            echo $SPACK_SPEC
            # run uber to build tpls
-           python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv/spack_configs/ci/ubuntu_16/
+           python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv/spack_configs/ci/ubuntu_16/ --mirror=https://www.ascent-dav.org/mirror/conduit/latest/
         displayName: 'Spack Build Tpls'
 
       - script: |

scripts/gen_uberenv_mirror.py

@@ -0,0 +1,139 @@
+###############################################################################
+# Copyright (c) 2015-2019, Lawrence Livermore National Security, LLC.
+#
+# Produced at the Lawrence Livermore National Laboratory
+#
+# LLNL-CODE-716457
+#
+# All rights reserved.
+#
+# This file is part of Ascent.
+#
+# For details, see: http://ascent.readthedocs.io/.
+#
+# Please also read ascent/LICENSE
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the disclaimer below.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the disclaimer (as noted below) in the
+#   documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of the LLNS/LLNL nor the names of its contributors may
+#   be used to endorse or promote products derived from this software without
+#   specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY,
+# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+###############################################################################
+import os
+import sys
+import subprocess
+import glob
+import datetime
+import shutil
+
+from os.path import join as pjoin
+
+
+def key_pkgs():
+    return  ["conduit"]
+
+def spec():
+    return  "+zfp+adios+silo ^silo~mpi"
+
+
+def timestamp(t=None,sep="_"):
+    """ Creates a timestamp that can easily be included in a filename. """
+    if t is None:
+        t = datetime.datetime.now()
+    #sargs = (t.year,t.month,t.day,t.hour,t.minute,t.second)
+    #sbase = "".join(["%04d",sep,"%02d",sep,"%02d",sep,"%02d",sep,"%02d",sep,"%02d"])
+    sargs = (t.year,t.month,t.day)
+    sbase = "".join(["%04d",sep,"%02d",sep,"%02d"])
+    return  sbase % sargs
+
+def sexe(cmd,ret_output=False,echo = True):
+    """ Helper for executing shell commands. """
+    if echo:
+        print("[exe: {}]".format(cmd))
+    if ret_output:
+        p = subprocess.Popen(cmd,
+                             shell=True,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.STDOUT)
+        res = p.communicate()[0]
+        res = res.decode('utf8')
+        return p.returncode,res
+    else:
+        return subprocess.call(cmd,shell=True)
+
+def dest_dir():
+    res = pjoin("uberenv_mirror","_tarballs")
+    if not os.path.isdir(res):
+        os.mkdir(res)
+    return res
+
+def copy_patches():
+    for pkg_name in key_pkgs():
+        patches = glob.glob( pjoin("uberenv_libs",
+                                   "spack",
+                                   "var",
+                                   "spack",
+                                   "repos",
+                                   "builtin",
+                                   "packages",
+                                   pkg_name,
+                                   "*.patch"))
+        for patch in patches:
+            des_path = pjoin(dest_dir(),pkg_name + "-" + os.path.split(patch)[1])
+            print("[copying patch: {0} to {1}]".format(patch,des_path))
+            shutil.copyfile(patch,des_path)
+
+def gen_key_tarballs():
+    tstamp = timestamp()
+    for pkg_name in key_pkgs():
+        pkg_tarball = glob.glob(pjoin("uberenv_mirror",pkg_name,"*.tar.gz"))
+        if len(pkg_tarball) != 1:
+            print("[Error: Could not find mirror tarball of {0}!]".format(pkg_name))
+            sys.exit(-1)
+        pkg_tarball = pkg_tarball[0]
+        src_fname = os.path.split(pkg_tarball)[1]
+        pkg_version =  src_fname[len(pkg_name)+1:]
+        des_fname =  pkg_name + "-" + tstamp + "-" + pkg_version
+        des_path = pjoin(dest_dir(),des_fname)
+        print("[copying source tarball: {0} to {1}]".format(pkg_tarball,des_path))
+        # copy to dated dir
+        shutil.copyfile(pkg_tarball,des_path)
+
+def gen_mirror():
+    cmd = 'python {0} --create-mirror --spec="{1}"'.format(pjoin("uberenv","uberenv.py"),
+                                                           spec())
+    cmd += ' --prefix=uberenv_libs --mirror=uberenv_mirror'
+    sexe(cmd)
+
+def main():
+    gen_mirror()
+    copy_patches()
+    gen_key_tarballs()
+
+
+
+
+if __name__ == "__main__":
+    main()

scripts/uberenv/uberenv.py

@@ -178,6 +178,13 @@ def parse_args():
         if not os.path.isdir(opts["spack_config_dir"]):
             print("[ERROR: invalid spack config dir: {} ]".format(opts["spack_config_dir"]))
             sys.exit(-1)
+    # if rel path is given for the mirror, we need to evaluate here -- before any
+    # chdirs to avoid confusion related to what it is relative to.
+    # (it should be relative to where uberenv is run from, so it matches what you expect
+    #  from shell completion, etc)
+    if not opts["mirror"] is None:
+        if not opts["mirror"].startswith("http") and not os.path.isabs(opts["mirror"]):
+            opts["mirror"] = os.path.abspath(opts["mirror"])
     return opts, extras
 
 
@@ -262,7 +269,6 @@ def create_spack_mirror(mirror_path,pkg_name,ignore_ssl_errors=False):
     if not mirror_path:
         print("[--create-mirror requires a mirror directory]")
         sys.exit(-1)
-    mirror_path = os.path.abspath(mirror_path)
 
     mirror_cmd = "spack/bin/spack "
     if ignore_ssl_errors:
@@ -292,7 +298,6 @@ def use_spack_mirror(spack_dir,
     """
     Configures spack to use mirror at a given path.
     """
-    mirror_path = os.path.abspath(mirror_path)
     existing_mirror_path = find_spack_mirror(spack_dir, mirror_name)
     if existing_mirror_path and mirror_path != existing_mirror_path:
         # Existing mirror has different URL, error out
@@ -479,7 +484,7 @@ def main():
     ##########################################################
     if opts["create_mirror"]:
         return create_spack_mirror(opts["mirror"],
-                                   uberenv_pkg_name,
+                                   uberenv_pkg_name + opts["spec"],
                                    opts["ignore_ssl_errors"])
     else:
         if not opts["mirror"] is None: