Update uberenv

scripts/uberenv/uberenv.py

@@ -60,37 +60,26 @@
 import json
 import datetime
 import glob
+import re
 
 from optparse import OptionParser
 
 from os import environ as env
 from os.path import join as pjoin
 
 
-def sexe(cmd, ret_output=False, print_output=True, echo=False):
+def sexe(cmd,ret_output=False,echo = False):
     """ Helper for executing shell commands. """
     if echo:
         print("[exe: {}]".format(cmd))
     if ret_output:
         p = subprocess.Popen(cmd,
                              shell=True,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.STDOUT,
-                             universal_newlines=True)
-        full_output = ""
-        if print_output:
-            while True:
-                output = p.stdout.readline()
-                full_output += output
-                if output == '' and p.poll() is not None:
-                    break
-                if output:
-                    print(output.strip('\n'))
-            rc = p.poll()
-        else:
-            full_output = p.communicate()[0]
-            rc = p.returncode
-        return rc, full_output
+                             stderr=subprocess.STDOUT)
+        res = p.communicate()[0]
+        res = res.decode('utf8')
+        return p.returncode,res
     else:
         return subprocess.call(cmd,shell=True)
 
@@ -292,6 +281,9 @@ def __init__(self, opts, extra_opts):
         self.pkg_final_phase = self.set_from_args_or_json("package_final_phase")
         self.pkg_src_dir = self.set_from_args_or_json("package_source_dir")
 
+        self.spec_hash = ""
+        self.use_install = False
+
         # Some additional setup for macos
         if is_darwin():
             if opts["macos_sdk_env_setup"]:
@@ -342,7 +334,15 @@ def setup_paths_and_dirs(self):
             sys.exit(-1)
 
 
-    def find_spack_pkg_path(self,pkg_name,spec):
+    def find_spack_pkg_path_from_hash(self, pkg_name, pkg_hash):
+        r,rout = sexe("spack/bin/spack find -p /{}".format(pkg_hash), ret_output = True)
+        for l in rout.split("\n"):
+            if l.startswith(pkg_name):
+                   return {"name": pkg_name, "path": l.split()[-1]}
+        print("[ERROR: failed to find package named '{}']".format(pkg_name))
+        sys.exit(-1)
+
+    def find_spack_pkg_path(self, pkg_name, spec = ""):
         r,rout = sexe("spack/bin/spack find -p " + pkg_name + spec,ret_output = True)
         for l in rout.split("\n"):
             # TODO: at least print a warning when several choices exist. This will
@@ -352,8 +352,9 @@ def find_spack_pkg_path(self,pkg_name,spec):
         print("[ERROR: failed to find package named '{}']".format(pkg_name))
         sys.exit(-1)
 
+    # Extract the first line of the full spec
     def read_spack_full_spec(self,pkg_name,spec):
-        rv, res = sexe("spack/bin/spack spec " + pkg_name + " " + spec, ret_output=True, print_output=False)
+        rv, res = sexe("spack/bin/spack spec " + pkg_name + " " + spec, ret_output=True)
         for l in res.split("\n"):
             if l.startswith(pkg_name) and l.count("@") > 0 and l.count("arch=") > 0:
                 return l.strip()
@@ -445,8 +446,8 @@ def patch(self):
             # let spack try to auto find compilers
             sexe("spack/bin/spack compiler find", echo=True)
 
+        # hot-copy our packages into spack
         if self.pkgs:
-            # hot-copy our packages into spack
             dest_spack_pkgs = pjoin(spack_dir,"var","spack","repos","builtin","packages")
             sexe("cp -Rf {} {}".format(self.pkgs,dest_spack_pkgs))
 
@@ -464,44 +465,53 @@ def clean_build(self):
         if self.opts["spack_clean"]:
             if self.project_opts.has_key("spack_clean_packages"):
                 for cln_pkg in self.project_opts["spack_clean_packages"]:
-                    if not self.find_spack_pkg_path(cln_pkg, self.opts["spec"]) is None:
+                    if not self.find_spack_pkg_path(cln_pkg) is None:
                         unist_cmd = "spack/bin/spack uninstall -f -y --all --dependents " + cln_pkg
                         res = sexe(unist_cmd, echo=True)
 
     def show_info(self):
-        spec_cmd = "spack/bin/spack spec " + self.pkg_name + self.opts["spec"]
-        return sexe(spec_cmd, echo=True)
+        # prints install status and 32 characters hash
+        options="--install-status --very-long"
+        spec_cmd = "spack/bin/spack spec {0} {1}{2}".format(options,self.pkg_name,self.opts["spec"])
+
+        res, out = sexe(spec_cmd, ret_output=True, echo=True)
+        print(out)
+
+        #Check if spec is already installed
+        for line in out.split("\n"):
+            # Example of matching line: ("status"  "hash"  "package"...)
+            # [+]  hf3cubkgl74ryc3qwen73kl4yfh2ijgd  serac@develop%clang@10.0.0-apple~debug~devtools~glvis arch=darwin-mojave-x86_64
+            if re.match(r"^(\[\+\]| - )  [a-z0-9]{32}  " + re.escape(self.pkg_name), line):
+                self.spec_hash = line.split("  ")[1]
+                # if spec already installed
+                if line.startswith("[+]"):
+                    pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name,self.spec_hash)
+                    install_path = pkg_path["path"]
+                    # testing that the path exists is mandatory until Spack team fixes
+                    # https://github.com/spack/spack/issues/16329
+                    if os.path.isdir(install_path):
+                        print("[Warning: {} {} has already been installed in {}]".format(self.pkg_name, self.opts["spec"],install_path))
+                        print("[Warning: Uberenv will proceed using this directory]".format(self.pkg_name))
+                        self.use_install = True
+
+        return res
 
     def install(self):
         # use the uberenv package to trigger the right builds
         # and build an host-config.cmake file
-        install_cmd = "spack/bin/spack "
-        if self.opts["ignore_ssl_errors"]:
-            install_cmd += "-k "
-        if not self.opts["install"]:
-            install_cmd += "dev-build --quiet -d {} -u {} ".format(self.pkg_src_dir,self.pkg_final_phase)
-        else:
-            install_cmd += "install "
-            if self.opts["run_tests"]:
-                install_cmd += "--test=root "
-        install_cmd += self.pkg_name + self.opts["spec"]
-        res, out = sexe(install_cmd, ret_output=True, echo=True)
-        if res != 0:
-            error_key="==> Error: Already installed in "
-            install_path=""
-            for line in out.split("\n"):
-                if line.startswith(error_key):
-                    install_path=line.replace(error_key,"")
-
-            if install_path and os.path.isdir(install_path):
-                print("[Warning: {} has already been install in {}]".format(self.pkg_name,install_path))
-                print("[Warning: Uberenv will proceed using this directory]".format(self.pkg_name))
-                self.opts["use_install"] = True
-            elif not install_path:
-                return res
+
+        if not self.use_install:
+            install_cmd = "spack/bin/spack "
+            if self.opts["ignore_ssl_errors"]:
+                install_cmd += "-k "
+            if not self.opts["install"]:
+                install_cmd += "dev-build --quiet -d {} -u {} ".format(self.pkg_src_dir,self.pkg_final_phase)
             else:
-                print("[ERROR: not a directory {}".format(install_path))
-                return res
+                install_cmd += "install "
+                if self.opts["run_tests"]:
+                    install_cmd += "--test=root "
+            install_cmd += self.pkg_name + self.opts["spec"]
+            res, out = sexe(install_cmd, ret_output=True, echo=True)
 
         full_spec = self.read_spack_full_spec(self.pkg_name,self.opts["spec"])
         if "spack_activate" in self.project_opts:
@@ -521,12 +531,12 @@ def install(self):
         # note: this assumes package extends python when +python
         # this may fail general cases
         if self.opts["install"] and "+python" in full_spec:
-            activate_cmd = "spack/bin/spack activate " + self.pkg_name + self.opts["spec"]
+            activate_cmd = "spack/bin/spack activate /" + self.spec_hash
             sexe(activate_cmd, echo=True)
         # if user opt'd for an install, we want to symlink the final
         # install to an easy place:
-        if self.opts["install"] or "use_install" in self.opts:
-            pkg_path = self.find_spack_pkg_path(self.pkg_name,self.opts["spec"])
+        if self.opts["install"] or self.use_install:
+            pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name, self.spec_hash)
             if self.pkg_name != pkg_path["name"]:
                 print("[ERROR: Could not find install of {}]".format(self.pkg_name))
                 return -1
@@ -553,6 +563,8 @@ def install(self):
                     os.symlink(pkg_path["path"],os.path.abspath(pkg_lnk_dir))
                     print("")
                     print("[install complete!]")
+        # otherwise we are in the "only dependencies" case and the host-config
+        # file has to be copied from the do-be-deleted spack-build dir.
         else:
             pattern = "*{}.cmake".format(self.pkg_name)
             build_dir = pjoin(self.pkg_src_dir,"spack-build")