do_install : allow for an arbitrary number of phases

lib/spack/llnl/util/tty/log.py

@@ -24,11 +24,11 @@
 ##############################################################################
 """Utility classes for logging the output of blocks of code.
 """
-import sys
+import multiprocessing
 import os
 import re
 import select
-import inspect
+import sys
 
 import llnl.util.tty as tty
 import llnl.util.tty.color as color
@@ -100,96 +100,111 @@ def __exit__(self, exc_type, exception, traceback):
 
 
 class log_output(object):
-    """Redirects output and error of enclosed block to a file.
+    """Spawns a daemon that reads from a pipe and writes to a file
 
     Usage:
-        with log_output(open('logfile.txt', 'w')):
-           # do things ... output will be logged.
+        # Spawns the daemon
+        with log_output('logfile.txt', 'w') as log_redirection:
+           # do things ... output is not redirected
+           with log_redirection:
+                # do things ... output will be logged
 
     or:
-        with log_output(open('logfile.txt', 'w'), echo=True):
-           # do things ... output will be logged
-           # and also printed to stdout.
-
-    Closes the provided stream when done with the block.
-    If echo is True, also prints the output to stdout.
+        with log_output('logfile.txt', echo=True) as log_redirection:
+           # do things ... output is not redirected
+           with log_redirection:
+               # do things ... output will be logged
+               # and also printed to stdout.
+
+    Opens a stream in 'w' mode at daemon spawning and closes it at
+    daemon joining. If echo is True, also prints the output to stdout.
     """
 
-    def __init__(self, stream, echo=False, force_color=False, debug=False):
-        self.stream = stream
-
-        # various output options
+    def __init__(self, filename, echo=False, force_color=False, debug=False):
+        self.filename = filename
+        # Various output options
         self.echo = echo
         self.force_color = force_color
         self.debug = debug
 
         # Default is to try file-descriptor reassignment unless the system
         # out/err streams do not have an associated file descriptor
         self.directAssignment = False
-
-    def trace(self, frame, event, arg):
-        """Jumps to __exit__ on the child process."""
-        raise _SkipWithBlock()
+        self.read, self.write = os.pipe()
+
+        # Sets a daemon that writes to file what it reads from a pipe
+        self.p = multiprocessing.Process(
+            target=self._spawn_writing_daemon,
+            args=(self.read,),
+            name='logger_daemon'
+        )
+        self.p.daemon = True
+        # Needed to un-summon the daemon
+        self.parent_pipe, self.child_pipe = multiprocessing.Pipe()
 
     def __enter__(self):
-        """Redirect output from the with block to a file.
-
-        This forks the with block as a separate process, with stdout
-        and stderr redirected back to the parent via a pipe.  If
-        echo is set, also writes to standard out.
-
-        """
-        # remember these values for later.
-        self._force_color = color._force_color
-        self._debug = tty._debug
-
-        read, write = os.pipe()
-
-        self.pid = os.fork()
-        if self.pid:
-            # Parent: read from child, skip the with block.
-            os.close(write)
-
-            read_file = os.fdopen(read, 'r', 0)
-            with self.stream as log_file:
-                with keyboard_input(sys.stdin):
-                    while True:
-                        rlist, w, x = select.select(
-                            [read_file, sys.stdin], [], [])
-                        if not rlist:
+        self.p.start()
+        return log_output.OutputRedirection(self)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.parent_pipe.send(True)
+        self.p.join(60.0)  # 1 minute to join the child
+
+    def _spawn_writing_daemon(self, read):
+        # Parent: read from child, skip the with block.
+        read_file = os.fdopen(read, 'r', 0)
+        with open(self.filename, 'w') as log_file:
+            with keyboard_input(sys.stdin):
+                while True:
+                    rlist, _, _ = select.select([read_file, sys.stdin], [], [])
+                    if not rlist:
+                        break
+
+                    # Allow user to toggle echo with 'v' key.
+                    # Currently ignores other chars.
+                    if sys.stdin in rlist:
+                        if sys.stdin.read(1) == 'v':
+                            self.echo = not self.echo
+
+                    # Handle output from the with block process.
+                    if read_file in rlist:
+                        line = read_file.readline()
+                        if not line:
+                            # For some reason we never reach this point...
                             break
 
-                        # Allow user to toggle echo with 'v' key.
-                        # Currently ignores other chars.
-                        if sys.stdin in rlist:
-                            if sys.stdin.read(1) == 'v':
-                                self.echo = not self.echo
+                        # Echo to stdout if requested.
+                        if self.echo:
+                            sys.stdout.write(line)
 
-                        # handle output from the with block process.
-                        if read_file in rlist:
-                            line = read_file.readline()
-                            if not line:
-                                break
+                        # Stripped output to log file.
+                        log_file.write(_strip(line))
+                        log_file.flush()
 
-                            # Echo to stdout if requested.
-                            if self.echo:
-                                sys.stdout.write(line)
+                    if self.child_pipe.poll():
+                        break
 
-                            # Stripped output to log file.
-                            log_file.write(_strip(line))
+    def __del__(self):
+        """Closes the pipes"""
+        os.close(self.write)
+        os.close(self.read)
 
-            read_file.flush()
-            read_file.close()
+    class OutputRedirection(object):
 
-            # Set a trace function to skip the with block.
-            sys.settrace(lambda *args, **keys: None)
-            frame = inspect.currentframe(1)
-            frame.f_trace = self.trace
+        def __init__(self, other):
+            self.__dict__.update(other.__dict__)
 
-        else:
-            # Child: redirect output, execute the with block.
-            os.close(read)
+        def __enter__(self):
+            """Redirect output from the with block to a file.
 
+            Hijacks stdout / stderr and writes to the pipe
+            connected to the logger daemon
+            """
+            # remember these values for later.
+            self._force_color = color._force_color
+            self._debug = tty._debug
+            # Redirect this output to a pipe
+            write = self.write
             try:
                 # Save old stdout and stderr
                 self._stdout = os.dup(sys.stdout.fileno())
@@ -205,53 +220,26 @@ def __enter__(self):
                 output_redirect = os.fdopen(write, 'w')
                 sys.stdout = output_redirect
                 sys.stderr = output_redirect
-
             if self.force_color:
                 color._force_color = True
-
             if self.debug:
                 tty._debug = True
 
-    def __exit__(self, exc_type, exception, traceback):
-        """Exits on child, handles skipping the with block on parent."""
-        # Child should just exit here.
-        if self.pid == 0:
+        def __exit__(self, exc_type, exception, traceback):
+            """Plugs back the original file descriptors
+            for stdout and stderr
+            """
             # Flush the log to disk.
             sys.stdout.flush()
             sys.stderr.flush()
-
-            if exception:
-                # Restore stdout on the child if there's an exception,
-                # and let it be raised normally.
-                #
-                # This assumes that even if the exception is caught,
-                # the child will exit with a nonzero return code.  If
-                # it doesn't, the child process will continue running.
-                #
-                # TODO: think about how this works outside install.
-                # TODO: ideally would propagate exception to parent...
-                if self.directAssignment:
-                    sys.stdout = self._stdout
-                    sys.stderr = self._stderr
-                else:
-                    os.dup2(self._stdout, sys.stdout.fileno())
-                    os.dup2(self._stderr, sys.stderr.fileno())
-
-                return False
-
+            if self.directAssignment:
+                # We seem to need this only to pass test/install.py
+                sys.stdout = self._stdout
+                sys.stderr = self._stderr
             else:
-                # Die quietly if there was no exception.
-                os._exit(0)
-
-        else:
-            # If the child exited badly, parent also should exit.
-            pid, returncode = os.waitpid(self.pid, 0)
-            if returncode != 0:
-                os._exit(1)
-
-        # restore output options.
-        color._force_color = self._force_color
-        tty._debug = self._debug
+                os.dup2(self._stdout, sys.stdout.fileno())
+                os.dup2(self._stderr, sys.stderr.fileno())
 
-        # Suppresses exception if it's our own.
-        return exc_type is _SkipWithBlock
+            # restore output options.
+            color._force_color = self._force_color
+            tty._debug = self._debug

lib/spack/spack/__init__.py

@@ -186,10 +186,17 @@
 #       packages should live.  This file is overloaded for spack core vs.
 #       for packages.
 #
-__all__ = ['Package', 'StagedPackage', 'CMakePackage',
-           'Version', 'when', 'ver', 'alldeps', 'nolink']
+__all__ = ['Package',
+           'CMakePackage',
+           'AutotoolsPackage',
+           'EditableMakefile',
+           'Version',
+           'when',
+           'ver',
+           'alldeps',
+           'nolink']
 from spack.package import Package, ExtensionConflictError
-from spack.package import StagedPackage, CMakePackage
+from spack.package import CMakePackage, AutotoolsPackage, EditableMakefile
 from spack.version import Version, ver
 from spack.spec import DependencySpec, alldeps, nolink
 from spack.multimethod import when

lib/spack/spack/build_environment.py

@@ -51,16 +51,14 @@
 Skimming this module is a nice way to get acquainted with the types of
 calls you can make from within the install() function.
 """
+import multiprocessing
 import os
-import sys
 import shutil
-import multiprocessing
-import platform
+import sys
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import *
-
 import spack
+from llnl.util.filesystem import *
 from spack.environment import EnvironmentModifications, validate
 from spack.util.environment import *
 from spack.util.executable import Executable, which
@@ -351,8 +349,8 @@ def set_module_variables_for_package(pkg, module):
     m.cmake = Executable('cmake')
     m.ctest = Executable('ctest')
 
-    # standard CMake arguments
-    m.std_cmake_args = get_std_cmake_args(pkg)
+    # Standard CMake arguments
+    m.std_cmake_args = spack.CMakePackage._std_args(pkg)
 
     # Put spack compiler paths in module scope.
     link_dir = spack.build_env_path
@@ -522,41 +520,26 @@ def child_fun():
     carries on.
     """
 
-    try:
-        pid = os.fork()
-    except OSError as e:
-        raise InstallError("Unable to fork build process: %s" % e)
-
-    if pid == 0:
-        # Give the child process the package's build environment.
-        setup_package(pkg, dirty=dirty)
-
+    def child_execution(child_connection):
         try:
-            # call the forked function.
+            setup_package(pkg, dirty=dirty)
             function()
-
-            # Use os._exit here to avoid raising a SystemExit exception,
-            # which interferes with unit tests.
-            os._exit(0)
-
-        except spack.error.SpackError as e:
-            e.die()
-
-        except:
-            # Child doesn't raise or return to main spack code.
-            # Just runs default exception handler and exits.
-            sys.excepthook(*sys.exc_info())
-            os._exit(1)
-
-    else:
-        # Parent process just waits for the child to complete.  If the
-        # child exited badly, assume it already printed an appropriate
-        # message.  Just make the parent exit with an error code.
-        pid, returncode = os.waitpid(pid, 0)
-        if returncode != 0:
-            message = "Installation process had nonzero exit code : {code}"
-            strcode = str(returncode)
-            raise InstallError(message.format(code=strcode))
+            child_connection.send([None, None, None])
+        except Exception as e:
+            child_connection.send([type(e), e, None])
+        finally:
+            child_connection.close()
+
+    parent_connection, child_connection = multiprocessing.Pipe()
+    p = multiprocessing.Process(
+        target=child_execution,
+        args=(child_connection,)
+    )
+    p.start()
+    exc_type, exception, traceback = parent_connection.recv()
+    p.join()
+    if exception is not None:
+        raise exception
 
 
 class InstallError(spack.error.SpackError):