Permalink
Browse files

Fully refactored subprocess handling on all platforms.

Now we have all process-related code in utils.process, which itself
imports from platform-specific files as needed.

On posix, we have reliable asynchronous delivery of stdout and stderr,
and on win32 at least we have the basics that subprocess.py provides,
since pexpect is not available.

We also now support robust killing of subprocesses that may capture
SIGINT: one SIGINT on our end is sent to the subprocess, but then we
kill it, to prevent a rogue subprocess from hijacking the ipython
console.

Note that on posix, we now depend on pexpect, but we ship our own copy
to users which we'll use if there's no system pexpect installed.

UNC path handling for windows was implemented as a context manager
called AvoidUNCPath.
  • Loading branch information...
1 parent 566a32b commit 06dcbd4381870cd44c9b76e7a7be2c0431086264 @fperez fperez committed Aug 31, 2010
@@ -57,7 +57,7 @@
from IPython.utils.io import ask_yes_no, rprint
from IPython.utils.ipstruct import Struct
from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
-from IPython.utils.process import system, getoutput, getoutputerror
+from IPython.utils.process import system, getoutput
from IPython.utils.strdispatch import StrDispatch
from IPython.utils.syspathcontext import prepended_to_syspath
from IPython.utils.text import num_ini_spaces
@@ -1667,12 +1667,6 @@ def getoutput(self, cmd):
raise OSError("Background processes not supported.")
return getoutput(self.var_expand(cmd, depth=2))
- def getoutputerror(self, cmd):
- """Get stdout and stderr from a subprocess."""
- if cmd.endswith('&'):
- raise OSError("Background processes not supported.")
- return getoutputerror(self.var_expand(cmd, depth=2))
-
#-------------------------------------------------------------------------
# Things related to aliases
#-------------------------------------------------------------------------
View
@@ -3072,9 +3072,7 @@ def magic_sc(self, parameter_s=''):
except ValueError:
var,cmd = '',''
# If all looks ok, proceed
- out,err = self.shell.getoutputerror(cmd)
- if err:
- print >> IPython.utils.io.Term.cerr, err
+ out = self.shell.getoutput(cmd)
if opts.has_key('l'):
out = SList(out.split('\n'))
else:
@@ -3122,10 +3120,9 @@ def magic_sx(self, parameter_s=''):
system commands."""
if parameter_s:
- out,err = self.shell.getoutputerror(parameter_s)
- if err:
- print >> IPython.utils.io.Term.cerr, err
- return SList(out.split('\n'))
+ out = self.shell.getoutput(parameter_s)
+ if out is not None:
+ return SList(out.splitlines())
def magic_r(self, parameter_s=''):
"""Repeat previous input.
View
@@ -37,7 +37,7 @@
from IPython.utils.cursesimport import use_curses
from IPython.utils.data import chop
import IPython.utils.io
-from IPython.utils.process import xsys
+from IPython.utils.process import system
from IPython.utils.terminal import get_terminal_size
@@ -210,7 +210,7 @@ def page_file(fname, start=0, pager_cmd=None):
try:
if os.environ['TERM'] in ['emacs','dumb']:
raise EnvironmentError
- xsys(pager_cmd + ' ' + fname)
+ system(pager_cmd + ' ' + fname)
except:
try:
if start > 0:
@@ -0,0 +1,119 @@
+"""Common utilities for the various process_* implementations.
+
+This file is only meant to be imported by the platform-specific implementations
+of subprocess utilities, and it contains tools that are common to all of them.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2010 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import subprocess
+import sys
+
+#-----------------------------------------------------------------------------
+# Function definitions
+#-----------------------------------------------------------------------------
+
+def read_no_interrupt(p):
+ """Read from a pipe ignoring EINTR errors.
+
+ This is necessary because when reading from pipes with GUI event loops
+ running in the background, often interrupts are raised that stop the
+ command from completing."""
+ import errno
+
+ try:
+ return p.read()
+ except IOError, err:
+ if err.errno != errno.EINTR:
+ raise
+
+
+def process_handler(cmd, callback, stderr=subprocess.PIPE):
+ """Open a command in a shell subprocess and execute a callback.
+
+ This function provides common scaffolding for creating subprocess.Popen()
+ calls. It creates a Popen object and then calls the callback with it.
+
+ Parameters
+ ----------
+ cmd : str
+ A string to be executed with the underlying system shell (by calling
+ :func:`Popen` with ``shell=True``.
+
+ callback : callable
+ A one-argument function that will be called with the Popen object.
+
+ stderr : file descriptor number, optional
+ By default this is set to ``subprocess.PIPE``, but you can also pass the
+ value ``subprocess.STDOUT`` to force the subprocess' stderr to go into
+ the same file descriptor as its stdout. This is useful to read stdout
+ and stderr combined in the order they are generated.
+
+ Returns
+ -------
+ The return value of the provided callback is returned.
+ """
+ sys.stdout.flush()
+ sys.stderr.flush()
+ p = subprocess.Popen(cmd, shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=stderr,
+ close_fds=True)
+
+ try:
+ out = callback(p)
+ except KeyboardInterrupt:
+ print('^C')
+ sys.stdout.flush()
+ sys.stderr.flush()
+ out = None
+ finally:
+ # Make really sure that we don't leave processes behind, in case the
+ # call above raises an exception
+ # We start by assuming the subprocess finished (to avoid NameErrors
+ # later depending on the path taken)
+ if p.returncode is None:
+ try:
+ p.terminate()
+ p.poll()
+ except OSError:
+ pass
+ # One last try on our way out
+ if p.returncode is None:
+ try:
+ p.kill()
+ except OSError:
+ pass
+
+ return out
+
+
+def getoutputerror(cmd):
+ """Return (standard output, standard error) of executing cmd in a shell.
+
+ Accepts the same arguments as os.system().
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ stdout : str
+ stderr : str
+ """
+
+ out_err = process_handler(cmd, lambda p: p.communicate())
+ if out_err is None:
+ out_err = '', ''
+ return out_err
@@ -0,0 +1,169 @@
+"""Posix-specific implementation of process utilities.
+
+This file is only meant to be imported by process.py, not by end-users.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2010 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib
+import subprocess as sp
+import sys
+
+# Third-party
+# We ship our own copy of pexpect (it's a single file) to minimize dependencies
+# for users, but it's only used if we don't find the system copy.
+try:
+ import pexpect
+except ImportError:
+ from IPython.external import pexpect
+
+# Our own
+from .autoattr import auto_attr
+
+#-----------------------------------------------------------------------------
+# Function definitions
+#-----------------------------------------------------------------------------
+
+def _find_cmd(cmd):
+ """Find the full path to a command using which."""
+
+ return sp.Popen(['/usr/bin/env', 'which', cmd],
+ stdout=sp.PIPE).communicate()[0]
+
+
+class ProcessHandler(object):
+ """Execute subprocesses under the control of pexpect.
+ """
+ # Timeout in seconds to wait on each reading of the subprocess' output.
+ # This should not be set too low to avoid cpu overusage from our side,
+ # since we read in a loop whose period is controlled by this timeout.
+ read_timeout = 0.05
+
+ # Timeout to give a process if we receive SIGINT, between sending the
+ # SIGINT to the process and forcefully terminating it.
+ terminate_timeout = 0.2
+
+ # File object where stdout and stderr of the subprocess will be written
+ logfile = None
+
+ # Shell to call for subprocesses to execute
+ sh = None
+
+ @auto_attr
+ def sh(self):
+ sh = pexpect.which('sh')
+ if sh is None:
+ raise OSError('"sh" shell not found')
+ return sh
+
+ def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
+ """Arguments are used for pexpect calls."""
+ self.read_timeout = (ProcessHandler.read_timeout if read_timeout is
+ None else read_timeout)
+ self.terminate_timeout = (ProcessHandler.terminate_timeout if
+ terminate_timeout is None else
+ terminate_timeout)
+ self.logfile = sys.stdout if logfile is None else logfile
+
+ def getoutput(self, cmd):
+ """Run a command and return its stdout/stderr as a string.
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ output : str
+ A string containing the combination of stdout and stderr from the
+ subprocess, in whatever order the subprocess originally wrote to its
+ file descriptors (so the order of the information in this string is the
+ correct order as would be seen if running the command in a terminal).
+ """
+ pcmd = self._make_cmd(cmd)
+ try:
+ return pexpect.run(pcmd).replace('\r\n', '\n')
+ except KeyboardInterrupt:
+ print('^C', file=sys.stderr, end='')
+
+ def system(self, cmd):
+ """Execute a command in a subshell.
+
+ Parameters
+ ----------
+ cmd : str
+ A command to be executed in the system shell.
+
+ Returns
+ -------
+ None : we explicitly do NOT return the subprocess status code, as this
+ utility is meant to be used extensively in IPython, where any return
+ value would trigger :func:`sys.displayhook` calls.
+ """
+ pcmd = self._make_cmd(cmd)
+ # Patterns to match on the output, for pexpect. We read input and
+ # allow either a short timeout or EOF
+ patterns = [pexpect.TIMEOUT, pexpect.EOF]
+ # the index of the EOF pattern in the list.
+ EOF_index = 1 # Fix this index if you change the list!!
+ # The size of the output stored so far in the process output buffer.
+ # Since pexpect only appends to this buffer, each time we print we
+ # record how far we've printed, so that next time we only print *new*
+ # content from the buffer.
+ out_size = 0
+ try:
+ # Since we're not really searching the buffer for text patterns, we
+ # can set pexpect's search window to be tiny and it won't matter.
+ # We only search for the 'patterns' timeout or EOF, which aren't in
+ # the text itself.
+ child = pexpect.spawn(pcmd, searchwindowsize=1)
+ flush = sys.stdout.flush
+ while True:
+ # res is the index of the pattern that caused the match, so we
+ # know whether we've finished (if we matched EOF) or not
+ res_idx = child.expect_list(patterns, self.read_timeout)
+ print(child.before[out_size:], end='')
+ flush()
+ # Update the pointer to what we've already printed
+ out_size = len(child.before)
+ if res_idx==EOF_index:
+ break
+ except KeyboardInterrupt:
+ # We need to send ^C to the process. The ascii code for '^C' is 3
+ # (the character is known as ETX for 'End of Text', see
+ # curses.ascii.ETX).
+ child.sendline(chr(3))
+ # Read and print any more output the program might produce on its
+ # way out.
+ try:
+ out_size = len(child.before)
+ child.expect_list(patterns, self.terminate_timeout)
+ print(child.before[out_size:], end='')
+ except KeyboardInterrupt:
+ # Impatient users tend to type it multiple times
+ pass
+ finally:
+ # Ensure the subprocess really is terminated
+ child.terminate(force=True)
+
+ def _make_cmd(self, cmd):
+ return '%s -c "%s"' % (self.sh, cmd)
+
+
+
+# Make objects with a functional interface for outside use
+__ph = ProcessHandler()
+
+system = __ph.system
+getoutput = __ph.getoutput
Oops, something went wrong.

0 comments on commit 06dcbd4

Please sign in to comment.