diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 2b5a82e0107fb6..d3a07de2385313 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -357,7 +357,8 @@ functions. start_new_session=False, pass_fds=(), *, group=None, \ extra_groups=None, user=None, umask=-1, \ encoding=None, errors=None, text=None, pipesize=-1, \ - process_group=None) + process_group=None, read_stdout_callback=None, \ + read_stderr_callback=None) Execute a child program in a new process. On POSIX, the class uses :meth:`os.execvpe`-like behavior to execute the child program. On Windows, @@ -684,6 +685,27 @@ functions. .. versionadded:: 3.10 The ``pipesize`` parameter was added. + *read_stdout_callback*, if supplied, is called upon :data:`PIPE` data + being read for *stdout* of the child process. It must expect four + args: The :class:`Popen` instance, an ``output_buffer`` list of bytes + to append accumulated data to, and the ``data`` (bytes) just read from + the pipe. In order to maintain standard Popen behavior of collecting pipe + data to be available at the finish in the *stdout* attribute, + ``output_buffer.append(data)`` must be called within the callback. + + *read_stderr_callback*, as above, but for *stderr*. + + .. versionadded: 3.11 + The ``read_stdout_callback`` and ``read_stderr_callback`` parameters + were added. + + Add a tee'ing handler that may be accessed by calling + ``tee_pipe_to(handle)``. It takes the handle of the file to clone to + as an argument, such as *sys.stdout* or *sys.stderr*. + + .. versionadded: 3.11 + The ``tee_pipe_to()`` method was added. + Popen objects are supported as context managers via the :keyword:`with` statement: on exit, standard file descriptors are closed, and the process is waited for. :: diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 1f203bd00d3500..fd81ffc3cd9c46 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -799,6 +799,12 @@ class Popen: encoding and errors: Text mode encoding and error handling to use for file objects stdin, stdout and stderr. + read_stdout_callback: Callable to handle reading from the output pipes. + It takes a Popen object, a list of bytes objects to append the + new bytes to, and the new bytes themselves. + + read_stderr_callback: As above, but for stderr. + Attributes: stdin, stdout, stderr, pid, returncode """ @@ -812,7 +818,8 @@ def __init__(self, args, bufsize=-1, executable=None, restore_signals=True, start_new_session=False, pass_fds=(), *, user=None, group=None, extra_groups=None, encoding=None, errors=None, text=None, umask=-1, pipesize=-1, - process_group=None): + process_group=None, read_stdout_callback=None, + read_stderr_callback=None): """Create new Popen instance.""" if not _can_fork_exec: raise OSError( @@ -864,6 +871,18 @@ def __init__(self, args, bufsize=-1, executable=None, self.encoding = encoding self.errors = errors self.pipesize = pipesize + if read_stdout_callback is not None: + if not callable(read_stdout_callback): + raise ValueError("read_stdout_callback not a callable") + self.read_stdout_callback = read_stdout_callback + else: + self.read_stdout_callback = Popen._read_common_handler + if read_stderr_callback is not None: + if not callable(read_stderr_callback): + raise ValueError("read_stderr_callback not a callable") + self.read_stderr_callback = read_stderr_callback + else: + self.read_stderr_callback = Popen._read_common_handler # Validate the combinations of text and universal_newlines if (text is not None and universal_newlines is not None @@ -1306,6 +1325,16 @@ def _close_pipe_fds(self, # Prevent a double close of these handles/fds from __init__ on error. self._closed_child_pipe_fds = True + def tee_pipe_to(output_fh): + def _tee_handler(self, buffer, data): + buffer.append(data) + output_fh.write(data) + return _tee_handler + + def _read_common_handler(self, buffer, data): + """Default handler for read_stdout_callback and read_stderr_callback.""" + buffer.append(data) + if _mswindows: # # Windows methods @@ -1565,7 +1594,12 @@ def _wait(self, timeout): def _readerthread(self, fh, buffer): - buffer.append(fh.read()) + is_stdout = fh == self.stdout + data = fh.read() + if is_stdout: + self.read_stdout_callback(self, buffer, data) + else: + self.read_stderr_callback(self, buffer, data) fh.close() @@ -2096,7 +2130,11 @@ def _communicate(self, input, endtime, orig_timeout): if not data: selector.unregister(key.fileobj) key.fileobj.close() - self._fileobj2output[key.fileobj].append(data) + else: + if key.fileobj == self.stdout: + self.read_stdout_callback(self, stdout, data) + else: + self.read_stderr_callback(self, stderr, data) self.wait(timeout=self._remaining_time(endtime)) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 3880125807f235..934bc58f1af322 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -53,6 +53,8 @@ mswindows = (sys.platform == "win32") +NEWLINE = b'\r\n' if mswindows else b'\n' + # # Depends on the following external programs: Python # @@ -322,26 +324,98 @@ def test_stdout_none(self): # parent's stdout. This test checks that the message printed by the # child goes to the parent stdout. The parent also checks that the # child's stdout is None. See #11963. - code = ('import sys; from subprocess import Popen, PIPE;' - 'p = Popen([sys.executable, "-c", "print(\'test_stdout_none\')"],' - ' stdin=PIPE, stderr=PIPE);' - 'p.wait(); assert p.stdout is None;') - p = subprocess.Popen([sys.executable, "-c", code], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stderr.close) - out, err = p.communicate() - self.assertEqual(p.returncode, 0, err) - self.assertEqual(out.rstrip(), b'test_stdout_none') + code = ('import sys\n' + 'from subprocess import Popen, PIPE\n' + 'with Popen([sys.executable, "-c", "print(\'test_stdout_none\')"],\n' + ' stdin=PIPE, stderr=PIPE) as p:\n' + ' p.wait()\n' + ' assert p.stdout is None\n') + with subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out, b'test_stdout_none' + NEWLINE) + self.assertEqual(err, b'') + + def test_stdout_tee(self): + # .stdout is PIPE when not redirected, and the child's stdout will + # be inherited from the parent. In order to test this we run a + # subprocess in a subprocess: + # this_test + # \-- subprocess created by this test (parent) + # \-- subprocess created by the parent subprocess (child) + # The parent doesn't specify stdout, so the child will use the + # parent's stdout. This test checks that the message printed by the + # child goes to the parent stdout. The parent also checks that the + # child's stdout is cloned. See #47222. + code = ('import sys\n' + 'from subprocess import Popen, PIPE\n' + 'with Popen([sys.executable, "-c", "print(\'test_stdout_teed\')"],\n' + ' stdout=PIPE, stderr=PIPE, read_stdout_callback=Popen.tee_pipe_to(sys.stdout.buffer)) as p:\n' + ' out, err = p.communicate()\n' + ' assert p.returncode == 0\n' + ' assert out.rstrip() == b\'test_stdout_teed\'\n' + ' assert err == b\'\'\n' + ) + with subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err.decode()) + self.assertEqual(out, b'test_stdout_teed' + NEWLINE) + self.assertEqual(err, b'') def test_stderr_none(self): # .stderr is None when not redirected - p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], - stdin=subprocess.PIPE, stdout=subprocess.PIPE) - self.addCleanup(p.stdout.close) - self.addCleanup(p.stdin.close) - p.wait() - self.assertEqual(p.stderr, None) + with subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) as p: + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + p.wait() + self.assertEqual(p.stderr, None) + + def test_invalid_read_callback(self): + with self.assertRaises(ValueError) as cm: + with subprocess.Popen([sys.executable, '-c', + 'import sys\n' + 'print(\'foo\')\n' + 'print(\'bar\', file=sys.stderr)\n'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + read_stdout_callback=True) as p: + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err.decode()) + e = cm.exception + self.assertEqual(e.args, ('read_stdout_callback not a callable', )) + + def test_custom_read_callback(self): + def my_read_stdout_callback(self, buffer, data): + buffer.append(b'<' + data + b'>') + def my_read_stderr_callback(self, buffer, data): + buffer.append(b'[' + data + b']') + + with subprocess.Popen([sys.executable, '-c', + 'import sys\n' + 'print(\'foo\')\n' + 'print(\'bar\', file=sys.stderr)\n'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + read_stdout_callback=my_read_stdout_callback, + read_stderr_callback=my_read_stderr_callback) as p: + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err.decode()) + self.assertEqual(out, b'') + self.assertEqual(err, b'[bar' + NEWLINE + b']') def _assert_python(self, pre_args, **kwargs): # We include sys.exit() to prevent the test runner from hanging @@ -679,7 +753,7 @@ def test_stdout_filedes_of_stdout(self): self.addCleanup(p.stderr.close) out, err = p.communicate() self.assertEqual(p.returncode, 0, err) - self.assertEqual(out.rstrip(), b'test with stdout=1') + self.assertEqual(out, b'test with stdout=1') def test_stdout_devnull(self): p = subprocess.Popen([sys.executable, "-c", @@ -2678,8 +2752,7 @@ def test_undecodable_env(self): stdout = subprocess.check_output( [sys.executable, "-c", script], env=env) - stdout = stdout.rstrip(b'\n\r') - self.assertEqual(stdout.decode('ascii'), ascii(decoded_value)) + self.assertEqual(stdout.decode('ascii'), ascii(decoded_value) + NEWLINE.decode()) # test bytes key = key.encode("ascii", "surrogateescape") @@ -2689,8 +2762,7 @@ def test_undecodable_env(self): stdout = subprocess.check_output( [sys.executable, "-c", script], env=env) - stdout = stdout.rstrip(b'\n\r') - self.assertEqual(stdout.decode('ascii'), ascii(encoded_value)) + self.assertEqual(stdout.decode('ascii'), ascii(encoded_value) + NEWLINE.decode()) def test_bytes_program(self): abs_program = os.fsencode(ZERO_RETURN_CMD[0]) diff --git a/Misc/NEWS.d/next/Library/2022-04-05-22-15-55.bpo-47222.I5CLHq.rst b/Misc/NEWS.d/next/Library/2022-04-05-22-15-55.bpo-47222.I5CLHq.rst new file mode 100644 index 00000000000000..e3398a82439b46 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-05-22-15-55.bpo-47222.I5CLHq.rst @@ -0,0 +1,3 @@ +In the :mod:`subprocess` :class:`Popen`, factor the read callback handler for the ``PIPE`` file descriptors into its own function. Also add class variables pointing to the *stdout* and *stderr* read handlers. Lastly, add constructor variable ``read_stdout_callback`` and ``read_stderr_callback`` to allow overriding the handler with a user-supplied function. + +Also add the method ``tee_pipe_to(handle)`` to this class, which indicates that output on the pipes should be copied to the parent's buffers, as well as being cloned onto the named *handle*.