CliRunner causes subprocess.run(..., stdout=sys.stderr) to raise UnsupportedOperation

[tucked]

$ tree
.
├── Pipfile
├── Pipfile.lock
├── sscce.py
└── test_sscce.py

0 directories, 4 files

Pipfile(.lock)

# Pipfile
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"

[packages]
click = {version = "*"}
pytest = {version = "*"}

[dev-packages]

[requires]
python_version = "3.8"

{
    "_meta": {
        "hash": {
            "sha256": "bf73778e75cb7de62f2ca45f686cd2b85cca37b4ab33417a54b77cc8a6ff3f97"
        },
        "pipfile-spec": 6,
        "requires": {
            "python_version": "3.8"
        },
        "sources": [
            {
                "name": "pypi",
                "url": "https://pypi.org/simple",
                "verify_ssl": true
            }
        ]
    },
    "default": {
        "attrs": {
            "hashes": [
                "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6",
                "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"
            ],
            "markers": "python_version >= '3.5'",
            "version": "==22.1.0"
        },
        "click": {
            "hashes": [
                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
            ],
            "index": "pypi",
            "markers": null,
            "version": "==8.1.3"
        },
        "exceptiongroup": {
            "hashes": [
                "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828",
                "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"
            ],
            "markers": "python_version < '3.11'",
            "version": "==1.0.4"
        },
        "iniconfig": {
            "hashes": [
                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
            ],
            "version": "==1.1.1"
        },
        "packaging": {
            "hashes": [
                "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
                "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
            ],
            "markers": "python_version >= '3.6'",
            "version": "==21.3"
        },
        "pluggy": {
            "hashes": [
                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
            ],
            "markers": "python_version >= '3.6'",
            "version": "==1.0.0"
        },
        "pyparsing": {
            "hashes": [
                "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
                "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
            ],
            "markers": "python_full_version >= '3.6.8'",
            "version": "==3.0.9"
        },
        "pytest": {
            "hashes": [
                "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71",
                "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"
            ],
            "index": "pypi",
            "markers": null,
            "version": "==7.2.0"
        },
        "tomli": {
            "hashes": [
                "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
                "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
            ],
            "markers": "python_version < '3.11'",
            "version": "==2.0.1"
        }
    },
    "develop": {}
}

tl;dr: click==8.1.3 on Ubuntu 20.04.5 LTS

"""sscce.py"""
import subprocess
import sys

import click


@click.command
def main():
    subprocess.run(["echo", "foo"], stdout=sys.stderr)


if __name__ == "__main__":
    main()

"""test_sscce.py"""
from click.testing import CliRunner

import sscce


def test_main():
    CliRunner().invoke(sscce.main, [], catch_exceptions=False)

# pipenv isn't needed unless you want the exact deps I used.
pipenv install --deploy
pipenv run pytest test_sscce.py

============================= test session starts ==============================
platform linux -- Python 3.8.12, pytest-7.2.0, pluggy-1.0.0
rootdir: /tmp/tmp.dR7fnRzrtk
collected 1 item

test_sscce.py F                                                          [100%]

=================================== FAILURES ===================================
__________________________________ test_main ___________________________________

    def test_main():
>       CliRunner().invoke(sscce.main, [], catch_exceptions=False)

test_sscce.py:8:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/ifs/home/dtucker/.local/share/virtualenvs/tmp.dR7fnRzrtk-jIAzfqD4/lib/python3.8/site-packages/click/testing.py:408: in invoke
    return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
/ifs/home/dtucker/.local/share/virtualenvs/tmp.dR7fnRzrtk-jIAzfqD4/lib/python3.8/site-packages/click/core.py:1055: in main
    rv = self.invoke(ctx)
/ifs/home/dtucker/.local/share/virtualenvs/tmp.dR7fnRzrtk-jIAzfqD4/lib/python3.8/site-packages/click/core.py:1404: in invoke
    return ctx.invoke(self.callback, **ctx.params)
/ifs/home/dtucker/.local/share/virtualenvs/tmp.dR7fnRzrtk-jIAzfqD4/lib/python3.8/site-packages/click/core.py:760: in invoke
    return __callback(*args, **kwargs)
sscce.py:10: in main
    subprocess.run(["echo", "foo"], stdout=sys.stderr)
/ifs/home/dtucker/.pyenv/versions/3.8.12/lib/python3.8/subprocess.py:493: in run
    with Popen(*popenargs, **kwargs) as process:
/ifs/home/dtucker/.pyenv/versions/3.8.12/lib/python3.8/subprocess.py:808: in __init__
    errread, errwrite) = self._get_handles(stdin, stdout, stderr)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <subprocess.Popen object at 0x7f5d37ccfb80>, stdin = None
stdout = <_io.TextIOWrapper name='<stdout>' mode='w' encoding='utf-8'>
stderr = None

    def _get_handles(self, stdin, stdout, stderr):
        """Construct and return tuple with IO objects:
        p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
        """
        p2cread, p2cwrite = -1, -1
        c2pread, c2pwrite = -1, -1
        errread, errwrite = -1, -1

        if stdin is None:
            pass
        elif stdin == PIPE:
            p2cread, p2cwrite = os.pipe()
        elif stdin == DEVNULL:
            p2cread = self._get_devnull()
        elif isinstance(stdin, int):
            p2cread = stdin
        else:
            # Assuming file-like object
            p2cread = stdin.fileno()

        if stdout is None:
            pass
        elif stdout == PIPE:
            c2pread, c2pwrite = os.pipe()
        elif stdout == DEVNULL:
            c2pwrite = self._get_devnull()
        elif isinstance(stdout, int):
            c2pwrite = stdout
        else:
            # Assuming file-like object
>           c2pwrite = stdout.fileno()
E           io.UnsupportedOperation: fileno

/ifs/home/dtucker/.pyenv/versions/3.8.12/lib/python3.8/subprocess.py:1489: UnsupportedOperation
=========================== short test summary info ============================
FAILED test_sscce.py::test_main - io.UnsupportedOperation: fileno
============================== 1 failed in 0.23s ===============================

The script works fine when invoked directly:

$ pipenv run python sscce.py > /dev/null
foo

[tucked]

@davidism, any thoughts?

[davidism]

Nope, haven't had a chance to look into this and likely won't any time soon. You're welcome to submit a PR for review though.

[tucked]

Here's a simplified repro

>>> subprocess.run(["date"], stdout=io.StringIO())
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/lib64/python3.6/subprocess.py", line 423, in run
    with Popen(*popenargs, **kwargs) as process:
  File "/usr/lib64/python3.6/subprocess.py", line 687, in __init__
    errread, errwrite) = self._get_handles(stdin, stdout, stderr)
  File "/usr/lib64/python3.6/subprocess.py", line 1204, in _get_handles
    c2pwrite = stdout.fileno()
io.UnsupportedOperation: fileno

From the subprocess.Popen docs

stdin, stdout and stderr specify the executed program’s standard input, standard output and standard error file handles, respectively. Valid values are PIPE, DEVNULL, an existing file descriptor (a positive integer), an existing file object with a valid file descriptor, and None.

This is the part that getting us into trouble (emphasis added):

an existing file object with a valid file descriptor

Notably, the sys.stderr docs say

be aware that the standard streams may be replaced with file-like objects like io.StringIO

So, I guess this is technically a bug in the program (i.e. it assumes that sys.stderr will always have a valid file descriptor, which is not a safe assumption).

One option for a workaround is to pipe the output and write it out manually:

sys.stderr.write(
    subprocess.run(["date"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8").stdout
)