Add tab-completion support to PEX repls. (#2321)

This works for all supported Python versions even when those versions
own REPL does not support tab-completion out of the box.

.github/workflows/ci.yml

@@ -10,6 +10,7 @@ env:
   # collide when attempting to load libpython<major>.<minor><flags>.so and lead to mysterious errors
   # importing builtins like `fcntl` as outlined in https://github.com/pantsbuild/pex/issues/1391.
   _PEX_TEST_PYENV_VERSIONS: "2.7 3.7 3.10"
+  _PEX_PEXPECT_TIMEOUT: 10
 concurrency:
   group: CI-${{ github.ref }}
   # Queue on all branches and tags, but only cancel overlapping PR burns.
@@ -100,7 +101,10 @@ jobs:
           ssh-private-key: ${{ env.SSH_PRIVATE_KEY }}
       - name: Run Tests
         run: |
-          BASE_MODE=pull CACHE_MODE=pull ./dtox.sh -e ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }}
+          # This is needed to get pexpect tests working under PyPy running under docker.
+          export TERM="xterm"
+          BASE_MODE=pull CACHE_MODE=pull \
+            ./dtox.sh -e ${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }}
   mac-tests:
     name: tox -e ${{ matrix.tox-env }}
     needs: org-check

CHANGES.md

@@ -1,5 +1,14 @@
 # Release Notes
 
+## 2.1.158
+
+This release adds support for tab completion to all PEX repls running
+under Pythons with the `readline` module available. This tab completion
+support is on-par with newer Python REPL out of the box tab completion
+support.
+
+* Add tab-completion support to PEX repls. (#2321)
+
 ## 2.1.157
 
 This release fixes a bug in `pex3 lock update` for updates that leave

dtox.sh

@@ -92,6 +92,13 @@ if [[ -n "${SSH_AUTH_SOCK:-}" ]]; then
   )
 fi
 
+if [[ -n "${TERM:-}" ]]; then
+  # Some integration tests need a TERM / terminfo. Propagate it when available.
+  DOCKER_ARGS+=(
+    --env TERM="${TERM}"
+  )
+fi
+
 # This ensures the current user owns the host .tox/ dir before launching the container, which
 # otherwise sets the ownership as root for undetermined reasons
 mkdir -p "${ROOT}/.tox"

pex/pex.py

@@ -7,10 +7,11 @@
 import itertools
 import os
 import sys
+import warnings
 from site import USER_SITE
 from types import ModuleType
 
-from pex import bootstrap
+from pex import bootstrap, pex_warnings
 from pex.bootstrap import Bootstrap
 from pex.common import die
 from pex.dist_metadata import CallableEntryPoint, Distribution, EntryPoint
@@ -656,20 +657,53 @@ def execute_interpreter(self):
                 sys.argv = args
                 return self.execute_content(arg, content)
         else:
-            if self._vars.PEX_INTERPRETER_HISTORY:
-                import atexit
+            try:
                 import readline
-
-                histfile = os.path.expanduser(self._vars.PEX_INTERPRETER_HISTORY_FILE)
-                try:
-                    readline.read_history_file(histfile)
-                    readline.set_history_length(1000)
-                except OSError as e:
-                    sys.stderr.write(
-                        "Failed to read history file at {} due to: {}".format(histfile, e)
+            except ImportError:
+                if self._vars.PEX_INTERPRETER_HISTORY:
+                    pex_warnings.warn(
+                        "PEX_INTERPRETER_HISTORY was requested which requires the `readline` "
+                        "module, but the current interpreter at {python} does not have readline "
+                        "support.".format(python=sys.executable)
                     )
+            else:
+                # This import is used for its side effects by the parse_and_bind lines below.
+                import rlcompleter  # NOQA
+
+                # N.B.: This hacky method of detecting use of libedit for the readline
+                # implementation is the recommended means.
+                # See https://docs.python.org/3/library/readline.html
+                if "libedit" in readline.__doc__:
+                    # Mac can use libedit, and libedit has different config syntax.
+                    readline.parse_and_bind("bind ^I rl_complete")
+                else:
+                    readline.parse_and_bind("tab: complete")
 
-                atexit.register(readline.write_history_file, histfile)
+                try:
+                    # Under current PyPy readline does not implement read_init_file and emits a
+                    # warning; so we squelch that noise.
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("ignore")
+                        readline.read_init_file()
+                except (IOError, OSError):
+                    # No init file (~/.inputrc for readline or ~/.editrc for libedit).
+                    pass
+
+                if self._vars.PEX_INTERPRETER_HISTORY:
+                    import atexit
+
+                    histfile = os.path.expanduser(self._vars.PEX_INTERPRETER_HISTORY_FILE)
+                    try:
+                        readline.read_history_file(histfile)
+                        readline.set_history_length(1000)
+                    except (IOError, OSError) as e:
+                        sys.stderr.write(
+                            "Failed to read history file at {path} due to: {err}\n".format(
+                                path=histfile, err=e
+                            )
+                        )
+
+                    atexit.register(readline.write_history_file, histfile)
 
             bootstrap.demote()
 

pex/variables.py

@@ -528,8 +528,6 @@ def PEX_INTERPRETER_HISTORY(self):
         IF PEX_INTERPRETER is true, use a command history file for REPL user convenience.
         The location of the history file is determined by PEX_INTERPRETER_HISTORY_FILE.
 
-        Note: Only supported on CPython interpreters.
-
         Default: false.
         """
         return self._get_bool("PEX_INTERPRETER_HISTORY")
@@ -540,7 +538,7 @@ def PEX_INTERPRETER_HISTORY_FILE(self):
         """File.
 
         IF PEX_INTERPRETER_HISTORY is true, use this history file.
-        The default is the standard CPython interpreter history location.
+        The default is the standard Python interpreter history location.
 
         Default: ~/.python_history.
         """

pex/venv/installer.py

@@ -677,6 +677,7 @@ def sys_executable_paths():
                     "__PEX_UNVENDORED__",
                     # These are _not_ used at runtime, but are present under testing / CI and
                     # simplest to add an exception for here and not warn about in CI runs.
+                    "_PEX_PEXPECT_TIMEOUT",
                     "_PEX_PIP_VERSION",
                     "_PEX_REQUIRES_PYTHON",
                     "_PEX_TEST_DEV_ROOT",
@@ -776,18 +777,55 @@ def sys_executable_paths():
                 # See https://docs.python.org/3/library/sys.html#sys.path
                 sys.path.insert(0, "")
 
-                if pex_interpreter_history:
-                    import atexit
+                try:
                     import readline
+                except ImportError:
+                    if pex_interpreter_history:
+                        pex_warnings.warn(
+                            "PEX_INTERPRETER_HISTORY was requested which requires the `readline` "
+                            "module, but the current interpreter at {{python}} does not have readline "
+                            "support.".format(python=sys.executable)
+                        )
+                else:
+                    # This import is used for its side effects by the line below.
+                    import rlcompleter
+
+                    # N.B.: This hacky method of detecting use of libedit for the readline
+                    # implementation is the recommended means.
+                    # See https://docs.python.org/3/library/readline.html
+                    if "libedit" in readline.__doc__:
+                        # Mac can use libedit, and libedit has different config syntax.
+                        readline.parse_and_bind("bind ^I rl_complete")
+                    else:
+                        readline.parse_and_bind("tab: complete")
 
-                    histfile = os.path.expanduser(pex_interpreter_history_file)
                     try:
-                        readline.read_history_file(histfile)
-                        readline.set_history_length(1000)
-                    except OSError:
+                        # Under current PyPy readline does not implement read_init_file and emits a
+                        # warning; so we squelch that noise.
+                        import warnings
+
+                        with warnings.catch_warnings():
+                            warnings.simplefilter("ignore")
+                            readline.read_init_file()
+                    except (IOError, OSError):
+                        # No init file (~/.inputrc for readline or ~/.editrc for libedit).
                         pass
 
-                    atexit.register(readline.write_history_file, histfile)
+                    if pex_interpreter_history:
+                        import atexit
+
+                        histfile = os.path.expanduser(pex_interpreter_history_file)
+                        try:
+                            readline.read_history_file(histfile)
+                            readline.set_history_length(1000)
+                        except (IOError, OSError) as e:
+                            sys.stderr.write(
+                                "Failed to read history file at {{path}} due to: {{err}}\\n".format(
+                                    path=histfile, err=e
+                                )
+                            )
+
+                        atexit.register(readline.write_history_file, histfile)
 
             if entry_point == PEX_INTERPRETER_ENTRYPOINT and len(sys.argv) > 1:
                 args = sys.argv[1:]

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.1.157"
+__version__ = "2.1.158"

tests/integration/conftest.py

@@ -20,6 +20,14 @@
     from typing import Any, Callable, ContextManager, Iterator, Optional, Tuple
 
 
+@pytest.fixture(scope="session")
+def pexpect_timeout():
+    # type: () -> int
+
+    # The default here of 5 provides enough margin for PyPy which has slow startup.
+    return int(os.environ.get("_PEX_PEXPECT_TIMEOUT", "5"))
+
+
 @pytest.fixture(scope="session")
 def is_pytest_xdist(worker_id):
     # type: (str) -> bool

tests/integration/test_integration.py

@@ -11,9 +11,10 @@
 import shutil
 import subprocess
 import sys
-from contextlib import contextmanager
+from contextlib import closing, contextmanager
 from textwrap import dedent
 
+import pexpect  # type: ignore[import]  # MyPy can't see the types under Python 2.7.
 import pytest
 
 from pex.common import is_exe, safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch
@@ -33,7 +34,6 @@
     IS_LINUX_ARM64,
     IS_MAC,
     IS_MAC_ARM64,
-    IS_PYPY,
     NOT_CPYTHON27,
     PY27,
     PY38,
@@ -43,6 +43,7 @@
     IntegResults,
     built_wheel,
     ensure_python_interpreter,
+    environment_as,
     get_dep_dist_names_from_pex,
     make_env,
     run_pex_command,
@@ -234,37 +235,100 @@ def test_pex_repl_built():
         assert b">>>" in stdout
 
 
-@pytest.mark.skipif(
-    IS_PYPY or IS_MAC,
-    reason="REPL history is only supported on CPython. It works on macOS in an interactive "
-    "terminal, but this test fails in CI on macOS with `Inappropriate ioctl for device`, "
-    "because readline.read_history_file expects a tty on stdout. The linux tests will have "
-    "to suffice for now.",
+try:
+    # This import is needed for the side effect of testing readline availability.
+    import readline  # NOQA
+
+    READLINE_AVAILABLE = True
+except ImportError:
+    READLINE_AVAILABLE = False
+
+readline_test = pytest.mark.skipif(
+    not READLINE_AVAILABLE,
+    reason="The readline module is not available, but is required for this test.",
 )
-@pytest.mark.parametrize("venv_pex", [False, True])
-def test_pex_repl_history(venv_pex):
-    # type: (...) -> None
-    """Tests enabling REPL command history."""
-    stdin_payload = b"import sys; import readline; print(readline.get_history_item(1)); sys.exit(3)"
 
-    with temporary_dir() as output_dir:
-        # Create a dummy temporary pex with no entrypoint.
-        pex_path = os.path.join(output_dir, "dummy.pex")
-        results = run_pex_command(
-            ["--disable-cache", "-o", pex_path] + (["--venv"] if venv_pex else [])
-        )
-        results.assert_success()
+empty_pex_test = pytest.mark.parametrize(
+    "empty_pex", [pytest.param([], id="PEX"), pytest.param(["--venv"], id="VENV")], indirect=True
+)
 
-        history_file = os.path.join(output_dir, ".python_history")
-        with open(history_file, "w") as fp:
-            fp.write("2 + 2\n")
 
-        # Test that the REPL can see the history.
-        env = {"PEX_INTERPRETER_HISTORY": "1", "PEX_INTERPRETER_HISTORY_FILE": history_file}
-        stdout, rc = run_simple_pex(pex_path, stdin=stdin_payload, env=env)
-        assert rc == 3, "Failed with: {}".format(stdout.decode("utf-8"))
-        assert b">>>" in stdout
-        assert b"2 + 2" in stdout
+@pytest.fixture
+def empty_pex(
+    tmpdir,  # type: Any
+    request,  # type: Any
+):
+    # type: (...) -> str
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    result = run_pex_command(
+        [
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "-o",
+            os.path.join(str(tmpdir), "pex"),
+            "--seed",
+            "verbose",
+        ]
+        + request.param
+    )
+    result.assert_success()
+    return cast(str, json.loads(result.output)["pex"])
+
+
+@readline_test
+@empty_pex_test
+def test_pex_repl_history(
+    tmpdir,  # type: Any
+    empty_pex,  # type: str
+    pexpect_timeout,  # type: int
+):
+    # type: (...) -> None
+
+    history_file = os.path.join(str(tmpdir), ".python_history")
+    with safe_open(history_file, "w") as fp:
+        # Mac can use libedit and libedit needs this header line or else the history file will fail
+        # to load with `OSError [Errno 22] invalid argument`.
+        # See: https://github.com/cdesjardins/libedit/blob/18b682734c11a2bd0a9911690fca522c96079712/src/history.c#L56
+        print("_HiStOrY_V2_", file=fp)
+        print("2 + 2", file=fp)
+
+    # Test that the REPL can see the history.
+    with open(os.path.join(str(tmpdir), "pexpect.log"), "wb") as log, environment_as(
+        PEX_INTERPRETER_HISTORY=1, PEX_INTERPRETER_HISTORY_FILE=history_file
+    ), closing(pexpect.spawn(empty_pex, dimensions=(24, 80), logfile=log)) as process:
+        process.expect_exact(b">>>", timeout=pexpect_timeout)
+        process.send(
+            b"\x1b[A"
+        )  # This is up-arrow and should net the most recent history line: 2 + 2.
+        process.sendline(b"")
+        process.expect_exact(b"4", timeout=pexpect_timeout)
+        process.expect_exact(b">>>", timeout=pexpect_timeout)
+
+
+@readline_test
+@empty_pex_test
+def test_pex_repl_tab_complete(
+    tmpdir,  # type: Any
+    empty_pex,  # type: str
+    pexpect_timeout,  # type: int
+):
+    # type: (...) -> None
+    subprocess_module_path = subprocess.check_output(
+        args=[sys.executable, "-c", "import subprocess; print(subprocess.__file__)"],
+    ).strip()
+    with open(os.path.join(str(tmpdir), "pexpect.log"), "wb") as log, closing(
+        pexpect.spawn(empty_pex, dimensions=(24, 80), logfile=log)
+    ) as process:
+        process.expect_exact(b">>>", timeout=pexpect_timeout)
+        process.send(b"impo\t")
+        process.expect_exact(b"rt", timeout=pexpect_timeout)
+        process.sendline(b" subprocess")
+        process.expect_exact(b">>>", timeout=pexpect_timeout)
+        process.sendline(b"print(subprocess.__file__)")
+        process.expect_exact(subprocess_module_path, timeout=pexpect_timeout)
+        process.expect_exact(b">>>", timeout=pexpect_timeout)
 
 
 @pytest.mark.skipif(WINDOWS, reason="No symlinks on windows")