From 7dc9b7ebd704aa530ba2a3200154cbca2ae35e5f Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 23 May 2026 10:08:11 -0500 Subject: [PATCH 1/6] tmux_cmd(test[encoding]): Reproduce non-UTF-8 locale separator corruption why: The previous xfail depended on locale.getencoding(), which only exists on Python 3.11+ and skipped libtmux's supported Python 3.10 runtime. what: - Recreate the strict xfail regression test with a temporary LC_CTYPE=C boundary - Assert FORMAT_SEPARATOR survives list-sessions output before parse_output consumes it - Keep the test skipped only when Python UTF-8 mode masks locale decoding --- tests/test_common.py | 57 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/test_common.py b/tests/test_common.py index 59f926eaa..1fe676f14 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -2,6 +2,7 @@ from __future__ import annotations +import locale import logging import re import sys @@ -713,3 +714,59 @@ def test_raise_if_stderr_str_shape_exact(session: libtmux.Session) -> None: assert str(excinfo.value) == "last-window: no last window" assert excinfo.value.args == ("no last window",) assert excinfo.value.subcommand == "last-window" + + +@pytest.mark.xfail( + reason=( + "tmux_cmd passes text=True without encoding='utf-8' to Popen; " + "on non-UTF-8 locales FORMAT_SEPARATOR is corrupted. " + "See: https://github.com/tmux-python/libtmux/issues/678" + ), + strict=True, +) +@pytest.mark.skipif( + sys.flags.utf8_mode != 0, + reason="PYTHONUTF8 mode forces UTF-8, masking the locale bug", +) +def test_tmux_cmd_format_separator_survives_non_utf8_locale( + session: Session, +) -> None: + """FORMAT_SEPARATOR must survive a non-UTF-8 locale round-trip through tmux_cmd. + + Regression test for the encoding bug introduced in commit 1a5e69a2 + (``tmux_cmd: Remove console_to_str(), use text=True``). When + ``subprocess.Popen`` receives ``text=True`` without an explicit + ``encoding="utf-8"``, CPython falls back to the process locale encoding. On + a ``C`` locale the FORMAT_SEPARATOR character U+241E (UTF-8 bytes + ``e2 90 9e``) is decoded as escaped bytes, corrupting every + ``parse_output()`` call downstream. + + The fix is to pass ``encoding="utf-8"`` to the ``subprocess.Popen`` + call in ``tmux_cmd.__init__``. + """ + from libtmux.formats import FORMAT_SEPARATOR + from libtmux.neo import get_output_format, parse_output + + server = session.server + + tmux_version = str(get_version(tmux_bin=server.tmux_bin)) + _fields, fmt_str = get_output_format("list-sessions", tmux_version) + + old_lc_ctype = locale.setlocale(locale.LC_CTYPE) + try: + locale.setlocale(locale.LC_CTYPE, "C") + proc = server.cmd("list-sessions", f"-F{fmt_str}") + finally: + locale.setlocale(locale.LC_CTYPE, old_lc_ctype) + assert proc.stdout + + line = proc.stdout[0] + + assert FORMAT_SEPARATOR in line, ( + f"FORMAT_SEPARATOR U+241E not found in output; " + f"got {line[:80]!r}... (likely decoded with wrong encoding)" + ) + + result = parse_output(line, "list-sessions", tmux_version) + assert isinstance(result, dict) + assert "session_id" in result From 589e97894c2a1a5e06a917d4a8fc90d291e8663a Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 23 May 2026 10:25:44 -0500 Subject: [PATCH 2/6] tmux_cmd(fix[encoding]): Decode tmux output as UTF-8 why: tmux format output is UTF-8, but text=True without an explicit encoding lets CPython decode with the process locale and corrupts non-ASCII separators under non-UTF-8 locales. what: - Pass encoding="utf-8" to subprocess.Popen in tmux_cmd --- src/libtmux/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libtmux/common.py b/src/libtmux/common.py index d05a0b2ea..06c92320f 100644 --- a/src/libtmux/common.py +++ b/src/libtmux/common.py @@ -333,6 +333,7 @@ def __init__(self, *args: t.Any, tmux_bin: str | None = None) -> None: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding="utf-8", errors="backslashreplace", ) stdout, stderr = self.process.communicate() From ca26b314d59ae03d509f348ea266c0c8e65e1cd0 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 23 May 2026 10:32:22 -0500 Subject: [PATCH 3/6] ControlMode(test[encoding]): Reproduce non-UTF-8 stdout decoding why: ControlMode opens its own tmux -C subprocess with text=True, so it has the same locale-decoding risk as tmux_cmd but through a separate stdout surface. what: - Add a strict xfail regression test for non-ASCII control protocol output under LC_CTYPE=C - Drive display-message through the control client and assert FORMAT_SEPARATOR survives stdout decoding --- tests/test_control_mode.py | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_control_mode.py b/tests/test_control_mode.py index 609357ed3..ce806a805 100644 --- a/tests/test_control_mode.py +++ b/tests/test_control_mode.py @@ -2,9 +2,16 @@ from __future__ import annotations +import locale +import os +import select +import sys import typing as t +import pytest + from libtmux._internal.control_mode import ControlMode +from libtmux.formats import FORMAT_SEPARATOR if t.TYPE_CHECKING: from libtmux.server import Server @@ -60,3 +67,41 @@ def test_control_mode_client_name_matches_spawned_client( assert first.client_name != second.client_name assert (str(first._proc.pid), first.client_name) in clients assert (str(second._proc.pid), second.client_name) in clients + + +@pytest.mark.xfail( + reason=( + "ControlMode passes text=True without encoding='utf-8' to Popen; " + "on non-UTF-8 locales control protocol output is decoded with the " + "locale encoding. See: https://github.com/tmux-python/libtmux/issues/678" + ), + strict=True, +) +@pytest.mark.skipif( + sys.flags.utf8_mode != 0, + reason="PYTHONUTF8 mode forces UTF-8, masking the locale bug", +) +def test_control_mode_stdout_preserves_non_ascii_output( + control_mode: t.Callable[[], ControlMode], +) -> None: + """Control-mode stdout must preserve non-ASCII tmux output.""" + old_lc_ctype = locale.setlocale(locale.LC_CTYPE) + try: + locale.setlocale(locale.LC_CTYPE, "C") + with control_mode() as ctl: + os.write( + ctl._write_fd, + f"display-message -p '{FORMAT_SEPARATOR}'\n".encode(), + ) + + for _ in range(20): + ready, _, _ = select.select([ctl.stdout], [], [], 1) + assert ready, "timed out waiting for control-mode output" + + line = ctl.stdout.readline() + if FORMAT_SEPARATOR in line: + break + else: + pytest.fail("FORMAT_SEPARATOR U+241E not found in control output") + finally: + locale.setlocale(locale.LC_CTYPE, old_lc_ctype) From 525aa5a75020f6da3c84cc514b77b3e448f609c6 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 23 May 2026 10:34:19 -0500 Subject: [PATCH 4/6] ControlMode(fix[encoding]): Decode control stdout as UTF-8 why: ControlMode reads tmux control protocol output through its own text-mode subprocess, so locale decoding can corrupt or reject UTF-8 output under non-UTF-8 locales. what: - Pass encoding="utf-8" to the control-mode subprocess.Popen call --- src/libtmux/_internal/control_mode.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libtmux/_internal/control_mode.py b/src/libtmux/_internal/control_mode.py index c4ef1d2da..6c1d2301e 100644 --- a/src/libtmux/_internal/control_mode.py +++ b/src/libtmux/_internal/control_mode.py @@ -86,6 +86,7 @@ def __enter__(self) -> ControlMode: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding="utf-8", ) finally: # subprocess owns read_fd now From 55c33af47ac419c878d0b2a0c1d73aab3cfbd7b6 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 23 May 2026 10:37:03 -0500 Subject: [PATCH 5/6] encoding(test): Promote UTF-8 locale regressions to passing tests why: tmux_cmd and ControlMode now decode tmux output as UTF-8 explicitly, so the locale regression checks should fail only on future regressions. what: - Remove strict xfail markers from the tmux_cmd and ControlMode UTF-8 regression tests - Keep the UTF-8-mode skip that avoids masked locale-decoding behavior --- tests/test_common.py | 12 ++---------- tests/test_control_mode.py | 8 -------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/tests/test_common.py b/tests/test_common.py index 1fe676f14..426b72d57 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -716,14 +716,6 @@ def test_raise_if_stderr_str_shape_exact(session: libtmux.Session) -> None: assert excinfo.value.subcommand == "last-window" -@pytest.mark.xfail( - reason=( - "tmux_cmd passes text=True without encoding='utf-8' to Popen; " - "on non-UTF-8 locales FORMAT_SEPARATOR is corrupted. " - "See: https://github.com/tmux-python/libtmux/issues/678" - ), - strict=True, -) @pytest.mark.skipif( sys.flags.utf8_mode != 0, reason="PYTHONUTF8 mode forces UTF-8, masking the locale bug", @@ -741,8 +733,8 @@ def test_tmux_cmd_format_separator_survives_non_utf8_locale( ``e2 90 9e``) is decoded as escaped bytes, corrupting every ``parse_output()`` call downstream. - The fix is to pass ``encoding="utf-8"`` to the ``subprocess.Popen`` - call in ``tmux_cmd.__init__``. + This test guards the explicit ``encoding="utf-8"`` passed to + ``subprocess.Popen`` in ``tmux_cmd.__init__``. """ from libtmux.formats import FORMAT_SEPARATOR from libtmux.neo import get_output_format, parse_output diff --git a/tests/test_control_mode.py b/tests/test_control_mode.py index ce806a805..f72f68466 100644 --- a/tests/test_control_mode.py +++ b/tests/test_control_mode.py @@ -69,14 +69,6 @@ def test_control_mode_client_name_matches_spawned_client( assert (str(second._proc.pid), second.client_name) in clients -@pytest.mark.xfail( - reason=( - "ControlMode passes text=True without encoding='utf-8' to Popen; " - "on non-UTF-8 locales control protocol output is decoded with the " - "locale encoding. See: https://github.com/tmux-python/libtmux/issues/678" - ), - strict=True, -) @pytest.mark.skipif( sys.flags.utf8_mode != 0, reason="PYTHONUTF8 mode forces UTF-8, masking the locale bug", From fc5e6c4e1dcb2d04c075069e825e9c3b0b216ffe Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sat, 23 May 2026 09:44:26 -0500 Subject: [PATCH 6/6] docs(CHANGES): tmux_cmd UTF-8 encoding fix for non-UTF-8 locales why: Document the user-visible bug fix for the 0.58.x release. what: - Add Fixes entry for tmux_cmd encoding on non-UTF-8 locales --- CHANGES | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES b/CHANGES index f55d5312f..4c08d8f1f 100644 --- a/CHANGES +++ b/CHANGES @@ -45,6 +45,17 @@ $ uvx --from 'libtmux' --prerelease allow python _Notes on the upcoming release will go here._ +### Fixes + +#### Subprocess encoding on non-UTF-8 locales (#679) + +{class}`~libtmux.common.tmux_cmd` and {class}`~libtmux._internal.control_mode.ControlMode` +now pass `encoding="utf-8"` to `subprocess.Popen`, ensuring tmux output +is decoded correctly regardless of the system locale. Previously, on +non-UTF-8 locales, the {data}`~libtmux.formats.FORMAT_SEPARATOR` character +(U+241E) was corrupted during decoding, causing list accessors +({attr}`~libtmux.Server.sessions`, etc.) to return empty results. + ## libtmux 0.57.1 (2026-05-18) Restores the "lenient-by-default" behavior for