From 58aaa863b3c621cf657d3868b8c65c85ffb0d079 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 24 Nov 2025 21:11:45 -0400 Subject: [PATCH 1/9] gh-141930 Retry writing .pyc files if incomplete Interruption by a system call or something is not completely unexpected. Retry the write with the remaining data. --- Lib/importlib/_bootstrap_external.py | 11 +-- Lib/test/test_importlib/test_util.py | 71 ++++++++++++++----- ...-11-24-21-09-30.gh-issue-141930.hIIzSd.rst | 2 + 3 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 192c0261408ead..a076c5ddc5cb4f 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -209,11 +209,12 @@ def _write_atomic(path, data, mode=0o666): # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. with _io.FileIO(fd, 'wb') as file: - bytes_written = file.write(data) - if bytes_written != len(data): - # Raise an OSError so the 'except' below cleans up the partially - # written file. - raise OSError("os.write() didn't write the full pyc file") + bytes_written = 0 + while bytes_written < len(data): + last_bytes_written = file.write(data[bytes_written:]) + if last_bytes_written is None or last_bytes_written < 1: + raise OSError("os.write() didn't write the full pyc file") + bytes_written += last_bytes_written _os.replace(path_tmp, path) except OSError: try: diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index a77ce234deec58..01728430adb451 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -788,31 +788,70 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) -class MiscTests(unittest.TestCase): - def test_atomic_write_should_notice_incomplete_writes(self): +class PatchAtomicWrites(): + def __init__(self, truncate_at_length=100, never_complete=False): + self.truncate_at_length = truncate_at_length + self.never_complete = never_complete + self.seen_write = False + self._children = [] + + def __enter__(self): import _pyio oldwrite = os.write - seen_write = False - - truncate_at_length = 100 # Emulate an os.write that only writes partial data. def write(fd, data): - nonlocal seen_write - seen_write = True - return oldwrite(fd, data[:truncate_at_length]) + if self.seen_write and self.never_complete: + return None + self.seen_write = True + return oldwrite(fd, data[:self.truncate_at_length]) # Need to patch _io to be _pyio, so that io.FileIO is affected by the # os.write patch. - with (support.swap_attr(_bootstrap_external, '_io', _pyio), - support.swap_attr(os, 'write', write)): - with self.assertRaises(OSError): - # Make sure we write something longer than the point where we - # truncate. - content = b'x' * (truncate_at_length * 2) - _bootstrap_external._write_atomic(os_helper.TESTFN, content) - assert seen_write + self.children = [ + support.swap_attr(_bootstrap_external, '_io', _pyio), + support.swap_attr(os, 'write', write) + ] + for child in self.children: + child.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for child in self.children: + child.__exit__(exc_type, exc_val, exc_tb) + + +class MiscTests(unittest.TestCase): + + def test_atomic_write_retries_incomplete_writes(self): + truncate_at_length = 100 + length = truncate_at_length * 2 + + with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm: + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * length + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + assert cm.seen_write + + assert os.stat(support.os_helper.TESTFN).st_size == length + os.unlink(support.os_helper.TESTFN) + + def test_atomic_write_errors_if_unable_to_complete(self): + truncate_at_length = 100 + + with ( + PatchAtomicWrites( + truncate_at_length=truncate_at_length, never_complete=True, + ) as cm, + self.assertRaises(OSError) + ): + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * (truncate_at_length * 2) + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + assert cm.seen_write with self.assertRaises(OSError): os.stat(support.os_helper.TESTFN) # Check that the file did not get written. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst new file mode 100644 index 00000000000000..160e4e954b5795 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst @@ -0,0 +1,2 @@ +Retry writing ``.pyc`` files if we get interrupted by a system call or +something that results in a truncated write. From 914caf1bef95fedf2dfd161538c0b04b526795a4 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Tue, 25 Nov 2025 15:43:42 -0400 Subject: [PATCH 2/9] Drop the check for None, this file is synchronous --- Lib/importlib/_bootstrap_external.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index a076c5ddc5cb4f..90cf4c1276542d 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -212,9 +212,9 @@ def _write_atomic(path, data, mode=0o666): bytes_written = 0 while bytes_written < len(data): last_bytes_written = file.write(data[bytes_written:]) - if last_bytes_written is None or last_bytes_written < 1: - raise OSError("os.write() didn't write the full pyc file") bytes_written += last_bytes_written + if last_bytes_written < 1: + raise OSError("os.write() didn't write the full pyc file") _os.replace(path_tmp, path) except OSError: try: From 614946562bd3d941d164e3465dbade7c6dcbdc87 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Tue, 25 Nov 2025 18:26:32 -0400 Subject: [PATCH 3/9] Use BufferedWriter's loop to write .pyc files --- Lib/importlib/_bootstrap_external.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 90cf4c1276542d..6d2c207be3aa56 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -208,13 +208,8 @@ def _write_atomic(path, data, mode=0o666): try: # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. - with _io.FileIO(fd, 'wb') as file: - bytes_written = 0 - while bytes_written < len(data): - last_bytes_written = file.write(data[bytes_written:]) - bytes_written += last_bytes_written - if last_bytes_written < 1: - raise OSError("os.write() didn't write the full pyc file") + with _io.FileIO(fd, 'wb') as file, _io.BufferedWriter(file) as writer: + writer.write(data) _os.replace(path_tmp, path) except OSError: try: From c8ba03f8de63437e82c820611e2a8c4e3c20930f Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Tue, 25 Nov 2025 18:31:04 -0400 Subject: [PATCH 4/9] Tweak news to mentioned BufferedWriter --- .../2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst index 160e4e954b5795..4e4315c043b811 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst @@ -1,2 +1 @@ -Retry writing ``.pyc`` files if we get interrupted by a system call or -something that results in a truncated write. +Use ``BufferedWriter`` to ensure that writes to ``.pyc`` files are retried, or an appropriate error is raised. From f60d6cba0b07c4b536dbadf28144c2b78c6ec358 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Wed, 26 Nov 2025 07:41:27 -0400 Subject: [PATCH 5/9] Use open() to write the bytecode --- Lib/importlib/_bootstrap_external.py | 4 ++-- .../2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 6d2c207be3aa56..2f9307cba4f086 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -208,8 +208,8 @@ def _write_atomic(path, data, mode=0o666): try: # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. - with _io.FileIO(fd, 'wb') as file, _io.BufferedWriter(file) as writer: - writer.write(data) + with _io.open(fd, 'wb') as file: + file.write(data) _os.replace(path_tmp, path) except OSError: try: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst index 4e4315c043b811..9454ea862e3266 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst @@ -1 +1 @@ -Use ``BufferedWriter`` to ensure that writes to ``.pyc`` files are retried, or an appropriate error is raised. +Use Python's regular file object to ensure that writes to ``.pyc`` files are retried, or an appropriate error is raised. From 2faad5658bb291bafc7d30baa0a4814516d9c3d1 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 27 Nov 2025 08:23:21 -0400 Subject: [PATCH 6/9] Convert to unittest style asserts --- Lib/test/test_importlib/test_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 01728430adb451..6b28f87d70ff17 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -789,7 +789,7 @@ def test_complete_multi_phase_init_module(self): class PatchAtomicWrites(): - def __init__(self, truncate_at_length=100, never_complete=False): + def __init__(self, truncate_at_length, never_complete=False): self.truncate_at_length = truncate_at_length self.never_complete = never_complete self.seen_write = False @@ -833,9 +833,9 @@ def test_atomic_write_retries_incomplete_writes(self): # truncate. content = b'x' * length _bootstrap_external._write_atomic(os_helper.TESTFN, content) - assert cm.seen_write + self.assertTrue(cm.seen_write) - assert os.stat(support.os_helper.TESTFN).st_size == length + self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length) os.unlink(support.os_helper.TESTFN) def test_atomic_write_errors_if_unable_to_complete(self): @@ -851,7 +851,7 @@ def test_atomic_write_errors_if_unable_to_complete(self): # truncate. content = b'x' * (truncate_at_length * 2) _bootstrap_external._write_atomic(os_helper.TESTFN, content) - assert cm.seen_write + self.assertTrue(cm.seen_write) with self.assertRaises(OSError): os.stat(support.os_helper.TESTFN) # Check that the file did not get written. From 5d5432bd0d28cfdaf1946b175ff07e94c0fbb7ca Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 27 Nov 2025 09:22:42 -0400 Subject: [PATCH 7/9] Tweak news, thanks @vstinner --- .../2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst index 9454ea862e3266..8c078a2bcc1bd1 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst @@ -1 +1,2 @@ -Use Python's regular file object to ensure that writes to ``.pyc`` files are retried, or an appropriate error is raised. +When importing a module, use Python's regular file object to ensure that +writes to ``.pyc`` files are retried, or an appropriate error is raised. From 0296299e7daf691b8a9333f8859a756673d718d1 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Thu, 27 Nov 2025 09:23:27 -0400 Subject: [PATCH 8/9] Tidy --- Lib/test/test_importlib/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 6b28f87d70ff17..0adab8d14e0452 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -788,7 +788,7 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) -class PatchAtomicWrites(): +class PatchAtomicWrites: def __init__(self, truncate_at_length, never_complete=False): self.truncate_at_length = truncate_at_length self.never_complete = never_complete From 1491620e63131d95d7766c59413809675c273f6d Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Thu, 27 Nov 2025 10:50:49 -0800 Subject: [PATCH 9/9] reword NEWS, avoid word "retried" --- .../2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst index 8c078a2bcc1bd1..06a12f98224e88 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst @@ -1,2 +1,2 @@ When importing a module, use Python's regular file object to ensure that -writes to ``.pyc`` files are retried, or an appropriate error is raised. +writes to ``.pyc`` files are complete or an appropriate error is raised.