From 7c655591f124b4aef36f144b6011a31f1415b00f Mon Sep 17 00:00:00 2001 From: Prathamesh Date: Fri, 29 Nov 2024 05:55:42 +0530 Subject: [PATCH 1/6] Fix high memory usage in SpooledTemporaryFile.writelines() when handling large iterators --- Lib/tempfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index b5a15f7b72c872..cc4ff8ec27b522 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -849,9 +849,9 @@ def write(self, s): def writelines(self, iterable): file = self._file - rv = file.writelines(iterable) - self._check(file) - return rv + for line in iterable: + file.write(line) + self._check(file) def detach(self): return self._file.detach() From 5609b5173e2eb82aa6ed350b632f761a9894e46b Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2024 00:47:40 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst new file mode 100644 index 00000000000000..40e048cc3009e8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst @@ -0,0 +1,2 @@ +Issue: The writelines() method in SpooledTemporaryFile only checked whether to roll over to a disk-backed file after the entire iterable is written. This led to unexpectedly high memory usage when handling large iterators because the in-memory buffer grows without limit until the iterable was exhausted. +Solution: Modify the writelines() method to write each line individually and check the buffer size after each write. This ensures that if the max_size is exceeded, the file rolls over to disk immediately, preventing excessive memory consumption. From c04e4b08421331a50a0a9890505be988ae94ffcd Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2024 00:53:51 +0000 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst new file mode 100644 index 00000000000000..40e048cc3009e8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst @@ -0,0 +1,2 @@ +Issue: The writelines() method in SpooledTemporaryFile only checked whether to roll over to a disk-backed file after the entire iterable is written. This led to unexpectedly high memory usage when handling large iterators because the in-memory buffer grows without limit until the iterable was exhausted. +Solution: Modify the writelines() method to write each line individually and check the buffer size after each write. This ensures that if the max_size is exceeded, the file rolls over to disk immediately, preventing excessive memory consumption. From f159e0ecdb059345071f98add035774a5c0103c0 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2024 00:59:49 +0000 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-11-29-00-59-48.gh-issue-127371.S9Zvnc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-59-48.gh-issue-127371.S9Zvnc.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-59-48.gh-issue-127371.S9Zvnc.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-59-48.gh-issue-127371.S9Zvnc.rst new file mode 100644 index 00000000000000..5f6c78492d39b1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-59-48.gh-issue-127371.S9Zvnc.rst @@ -0,0 +1 @@ +The ``writelines()`` method in ``SpooledTemporaryFile`` previously only checked whether to roll over to a disk-backed file after the entire iterable was written. This led to high memory usage when handling large iterators, as the in-memory buffer grew without limit. The method was modified to write each line individually and check the buffer size after each write. If the maximum size is exceeded, the file rolls over to disk immediately, preventing excessive memory consumption. From 4c8d41beff65bfbd9a0344f0070467abaed78ae6 Mon Sep 17 00:00:00 2001 From: Prathamesh Padiyar Date: Fri, 29 Nov 2024 06:35:44 +0530 Subject: [PATCH 5/6] Delete Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst --- .../2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst deleted file mode 100644 index 40e048cc3009e8..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-47-39.gh-issue-127371.EIwXpz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Issue: The writelines() method in SpooledTemporaryFile only checked whether to roll over to a disk-backed file after the entire iterable is written. This led to unexpectedly high memory usage when handling large iterators because the in-memory buffer grows without limit until the iterable was exhausted. -Solution: Modify the writelines() method to write each line individually and check the buffer size after each write. This ensures that if the max_size is exceeded, the file rolls over to disk immediately, preventing excessive memory consumption. From a2b346cf41892e4ba8409797ac87c9b1f666b696 Mon Sep 17 00:00:00 2001 From: Prathamesh Padiyar Date: Fri, 29 Nov 2024 06:35:59 +0530 Subject: [PATCH 6/6] Delete Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst --- .../2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst deleted file mode 100644 index 40e048cc3009e8..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-29-00-53-50.gh-issue-127371.EIwXpz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Issue: The writelines() method in SpooledTemporaryFile only checked whether to roll over to a disk-backed file after the entire iterable is written. This led to unexpectedly high memory usage when handling large iterators because the in-memory buffer grows without limit until the iterable was exhausted. -Solution: Modify the writelines() method to write each line individually and check the buffer size after each write. This ensures that if the max_size is exceeded, the file rolls over to disk immediately, preventing excessive memory consumption.