From 418759d43f8f036a077458b893eed1e4893c3d78 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 23 Jan 2022 12:09:15 +0530 Subject: [PATCH 01/25] free co_quickened at exit --- Python/pylifecycle.c | 4 ++++ Tools/scripts/deepfreeze.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9d10f94efa732ef..533504e310c970e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -31,6 +31,7 @@ #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() extern void _PyIO_Fini(void); +extern void _Py_Deepfreeze_Fini(void); #include // setlocale() #include // getenv() @@ -1726,6 +1727,9 @@ finalize_interp_clear(PyThreadState *tstate) } finalize_interp_types(tstate->interp); + if (is_main_interp) { + _Py_Deepfreeze_Fini(); + } } diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index a1ef85ea891a27d..6618990e17dd92f 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -109,6 +109,7 @@ def __init__(self, file: TextIO) -> None: self.cache: Dict[tuple[type, object, str], str] = {} self.hits, self.misses = 0, 0 self.patchups: list[str] = [] + self.deallocs: list[str] = [] self.write('#include "Python.h"') self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') @@ -277,6 +278,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_varnames = {co_varnames},") self.write(f".co_cellvars = {co_cellvars},") self.write(f".co_freevars = {co_freevars},") + self.deallocs.append(f"PyMem_Free({name}.co_quickened);") return f"& {name}.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: @@ -440,6 +442,9 @@ def generate(args: list[str], output: TextIO) -> None: else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) + with printer.block(f"void _Py_Deepfreeze_Fini(void)"): + for p in printer.deallocs: + printer.write(p) if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}") From 0ae535eeeba8db9d8d71b6add598852889fac034 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 23 Jan 2022 20:03:24 +0530 Subject: [PATCH 02/25] clear more --- PCbuild/_freeze_module.vcxproj | 2 +- Tools/scripts/deepfreeze.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 0a74f5850a1e8e1..6b0619fb5bb8566 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -88,7 +88,7 @@ - Py_NO_ENABLE_SHARED;Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) + Py_NO_ENABLE_SHARED;Py_BUILD_CORE;Py_NOT_FINI_DEEPFREEZE;_CONSOLE;%(PreprocessorDefinitions) Disabled false diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 6618990e17dd92f..d1c62a881a2bca2 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -279,6 +279,8 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_cellvars = {co_cellvars},") self.write(f".co_freevars = {co_freevars},") self.deallocs.append(f"PyMem_Free({name}.co_quickened);") + self.deallocs.append(f"PyMem_Free({name}.co_extra);") + self.deallocs.append(f"PyObject_ClearWeakRefs((PyObject *)&{name});") return f"& {name}.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: From 1f86641f6eaaf1542a62c34e91a104fa1455a072 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 23 Jan 2022 20:16:40 +0530 Subject: [PATCH 03/25] add guard --- Python/pylifecycle.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 533504e310c970e..9d82a4e70b63f3a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -31,8 +31,9 @@ #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() extern void _PyIO_Fini(void); +#ifndef Py_NOT_FINI_DEEPFREEZE extern void _Py_Deepfreeze_Fini(void); - +#endif #include // setlocale() #include // getenv() @@ -1727,9 +1728,11 @@ finalize_interp_clear(PyThreadState *tstate) } finalize_interp_types(tstate->interp); + #ifndef Py_NOT_FINI_DEEPFREEZE if (is_main_interp) { _Py_Deepfreeze_Fini(); } + #endif } From 4213d7e1ec81d4f0b5c89949b38decc64636ecfd Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 24 Jan 2022 09:58:09 +0530 Subject: [PATCH 04/25] add a dummy finalizer --- PCbuild/_freeze_module.vcxproj | 2 +- Programs/_bootstrap_python.c | 4 ++++ Programs/_freeze_module.c | 5 ++++- Python/pylifecycle.c | 4 ---- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 6b0619fb5bb8566..0a74f5850a1e8e1 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -88,7 +88,7 @@ - Py_NO_ENABLE_SHARED;Py_BUILD_CORE;Py_NOT_FINI_DEEPFREEZE;_CONSOLE;%(PreprocessorDefinitions) + Py_NO_ENABLE_SHARED;Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) Disabled false diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index f2103fd5aaae51b..f41a11c9d2bdaf4 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -103,3 +103,7 @@ main(int argc, char **argv) } Py_ExitStatusException(status); } + + +void _Py_Deepfreeze_Fini(void) { +}; diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index b2f1a24016fac87..8dc0f83cb68f79b 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -234,4 +234,7 @@ main(int argc, char *argv[]) PyErr_Print(); Py_Finalize(); return 1; -} +}; + +void _Py_Deepfreeze_Fini(void) { +}; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9d82a4e70b63f3a..c71352ef99fdc3d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -31,9 +31,7 @@ #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() extern void _PyIO_Fini(void); -#ifndef Py_NOT_FINI_DEEPFREEZE extern void _Py_Deepfreeze_Fini(void); -#endif #include // setlocale() #include // getenv() @@ -1728,11 +1726,9 @@ finalize_interp_clear(PyThreadState *tstate) } finalize_interp_types(tstate->interp); - #ifndef Py_NOT_FINI_DEEPFREEZE if (is_main_interp) { _Py_Deepfreeze_Fini(); } - #endif } From ac0b05e25c58bbaf5b212824a7ca19fbf6a3cc95 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 24 Jan 2022 04:48:55 +0000 Subject: [PATCH 05/25] check if weakrefs are null --- Tools/scripts/deepfreeze.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index d1c62a881a2bca2..dd778ecd753793a 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -280,7 +280,9 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_freevars = {co_freevars},") self.deallocs.append(f"PyMem_Free({name}.co_quickened);") self.deallocs.append(f"PyMem_Free({name}.co_extra);") + self.deallocs.append(f"if ({name}.co_weakreflist != NULL) {{") self.deallocs.append(f"PyObject_ClearWeakRefs((PyObject *)&{name});") + self.deallocs.append("};") return f"& {name}.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: From 105d20b93f1caa92083f6430712808e5af67746f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Mon, 24 Jan 2022 20:55:27 +0530 Subject: [PATCH 06/25] finalized --- Modules/main.c | 3 +++ Python/pylifecycle.c | 5 +---- Tools/scripts/deepfreeze.py | 11 ++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Modules/main.c b/Modules/main.c index b9bcea393abe3cf..837f1b5a8d8cf6c 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -28,6 +28,8 @@ #ifdef __cplusplus extern "C" { #endif +/* For deepfrozen modules */ +extern void _Py_Deepfreeze_Fini(void); /* --- pymain_init() ---------------------------------------------- */ @@ -621,6 +623,7 @@ pymain_free(void) _PyPathConfig_ClearGlobal(); _Py_ClearStandardStreamEncoding(); _Py_ClearArgcArgv(); + _Py_Deepfreeze_Fini(); _PyRuntime_Finalize(); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c71352ef99fdc3d..9d10f94efa732ef 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -31,7 +31,7 @@ #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes() extern void _PyIO_Fini(void); -extern void _Py_Deepfreeze_Fini(void); + #include // setlocale() #include // getenv() @@ -1726,9 +1726,6 @@ finalize_interp_clear(PyThreadState *tstate) } finalize_interp_types(tstate->interp); - if (is_main_interp) { - _Py_Deepfreeze_Fini(); - } } diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index dd778ecd753793a..621c03ca8d805c8 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -115,6 +115,11 @@ def __init__(self, file: TextIO) -> None: self.write('#include "internal/pycore_code.h"') self.write('#include "internal/pycore_long.h"') self.write("") + with self.block("static void dealloc_codeobject(PyCodeObject co)", ";"): + self.write("PyMem_Free(co.co_quickened);") + self.write("PyMem_Free(co.co_extra);") + with self.block("if (co.co_weakreflist != NULL)", ";"): + self.write("PyObject_ClearWeakRefs((PyObject *)&co);") @contextlib.contextmanager def indent(self) -> None: @@ -278,11 +283,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_varnames = {co_varnames},") self.write(f".co_cellvars = {co_cellvars},") self.write(f".co_freevars = {co_freevars},") - self.deallocs.append(f"PyMem_Free({name}.co_quickened);") - self.deallocs.append(f"PyMem_Free({name}.co_extra);") - self.deallocs.append(f"if ({name}.co_weakreflist != NULL) {{") - self.deallocs.append(f"PyObject_ClearWeakRefs((PyObject *)&{name});") - self.deallocs.append("};") + self.deallocs.append(f"dealloc_codeobject({name});") return f"& {name}.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: From 41887ce63c21d5b01bb3c95e7ef0d2b3f2884c85 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 15:39:35 +0000 Subject: [PATCH 07/25] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst new file mode 100644 index 000000000000000..26079839a5f257a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-24-15-39-34.bpo-46476.cvP1Mr.rst @@ -0,0 +1 @@ +Fix memory leak in code objects generated by deepfreeze. Patch by Kumar Aditya. \ No newline at end of file From 16e1c158e55585c79e36e0fc20fba8912a412a7f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:29:19 +0000 Subject: [PATCH 08/25] fix style --- Programs/_bootstrap_python.c | 4 ++-- Programs/_freeze_module.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index f41a11c9d2bdaf4..70acd07153fb271 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -104,6 +104,6 @@ main(int argc, char **argv) Py_ExitStatusException(status); } - +/* Empty finalizer for deepfrozen modules*/ void _Py_Deepfreeze_Fini(void) { -}; +} diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 8dc0f83cb68f79b..f03f581a1253b60 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -234,7 +234,8 @@ main(int argc, char *argv[]) PyErr_Print(); Py_Finalize(); return 1; -}; +} +/* Empty finalizer for deepfrozen modules*/ void _Py_Deepfreeze_Fini(void) { -}; +} From a5e3c5597efc17cbbd84de5a399bf8a7f49ae24d Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:34:22 +0000 Subject: [PATCH 09/25] fix style --- Programs/_bootstrap_python.c | 4 +++- Programs/_freeze_module.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 70acd07153fb271..3a0c71ea442fd4e 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -105,5 +105,7 @@ main(int argc, char **argv) } /* Empty finalizer for deepfrozen modules*/ -void _Py_Deepfreeze_Fini(void) { +void +_Py_Deepfreeze_Fini(void) +{ } diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index f03f581a1253b60..98b14180839ec77 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -237,5 +237,7 @@ main(int argc, char *argv[]) } /* Empty finalizer for deepfrozen modules*/ -void _Py_Deepfreeze_Fini(void) { +void +_Py_Deepfreeze_Fini(void) +{ } From d8cc4bdde780f527a6438aa0d80f830620fe0170 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:35:05 +0000 Subject: [PATCH 10/25] add blank line --- Modules/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/main.c b/Modules/main.c index 837f1b5a8d8cf6c..3fee01ee6c5975d 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -28,6 +28,7 @@ #ifdef __cplusplus extern "C" { #endif + /* For deepfrozen modules */ extern void _Py_Deepfreeze_Fini(void); From f5a2df04d1821ddde25a4ce7ac69f7eb439eb1aa Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:38:34 +0000 Subject: [PATCH 11/25] take ptr --- Tools/scripts/deepfreeze.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 621c03ca8d805c8..2523d7998206824 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -115,11 +115,13 @@ def __init__(self, file: TextIO) -> None: self.write('#include "internal/pycore_code.h"') self.write('#include "internal/pycore_long.h"') self.write("") - with self.block("static void dealloc_codeobject(PyCodeObject co)", ";"): - self.write("PyMem_Free(co.co_quickened);") - self.write("PyMem_Free(co.co_extra);") - with self.block("if (co.co_weakreflist != NULL)", ";"): - self.write("PyObject_ClearWeakRefs((PyObject *)&co);") + with self.block("static void dealloc_codeobject(PyCodeObject *co)"): + self.write("PyMem_Free(co->co_quickened);") + self.write("co->co_quickened = NULL;") + self.write("PyMem_Free(co->co_extra);") + self.write("co->co_extra = NULL;") + with self.block("if (co->co_weakreflist != NULL)"): + self.write("PyObject_ClearWeakRefs((PyObject *)co);") @contextlib.contextmanager def indent(self) -> None: @@ -283,7 +285,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_varnames = {co_varnames},") self.write(f".co_cellvars = {co_cellvars},") self.write(f".co_freevars = {co_freevars},") - self.deallocs.append(f"dealloc_codeobject({name});") + self.deallocs.append(f"dealloc_codeobject(&{name});") return f"& {name}.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: From 985b2febcace33f058105f11568ec55df99267de Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:41:04 +0000 Subject: [PATCH 12/25] move deallocator down --- Tools/scripts/deepfreeze.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 2523d7998206824..541dc93ae7acaad 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -114,14 +114,7 @@ def __init__(self, file: TextIO) -> None: self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') self.write('#include "internal/pycore_long.h"') - self.write("") - with self.block("static void dealloc_codeobject(PyCodeObject *co)"): - self.write("PyMem_Free(co->co_quickened);") - self.write("co->co_quickened = NULL;") - self.write("PyMem_Free(co->co_extra);") - self.write("co->co_extra = NULL;") - with self.block("if (co->co_weakreflist != NULL)"): - self.write("PyObject_ClearWeakRefs((PyObject *)co);") + self.write("") @contextlib.contextmanager def indent(self) -> None: @@ -449,6 +442,13 @@ def generate(args: list[str], output: TextIO) -> None: else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) + with printer.block("static void dealloc_codeobject(PyCodeObject *co)"): + printer.write("PyMem_Free(co->co_quickened);") + printer.write("co->co_quickened = NULL;") + printer.write("PyMem_Free(co->co_extra);") + printer.write("co->co_extra = NULL;") + with printer.block("if (co->co_weakreflist != NULL)"): + printer.write("PyObject_ClearWeakRefs((PyObject *)co);") with printer.block(f"void _Py_Deepfreeze_Fini(void)"): for p in printer.deallocs: printer.write(p) From c27d31c6efb8103fb699e199194316f400e1b2cf Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:43:00 +0000 Subject: [PATCH 13/25] set weakreflist to NULL --- Tools/scripts/deepfreeze.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 541dc93ae7acaad..a7bb70659b460d6 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -449,6 +449,7 @@ def generate(args: list[str], output: TextIO) -> None: printer.write("co->co_extra = NULL;") with printer.block("if (co->co_weakreflist != NULL)"): printer.write("PyObject_ClearWeakRefs((PyObject *)co);") + printer.write("co->co_weakreflist = NULL;") with printer.block(f"void _Py_Deepfreeze_Fini(void)"): for p in printer.deallocs: printer.write(p) From 1a878cc9773a101d5ab66cd98d3f0abde0940599 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:44:45 +0000 Subject: [PATCH 14/25] add a new line in function decl --- Tools/scripts/deepfreeze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index a7bb70659b460d6..d580eb652da7f2e 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -442,7 +442,7 @@ def generate(args: list[str], output: TextIO) -> None: else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) - with printer.block("static void dealloc_codeobject(PyCodeObject *co)"): + with printer.block("static void \ndealloc_codeobject(PyCodeObject *co)"): printer.write("PyMem_Free(co->co_quickened);") printer.write("co->co_quickened = NULL;") printer.write("PyMem_Free(co->co_extra);") @@ -450,7 +450,7 @@ def generate(args: list[str], output: TextIO) -> None: with printer.block("if (co->co_weakreflist != NULL)"): printer.write("PyObject_ClearWeakRefs((PyObject *)co);") printer.write("co->co_weakreflist = NULL;") - with printer.block(f"void _Py_Deepfreeze_Fini(void)"): + with printer.block(f"void \n_Py_Deepfreeze_Fini(void)"): for p in printer.deallocs: printer.write(p) if verbose: From 7f73917b361996fb57715b54617fda272b5f19e2 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:46:06 +0000 Subject: [PATCH 15/25] add tiran's comments --- Programs/_freeze_module.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 98b14180839ec77..99a1d4dfc261f39 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -22,6 +22,12 @@ #include #endif +/* Empty finalizer for deepfrozen modules */ +void +_Py_Deepfreeze_Fini(void) +{ +} + /* To avoid a circular dependency on frozen.o, we create our own structure of frozen modules instead, left deliberately blank so as to avoid unintentional import of a stale version of _frozen_importlib. */ @@ -236,8 +242,3 @@ main(int argc, char *argv[]) return 1; } -/* Empty finalizer for deepfrozen modules*/ -void -_Py_Deepfreeze_Fini(void) -{ -} From 48fad74f04d837402694c72b846c260004bcc3f3 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:46:42 +0000 Subject: [PATCH 16/25] ditto --- Programs/_bootstrap_python.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 3a0c71ea442fd4e..f4d0167b62e7123 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -14,6 +14,12 @@ #include "Python/frozen_modules/importlib._bootstrap_external.h" /* End includes */ +/* Empty finalizer for deepfrozen modules*/ +void +_Py_Deepfreeze_Fini(void) +{ +} + /* Note that a negative size indicates a package. */ static const struct _frozen bootstrap_modules[] = { @@ -104,8 +110,3 @@ main(int argc, char **argv) Py_ExitStatusException(status); } -/* Empty finalizer for deepfrozen modules*/ -void -_Py_Deepfreeze_Fini(void) -{ -} From 49d1af338eb2973de055e4a3941a1287102a2c9f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 04:48:19 +0000 Subject: [PATCH 17/25] remove trailing whitespace --- Tools/scripts/deepfreeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index d580eb652da7f2e..05bf53395cb4fdc 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -114,7 +114,7 @@ def __init__(self, file: TextIO) -> None: self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') self.write('#include "internal/pycore_long.h"') - self.write("") + self.write("") @contextlib.contextmanager def indent(self) -> None: From 1518f2339df8d2d598ddbcae3989be3c7d5d70af Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 05:15:12 +0000 Subject: [PATCH 18/25] new idea --- Tools/scripts/deepfreeze.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 05bf53395cb4fdc..bd237212baac571 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -278,7 +278,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_varnames = {co_varnames},") self.write(f".co_cellvars = {co_cellvars},") self.write(f".co_freevars = {co_freevars},") - self.deallocs.append(f"dealloc_codeobject(&{name});") + self.deallocs.append(f"dealloc_codeobject(&{name}, (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval);") return f"& {name}.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: @@ -442,11 +442,12 @@ def generate(args: list[str], output: TextIO) -> None: else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) - with printer.block("static void \ndealloc_codeobject(PyCodeObject *co)"): + with printer.block("static void \ndealloc_codeobject(PyCodeObject *co, _Py_CODEUNIT *firstinstr)"): printer.write("PyMem_Free(co->co_quickened);") printer.write("co->co_quickened = NULL;") printer.write("PyMem_Free(co->co_extra);") printer.write("co->co_extra = NULL;") + printer.write("co->co_firstinstr = firstinstr;") with printer.block("if (co->co_weakreflist != NULL)"): printer.write("PyObject_ClearWeakRefs((PyObject *)co);") printer.write("co->co_weakreflist = NULL;") From 5311948803dbda2c959c91254811272f19c5246c Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 05:23:34 +0000 Subject: [PATCH 19/25] fix it for embedding --- Modules/main.c | 4 ---- Python/pylifecycle.c | 5 ++++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Modules/main.c b/Modules/main.c index 3fee01ee6c5975d..b9bcea393abe3cf 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -29,9 +29,6 @@ extern "C" { #endif -/* For deepfrozen modules */ -extern void _Py_Deepfreeze_Fini(void); - /* --- pymain_init() ---------------------------------------------- */ static PyStatus @@ -624,7 +621,6 @@ pymain_free(void) _PyPathConfig_ClearGlobal(); _Py_ClearStandardStreamEncoding(); _Py_ClearArgcArgv(); - _Py_Deepfreeze_Fini(); _PyRuntime_Finalize(); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9d10f94efa732ef..a181c9c1794aa46 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -74,6 +74,8 @@ _Py_IDENTIFIER(threading); extern "C" { #endif +/* Deepfreeze finalizer */ +extern void _Py_Deepfreeze_Fini(void); /* Forward declarations */ static PyStatus add_main_module(PyInterpreterState *interp); @@ -1724,7 +1726,8 @@ finalize_interp_clear(PyThreadState *tstate) _PyArg_Fini(); _Py_ClearFileSystemEncoding(); } - + /* Finalize deepfrozen code objects */ + _Py_Deepfreeze_Fini(); finalize_interp_types(tstate->interp); } From 2febbe090e60b100732cdeb0bc15c44d2db24f6f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 13:38:01 +0530 Subject: [PATCH 20/25] address review --- Include/internal/pycore_pylifecycle.h | 1 + Python/pylifecycle.c | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 35e560b42ee0ae7..439bc5b470bb266 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -83,6 +83,7 @@ extern void _PyWarnings_Fini(PyInterpreterState *interp); extern void _PyAST_Fini(PyInterpreterState *interp); extern void _PyAtExit_Fini(PyInterpreterState *interp); extern void _PyThread_FiniType(PyInterpreterState *interp); +extern void _Py_Deepfreeze_Fini(void); extern PyStatus _PyGILState_Init(_PyRuntimeState *runtime); extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a181c9c1794aa46..f95d2ac86898b75 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -74,9 +74,6 @@ _Py_IDENTIFIER(threading); extern "C" { #endif -/* Deepfreeze finalizer */ -extern void _Py_Deepfreeze_Fini(void); - /* Forward declarations */ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); @@ -1725,9 +1722,9 @@ finalize_interp_clear(PyThreadState *tstate) _Py_HashRandomization_Fini(); _PyArg_Fini(); _Py_ClearFileSystemEncoding(); + _Py_Deepfreeze_Fini(); } - /* Finalize deepfrozen code objects */ - _Py_Deepfreeze_Fini(); + finalize_interp_types(tstate->interp); } From e4a7e97c8db9f1ccbdad58f08db5af7034da4081 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 13:39:07 +0530 Subject: [PATCH 21/25] whitespace --- Tools/scripts/deepfreeze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index bd237212baac571..57c811a2bb96e46 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -442,7 +442,7 @@ def generate(args: list[str], output: TextIO) -> None: else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) - with printer.block("static void \ndealloc_codeobject(PyCodeObject *co, _Py_CODEUNIT *firstinstr)"): + with printer.block("static void\ndealloc_codeobject(PyCodeObject *co, _Py_CODEUNIT *firstinstr)"): printer.write("PyMem_Free(co->co_quickened);") printer.write("co->co_quickened = NULL;") printer.write("PyMem_Free(co->co_extra);") @@ -451,7 +451,7 @@ def generate(args: list[str], output: TextIO) -> None: with printer.block("if (co->co_weakreflist != NULL)"): printer.write("PyObject_ClearWeakRefs((PyObject *)co);") printer.write("co->co_weakreflist = NULL;") - with printer.block(f"void \n_Py_Deepfreeze_Fini(void)"): + with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"): for p in printer.deallocs: printer.write(p) if verbose: From cfe8a56ddc2e1f3258814c759baa02712ce0c9f6 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 25 Jan 2022 13:42:22 +0530 Subject: [PATCH 22/25] whitespace in pylifecycle.c --- Python/pylifecycle.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f95d2ac86898b75..106e11ea12ff533 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -74,6 +74,7 @@ _Py_IDENTIFIER(threading); extern "C" { #endif + /* Forward declarations */ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); @@ -1724,7 +1725,7 @@ finalize_interp_clear(PyThreadState *tstate) _Py_ClearFileSystemEncoding(); _Py_Deepfreeze_Fini(); } - + finalize_interp_types(tstate->interp); } From b414b8af5df60f299a76c1f9c4899dfd6184d91a Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 27 Jan 2022 13:43:40 +0530 Subject: [PATCH 23/25] move to Objects/codeobject.c --- Include/internal/pycore_code.h | 2 ++ Objects/codeobject.c | 15 +++++++++++++++ Tools/scripts/deepfreeze.py | 11 +---------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index dfc75300315e29a..884a91f3fb7edff 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -276,6 +276,8 @@ void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache); void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache); +/* Deallocator function for static codeobjects used in deepfreeze.py */ +void _PyStaticCode_Dealloc(PyCodeObject *co, _Py_CODEUNIT *firstinstr); #ifdef Py_STATS diff --git a/Objects/codeobject.c b/Objects/codeobject.c index a413b183be8edca..ad055514b206304 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1906,3 +1906,18 @@ _PyCode_ConstantKey(PyObject *op) } return key; } + + +void +_PyStaticCode_Dealloc(PyCodeObject *co, _Py_CODEUNIT *firstinstr) { + PyMem_Free(co->co_quickened); + co->co_quickened = NULL; + PyMem_Free(co->co_extra); + co->co_extra = NULL; + co->co_firstinstr = firstinstr; + if (co->co_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject *)co); + co->co_weakreflist = NULL; + } + co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; +} \ No newline at end of file diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 57c811a2bb96e46..78849390f12604d 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -278,7 +278,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_varnames = {co_varnames},") self.write(f".co_cellvars = {co_cellvars},") self.write(f".co_freevars = {co_freevars},") - self.deallocs.append(f"dealloc_codeobject(&{name}, (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval);") + self.deallocs.append(f"_PyStaticCode_Dealloc(&{name}, (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval);") return f"& {name}.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: @@ -442,15 +442,6 @@ def generate(args: list[str], output: TextIO) -> None: else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) - with printer.block("static void\ndealloc_codeobject(PyCodeObject *co, _Py_CODEUNIT *firstinstr)"): - printer.write("PyMem_Free(co->co_quickened);") - printer.write("co->co_quickened = NULL;") - printer.write("PyMem_Free(co->co_extra);") - printer.write("co->co_extra = NULL;") - printer.write("co->co_firstinstr = firstinstr;") - with printer.block("if (co->co_weakreflist != NULL)"): - printer.write("PyObject_ClearWeakRefs((PyObject *)co);") - printer.write("co->co_weakreflist = NULL;") with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"): for p in printer.deallocs: printer.write(p) From 133a46f24d9761d461f0b5718651ced520f62009 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 27 Jan 2022 08:15:31 +0000 Subject: [PATCH 24/25] newline --- Objects/codeobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index ad055514b206304..d87eed79b27693c 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1907,7 +1907,6 @@ _PyCode_ConstantKey(PyObject *op) return key; } - void _PyStaticCode_Dealloc(PyCodeObject *co, _Py_CODEUNIT *firstinstr) { PyMem_Free(co->co_quickened); From 1893b35cb1377e27799dc2ff934a094513b482ef Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 27 Jan 2022 08:25:59 +0000 Subject: [PATCH 25/25] fix style --- Objects/codeobject.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d87eed79b27693c..f983d66db05d6ea 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1908,7 +1908,8 @@ _PyCode_ConstantKey(PyObject *op) } void -_PyStaticCode_Dealloc(PyCodeObject *co, _Py_CODEUNIT *firstinstr) { +_PyStaticCode_Dealloc(PyCodeObject *co, _Py_CODEUNIT *firstinstr) +{ PyMem_Free(co->co_quickened); co->co_quickened = NULL; PyMem_Free(co->co_extra); @@ -1919,4 +1920,4 @@ _PyStaticCode_Dealloc(PyCodeObject *co, _Py_CODEUNIT *firstinstr) { co->co_weakreflist = NULL; } co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; -} \ No newline at end of file +}