From 52b71d0de1200f16316aa950ba5c81d61af6374f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Nov 2018 12:36:31 +0100 Subject: [PATCH 01/12] bpo-35265: Add _PyConfigCtx * Add _PyConfigCtx structure * Add "ctx" context (_PyConfigCtx) to _PyCoreConfig, _PyPathConfig, _Py_PreInit and PyCalculatePath. * Move _PyCoreConfig.utf8_mode into _PyConfigCtx * Replace _PyCoreConfig_INIT with _PyCoreConfig_Init() * Replace _PyPathConfig_INIT with _PyPathConfig_StaticInit() * Add new functions: * _PyMem_RawMallocCtx() * _PyMem_RawCallocCtx() * _PyMem_RawReallocCtx() * _PyMem_RawFreeCtx() * Add ctx paramter to functions: * _Py_DecodeLocaleEx() * _Py_EncodeLocaleEx() * _PyMem_RawWcsdup() * Use ctx in Python initialization code --- Include/coreconfig.h | 37 +++++-- Include/fileutils.h | 4 + Include/internal/pycore_fileutils.h | 7 ++ Include/internal/pycore_pathconfig.h | 21 ++-- Include/pymem.h | 4 +- Modules/_testcapimodule.c | 6 +- Modules/getpath.c | 62 ++++++------ Modules/main.c | 139 +++++++++++++-------------- Objects/obmalloc.c | 49 +++++++++- Objects/unicodeobject.c | 32 +++--- PC/getpathp.c | 58 ++++++----- Programs/_freeze_importlib.c | 3 +- Programs/_testembed.c | 13 ++- Python/coreconfig.c | 101 +++++++++++-------- Python/fileutils.c | 81 +++++++++------- Python/frozenmain.c | 3 +- Python/pathconfig.c | 113 ++++++++++++++-------- Python/pylifecycle.c | 9 +- Python/pystate.c | 2 +- Python/sysmodule.c | 96 ++++++++++-------- 20 files changed, 519 insertions(+), 321 deletions(-) diff --git a/Include/coreconfig.h b/Include/coreconfig.h index 0ed3222a15e142e..27ee53b658f8456 100644 --- a/Include/coreconfig.h +++ b/Include/coreconfig.h @@ -36,6 +36,30 @@ typedef struct { typedef struct { + /* Enable UTF-8 mode? + Set by -X utf8 command line option and PYTHONUTF8 environment variable. + If set to -1 (default), inherit Py_UTF8Mode value. */ + int utf8_mode; + + PyMemAllocatorEx raw_alloc; +} _PyConfigCtx; + +PyAPI_FUNC(void) _PyConfigCtx_Init(_PyConfigCtx *ctx); + + +#ifndef Py_LIMITED_API +/* Functions implemented in obmalloc.c */ +PyAPI_FUNC(void *) _PyMem_RawMallocCtx(const _PyConfigCtx *ctx, size_t size); +PyAPI_FUNC(void *) _PyMem_RawCallocCtx(const _PyConfigCtx *ctx, size_t nelem, size_t elsize); +PyAPI_FUNC(void *) _PyMem_RawReallocCtx(const _PyConfigCtx *ctx, void *ptr, size_t new_size); +PyAPI_FUNC(void) _PyMem_RawFreeCtx(const _PyConfigCtx *ctx, void *ptr); +PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const _PyConfigCtx *ctx, const wchar_t *str); +#endif /* !Py_LIMITED_API */ + + +typedef struct { + _PyConfigCtx ctx; + /* Install signal handlers? Yes by default. */ int install_signal_handlers; @@ -102,11 +126,6 @@ typedef struct { char *filesystem_encoding; char *filesystem_errors; - /* Enable UTF-8 mode? - Set by -X utf8 command line option and PYTHONUTF8 environment variable. - If set to -1 (default), inherit Py_UTF8Mode value. */ - int utf8_mode; - wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */ wchar_t *program_name; /* Program name, see also Py_GetProgramName() */ @@ -309,7 +328,8 @@ typedef struct { # define _PyCoreConfig_WINDOWS_INIT #endif -#define _PyCoreConfig_INIT \ +/* _PyCoreConfig_STATIC_INIT must not be used: use _PyCoreConfig_Init() */ +#define _PyCoreConfig_STATIC_INIT \ (_PyCoreConfig){ \ .install_signal_handlers = 1, \ .use_environment = -1, \ @@ -317,7 +337,6 @@ typedef struct { .faulthandler = -1, \ .tracemalloc = -1, \ .coerce_c_locale = -1, \ - .utf8_mode = -1, \ .argc = -1, \ .nmodule_search_path = -1, \ .isolated = -1, \ @@ -336,7 +355,9 @@ typedef struct { ._install_importlib = 1, \ ._check_hash_pycs_mode = "default", \ ._frozen = -1} -/* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */ +/* Note: _PyCoreConfig_STATIC_INIT sets other fields to 0/NULL */ + +PyAPI_FUNC(void) _PyCoreConfig_Init(_PyCoreConfig *config); #ifndef Py_LIMITED_API diff --git a/Include/fileutils.h b/Include/fileutils.h index fdd60fffcd5514e..4a65ad37497b23b 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -34,7 +34,9 @@ typedef enum { PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); + PyAPI_FUNC(int) _Py_DecodeLocaleEx( + const _PyConfigCtx *ctx, const char *arg, wchar_t **wstr, size_t *wlen, @@ -43,6 +45,7 @@ PyAPI_FUNC(int) _Py_DecodeLocaleEx( _Py_error_handler errors); PyAPI_FUNC(int) _Py_EncodeLocaleEx( + const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, @@ -51,6 +54,7 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( _Py_error_handler errors); #endif + #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _Py_device_encoding(int); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index d577e099d1f5513..ec67d7dc148e1fd 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -8,7 +8,13 @@ extern "C" { # error "Py_BUILD_CORE must be defined to include this header" #endif +PyAPI_FUNC(wchar_t*) _Py_DecodeLocaleCtx( + const _PyConfigCtx *ctx, + const char* arg, + size_t *wlen); + PyAPI_FUNC(int) _Py_DecodeUTF8Ex( + const _PyConfigCtx *ctx, const char *arg, Py_ssize_t arglen, wchar_t **wstr, @@ -17,6 +23,7 @@ PyAPI_FUNC(int) _Py_DecodeUTF8Ex( _Py_error_handler errors); PyAPI_FUNC(int) _Py_EncodeUTF8Ex( + const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 267e690976da87d..497e5310a0bef83 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -9,12 +9,15 @@ extern "C" { #endif PyAPI_FUNC(void) _Py_wstrlist_clear( + const _PyConfigCtx *ctx, int len, wchar_t **list); PyAPI_FUNC(wchar_t**) _Py_wstrlist_copy( + const _PyConfigCtx *ctx, int len, wchar_t **list); PyAPI_FUNC(_PyInitError) _Py_wstrlist_append( + const _PyConfigCtx *ctx, int *len, wchar_t ***list, const wchar_t *str); @@ -23,6 +26,8 @@ PyAPI_FUNC(PyObject*) _Py_wstrlist_as_pylist( wchar_t **list); typedef struct _PyPathConfig { + _PyConfigCtx ctx; + /* Full path to the Python program */ wchar_t *program_full_path; wchar_t *prefix; @@ -44,17 +49,21 @@ typedef struct _PyPathConfig { int site_import; } _PyPathConfig; -#define _PyPathConfig_INIT \ - {.module_search_path = NULL, \ - .isolated = -1, \ - .site_import = -1} -/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ +#define _PyPathConfig_STATIC_INIT \ + (_PyPathConfig){ \ + .module_search_path = NULL, \ + .isolated = -1, \ + .site_import = -1} +/* Note: _PyPathConfig_STATIC_INIT sets other fields to 0/NULL */ + +PyAPI_FUNC(void) _PyPathConfig_StaticInit( + _PyPathConfig *config); PyAPI_DATA(_PyPathConfig) _Py_path_config; PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); PyAPI_FUNC(_PyInitError) _PyPathConfig_SetGlobal( - const struct _PyPathConfig *config); + const _PyPathConfig *config); PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate_impl( _PyPathConfig *config, diff --git a/Include/pymem.h b/Include/pymem.h index 23457adb5a45e9b..09bfbf7280dbe35 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -79,9 +79,6 @@ PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); /* strdup() using PyMem_Malloc() */ PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); - -/* wcsdup() using PyMem_RawMalloc() */ -PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str); #endif /* Macros. */ @@ -198,6 +195,7 @@ PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain, PyAPI_FUNC(void) PyMem_SetupDebugHooks(void); #endif /* Py_LIMITED_API */ + /* bpo-35053: expose _Py_tracemalloc_config for performance: _Py_NewReference() needs an efficient check to test if tracemalloc is tracing. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 56a08a418b361f3..75ce05c2138edf4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4622,7 +4622,8 @@ encode_locale_ex(PyObject *self, PyObject *args) char *str = NULL; size_t error_pos; const char *reason = NULL; - int ret = _Py_EncodeLocaleEx(wstr, + PyInterpreterState *interp = _PyInterpreterState_Get(); + int ret = _Py_EncodeLocaleEx(&interp->core_config.ctx, wstr, &str, &error_pos, &reason, current_locale, error_handler); PyMem_Free(wstr); @@ -4666,7 +4667,8 @@ decode_locale_ex(PyObject *self, PyObject *args) wchar_t *wstr = NULL; size_t wlen = 0; const char *reason = NULL; - int ret = _Py_DecodeLocaleEx(str, + PyInterpreterState *interp = _PyInterpreterState_Get(); + int ret = _Py_DecodeLocaleEx(&interp->core_config.ctx, str, &wstr, &wlen, &reason, current_locale, error_handler); diff --git a/Modules/getpath.c b/Modules/getpath.c index 18df795c93fc96d..3b20261cf0af153 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -118,6 +118,8 @@ extern "C" { : _Py_INIT_NO_MEMORY() typedef struct { + _PyConfigCtx ctx; + wchar_t *path_env; /* PATH environment variable */ wchar_t *pythonpath; /* PYTHONPATH define */ @@ -142,6 +144,7 @@ _Py_wstat(const wchar_t* path, struct stat *buf) { int err; char *fname; + /* FIXME: use ctx? */ fname = _Py_EncodeLocaleRaw(path, NULL); if (fname == NULL) { errno = EINVAL; @@ -337,7 +340,7 @@ add_exe_suffix(wchar_t *progpath) bytes long. */ static int -search_for_prefix(const _PyCoreConfig *core_config, +search_for_prefix(const _PyCoreConfig *core_config, _PyPathConfig *config, PyCalculatePath *calculate, wchar_t *prefix) { size_t n; @@ -362,12 +365,12 @@ search_for_prefix(const _PyCoreConfig *core_config, joinpath(prefix, L"Modules/Setup.local"); if (isfile(prefix)) { /* Check VPATH to see if argv0_path is in the build directory. */ - vpath = Py_DecodeLocale(VPATH, NULL); + vpath = _Py_DecodeLocaleCtx(&config->ctx, VPATH, NULL); if (vpath != NULL) { wcsncpy(prefix, calculate->argv0_path, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; joinpath(prefix, vpath); - PyMem_RawFree(vpath); + _PyMem_RawFreeCtx(&config->ctx, vpath); joinpath(prefix, L"Lib"); joinpath(prefix, LANDMARK); if (ismodule(prefix)) { @@ -404,10 +407,10 @@ search_for_prefix(const _PyCoreConfig *core_config, static void -calculate_prefix(const _PyCoreConfig *core_config, +calculate_prefix(const _PyCoreConfig *core_config, _PyPathConfig *config, PyCalculatePath *calculate, wchar_t *prefix) { - calculate->prefix_found = search_for_prefix(core_config, calculate, prefix); + calculate->prefix_found = search_for_prefix(core_config, config, calculate, prefix); if (!calculate->prefix_found) { if (!core_config->_frozen) { fprintf(stderr, @@ -599,12 +602,12 @@ calculate_program_full_path(const _PyCoreConfig *core_config, execpath[0] == SEP) { size_t len; - wchar_t *path = Py_DecodeLocale(execpath, &len); + wchar_t *path = _Py_DecodeLocaleCtx(&config->ctx, execpath, &len); if (path == NULL) { return DECODE_LOCALE_ERR("executable path", len); } wcsncpy(program_full_path, path, MAXPATHLEN); - PyMem_RawFree(path); + _PyMem_RawFreeCtx(&config->ctx, path); } #endif /* __APPLE__ */ else if (calculate->path_env) { @@ -653,7 +656,7 @@ calculate_program_full_path(const _PyCoreConfig *core_config, } #endif - config->program_full_path = _PyMem_RawWcsdup(program_full_path); + config->program_full_path = _PyMem_RawWcsdup(&config->ctx, program_full_path); if (config->program_full_path == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -662,7 +665,10 @@ calculate_program_full_path(const _PyCoreConfig *core_config, static _PyInitError -calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_path) +calculate_argv0_path(PyCalculatePath *calculate, + const _PyCoreConfig *core_config, + _PyPathConfig *config, + const wchar_t *program_full_path) { wcsncpy(calculate->argv0_path, program_full_path, MAXPATHLEN); calculate->argv0_path[MAXPATHLEN] = '\0'; @@ -688,7 +694,7 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat ** build-directory-specific logic to find Lib and such. */ size_t len; - wchar_t* wbuf = Py_DecodeLocale(modPath, &len); + wchar_t* wbuf = _Py_DecodeLocaleCtx(&config->ctx, modPath, &len); if (wbuf == NULL) { return DECODE_LOCALE_ERR("framework location", len); } @@ -706,7 +712,7 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat /* Use the location of the library as the program_full_path */ wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); } - PyMem_RawFree(wbuf); + _PyMem_RawFreeCtx(&config->ctx, wbuf); } #endif @@ -834,7 +840,7 @@ calculate_module_search_path(const _PyCoreConfig *core_config, bufsz += wcslen(exec_prefix) + 1; /* Allocate the buffer */ - wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); + wchar_t *buf = _PyMem_RawMallocCtx(&config->ctx, bufsz * sizeof(wchar_t)); if (buf == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -891,30 +897,31 @@ calculate_module_search_path(const _PyCoreConfig *core_config, static _PyInitError calculate_init(PyCalculatePath *calculate, + _PyPathConfig *config, const _PyCoreConfig *core_config) { size_t len; const char *path = getenv("PATH"); if (path) { - calculate->path_env = Py_DecodeLocale(path, &len); + calculate->path_env = _Py_DecodeLocaleCtx(&calculate->ctx, path, &len); if (!calculate->path_env) { return DECODE_LOCALE_ERR("PATH environment variable", len); } } - calculate->pythonpath = Py_DecodeLocale(PYTHONPATH, &len); + calculate->pythonpath = _Py_DecodeLocaleCtx(&config->ctx, PYTHONPATH, &len); if (!calculate->pythonpath) { return DECODE_LOCALE_ERR("PYTHONPATH define", len); } - calculate->prefix = Py_DecodeLocale(PREFIX, &len); + calculate->prefix = _Py_DecodeLocaleCtx(&config->ctx, PREFIX, &len); if (!calculate->prefix) { return DECODE_LOCALE_ERR("PREFIX define", len); } - calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len); + calculate->exec_prefix = _Py_DecodeLocaleCtx(&config->ctx, EXEC_PREFIX, &len); if (!calculate->prefix) { return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); } - calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len); + calculate->lib_python = _Py_DecodeLocaleCtx(&config->ctx, "lib/python" VERSION, &len); if (!calculate->lib_python) { return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); } @@ -925,11 +932,11 @@ calculate_init(PyCalculatePath *calculate, static void calculate_free(PyCalculatePath *calculate) { - PyMem_RawFree(calculate->pythonpath); - PyMem_RawFree(calculate->prefix); - PyMem_RawFree(calculate->exec_prefix); - PyMem_RawFree(calculate->lib_python); - PyMem_RawFree(calculate->path_env); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->pythonpath); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->prefix); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->exec_prefix); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->lib_python); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->path_env); } @@ -944,7 +951,7 @@ calculate_path_impl(const _PyCoreConfig *core_config, return err; } - err = calculate_argv0_path(calculate, config->program_full_path); + err = calculate_argv0_path(calculate, core_config, config, config->program_full_path); if (_Py_INIT_FAILED(err)) { return err; } @@ -953,7 +960,7 @@ calculate_path_impl(const _PyCoreConfig *core_config, wchar_t prefix[MAXPATHLEN+1]; memset(prefix, 0, sizeof(prefix)); - calculate_prefix(core_config, calculate, prefix); + calculate_prefix(core_config, config, calculate, prefix); calculate_zip_path(calculate, prefix); @@ -976,14 +983,14 @@ calculate_path_impl(const _PyCoreConfig *core_config, calculate_reduce_prefix(calculate, prefix); - config->prefix = _PyMem_RawWcsdup(prefix); + config->prefix = _PyMem_RawWcsdup(&config->ctx, prefix); if (config->prefix == NULL) { return _Py_INIT_NO_MEMORY(); } calculate_reduce_exec_prefix(calculate, exec_prefix); - config->exec_prefix = _PyMem_RawWcsdup(exec_prefix); + config->exec_prefix = _PyMem_RawWcsdup(&config->ctx, exec_prefix); if (config->exec_prefix == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -997,8 +1004,9 @@ _PyPathConfig_Calculate_impl(_PyPathConfig *config, const _PyCoreConfig *core_co { PyCalculatePath calculate; memset(&calculate, 0, sizeof(calculate)); + _PyConfigCtx_Init(&calculate.ctx); - _PyInitError err = calculate_init(&calculate, core_config); + _PyInitError err = calculate_init(&calculate, config, core_config); if (_Py_INIT_FAILED(err)) { goto done; } diff --git a/Modules/main.c b/Modules/main.c index 8e66ddded419a31..1ec77fea7419b7e 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -2,6 +2,7 @@ #include "Python.h" #include "osdefs.h" +#include "pycore_fileutils.h" #include "pycore_getopt.h" #include "pycore_pathconfig.h" #include "pycore_pylifecycle.h" @@ -315,6 +316,8 @@ typedef struct { /* Structure used by Py_Main() to pass data to subfunctions */ typedef struct { + _PyConfigCtx ctx; + /* Input arguments */ int argc; int use_bytes_argv; @@ -346,9 +349,9 @@ typedef struct { static wchar_t* -pymain_wstrdup(_PyMain *pymain, const wchar_t *str) +pymain_wstrdup(_PyMain *pymain, _PyConfigCtx *ctx, const wchar_t *str) { - wchar_t *str2 = _PyMem_RawWcsdup(str); + wchar_t *str2 = _PyMem_RawWcsdup(ctx, str); if (str2 == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return NULL; @@ -374,9 +377,9 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, for (int i = 0; i < pymain->argc; i++) { size_t len; - wchar_t *arg = Py_DecodeLocale(pymain->bytes_argv[i], &len); + wchar_t *arg = _Py_DecodeLocaleCtx(&config->ctx, pymain->bytes_argv[i], &len); if (arg == NULL) { - _Py_wstrlist_clear(i, argv); + _Py_wstrlist_clear(&config->ctx, i, argv); pymain->err = DECODE_LOCALE_ERR("command line arguments", (Py_ssize_t)len); return -1; @@ -398,7 +401,7 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, else { program = L""; } - config->program = pymain_wstrdup(pymain, program); + config->program = pymain_wstrdup(pymain, &config->ctx, program); if (config->program == NULL) { return -1; } @@ -408,21 +411,21 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, static void -pymain_clear_cmdline(_PyMain *pymain, _PyCmdline *cmdline) +pymain_clear_cmdline(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _Py_wstrlist_clear(cmdline->nwarnoption, cmdline->warnoptions); + _Py_wstrlist_clear(&config->ctx, cmdline->nwarnoption, cmdline->warnoptions); cmdline->nwarnoption = 0; cmdline->warnoptions = NULL; - _Py_wstrlist_clear(cmdline->nenv_warnoption, cmdline->env_warnoptions); + _Py_wstrlist_clear(&config->ctx, cmdline->nenv_warnoption, cmdline->env_warnoptions); cmdline->nenv_warnoption = 0; cmdline->env_warnoptions = NULL; if (pymain->use_bytes_argv && cmdline->argv != NULL) { - _Py_wstrlist_clear(pymain->argc, cmdline->argv); + _Py_wstrlist_clear(&config->ctx, pymain->argc, cmdline->argv); } cmdline->argv = NULL; @@ -435,7 +438,7 @@ pymain_clear_pymain(_PyMain *pymain) { #define CLEAR(ATTR) \ do { \ - PyMem_RawFree(ATTR); \ + _PyMem_RawFreeCtx(&pymain->ctx, ATTR); \ ATTR = NULL; \ } while (0) @@ -445,20 +448,6 @@ pymain_clear_pymain(_PyMain *pymain) #undef CLEAR } -static void -pymain_clear_config(_PyCoreConfig *config) -{ - /* Clear core config with the memory allocator - used by pymain_read_conf() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _PyCoreConfig_Clear(config); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - static void pymain_free(_PyMain *pymain) { @@ -471,17 +460,12 @@ pymain_free(_PyMain *pymain) _PyPathConfig_ClearGlobal(); _Py_ClearStandardStreamEncoding(); - /* Force the allocator used by pymain_read_conf() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - pymain_clear_pymain(pymain); - _Py_wstrlist_clear(orig_argc, orig_argv); + _Py_wstrlist_clear(&pymain->ctx, orig_argc, orig_argv); orig_argc = 0; orig_argv = NULL; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering @@ -526,21 +510,21 @@ pymain_sys_path_add_path0(PyInterpreterState *interp, PyObject *path0) _PyInitError -_Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) +_Py_wstrlist_append(const _PyConfigCtx *ctx, int *len, wchar_t ***list, const wchar_t *str) { if (*len == INT_MAX) { /* len+1 would overflow */ return _Py_INIT_NO_MEMORY(); } - wchar_t *str2 = _PyMem_RawWcsdup(str); + wchar_t *str2 = _PyMem_RawWcsdup(ctx, str); if (str2 == NULL) { return _Py_INIT_NO_MEMORY(); } size_t size = (*len + 1) * sizeof(list[0]); - wchar_t **list2 = (wchar_t **)PyMem_RawRealloc(*list, size); + wchar_t **list2 = (wchar_t **)_PyMem_RawReallocCtx(ctx, *list, size); if (list2 == NULL) { - PyMem_RawFree(str2); + _PyMem_RawFreeCtx(ctx, str2); return _Py_INIT_NO_MEMORY(); } list2[*len] = str2; @@ -551,9 +535,10 @@ _Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) static int -pymain_wstrlist_append(_PyMain *pymain, int *len, wchar_t ***list, const wchar_t *str) +pymain_wstrlist_append(_PyMain *pymain, _PyCoreConfig *config, + int *len, wchar_t ***list, const wchar_t *str) { - _PyInitError err = _Py_wstrlist_append(len, list, str); + _PyInitError err = _Py_wstrlist_append(&config->ctx, len, list, str); if (_Py_INIT_FAILED(err)) { pymain->err = err; return -1; @@ -584,7 +569,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, that look like options are left for the command to interpret. */ size_t len = wcslen(_PyOS_optarg) + 1 + 1; - wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); + wchar_t *command = _PyMem_RawMallocCtx(&pymain->ctx, sizeof(wchar_t) * len); if (command == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; @@ -600,7 +585,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, /* -m is the last option; following arguments that look like options are left for the module to interpret. */ - pymain->module = pymain_wstrdup(pymain, _PyOS_optarg); + pymain->module = pymain_wstrdup(pymain, &pymain->ctx, _PyOS_optarg); if (pymain->module == NULL) { return -1; } @@ -689,7 +674,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, break; case 'W': - if (pymain_wstrlist_append(pymain, + if (pymain_wstrlist_append(pymain, config, &cmdline->nwarnoption, &cmdline->warnoptions, _PyOS_optarg) < 0) { @@ -698,7 +683,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, break; case 'X': - if (pymain_wstrlist_append(pymain, + if (pymain_wstrlist_append(pymain, config, &config->nxoption, &config->xoptions, _PyOS_optarg) < 0) { @@ -726,7 +711,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, && _PyOS_optind < pymain->argc && wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0) { - pymain->filename = pymain_wstrdup(pymain, cmdline->argv[_PyOS_optind]); + pymain->filename = pymain_wstrdup(pymain, &pymain->ctx, cmdline->argv[_PyOS_optind]); if (pymain->filename == NULL) { return -1; } @@ -797,7 +782,8 @@ static _PyInitError config_add_warnings_optlist(_PyCoreConfig *config, int len, wchar_t **options) { for (int i = 0; i < len; i++) { - _PyInitError err = _Py_wstrlist_append(&config->nwarnoption, + _PyInitError err = _Py_wstrlist_append(&config->ctx, + &config->nwarnoption, &config->warnoptions, options[i]); if (_Py_INIT_FAILED(err)) { @@ -831,7 +817,8 @@ config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) */ if (config->dev_mode) { - err = _Py_wstrlist_append(&config->nwarnoption, + err = _Py_wstrlist_append(&config->ctx, + &config->nwarnoption, &config->warnoptions, L"default"); if (_Py_INIT_FAILED(err)) { @@ -865,7 +852,8 @@ config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) else { filter = L"default::BytesWarning"; } - err = _Py_wstrlist_append(&config->nwarnoption, + err = _Py_wstrlist_append(&config->ctx, + &config->nwarnoption, &config->warnoptions, filter); if (_Py_INIT_FAILED(err)) { @@ -900,7 +888,8 @@ cmdline_init_env_warnoptions(_PyMain *pymain, const _PyCoreConfig *config, warning != NULL; warning = WCSTOK(NULL, L",", &context)) { - _PyInitError err = _Py_wstrlist_append(&cmdline->nenv_warnoption, + _PyInitError err = _Py_wstrlist_append(&config->ctx, + &cmdline->nenv_warnoption, &cmdline->env_warnoptions, warning); if (_Py_INIT_FAILED(err)) { @@ -982,10 +971,10 @@ pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdlin /* Ensure at least one (empty) argument is seen */ static wchar_t *empty_argv[1] = {L""}; argc = 1; - argv = _Py_wstrlist_copy(1, empty_argv); + argv = _Py_wstrlist_copy(&config->ctx, 1, empty_argv); } else { - argv = _Py_wstrlist_copy(argc, &cmdline->argv[_PyOS_optind]); + argv = _Py_wstrlist_copy(&config->ctx, argc, &cmdline->argv[_PyOS_optind]); } if (argv == NULL) { @@ -1003,9 +992,9 @@ pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdlin arg0 = L"-m"; } if (arg0 != NULL) { - arg0 = _PyMem_RawWcsdup(arg0); + arg0 = _PyMem_RawWcsdup(&config->ctx, arg0); if (arg0 == NULL) { - _Py_wstrlist_clear(argc, argv); + _Py_wstrlist_clear(&config->ctx, argc, argv); pymain->err = _Py_INIT_NO_MEMORY(); return -1; } @@ -1293,11 +1282,13 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, #ifdef MS_WINDOWS int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag; #endif - _PyCoreConfig save_config = _PyCoreConfig_INIT; + _PyCoreConfig save_config; int res = -1; int locale_coerced = 0; int loops = 0; + _PyCoreConfig_Init(&save_config); + if (_PyCoreConfig_Copy(&save_config, config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); goto done; @@ -1307,7 +1298,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, _Py_SetLocaleFromEnv(LC_CTYPE); while (1) { - int utf8_mode = config->utf8_mode; + int utf8_mode = config->ctx.utf8_mode; int encoding_changed = 0; /* Watchdog to prevent an infinite loop */ @@ -1320,7 +1311,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */ - Py_UTF8Mode = config->utf8_mode; + Py_UTF8Mode = config->ctx.utf8_mode; #ifdef MS_WINDOWS Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; #endif @@ -1352,13 +1343,13 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, } if (utf8_mode == -1) { - if (config->utf8_mode == 1) { + if (config->ctx.utf8_mode == 1) { /* UTF-8 Mode enabled */ encoding_changed = 1; } } else { - if (config->utf8_mode != utf8_mode) { + if (config->ctx.utf8_mode != utf8_mode) { encoding_changed = 1; } } @@ -1369,15 +1360,15 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, /* Reset the configuration before reading again the configuration, just keep UTF-8 Mode value. */ - int new_utf8_mode = config->utf8_mode; + int new_utf8_mode = config->ctx.utf8_mode; int new_coerce_c_locale = config->coerce_c_locale; if (_PyCoreConfig_Copy(config, &save_config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); goto done; } - pymain_clear_cmdline(pymain, cmdline); + pymain_clear_cmdline(pymain, config, cmdline); memset(cmdline, 0, sizeof(*cmdline)); - config->utf8_mode = new_utf8_mode; + config->ctx.utf8_mode = new_utf8_mode; config->coerce_c_locale = new_coerce_c_locale; /* The encoding changed: read again the configuration @@ -1686,11 +1677,6 @@ static int pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { - pymain->err = _PyRuntime_Initialize(); - if (_Py_INIT_FAILED(pymain->err)) { - return -1; - } - int res = pymain_read_conf(pymain, config, cmdline); if (res < 0) { return -1; @@ -1712,7 +1698,7 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, } /* For Py_GetArgcArgv(). Cleared by pymain_free(). */ - orig_argv = _Py_wstrlist_copy(pymain->argc, cmdline->argv); + orig_argv = _Py_wstrlist_copy(&pymain->ctx, pymain->argc, cmdline->argv); if (orig_argv == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; @@ -1736,8 +1722,6 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, static int pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) { - /* Force default allocator, since pymain_free() and pymain_clear_config() - must use the same allocator than this function. */ PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); #ifdef Py_DEBUG @@ -1750,7 +1734,7 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) int res = pymain_cmdline_impl(pymain, config, &cmdline); - pymain_clear_cmdline(pymain, &cmdline); + pymain_clear_cmdline(pymain, config, &cmdline); #ifdef Py_DEBUG /* Make sure that PYMEM_DOMAIN_RAW has not been modified */ @@ -1766,6 +1750,12 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) static int pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) { + pymain->err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(pymain->err)) { + return -1; + } + _PyConfigCtx_Init(&pymain->ctx); + /* 754 requires that FP exceptions run in "no stop" mode by default, * and until C vendors implement C99's ways to control FP exceptions, * Python requires non-stop mode. Alas, some platforms enable FP @@ -1775,17 +1765,19 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) fedisableexcept(FE_OVERFLOW); #endif - _PyCoreConfig local_config = _PyCoreConfig_INIT; + _PyCoreConfig local_config; _PyCoreConfig *config = &local_config; + _PyCoreConfig_Init(&local_config); + _PyCoreConfig_GetGlobalConfig(config); int cmd_res = pymain_cmdline(pymain, config); if (cmd_res < 0) { - _Py_FatalInitError(pymain->err); + return -1; } if (cmd_res == 1) { - pymain_clear_config(config); + _PyCoreConfig_Clear(config); return 1; } @@ -1796,15 +1788,15 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) PyInterpreterState *interp; pymain->err = _Py_InitializeCore(&interp, config); if (_Py_INIT_FAILED(pymain->err)) { - _Py_FatalInitError(pymain->err); + return -1; } *interp_p = interp; - pymain_clear_config(config); + _PyCoreConfig_Clear(config); config = &interp->core_config; if (pymain_init_python_main(pymain, config, interp) < 0) { - _Py_FatalInitError(pymain->err); + return -1; } return 0; } @@ -1815,6 +1807,9 @@ pymain_main(_PyMain *pymain) { PyInterpreterState *interp; int res = pymain_init(pymain, &interp); + if (res < 0) { + _Py_FatalInitError(pymain->err); + } if (res != 1) { if (pymain_run_python(pymain, interp) < 0) { _Py_FatalInitError(pymain->err); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 1c2a32050f9381e..d71a7529e5ab003 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -513,6 +513,7 @@ PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) _PyObject_Arena = *allocator; } + void * PyMem_RawMalloc(size_t size) { @@ -551,6 +552,44 @@ void PyMem_RawFree(void *ptr) } +void* +_PyMem_RawMallocCtx(const _PyConfigCtx *ctx, size_t size) +{ + const PyMemAllocatorEx *alloc = &ctx->raw_alloc; + /* see PyMem_RawMalloc() */ + if (size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return alloc->malloc(alloc->ctx, size); +} + +void* +_PyMem_RawCallocCtx(const _PyConfigCtx *ctx, size_t nelem, size_t elsize) +{ + const PyMemAllocatorEx *alloc = &ctx->raw_alloc; + /* see PyMem_RawMalloc() */ + if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) + return NULL; + return alloc->calloc(alloc->ctx, nelem, elsize); +} + +void* +_PyMem_RawReallocCtx(const _PyConfigCtx *ctx, void *ptr, size_t new_size) +{ + const PyMemAllocatorEx *alloc = &ctx->raw_alloc; + /* see PyMem_RawMalloc() */ + if (new_size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return alloc->realloc(alloc->ctx, ptr, new_size); +} + +void +_PyMem_RawFreeCtx(const _PyConfigCtx *ctx, void *ptr) +{ + const PyMemAllocatorEx *alloc = &ctx->raw_alloc; + alloc->free(alloc->ctx, ptr); +} + + void * PyMem_Malloc(size_t size) { @@ -586,7 +625,7 @@ PyMem_Free(void *ptr) wchar_t* -_PyMem_RawWcsdup(const wchar_t *str) +_PyMem_RawWcsdup(const _PyConfigCtx *ctx, const wchar_t *str) { assert(str != NULL); @@ -596,7 +635,13 @@ _PyMem_RawWcsdup(const wchar_t *str) } size_t size = (len + 1) * sizeof(wchar_t); - wchar_t *str2 = PyMem_RawMalloc(size); + wchar_t *str2; + if (ctx != NULL) { + str2 = _PyMem_RawMallocCtx(ctx, size); + } + else { + str2 = PyMem_RawMalloc(size); + } if (str2 == NULL) { return NULL; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 04ca5f3344470e7..4b20c78264395a0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3424,7 +3424,8 @@ unicode_encode_locale(PyObject *unicode, const char *errors, char *str; size_t error_pos; const char *reason; - int res = _Py_EncodeLocaleEx(wstr, &str, &error_pos, &reason, + PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); + int res = _Py_EncodeLocaleEx(&interp->core_config.ctx, wstr, &str, &error_pos, &reason, current_locale, error_handler); if (res != 0) { if (res == -2) { @@ -3636,7 +3637,10 @@ unicode_decode_locale(const char *str, Py_ssize_t len, const char *errors, wchar_t *wstr; size_t wlen; const char *reason; - int res = _Py_DecodeLocaleEx(str, &wstr, &wlen, &reason, + + PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); + int res = _Py_DecodeLocaleEx(&interp->core_config.ctx, + str, &wstr, &wlen, &reason, current_locale, error_handler); if (res != 0) { if (res == -2) { @@ -4904,7 +4908,8 @@ PyUnicode_DecodeUTF8Stateful(const char *s, non-NULL, write the start of the illegal byte sequence into *wlen. If reason is not NULL, write the decoding error message into *reason. */ int -_Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, +_Py_DecodeUTF8Ex(const _PyConfigCtx *ctx, const char *s, Py_ssize_t size, + wchar_t **wstr, size_t *wlen, const char **reason, _Py_error_handler errors) { const char *orig_s = s; @@ -4934,7 +4939,7 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, return -1; } - unicode = PyMem_RawMalloc((size + 1) * sizeof(wchar_t)); + unicode = _PyMem_RawMallocCtx(ctx, (size + 1) * sizeof(wchar_t)); if (!unicode) { return -1; } @@ -4980,7 +4985,7 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, unicode[outpos++] = ch; } else { - PyMem_RawFree(unicode ); + _PyMem_RawFreeCtx(ctx, unicode); if (reason != NULL) { switch (ch) { case 0: @@ -5011,11 +5016,15 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, return 0; } +/* FIXME: add ctx parameter */ wchar_t* _Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen) { wchar_t *wstr; - int res = _Py_DecodeUTF8Ex(arg, arglen, &wstr, NULL, NULL, 1); + _PyConfigCtx ctx; + + _PyConfigCtx_Init(&ctx); + int res = _Py_DecodeUTF8Ex(&ctx, arg, arglen, &wstr, NULL, NULL, 1); if (res != 0) { return NULL; } @@ -5034,7 +5043,8 @@ _Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen) On memory allocation failure, return -1. */ int -_Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, +_Py_EncodeUTF8Ex(const _PyConfigCtx *ctx, const wchar_t *text, + char **str, size_t *error_pos, const char **reason, int raw_malloc, _Py_error_handler errors) { const Py_ssize_t max_char_size = 4; @@ -5063,7 +5073,7 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, } char *bytes; if (raw_malloc) { - bytes = PyMem_RawMalloc((len + 1) * max_char_size); + bytes = _PyMem_RawMallocCtx(ctx, (len + 1) * max_char_size); } else { bytes = PyMem_Malloc((len + 1) * max_char_size); @@ -5108,7 +5118,7 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, *reason = "encoding error"; } if (raw_malloc) { - PyMem_RawFree(bytes); + _PyMem_RawFreeCtx(ctx, bytes); } else { PyMem_Free(bytes); @@ -5136,7 +5146,7 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, size_t final_size = (p - bytes); char *bytes2; if (raw_malloc) { - bytes2 = PyMem_RawRealloc(bytes, final_size); + bytes2 = _PyMem_RawReallocCtx(ctx, bytes, final_size); } else { bytes2 = PyMem_Realloc(bytes, final_size); @@ -5146,7 +5156,7 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, *error_pos = (size_t)-1; } if (raw_malloc) { - PyMem_RawFree(bytes); + _PyMem_RawFreeCtx(ctx, bytes); } else { PyMem_Free(bytes); diff --git a/PC/getpathp.c b/PC/getpathp.c index ee9d3d258f68573..7f6111b88ad0cec 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -118,6 +118,8 @@ #endif typedef struct { + _PyConfigCtx ctx; + const wchar_t *path_env; /* PATH environment variable */ const wchar_t *home; /* PYTHONHOME environment variable */ @@ -353,7 +355,7 @@ extern const char *PyWin_DLLVersionString; in advance. It could be simplied now Win16/Win32s is dead! */ static wchar_t * -getpythonregpath(HKEY keyBase, int skipcore) +getpythonregpath(_PyConfigCtx *ctx, HKEY keyBase, int skipcore) { HKEY newKey = 0; DWORD dataSize = 0; @@ -375,7 +377,7 @@ getpythonregpath(HKEY keyBase, int skipcore) keyBufLen = sizeof(keyPrefix) + sizeof(WCHAR)*(versionLen-1) + sizeof(keySuffix); - keyBuf = keyBufPtr = PyMem_RawMalloc(keyBufLen); + keyBuf = keyBufPtr = _PyMem_RawMallocCtx(ctx, keyBufLen); if (keyBuf==NULL) { goto done; } @@ -407,7 +409,7 @@ getpythonregpath(HKEY keyBase, int skipcore) /* Allocate a temp array of char buffers, so we only need to loop reading the registry once */ - ppPaths = PyMem_RawMalloc( sizeof(WCHAR *) * numKeys ); + ppPaths = _PyMem_RawMallocCtx(ctx, sizeof(WCHAR *) * numKeys); if (ppPaths==NULL) { goto done; } @@ -435,7 +437,7 @@ getpythonregpath(HKEY keyBase, int skipcore) /* Find the value of the buffer size, malloc, then read it */ RegQueryValueExW(subKey, NULL, 0, NULL, NULL, &reqdSize); if (reqdSize) { - ppPaths[index] = PyMem_RawMalloc(reqdSize); + ppPaths[index] = _PyMem_RawMallocCtx(ctx, reqdSize); if (ppPaths[index]) { RegQueryValueExW(subKey, NULL, 0, NULL, (LPBYTE)ppPaths[index], @@ -452,7 +454,7 @@ getpythonregpath(HKEY keyBase, int skipcore) } /* original datasize from RegQueryInfo doesn't include the \0 */ - dataBuf = PyMem_RawMalloc((dataSize+1) * sizeof(WCHAR)); + dataBuf = _PyMem_RawMallocCtx(ctx, (dataSize+1) * sizeof(WCHAR)); if (dataBuf) { WCHAR *szCur = dataBuf; /* Copy our collected strings */ @@ -484,7 +486,7 @@ getpythonregpath(HKEY keyBase, int skipcore) rc = RegQueryValueExW(newKey, NULL, 0, NULL, (LPBYTE)szCur, &dataSize); if (rc != ERROR_SUCCESS) { - PyMem_RawFree(dataBuf); + _PyMem_RawFreeCtx(ctx, dataBuf); goto done; } } @@ -494,14 +496,15 @@ getpythonregpath(HKEY keyBase, int skipcore) done: /* Loop freeing my temp buffers */ if (ppPaths) { - for(index=0; indexdll_path = _PyMem_RawWcsdup(dll_path); + config->dll_path = _PyMem_RawWcsdup(&config->ctx, dll_path); if (config->dll_path == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -544,7 +547,7 @@ get_program_full_path(const _PyCoreConfig *core_config, return _Py_INIT_ERR("Cannot determine program path"); } - config->program_full_path = PyMem_RawMalloc( + config->program_full_path = _PyMem_RawMallocCtx(&config->ctx, sizeof(wchar_t) * (MAXPATHLEN + 1)); return canonicalize(config->program_full_path, @@ -568,7 +571,7 @@ read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path) size_t bufsiz = MAXPATHLEN; size_t prefixlen = wcslen(prefix); - wchar_t *buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t)); + wchar_t *buf = _PyMem_RawMallocCtx(&config->ctx, bufsiz * sizeof(wchar_t)); buf[0] = '\0'; while (!feof(sp_file)) { @@ -596,16 +599,16 @@ read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path) } DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0); - wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t)); + wchar_t *wline = _PyMem_RawMallocCtx(&config->ctx, (wn + 1) * sizeof(wchar_t)); wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1); wline[wn] = '\0'; size_t usedsiz = wcslen(buf); while (usedsiz + wn + prefixlen + 4 > bufsiz) { bufsiz += MAXPATHLEN; - buf = (wchar_t*)PyMem_RawRealloc(buf, (bufsiz + 1) * sizeof(wchar_t)); + buf = _PyMem_RawReallocCtx(&config->ctx, buf, (bufsiz + 1) * sizeof(wchar_t)); if (!buf) { - PyMem_RawFree(wline); + _PyMem_RawFreeCtx(&config->ctx, wline); goto error; } } @@ -627,7 +630,7 @@ read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path) wchar_t *b = &buf[usedsiz]; join(b, wline); - PyMem_RawFree(wline); + _PyMem_RawFreeCtx(&config->ctx, wline); } fclose(sp_file); @@ -635,7 +638,7 @@ read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path) return 1; error: - PyMem_RawFree(buf); + _PyMem_RawFreeCtx(&config->ctx, buf); fclose(sp_file); return 0; } @@ -756,8 +759,8 @@ calculate_module_search_path(const _PyCoreConfig *core_config, { int skiphome = calculate->home==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED - calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); - calculate->user_path = getpythonregpath(HKEY_CURRENT_USER, skiphome); + calculate->machine_path = getpythonregpath(&calculate->ctx, HKEY_LOCAL_MACHINE, skiphome); + calculate->user_path = getpythonregpath(&calculate->ctx, HKEY_CURRENT_USER, skiphome); #endif /* We only use the default relative PYTHONPATH if we haven't anything better to use! */ @@ -805,7 +808,7 @@ calculate_module_search_path(const _PyCoreConfig *core_config, } wchar_t *buf, *start_buf; - buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); + buf = _PyMem_RawMallocCtx(&config->ctx, bufsz * sizeof(wchar_t)); if (buf == NULL) { /* We can't exit, so print a warning and limp along */ fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n"); @@ -978,7 +981,7 @@ calculate_path_impl(const _PyCoreConfig *core_config, } done: - config->prefix = _PyMem_RawWcsdup(prefix); + config->prefix = _PyMem_RawWcsdup(&config->ctx, prefix); if (config->prefix == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -990,8 +993,14 @@ calculate_path_impl(const _PyCoreConfig *core_config, static void calculate_free(PyCalculatePath *calculate) { - PyMem_RawFree(calculate->machine_path); - PyMem_RawFree(calculate->user_path); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->machine_path); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->user_path); +} + + +static void +_PyCalculatePath_Init(PyCalculatePath *calculate) +{ } @@ -1000,6 +1009,7 @@ _PyPathConfig_Calculate_impl(_PyPathConfig *config, const _PyCoreConfig *core_co { PyCalculatePath calculate; memset(&calculate, 0, sizeof(calculate)); + _PyConfigCtx_Init(&calculate.ctx); calculate_init(&calculate, core_config); diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 3f43757ecb1ff5e..c9c8eaa2c3dee27 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -76,7 +76,8 @@ main(int argc, char *argv[]) } text[text_size] = '\0'; - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); config.user_site_directory = 0; config.site_import = 0; config.use_environment = 0; diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 22212bff52d4a1c..f488567c7d905d3 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -427,7 +427,8 @@ static int test_init_global_config(void) static int test_init_from_config(void) { /* Test _Py_InitializeFromConfig() */ - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); config.install_signal_handlers = 0; /* FIXME: test use_environment */ @@ -461,7 +462,7 @@ static int test_init_from_config(void) putenv("PYTHONUTF8=0"); Py_UTF8Mode = 0; - config.utf8_mode = 1; + config.ctx.utf8_mode = 1; putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix"); config.pycache_prefix = L"conf_pycache_prefix"; @@ -604,11 +605,12 @@ static int test_init_env(void) static int test_init_isolated(void) { /* Test _PyCoreConfig.isolated=1 */ - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); /* Set coerce_c_locale and utf8_mode to not depend on the locale */ config.coerce_c_locale = 0; - config.utf8_mode = 0; + config.ctx.utf8_mode = 0; /* Use path starting with "./" avoids a search along the PATH */ config.program_name = L"./_testembed"; @@ -628,7 +630,8 @@ static int test_init_isolated(void) static int test_init_dev_mode(void) { - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); putenv("PYTHONFAULTHANDLER="); putenv("PYTHONMALLOC="); config.dev_mode = 1; diff --git a/Python/coreconfig.c b/Python/coreconfig.c index a040a865ae1ddb1..27081309afd9f41 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -55,6 +55,21 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif +void _PyConfigCtx_Init(_PyConfigCtx *ctx) +{ + /* FIXME: don't magically initialize the runtime? */ + _PyInitError err; + err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } + + ctx->utf8_mode = -1; + + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &ctx->raw_alloc); +} + + PyObject * _Py_GetGlobalVariablesAsDict(void) { @@ -128,28 +143,28 @@ _Py_GetGlobalVariablesAsDict(void) void -_Py_wstrlist_clear(int len, wchar_t **list) +_Py_wstrlist_clear(const _PyConfigCtx *ctx, int len, wchar_t **list) { for (int i=0; i < len; i++) { - PyMem_RawFree(list[i]); + _PyMem_RawFreeCtx(ctx, list[i]); } - PyMem_RawFree(list); + _PyMem_RawFreeCtx(ctx, list); } wchar_t** -_Py_wstrlist_copy(int len, wchar_t **list) +_Py_wstrlist_copy(const _PyConfigCtx *ctx, int len, wchar_t **list) { assert((len > 0 && list != NULL) || len == 0); size_t size = len * sizeof(list[0]); - wchar_t **list_copy = PyMem_RawMalloc(size); + wchar_t **list_copy = _PyMem_RawMallocCtx(ctx, size); if (list_copy == NULL) { return NULL; } for (int i=0; i < len; i++) { - wchar_t* arg = _PyMem_RawWcsdup(list[i]); + wchar_t* arg = _PyMem_RawWcsdup(ctx, list[i]); if (arg == NULL) { - _Py_wstrlist_clear(i, list_copy); + _Py_wstrlist_clear(ctx, i, list_copy); return NULL; } list_copy[i] = arg; @@ -281,18 +296,26 @@ _Py_ClearStandardStreamEncoding(void) } +void +_PyCoreConfig_Init(_PyCoreConfig *config) +{ + *config = _PyCoreConfig_STATIC_INIT; + _PyConfigCtx_Init(&config->ctx); +} + + /* Free memory allocated in config, but don't clear all attributes */ void _PyCoreConfig_Clear(_PyCoreConfig *config) { #define CLEAR(ATTR) \ do { \ - PyMem_RawFree(ATTR); \ + _PyMem_RawFreeCtx(&config->ctx, ATTR); \ ATTR = NULL; \ } while (0) #define CLEAR_WSTRLIST(LEN, LIST) \ do { \ - _Py_wstrlist_clear(LEN, LIST); \ + _Py_wstrlist_clear(&config->ctx, LEN, LIST); \ LEN = 0; \ LIST = NULL; \ } while (0) @@ -334,6 +357,8 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) { _PyCoreConfig_Clear(config); + memcpy(&config->ctx, &config2->ctx, sizeof(config->ctx)); + #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR #define COPY_STR_ATTR(ATTR) \ do { \ @@ -347,16 +372,17 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) #define COPY_WSTR_ATTR(ATTR) \ do { \ if (config2->ATTR != NULL) { \ - config->ATTR = _PyMem_RawWcsdup(config2->ATTR); \ + config->ATTR = _PyMem_RawWcsdup(&config->ctx, config2->ATTR); \ if (config->ATTR == NULL) { \ return -1; \ } \ } \ } while (0) + /* Pass the context of the current config, not of the copied config */ #define COPY_WSTRLIST(LEN, LIST) \ do { \ if (config2->LIST != NULL) { \ - config->LIST = _Py_wstrlist_copy(config2->LEN, config2->LIST); \ + config->LIST = _Py_wstrlist_copy(&config->ctx, config2->LEN, config2->LIST); \ if (config->LIST == NULL) { \ return -1; \ } \ @@ -381,7 +407,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(coerce_c_locale); COPY_ATTR(coerce_c_locale_warn); - COPY_ATTR(utf8_mode); COPY_WSTR_ATTR(pycache_prefix); COPY_WSTR_ATTR(module_search_path_env); @@ -472,7 +497,7 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, return 0; } - wchar_t *copy = _PyMem_RawWcsdup(var); + wchar_t *copy = _PyMem_RawWcsdup(&config->ctx, var); if (copy == NULL) { return -1; } @@ -486,7 +511,7 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, } size_t len; - wchar_t *wvar = Py_DecodeLocale(var, &len); + wchar_t *wvar = _Py_DecodeLocaleCtx(&config->ctx, var, &len); if (!wvar) { if (len == (size_t)-2) { return -2; @@ -513,7 +538,7 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) config->ATTR = !(VALUE); \ } - COPY_FLAG(utf8_mode, Py_UTF8Mode); + COPY_FLAG(ctx.utf8_mode, Py_UTF8Mode); COPY_FLAG(isolated, Py_IsolatedFlag); COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); @@ -552,7 +577,7 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) VAR = !config->ATTR; \ } - COPY_FLAG(utf8_mode, Py_UTF8Mode); + COPY_FLAG(ctx.utf8_mode, Py_UTF8Mode); COPY_FLAG(isolated, Py_IsolatedFlag); COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); @@ -592,7 +617,7 @@ config_init_program_name(_PyCoreConfig *config) /* If Py_SetProgramName() was called, use its value */ const wchar_t *program_name = _Py_path_config.program_name; if (program_name != NULL) { - config->program_name = _PyMem_RawWcsdup(program_name); + config->program_name = _PyMem_RawWcsdup(&config->ctx, program_name); if (config->program_name == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -612,7 +637,7 @@ config_init_program_name(_PyCoreConfig *config) const char *p = _PyCoreConfig_GetEnv(config, "PYTHONEXECUTABLE"); if (p != NULL) { size_t len; - wchar_t* program_name = Py_DecodeLocale(p, &len); + wchar_t* program_name = _Py_DecodeLocaleCtx(&config->ctx, p, &len); if (program_name == NULL) { return DECODE_LOCALE_ERR("PYTHONEXECUTABLE environment " "variable", (Py_ssize_t)len); @@ -628,7 +653,7 @@ config_init_program_name(_PyCoreConfig *config) * the argv0 of the stub executable */ size_t len; - wchar_t* program_name = Py_DecodeLocale(pyvenv_launcher, &len); + wchar_t* program_name = _Py_DecodeLocaleCtx(&config->ctx, pyvenv_launcher, &len); if (program_name == NULL) { return DECODE_LOCALE_ERR("__PYVENV_LAUNCHER__ environment " "variable", (Py_ssize_t)len); @@ -642,7 +667,7 @@ config_init_program_name(_PyCoreConfig *config) /* Use argv[0] by default, if available */ if (config->program != NULL) { - config->program_name = _PyMem_RawWcsdup(config->program); + config->program_name = _PyMem_RawWcsdup(&config->ctx, config->program); if (config->program_name == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -655,7 +680,7 @@ config_init_program_name(_PyCoreConfig *config) #else const wchar_t *default_program_name = L"python3"; #endif - config->program_name = _PyMem_RawWcsdup(default_program_name); + config->program_name = _PyMem_RawWcsdup(&config->ctx, default_program_name); if (config->program_name == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -694,7 +719,7 @@ config_init_home(_PyCoreConfig *config) /* If Py_SetPythonHome() was called, use its value */ home = _Py_path_config.home; if (home) { - config->home = _PyMem_RawWcsdup(home); + config->home = _PyMem_RawWcsdup(&config->ctx, home); if (config->home == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -752,17 +777,17 @@ config_init_utf8_mode(_PyCoreConfig *config) if (sep) { xopt = sep + 1; if (wcscmp(xopt, L"1") == 0) { - config->utf8_mode = 1; + config->ctx.utf8_mode = 1; } else if (wcscmp(xopt, L"0") == 0) { - config->utf8_mode = 0; + config->ctx.utf8_mode = 0; } else { return _Py_INIT_USER_ERR("invalid -X utf8 option value"); } } else { - config->utf8_mode = 1; + config->ctx.utf8_mode = 1; } return _Py_INIT_OK(); } @@ -770,10 +795,10 @@ config_init_utf8_mode(_PyCoreConfig *config) const char *opt = _PyCoreConfig_GetEnv(config, "PYTHONUTF8"); if (opt) { if (strcmp(opt, "1") == 0) { - config->utf8_mode = 1; + config->ctx.utf8_mode = 1; } else if (strcmp(opt, "0") == 0) { - config->utf8_mode = 0; + config->ctx.utf8_mode = 0; } else { return _Py_INIT_USER_ERR("invalid PYTHONUTF8 environment " @@ -976,7 +1001,7 @@ config_init_pycache_prefix(_PyCoreConfig *config) if (xoption) { const wchar_t *sep = wcschr(xoption, L'='); if (sep && wcslen(sep) > 1) { - config->pycache_prefix = _PyMem_RawWcsdup(sep + 1); + config->pycache_prefix = _PyMem_RawWcsdup(&config->ctx, sep + 1); if (config->pycache_prefix == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1052,14 +1077,14 @@ config_init_locale(_PyCoreConfig *config) } #ifndef MS_WINDOWS - if (config->utf8_mode < 0) { + if (config->ctx.utf8_mode < 0) { /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ const char *ctype_loc = setlocale(LC_CTYPE, NULL); if (ctype_loc != NULL && (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0)) { - config->utf8_mode = 1; + config->ctx.utf8_mode = 1; } } #endif @@ -1187,7 +1212,7 @@ config_init_stdio_encoding(_PyCoreConfig *config) } /* UTF-8 Mode uses UTF-8/surrogateescape */ - if (config->utf8_mode) { + if (config->ctx.utf8_mode) { if (config->stdio_encoding == NULL) { config->stdio_encoding = _PyMem_RawStrdup("utf-8"); if (config->stdio_encoding == NULL) { @@ -1260,7 +1285,7 @@ config_init_fs_encoding(_PyCoreConfig *config) } #else if (config->filesystem_encoding == NULL) { - if (config->utf8_mode) { + if (config->ctx.utf8_mode) { /* UTF-8 Mode use: utf-8/surrogateescape */ config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); /* errors defaults to surrogateescape above */ @@ -1325,7 +1350,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config) #ifdef MS_WINDOWS if (config->legacy_windows_fs_encoding) { - config->utf8_mode = 0; + config->ctx.utf8_mode = 0; } #endif @@ -1349,7 +1374,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config) return err; } - if (config->utf8_mode < 0) { + if (config->ctx.utf8_mode < 0) { err = config_init_utf8_mode(config); if (_Py_INIT_FAILED(err)) { return err; @@ -1370,7 +1395,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config) } } - if (config->utf8_mode < 0 || config->coerce_c_locale < 0) { + if (config->ctx.utf8_mode < 0 || config->coerce_c_locale < 0) { config_init_locale(config); } @@ -1403,8 +1428,8 @@ _PyCoreConfig_Read(_PyCoreConfig *config) if (config->coerce_c_locale < 0) { config->coerce_c_locale = 0; } - if (config->utf8_mode < 0) { - config->utf8_mode = 0; + if (config->ctx.utf8_mode < 0) { + config->ctx.utf8_mode = 0; } if (config->argc < 0) { config->argc = 0; @@ -1492,7 +1517,7 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_INT(coerce_c_locale_warn); SET_ITEM_STR(filesystem_encoding); SET_ITEM_STR(filesystem_errors); - SET_ITEM_INT(utf8_mode); + SET_ITEM("utf8_mode", PyLong_FromLong(config->ctx.utf8_mode)); SET_ITEM_WSTR(pycache_prefix); SET_ITEM_WSTR(program_name); SET_ITEM_WSTRLIST(argc, argv); diff --git a/Python/fileutils.c b/Python/fileutils.c index c9a8e58dd1221d6..594bbf027f13dad 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -232,7 +232,7 @@ _Py_GetForceASCII(void) static int -encode_ascii(const wchar_t *text, char **str, +encode_ascii(const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, const char **reason, int raw_malloc, _Py_error_handler errors) { @@ -249,7 +249,7 @@ encode_ascii(const wchar_t *text, char **str, /* +1 for NULL byte */ if (raw_malloc) { - result = PyMem_RawMalloc(len + 1); + result = _PyMem_RawMallocCtx(ctx, len + 1); } else { result = PyMem_Malloc(len + 1); @@ -272,7 +272,7 @@ encode_ascii(const wchar_t *text, char **str, } else { if (raw_malloc) { - PyMem_RawFree(result); + _PyMem_RawFreeCtx(ctx, result); } else { PyMem_Free(result); @@ -301,7 +301,7 @@ _Py_GetForceASCII(void) #if !defined(HAVE_MBRTOWC) || defined(USE_FORCE_ASCII) static int -decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, +decode_ascii(const _PyConfigCtx *ctx, const char *arg, wchar_t **wstr, size_t *wlen, const char **reason, _Py_error_handler errors) { wchar_t *res; @@ -317,7 +317,7 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) { return -1; } - res = PyMem_RawMalloc(argsize * sizeof(wchar_t)); + res = _PyMem_RawMallocCtx(ctx, argsize * sizeof(wchar_t)); if (!res) { return -1; } @@ -353,8 +353,9 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, #endif /* !HAVE_MBRTOWC */ static int -decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, - const char **reason, _Py_error_handler errors) +decode_current_locale(const _PyConfigCtx *ctx, const char* arg, wchar_t **wstr, + size_t *wlen, const char **reason, + _Py_error_handler errors) { wchar_t *res; size_t argsize; @@ -383,7 +384,7 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { return -1; } - res = (wchar_t *)PyMem_RawMalloc((argsize + 1) * sizeof(wchar_t)); + res = (wchar_t *)_PyMem_RawMallocCtx(ctx, (argsize + 1) * sizeof(wchar_t)); if (!res) { return -1; } @@ -417,7 +418,7 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) { return -1; } - res = (wchar_t*)PyMem_RawMalloc(argsize * sizeof(wchar_t)); + res = (wchar_t*)_PyMem_RawMallocCtx(ctx, argsize * sizeof(wchar_t)); if (!res) { return -1; } @@ -490,7 +491,7 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, /* Cannot use C locale for escaping; manually escape as if charset is ASCII (i.e. escape all bytes > 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ - return decode_ascii(arg, wstr, wlen, reason, errors); + return decode_ascii(ctx, arg, wstr, wlen, reason, errors); #endif /* HAVE_MBRTOWC */ } @@ -518,29 +519,29 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, Use the Py_EncodeLocaleEx() function to encode the character string back to a byte string. */ int -_Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen, +_Py_DecodeLocaleEx(const _PyConfigCtx *ctx, const char* arg, wchar_t **wstr, size_t *wlen, const char **reason, int current_locale, _Py_error_handler errors) { if (current_locale) { #ifdef __ANDROID__ - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, + return _Py_DecodeUTF8Ex(ctx, arg, strlen(arg), wstr, wlen, reason, errors); #else - return decode_current_locale(arg, wstr, wlen, reason, errors); + return decode_current_locale(ctx, arg, wstr, wlen, reason, errors); #endif } #if defined(__APPLE__) || defined(__ANDROID__) - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, + return _Py_DecodeUTF8Ex(ctx, arg, strlen(arg), wstr, wlen, reason, errors); #else - int use_utf8 = (Py_UTF8Mode == 1); + int use_utf8 = (ctx->utf8_mode == 1); #ifdef MS_WINDOWS use_utf8 |= !Py_LegacyWindowsFSEncodingFlag; #endif if (use_utf8) { - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, + return _Py_DecodeUTF8Ex(ctx, arg, strlen(arg), wstr, wlen, reason, errors); } @@ -551,11 +552,11 @@ _Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen, if (force_ascii) { /* force ASCII encoding to workaround mbstowcs() issue */ - return decode_ascii(arg, wstr, wlen, reason, errors); + return decode_ascii(ctx, arg, wstr, wlen, reason, errors); } #endif - return decode_current_locale(arg, wstr, wlen, reason, errors); + return decode_current_locale(ctx, arg, wstr, wlen, reason, errors); #endif /* __APPLE__ or __ANDROID__ */ } @@ -580,10 +581,10 @@ _Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen, Use the Py_EncodeLocale() function to encode the character string back to a byte string. */ wchar_t* -Py_DecodeLocale(const char* arg, size_t *wlen) +_Py_DecodeLocaleCtx(const _PyConfigCtx *ctx, const char* arg, size_t *wlen) { wchar_t *wstr; - int res = _Py_DecodeLocaleEx(arg, &wstr, wlen, + int res = _Py_DecodeLocaleEx(ctx, arg, &wstr, wlen, NULL, 0, _Py_ERROR_SURROGATEESCAPE); if (res != 0) { @@ -596,9 +597,17 @@ Py_DecodeLocale(const char* arg, size_t *wlen) return wstr; } +wchar_t* +Py_DecodeLocale(const char* arg, size_t *wlen) +{ + _PyConfigCtx ctx; + _PyConfigCtx_Init(&ctx); + return _Py_DecodeLocaleCtx(&ctx, arg, wlen); +} + static int -encode_current_locale(const wchar_t *text, char **str, +encode_current_locale(const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, const char **reason, int raw_malloc, _Py_error_handler errors) { @@ -661,7 +670,7 @@ encode_current_locale(const wchar_t *text, char **str, size += 1; /* nul byte at the end */ if (raw_malloc) { - result = PyMem_RawMalloc(size); + result = _PyMem_RawMallocCtx(ctx, size); } else { result = PyMem_Malloc(size); @@ -676,7 +685,7 @@ encode_current_locale(const wchar_t *text, char **str, encode_error: if (raw_malloc) { - PyMem_RawFree(result); + _PyMem_RawFreeCtx(ctx, result); } else { PyMem_Free(result); @@ -709,30 +718,31 @@ encode_current_locale(const wchar_t *text, char **str, -3: the error handler 'errors' is not supported. */ static int -encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos, +encode_locale_ex(const _PyConfigCtx *ctx, const wchar_t *text, + char **str, size_t *error_pos, const char **reason, int raw_malloc, int current_locale, _Py_error_handler errors) { if (current_locale) { #ifdef __ANDROID__ - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, + return _Py_EncodeUTF8Ex(ctx, text, str, error_pos, reason, raw_malloc, errors); #else - return encode_current_locale(text, str, error_pos, reason, + return encode_current_locale(ctx, text, str, error_pos, reason, raw_malloc, errors); #endif } #if defined(__APPLE__) || defined(__ANDROID__) - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, + return _Py_EncodeUTF8Ex(ctx, text, str, error_pos, reason, raw_malloc, errors); #else - int use_utf8 = (Py_UTF8Mode == 1); + int use_utf8 = (ctx->utf8_mode == 1); #ifdef MS_WINDOWS use_utf8 |= !Py_LegacyWindowsFSEncodingFlag; #endif if (use_utf8) { - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, + return _Py_EncodeUTF8Ex(ctx, text, str, error_pos, reason, raw_malloc, errors); } @@ -742,12 +752,12 @@ encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos, } if (force_ascii) { - return encode_ascii(text, str, error_pos, reason, + return encode_ascii(ctx, text, str, error_pos, reason, raw_malloc, errors); } #endif - return encode_current_locale(text, str, error_pos, reason, + return encode_current_locale(ctx, text, str, error_pos, reason, raw_malloc, errors); #endif /* __APPLE__ or __ANDROID__ */ } @@ -757,7 +767,10 @@ encode_locale(const wchar_t *text, size_t *error_pos, int raw_malloc, int current_locale) { char *str; - int res = encode_locale_ex(text, &str, error_pos, NULL, + _PyConfigCtx ctx; + + _PyConfigCtx_Init(&ctx); + int res = encode_locale_ex(&ctx, text, &str, error_pos, NULL, raw_malloc, current_locale, _Py_ERROR_SURROGATEESCAPE); if (res != -2 && error_pos) { @@ -798,11 +811,11 @@ _Py_EncodeLocaleRaw(const wchar_t *text, size_t *error_pos) int -_Py_EncodeLocaleEx(const wchar_t *text, char **str, +_Py_EncodeLocaleEx(const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, const char **reason, int current_locale, _Py_error_handler errors) { - return encode_locale_ex(text, str, error_pos, reason, 1, + return encode_locale_ex(ctx, text, str, error_pos, reason, 1, current_locale, errors); } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 616090965b1351b..e9ed076bff634b3 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -41,7 +41,8 @@ Py_FrozenMain(int argc, char **argv) } } - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); config._frozen = 1; /* Suppress errors from getpath.c */ if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 6a8688059825bf9..5b3ad1089008d75 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -13,14 +13,32 @@ extern "C" { #endif -_PyPathConfig _Py_path_config = _PyPathConfig_INIT; +_PyPathConfig _Py_path_config = _PyPathConfig_STATIC_INIT; +void +_PyPathConfig_StaticInit(_PyPathConfig *config) +{ + *config = _PyPathConfig_STATIC_INIT; + _PyConfigCtx_Init(&config->ctx); +} + + +static void +pathconfig_global_static_init(void) +{ + if (_Py_path_config.ctx.raw_alloc.malloc != NULL) { + /* Already initialized */ + return; + } + _PyPathConfig_StaticInit(&_Py_path_config); +} + static int -copy_wstr(wchar_t **dst, const wchar_t *src) +copy_wstr(const _PyConfigCtx *ctx, wchar_t **dst, const wchar_t *src) { if (src != NULL) { - *dst = _PyMem_RawWcsdup(src); + *dst = _PyMem_RawWcsdup(ctx, src); if (*dst == NULL) { return -1; } @@ -35,15 +53,9 @@ copy_wstr(wchar_t **dst, const wchar_t *src) static void _PyPathConfig_Clear(_PyPathConfig *config) { - /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, - since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be - called before Py_Initialize() which can changes the memory allocator. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - #define CLEAR(ATTR) \ do { \ - PyMem_RawFree(ATTR); \ + _PyMem_RawFreeCtx(&config->ctx, ATTR); \ ATTR = NULL; \ } while (0) @@ -58,8 +70,6 @@ _PyPathConfig_Clear(_PyPathConfig *config) CLEAR(config->home); CLEAR(config->program_name); #undef CLEAR - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -69,7 +79,8 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, const _PyCoreConfig *core_config) { _PyInitError err; - _PyPathConfig new_config = _PyPathConfig_INIT; + _PyPathConfig new_config; + _PyPathConfig_StaticInit(&new_config); PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -82,11 +93,11 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, } /* Copy home and program_name from core_config */ - if (copy_wstr(&new_config.home, core_config->home) < 0) { + if (copy_wstr(&new_config.ctx, &new_config.home, core_config->home) < 0) { err = _Py_INIT_NO_MEMORY(); goto err; } - if (copy_wstr(&new_config.program_name, core_config->program_name) < 0) { + if (copy_wstr(&new_config.ctx, &new_config.program_name, core_config->program_name) < 0) { err = _Py_INIT_NO_MEMORY(); goto err; } @@ -109,15 +120,18 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, _PyInitError _PyPathConfig_SetGlobal(const _PyPathConfig *config) { + pathconfig_global_static_init(); + _PyInitError err; - _PyPathConfig new_config = _PyPathConfig_INIT; + _PyPathConfig new_config; + _PyPathConfig_StaticInit(&new_config); PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); #define COPY_ATTR(ATTR) \ do { \ - if (copy_wstr(&new_config.ATTR, config->ATTR) < 0) { \ + if (copy_wstr(&new_config.ctx, &new_config.ATTR, config->ATTR) < 0) { \ _PyPathConfig_Clear(&new_config); \ err = _Py_INIT_NO_MEMORY(); \ goto done; \ @@ -150,6 +164,7 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) void _PyPathConfig_ClearGlobal(void) { + pathconfig_global_static_init(); _PyPathConfig_Clear(&_Py_path_config); } @@ -193,7 +208,8 @@ _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); _PyInitError err; - _PyPathConfig path_config = _PyPathConfig_INIT; + _PyPathConfig path_config; + _PyPathConfig_StaticInit(&path_config); path_config.module_search_path = wstrlist_join(DELIM, core_config->nmodule_search_path, @@ -202,25 +218,25 @@ _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) goto no_memory; } - if (copy_wstr(&path_config.program_full_path, core_config->executable) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.program_full_path, core_config->executable) < 0) { goto no_memory; } - if (copy_wstr(&path_config.prefix, core_config->prefix) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.prefix, core_config->prefix) < 0) { goto no_memory; } #ifdef MS_WINDOWS - if (copy_wstr(&path_config.dll_path, core_config->dll_path) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.dll_path, core_config->dll_path) < 0) { goto no_memory; } #else - if (copy_wstr(&path_config.exec_prefix, core_config->exec_prefix) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.exec_prefix, core_config->exec_prefix) < 0) { goto no_memory; } #endif - if (copy_wstr(&path_config.program_name, core_config->program_name) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.program_name, core_config->program_name) < 0) { goto no_memory; } - if (copy_wstr(&path_config.home, core_config->home) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.home, core_config->home) < 0) { goto no_memory; } @@ -268,7 +284,8 @@ core_config_init_module_search_paths(_PyCoreConfig *config, memcpy(path, sys_path, path_len * sizeof(wchar_t)); path[path_len] = L'\0'; - _PyInitError err = _Py_wstrlist_append(&config->nmodule_search_path, + _PyInitError err = _Py_wstrlist_append(&config->ctx, + &config->nmodule_search_path, &config->module_search_paths, path); PyMem_RawFree(path); @@ -288,9 +305,10 @@ core_config_init_module_search_paths(_PyCoreConfig *config, static _PyInitError _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) { - _PyPathConfig path_config = _PyPathConfig_INIT; + _PyPathConfig path_config; _PyInitError err; + _PyPathConfig_StaticInit(&path_config); err = _PyPathConfig_Calculate(&path_config, config); if (_Py_INIT_FAILED(err)) { goto error; @@ -304,14 +322,15 @@ _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) } if (config->executable == NULL) { - if (copy_wstr(&config->executable, + if (copy_wstr(&config->ctx, + &config->executable, path_config.program_full_path) < 0) { goto no_memory; } } if (config->prefix == NULL) { - if (copy_wstr(&config->prefix, path_config.prefix) < 0) { + if (copy_wstr(&config->ctx, &config->prefix, path_config.prefix) < 0) { goto no_memory; } } @@ -322,14 +341,14 @@ _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) #else wchar_t *exec_prefix = path_config.exec_prefix; #endif - if (copy_wstr(&config->exec_prefix, exec_prefix) < 0) { + if (copy_wstr(&config->ctx, &config->exec_prefix, exec_prefix) < 0) { goto no_memory; } } #ifdef MS_WINDOWS if (config->dll_path == NULL) { - if (copy_wstr(&config->dll_path, path_config.dll_path) < 0) { + if (copy_wstr(&config->ctx, &config->dll_path, path_config.dll_path) < 0) { goto no_memory; } } @@ -373,13 +392,13 @@ _PyCoreConfig_InitPathConfig(_PyCoreConfig *config) } if (config->base_prefix == NULL) { - if (copy_wstr(&config->base_prefix, config->prefix) < 0) { + if (copy_wstr(&config->ctx, &config->base_prefix, config->prefix) < 0) { return _Py_INIT_NO_MEMORY(); } } if (config->base_exec_prefix == NULL) { - if (copy_wstr(&config->base_exec_prefix, config->exec_prefix) < 0) { + if (copy_wstr(&config->ctx, &config->base_exec_prefix, config->exec_prefix) < 0) { return _Py_INIT_NO_MEMORY(); } } @@ -390,13 +409,16 @@ _PyCoreConfig_InitPathConfig(_PyCoreConfig *config) static void pathconfig_global_init(void) { + pathconfig_global_static_init(); + if (_Py_path_config.module_search_path != NULL) { /* Already initialized */ return; } _PyInitError err; - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); err = _PyCoreConfig_Read(&config); if (_Py_INIT_FAILED(err)) { @@ -422,6 +444,8 @@ pathconfig_global_init(void) void Py_SetPath(const wchar_t *path) { + pathconfig_global_static_init(); + if (path == NULL) { _PyPathConfig_Clear(&_Py_path_config); return; @@ -431,18 +455,19 @@ Py_SetPath(const wchar_t *path) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); _PyPathConfig new_config; - new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName()); + _PyPathConfig_StaticInit(&new_config); + new_config.program_full_path = _PyMem_RawWcsdup(&new_config.ctx, Py_GetProgramName()); int alloc_error = (new_config.program_full_path == NULL); - new_config.prefix = _PyMem_RawWcsdup(L""); + new_config.prefix = _PyMem_RawWcsdup(&new_config.ctx, L""); alloc_error |= (new_config.prefix == NULL); #ifdef MS_WINDOWS - new_config.dll_path = _PyMem_RawWcsdup(L""); + new_config.dll_path = _PyMem_RawWcsdup(&new_config.ctx, L""); alloc_error |= (new_config.dll_path == NULL); #else - new_config.exec_prefix = _PyMem_RawWcsdup(L""); + new_config.exec_prefix = _PyMem_RawWcsdup(&new_config.ctx, L""); alloc_error |= (new_config.exec_prefix == NULL); #endif - new_config.module_search_path = _PyMem_RawWcsdup(path); + new_config.module_search_path = _PyMem_RawWcsdup(&new_config.ctx, path); alloc_error |= (new_config.module_search_path == NULL); /* steal the home and program_name values (to leave them unchanged) */ @@ -465,6 +490,8 @@ Py_SetPath(const wchar_t *path) void Py_SetPythonHome(const wchar_t *home) { + pathconfig_global_static_init(); + if (home == NULL) { return; } @@ -472,8 +499,8 @@ Py_SetPythonHome(const wchar_t *home) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - PyMem_RawFree(_Py_path_config.home); - _Py_path_config.home = _PyMem_RawWcsdup(home); + _PyMem_RawFreeCtx(&_Py_path_config.ctx, _Py_path_config.home); + _Py_path_config.home = _PyMem_RawWcsdup(&_Py_path_config.ctx, home); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -486,6 +513,8 @@ Py_SetPythonHome(const wchar_t *home) void Py_SetProgramName(const wchar_t *program_name) { + pathconfig_global_static_init(); + if (program_name == NULL || program_name[0] == L'\0') { return; } @@ -493,8 +522,8 @@ Py_SetProgramName(const wchar_t *program_name) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - PyMem_RawFree(_Py_path_config.program_name); - _Py_path_config.program_name = _PyMem_RawWcsdup(program_name); + _PyMem_RawFreeCtx(&_Py_path_config.ctx, _Py_path_config.program_name); + _Py_path_config.program_name = _PyMem_RawWcsdup(&_Py_path_config.ctx, program_name); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 58e16473100e3a6..589d3f1f3743739 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -579,6 +579,8 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p, return _Py_INIT_ERR("failed to copy core config"); } core_config = &interp->core_config; + /* FIXME: update ctx just after _PyMem_SetupAllocators() */ + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &interp->core_config.ctx.raw_alloc); PyThreadState *tstate = PyThreadState_New(interp); if (tstate == NULL) @@ -706,7 +708,8 @@ _Py_InitializeCore(PyInterpreterState **interp_p, /* Copy the configuration, since _PyCoreConfig_Read() modifies it (and the input configuration is read only). */ - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); /* Set LC_CTYPE to the user preferred locale */ _Py_SetLocaleFromEnv(LC_CTYPE); @@ -898,7 +901,9 @@ Py_InitializeEx(int install_sigs) } _PyInitError err; - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + + _PyCoreConfig_Init(&config); config.install_signal_handlers = install_sigs; err = _Py_InitializeFromConfig(&config); diff --git a/Python/pystate.c b/Python/pystate.c index f86f5a96f07450d..d44a4ebab34838c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -148,7 +148,7 @@ PyInterpreterState_New(void) interp->codec_error_registry = NULL; interp->codecs_initialized = 0; interp->fscodec_initialized = 0; - interp->core_config = _PyCoreConfig_INIT; + _PyCoreConfig_Init(&interp->core_config); interp->config = _PyMainInterpreterConfig_INIT; interp->importlib = NULL; interp->import_func = NULL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 2284e88d4c11800..25d915a05bb2619 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1661,47 +1661,58 @@ struct _preinit_entry { typedef struct _preinit_entry *_Py_PreInitEntry; -static _Py_PreInitEntry _preinit_warnoptions = NULL; -static _Py_PreInitEntry _preinit_xoptions = NULL; +typedef struct { + _PyConfigCtx ctx; + _Py_PreInitEntry list; +} _Py_PreInit; + +#define _Py_PreInit_STATIC_INIT (_Py_PreInit){.list = NULL} + +static _Py_PreInit _preinit_warnoptions = _Py_PreInit_STATIC_INIT; +static _Py_PreInit _preinit_xoptions = _Py_PreInit_STATIC_INIT; + +static void +preinit_static_init(_Py_PreInit *preinit) +{ + if (preinit->ctx.raw_alloc.malloc != NULL) { + /* Already initialized */ + return; + } + _PyConfigCtx_Init(&preinit->ctx); +} + static _Py_PreInitEntry -_alloc_preinit_entry(const wchar_t *value) +preinit_alloc(_Py_PreInit *preinit, const wchar_t *value) { /* To get this to work, we have to initialize the runtime implicitly */ _PyRuntime_Initialize(); - /* Force default allocator, so we can ensure that it also gets used to - * destroy the linked list in _clear_preinit_entries. - */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _Py_PreInitEntry node = PyMem_RawCalloc(1, sizeof(*node)); + _Py_PreInitEntry node = _PyMem_RawCallocCtx(&preinit->ctx, 1, sizeof(*node)); if (node != NULL) { - node->value = _PyMem_RawWcsdup(value); + node->value = _PyMem_RawWcsdup(&preinit->ctx, value); if (node->value == NULL) { - PyMem_RawFree(node); + _PyMem_RawFreeCtx(&preinit->ctx, node); node = NULL; }; }; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return node; }; static int -_append_preinit_entry(_Py_PreInitEntry *optionlist, const wchar_t *value) +preinit_append(_Py_PreInit *preinit, const wchar_t *value) { - _Py_PreInitEntry new_entry = _alloc_preinit_entry(value); + _Py_PreInitEntry new_entry = preinit_alloc(preinit, value); if (new_entry == NULL) { return -1; } /* We maintain the linked list in this order so it's easy to play back * the add commands in the same order later on in _Py_InitializeCore */ - _Py_PreInitEntry last_entry = *optionlist; + _Py_PreInitEntry last_entry = preinit->list; if (last_entry == NULL) { - *optionlist = new_entry; + preinit->list = new_entry; } else { while (last_entry->next != NULL) { last_entry = last_entry->next; @@ -1712,50 +1723,44 @@ _append_preinit_entry(_Py_PreInitEntry *optionlist, const wchar_t *value) }; static void -_clear_preinit_entries(_Py_PreInitEntry *optionlist) +preinit_clear(_Py_PreInit *preinit) { - _Py_PreInitEntry current = *optionlist; - *optionlist = NULL; - /* Deallocate the nodes and their contents using the default allocator */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _Py_PreInitEntry current = preinit->list; + preinit->list = NULL; + while (current != NULL) { _Py_PreInitEntry next = current->next; - PyMem_RawFree(current->value); - PyMem_RawFree(current); + _PyMem_RawFreeCtx(&preinit->ctx, current->value); + _PyMem_RawFreeCtx(&preinit->ctx, current); current = next; } - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); }; -static void -_clear_all_preinit_options(void) -{ - _clear_preinit_entries(&_preinit_warnoptions); - _clear_preinit_entries(&_preinit_xoptions); -} - static int -_PySys_ReadPreInitOptions(void) +sys_read_preinit_options(void) { + preinit_static_init(&_preinit_warnoptions); + preinit_static_init(&_preinit_xoptions); + /* Rerun the add commands with the actual sys module available */ PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { /* Still don't have a thread state, so something is wrong! */ return -1; } - _Py_PreInitEntry entry = _preinit_warnoptions; + _Py_PreInitEntry entry = _preinit_warnoptions.list; while (entry != NULL) { PySys_AddWarnOption(entry->value); entry = entry->next; } - entry = _preinit_xoptions; + entry = _preinit_xoptions.list; while (entry != NULL) { PySys_AddXOption(entry->value); entry = entry->next; } - _clear_all_preinit_options(); + preinit_clear(&_preinit_warnoptions); + preinit_clear(&_preinit_xoptions); return 0; }; @@ -1790,9 +1795,11 @@ get_warnoptions(void) void PySys_ResetWarnOptions(void) { + preinit_static_init(&_preinit_warnoptions); + PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { - _clear_preinit_entries(&_preinit_warnoptions); + preinit_clear(&_preinit_warnoptions); return; } @@ -1829,9 +1836,11 @@ PySys_AddWarnOptionUnicode(PyObject *option) void PySys_AddWarnOption(const wchar_t *s) { + preinit_static_init(&_preinit_warnoptions); + PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { - _append_preinit_entry(&_preinit_warnoptions, s); + preinit_append(&_preinit_warnoptions, s); return; } PyObject *unicode; @@ -1916,9 +1925,11 @@ _PySys_AddXOptionWithError(const wchar_t *s) void PySys_AddXOption(const wchar_t *s) { + preinit_static_init(&_preinit_xoptions); + PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { - _append_preinit_entry(&_preinit_xoptions, s); + preinit_append(&_preinit_xoptions, s); return; } if (_PySys_AddXOptionWithError(s) < 0) { @@ -2101,7 +2112,7 @@ make_flags(void) SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0); SetFlag(config->isolated); PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode)); - SetFlag(config->utf8_mode); + SetFlag(config->ctx.utf8_mode); #undef SetFlag if (PyErr_Occurred()) { @@ -2555,8 +2566,9 @@ _PySys_EndInit(PyObject *sysdict, PyInterpreterState *interp) /* Transfer any sys.warnoptions and sys._xoptions set directly * by an embedding application from the linked list to the module. */ - if (_PySys_ReadPreInitOptions() != 0) + if (sys_read_preinit_options() < 0) { return -1; + } if (PyErr_Occurred()) return -1; From ae5a33c1514da3575cc8c17bd28369b36f4f00ef Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Nov 2018 15:31:54 +0100 Subject: [PATCH 02/12] Some bugfixes --- Objects/unicodeobject.c | 4 ++-- Python/pylifecycle.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4b20c78264395a0..78962bc77de13db 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3453,7 +3453,7 @@ unicode_encode_locale(PyObject *unicode, const char *errors, PyMem_Free(wstr); PyObject *bytes = PyBytes_FromString(str); - PyMem_RawFree(str); + _PyMem_RawFreeCtx(&interp->core_config.ctx, str); return bytes; } @@ -3665,7 +3665,7 @@ unicode_decode_locale(const char *str, Py_ssize_t len, const char *errors, } PyObject *unicode = PyUnicode_FromWideChar(wstr, wlen); - PyMem_RawFree(wstr); + _PyMem_RawFreeCtx(&interp->core_config.ctx, wstr); return unicode; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 589d3f1f3743739..35a06a8ba4291a1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -575,12 +575,13 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p, } *interp_p = interp; + /* Update the memory allocator of the config and copy the new config */ + _PyCoreConfig_Clear(&interp->core_config); + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &interp->core_config.ctx.raw_alloc); if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) { return _Py_INIT_ERR("failed to copy core config"); } core_config = &interp->core_config; - /* FIXME: update ctx just after _PyMem_SetupAllocators() */ - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &interp->core_config.ctx.raw_alloc); PyThreadState *tstate = PyThreadState_New(interp); if (tstate == NULL) From f77ba8c10c966a94a79f212d49be54886facd29d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Nov 2018 18:20:51 +0100 Subject: [PATCH 03/12] _PyCoreConfig_Copy() don't copy memory allocator --- Python/coreconfig.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 27081309afd9f41..067bacf07332bd0 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -69,6 +69,14 @@ void _PyConfigCtx_Init(_PyConfigCtx *ctx) PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &ctx->raw_alloc); } +static void +_PyConfigCtx_Copy(_PyConfigCtx *ctx, const _PyConfigCtx *ctx2) +{ + /* Don't copy raw_alloc, each context keeps is own memory allocators. + Memory allocators must be copied or modified manually. */ + ctx->utf8_mode = ctx2->utf8_mode; +} + PyObject * _Py_GetGlobalVariablesAsDict(void) @@ -357,7 +365,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) { _PyCoreConfig_Clear(config); - memcpy(&config->ctx, &config2->ctx, sizeof(config->ctx)); + _PyConfigCtx_Copy(&config->ctx, &config2->ctx); #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR #define COPY_STR_ATTR(ATTR) \ From 96698128baeaad3350aafe9ecdd212ebd1b9cb48 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Nov 2018 18:29:13 +0100 Subject: [PATCH 04/12] Try to fix compilation on Windows Fix "initializer is not a constant" error. --- Include/internal/pycore_pathconfig.h | 4 +- PC/getpathp.c | 4 +- Python/coreconfig.c | 7 +++- Python/pathconfig.c | 60 ++++++++++++++++------------ Python/sysmodule.c | 7 ++-- 5 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 497e5310a0bef83..cda7e325d4521f1 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -59,12 +59,12 @@ typedef struct _PyPathConfig { PyAPI_FUNC(void) _PyPathConfig_StaticInit( _PyPathConfig *config); -PyAPI_DATA(_PyPathConfig) _Py_path_config; - PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); PyAPI_FUNC(_PyInitError) _PyPathConfig_SetGlobal( const _PyPathConfig *config); +PyAPI_FUNC(_PyPathConfig*) _PyPathConfig_GetGlobal(); + PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate_impl( _PyPathConfig *config, const _PyCoreConfig *core_config); diff --git a/PC/getpathp.c b/PC/getpathp.c index 7f6111b88ad0cec..6d5234c18876bc2 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -1044,9 +1044,11 @@ _Py_CheckPython3(void) } python3_checked = 1; + /* If there is a python3.dll next to the python3y.dll, assume this is a build tree; use that DLL */ - wcscpy(py3path, _Py_path_config.dll_path); + _PyPathConfig* path_config = _PyPathConfig_GetGlobal(); + wcscpy(py3path, path_config->dll_path); s = wcsrchr(py3path, L'\\'); if (!s) { s = py3path; diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 067bacf07332bd0..6600274ebc0b305 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -622,8 +622,10 @@ config_init_program_name(_PyCoreConfig *config) { assert(config->program_name == NULL); + _PyPathConfig* path_config = _PyPathConfig_GetGlobal(); + /* If Py_SetProgramName() was called, use its value */ - const wchar_t *program_name = _Py_path_config.program_name; + const wchar_t *program_name = path_config->program_name; if (program_name != NULL) { config->program_name = _PyMem_RawWcsdup(&config->ctx, program_name); if (config->program_name == NULL) { @@ -723,9 +725,10 @@ static _PyInitError config_init_home(_PyCoreConfig *config) { wchar_t *home; + _PyPathConfig* path_config = _PyPathConfig_GetGlobal(); /* If Py_SetPythonHome() was called, use its value */ - home = _Py_path_config.home; + home = path_config->home; if (home) { config->home = _PyMem_RawWcsdup(&config->ctx, home); if (config->home == NULL) { diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 5b3ad1089008d75..4d9f5616e486826 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -13,7 +13,8 @@ extern "C" { #endif -_PyPathConfig _Py_path_config = _PyPathConfig_STATIC_INIT; +/* static initializes all fields to 0/NULL */ +static _PyPathConfig global_path_config; void @@ -27,11 +28,11 @@ _PyPathConfig_StaticInit(_PyPathConfig *config) static void pathconfig_global_static_init(void) { - if (_Py_path_config.ctx.raw_alloc.malloc != NULL) { + if (global_path_config.ctx.raw_alloc.malloc != NULL) { /* Already initialized */ return; } - _PyPathConfig_StaticInit(&_Py_path_config); + _PyPathConfig_StaticInit(&global_path_config); } static int @@ -149,9 +150,9 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) COPY_ATTR(program_name); COPY_ATTR(home); - _PyPathConfig_Clear(&_Py_path_config); + _PyPathConfig_Clear(&global_path_config); /* Steal new_config strings; don't clear new_config */ - _Py_path_config = new_config; + global_path_config = new_config; err = _Py_INIT_OK(); @@ -160,12 +161,19 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) return err; } +_PyPathConfig* +_PyPathConfig_GetGlobal(void) +{ + pathconfig_global_static_init(); + return &global_path_config; +} + void _PyPathConfig_ClearGlobal(void) { pathconfig_global_static_init(); - _PyPathConfig_Clear(&_Py_path_config); + _PyPathConfig_Clear(&global_path_config); } @@ -411,7 +419,7 @@ pathconfig_global_init(void) { pathconfig_global_static_init(); - if (_Py_path_config.module_search_path != NULL) { + if (global_path_config.module_search_path != NULL) { /* Already initialized */ return; } @@ -447,7 +455,7 @@ Py_SetPath(const wchar_t *path) pathconfig_global_static_init(); if (path == NULL) { - _PyPathConfig_Clear(&_Py_path_config); + _PyPathConfig_Clear(&global_path_config); return; } @@ -471,13 +479,13 @@ Py_SetPath(const wchar_t *path) alloc_error |= (new_config.module_search_path == NULL); /* steal the home and program_name values (to leave them unchanged) */ - new_config.home = _Py_path_config.home; - _Py_path_config.home = NULL; - new_config.program_name = _Py_path_config.program_name; - _Py_path_config.program_name = NULL; + new_config.home = global_path_config.home; + global_path_config.home = NULL; + new_config.program_name = global_path_config.program_name; + global_path_config.program_name = NULL; - _PyPathConfig_Clear(&_Py_path_config); - _Py_path_config = new_config; + _PyPathConfig_Clear(&global_path_config); + global_path_config = new_config; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -499,12 +507,12 @@ Py_SetPythonHome(const wchar_t *home) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyMem_RawFreeCtx(&_Py_path_config.ctx, _Py_path_config.home); - _Py_path_config.home = _PyMem_RawWcsdup(&_Py_path_config.ctx, home); + _PyMem_RawFreeCtx(&global_path_config.ctx, global_path_config.home); + global_path_config.home = _PyMem_RawWcsdup(&global_path_config.ctx, home); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.home == NULL) { + if (global_path_config.home == NULL) { Py_FatalError("Py_SetPythonHome() failed: out of memory"); } } @@ -522,12 +530,12 @@ Py_SetProgramName(const wchar_t *program_name) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyMem_RawFreeCtx(&_Py_path_config.ctx, _Py_path_config.program_name); - _Py_path_config.program_name = _PyMem_RawWcsdup(&_Py_path_config.ctx, program_name); + _PyMem_RawFreeCtx(&global_path_config.ctx, global_path_config.program_name); + global_path_config.program_name = _PyMem_RawWcsdup(&global_path_config.ctx, program_name); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.program_name == NULL) { + if (global_path_config.program_name == NULL) { Py_FatalError("Py_SetProgramName() failed: out of memory"); } } @@ -537,7 +545,7 @@ wchar_t * Py_GetPath(void) { pathconfig_global_init(); - return _Py_path_config.module_search_path; + return global_path_config.module_search_path; } @@ -545,7 +553,7 @@ wchar_t * Py_GetPrefix(void) { pathconfig_global_init(); - return _Py_path_config.prefix; + return global_path_config.prefix; } @@ -556,7 +564,7 @@ Py_GetExecPrefix(void) return Py_GetPrefix(); #else pathconfig_global_init(); - return _Py_path_config.exec_prefix; + return global_path_config.exec_prefix; #endif } @@ -565,7 +573,7 @@ wchar_t * Py_GetProgramFullPath(void) { pathconfig_global_init(); - return _Py_path_config.program_full_path; + return global_path_config.program_full_path; } @@ -573,7 +581,7 @@ wchar_t* Py_GetPythonHome(void) { pathconfig_global_init(); - return _Py_path_config.home; + return global_path_config.home; } @@ -581,7 +589,7 @@ wchar_t * Py_GetProgramName(void) { pathconfig_global_init(); - return _Py_path_config.program_name; + return global_path_config.program_name; } /* Compute argv[0] which will be prepended to sys.argv */ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 25d915a05bb2619..c0526b097ed5fd4 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1666,10 +1666,9 @@ typedef struct { _Py_PreInitEntry list; } _Py_PreInit; -#define _Py_PreInit_STATIC_INIT (_Py_PreInit){.list = NULL} - -static _Py_PreInit _preinit_warnoptions = _Py_PreInit_STATIC_INIT; -static _Py_PreInit _preinit_xoptions = _Py_PreInit_STATIC_INIT; +/* "static" initializes all fields to 0/NULL */ +static _Py_PreInit _preinit_warnoptions; +static _Py_PreInit _preinit_xoptions; static void preinit_static_init(_Py_PreInit *preinit) From 6884371fbcea33d14b124660585bf9c1e7cbc603 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Nov 2018 18:55:14 +0100 Subject: [PATCH 05/12] Fix _PyPathConfig_GetGlobal() prototype --- Include/internal/pycore_pathconfig.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index cda7e325d4521f1..b690597ffbd165f 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -63,7 +63,7 @@ PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); PyAPI_FUNC(_PyInitError) _PyPathConfig_SetGlobal( const _PyPathConfig *config); -PyAPI_FUNC(_PyPathConfig*) _PyPathConfig_GetGlobal(); +PyAPI_FUNC(_PyPathConfig*) _PyPathConfig_GetGlobal(void); PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate_impl( _PyPathConfig *config, From 2c2006c32469eeeb5d288f764d01aca5b5f24dd2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Nov 2018 19:10:21 +0100 Subject: [PATCH 06/12] Exclude coreconfig.h from Py_LIMITED_API --- Include/coreconfig.h | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Include/coreconfig.h b/Include/coreconfig.h index 27ee53b658f8456..a43484a64d6d858 100644 --- a/Include/coreconfig.h +++ b/Include/coreconfig.h @@ -1,11 +1,12 @@ #ifndef Py_PYCORECONFIG_H #define Py_PYCORECONFIG_H +#ifndef Py_LIMITED_API + #ifdef __cplusplus extern "C" { #endif -#ifndef Py_LIMITED_API typedef struct { const char *prefix; const char *msg; @@ -32,8 +33,6 @@ typedef struct { #define _Py_INIT_FAILED(err) \ (err.msg != NULL) -#endif /* !defined(Py_LIMITED_API) */ - typedef struct { /* Enable UTF-8 mode? @@ -47,14 +46,12 @@ typedef struct { PyAPI_FUNC(void) _PyConfigCtx_Init(_PyConfigCtx *ctx); -#ifndef Py_LIMITED_API /* Functions implemented in obmalloc.c */ PyAPI_FUNC(void *) _PyMem_RawMallocCtx(const _PyConfigCtx *ctx, size_t size); PyAPI_FUNC(void *) _PyMem_RawCallocCtx(const _PyConfigCtx *ctx, size_t nelem, size_t elsize); PyAPI_FUNC(void *) _PyMem_RawReallocCtx(const _PyConfigCtx *ctx, void *ptr, size_t new_size); PyAPI_FUNC(void) _PyMem_RawFreeCtx(const _PyConfigCtx *ctx, void *ptr); PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const _PyConfigCtx *ctx, const wchar_t *str); -#endif /* !Py_LIMITED_API */ typedef struct { @@ -360,7 +357,6 @@ typedef struct { PyAPI_FUNC(void) _PyCoreConfig_Init(_PyCoreConfig *config); -#ifndef Py_LIMITED_API PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config); PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); PyAPI_FUNC(int) _PyCoreConfig_Copy( @@ -383,9 +379,9 @@ PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( /* Used by _testcapi.get_global_config() and _testcapi.get_core_config() */ PyAPI_FUNC(PyObject *) _Py_GetGlobalVariablesAsDict(void); PyAPI_FUNC(PyObject *) _PyCoreConfig_AsDict(const _PyCoreConfig *config); -#endif #ifdef __cplusplus } #endif +#endif /* !Py_LIMITED_API */ #endif /* !Py_PYCORECONFIG_H */ From 1f03f4d580780eca3c47c74570dfcddd4c50db7a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 17 Nov 2018 01:05:35 +0100 Subject: [PATCH 07/12] Bugfixes --- Modules/getpath.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index 3b20261cf0af153..05d28f5ec17cc5e 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -144,7 +144,6 @@ _Py_wstat(const wchar_t* path, struct stat *buf) { int err; char *fname; - /* FIXME: use ctx? */ fname = _Py_EncodeLocaleRaw(path, NULL); if (fname == NULL) { errno = EINVAL; @@ -365,12 +364,12 @@ search_for_prefix(const _PyCoreConfig *core_config, _PyPathConfig *config, joinpath(prefix, L"Modules/Setup.local"); if (isfile(prefix)) { /* Check VPATH to see if argv0_path is in the build directory. */ - vpath = _Py_DecodeLocaleCtx(&config->ctx, VPATH, NULL); + vpath = Py_DecodeLocale(VPATH, NULL); if (vpath != NULL) { wcsncpy(prefix, calculate->argv0_path, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; joinpath(prefix, vpath); - _PyMem_RawFreeCtx(&config->ctx, vpath); + PyMem_RawFree(vpath); joinpath(prefix, L"Lib"); joinpath(prefix, LANDMARK); if (ismodule(prefix)) { @@ -602,12 +601,12 @@ calculate_program_full_path(const _PyCoreConfig *core_config, execpath[0] == SEP) { size_t len; - wchar_t *path = _Py_DecodeLocaleCtx(&config->ctx, execpath, &len); + wchar_t *path = Py_DecodeLocaleCtx(execpath, &len); if (path == NULL) { return DECODE_LOCALE_ERR("executable path", len); } wcsncpy(program_full_path, path, MAXPATHLEN); - _PyMem_RawFreeCtx(&config->ctx, path); + PyMem_RawFree(path); } #endif /* __APPLE__ */ else if (calculate->path_env) { @@ -694,7 +693,7 @@ calculate_argv0_path(PyCalculatePath *calculate, ** build-directory-specific logic to find Lib and such. */ size_t len; - wchar_t* wbuf = _Py_DecodeLocaleCtx(&config->ctx, modPath, &len); + wchar_t* wbuf = Py_DecodeLocaleCtx(modPath, &len); if (wbuf == NULL) { return DECODE_LOCALE_ERR("framework location", len); } @@ -712,7 +711,7 @@ calculate_argv0_path(PyCalculatePath *calculate, /* Use the location of the library as the program_full_path */ wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); } - _PyMem_RawFreeCtx(&config->ctx, wbuf); + PyMem_RawFree(wbuf); } #endif @@ -900,28 +899,29 @@ calculate_init(PyCalculatePath *calculate, _PyPathConfig *config, const _PyCoreConfig *core_config) { + _PyConfigCtx *ctx = &calculate->ctx; size_t len; const char *path = getenv("PATH"); if (path) { - calculate->path_env = _Py_DecodeLocaleCtx(&calculate->ctx, path, &len); + calculate->path_env = _Py_DecodeLocaleCtx(ctx, path, &len); if (!calculate->path_env) { return DECODE_LOCALE_ERR("PATH environment variable", len); } } - calculate->pythonpath = _Py_DecodeLocaleCtx(&config->ctx, PYTHONPATH, &len); + calculate->pythonpath = _Py_DecodeLocaleCtx(ctx, PYTHONPATH, &len); if (!calculate->pythonpath) { return DECODE_LOCALE_ERR("PYTHONPATH define", len); } - calculate->prefix = _Py_DecodeLocaleCtx(&config->ctx, PREFIX, &len); + calculate->prefix = _Py_DecodeLocaleCtx(ctx, PREFIX, &len); if (!calculate->prefix) { return DECODE_LOCALE_ERR("PREFIX define", len); } - calculate->exec_prefix = _Py_DecodeLocaleCtx(&config->ctx, EXEC_PREFIX, &len); + calculate->exec_prefix = _Py_DecodeLocaleCtx(ctx, EXEC_PREFIX, &len); if (!calculate->prefix) { return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); } - calculate->lib_python = _Py_DecodeLocaleCtx(&config->ctx, "lib/python" VERSION, &len); + calculate->lib_python = _Py_DecodeLocaleCtx(ctx, "lib/python" VERSION, &len); if (!calculate->lib_python) { return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); } From 048414383028ad3bccdaddfeebb0e2431d48bd7c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 17 Nov 2018 01:06:47 +0100 Subject: [PATCH 08/12] Add _PyMem_RawStrdupCtx() --- Include/coreconfig.h | 1 + Objects/obmalloc.c | 13 +++++++++++++ Python/coreconfig.c | 45 +++++++++++++++++++++----------------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/Include/coreconfig.h b/Include/coreconfig.h index a43484a64d6d858..ce44e3dd90e0ccb 100644 --- a/Include/coreconfig.h +++ b/Include/coreconfig.h @@ -51,6 +51,7 @@ PyAPI_FUNC(void *) _PyMem_RawMallocCtx(const _PyConfigCtx *ctx, size_t size); PyAPI_FUNC(void *) _PyMem_RawCallocCtx(const _PyConfigCtx *ctx, size_t nelem, size_t elsize); PyAPI_FUNC(void *) _PyMem_RawReallocCtx(const _PyConfigCtx *ctx, void *ptr, size_t new_size); PyAPI_FUNC(void) _PyMem_RawFreeCtx(const _PyConfigCtx *ctx, void *ptr); +PyAPI_FUNC(char *) _PyMem_RawStrdupCtx(const _PyConfigCtx *ctx, const char *str); PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const _PyConfigCtx *ctx, const wchar_t *str); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index d71a7529e5ab003..9823295a53052c8 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -650,6 +650,19 @@ _PyMem_RawWcsdup(const _PyConfigCtx *ctx, const wchar_t *str) return str2; } +char * +_PyMem_RawStrdupCtx(const _PyConfigCtx *ctx, const char *str) +{ + assert(str != NULL); + size_t size = strlen(str) + 1; + char *copy = _PyMem_RawMallocCtx(ctx, size); + if (copy == NULL) { + return NULL; + } + memcpy(copy, str, size); + return copy; +} + char * _PyMem_RawStrdup(const char *str) { diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 6600274ebc0b305..b96bd0b823e7ccf 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -57,15 +57,12 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ void _PyConfigCtx_Init(_PyConfigCtx *ctx) { - /* FIXME: don't magically initialize the runtime? */ - _PyInitError err; - err = _PyRuntime_Initialize(); + _PyInitError err = _PyRuntime_Initialize(); if (_Py_INIT_FAILED(err)) { _Py_FatalInitError(err); } ctx->utf8_mode = -1; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &ctx->raw_alloc); } @@ -371,7 +368,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) #define COPY_STR_ATTR(ATTR) \ do { \ if (config2->ATTR != NULL) { \ - config->ATTR = _PyMem_RawStrdup(config2->ATTR); \ + config->ATTR = _PyMem_RawStrdupCtx(&config->ctx, config2->ATTR); \ if (config->ATTR == NULL) { \ return -1; \ } \ @@ -1130,7 +1127,7 @@ get_stdio_errors(const _PyCoreConfig *config) static _PyInitError -get_locale_encoding(char **locale_encoding) +get_locale_encoding(_PyConfigCtx *ctx, char **locale_encoding) { #ifdef MS_WINDOWS char encoding[20]; @@ -1144,7 +1141,7 @@ get_locale_encoding(char **locale_encoding) "nl_langinfo(CODESET) failed"); } #endif - *locale_encoding = _PyMem_RawStrdup(encoding); + *locale_encoding = _PyMem_RawStrdupCtx(ctx, encoding); if (*locale_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1158,14 +1155,14 @@ config_init_stdio_encoding(_PyCoreConfig *config) /* If Py_SetStandardStreamEncoding() have been called, use these parameters. */ if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) { - config->stdio_encoding = _PyMem_RawStrdup(_Py_StandardStreamEncoding); + config->stdio_encoding = _PyMem_RawStrdupCtx(&config->ctx, _Py_StandardStreamEncoding); if (config->stdio_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } } if (config->stdio_errors == NULL && _Py_StandardStreamErrors != NULL) { - config->stdio_errors = _PyMem_RawStrdup(_Py_StandardStreamErrors); + config->stdio_errors = _PyMem_RawStrdupCtx(&config->ctx, _Py_StandardStreamErrors); if (config->stdio_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1195,7 +1192,7 @@ config_init_stdio_encoding(_PyCoreConfig *config) /* Does PYTHONIOENCODING contain an encoding? */ if (pythonioencoding[0]) { if (config->stdio_encoding == NULL) { - config->stdio_encoding = _PyMem_RawStrdup(pythonioencoding); + config->stdio_encoding = _PyMem_RawStrdupCtx(&config->ctx, pythonioencoding); if (config->stdio_encoding == NULL) { PyMem_RawFree(pythonioencoding); return _Py_INIT_NO_MEMORY(); @@ -1212,7 +1209,7 @@ config_init_stdio_encoding(_PyCoreConfig *config) } if (config->stdio_errors == NULL && err != NULL) { - config->stdio_errors = _PyMem_RawStrdup(err); + config->stdio_errors = _PyMem_RawStrdupCtx(&config->ctx, err); if (config->stdio_errors == NULL) { PyMem_RawFree(pythonioencoding); return _Py_INIT_NO_MEMORY(); @@ -1225,13 +1222,13 @@ config_init_stdio_encoding(_PyCoreConfig *config) /* UTF-8 Mode uses UTF-8/surrogateescape */ if (config->ctx.utf8_mode) { if (config->stdio_encoding == NULL) { - config->stdio_encoding = _PyMem_RawStrdup("utf-8"); + config->stdio_encoding = _PyMem_RawStrdupCtx(&config->ctx, "utf-8"); if (config->stdio_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } } if (config->stdio_errors == NULL) { - config->stdio_errors = _PyMem_RawStrdup("surrogateescape"); + config->stdio_errors = _PyMem_RawStrdupCtx(&config->ctx, "surrogateescape"); if (config->stdio_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1240,14 +1237,14 @@ config_init_stdio_encoding(_PyCoreConfig *config) /* Choose the default error handler based on the current locale. */ if (config->stdio_encoding == NULL) { - _PyInitError err = get_locale_encoding(&config->stdio_encoding); + _PyInitError err = get_locale_encoding(&config->ctx, &config->stdio_encoding); if (_Py_INIT_FAILED(err)) { return err; } } if (config->stdio_errors == NULL) { const char *errors = get_stdio_errors(config); - config->stdio_errors = _PyMem_RawStrdup(errors); + config->stdio_errors = _PyMem_RawStrdupCtx(&config->ctx, errors); if (config->stdio_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1264,13 +1261,13 @@ config_init_fs_encoding(_PyCoreConfig *config) if (config->legacy_windows_fs_encoding) { /* Legacy Windows filesystem encoding: mbcs/replace */ if (config->filesystem_encoding == NULL) { - config->filesystem_encoding = _PyMem_RawStrdup("mbcs"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "mbcs"); if (config->filesystem_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } } if (config->filesystem_errors == NULL) { - config->filesystem_errors = _PyMem_RawStrdup("replace"); + config->filesystem_errors = _PyMem_RawStrdupCtx(&config->ctx, "replace"); if (config->filesystem_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1282,14 +1279,14 @@ config_init_fs_encoding(_PyCoreConfig *config) Note: UTF-8 Mode takes the same code path and the Legacy Windows FS encoding has the priortiy over UTF-8 Mode. */ if (config->filesystem_encoding == NULL) { - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "utf-8"); if (config->filesystem_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } } if (config->filesystem_errors == NULL) { - config->filesystem_errors = _PyMem_RawStrdup("surrogatepass"); + config->filesystem_errors = _PyMem_RawStrdupCtx(&config->ctx, "surrogatepass"); if (config->filesystem_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1298,19 +1295,19 @@ config_init_fs_encoding(_PyCoreConfig *config) if (config->filesystem_encoding == NULL) { if (config->ctx.utf8_mode) { /* UTF-8 Mode use: utf-8/surrogateescape */ - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "utf-8"); /* errors defaults to surrogateescape above */ } else if (_Py_GetForceASCII()) { - config->filesystem_encoding = _PyMem_RawStrdup("ascii"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "ascii"); } else { /* macOS and Android use UTF-8, other platforms use the locale encoding. */ #if defined(__APPLE__) || defined(__ANDROID__) - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "utf-8"); #else - _PyInitError err = get_locale_encoding(&config->filesystem_encoding); + _PyInitError err = get_locale_encoding(&config->ctx, &config->filesystem_encoding); if (_Py_INIT_FAILED(err)) { return err; } @@ -1324,7 +1321,7 @@ config_init_fs_encoding(_PyCoreConfig *config) if (config->filesystem_errors == NULL) { /* by default, use the "surrogateescape" error handler */ - config->filesystem_errors = _PyMem_RawStrdup("surrogateescape"); + config->filesystem_errors = _PyMem_RawStrdupCtx(&config->ctx, "surrogateescape"); if (config->filesystem_errors == NULL) { return _Py_INIT_NO_MEMORY(); } From aaf65b25d7f89905aade8f29aa8048ed20b34a00 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 17 Nov 2018 01:12:19 +0100 Subject: [PATCH 09/12] _testembed: unset PYTHONMALLOC --- Programs/_testembed.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index f488567c7d905d3..b52be36dc000d62 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -139,6 +139,8 @@ static int test_forced_io_encoding(void) static int test_pre_initialization_api(void) { + unsetenv("PYTHONMALLOC"); + /* Leading "./" ensures getpath.c can still find the standard library */ _Py_EMBED_PREINIT_CHECK("Checking Py_DecodeLocale\n"); wchar_t *program = Py_DecodeLocale("./spam", NULL); @@ -235,6 +237,8 @@ static void bpo20891_thread(void *lockp) static int test_bpo20891(void) { + unsetenv("PYTHONMALLOC"); + /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread before calling PyEval_InitThreads() must not crash. PyGILState_Ensure() must call PyEval_InitThreads() for us in this case. */ @@ -278,6 +282,8 @@ static int test_initialize_twice(void) static int test_initialize_pymain(void) { + unsetenv("PYTHONMALLOC"); + wchar_t *argv[] = {L"PYTHON", L"-c", L"import sys; print(f'Py_Main() after Py_Initialize: sys.argv={sys.argv}')", L"arg2"}; From f99e036ad4dd18d8a8d991232cdb081f23ccad62 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 17 Nov 2018 01:14:19 +0100 Subject: [PATCH 10/12] Cleanup _PyMem_SetupDebugHooksDomain() --- Objects/obmalloc.c | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 9823295a53052c8..871df7b655c34ed 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -204,11 +204,11 @@ static struct { }; #define PYDBGRAW_ALLOC \ - {&_PyMem_Debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree} + (PyMemAllocatorEx){&_PyMem_Debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree} #define PYDBGMEM_ALLOC \ - {&_PyMem_Debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} + (PyMemAllocatorEx){&_PyMem_Debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} #define PYDBGOBJ_ALLOC \ - {&_PyMem_Debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} + (PyMemAllocatorEx){&_PyMem_Debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} #ifdef Py_DEBUG static PyMemAllocatorEx _PyMem_Raw = PYDBGRAW_ALLOC; @@ -376,16 +376,6 @@ _PyMem_GetAllocatorsName(void) } -#undef MALLOC_ALLOC -#undef PYMALLOC_ALLOC -#undef PYRAW_ALLOC -#undef PYMEM_ALLOC -#undef PYOBJ_ALLOC -#undef PYDBGRAW_ALLOC -#undef PYDBGMEM_ALLOC -#undef PYDBGOBJ_ALLOC - - static PyObjectArenaAllocator _PyObject_Arena = {NULL, #ifdef MS_WINDOWS _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree @@ -427,11 +417,7 @@ _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain) } PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc); - alloc.ctx = &_PyMem_Debug.raw; - alloc.malloc = _PyMem_DebugRawMalloc; - alloc.calloc = _PyMem_DebugRawCalloc; - alloc.realloc = _PyMem_DebugRawRealloc; - alloc.free = _PyMem_DebugRawFree; + alloc = PYDBGRAW_ALLOC ; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); } else if (domain == PYMEM_DOMAIN_MEM) { @@ -440,11 +426,7 @@ _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain) } PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); - alloc.ctx = &_PyMem_Debug.mem; - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; + alloc = PYDBGMEM_ALLOC; PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); } else if (domain == PYMEM_DOMAIN_OBJ) { @@ -453,11 +435,7 @@ _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain) } PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); - alloc.ctx = &_PyMem_Debug.obj; - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; + alloc = PYDBGOBJ_ALLOC; PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); } } From 2d9b47e6dfa2ac085930eaa5bac2c2fba6b28efd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 17 Nov 2018 01:16:14 +0100 Subject: [PATCH 11/12] _PyPathConfig_Calculate() leaves malloc unchanged --- Python/pathconfig.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 4d9f5616e486826..d7583ba5e8fa948 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -83,9 +83,6 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, _PyPathConfig new_config; _PyPathConfig_StaticInit(&new_config); - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - /* Calculate program_full_path, prefix, exec_prefix (Unix) or dll_path (Windows), and module_search_path */ err = _PyPathConfig_Calculate_impl(&new_config, core_config); @@ -106,14 +103,10 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, _PyPathConfig_Clear(path_config); *path_config = new_config; - err = _Py_INIT_OK(); - goto done; + return _Py_INIT_OK(); err: _PyPathConfig_Clear(&new_config); - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return err; } From abe35405e058d04fdc919c575bf575b90cb5953c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 17 Nov 2018 01:18:16 +0100 Subject: [PATCH 12/12] _PyCmdline: add ctx --- Modules/main.c | 44 +++++++++++++++----------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/Modules/main.c b/Modules/main.c index 1ec77fea7419b7e..fb2833032b33fa1 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -305,6 +305,7 @@ pymain_run_command(wchar_t *command, PyCompilerFlags *cf) /* Main program */ typedef struct { + _PyConfigCtx ctx; wchar_t **argv; int nwarnoption; /* Number of -W command line options */ wchar_t **warnoptions; /* Command line -W options */ @@ -369,7 +370,7 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, if (pymain->use_bytes_argv) { /* +1 for a the NULL terminator */ size_t size = sizeof(wchar_t*) * (pymain->argc + 1); - wchar_t** argv = (wchar_t **)PyMem_RawMalloc(size); + wchar_t** argv = (wchar_t **)_PyMem_RawMallocCtx(&cmdline->ctx, size); if (argv == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; @@ -377,9 +378,9 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, for (int i = 0; i < pymain->argc; i++) { size_t len; - wchar_t *arg = _Py_DecodeLocaleCtx(&config->ctx, pymain->bytes_argv[i], &len); + wchar_t *arg = _Py_DecodeLocaleCtx(&cmdline->ctx, pymain->bytes_argv[i], &len); if (arg == NULL) { - _Py_wstrlist_clear(&config->ctx, i, argv); + _Py_wstrlist_clear(&cmdline->ctx, i, argv); pymain->err = DECODE_LOCALE_ERR("command line arguments", (Py_ssize_t)len); return -1; @@ -411,25 +412,23 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, static void -pymain_clear_cmdline(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) +pymain_clear_cmdline(_PyMain *pymain, _PyCmdline *cmdline) { - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _Py_wstrlist_clear(&config->ctx, cmdline->nwarnoption, cmdline->warnoptions); + _Py_wstrlist_clear(&cmdline->ctx, cmdline->nwarnoption, cmdline->warnoptions); cmdline->nwarnoption = 0; cmdline->warnoptions = NULL; - _Py_wstrlist_clear(&config->ctx, cmdline->nenv_warnoption, cmdline->env_warnoptions); + _Py_wstrlist_clear(&cmdline->ctx, cmdline->nenv_warnoption, cmdline->env_warnoptions); cmdline->nenv_warnoption = 0; cmdline->env_warnoptions = NULL; if (pymain->use_bytes_argv && cmdline->argv != NULL) { - _Py_wstrlist_clear(&config->ctx, pymain->argc, cmdline->argv); + _Py_wstrlist_clear(&cmdline->ctx, pymain->argc, cmdline->argv); } cmdline->argv = NULL; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + memset(cmdline, 0, sizeof(*cmdline)); + _PyConfigCtx_Init(&cmdline->ctx); } @@ -1311,7 +1310,9 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */ + cmdline->ctx.utf8_mode = config->ctx.utf8_mode; Py_UTF8Mode = config->ctx.utf8_mode; + /* FIXME: put legacy_windows_fs_encoding into ctx */ #ifdef MS_WINDOWS Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; #endif @@ -1366,8 +1367,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, pymain->err = _Py_INIT_NO_MEMORY(); goto done; } - pymain_clear_cmdline(pymain, config, cmdline); - memset(cmdline, 0, sizeof(*cmdline)); + pymain_clear_cmdline(pymain, cmdline); config->ctx.utf8_mode = new_utf8_mode; config->coerce_c_locale = new_coerce_c_locale; @@ -1722,27 +1722,13 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, static int pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) { - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -#ifdef Py_DEBUG - PyMemAllocatorEx default_alloc; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &default_alloc); -#endif - _PyCmdline cmdline; memset(&cmdline, 0, sizeof(cmdline)); + _PyConfigCtx_Init(&cmdline.ctx); int res = pymain_cmdline_impl(pymain, config, &cmdline); - pymain_clear_cmdline(pymain, config, &cmdline); - -#ifdef Py_DEBUG - /* Make sure that PYMEM_DOMAIN_RAW has not been modified */ - PyMemAllocatorEx cur_alloc; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &cur_alloc); - assert(memcmp(&cur_alloc, &default_alloc, sizeof(cur_alloc)) == 0); -#endif - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + pymain_clear_cmdline(pymain, &cmdline); return res; }