diff --git a/Include/Python.h b/Include/Python.h index aa60175f5235f37..55b06aeea9846f8 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -128,7 +128,7 @@ #include "codecs.h" #include "pyerrors.h" -#include "coreconfig.h" +#include "cpython/coreconfig.h" #include "pystate.h" #include "context.h" diff --git a/Include/coreconfig.h b/Include/cpython/coreconfig.h similarity index 90% rename from Include/coreconfig.h rename to Include/cpython/coreconfig.h index 0ed3222a15e142e..59e54ca4a053f5d 100644 --- a/Include/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -1,15 +1,15 @@ #ifndef Py_PYCORECONFIG_H #define Py_PYCORECONFIG_H +#ifndef Py_LIMITED_API #ifdef __cplusplus extern "C" { #endif - -#ifndef Py_LIMITED_API typedef struct { const char *prefix; const char *msg; int user_err; + int exitcode; } _PyInitError; /* Almost all errors causing Python initialization to fail */ @@ -21,48 +21,50 @@ typedef struct { #endif #define _Py_INIT_OK() \ - (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0} + (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0, .exitcode = -1} #define _Py_INIT_ERR(MSG) \ - (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 0} + (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 0, .exitcode = -1} /* Error that can be fixed by the user like invalid input parameter. Don't abort() the process on such error. */ #define _Py_INIT_USER_ERR(MSG) \ - (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1} + (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1, .exitcode = -1} #define _Py_INIT_NO_MEMORY() _Py_INIT_USER_ERR("memory allocation failed") +#define _Py_INIT_EXIT(EXITCODE) \ + (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0, .exitcode = (EXITCODE)} #define _Py_INIT_FAILED(err) \ - (err.msg != NULL) + (err.msg != NULL || err.exitcode != -1) -#endif /* !defined(Py_LIMITED_API) */ typedef struct { - /* Install signal handlers? Yes by default. */ - int install_signal_handlers; + int argc; + int use_bytes_argv; + char * const * bytes_argv; + wchar_t * const * wchar_argv; +} _PyArgv; + + +typedef struct { + char *allocator; /* Memory allocator: PYTHONMALLOC */ + int dev_mode; /* Development mode. PYTHONDEVMODE, -X dev */ /* If greater than 0: use environment variables. Set to 0 by -E command line option. If set to -1 (default), it is set to !Py_IgnoreEnvironmentFlag. */ int use_environment; - int use_hash_seed; /* PYTHONHASHSEED=x */ - unsigned long hash_seed; - - const char *allocator; /* Memory allocator: PYTHONMALLOC */ - int dev_mode; /* PYTHONDEVMODE, -X dev */ + /* If greater than 0, enable isolated mode: sys.path contains + neither the script's directory nor the user's site-packages directory. - /* Enable faulthandler? - Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */ - int faulthandler; + Set to 1 by the -I command line option. If set to -1 (default), inherit + Py_IsolatedFlag value. */ + int isolated; - /* Enable tracemalloc? - Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */ - int tracemalloc; + /* Enable UTF-8 mode? + Set by -X utf8 command line option and PYTHONUTF8 environment variable. + If set to -1 (default), inherit Py_UTF8Mode value. */ + int utf8_mode; - int import_time; /* PYTHONPROFILEIMPORTTIME, -X importtime */ - int show_ref_count; /* -X showrefcount */ - int show_alloc_count; /* -X showalloccount */ - int dump_refs; /* PYTHONDUMPREFS */ - int malloc_stats; /* PYTHONMALLOCSTATS */ int coerce_c_locale; /* PYTHONCOERCECLOCALE, -1 means unknown */ int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */ @@ -102,10 +104,81 @@ typedef struct { char *filesystem_encoding; char *filesystem_errors; - /* Enable UTF-8 mode? - Set by -X utf8 command line option and PYTHONUTF8 environment variable. - If set to -1 (default), inherit Py_UTF8Mode value. */ - int utf8_mode; + /* Encoding of sys.stdin, sys.stdout and sys.stderr. + Value set from PYTHONIOENCODING environment variable and + Py_SetStandardStreamEncoding() function. + See also 'stdio_errors' attribute. */ + char *stdio_encoding; + + /* Error handler of sys.stdin and sys.stdout. + Value set from PYTHONIOENCODING environment variable and + Py_SetStandardStreamEncoding() function. + See also 'stdio_encoding' attribute. */ + char *stdio_errors; + +#ifdef MS_WINDOWS + /* If greater than 1, use the "mbcs" encoding instead of the UTF-8 + encoding for the filesystem encoding. + + Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is + set to a non-empty string. If set to -1 (default), inherit + Py_LegacyWindowsFSEncodingFlag value. + + See PEP 529 for more details. */ + int legacy_windows_fs_encoding; + + /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys + standard streams. + + Set to 1 if the PYTHONLEGACYWINDOWSSTDIO environment variable is set to + a non-empty string. If set to -1 (default), inherit + Py_LegacyWindowsStdioFlag value. + + See PEP 528 for more details. */ + int legacy_windows_stdio; +#endif +} _PyPreConfig; + +#ifdef MS_WINDOWS +# define _PyPreConfig_WINDOWS_INIT \ + .legacy_windows_fs_encoding = -1, \ + .legacy_windows_stdio = -1, +#else +# define _PyPreConfig_WINDOWS_INIT +#endif + +#define _PyPreConfig_INIT \ + (_PyPreConfig){ \ + _PyPreConfig_WINDOWS_INIT \ + .use_environment = -1, \ + .isolated = -1, \ + .utf8_mode = -1, \ + .coerce_c_locale = -1 \ + } + + +typedef struct { + _PyPreConfig preconfig; + + /* Install signal handlers? Yes by default. */ + int install_signal_handlers; + + int use_hash_seed; /* PYTHONHASHSEED=x */ + unsigned long hash_seed; + + /* Enable faulthandler? + Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */ + int faulthandler; + + /* Enable tracemalloc? + Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */ + int tracemalloc; + + int import_time; /* PYTHONPROFILEIMPORTTIME, -X importtime */ + int show_ref_count; /* -X showrefcount */ + int show_alloc_count; /* -X showalloccount */ + int dump_refs; /* PYTHONDUMPREFS */ + int malloc_stats; /* PYTHONMALLOCSTATS */ wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */ @@ -139,13 +212,6 @@ typedef struct { wchar_t *dll_path; /* Windows DLL path */ #endif - /* If greater than 0, enable isolated mode: sys.path contains - neither the script's directory nor the user's site-packages directory. - - Set to 1 by the -I command line option. If set to -1 (default), inherit - Py_IsolatedFlag value. */ - int isolated; - /* If equal to zero, disable the import of the module site and the site-dependent manipulations of sys.path that it entails. Also disable these manipulations if site is explicitly imported later (call @@ -239,39 +305,12 @@ typedef struct { If set to -1 (default), it is set to !Py_UnbufferedStdioFlag. */ int buffered_stdio; - /* Encoding of sys.stdin, sys.stdout and sys.stderr. - Value set from PYTHONIOENCODING environment variable and - Py_SetStandardStreamEncoding() function. - See also 'stdio_errors' attribute. */ - char *stdio_encoding; - - /* Error handler of sys.stdin and sys.stdout. - Value set from PYTHONIOENCODING environment variable and - Py_SetStandardStreamEncoding() function. - See also 'stdio_encoding' attribute. */ - char *stdio_errors; - -#ifdef MS_WINDOWS - /* If greater than 1, use the "mbcs" encoding instead of the UTF-8 - encoding for the filesystem encoding. - - Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is - set to a non-empty string. If set to -1 (default), inherit - Py_LegacyWindowsFSEncodingFlag value. - - See PEP 529 for more details. */ - int legacy_windows_fs_encoding; - - /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys - standard streams. - - Set to 1 if the PYTHONLEGACYWINDOWSSTDIO environment variable is set to - a non-empty string. If set to -1 (default), inherit - Py_LegacyWindowsStdioFlag value. + /* Skip the first line of the source: -x command line option */ + int skip_source_first_line; - See PEP 528 for more details. */ - int legacy_windows_stdio; -#endif + wchar_t *run_command; /* -c command line argument */ + wchar_t *run_module; /* -m command line argument */ + wchar_t *run_filename; /* Trailing command line argument without -c or -m */ /* --- Private fields -------- */ @@ -301,26 +340,15 @@ typedef struct { } _PyCoreConfig; -#ifdef MS_WINDOWS -# define _PyCoreConfig_WINDOWS_INIT \ - .legacy_windows_fs_encoding = -1, \ - .legacy_windows_stdio = -1, -#else -# define _PyCoreConfig_WINDOWS_INIT -#endif - #define _PyCoreConfig_INIT \ (_PyCoreConfig){ \ + .preconfig = _PyPreConfig_INIT, \ .install_signal_handlers = 1, \ - .use_environment = -1, \ .use_hash_seed = -1, \ .faulthandler = -1, \ .tracemalloc = -1, \ - .coerce_c_locale = -1, \ - .utf8_mode = -1, \ .argc = -1, \ .nmodule_search_path = -1, \ - .isolated = -1, \ .site_import = -1, \ .bytes_warning = -1, \ .inspect = -1, \ @@ -332,39 +360,17 @@ typedef struct { .quiet = -1, \ .user_site_directory = -1, \ .buffered_stdio = -1, \ - _PyCoreConfig_WINDOWS_INIT \ ._install_importlib = 1, \ ._check_hash_pycs_mode = "default", \ ._frozen = -1} /* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */ - -#ifndef Py_LIMITED_API -PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config); -PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); -PyAPI_FUNC(int) _PyCoreConfig_Copy( - _PyCoreConfig *config, - const _PyCoreConfig *config2); -PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(_PyCoreConfig *config); -PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPathConfig( - const _PyCoreConfig *config); -PyAPI_FUNC(void) _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config); -PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config); -PyAPI_FUNC(const char*) _PyCoreConfig_GetEnv( - const _PyCoreConfig *config, - const char *name); -PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( - const _PyCoreConfig *config, - wchar_t **dest, - wchar_t *wname, - char *name); - /* Used by _testcapi.get_global_config() and _testcapi.get_core_config() */ PyAPI_FUNC(PyObject *) _Py_GetGlobalVariablesAsDict(void); PyAPI_FUNC(PyObject *) _PyCoreConfig_AsDict(const _PyCoreConfig *config); -#endif #ifdef __cplusplus } #endif -#endif /* !Py_PYCORECONFIG_H */ +#endif /* !Py_LIMITED_API */ +#endif /* !Py_PYCORECONFIG_H */ diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index a3fdeefde01b40d..e401f80b72c661d 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -39,7 +39,7 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter( PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig( const _PyCoreConfig *config); -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalInitError(_PyInitError err); +PyAPI_FUNC(void) _Py_NO_RETURN _Py_ExitInitError(_PyInitError err); /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. @@ -64,7 +64,7 @@ PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size); /* Legacy locale support */ PyAPI_FUNC(void) _Py_CoerceLegacyLocale(int warn); -PyAPI_FUNC(int) _Py_LegacyLocaleDetected(void); +PyAPI_FUNC(int) _Py_LegacyLocaleDetected(const char *ctype_locale); PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); #ifdef __cplusplus diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0da59d96d1f75ee..a354e046eaf6515 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -6,6 +6,8 @@ extern "C" { #endif +#include "coreconfig.h" + /* Placeholders while working on the new configuration API * * See PEP 432 for final anticipated contents diff --git a/Include/fileobject.h b/Include/fileobject.h index 89e8dd6a2850ca9..fed4576e673f8e0 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -27,6 +27,9 @@ PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding; PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; #endif PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; +#ifdef Py_BUILD_CORE +PyAPI_DATA(int) _Py_HasFileSystemDefaultEncodeErrors; +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 PyAPI_DATA(int) Py_UTF8Mode; diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h new file mode 100644 index 000000000000000..db23995c4defacd --- /dev/null +++ b/Include/internal/pycore_coreconfig.h @@ -0,0 +1,56 @@ +#ifndef Py_INTERNAL_CORECONFIG_H +#define Py_INTERNAL_CORECONFIG_H +#ifdef __cplusplus +extern "C" { +#endif + +#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN) +# error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN defined" +#endif + +/* helpers */ +PyAPI_FUNC(int) _Py_str_to_int(const char *str, int *result); +PyAPI_FUNC(const wchar_t*) _Py_GetXOption(int nxoption, wchar_t **xoptions, wchar_t *name); +PyAPI_FUNC(const wchar_t*) _Py_GetProgramFromArgv(int argc, wchar_t * const *argv); +PyAPI_FUNC(_PyInitError) _PyArgv_Decode(const _PyArgv *args, int *argc_p, wchar_t*** argv_p); + +/* _PyPreConfig */ +PyAPI_FUNC(const char*) _PyPreConfig_GetEnv(const _PyPreConfig *config, const char *name); +PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config); +PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config); +PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2); +PyAPI_FUNC(void) _PyPreConfig_GetGlobalConfig(_PyPreConfig *config); +PyAPI_FUNC(void) _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config); +PyAPI_FUNC(_PyInitError) _PyPreConfig_InitFromArgv(_PyPreConfig *config, const _PyArgv *args); +PyAPI_FUNC(_PyInitError) _PyPreConfig_SetAllocator(_PyPreConfig *config); +PyAPI_FUNC(void) _PyPreConfig_GetEnvFlag(_PyPreConfig *config, int *flag, const char *name); + +/* _PyCoreConfig */ +PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig); +PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); +PyAPI_FUNC(int) _PyCoreConfig_Copy( + _PyCoreConfig *config, + const _PyCoreConfig *config2); +PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(_PyCoreConfig *config); +PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPathConfig( + const _PyCoreConfig *config); +PyAPI_FUNC(void) _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config); +PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config); +PyAPI_FUNC(const char*) _PyCoreConfig_GetEnv( + const _PyCoreConfig *config, + const char *name); +PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( + const _PyCoreConfig *config, + wchar_t **dest, + wchar_t *wname, + char *name); +PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv( + _PyCoreConfig *config, + const _PyArgv *args, + const _PyPreConfig *preconfig); +PyAPI_FUNC(void) _Py_ClearArgcArgv(void); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CORECONFIG_H */ diff --git a/Include/internal/pycore_getopt.h b/Include/internal/pycore_getopt.h index e6f4654a666b6b5..564f545998e2b02 100644 --- a/Include/internal/pycore_getopt.h +++ b/Include/internal/pycore_getopt.h @@ -11,13 +11,6 @@ extern wchar_t *_PyOS_optarg; extern void _PyOS_ResetGetOpt(void); -typedef struct { - const wchar_t *name; - int has_arg; - int val; -} _PyOS_LongOption; - -extern int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring, - const _PyOS_LongOption *longopts, int *longindex); +extern int _PyOS_GetOpt(int argc, wchar_t **argv, int *longindex); #endif /* !Py_INTERNAL_PYGETOPT_H */ diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index c0731525ca1002e..894a05e3e895e7c 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -13,7 +13,7 @@ PyAPI_FUNC(void) _Py_wstrlist_clear( wchar_t **list); PyAPI_FUNC(wchar_t**) _Py_wstrlist_copy( int len, - wchar_t **list); + wchar_t * const * list); PyAPI_FUNC(_PyInitError) _Py_wstrlist_append( int *len, wchar_t ***list, diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1e7da87cd75c0aa..cde2ea6194d8315 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -1,5 +1,5 @@ -#ifndef Py_INTERNAL_MEM_H -#define Py_INTERNAL_MEM_H +#ifndef Py_INTERNAL_PYMEM_H +#define Py_INTERNAL_PYMEM_H #ifdef __cplusplus extern "C" { #endif @@ -158,4 +158,4 @@ PyAPI_FUNC(int) _PyMem_SetDefaultAllocator( #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_MEM_H */ +#endif /* !Py_INTERNAL_PYMEM_H */ diff --git a/Include/pystate.h b/Include/pystate.h index a541dc8f00969d7..a79a2e60e210770 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -8,7 +8,6 @@ extern "C" { #endif #include "pythread.h" -#include "coreconfig.h" /* This limitation is for performance and simplicity. If needed it can be removed (with effort). */ diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d35b9f4d5a2f2fd..6c245ebcd1940b3 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -322,6 +322,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'stdio_encoding': GET_DEFAULT_CONFIG, 'stdio_errors': GET_DEFAULT_CONFIG, + 'skip_source_first_line': 0, + 'run_command': None, + 'run_module': None, + 'run_filename': None, + '_install_importlib': 1, '_check_hash_pycs_mode': 'default', '_frozen': 0, diff --git a/Makefile.pre.in b/Makefile.pre.in index 1c536f813f8df16..5ae69480bf16a46 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -357,6 +357,7 @@ PYTHON_OBJS= \ Python/mystrtoul.o \ Python/pathconfig.o \ Python/peephole.o \ + Python/preconfig.o \ Python/pyarena.o \ Python/pyctype.o \ Python/pyfpe.o \ @@ -994,7 +995,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/codecs.h \ $(srcdir)/Include/compile.h \ $(srcdir)/Include/complexobject.h \ - $(srcdir)/Include/coreconfig.h \ $(srcdir)/Include/descrobject.h \ $(srcdir)/Include/dictobject.h \ $(srcdir)/Include/dtoa.h \ @@ -1071,6 +1071,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/Python-ast.h \ \ $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/coreconfig.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ @@ -1085,6 +1086,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_condvar.h \ $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_coreconfig.h \ $(srcdir)/Include/internal/pycore_fileutils.h \ $(srcdir)/Include/internal/pycore_getopt.h \ $(srcdir)/Include/internal/pycore_gil.h \ diff --git a/Modules/main.c b/Modules/main.c index f6e78890085b6f0..c590ebcf73e5436 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1,14 +1,12 @@ /* Python interpreter main program */ #include "Python.h" -#include "osdefs.h" -#include "pycore_getopt.h" +#include "pycore_coreconfig.h" #include "pycore_pathconfig.h" #include "pycore_pylifecycle.h" #include "pycore_pymem.h" #include "pycore_pystate.h" -#include #ifdef HAVE_SIGNAL_H #include #endif @@ -35,12 +33,6 @@ # include #endif -#if defined(MS_WINDOWS) -# define PYTHONHOMEHELP "\\python{major}{minor}" -#else -# define PYTHONHOMEHELP "/lib/pythonX.X" -#endif - #define COPYRIGHT \ "Type \"help\", \"copyright\", \"credits\" or \"license\" " \ "for more information." @@ -49,123 +41,6 @@ extern "C" { #endif -#define DECODE_LOCALE_ERR(NAME, LEN) \ - (((LEN) == -2) \ - ? _Py_INIT_USER_ERR("cannot decode " NAME) \ - : _Py_INIT_NO_MEMORY()) - - -#ifdef MS_WINDOWS -#define WCSTOK wcstok_s -#else -#define WCSTOK wcstok -#endif - -/* For Py_GetArgcArgv(); set by main() */ -static wchar_t **orig_argv = NULL; -static int orig_argc = 0; - -/* command line options */ -#define BASE_OPTS L"bBc:dEhiIJm:OqRsStuvVW:xX:?" - -#define PROGRAM_OPTS BASE_OPTS - -static const _PyOS_LongOption longoptions[] = { - {L"check-hash-based-pycs", 1, 0}, - {NULL, 0, 0}, -}; - -/* Short usage message (with %s for argv0) */ -static const char usage_line[] = -"usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; - -/* Long usage message, split into parts < 512 bytes */ -static const char usage_1[] = "\ -Options and arguments (and corresponding environment variables):\n\ --b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ - and comparing bytes/bytearray with str. (-bb: issue errors)\n\ --B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ --c cmd : program passed in as string (terminates option list)\n\ --d : debug output from parser; also PYTHONDEBUG=x\n\ --E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ --h : print this help message and exit (also --help)\n\ -"; -static const char usage_2[] = "\ --i : inspect interactively after running script; forces a prompt even\n\ - if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ --I : isolate Python from the user's environment (implies -E and -s)\n\ --m mod : run library module as a script (terminates option list)\n\ --O : remove assert and __debug__-dependent statements; add .opt-1 before\n\ - .pyc extension; also PYTHONOPTIMIZE=x\n\ --OO : do -O changes and also discard docstrings; add .opt-2 before\n\ - .pyc extension\n\ --q : don't print version and copyright messages on interactive startup\n\ --s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ --S : don't imply 'import site' on initialization\n\ -"; -static const char usage_3[] = "\ --u : force the stdout and stderr streams to be unbuffered;\n\ - this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ --v : verbose (trace import statements); also PYTHONVERBOSE=x\n\ - can be supplied multiple times to increase verbosity\n\ --V : print the Python version number and exit (also --version)\n\ - when given twice, print more information about the build\n\ --W arg : warning control; arg is action:message:category:module:lineno\n\ - also PYTHONWARNINGS=arg\n\ --x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ --X opt : set implementation-specific option\n\ ---check-hash-based-pycs always|default|never:\n\ - control how Python invalidates hash-based .pyc files\n\ -"; -static const char usage_4[] = "\ -file : program read from script file\n\ -- : program read from stdin (default; interactive mode if a tty)\n\ -arg ...: arguments passed to program in sys.argv[1:]\n\n\ -Other environment variables:\n\ -PYTHONSTARTUP: file executed on interactive startup (no default)\n\ -PYTHONPATH : '%lc'-separated list of directories prefixed to the\n\ - default module search path. The result is sys.path.\n\ -"; -static const char usage_5[] = -"PYTHONHOME : alternate directory (or %lc).\n" -" The default module search path uses %s.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" -"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n"; -static const char usage_6[] = -"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" -" to seed the hashes of str, bytes and datetime objects. It can also be\n" -" set to an integer in the range [0,4294967295] to get hash values with a\n" -" predictable seed.\n" -"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" -" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" -" hooks.\n" -"PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" -" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" -" locale coercion and locale compatibility warnings on stderr.\n" -"PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" -" debugger. It can be set to the callable of your debugger of choice.\n" -"PYTHONDEVMODE: enable the development mode.\n" -"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n"; - -static void -pymain_usage(int error, const wchar_t* program) -{ - FILE *f = error ? stderr : stdout; - - fprintf(f, usage_line, program); - if (error) - fprintf(f, "Try `python -h' for more information.\n"); - else { - fputs(usage_1, f); - fputs(usage_2, f); - fputs(usage_3, f); - fprintf(f, usage_4, (wint_t)DELIM); - fprintf(f, usage_5, (wint_t)DELIM, PYTHONHOMEHELP); - fputs(usage_6, f); - } -} - static void pymain_run_interactive_hook(void) @@ -248,6 +123,7 @@ pymain_run_module(const wchar_t *modname, int set_argv0) return 0; } + static PyObject * pymain_get_importer(const wchar_t *filename) { @@ -308,166 +184,15 @@ pymain_run_command(wchar_t *command, PyCompilerFlags *cf) } -/* Main program */ - -typedef struct { - wchar_t **argv; - int nwarnoption; /* Number of -W command line options */ - wchar_t **warnoptions; /* Command line -W options */ - int nenv_warnoption; /* Number of PYTHONWARNINGS environment variables */ - wchar_t **env_warnoptions; /* PYTHONWARNINGS environment variables */ - int print_help; /* -h, -? options */ - int print_version; /* -V option */ -} _PyCmdline; - -/* Structure used by Py_Main() to pass data to subfunctions */ -typedef struct { - /* Input arguments */ - int argc; - int use_bytes_argv; - char **bytes_argv; - wchar_t **wchar_argv; - - /* Exit status or "exit code": result of pymain_main() */ - int status; - /* Error message if a function failed */ - _PyInitError err; - - /* non-zero is stdin is a TTY or if -i option is used */ - int stdin_is_interactive; - int skip_first_line; /* -x option */ - wchar_t *filename; /* Trailing arg without -c or -m */ - wchar_t *command; /* -c argument */ - wchar_t *module; /* -m argument */ -} _PyMain; - -#define _PyMain_INIT {.err = _Py_INIT_OK()} -/* Note: _PyMain_INIT sets other fields to 0/NULL */ - - /* Non-zero if filename, command (-c) or module (-m) is set on the command line */ -#define RUN_CODE(pymain) \ - (pymain->command != NULL || pymain->filename != NULL \ - || pymain->module != NULL) - - -static wchar_t* -pymain_wstrdup(_PyMain *pymain, const wchar_t *str) -{ - wchar_t *str2 = _PyMem_RawWcsdup(str); - if (str2 == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return NULL; - } - return str2; -} - - -static int -pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - assert(cmdline->argv == NULL); - - if (pymain->use_bytes_argv) { - /* +1 for a the NULL terminator */ - size_t size = sizeof(wchar_t*) * (pymain->argc + 1); - wchar_t** argv = (wchar_t **)PyMem_RawMalloc(size); - if (argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; - } - - for (int i = 0; i < pymain->argc; i++) { - size_t len; - wchar_t *arg = Py_DecodeLocale(pymain->bytes_argv[i], &len); - if (arg == NULL) { - _Py_wstrlist_clear(i, argv); - pymain->err = DECODE_LOCALE_ERR("command line arguments", - (Py_ssize_t)len); - return -1; - } - argv[i] = arg; - } - argv[pymain->argc] = NULL; - - cmdline->argv = argv; - } - else { - cmdline->argv = pymain->wchar_argv; - } - - wchar_t *program; - if (pymain->argc >= 1 && cmdline->argv != NULL) { - program = cmdline->argv[0]; - } - else { - program = L""; - } - config->program = pymain_wstrdup(pymain, program); - if (config->program == NULL) { - return -1; - } - - return 0; -} +#define RUN_CODE(config) \ + (config->run_command != NULL || config->run_filename != NULL \ + || config->run_module != NULL) static void -pymain_clear_cmdline(_PyMain *pymain, _PyCmdline *cmdline) -{ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _Py_wstrlist_clear(cmdline->nwarnoption, cmdline->warnoptions); - cmdline->nwarnoption = 0; - cmdline->warnoptions = NULL; - - _Py_wstrlist_clear(cmdline->nenv_warnoption, cmdline->env_warnoptions); - cmdline->nenv_warnoption = 0; - cmdline->env_warnoptions = NULL; - - if (pymain->use_bytes_argv && cmdline->argv != NULL) { - _Py_wstrlist_clear(pymain->argc, cmdline->argv); - } - cmdline->argv = NULL; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - -static void -pymain_clear_pymain(_PyMain *pymain) -{ -#define CLEAR(ATTR) \ - do { \ - PyMem_RawFree(ATTR); \ - ATTR = NULL; \ - } while (0) - - CLEAR(pymain->filename); - CLEAR(pymain->command); - CLEAR(pymain->module); -#undef CLEAR -} - -static void -pymain_clear_config(_PyCoreConfig *config) -{ - /* Clear core config with the memory allocator - used by pymain_read_conf() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _PyCoreConfig_Clear(config); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - -static void -pymain_free(_PyMain *pymain) +pymain_free(void) { _PyImport_Fini2(); @@ -478,17 +203,7 @@ pymain_free(_PyMain *pymain) _PyPathConfig_ClearGlobal(); _Py_ClearStandardStreamEncoding(); - /* Force the allocator used by pymain_read_conf() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - pymain_clear_pymain(pymain); - - _Py_wstrlist_clear(orig_argc, orig_argv); - orig_argc = 0; - orig_argv = NULL; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _Py_ClearArgcArgv(); #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering @@ -536,220 +251,6 @@ pymain_sys_path_add_path0(PyInterpreterState *interp, PyObject *path0) } -_PyInitError -_Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) -{ - if (*len == INT_MAX) { - /* len+1 would overflow */ - return _Py_INIT_NO_MEMORY(); - } - wchar_t *str2 = _PyMem_RawWcsdup(str); - if (str2 == NULL) { - return _Py_INIT_NO_MEMORY(); - } - - size_t size = (*len + 1) * sizeof(list[0]); - wchar_t **list2 = (wchar_t **)PyMem_RawRealloc(*list, size); - if (list2 == NULL) { - PyMem_RawFree(str2); - return _Py_INIT_NO_MEMORY(); - } - list2[*len] = str2; - *list = list2; - (*len)++; - return _Py_INIT_OK(); -} - - -static int -pymain_wstrlist_append(_PyMain *pymain, int *len, wchar_t ***list, const wchar_t *str) -{ - _PyInitError err = _Py_wstrlist_append(len, list, str); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - return 0; -} - - -/* Parse the command line arguments - Return 0 on success. - Return 1 if parsing failed. - Set pymain->err and return -1 on other errors. */ -static int -pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - _PyOS_ResetGetOpt(); - do { - int longindex = -1; - int c = _PyOS_GetOpt(pymain->argc, cmdline->argv, PROGRAM_OPTS, - longoptions, &longindex); - if (c == EOF) { - break; - } - - if (c == 'c') { - /* -c is the last option; following arguments - that look like options are left for the - command to interpret. */ - size_t len = wcslen(_PyOS_optarg) + 1 + 1; - wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); - if (command == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; - } - memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); - command[len - 2] = '\n'; - command[len - 1] = 0; - pymain->command = command; - break; - } - - if (c == 'm') { - /* -m is the last option; following arguments - that look like options are left for the - module to interpret. */ - pymain->module = pymain_wstrdup(pymain, _PyOS_optarg); - if (pymain->module == NULL) { - return -1; - } - break; - } - - switch (c) { - case 0: - // Handle long option. - assert(longindex == 0); // Only one long option now. - if (!wcscmp(_PyOS_optarg, L"always")) { - config->_check_hash_pycs_mode = "always"; - } else if (!wcscmp(_PyOS_optarg, L"never")) { - config->_check_hash_pycs_mode = "never"; - } else if (!wcscmp(_PyOS_optarg, L"default")) { - config->_check_hash_pycs_mode = "default"; - } else { - fprintf(stderr, "--check-hash-based-pycs must be one of " - "'default', 'always', or 'never'\n"); - return 1; - } - break; - - case 'b': - config->bytes_warning++; - break; - - case 'd': - config->parser_debug++; - break; - - case 'i': - config->inspect++; - config->interactive++; - break; - - case 'I': - config->isolated++; - break; - - /* case 'J': reserved for Jython */ - - case 'O': - config->optimization_level++; - break; - - case 'B': - config->write_bytecode = 0; - break; - - case 's': - config->user_site_directory = 0; - break; - - case 'S': - config->site_import = 0; - break; - - case 'E': - config->use_environment = 0; - break; - - case 't': - /* ignored for backwards compatibility */ - break; - - case 'u': - config->buffered_stdio = 0; - break; - - case 'v': - config->verbose++; - break; - - case 'x': - pymain->skip_first_line = 1; - break; - - case 'h': - case '?': - cmdline->print_help++; - break; - - case 'V': - cmdline->print_version++; - break; - - case 'W': - if (pymain_wstrlist_append(pymain, - &cmdline->nwarnoption, - &cmdline->warnoptions, - _PyOS_optarg) < 0) { - return -1; - } - break; - - case 'X': - if (pymain_wstrlist_append(pymain, - &config->nxoption, - &config->xoptions, - _PyOS_optarg) < 0) { - return -1; - } - break; - - case 'q': - config->quiet++; - break; - - case 'R': - config->use_hash_seed = 0; - break; - - /* This space reserved for other options */ - - default: - /* unknown argument: parsing failed */ - return 1; - } - } while (1); - - if (pymain->command == NULL && pymain->module == NULL - && _PyOS_optind < pymain->argc - && wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0) - { - pymain->filename = pymain_wstrdup(pymain, cmdline->argv[_PyOS_optind]); - if (pymain->filename == NULL) { - return -1; - } - } - - /* -c and -m options are exclusive */ - assert(!(pymain->command != NULL && pymain->module != NULL)); - - return 0; -} - - static int add_xoption(PyObject *opts, const wchar_t *s) { @@ -783,7 +284,7 @@ add_xoption(PyObject *opts, const wchar_t *s) static PyObject* -config_create_xoptions_dict(const _PyCoreConfig *config) +mainconfig_create_xoptions_dict(const _PyCoreConfig *config) { int nxoption = config->nxoption; wchar_t **xoptions = config->xoptions; @@ -804,132 +305,16 @@ config_create_xoptions_dict(const _PyCoreConfig *config) } -static _PyInitError -config_add_warnings_optlist(_PyCoreConfig *config, int len, wchar_t **options) -{ - for (int i = 0; i < len; i++) { - _PyInitError err = _Py_wstrlist_append(&config->nwarnoption, - &config->warnoptions, - options[i]); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - return _Py_INIT_OK(); -} - - -static _PyInitError -config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) -{ - _PyInitError err; - - assert(config->nwarnoption == 0); - - /* The priority order for warnings configuration is (highest precedence - * first): - * - * - the BytesWarning filter, if needed ('-b', '-bb') - * - any '-W' command line options; then - * - the 'PYTHONWARNINGS' environment variable; then - * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then - * - any implicit filters added by _warnings.c/warnings.py - * - * All settings except the last are passed to the warnings module via - * the `sys.warnoptions` list. Since the warnings module works on the basis - * of "the most recently added filter will be checked first", we add - * the lowest precedence entries first so that later entries override them. - */ - - if (config->dev_mode) { - err = _Py_wstrlist_append(&config->nwarnoption, - &config->warnoptions, - L"default"); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - - err = config_add_warnings_optlist(config, - cmdline->nenv_warnoption, - cmdline->env_warnoptions); - if (_Py_INIT_FAILED(err)) { - return err; - } - - err = config_add_warnings_optlist(config, - cmdline->nwarnoption, - cmdline->warnoptions); - if (_Py_INIT_FAILED(err)) { - return err; - } - - /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c - * don't even try to emit a warning, so we skip setting the filter in that - * case. - */ - if (config->bytes_warning) { - wchar_t *filter; - if (config->bytes_warning> 1) { - filter = L"error::BytesWarning"; - } - else { - filter = L"default::BytesWarning"; - } - err = _Py_wstrlist_append(&config->nwarnoption, - &config->warnoptions, - filter); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - return _Py_INIT_OK(); -} - - -/* Get warning options from PYTHONWARNINGS environment variable. - Return 0 on success. - Set pymain->err and return -1 on error. */ -static _PyInitError -cmdline_init_env_warnoptions(_PyMain *pymain, const _PyCoreConfig *config, - _PyCmdline *cmdline) +static int +stdin_is_interactive(const _PyCoreConfig *config) { - wchar_t *env; - int res = _PyCoreConfig_GetEnvDup(config, &env, - L"PYTHONWARNINGS", "PYTHONWARNINGS"); - if (res < 0) { - return DECODE_LOCALE_ERR("PYTHONWARNINGS", res); - } - - if (env == NULL) { - return _Py_INIT_OK(); - } - - - wchar_t *warning, *context = NULL; - for (warning = WCSTOK(env, L",", &context); - warning != NULL; - warning = WCSTOK(NULL, L",", &context)) - { - _PyInitError err = _Py_wstrlist_append(&cmdline->nenv_warnoption, - &cmdline->env_warnoptions, - warning); - if (_Py_INIT_FAILED(err)) { - PyMem_RawFree(env); - return err; - } - } - PyMem_RawFree(env); - return _Py_INIT_OK(); + return (isatty(fileno(stdin)) || config->interactive); } static void -pymain_init_stdio(_PyMain *pymain, _PyCoreConfig *config) +config_init_stdio(_PyCoreConfig *config) { - pymain->stdin_is_interactive = (isatty(fileno(stdin)) - || config->interactive); - #if defined(MS_WINDOWS) || defined(__CYGWIN__) /* don't translate newlines (\r\n <=> \n) */ _setmode(fileno(stdin), O_BINARY); @@ -965,13 +350,13 @@ pymain_init_stdio(_PyMain *pymain, _PyCoreConfig *config) static void -pymain_header(_PyMain *pymain, const _PyCoreConfig *config) +pymain_header(const _PyCoreConfig *config) { if (config->quiet) { return; } - if (!config->verbose && (RUN_CODE(pymain) || !pymain->stdin_is_interactive)) { + if (!config->verbose && (RUN_CODE(config) || !stdin_is_interactive(config))) { return; } @@ -982,85 +367,13 @@ pymain_header(_PyMain *pymain, const _PyCoreConfig *config) } -static int -pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) -{ - /* Copy argv to be able to modify it (to force -c/-m) */ - int argc = pymain->argc - _PyOS_optind; - wchar_t **argv; - - if (argc <= 0 || cmdline->argv == NULL) { - /* Ensure at least one (empty) argument is seen */ - static wchar_t *empty_argv[1] = {L""}; - argc = 1; - argv = _Py_wstrlist_copy(1, empty_argv); - } - else { - argv = _Py_wstrlist_copy(argc, &cmdline->argv[_PyOS_optind]); - } - - if (argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; - } - - wchar_t *arg0 = NULL; - if (pymain->command != NULL) { - /* Force sys.argv[0] = '-c' */ - arg0 = L"-c"; - } - else if (pymain->module != NULL) { - /* Force sys.argv[0] = '-m'*/ - arg0 = L"-m"; - } - if (arg0 != NULL) { - arg0 = _PyMem_RawWcsdup(arg0); - if (arg0 == NULL) { - _Py_wstrlist_clear(argc, argv); - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; - } - - assert(argc >= 1); - PyMem_RawFree(argv[0]); - argv[0] = arg0; - } - - config->argc = argc; - config->argv = argv; - return 0; -} - - -PyObject* -_Py_wstrlist_as_pylist(int len, wchar_t **list) -{ - assert(list != NULL || len < 1); - - PyObject *pylist = PyList_New(len); - if (pylist == NULL) { - return NULL; - } - - for (int i = 0; i < len; i++) { - PyObject *v = PyUnicode_FromWideChar(list[i], -1); - if (v == NULL) { - Py_DECREF(pylist); - return NULL; - } - PyList_SET_ITEM(pylist, i, v); - } - return pylist; -} - - static void -pymain_import_readline(_PyMain *pymain, const _PyCoreConfig *config) +pymain_import_readline(const _PyCoreConfig *config) { - if (config->isolated) { + if (config->preconfig.isolated) { return; } - if (!config->inspect && RUN_CODE(pymain)) { + if (!config->inspect && RUN_CODE(config)) { return; } if (!isatty(fileno(stdin))) { @@ -1078,7 +391,7 @@ pymain_import_readline(_PyMain *pymain, const _PyCoreConfig *config) static void -pymain_run_startup(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +pymain_run_startup(_PyCoreConfig *config, PyCompilerFlags *cf) { const char *startup = _PyCoreConfig_GetEnv(config, "PYTHONSTARTUP"); if (startup == NULL) { @@ -1103,10 +416,10 @@ pymain_run_startup(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) } -static void -pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +static int +pymain_run_file(_PyCoreConfig *config, PyCompilerFlags *cf) { - const wchar_t *filename = pymain->filename; + const wchar_t *filename = config->run_filename; FILE *fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; @@ -1120,11 +433,10 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) fprintf(stderr, "%ls: can't open file '%s': [Errno %d] %s\n", config->program, cfilename, err, strerror(err)); PyMem_RawFree(cfilename_buffer); - pymain->status = 2; - return; + return 2; } - if (pymain->skip_first_line) { + if (config->skip_source_first_line) { int ch; /* Push back first newline so line numbers remain the same */ while ((ch = getc(fp)) != EOF) { @@ -1140,17 +452,15 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", config->program, filename); - pymain->status = 1; fclose(fp); - return; + return 1; } /* call pending calls like signal handlers (SIGINT) */ if (Py_MakePendingCalls() == -1) { PyErr_Print(); - pymain->status = 1; fclose(fp); - return; + return 1; } PyObject *unicode, *bytes = NULL; @@ -1172,34 +482,33 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) /* PyRun_AnyFileExFlags(closeit=1) calls fclose(fp) before running code */ int run = PyRun_AnyFileExFlags(fp, filename_str, 1, cf); Py_XDECREF(bytes); - pymain->status = (run != 0); + return (run != 0); } -static void -pymain_run_stdin(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +static int +pymain_run_stdin(_PyCoreConfig *config, PyCompilerFlags *cf) { - if (pymain->stdin_is_interactive) { + if (stdin_is_interactive(config)) { Py_InspectFlag = 0; /* do exit on SystemExit */ config->inspect = 0; - pymain_run_startup(pymain, config, cf); + pymain_run_startup(config, cf); pymain_run_interactive_hook(); } /* call pending calls like signal handlers (SIGINT) */ if (Py_MakePendingCalls() == -1) { PyErr_Print(); - pymain->status = 1; - return; + return 1; } int run = PyRun_AnyFileExFlags(stdin, "", 0, cf); - pymain->status = (run != 0); + return (run != 0); } static void -pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +pymain_repl(int *exitcode, _PyCoreConfig *config, PyCompilerFlags *cf) { /* Check this environment variable at the end, to give programs the opportunity to set it from Python. */ @@ -1208,7 +517,7 @@ pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) config->inspect = 1; } - if (!(Py_InspectFlag && pymain->stdin_is_interactive && RUN_CODE(pymain))) { + if (!(Py_InspectFlag && stdin_is_interactive(config) && RUN_CODE(config))) { return; } @@ -1217,193 +526,7 @@ pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) pymain_run_interactive_hook(); int res = PyRun_AnyFileFlags(stdin, "", cf); - pymain->status = (res != 0); -} - - -/* Parse the command line. - Handle --version and --help options directly. - - Return 1 if Python must exit. - Return 0 on success. - Set pymain->err and return -1 on failure. */ -static int -pymain_parse_cmdline(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - int res = pymain_parse_cmdline_impl(pymain, config, cmdline); - if (res < 0) { - return -1; - } - if (res) { - pymain_usage(1, config->program); - pymain->status = 2; - return 1; - } - - if (pymain->command != NULL || pymain->module != NULL) { - /* Backup _PyOS_optind */ - _PyOS_optind--; - } - - return 0; -} - - -/* Parse command line options and environment variables. - This code must not use Python runtime apart PyMem_Raw memory allocator. - - Return 0 on success. - Return 1 if Python is done and must exit. - Set pymain->err and return -1 on error. */ -static int -pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - _PyInitError err; - - int res = pymain_parse_cmdline(pymain, config, cmdline); - if (res != 0) { - return res; - } - - if (pymain_init_core_argv(pymain, config, cmdline) < 0) { - return -1; - } - - err = _PyCoreConfig_Read(config); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - - if (config->use_environment) { - err = cmdline_init_env_warnoptions(pymain, config, cmdline); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - } - - err = config_init_warnoptions(config, cmdline); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - return 0; -} - - -/* Read the configuration and initialize the LC_CTYPE locale: - enable UTF-8 mode (PEP 540) and/or coerce the C locale (PEP 538). */ -static int -pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - int init_utf8_mode = Py_UTF8Mode; -#ifdef MS_WINDOWS - int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag; -#endif - _PyCoreConfig save_config = _PyCoreConfig_INIT; - int res = -1; - int locale_coerced = 0; - int loops = 0; - - if (_PyCoreConfig_Copy(&save_config, config) < 0) { - pymain->err = _Py_INIT_NO_MEMORY(); - goto done; - } - - /* Set LC_CTYPE to the user preferred locale */ - _Py_SetLocaleFromEnv(LC_CTYPE); - - while (1) { - int utf8_mode = config->utf8_mode; - int encoding_changed = 0; - - /* Watchdog to prevent an infinite loop */ - loops++; - if (loops == 3) { - pymain->err = _Py_INIT_ERR("Encoding changed twice while " - "reading the configuration"); - goto done; - } - - /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend - on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */ - Py_UTF8Mode = config->utf8_mode; -#ifdef MS_WINDOWS - Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; -#endif - - if (pymain_init_cmdline_argv(pymain, config, cmdline) < 0) { - goto done; - } - - int conf_res = pymain_read_conf_impl(pymain, config, cmdline); - if (conf_res != 0) { - res = conf_res; - goto done; - } - - /* The legacy C locale assumes ASCII as the default text encoding, which - * causes problems not only for the CPython runtime, but also other - * components like GNU readline. - * - * Accordingly, when the CLI detects it, it attempts to coerce it to a - * more capable UTF-8 based alternative. - * - * See the documentation of the PYTHONCOERCECLOCALE setting for more - * details. - */ - if (config->coerce_c_locale && !locale_coerced) { - locale_coerced = 1; - _Py_CoerceLegacyLocale(config->coerce_c_locale_warn); - encoding_changed = 1; - } - - if (utf8_mode == -1) { - if (config->utf8_mode == 1) { - /* UTF-8 Mode enabled */ - encoding_changed = 1; - } - } - else { - if (config->utf8_mode != utf8_mode) { - encoding_changed = 1; - } - } - - if (!encoding_changed) { - break; - } - - /* Reset the configuration before reading again the configuration, - just keep UTF-8 Mode value. */ - int new_utf8_mode = config->utf8_mode; - int new_coerce_c_locale = config->coerce_c_locale; - if (_PyCoreConfig_Copy(config, &save_config) < 0) { - pymain->err = _Py_INIT_NO_MEMORY(); - goto done; - } - pymain_clear_cmdline(pymain, cmdline); - pymain_clear_pymain(pymain); - memset(cmdline, 0, sizeof(*cmdline)); - config->utf8_mode = new_utf8_mode; - config->coerce_c_locale = new_coerce_c_locale; - - /* The encoding changed: read again the configuration - with the new encoding */ - } - res = 0; - -done: - _PyCoreConfig_Clear(&save_config); - Py_UTF8Mode = init_utf8_mode ; -#ifdef MS_WINDOWS - Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding; -#endif - return res; + *exitcode = (res != 0); } @@ -1548,7 +671,7 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config, } if (main_config->xoptions == NULL) { - main_config->xoptions = config_create_xoptions_dict(config); + main_config->xoptions = mainconfig_create_xoptions_dict(config); if (main_config->xoptions == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1604,61 +727,55 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config, } -static int -pymain_init_python_main(_PyMain *pymain, _PyCoreConfig *config, - PyInterpreterState *interp) +static _PyInitError +config_init_python_main(PyInterpreterState *interp) { _PyInitError err; _PyMainInterpreterConfig main_config = _PyMainInterpreterConfig_INIT; - err = _PyMainInterpreterConfig_Read(&main_config, config); + err = _PyMainInterpreterConfig_Read(&main_config, &interp->core_config); if (!_Py_INIT_FAILED(err)) { err = _Py_InitializeMainInterpreter(interp, &main_config); } _PyMainInterpreterConfig_Clear(&main_config); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - return 0; + return err; } -static int -pymain_run_python(_PyMain *pymain, PyInterpreterState *interp) +static _PyInitError +pymain_run_python(int *exitcode, PyInterpreterState *interp) { - int res = 0; + _PyInitError err; _PyCoreConfig *config = &interp->core_config; PyObject *main_importer_path = NULL; - if (pymain->filename != NULL) { + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be prepended to sys.path. Otherwise, main_importer_path is set to NULL. */ - main_importer_path = pymain_get_importer(pymain->filename); + main_importer_path = pymain_get_importer(config->run_filename); } if (main_importer_path != NULL) { if (pymain_sys_path_add_path0(interp, main_importer_path) < 0) { - pymain->status = 1; + err = _Py_INIT_EXIT(1); goto done; } } - else if (!config->isolated) { + else if (!config->preconfig.isolated) { PyObject *path0 = _PyPathConfig_ComputeArgv0(config->argc, config->argv); if (path0 == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - res = -1; + err = _Py_INIT_NO_MEMORY(); goto done; } if (pymain_sys_path_add_path0(interp, path0) < 0) { Py_DECREF(path0); - pymain->status = 1; + err = _Py_INIT_EXIT(1); goto done; } Py_DECREF(path0); @@ -1666,118 +783,76 @@ pymain_run_python(_PyMain *pymain, PyInterpreterState *interp) PyCompilerFlags cf = {.cf_flags = 0}; - pymain_header(pymain, config); - pymain_import_readline(pymain, config); + pymain_header(config); + pymain_import_readline(config); - if (pymain->command) { - pymain->status = pymain_run_command(pymain->command, &cf); + if (config->run_command) { + *exitcode = pymain_run_command(config->run_command, &cf); } - else if (pymain->module) { - pymain->status = (pymain_run_module(pymain->module, 1) != 0); + else if (config->run_module) { + *exitcode = (pymain_run_module(config->run_module, 1) != 0); } else if (main_importer_path != NULL) { int sts = pymain_run_module(L"__main__", 0); - pymain->status = (sts != 0); + *exitcode = (sts != 0); } - else if (pymain->filename != NULL) { - pymain_run_file(pymain, config, &cf); + else if (config->run_filename != NULL) { + *exitcode = pymain_run_file(config, &cf); } else { - pymain_run_stdin(pymain, config, &cf); + *exitcode = pymain_run_stdin(config, &cf); } - pymain_repl(pymain, config, &cf); + pymain_repl(exitcode, config, &cf); + err = _Py_INIT_OK(); done: Py_XDECREF(main_importer_path); - return res; + return err; } -static int -pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) +static _PyInitError +pymain_init_core(const _PyArgv *args, const _PyPreConfig *preconfig, + PyInterpreterState **interp_p) { - pymain->err = _PyRuntime_Initialize(); - if (_Py_INIT_FAILED(pymain->err)) { - return -1; - } - - int res = pymain_read_conf(pymain, config, cmdline); - if (res < 0) { - return -1; - } - if (res > 0) { - /* --help or --version command: we are done */ - return 1; - } - - if (cmdline->print_help) { - pymain_usage(0, config->program); - return 1; - } + _PyInitError err; + _PyCoreConfig local_config = _PyCoreConfig_INIT; + _PyCoreConfig *config = &local_config; - if (cmdline->print_version) { - printf("Python %s\n", - (cmdline->print_version >= 2) ? Py_GetVersion() : PY_VERSION); - return 1; - } + _PyCoreConfig_GetGlobalConfig(config); - /* For Py_GetArgcArgv(). Cleared by pymain_free(). */ - orig_argv = _Py_wstrlist_copy(pymain->argc, cmdline->argv); - if (orig_argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; + err = _PyCoreConfig_ReadFromArgv(config, args, preconfig); + if (_Py_INIT_FAILED(err)) { + goto done; } - orig_argc = pymain->argc; - return 0; -} - -/* Read the configuration into _PyCoreConfig and _PyMain, initialize the - LC_CTYPE locale and Py_DecodeLocale(). - - Configuration: - - * Command line arguments - * Environment variables - * Py_xxx global configuration variables - - _PyCmdline is a temporary structure used to prioritize these - variables. */ -static int -pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) -{ - /* Force default allocator, since pymain_free() and pymain_clear_config() - must use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -#ifdef Py_DEBUG - PyMemAllocatorEx default_alloc; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &default_alloc); -#endif + _PyCoreConfig_SetGlobalConfig(config); - _PyCmdline cmdline; - memset(&cmdline, 0, sizeof(cmdline)); + config_init_stdio(config); - int res = pymain_cmdline_impl(pymain, config, &cmdline); + err = _Py_InitializeCore(interp_p, config); + if (_Py_INIT_FAILED(err)) { + goto done; + } - pymain_clear_cmdline(pymain, &cmdline); + err = config_init_python_main(*interp_p); + if (_Py_INIT_FAILED(err)) { + goto done; + } + err = _Py_INIT_OK(); -#ifdef Py_DEBUG - /* Make sure that PYMEM_DOMAIN_RAW has not been modified */ - PyMemAllocatorEx cur_alloc; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &cur_alloc); - assert(memcmp(&cur_alloc, &default_alloc, sizeof(cur_alloc)) == 0); -#endif - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return res; +done: + _PyCoreConfig_Clear(config); + return err; } -static int -pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) +static _PyInitError +pymain_init(const _PyArgv *args, PyInterpreterState **interp_p) { + _PyInitError err; + /* 754 requires that FP exceptions run in "no stop" mode by default, * and until C vendors implement C99's ways to control FP exceptions, * Python requires non-stop mode. Alas, some platforms enable FP @@ -1787,59 +862,69 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) fedisableexcept(FE_OVERFLOW); #endif - _PyCoreConfig local_config = _PyCoreConfig_INIT; - _PyCoreConfig *config = &local_config; + err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } - _PyCoreConfig_GetGlobalConfig(config); + _PyPreConfig local_preconfig = _PyPreConfig_INIT; + _PyPreConfig *preconfig = &local_preconfig; - int cmd_res = pymain_cmdline(pymain, config); - if (cmd_res < 0) { - _Py_FatalInitError(pymain->err); - } - if (cmd_res == 1) { - pymain_clear_config(config); - return 1; + _PyPreConfig_GetGlobalConfig(preconfig); + + err = _PyPreConfig_InitFromArgv(preconfig, args); + if (_Py_INIT_FAILED(err)) { + goto done; } - _PyCoreConfig_SetGlobalConfig(config); + _PyPreConfig_SetGlobalConfig(preconfig); - pymain_init_stdio(pymain, config); + /* Don't change the memory allocator if Py_Initialize() has already been + called. */ + if (preconfig->allocator != NULL && !_PyRuntime.core_initialized) { + err = _PyPreConfig_SetAllocator(preconfig); + if (_Py_INIT_FAILED(err)) { + goto done; + } + } - PyInterpreterState *interp; - pymain->err = _Py_InitializeCore(&interp, config); - if (_Py_INIT_FAILED(pymain->err)) { - _Py_FatalInitError(pymain->err); + err = pymain_init_core(args, preconfig, interp_p); + if (_Py_INIT_FAILED(err)) { + goto done; } - *interp_p = interp; - pymain_clear_config(config); - config = &interp->core_config; + err = _Py_INIT_OK(); - if (pymain_init_python_main(pymain, config, interp) < 0) { - _Py_FatalInitError(pymain->err); - } - return 0; +done: + _PyPreConfig_Clear(preconfig); + return err; } static int -pymain_main(_PyMain *pymain) +pymain_main(const _PyArgv *args) { + _PyInitError err; + PyInterpreterState *interp; - int res = pymain_init(pymain, &interp); - if (res != 1) { - if (pymain_run_python(pymain, interp) < 0) { - _Py_FatalInitError(pymain->err); - } + err = pymain_init(args, &interp); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } - if (Py_FinalizeEx() < 0) { - /* Value unlikely to be confused with a non-error exit status or - other special meaning */ - pymain->status = 120; - } + int exitcode = 0; + err = pymain_run_python(&exitcode, interp); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + + if (Py_FinalizeEx() < 0) { + /* Value unlikely to be confused with a non-error exit status or + other special meaning */ + exitcode = 120; } - pymain_free(pymain); + pymain_free(); if (_Py_UnhandledKeyboardInterrupt) { /* https://bugs.python.org/issue1054041 - We need to exit via the @@ -1858,52 +943,37 @@ pymain_main(_PyMain *pymain) #ifdef MS_WINDOWS /* cmd.exe detects this, prints ^C, and offers to terminate. */ /* https://msdn.microsoft.com/en-us/library/cc704588.aspx */ - pymain->status = STATUS_CONTROL_C_EXIT; + exitcode = STATUS_CONTROL_C_EXIT; #else - pymain->status = SIGINT + 128; + exitcode = SIGINT + 128; #endif /* !MS_WINDOWS */ } - return pymain->status; + return exitcode; } int Py_Main(int argc, wchar_t **argv) { - _PyMain pymain = _PyMain_INIT; - pymain.use_bytes_argv = 0; - pymain.argc = argc; - pymain.wchar_argv = argv; - - return pymain_main(&pymain); + _PyArgv args = { + .argc = argc, + .use_bytes_argv = 0, + .bytes_argv = NULL, + .wchar_argv = argv}; + return pymain_main(&args); } int _Py_UnixMain(int argc, char **argv) { - _PyMain pymain = _PyMain_INIT; - pymain.use_bytes_argv = 1; - pymain.argc = argc; - pymain.bytes_argv = argv; - - return pymain_main(&pymain); -} - - -/* this is gonna seem *real weird*, but if you put some other code between - Py_Main() and Py_GetArgcArgv() you will need to adjust the test in the - while statement in Misc/gdbinit:ppystack */ - -/* Make the *original* argc/argv available to other modules. - This is rare, but it is needed by the secureware extension. */ - -void -Py_GetArgcArgv(int *argc, wchar_t ***argv) -{ - *argc = orig_argc; - *argv = orig_argv; + _PyArgv args = { + .argc = argc, + .use_bytes_argv = 1, + .bytes_argv = argv, + .wchar_argv = NULL}; + return pymain_main(&args); } #ifdef __cplusplus diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8141ce757412cbf..e059b134dde85ad 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3507,7 +3507,7 @@ PyUnicode_EncodeFSDefault(PyObject *unicode) PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); const _PyCoreConfig *config = &interp->core_config; #if defined(__APPLE__) - return _PyUnicode_AsUTF8String(unicode, config->filesystem_errors); + return _PyUnicode_AsUTF8String(unicode, config->preconfig.filesystem_errors); #else /* Bootstrap check: if the filesystem codec is implemented in Python, we cannot use it to encode and decode filenames before it is loaded. Load @@ -3516,12 +3516,12 @@ PyUnicode_EncodeFSDefault(PyObject *unicode) initialized and the Python codec is loaded. See initfsencoding(). */ if (interp->fscodec_initialized) { return PyUnicode_AsEncodedString(unicode, - config->filesystem_encoding, - config->filesystem_errors); + config->preconfig.filesystem_encoding, + config->preconfig.filesystem_errors); } else { return unicode_encode_locale(unicode, - config->filesystem_errors, 0); + config->preconfig.filesystem_errors, 0); } #endif } @@ -3731,7 +3731,7 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); const _PyCoreConfig *config = &interp->core_config; #if defined(__APPLE__) - return PyUnicode_DecodeUTF8Stateful(s, size, config->filesystem_errors, NULL); + return PyUnicode_DecodeUTF8Stateful(s, size, config->preconfig.filesystem_errors, NULL); #else /* Bootstrap check: if the filesystem codec is implemented in Python, we cannot use it to encode and decode filenames before it is loaded. Load @@ -3740,12 +3740,12 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) initialized and the Python codec is loaded. See initfsencoding(). */ if (interp->fscodec_initialized) { return PyUnicode_Decode(s, size, - config->filesystem_encoding, - config->filesystem_errors); + config->preconfig.filesystem_encoding, + config->preconfig.filesystem_errors); } else { return unicode_decode_locale(s, size, - config->filesystem_errors, 0); + config->preconfig.filesystem_errors, 0); } #endif } diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b1b9944d1e91903..992c84f8cc81eed 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -110,8 +110,8 @@ - + @@ -141,6 +141,7 @@ + @@ -424,6 +425,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 9dbd0669f76d9b6..df1ae5c3d78dfc1 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -81,7 +81,7 @@ Include - + Include @@ -174,6 +174,9 @@ Include + + Include + Include @@ -986,6 +989,9 @@ Python + + Python + Python diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 3f43757ecb1ff5e..e6c51a4a1dfbe82 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -77,9 +77,9 @@ main(int argc, char *argv[]) text[text_size] = '\0'; _PyCoreConfig config = _PyCoreConfig_INIT; + config.preconfig.use_environment = 0; config.user_site_directory = 0; config.site_import = 0; - config.use_environment = 0; config.program_name = L"./_freeze_importlib"; /* Don't install importlib, since it could execute outdated bytecode. */ config._install_importlib = 0; @@ -89,7 +89,7 @@ main(int argc, char *argv[]) /* No need to call _PyCoreConfig_Clear() since we didn't allocate any memory: program_name is a constant string. */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } sprintf(buf, "", name); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index e28d94c175e6a5f..81ccc6a53a23c76 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -139,6 +139,9 @@ static int test_forced_io_encoding(void) static int test_pre_initialization_api(void) { + /* the test doesn't support custom memory allocators */ + putenv("PYTHONMALLOC="); + /* Leading "./" ensures getpath.c can still find the standard library */ _Py_EMBED_PREINIT_CHECK("Checking Py_DecodeLocale\n"); wchar_t *program = Py_DecodeLocale("./spam", NULL); @@ -235,6 +238,9 @@ static void bpo20891_thread(void *lockp) static int test_bpo20891(void) { + /* the test doesn't support custom memory allocators */ + putenv("PYTHONMALLOC="); + /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread before calling PyEval_InitThreads() must not crash. PyGILState_Ensure() must call PyEval_InitThreads() for us in this case. */ @@ -437,7 +443,7 @@ static int test_init_from_config(void) config.hash_seed = 123; putenv("PYTHONMALLOC=malloc"); - config.allocator = "malloc_debug"; + config.preconfig.allocator = "malloc_debug"; /* dev_mode=1 is tested in test_init_dev_mode() */ @@ -461,7 +467,7 @@ static int test_init_from_config(void) putenv("PYTHONUTF8=0"); Py_UTF8Mode = 0; - config.utf8_mode = 1; + config.preconfig.utf8_mode = 1; putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix"); config.pycache_prefix = L"conf_pycache_prefix"; @@ -536,10 +542,10 @@ static int test_init_from_config(void) #ifdef MS_WINDOWS /* Py_SetStandardStreamEncoding() sets Py_LegacyWindowsStdioFlag to 1. Force it to 0 through the config. */ - config.legacy_windows_stdio = 0; + config.preconfig.legacy_windows_stdio = 0; #endif - config.stdio_encoding = "iso8859-1"; - config.stdio_errors = "replace"; + config.preconfig.stdio_encoding = "iso8859-1"; + config.preconfig.stdio_errors = "replace"; putenv("PYTHONNOUSERSITE="); Py_NoUserSiteDirectory = 0; @@ -553,7 +559,7 @@ static int test_init_from_config(void) _PyInitError err = _Py_InitializeFromConfig(&config); /* Don't call _PyCoreConfig_Clear() since all strings are static */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); @@ -607,18 +613,18 @@ static int test_init_isolated(void) _PyCoreConfig config = _PyCoreConfig_INIT; /* Set coerce_c_locale and utf8_mode to not depend on the locale */ - config.coerce_c_locale = 0; - config.utf8_mode = 0; + config.preconfig.coerce_c_locale = 0; + config.preconfig.utf8_mode = 0; /* Use path starting with "./" avoids a search along the PATH */ config.program_name = L"./_testembed"; Py_IsolatedFlag = 0; - config.isolated = 1; + config.preconfig.isolated = 1; test_init_env_putenvs(); _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); @@ -631,11 +637,11 @@ static int test_init_dev_mode(void) _PyCoreConfig config = _PyCoreConfig_INIT; putenv("PYTHONFAULTHANDLER="); putenv("PYTHONMALLOC="); - config.dev_mode = 1; + config.preconfig.dev_mode = 1; config.program_name = L"./_testembed"; _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 2fb4e3fd196ea68..3a6afdb44a155fd 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -1,17 +1,17 @@ #include "Python.h" +#include "osdefs.h" +#include "pycore_coreconfig.h" #include "pycore_fileutils.h" +#include "pycore_getopt.h" #include "pycore_pylifecycle.h" #include "pycore_pymem.h" #include "pycore_pathconfig.h" #include "pycore_pystate.h" -#include -#ifdef HAVE_LANGINFO_H -# include -#endif -#include /* setlocale() */ -#ifdef HAVE_LANGINFO_H -#include /* nl_langinfo(CODESET) */ +#ifdef MS_WINDOWS +# define WCSTOK wcstok_s +#else +# define WCSTOK wcstok #endif @@ -23,13 +23,6 @@ /* Global configuration variables */ -/* The filesystem encoding is chosen by config_init_fs_encoding(), - see also initfsencoding(). */ -const char *Py_FileSystemDefaultEncoding = NULL; -int Py_HasFileSystemDefaultEncoding = 0; -const char *Py_FileSystemDefaultEncodeErrors = NULL; -static int _Py_HasFileSystemDefaultEncodeErrors = 0; - /* UTF-8 mode (PEP 540): if equals to 1, use the UTF-8 encoding, and change stdin and stdout error handler to "surrogateescape". It is equal to -1 by default: unknown, will be set by Py_Main() */ @@ -55,6 +48,136 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif +/* For Py_GetArgcArgv(); set by _Py_SetArgcArgv() */ +static wchar_t **orig_argv = NULL; +static int orig_argc = 0; + + +void +_Py_wstrlist_clear(int len, wchar_t **list) +{ + for (int i=0; i < len; i++) { + PyMem_RawFree(list[i]); + } + PyMem_RawFree(list); +} + + +wchar_t** +_Py_wstrlist_copy(int len, wchar_t * const *list) +{ + assert((len > 0 && list != NULL) || len == 0); + size_t size = len * sizeof(list[0]); + wchar_t **list_copy = PyMem_RawMalloc(size); + if (list_copy == NULL) { + return NULL; + } + for (int i=0; i < len; i++) { + wchar_t* arg = _PyMem_RawWcsdup(list[i]); + if (arg == NULL) { + _Py_wstrlist_clear(i, list_copy); + return NULL; + } + list_copy[i] = arg; + } + return list_copy; +} + + +_PyInitError +_Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) +{ + if (*len == INT_MAX) { + /* len+1 would overflow */ + return _Py_INIT_NO_MEMORY(); + } + wchar_t *str2 = _PyMem_RawWcsdup(str); + if (str2 == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + size_t size = (*len + 1) * sizeof(list[0]); + wchar_t **list2 = (wchar_t **)PyMem_RawRealloc(*list, size); + if (list2 == NULL) { + PyMem_RawFree(str2); + return _Py_INIT_NO_MEMORY(); + } + list2[*len] = str2; + *list = list2; + (*len)++; + return _Py_INIT_OK(); +} + + +PyObject* +_Py_wstrlist_as_pylist(int len, wchar_t **list) +{ + assert(list != NULL || len < 1); + + PyObject *pylist = PyList_New(len); + if (pylist == NULL) { + return NULL; + } + + for (int i = 0; i < len; i++) { + PyObject *v = PyUnicode_FromWideChar(list[i], -1); + if (v == NULL) { + Py_DECREF(pylist); + return NULL; + } + PyList_SET_ITEM(pylist, i, v); + } + return pylist; +} + + +void +_Py_ClearArgcArgv(void) +{ + /* Force the allocator used by _Py_SetArgcArgv() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + _Py_wstrlist_clear(orig_argc, orig_argv); + orig_argc = 0; + orig_argv = NULL; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +static int +_Py_SetArgcArgv(int argc, wchar_t * const * argv) +{ + /* _Py_SetArgcArgv() is only called once */ + assert(orig_argv == NULL); + + int res = 0; + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + wchar_t **argv_copy = _Py_wstrlist_copy(argc, argv); + if (argv_copy == NULL) { + res = -1; + } + orig_argv = argv_copy; + orig_argc = argc; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return res; +} + + +/* Make the *original* argc/argv available to other modules. + This is rare, but it is needed by the secureware extension. */ +void +Py_GetArgcArgv(int *argc, wchar_t ***argv) +{ + *argc = orig_argc; + *argv = orig_argv; +} + + PyObject * _Py_GetGlobalVariablesAsDict(void) { @@ -127,164 +250,12 @@ _Py_GetGlobalVariablesAsDict(void) } -void -_Py_wstrlist_clear(int len, wchar_t **list) -{ - for (int i=0; i < len; i++) { - PyMem_RawFree(list[i]); - } - PyMem_RawFree(list); -} - - -wchar_t** -_Py_wstrlist_copy(int len, wchar_t **list) -{ - assert((len > 0 && list != NULL) || len == 0); - size_t size = len * sizeof(list[0]); - wchar_t **list_copy = PyMem_RawMalloc(size); - if (list_copy == NULL) { - return NULL; - } - for (int i=0; i < len; i++) { - wchar_t* arg = _PyMem_RawWcsdup(list[i]); - if (arg == NULL) { - _Py_wstrlist_clear(i, list_copy); - return NULL; - } - list_copy[i] = arg; - } - return list_copy; -} - - -void -_Py_ClearFileSystemEncoding(void) -{ - if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { - PyMem_RawFree((char*)Py_FileSystemDefaultEncoding); - Py_FileSystemDefaultEncoding = NULL; - } - if (!_Py_HasFileSystemDefaultEncodeErrors && Py_FileSystemDefaultEncodeErrors) { - PyMem_RawFree((char*)Py_FileSystemDefaultEncodeErrors); - Py_FileSystemDefaultEncodeErrors = NULL; - } -} - - -/* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors - global configuration variables. */ -int -_Py_SetFileSystemEncoding(const char *encoding, const char *errors) -{ - char *encoding2 = _PyMem_RawStrdup(encoding); - if (encoding2 == NULL) { - return -1; - } - - char *errors2 = _PyMem_RawStrdup(errors); - if (errors2 == NULL) { - PyMem_RawFree(encoding2); - return -1; - } - - _Py_ClearFileSystemEncoding(); - - Py_FileSystemDefaultEncoding = encoding2; - Py_HasFileSystemDefaultEncoding = 0; - - Py_FileSystemDefaultEncodeErrors = errors2; - _Py_HasFileSystemDefaultEncodeErrors = 0; - return 0; -} - - -/* Helper to allow an embedding application to override the normal - * mechanism that attempts to figure out an appropriate IO encoding - */ - -static char *_Py_StandardStreamEncoding = NULL; -static char *_Py_StandardStreamErrors = NULL; - -int -Py_SetStandardStreamEncoding(const char *encoding, const char *errors) -{ - if (Py_IsInitialized()) { - /* This is too late to have any effect */ - return -1; - } - - int res = 0; - - /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(), - but Py_Initialize() can change the allocator. Use a known allocator - to be able to release the memory later. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Can't call PyErr_NoMemory() on errors, as Python hasn't been - * initialised yet. - * - * However, the raw memory allocators are initialised appropriately - * as C static variables, so _PyMem_RawStrdup is OK even though - * Py_Initialize hasn't been called yet. - */ - if (encoding) { - _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding); - if (!_Py_StandardStreamEncoding) { - res = -2; - goto done; - } - } - if (errors) { - _Py_StandardStreamErrors = _PyMem_RawStrdup(errors); - if (!_Py_StandardStreamErrors) { - if (_Py_StandardStreamEncoding) { - PyMem_RawFree(_Py_StandardStreamEncoding); - } - res = -3; - goto done; - } - } -#ifdef MS_WINDOWS - if (_Py_StandardStreamEncoding) { - /* Overriding the stream encoding implies legacy streams */ - Py_LegacyWindowsStdioFlag = 1; - } -#endif - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - return res; -} - - -void -_Py_ClearStandardStreamEncoding(void) -{ - /* Use the same allocator than Py_SetStandardStreamEncoding() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* We won't need them anymore. */ - if (_Py_StandardStreamEncoding) { - PyMem_RawFree(_Py_StandardStreamEncoding); - _Py_StandardStreamEncoding = NULL; - } - if (_Py_StandardStreamErrors) { - PyMem_RawFree(_Py_StandardStreamErrors); - _Py_StandardStreamErrors = NULL; - } - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - /* Free memory allocated in config, but don't clear all attributes */ void _PyCoreConfig_Clear(_PyCoreConfig *config) { + _PyPreConfig_Clear(&config->preconfig); + #define CLEAR(ATTR) \ do { \ PyMem_RawFree(ATTR); \ @@ -320,10 +291,9 @@ _PyCoreConfig_Clear(_PyCoreConfig *config) #endif CLEAR(config->base_exec_prefix); - CLEAR(config->filesystem_encoding); - CLEAR(config->filesystem_errors); - CLEAR(config->stdio_encoding); - CLEAR(config->stdio_errors); + CLEAR(config->run_command); + CLEAR(config->run_module); + CLEAR(config->run_filename); #undef CLEAR #undef CLEAR_WSTRLIST } @@ -334,6 +304,10 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) { _PyCoreConfig_Clear(config); + if (_PyPreConfig_Copy(&config->preconfig, &config2->preconfig) < 0) { + return -1; + } + #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR #define COPY_STR_ATTR(ATTR) \ do { \ @@ -365,12 +339,9 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) } while (0) COPY_ATTR(install_signal_handlers); - COPY_ATTR(use_environment); COPY_ATTR(use_hash_seed); COPY_ATTR(hash_seed); COPY_ATTR(_install_importlib); - COPY_ATTR(allocator); - COPY_ATTR(dev_mode); COPY_ATTR(faulthandler); COPY_ATTR(tracemalloc); COPY_ATTR(import_time); @@ -379,10 +350,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(dump_refs); COPY_ATTR(malloc_stats); - COPY_ATTR(coerce_c_locale); - COPY_ATTR(coerce_c_locale_warn); - COPY_ATTR(utf8_mode); - COPY_WSTR_ATTR(pycache_prefix); COPY_WSTR_ATTR(module_search_path_env); COPY_WSTR_ATTR(home); @@ -403,7 +370,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) #endif COPY_WSTR_ATTR(base_exec_prefix); - COPY_ATTR(isolated); COPY_ATTR(site_import); COPY_ATTR(bytes_warning); COPY_ATTR(inspect); @@ -415,14 +381,12 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(quiet); COPY_ATTR(user_site_directory); COPY_ATTR(buffered_stdio); - COPY_STR_ATTR(filesystem_encoding); - COPY_STR_ATTR(filesystem_errors); - COPY_STR_ATTR(stdio_encoding); - COPY_STR_ATTR(stdio_errors); -#ifdef MS_WINDOWS - COPY_ATTR(legacy_windows_fs_encoding); - COPY_ATTR(legacy_windows_stdio); -#endif + + COPY_ATTR(skip_source_first_line); + COPY_WSTR_ATTR(run_command); + COPY_WSTR_ATTR(run_module); + COPY_WSTR_ATTR(run_filename); + COPY_ATTR(_check_hash_pycs_mode); COPY_ATTR(_frozen); @@ -437,19 +401,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) const char* _PyCoreConfig_GetEnv(const _PyCoreConfig *config, const char *name) { - assert(config->use_environment >= 0); - - if (!config->use_environment) { - return NULL; - } - - const char *var = getenv(name); - if (var && var[0] != '\0') { - return var; - } - else { - return NULL; - } + return _PyPreConfig_GetEnv(&config->preconfig, name); } @@ -458,9 +410,9 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, wchar_t **dest, wchar_t *wname, char *name) { - assert(config->use_environment >= 0); + assert(config->preconfig.use_environment >= 0); - if (!config->use_environment) { + if (!config->preconfig.use_environment) { *dest = NULL; return 0; } @@ -513,8 +465,8 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) config->ATTR = !(VALUE); \ } - COPY_FLAG(utf8_mode, Py_UTF8Mode); - COPY_FLAG(isolated, Py_IsolatedFlag); + _PyPreConfig_GetGlobalConfig(&config->preconfig); + COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); COPY_FLAG(interactive, Py_InteractiveFlag); @@ -522,13 +474,8 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) COPY_FLAG(parser_debug, Py_DebugFlag); COPY_FLAG(verbose, Py_VerboseFlag); COPY_FLAG(quiet, Py_QuietFlag); -#ifdef MS_WINDOWS - COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); - COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); -#endif COPY_FLAG(_frozen, Py_FrozenFlag); - COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag); COPY_NOT_FLAG(site_import, Py_NoSiteFlag); COPY_NOT_FLAG(write_bytecode, Py_DontWriteBytecodeFlag); @@ -552,8 +499,8 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) VAR = !config->ATTR; \ } - COPY_FLAG(utf8_mode, Py_UTF8Mode); - COPY_FLAG(isolated, Py_IsolatedFlag); + _PyPreConfig_SetGlobalConfig(&config->preconfig); + COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); COPY_FLAG(interactive, Py_InteractiveFlag); @@ -561,13 +508,8 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) COPY_FLAG(parser_debug, Py_DebugFlag); COPY_FLAG(verbose, Py_VerboseFlag); COPY_FLAG(quiet, Py_QuietFlag); -#ifdef MS_WINDOWS - COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); - COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); -#endif COPY_FLAG(_frozen, Py_FrozenFlag); - COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag); COPY_NOT_FLAG(site_import, Py_NoSiteFlag); COPY_NOT_FLAG(write_bytecode, Py_DontWriteBytecodeFlag); @@ -662,6 +604,7 @@ config_init_program_name(_PyCoreConfig *config) return _Py_INIT_OK(); } + static _PyInitError config_init_executable(_PyCoreConfig *config) { @@ -680,26 +623,11 @@ config_init_executable(_PyCoreConfig *config) return _Py_INIT_OK(); } + static const wchar_t* config_get_xoption(const _PyCoreConfig *config, wchar_t *name) { - int nxoption = config->nxoption; - wchar_t **xoptions = config->xoptions; - for (int i=0; i < nxoption; i++) { - wchar_t *option = xoptions[i]; - size_t len; - wchar_t *sep = wcschr(option, L'='); - if (sep != NULL) { - len = (sep - option); - } - else { - len = wcslen(option); - } - if (wcsncmp(option, name, len) == 0 && name[len] == L'\0') { - return option; - } - } - return NULL; + return _Py_GetXOption(config->nxoption, config->xoptions, name); } @@ -760,51 +688,8 @@ config_init_hash_seed(_PyCoreConfig *config) } -static _PyInitError -config_init_utf8_mode(_PyCoreConfig *config) -{ - const wchar_t *xopt = config_get_xoption(config, L"utf8"); - if (xopt) { - wchar_t *sep = wcschr(xopt, L'='); - if (sep) { - xopt = sep + 1; - if (wcscmp(xopt, L"1") == 0) { - config->utf8_mode = 1; - } - else if (wcscmp(xopt, L"0") == 0) { - config->utf8_mode = 0; - } - else { - return _Py_INIT_USER_ERR("invalid -X utf8 option value"); - } - } - else { - config->utf8_mode = 1; - } - return _Py_INIT_OK(); - } - - const char *opt = _PyCoreConfig_GetEnv(config, "PYTHONUTF8"); - if (opt) { - if (strcmp(opt, "1") == 0) { - config->utf8_mode = 1; - } - else if (strcmp(opt, "0") == 0) { - config->utf8_mode = 0; - } - else { - return _Py_INIT_USER_ERR("invalid PYTHONUTF8 environment " - "variable value"); - } - return _Py_INIT_OK(); - } - - return _Py_INIT_OK(); -} - - -static int -config_str_to_int(const char *str, int *result) +int +_Py_str_to_int(const char *str, int *result) { const char *endptr = str; errno = 0; @@ -839,62 +724,35 @@ config_wstr_to_int(const wchar_t *wstr, int *result) } -static void -get_env_flag(_PyCoreConfig *config, int *flag, const char *name) -{ - const char *var = _PyCoreConfig_GetEnv(config, name); - if (!var) { - return; - } - int value; - if (config_str_to_int(var, &value) < 0 || value < 0) { - /* PYTHONDEBUG=text and PYTHONDEBUG=-2 behave as PYTHONDEBUG=1 */ - value = 1; - } - if (*flag < value) { - *flag = value; - } -} - - static _PyInitError config_read_env_vars(_PyCoreConfig *config) { + _PyPreConfig *preconfig = &config->preconfig; + /* Get environment variables */ - get_env_flag(config, &config->parser_debug, "PYTHONDEBUG"); - get_env_flag(config, &config->verbose, "PYTHONVERBOSE"); - get_env_flag(config, &config->optimization_level, "PYTHONOPTIMIZE"); - get_env_flag(config, &config->inspect, "PYTHONINSPECT"); + _PyPreConfig_GetEnvFlag(preconfig, &config->parser_debug, "PYTHONDEBUG"); + _PyPreConfig_GetEnvFlag(preconfig, &config->verbose, "PYTHONVERBOSE"); + _PyPreConfig_GetEnvFlag(preconfig, &config->optimization_level, "PYTHONOPTIMIZE"); + _PyPreConfig_GetEnvFlag(preconfig, &config->inspect, "PYTHONINSPECT"); int dont_write_bytecode = 0; - get_env_flag(config, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); + _PyPreConfig_GetEnvFlag(preconfig, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); if (dont_write_bytecode) { config->write_bytecode = 0; } int no_user_site_directory = 0; - get_env_flag(config, &no_user_site_directory, "PYTHONNOUSERSITE"); + _PyPreConfig_GetEnvFlag(preconfig, &no_user_site_directory, "PYTHONNOUSERSITE"); if (no_user_site_directory) { config->user_site_directory = 0; } int unbuffered_stdio = 0; - get_env_flag(config, &unbuffered_stdio, "PYTHONUNBUFFERED"); + _PyPreConfig_GetEnvFlag(preconfig, &unbuffered_stdio, "PYTHONUNBUFFERED"); if (unbuffered_stdio) { config->buffered_stdio = 0; } -#ifdef MS_WINDOWS - get_env_flag(config, &config->legacy_windows_fs_encoding, - "PYTHONLEGACYWINDOWSFSENCODING"); - get_env_flag(config, &config->legacy_windows_stdio, - "PYTHONLEGACYWINDOWSSTDIO"); -#endif - - if (config->allocator == NULL) { - config->allocator = _PyCoreConfig_GetEnv(config, "PYTHONMALLOC"); - } - if (_PyCoreConfig_GetEnv(config, "PYTHONDUMPREFS")) { config->dump_refs = 1; } @@ -902,23 +760,6 @@ config_read_env_vars(_PyCoreConfig *config) config->malloc_stats = 1; } - const char *env = _PyCoreConfig_GetEnv(config, "PYTHONCOERCECLOCALE"); - if (env) { - if (strcmp(env, "0") == 0) { - if (config->coerce_c_locale < 0) { - config->coerce_c_locale = 0; - } - } - else if (strcmp(env, "warn") == 0) { - config->coerce_c_locale_warn = 1; - } - else { - if (config->coerce_c_locale < 0) { - config->coerce_c_locale = 1; - } - } - } - wchar_t *path; int res = _PyCoreConfig_GetEnvDup(config, &path, L"PYTHONPATH", "PYTHONPATH"); @@ -946,7 +787,7 @@ config_init_tracemalloc(_PyCoreConfig *config) const char *env = _PyCoreConfig_GetEnv(config, "PYTHONTRACEMALLOC"); if (env) { - if (!config_str_to_int(env, &nframe)) { + if (!_Py_str_to_int(env, &nframe)) { valid = (nframe >= 0); } else { @@ -1034,11 +875,6 @@ config_read_complex_options(_PyCoreConfig *config) || config_get_xoption(config, L"importtime")) { config->import_time = 1; } - if (config_get_xoption(config, L"dev" ) || - _PyCoreConfig_GetEnv(config, "PYTHONDEVMODE")) - { - config->dev_mode = 1; - } _PyInitError err; if (config->tracemalloc < 0) { @@ -1058,301 +894,71 @@ config_read_complex_options(_PyCoreConfig *config) } -static void -config_init_locale(_PyCoreConfig *config) -{ - /* Test also if coerce_c_locale equals 1: PYTHONCOERCECLOCALE=1 doesn't - imply that the C locale is always coerced. It is only coerced if - if the LC_CTYPE locale is "C". */ - if (config->coerce_c_locale != 0) { - /* The C locale enables the C locale coercion (PEP 538) */ - if (_Py_LegacyLocaleDetected()) { - config->coerce_c_locale = 1; - } - else { - config->coerce_c_locale = 0; - } - } - -#ifndef MS_WINDOWS - if (config->utf8_mode < 0) { - /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ - const char *ctype_loc = setlocale(LC_CTYPE, NULL); - if (ctype_loc != NULL - && (strcmp(ctype_loc, "C") == 0 - || strcmp(ctype_loc, "POSIX") == 0)) - { - config->utf8_mode = 1; - } - } -#endif -} - - -static const char * -get_stdio_errors(const _PyCoreConfig *config) -{ -#ifndef MS_WINDOWS - const char *loc = setlocale(LC_CTYPE, NULL); - if (loc != NULL) { - /* surrogateescape is the default in the legacy C and POSIX locales */ - if (strcmp(loc, "C") == 0 || strcmp(loc, "POSIX") == 0) { - return "surrogateescape"; - } - -#ifdef PY_COERCE_C_LOCALE - /* surrogateescape is the default in locale coercion target locales */ - if (_Py_IsLocaleCoercionTarget(loc)) { - return "surrogateescape"; - } -#endif - } - - return "strict"; -#else - /* On Windows, always use surrogateescape by default */ - return "surrogateescape"; -#endif -} - - -static _PyInitError -get_locale_encoding(char **locale_encoding) -{ -#ifdef MS_WINDOWS - char encoding[20]; - PyOS_snprintf(encoding, sizeof(encoding), "cp%d", GetACP()); -#elif defined(__ANDROID__) - const char *encoding = "UTF-8"; -#else - const char *encoding = nl_langinfo(CODESET); - if (!encoding || encoding[0] == '\0') { - return _Py_INIT_USER_ERR("failed to get the locale encoding: " - "nl_langinfo(CODESET) failed"); - } -#endif - *locale_encoding = _PyMem_RawStrdup(encoding); - if (*locale_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - return _Py_INIT_OK(); -} - +/* Read configuration settings from standard locations + * + * This function doesn't make any changes to the interpreter state - it + * merely populates any missing configuration settings. This allows an + * embedding application to completely override a config option by + * setting it before calling this function, or else modify the default + * setting before passing the fully populated config to Py_EndInitialization. + * + * More advanced selective initialization tricks are possible by calling + * this function multiple times with various preconfigured settings. + */ static _PyInitError -config_init_stdio_encoding(_PyCoreConfig *config) +_PyCoreConfig_ReadPreConfig(_PyCoreConfig *config) { - /* If Py_SetStandardStreamEncoding() have been called, use these - parameters. */ - if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) { - config->stdio_encoding = _PyMem_RawStrdup(_Py_StandardStreamEncoding); - if (config->stdio_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - - if (config->stdio_errors == NULL && _Py_StandardStreamErrors != NULL) { - config->stdio_errors = _PyMem_RawStrdup(_Py_StandardStreamErrors); - if (config->stdio_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - - if (config->stdio_encoding != NULL && config->stdio_errors != NULL) { - return _Py_INIT_OK(); - } - - /* PYTHONIOENCODING environment variable */ - const char *opt = _PyCoreConfig_GetEnv(config, "PYTHONIOENCODING"); - if (opt) { - char *pythonioencoding = _PyMem_RawStrdup(opt); - if (pythonioencoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - - char *err = strchr(pythonioencoding, ':'); - if (err) { - *err = '\0'; - err++; - if (!err[0]) { - err = NULL; - } - } - - /* Does PYTHONIOENCODING contain an encoding? */ - if (pythonioencoding[0]) { - if (config->stdio_encoding == NULL) { - config->stdio_encoding = _PyMem_RawStrdup(pythonioencoding); - if (config->stdio_encoding == NULL) { - PyMem_RawFree(pythonioencoding); - return _Py_INIT_NO_MEMORY(); - } - } - - /* If the encoding is set but not the error handler, - use "strict" error handler by default. - PYTHONIOENCODING=latin1 behaves as - PYTHONIOENCODING=latin1:strict. */ - if (!err) { - err = "strict"; - } - } - - if (config->stdio_errors == NULL && err != NULL) { - config->stdio_errors = _PyMem_RawStrdup(err); - if (config->stdio_errors == NULL) { - PyMem_RawFree(pythonioencoding); - return _Py_INIT_NO_MEMORY(); - } - } + _PyInitError err; + _PyPreConfig local_preconfig = _PyPreConfig_INIT; + _PyPreConfig_GetGlobalConfig(&local_preconfig); - PyMem_RawFree(pythonioencoding); + if (_PyPreConfig_Copy(&local_preconfig, &config->preconfig) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; } - /* UTF-8 Mode uses UTF-8/surrogateescape */ - if (config->utf8_mode) { - if (config->stdio_encoding == NULL) { - config->stdio_encoding = _PyMem_RawStrdup("utf-8"); - if (config->stdio_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - if (config->stdio_errors == NULL) { - config->stdio_errors = _PyMem_RawStrdup("surrogateescape"); - if (config->stdio_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } + err = _PyPreConfig_Read(&local_preconfig); + if (_Py_INIT_FAILED(err)) { + goto done; } - /* Choose the default error handler based on the current locale. */ - if (config->stdio_encoding == NULL) { - _PyInitError err = get_locale_encoding(&config->stdio_encoding); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - if (config->stdio_errors == NULL) { - const char *errors = get_stdio_errors(config); - config->stdio_errors = _PyMem_RawStrdup(errors); - if (config->stdio_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } + if (_PyPreConfig_Copy(&config->preconfig, &local_preconfig) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; } + err = _Py_INIT_OK(); - return _Py_INIT_OK(); +done: + _PyPreConfig_Clear(&local_preconfig); + return err; } - -static _PyInitError -config_init_fs_encoding(_PyCoreConfig *config) +_PyInitError +_PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig) { -#ifdef MS_WINDOWS - if (config->legacy_windows_fs_encoding) { - /* Legacy Windows filesystem encoding: mbcs/replace */ - if (config->filesystem_encoding == NULL) { - config->filesystem_encoding = _PyMem_RawStrdup("mbcs"); - if (config->filesystem_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - if (config->filesystem_errors == NULL) { - config->filesystem_errors = _PyMem_RawStrdup("replace"); - if (config->filesystem_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - } - - /* Windows defaults to utf-8/surrogatepass (PEP 529). - - Note: UTF-8 Mode takes the same code path and the Legacy Windows FS - encoding has the priortiy over UTF-8 Mode. */ - if (config->filesystem_encoding == NULL) { - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); - if (config->filesystem_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } + _PyInitError err; - if (config->filesystem_errors == NULL) { - config->filesystem_errors = _PyMem_RawStrdup("surrogatepass"); - if (config->filesystem_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } -#else - if (config->filesystem_encoding == NULL) { - if (config->utf8_mode) { - /* UTF-8 Mode use: utf-8/surrogateescape */ - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); - /* errors defaults to surrogateescape above */ - } - else if (_Py_GetForceASCII()) { - config->filesystem_encoding = _PyMem_RawStrdup("ascii"); - } - else { - /* macOS and Android use UTF-8, - other platforms use the locale encoding. */ -#if defined(__APPLE__) || defined(__ANDROID__) - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); -#else - _PyInitError err = get_locale_encoding(&config->filesystem_encoding); - if (_Py_INIT_FAILED(err)) { - return err; - } -#endif - } + _PyCoreConfig_GetGlobalConfig(config); - if (config->filesystem_encoding == NULL) { + if (preconfig) { + if (_PyPreConfig_Copy(&config->preconfig, preconfig) < 0) { return _Py_INIT_NO_MEMORY(); } } - - if (config->filesystem_errors == NULL) { - /* by default, use the "surrogateescape" error handler */ - config->filesystem_errors = _PyMem_RawStrdup("surrogateescape"); - if (config->filesystem_errors == NULL) { - return _Py_INIT_NO_MEMORY(); + else { + err = _PyCoreConfig_ReadPreConfig(config); + if (_Py_INIT_FAILED(err)) { + return err; } } -#endif - return _Py_INIT_OK(); -} -/* Read configuration settings from standard locations - * - * This function doesn't make any changes to the interpreter state - it - * merely populates any missing configuration settings. This allows an - * embedding application to completely override a config option by - * setting it before calling this function, or else modify the default - * setting before passing the fully populated config to Py_EndInitialization. - * - * More advanced selective initialization tricks are possible by calling - * this function multiple times with various preconfigured settings. - */ - -_PyInitError -_PyCoreConfig_Read(_PyCoreConfig *config) -{ - _PyInitError err; - - _PyCoreConfig_GetGlobalConfig(config); - assert(config->use_environment >= 0); - - if (config->isolated > 0) { - config->use_environment = 0; + if (config->preconfig.isolated > 0) { config->user_site_directory = 0; } -#ifdef MS_WINDOWS - if (config->legacy_windows_fs_encoding) { - config->utf8_mode = 0; - } -#endif - - if (config->use_environment) { + if (config->preconfig.use_environment) { err = config_read_env_vars(config); if (_Py_INIT_FAILED(err)) { return err; @@ -1372,13 +978,6 @@ _PyCoreConfig_Read(_PyCoreConfig *config) return err; } - if (config->utf8_mode < 0) { - err = config_init_utf8_mode(config); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - if (config->home == NULL) { err = config_init_home(config); if (_Py_INIT_FAILED(err)) { @@ -1400,10 +999,6 @@ _PyCoreConfig_Read(_PyCoreConfig *config) } } - if (config->coerce_c_locale != 0 || config->utf8_mode < 0) { - config_init_locale(config); - } - if (config->_install_importlib) { err = _PyCoreConfig_InitPathConfig(config); if (_Py_INIT_FAILED(err)) { @@ -1412,13 +1007,10 @@ _PyCoreConfig_Read(_PyCoreConfig *config) } /* default values */ - if (config->dev_mode) { + if (config->preconfig.dev_mode) { if (config->faulthandler < 0) { config->faulthandler = 1; } - if (config->allocator == NULL) { - config->allocator = "debug"; - } } if (config->use_hash_seed < 0) { config->use_hash_seed = 0; @@ -1430,55 +1022,21 @@ _PyCoreConfig_Read(_PyCoreConfig *config) if (config->tracemalloc < 0) { config->tracemalloc = 0; } - if (config->coerce_c_locale < 0) { - config->coerce_c_locale = 0; - } - if (config->utf8_mode < 0) { - config->utf8_mode = 0; - } if (config->argc < 0) { config->argc = 0; } - if (config->filesystem_encoding == NULL || config->filesystem_errors == NULL) { - err = config_init_fs_encoding(config); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - - err = config_init_stdio_encoding(config); - if (_Py_INIT_FAILED(err)) { - return err; - } - - assert(config->coerce_c_locale >= 0); - assert(config->use_environment >= 0); - assert(config->filesystem_encoding != NULL); - assert(config->filesystem_errors != NULL); - assert(config->stdio_encoding != NULL); - assert(config->stdio_errors != NULL); assert(config->_check_hash_pycs_mode != NULL); return _Py_INIT_OK(); } -PyObject * -_PyCoreConfig_AsDict(const _PyCoreConfig *config) -{ - PyObject *dict, *obj; - - dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - #define SET_ITEM(KEY, EXPR) \ do { \ - obj = (EXPR); \ + PyObject *obj = (EXPR); \ if (obj == NULL) { \ - return NULL; \ + goto fail; \ } \ int res = PyDict_SetItemString(dict, (KEY), obj); \ Py_DECREF(obj); \ @@ -1505,12 +1063,49 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) #define SET_ITEM_WSTRLIST(NOPTION, OPTIONS) \ SET_ITEM(#OPTIONS, _Py_wstrlist_as_pylist(config->NOPTION, config->OPTIONS)) - SET_ITEM_INT(install_signal_handlers); +static int +_PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict) +{ + SET_ITEM_STR(allocator); + SET_ITEM_INT(dev_mode); SET_ITEM_INT(use_environment); + SET_ITEM_INT(isolated); + SET_ITEM_INT(utf8_mode); + SET_ITEM_INT(coerce_c_locale); + SET_ITEM_INT(coerce_c_locale_warn); + SET_ITEM_STR(filesystem_encoding); + SET_ITEM_STR(filesystem_errors); + SET_ITEM_STR(stdio_encoding); + SET_ITEM_STR(stdio_errors); +#ifdef MS_WINDOWS + SET_ITEM_INT(legacy_windows_fs_encoding); + SET_ITEM_INT(legacy_windows_stdio); +#endif + return 0; + +fail: + return -1; +} + + +PyObject * +_PyCoreConfig_AsDict(const _PyCoreConfig *config) +{ + PyObject *dict; + + dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + + if (_PyPreConfig_AsDict(&config->preconfig, dict) < 0) { + Py_DECREF(dict); + return NULL; + } + + SET_ITEM_INT(install_signal_handlers); SET_ITEM_INT(use_hash_seed); SET_ITEM_UINT(hash_seed); - SET_ITEM_STR(allocator); - SET_ITEM_INT(dev_mode); SET_ITEM_INT(faulthandler); SET_ITEM_INT(tracemalloc); SET_ITEM_INT(import_time); @@ -1518,11 +1113,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_INT(show_alloc_count); SET_ITEM_INT(dump_refs); SET_ITEM_INT(malloc_stats); - SET_ITEM_INT(coerce_c_locale); - SET_ITEM_INT(coerce_c_locale_warn); - SET_ITEM_STR(filesystem_encoding); - SET_ITEM_STR(filesystem_errors); - SET_ITEM_INT(utf8_mode); SET_ITEM_WSTR(pycache_prefix); SET_ITEM_WSTR(program_name); SET_ITEM_WSTRLIST(argc, argv); @@ -1540,7 +1130,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) #ifdef MS_WINDOWS SET_ITEM_WSTR(dll_path); #endif - SET_ITEM_INT(isolated); SET_ITEM_INT(site_import); SET_ITEM_INT(bytes_warning); SET_ITEM_INT(inspect); @@ -1552,12 +1141,10 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_INT(quiet); SET_ITEM_INT(user_site_directory); SET_ITEM_INT(buffered_stdio); - SET_ITEM_STR(stdio_encoding); - SET_ITEM_STR(stdio_errors); -#ifdef MS_WINDOWS - SET_ITEM_INT(legacy_windows_fs_encoding); - SET_ITEM_INT(legacy_windows_stdio); -#endif + SET_ITEM_INT(skip_source_first_line); + SET_ITEM_WSTR(run_command); + SET_ITEM_WSTR(run_module); + SET_ITEM_WSTR(run_filename); SET_ITEM_INT(_install_importlib); SET_ITEM_STR(_check_hash_pycs_mode); SET_ITEM_INT(_frozen); @@ -1567,6 +1154,7 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) fail: Py_DECREF(dict); return NULL; +} #undef FROM_STRING #undef FROM_WSTRING @@ -1576,4 +1164,591 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) #undef SET_ITEM_STR #undef SET_ITEM_WSTR #undef SET_ITEM_WSTRLIST + + +typedef struct { + int argc; + wchar_t **argv; + int nwarnoption; /* Number of -W command line options */ + wchar_t **warnoptions; /* Command line -W options */ + int nenv_warnoption; /* Number of PYTHONWARNINGS environment variables */ + wchar_t **env_warnoptions; /* PYTHONWARNINGS environment variables */ + int print_help; /* -h, -? options */ + int print_version; /* -V option */ +} _PyCmdline; + + +static void +cmdline_clear(_PyCmdline *cmdline) +{ + _Py_wstrlist_clear(cmdline->argc, cmdline->argv); + cmdline->argc = 0; + cmdline->argv = NULL; + + _Py_wstrlist_clear(cmdline->nwarnoption, cmdline->warnoptions); + cmdline->nwarnoption = 0; + cmdline->warnoptions = NULL; + + _Py_wstrlist_clear(cmdline->nenv_warnoption, cmdline->env_warnoptions); + cmdline->nenv_warnoption = 0; + cmdline->env_warnoptions = NULL; +} + + +#if defined(MS_WINDOWS) +# define PYTHONHOMEHELP "\\python{major}{minor}" +#else +# define PYTHONHOMEHELP "/lib/pythonX.X" +#endif + +/* Short usage message (with %s for argv0) */ +static const char usage_line[] = +"usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; + +/* Long usage message, split into parts < 512 bytes */ +static const char usage_1[] = "\ +Options and arguments (and corresponding environment variables):\n\ +-b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ + and comparing bytes/bytearray with str. (-bb: issue errors)\n\ +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ +-c cmd : program passed in as string (terminates option list)\n\ +-d : debug output from parser; also PYTHONDEBUG=x\n\ +-E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ +-h : print this help message and exit (also --help)\n\ +"; +static const char usage_2[] = "\ +-i : inspect interactively after running script; forces a prompt even\n\ + if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ +-I : isolate Python from the user's environment (implies -E and -s)\n\ +-m mod : run library module as a script (terminates option list)\n\ +-O : remove assert and __debug__-dependent statements; add .opt-1 before\n\ + .pyc extension; also PYTHONOPTIMIZE=x\n\ +-OO : do -O changes and also discard docstrings; add .opt-2 before\n\ + .pyc extension\n\ +-q : don't print version and copyright messages on interactive startup\n\ +-s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ +-S : don't imply 'import site' on initialization\n\ +"; +static const char usage_3[] = "\ +-u : force the stdout and stderr streams to be unbuffered;\n\ + this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ +-v : verbose (trace import statements); also PYTHONVERBOSE=x\n\ + can be supplied multiple times to increase verbosity\n\ +-V : print the Python version number and exit (also --version)\n\ + when given twice, print more information about the build\n\ +-W arg : warning control; arg is action:message:category:module:lineno\n\ + also PYTHONWARNINGS=arg\n\ +-x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ +-X opt : set implementation-specific option\n\ +--check-hash-based-pycs always|default|never:\n\ + control how Python invalidates hash-based .pyc files\n\ +"; +static const char usage_4[] = "\ +file : program read from script file\n\ +- : program read from stdin (default; interactive mode if a tty)\n\ +arg ...: arguments passed to program in sys.argv[1:]\n\n\ +Other environment variables:\n\ +PYTHONSTARTUP: file executed on interactive startup (no default)\n\ +PYTHONPATH : '%lc'-separated list of directories prefixed to the\n\ + default module search path. The result is sys.path.\n\ +"; +static const char usage_5[] = +"PYTHONHOME : alternate directory (or %lc).\n" +" The default module search path uses %s.\n" +"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" +"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n"; +static const char usage_6[] = +"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" +" to seed the hashes of str, bytes and datetime objects. It can also be\n" +" set to an integer in the range [0,4294967295] to get hash values with a\n" +" predictable seed.\n" +"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" +" hooks.\n" +"PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" +" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" +" locale coercion and locale compatibility warnings on stderr.\n" +"PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" +" debugger. It can be set to the callable of your debugger of choice.\n" +"PYTHONDEVMODE: enable the development mode.\n" +"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n"; + + +static void +python_cmdline_usage(const wchar_t* program, int error) +{ + FILE *f = error ? stderr : stdout; + + fprintf(f, usage_line, program); + if (error) + fprintf(f, "Try `python -h' for more information.\n"); + else { + fputs(usage_1, f); + fputs(usage_2, f); + fputs(usage_3, f); + fprintf(f, usage_4, (wint_t)DELIM); + fprintf(f, usage_5, (wint_t)DELIM, PYTHONHOMEHELP); + fputs(usage_6, f); + } +} + + +/* Parse the command line arguments. */ +static _PyInitError +config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline, int *need_usage) +{ + _PyInitError err; + _PyOS_ResetGetOpt(); + do { + int longindex = -1; + int c = _PyOS_GetOpt(cmdline->argc, cmdline->argv, &longindex); + if (c == EOF) { + break; + } + + if (c == 'c') { + /* -c is the last option; following arguments + that look like options are left for the + command to interpret. */ + size_t len = wcslen(_PyOS_optarg) + 1 + 1; + wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); + if (command == NULL) { + return _Py_INIT_NO_MEMORY(); + } + memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); + command[len - 2] = '\n'; + command[len - 1] = 0; + config->run_command = command; + break; + } + + if (c == 'm') { + /* -m is the last option; following arguments + that look like options are left for the + module to interpret. */ + config->run_module = _PyMem_RawWcsdup(_PyOS_optarg); + if (config->run_module == NULL) { + return _Py_INIT_NO_MEMORY(); + } + break; + } + + switch (c) { + case 0: + // Handle long option. + assert(longindex == 0); // Only one long option now. + if (!wcscmp(_PyOS_optarg, L"always")) { + config->_check_hash_pycs_mode = "always"; + } else if (!wcscmp(_PyOS_optarg, L"never")) { + config->_check_hash_pycs_mode = "never"; + } else if (!wcscmp(_PyOS_optarg, L"default")) { + config->_check_hash_pycs_mode = "default"; + } else { + fprintf(stderr, "--check-hash-based-pycs must be one of " + "'default', 'always', or 'never'\n"); + *need_usage = 1; + return _Py_INIT_OK(); + } + break; + + case 'I': + case 'E': + /* arguments parsed by preconfig_parse_cmdline() */ + break; + + case 'b': + config->bytes_warning++; + break; + + case 'd': + config->parser_debug++; + break; + + case 'i': + config->inspect++; + config->interactive++; + break; + + /* case 'J': reserved for Jython */ + + case 'O': + config->optimization_level++; + break; + + case 'B': + config->write_bytecode = 0; + break; + + case 's': + config->user_site_directory = 0; + break; + + case 'S': + config->site_import = 0; + break; + + case 't': + /* ignored for backwards compatibility */ + break; + + case 'u': + config->buffered_stdio = 0; + break; + + case 'v': + config->verbose++; + break; + + case 'x': + config->skip_source_first_line = 1; + break; + + case 'h': + case '?': + cmdline->print_help++; + break; + + case 'V': + cmdline->print_version++; + break; + + case 'W': + err = _Py_wstrlist_append(&cmdline->nwarnoption, + &cmdline->warnoptions, + _PyOS_optarg); + if (_Py_INIT_FAILED(err)) { + return err; + } + break; + + case 'X': + err = _Py_wstrlist_append(&config->nxoption, + &config->xoptions, + _PyOS_optarg); + if (_Py_INIT_FAILED(err)) { + return err; + } + break; + + case 'q': + config->quiet++; + break; + + case 'R': + config->use_hash_seed = 0; + break; + + /* This space reserved for other options */ + + default: + /* unknown argument: parsing failed */ + *need_usage = 1; + return _Py_INIT_OK(); + } + } while (1); + + if (config->run_command == NULL && config->run_module == NULL + && _PyOS_optind < cmdline->argc + && wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0) + { + config->run_filename = _PyMem_RawWcsdup(cmdline->argv[_PyOS_optind]); + if (config->run_filename == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->run_command != NULL || config->run_module != NULL) { + /* Backup _PyOS_optind */ + _PyOS_optind--; + } + + /* -c and -m options are exclusive */ + assert(!(config->run_command != NULL && config->run_module != NULL)); + return _Py_INIT_OK(); +} + + +static _PyInitError +config_init_argv(_PyCoreConfig *config, _PyCmdline *cmdline) +{ + /* Copy argv to be able to modify it (to force -c/-m) */ + int argc = cmdline->argc - _PyOS_optind; + wchar_t **argv; + + if (argc <= 0 || cmdline->argv == NULL) { + /* Ensure at least one (empty) argument is seen */ + static wchar_t *empty_argv[1] = {L""}; + argc = 1; + argv = _Py_wstrlist_copy(1, empty_argv); + } + else { + argv = _Py_wstrlist_copy(argc, &cmdline->argv[_PyOS_optind]); + } + + if (argv == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + wchar_t *arg0 = NULL; + if (config->run_command != NULL) { + /* Force sys.argv[0] = '-c' */ + arg0 = L"-c"; + } + else if (config->run_module != NULL) { + /* Force sys.argv[0] = '-m'*/ + arg0 = L"-m"; + } + if (arg0 != NULL) { + arg0 = _PyMem_RawWcsdup(arg0); + if (arg0 == NULL) { + _Py_wstrlist_clear(argc, argv); + return _Py_INIT_NO_MEMORY(); + } + + assert(argc >= 1); + PyMem_RawFree(argv[0]); + argv[0] = arg0; + } + + config->argc = argc; + config->argv = argv; + return _Py_INIT_OK(); +} + + +static _PyInitError +config_init_program(_PyCoreConfig *config, _PyCmdline *cmdline) +{ + const wchar_t *program = _Py_GetProgramFromArgv(cmdline->argc, cmdline->argv); + config->program = _PyMem_RawWcsdup(program); + if (config->program == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +/* Get warning options from PYTHONWARNINGS environment variable. + Return 0 on success. + Set pymain->err and return -1 on error. */ +static _PyInitError +cmdline_init_env_warnoptions(const _PyCoreConfig *config, _PyCmdline *cmdline) +{ + wchar_t *env; + int res = _PyCoreConfig_GetEnvDup(config, &env, + L"PYTHONWARNINGS", "PYTHONWARNINGS"); + if (res < 0) { + return DECODE_LOCALE_ERR("PYTHONWARNINGS", res); + } + + if (env == NULL) { + return _Py_INIT_OK(); + } + + + wchar_t *warning, *context = NULL; + for (warning = WCSTOK(env, L",", &context); + warning != NULL; + warning = WCSTOK(NULL, L",", &context)) + { + _PyInitError err = _Py_wstrlist_append(&cmdline->nenv_warnoption, + &cmdline->env_warnoptions, + warning); + if (_Py_INIT_FAILED(err)) { + PyMem_RawFree(env); + return err; + } + } + PyMem_RawFree(env); + return _Py_INIT_OK(); +} + + +static _PyInitError +config_add_warnings_optlist(_PyCoreConfig *config, int len, wchar_t **options) +{ + for (int i = 0; i < len; i++) { + _PyInitError err = _Py_wstrlist_append(&config->nwarnoption, + &config->warnoptions, + options[i]); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + return _Py_INIT_OK(); +} + + +static _PyInitError +config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) +{ + _PyInitError err; + + assert(config->nwarnoption == 0); + + /* The priority order for warnings configuration is (highest precedence + * first): + * + * - the BytesWarning filter, if needed ('-b', '-bb') + * - any '-W' command line options; then + * - the 'PYTHONWARNINGS' environment variable; then + * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then + * - any implicit filters added by _warnings.c/warnings.py + * + * All settings except the last are passed to the warnings module via + * the `sys.warnoptions` list. Since the warnings module works on the basis + * of "the most recently added filter will be checked first", we add + * the lowest precedence entries first so that later entries override them. + */ + + if (config->preconfig.dev_mode) { + err = _Py_wstrlist_append(&config->nwarnoption, + &config->warnoptions, + L"default"); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + err = config_add_warnings_optlist(config, + cmdline->nenv_warnoption, + cmdline->env_warnoptions); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = config_add_warnings_optlist(config, + cmdline->nwarnoption, + cmdline->warnoptions); + if (_Py_INIT_FAILED(err)) { + return err; + } + + /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c + * don't even try to emit a warning, so we skip setting the filter in that + * case. + */ + if (config->bytes_warning) { + wchar_t *filter; + if (config->bytes_warning> 1) { + filter = L"error::BytesWarning"; + } + else { + filter = L"default::BytesWarning"; + } + err = _Py_wstrlist_append(&config->nwarnoption, + &config->warnoptions, + filter); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + return _Py_INIT_OK(); +} + + +static _PyInitError +config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline, + const _PyPreConfig *preconfig) +{ + _PyInitError err; + + int need_usage = 0; + err = config_parse_cmdline(config, cmdline, &need_usage); + if (_Py_INIT_FAILED(err)) { + return err; + } + if (need_usage) { + const wchar_t *program = _Py_GetProgramFromArgv(cmdline->argc, cmdline->argv); + python_cmdline_usage(program, 1); + return _Py_INIT_EXIT(2); + } + + if (cmdline->print_help) { + const wchar_t *program = _Py_GetProgramFromArgv(cmdline->argc, cmdline->argv); + python_cmdline_usage(program, 0); + return _Py_INIT_EXIT(0); + } + + if (cmdline->print_version) { + printf("Python %s\n", + (cmdline->print_version >= 2) ? Py_GetVersion() : PY_VERSION); + return _Py_INIT_EXIT(0); + } + + err = config_init_argv(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = config_init_program(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = _PyCoreConfig_Read(config, preconfig); + if (_Py_INIT_FAILED(err)) { + return err; + } + + if (config->preconfig.use_environment) { + err = cmdline_init_env_warnoptions(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + err = config_init_warnoptions(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + return _Py_INIT_OK(); +} + + +static _PyInitError +config_fromargv(_PyCoreConfig *config, const _PyArgv *args, + const _PyPreConfig *preconfig) +{ + _PyInitError err; + + _PyCmdline cmdline; + memset(&cmdline, 0, sizeof(cmdline)); + + err = _PyArgv_Decode(args, &cmdline.argc, &cmdline.argv); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = config_from_cmdline(config, &cmdline, preconfig); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + if (_Py_SetArgcArgv(cmdline.argc, cmdline.argv)) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + + err = _Py_INIT_OK(); + +done: + cmdline_clear(&cmdline); + return err; +} + + +/* Read the configuration into _PyCoreConfig. */ +_PyInitError +_PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, + const _PyArgv *args, + const _PyPreConfig *preconfig) +{ + _PyInitError err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } + + return config_fromargv(config, args, preconfig); } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 616090965b1351b..6554aa75b038f8a 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -86,7 +86,7 @@ Py_FrozenMain(int argc, char **argv) /* No need to call _PyCoreConfig_Clear() since we didn't allocate any memory: program_name is a constant string. */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } #ifdef MS_WINDOWS diff --git a/Python/getopt.c b/Python/getopt.c index c165a94a2d84ca7..e35a0ea8c8a60c8 100644 --- a/Python/getopt.c +++ b/Python/getopt.c @@ -37,6 +37,21 @@ extern "C" { #endif +/* Python command line options */ +#define OPTIONS L"bBc:dEhiIJm:OqRsStuvVW:xX:?" + +typedef struct { + const wchar_t *name; + int has_arg; + int val; +} _PyOS_LongOption; + +static const _PyOS_LongOption longopts[] = { + {L"check-hash-based-pycs", 1, 0}, + {NULL, 0, 0}, +}; + + int _PyOS_opterr = 1; /* generate error messages */ int _PyOS_optind = 1; /* index into argv array */ wchar_t *_PyOS_optarg = NULL; /* optional argument */ @@ -51,8 +66,7 @@ void _PyOS_ResetGetOpt(void) opt_ptr = L""; } -int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring, - const _PyOS_LongOption *longopts, int *longindex) +int _PyOS_GetOpt(int argc, wchar_t **argv, int *longindex) { wchar_t *ptr; wchar_t option; @@ -128,7 +142,7 @@ int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring, return '_'; } - if ((ptr = wcschr(optstring, option)) == NULL) { + if ((ptr = wcschr(OPTIONS, option)) == NULL) { if (_PyOS_opterr) fprintf(stderr, "Unknown option: -%c\n", (char)option); return '_'; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index c9bddcf6c674d6c..a1bed77b31d1b63 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -2,6 +2,7 @@ #include "Python.h" #include "osdefs.h" +#include "pycore_coreconfig.h" #include "pycore_fileutils.h" #include "pycore_pathconfig.h" #include "pycore_pymem.h" @@ -35,12 +36,6 @@ copy_wstr(wchar_t **dst, const wchar_t *src) static void _PyPathConfig_Clear(_PyPathConfig *config) { - /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, - since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be - called before Py_Initialize() which can changes the memory allocator. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - #define CLEAR(ATTR) \ do { \ PyMem_RawFree(ATTR); \ @@ -57,8 +52,6 @@ _PyPathConfig_Clear(_PyPathConfig *config) CLEAR(config->home); CLEAR(config->program_name); #undef CLEAR - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -70,37 +63,29 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, _PyInitError err; _PyPathConfig new_config = _PyPathConfig_INIT; - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - /* Calculate program_full_path, prefix, exec_prefix, dll_path (Windows), and module_search_path */ err = _PyPathConfig_Calculate_impl(&new_config, core_config); if (_Py_INIT_FAILED(err)) { - goto err; + goto fail; } /* Copy home and program_name from core_config */ if (copy_wstr(&new_config.home, core_config->home) < 0) { err = _Py_INIT_NO_MEMORY(); - goto err; + goto fail; } if (copy_wstr(&new_config.program_name, core_config->program_name) < 0) { err = _Py_INIT_NO_MEMORY(); - goto err; + goto fail; } _PyPathConfig_Clear(path_config); *path_config = new_config; + return _Py_INIT_OK(); - err = _Py_INIT_OK(); - goto done; - -err: +fail: _PyPathConfig_Clear(&new_config); - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return err; } @@ -148,7 +133,15 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) void _PyPathConfig_ClearGlobal(void) { + /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, + since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be + called before Py_Initialize() which can changes the memory allocator. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyPathConfig_Clear(&_Py_path_config); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -187,9 +180,6 @@ wstrlist_join(wchar_t sep, int count, wchar_t **list) _PyInitError _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) { - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyInitError err; _PyPathConfig path_config = _PyPathConfig_INIT; @@ -234,7 +224,6 @@ _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) done: _PyPathConfig_Clear(&path_config); - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return err; } @@ -329,7 +318,7 @@ _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) #endif if (path_config.isolated != -1) { - config->isolated = path_config.isolated; + config->preconfig.isolated = path_config.isolated; } if (path_config.site_import != -1) { config->site_import = path_config.site_import; @@ -392,7 +381,7 @@ pathconfig_global_init(void) _PyInitError err; _PyCoreConfig config = _PyCoreConfig_INIT; - err = _PyCoreConfig_Read(&config); + err = _PyCoreConfig_Read(&config, NULL); if (_Py_INIT_FAILED(err)) { goto error; } @@ -407,7 +396,7 @@ pathconfig_global_init(void) error: _PyCoreConfig_Clear(&config); - _Py_FatalInitError(err); + _Py_ExitInitError(err); } @@ -417,7 +406,7 @@ void Py_SetPath(const wchar_t *path) { if (path == NULL) { - _PyPathConfig_Clear(&_Py_path_config); + _PyPathConfig_ClearGlobal(); return; } diff --git a/Python/preconfig.c b/Python/preconfig.c new file mode 100644 index 000000000000000..0d0a0d5044f743d --- /dev/null +++ b/Python/preconfig.c @@ -0,0 +1,1084 @@ +#include +#include "pycore_coreconfig.h" +#include "pycore_fileutils.h" +#include "pycore_getopt.h" +#include "pycore_pylifecycle.h" +#include "pycore_pystate.h" +#include +#ifdef HAVE_LANGINFO_H +# include +#endif + +#define DECODE_LOCALE_ERR(NAME, LEN) \ + (((LEN) == -2) \ + ? _Py_INIT_USER_ERR("cannot decode " NAME) \ + : _Py_INIT_NO_MEMORY()) + + +/* The filesystem encoding is chosen by config_init_fs_encoding(), + see also initfsencoding(). */ +const char *Py_FileSystemDefaultEncoding = NULL; +int Py_HasFileSystemDefaultEncoding = 0; +const char *Py_FileSystemDefaultEncodeErrors = NULL; +int _Py_HasFileSystemDefaultEncodeErrors = 0; + + +void +_Py_ClearFileSystemEncoding(void) +{ + if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { + PyMem_RawFree((char*)Py_FileSystemDefaultEncoding); + Py_FileSystemDefaultEncoding = NULL; + } + if (!_Py_HasFileSystemDefaultEncodeErrors && Py_FileSystemDefaultEncodeErrors) { + PyMem_RawFree((char*)Py_FileSystemDefaultEncodeErrors); + Py_FileSystemDefaultEncodeErrors = NULL; + } +} + + +/* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors + global configuration variables. */ +int +_Py_SetFileSystemEncoding(const char *encoding, const char *errors) +{ + char *encoding2 = _PyMem_RawStrdup(encoding); + if (encoding2 == NULL) { + return -1; + } + + char *errors2 = _PyMem_RawStrdup(errors); + if (errors2 == NULL) { + PyMem_RawFree(encoding2); + return -1; + } + + _Py_ClearFileSystemEncoding(); + + Py_FileSystemDefaultEncoding = encoding2; + Py_HasFileSystemDefaultEncoding = 0; + + Py_FileSystemDefaultEncodeErrors = errors2; + _Py_HasFileSystemDefaultEncodeErrors = 0; + return 0; +} + + +/* Helper to allow an embedding application to override the normal + * mechanism that attempts to figure out an appropriate IO encoding + */ + +static char *_Py_StandardStreamEncoding = NULL; +static char *_Py_StandardStreamErrors = NULL; + + +int +Py_SetStandardStreamEncoding(const char *encoding, const char *errors) +{ + if (Py_IsInitialized()) { + /* This is too late to have any effect */ + return -1; + } + + int res = 0; + + /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(), + but Py_Initialize() can change the allocator. Use a known allocator + to be able to release the memory later. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Can't call PyErr_NoMemory() on errors, as Python hasn't been + * initialised yet. + * + * However, the raw memory allocators are initialised appropriately + * as C static variables, so _PyMem_RawStrdup is OK even though + * Py_Initialize hasn't been called yet. + */ + if (encoding) { + _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding); + if (!_Py_StandardStreamEncoding) { + res = -2; + goto done; + } + } + if (errors) { + _Py_StandardStreamErrors = _PyMem_RawStrdup(errors); + if (!_Py_StandardStreamErrors) { + if (_Py_StandardStreamEncoding) { + PyMem_RawFree(_Py_StandardStreamEncoding); + } + res = -3; + goto done; + } + } +#ifdef MS_WINDOWS + if (_Py_StandardStreamEncoding) { + /* Overriding the stream encoding implies legacy streams */ + Py_LegacyWindowsStdioFlag = 1; + } +#endif + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + return res; +} + + +void +_Py_ClearStandardStreamEncoding(void) +{ + /* Use the same allocator than Py_SetStandardStreamEncoding() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* We won't need them anymore. */ + if (_Py_StandardStreamEncoding) { + PyMem_RawFree(_Py_StandardStreamEncoding); + _Py_StandardStreamEncoding = NULL; + } + if (_Py_StandardStreamErrors) { + PyMem_RawFree(_Py_StandardStreamErrors); + _Py_StandardStreamErrors = NULL; + } + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +const wchar_t* +_Py_GetProgramFromArgv(int argc, wchar_t * const *argv) +{ + if (argc >= 1 && argv != NULL) { + return argv[0]; + } + else { + return L""; + } +} + + +const wchar_t* +_Py_GetXOption(int nxoption, wchar_t **xoptions, wchar_t *name) +{ + for (int i=0; i < nxoption; i++) { + wchar_t *option = xoptions[i]; + size_t len; + wchar_t *sep = wcschr(option, L'='); + if (sep != NULL) { + len = (sep - option); + } + else { + len = wcslen(option); + } + if (wcsncmp(option, name, len) == 0 && name[len] == L'\0') { + return option; + } + } + return NULL; +} + + +_PyInitError +_PyArgv_Decode(const _PyArgv *args, int *argc_p, wchar_t*** argv_p) +{ + wchar_t** argv; + if (args->use_bytes_argv) { + /* +1 for a the NULL terminator */ + size_t size = sizeof(wchar_t*) * (args->argc + 1); + argv = (wchar_t **)PyMem_RawMalloc(size); + if (argv == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + for (int i = 0; i < args->argc; i++) { + size_t len; + wchar_t *arg = Py_DecodeLocale(args->bytes_argv[i], &len); + if (arg == NULL) { + _Py_wstrlist_clear(i, argv); + return DECODE_LOCALE_ERR("command line arguments", + (Py_ssize_t)len); + } + argv[i] = arg; + } + argv[args->argc] = NULL; + } + else { + argv = _Py_wstrlist_copy(args->argc, args->wchar_argv); + if (argv == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + *argc_p = args->argc; + *argv_p = argv; + return _Py_INIT_OK(); +} + + +#if !defined(__ANDROID__) +# define _PyPreCmdline_USE_LOCALE +#endif + +typedef struct { + int argc; + wchar_t **argv; + int nxoption; /* Number of -X options */ + wchar_t **xoptions; /* -X options */ +#ifdef _PyPreCmdline_USE_LOCALE + char *ctype_locale; +#endif +} _PyPreCmdline; + + +static void +precmdline_clear(_PyPreCmdline *cmdline) +{ + _Py_wstrlist_clear(cmdline->argc, cmdline->argv); + cmdline->argc = 0; + cmdline->argv = NULL; + + _Py_wstrlist_clear(cmdline->nxoption, cmdline->xoptions); + cmdline->nxoption = 0; + cmdline->xoptions = NULL; + +#ifdef _PyPreCmdline_USE_LOCALE + PyMem_RawFree(cmdline->ctype_locale); + cmdline->ctype_locale = NULL; +#endif +} + + +static _PyInitError +precmdline_init(const _PyArgv *args, _PyPreCmdline *cmdline) +{ + _PyInitError err = _PyArgv_Decode(args, &cmdline->argc, &cmdline->argv); + if (_Py_INIT_FAILED(err)) { + return err; + } + +#ifdef _PyPreCmdline_USE_LOCALE + const char *loc = setlocale(LC_CTYPE, NULL); + if (loc != NULL) { + cmdline->ctype_locale = _PyMem_RawStrdup(loc); + if (cmdline->ctype_locale == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } +#endif + + return _Py_INIT_OK(); +} + + +void +_PyPreConfig_Clear(_PyPreConfig *config) +{ +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->allocator); + CLEAR(config->filesystem_encoding); + CLEAR(config->filesystem_errors); + CLEAR(config->stdio_encoding); + CLEAR(config->stdio_errors); +#undef CLEAR +} + + +int +_PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2) +{ + _PyPreConfig_Clear(config); + +#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR +#define COPY_STR_ATTR(ATTR) \ + do { \ + if (config2->ATTR != NULL) { \ + config->ATTR = _PyMem_RawStrdup(config2->ATTR); \ + if (config->ATTR == NULL) { \ + return -1; \ + } \ + } \ + } while (0) + + COPY_ATTR(use_environment); + COPY_ATTR(isolated); + COPY_ATTR(dev_mode); + COPY_STR_ATTR(allocator); + COPY_ATTR(utf8_mode); + COPY_ATTR(coerce_c_locale); + COPY_ATTR(coerce_c_locale_warn); + COPY_STR_ATTR(filesystem_encoding); + COPY_STR_ATTR(filesystem_errors); + COPY_STR_ATTR(stdio_encoding); + COPY_STR_ATTR(stdio_errors); +#ifdef MS_WINDOWS + COPY_ATTR(legacy_windows_fs_encoding); + COPY_ATTR(legacy_windows_stdio); +#endif + return 0; + +#undef COPY_ATTR +#undef COPY_STR_ATTR +} + + +void +_PyPreConfig_GetGlobalConfig(_PyPreConfig *config) +{ +#define COPY_FLAG(ATTR, VALUE) \ + if (config->ATTR == -1) { \ + config->ATTR = VALUE; \ + } +#define COPY_NOT_FLAG(ATTR, VALUE) \ + if (config->ATTR == -1) { \ + config->ATTR = !(VALUE); \ + } + + COPY_FLAG(utf8_mode, Py_UTF8Mode); + COPY_FLAG(isolated, Py_IsolatedFlag); +#ifdef MS_WINDOWS + COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); + COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); +#endif + COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); + +#undef COPY_FLAG +#undef COPY_NOT_FLAG +} + + +void +_PyPreConfig_SetGlobalConfig(const _PyPreConfig *config) +{ +#define COPY_FLAG(ATTR, VAR) \ + if (config->ATTR != -1) { \ + VAR = config->ATTR; \ + } +#define COPY_NOT_FLAG(ATTR, VAR) \ + if (config->ATTR != -1) { \ + VAR = !config->ATTR; \ + } + + COPY_FLAG(utf8_mode, Py_UTF8Mode); + COPY_FLAG(isolated, Py_IsolatedFlag); +#ifdef MS_WINDOWS + COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); + COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); +#endif + COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); + +#undef COPY_FLAG +#undef COPY_NOT_FLAG +} + + +const char* +_PyPreConfig_GetEnv(const _PyPreConfig *config, const char *name) +{ + assert(config->use_environment >= 0); + + if (!config->use_environment) { + return NULL; + } + + const char *var = getenv(name); + if (var && var[0] != '\0') { + return var; + } + else { + return NULL; + } +} + + +static void +preconfig_init_locale(_PyPreConfig *config, const char *ctype_locale) +{ + /* Test also if coerce_c_locale equals 1: PYTHONCOERCECLOCALE=1 doesn't + imply that the C locale is always coerced. It is only coerced if + if the LC_CTYPE locale is "C". */ + if (config->coerce_c_locale != 0) { + /* The C locale enables the C locale coercion (PEP 538) */ + if (_Py_LegacyLocaleDetected(ctype_locale)) { + config->coerce_c_locale = 1; + } + else { + config->coerce_c_locale = 0; + } + } + +#ifndef MS_WINDOWS + if (config->utf8_mode < 0) { + /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ + if (ctype_locale != NULL + && (strcmp(ctype_locale, "C") == 0 + || strcmp(ctype_locale, "POSIX") == 0)) + { + config->utf8_mode = 1; + } + } +#endif +} + + +static _PyInitError +preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline) +{ + const wchar_t *xopt; + if (cmdline) { + xopt = _Py_GetXOption(cmdline->nxoption, cmdline->xoptions, L"utf8"); + } + else { + xopt = NULL; + } + if (xopt) { + wchar_t *sep = wcschr(xopt, L'='); + if (sep) { + xopt = sep + 1; + if (wcscmp(xopt, L"1") == 0) { + config->utf8_mode = 1; + } + else if (wcscmp(xopt, L"0") == 0) { + config->utf8_mode = 0; + } + else { + return _Py_INIT_USER_ERR("invalid -X utf8 option value"); + } + } + else { + config->utf8_mode = 1; + } + return _Py_INIT_OK(); + } + + const char *opt = _PyPreConfig_GetEnv(config, "PYTHONUTF8"); + if (opt) { + if (strcmp(opt, "1") == 0) { + config->utf8_mode = 1; + } + else if (strcmp(opt, "0") == 0) { + config->utf8_mode = 0; + } + else { + return _Py_INIT_USER_ERR("invalid PYTHONUTF8 environment " + "variable value"); + } + return _Py_INIT_OK(); + } + + return _Py_INIT_OK(); +} + + +static _PyInitError +get_locale_encoding(char **locale_encoding) +{ +#ifdef MS_WINDOWS + char encoding[20]; + PyOS_snprintf(encoding, sizeof(encoding), "cp%d", GetACP()); +#elif defined(__ANDROID__) + const char *encoding = "UTF-8"; +#else + const char *encoding = nl_langinfo(CODESET); + if (!encoding || encoding[0] == '\0') { + return _Py_INIT_USER_ERR("failed to get the locale encoding: " + "nl_langinfo(CODESET) failed"); + } +#endif + *locale_encoding = _PyMem_RawStrdup(encoding); + if (*locale_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +static _PyInitError +preconfig_init_fs_encoding(_PyPreConfig *config) +{ +#ifdef MS_WINDOWS + if (config->legacy_windows_fs_encoding) { + /* Legacy Windows filesystem encoding: mbcs/replace */ + if (config->filesystem_encoding == NULL) { + config->filesystem_encoding = _PyMem_RawStrdup("mbcs"); + if (config->filesystem_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + if (config->filesystem_errors == NULL) { + config->filesystem_errors = _PyMem_RawStrdup("replace"); + if (config->filesystem_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } + + /* Windows defaults to utf-8/surrogatepass (PEP 529). + + Note: UTF-8 Mode takes the same code path and the Legacy Windows FS + encoding has the priortiy over UTF-8 Mode. */ + if (config->filesystem_encoding == NULL) { + config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + if (config->filesystem_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->filesystem_errors == NULL) { + config->filesystem_errors = _PyMem_RawStrdup("surrogatepass"); + if (config->filesystem_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } +#else + if (config->filesystem_encoding == NULL) { + if (config->utf8_mode) { + /* UTF-8 Mode use: utf-8/surrogateescape */ + config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + /* errors defaults to surrogateescape above */ + } + else if (_Py_GetForceASCII()) { + config->filesystem_encoding = _PyMem_RawStrdup("ascii"); + } + else { + /* macOS and Android use UTF-8, + other platforms use the locale encoding. */ +#if defined(__APPLE__) || defined(__ANDROID__) + config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); +#else + _PyInitError err = get_locale_encoding(&config->filesystem_encoding); + if (_Py_INIT_FAILED(err)) { + return err; + } +#endif + } + + if (config->filesystem_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->filesystem_errors == NULL) { + /* by default, use the "surrogateescape" error handler */ + config->filesystem_errors = _PyMem_RawStrdup("surrogateescape"); + if (config->filesystem_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } +#endif + return _Py_INIT_OK(); +} + + +static const char * +get_stdio_errors(void) +{ +#ifndef MS_WINDOWS + const char *loc = setlocale(LC_CTYPE, NULL); + if (loc != NULL) { + /* surrogateescape is the default in the legacy C and POSIX locales */ + if (strcmp(loc, "C") == 0 || strcmp(loc, "POSIX") == 0) { + return "surrogateescape"; + } + +#ifdef PY_COERCE_C_LOCALE + /* surrogateescape is the default in locale coercion target locales */ + if (_Py_IsLocaleCoercionTarget(loc)) { + return "surrogateescape"; + } +#endif + } + + return "strict"; +#else + /* On Windows, always use surrogateescape by default */ + return "surrogateescape"; +#endif +} + + +static _PyInitError +preconfig_init_stdio_encoding(_PyPreConfig *config) +{ + /* If Py_SetStandardStreamEncoding() have been called, use these + parameters. */ + if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) { + config->stdio_encoding = _PyMem_RawStrdup(_Py_StandardStreamEncoding); + if (config->stdio_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->stdio_errors == NULL && _Py_StandardStreamErrors != NULL) { + config->stdio_errors = _PyMem_RawStrdup(_Py_StandardStreamErrors); + if (config->stdio_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->stdio_encoding != NULL && config->stdio_errors != NULL) { + return _Py_INIT_OK(); + } + + /* PYTHONIOENCODING environment variable */ + const char *opt = _PyPreConfig_GetEnv(config, "PYTHONIOENCODING"); + if (opt) { + char *pythonioencoding = _PyMem_RawStrdup(opt); + if (pythonioencoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + char *err = strchr(pythonioencoding, ':'); + if (err) { + *err = '\0'; + err++; + if (!err[0]) { + err = NULL; + } + } + + /* Does PYTHONIOENCODING contain an encoding? */ + if (pythonioencoding[0]) { + if (config->stdio_encoding == NULL) { + config->stdio_encoding = _PyMem_RawStrdup(pythonioencoding); + if (config->stdio_encoding == NULL) { + PyMem_RawFree(pythonioencoding); + return _Py_INIT_NO_MEMORY(); + } + } + + /* If the encoding is set but not the error handler, + use "strict" error handler by default. + PYTHONIOENCODING=latin1 behaves as + PYTHONIOENCODING=latin1:strict. */ + if (!err) { + err = "strict"; + } + } + + if (config->stdio_errors == NULL && err != NULL) { + config->stdio_errors = _PyMem_RawStrdup(err); + if (config->stdio_errors == NULL) { + PyMem_RawFree(pythonioencoding); + return _Py_INIT_NO_MEMORY(); + } + } + + PyMem_RawFree(pythonioencoding); + } + + /* UTF-8 Mode uses UTF-8/surrogateescape */ + if (config->utf8_mode) { + if (config->stdio_encoding == NULL) { + config->stdio_encoding = _PyMem_RawStrdup("utf-8"); + if (config->stdio_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + if (config->stdio_errors == NULL) { + config->stdio_errors = _PyMem_RawStrdup("surrogateescape"); + if (config->stdio_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } + + /* Choose the default error handler based on the current locale. */ + if (config->stdio_encoding == NULL) { + _PyInitError err = get_locale_encoding(&config->stdio_encoding); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + if (config->stdio_errors == NULL) { + const char *errors = get_stdio_errors(); + config->stdio_errors = _PyMem_RawStrdup(errors); + if (config->stdio_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + return _Py_INIT_OK(); +} + + +void +_PyPreConfig_GetEnvFlag(_PyPreConfig *config, int *flag, const char *name) +{ + const char *var = _PyPreConfig_GetEnv(config, name); + if (!var) { + return; + } + int value; + if (_Py_str_to_int(var, &value) < 0 || value < 0) { + /* PYTHONDEBUG=text and PYTHONDEBUG=-2 behave as PYTHONDEBUG=1 */ + value = 1; + } + if (*flag < value) { + *flag = value; + } +} + + +static _PyInitError +preconfig_read_env_vars(_PyPreConfig *config) +{ + if (config->allocator == NULL) { + const char *allocator = _PyPreConfig_GetEnv(config, "PYTHONMALLOC"); + if (allocator) { + config->allocator = _PyMem_RawStrdup(allocator); + if (config->allocator == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } + +#ifdef MS_WINDOWS + _PyPreConfig_GetEnvFlag(config, &config->legacy_windows_fs_encoding, + "PYTHONLEGACYWINDOWSFSENCODING"); + _PyPreConfig_GetEnvFlag(config, &config->legacy_windows_stdio, + "PYTHONLEGACYWINDOWSSTDIO"); +#endif + + const char *env = _PyPreConfig_GetEnv(config, "PYTHONCOERCECLOCALE"); + if (env) { + if (strcmp(env, "0") == 0) { + if (config->coerce_c_locale < 0) { + config->coerce_c_locale = 0; + } + } + else if (strcmp(env, "warn") == 0) { + config->coerce_c_locale_warn = 1; + } + else { + if (config->coerce_c_locale < 0) { + config->coerce_c_locale = 1; + } + } + } + + return _Py_INIT_OK(); +} + + +static _PyInitError +preconfig_read_from_cmdline(_PyPreConfig *config, const _PyPreCmdline *cmdline) +{ + assert(config->use_environment >= 0); + + _PyPreConfig_GetGlobalConfig(config); + + _PyInitError err; + + if (config->isolated > 0) { + config->use_environment = 0; + } + +#ifdef MS_WINDOWS + if (config->legacy_windows_fs_encoding) { + config->utf8_mode = 0; + } +#endif + + if (config->use_environment) { + err = preconfig_read_env_vars(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + if ((cmdline != NULL && _Py_GetXOption(cmdline->nxoption, cmdline->xoptions, L"dev")) || + _PyPreConfig_GetEnv(config, "PYTHONDEVMODE")) + { + config->dev_mode = 1; + } + + if (config->utf8_mode < 0) { + err = preconfig_init_utf8_mode(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + if (config->coerce_c_locale != 0 || config->utf8_mode < 0) { + const char *ctype_locale; + if (cmdline) { + ctype_locale = cmdline->ctype_locale; + } + else { + ctype_locale = setlocale(LC_CTYPE, NULL); + } + preconfig_init_locale(config, ctype_locale); + } + + if (config->coerce_c_locale < 0) { + config->coerce_c_locale = 0; + } + if (config->utf8_mode < 0) { + config->utf8_mode = 0; + } + + if (config->filesystem_encoding == NULL || config->filesystem_errors == NULL) { + err = preconfig_init_fs_encoding(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + err = preconfig_init_stdio_encoding(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + /* default values */ + if (config->dev_mode) { + if (config->allocator == NULL) { + config->allocator = _PyMem_RawStrdup("debug"); + if (config->allocator == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } + + assert(config->coerce_c_locale >= 0); + assert(config->use_environment >= 0); + assert(config->filesystem_encoding != NULL); + assert(config->filesystem_errors != NULL); + assert(config->stdio_encoding != NULL); + assert(config->stdio_errors != NULL); + + return _Py_INIT_OK(); +} + + +_PyInitError +_PyPreConfig_Read(_PyPreConfig *config) +{ + return preconfig_read_from_cmdline(config, NULL); +} + + +/* Parse _PyPreConfig command line options like -E */ +static _PyInitError +preconfig_parse_cmdline(_PyPreConfig *config, _PyPreCmdline *cmdline) +{ + _PyOS_ResetGetOpt(); + _PyOS_opterr = 0; + do { + int longindex = -1; + int c = _PyOS_GetOpt(cmdline->argc, cmdline->argv, &longindex); + if (c == EOF || c == 'c' || c == 'm') { + break; + } + + switch (c) { + case 'I': + config->isolated++; + break; + + case 'E': + config->use_environment = 0; + break; + + case 'X': + { + _PyInitError err; + err = _Py_wstrlist_append(&cmdline->nxoption, + &cmdline->xoptions, + _PyOS_optarg); + if (_Py_INIT_FAILED(err)) { + return err; + } + break; + } + + default: + /* other arguments will be parsed by + cmdline_parse_impl() */ + break; + } + } while (1); + return _Py_INIT_OK(); +} + + +static _PyInitError +_PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args) +{ + _PyInitError err; + + _PyPreCmdline cmdline; + memset(&cmdline, 0, sizeof(cmdline)); + + err = precmdline_init(args, &cmdline); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = preconfig_parse_cmdline(config, &cmdline); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = preconfig_read_from_cmdline(config, &cmdline); + +done: + precmdline_clear(&cmdline); + return err; +} + + +/* Read the preconfiguration and initialize the LC_CTYPE locale: + coerce the C locale (PEP 538). */ +_PyInitError +_PyPreConfig_InitFromArgv(_PyPreConfig *config, const _PyArgv *args) +{ + _PyInitError err; + + err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } + + int init_utf8_mode = Py_UTF8Mode; +#ifdef MS_WINDOWS + int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag; +#endif + _PyPreConfig save_config = _PyPreConfig_INIT; + int locale_coerced = 0; + int loops = 0; + + if (_PyPreConfig_Copy(&save_config, config) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + + /* Set LC_CTYPE to the user preferred locale */ + _Py_SetLocaleFromEnv(LC_CTYPE); + + while (1) { + int utf8_mode = config->utf8_mode; + + /* Watchdog to prevent an infinite loop */ + loops++; + if (loops == 3) { + err = _Py_INIT_ERR("Encoding changed twice while " + "reading the configuration"); + goto done; + } + + /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend + on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */ + Py_UTF8Mode = config->utf8_mode; +#ifdef MS_WINDOWS + Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; +#endif + + err = _PyPreConfig_ReadFromArgv(config, args); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + /* The legacy C locale assumes ASCII as the default text encoding, which + * causes problems not only for the CPython runtime, but also other + * components like GNU readline. + * + * Accordingly, when the CLI detects it, it attempts to coerce it to a + * more capable UTF-8 based alternative. + * + * See the documentation of the PYTHONCOERCECLOCALE setting for more + * details. + */ + int encoding_changed = 0; + if (config->coerce_c_locale && !locale_coerced) { + locale_coerced = 1; + _Py_CoerceLegacyLocale(config->coerce_c_locale_warn); + encoding_changed = 1; + } + + if (utf8_mode == -1) { + if (config->utf8_mode == 1) { + /* UTF-8 Mode enabled */ + encoding_changed = 1; + } + } + else { + if (config->utf8_mode != utf8_mode) { + encoding_changed = 1; + } + } + + if (!encoding_changed) { + break; + } + + /* Reset the configuration before reading again the configuration, + just keep UTF-8 Mode value. */ + int new_utf8_mode = config->utf8_mode; + int new_coerce_c_locale = config->coerce_c_locale; + if (_PyPreConfig_Copy(config, &save_config) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + config->utf8_mode = new_utf8_mode; + config->coerce_c_locale = new_coerce_c_locale; + + /* The encoding changed: read again the configuration + with the new encoding */ + } + err = _Py_INIT_OK(); + +done: + _PyPreConfig_Clear(&save_config); + Py_UTF8Mode = init_utf8_mode ; +#ifdef MS_WINDOWS + Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding; +#endif + return err; +} + + +_PyInitError +_PyPreConfig_SetAllocator(_PyPreConfig *config) +{ + assert(!_PyRuntime.core_initialized); + + PyMemAllocatorEx old_alloc; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_PyMem_SetupAllocators(config->allocator) < 0) { + return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator"); + } + + /* free config with the old allocator and re-allocate the config + with the new allocator */ + PyMemAllocatorEx new_alloc; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &new_alloc); + + _PyInitError err; + _PyPreConfig config2 = _PyPreConfig_INIT; + if (_PyPreConfig_Copy(&config2, config) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyPreConfig_Clear(config); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &new_alloc); + + if (_PyPreConfig_Copy(config, &config2) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + err = _Py_INIT_OK(); + +done: + _PyPreConfig_Clear(&config2); + return err; +} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index eb0fdddb4b3acf3..80d6c3952143c77 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -5,6 +5,7 @@ #include "Python-ast.h" #undef Yield /* undefine macro conflicting with */ #include "pycore_context.h" +#include "pycore_coreconfig.h" #include "pycore_fileutils.h" #include "pycore_hamt.h" #include "pycore_pathconfig.h" @@ -261,7 +262,7 @@ initexternalimport(PyInterpreterState *interp) */ int -_Py_LegacyLocaleDetected(void) +_Py_LegacyLocaleDetected(const char *ctype_locale) { #ifndef MS_WINDOWS /* On non-Windows systems, the C locale is considered a legacy locale */ @@ -269,8 +270,7 @@ _Py_LegacyLocaleDetected(void) * the POSIX locale as a simple alias for the C locale, so * we may also want to check for that explicitly. */ - const char *ctype_loc = setlocale(LC_CTYPE, NULL); - return ctype_loc != NULL && strcmp(ctype_loc, "C") == 0; + return (ctype_locale != NULL && strcmp(ctype_locale, "C") == 0); #else /* Windows uses code pages instead of locales, so no locale is legacy */ return 0; @@ -286,7 +286,8 @@ static const char *_C_LOCALE_WARNING = static void _emit_stderr_warning_for_legacy_locale(const _PyCoreConfig *core_config) { - if (core_config->coerce_c_locale_warn && _Py_LegacyLocaleDetected()) { + const char *ctype_loc = setlocale(LC_CTYPE, NULL); + if (core_config->preconfig.coerce_c_locale_warn && _Py_LegacyLocaleDetected(ctype_loc)) { PySys_FormatStderr("%s", _C_LOCALE_WARNING); } } @@ -479,16 +480,6 @@ _Py_Initialize_ReconfigureCore(PyInterpreterState **interp_p, } *interp_p = interp; - /* bpo-34008: For backward compatibility reasons, calling Py_Main() after - Py_Initialize() ignores the new configuration. */ - if (core_config->allocator != NULL) { - const char *allocator = _PyMem_GetAllocatorsName(); - if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) { - return _Py_INIT_USER_ERR("cannot modify memory allocator " - "after first Py_Initialize()"); - } - } - _PyCoreConfig_SetGlobalConfig(core_config); if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) { @@ -520,12 +511,6 @@ pycore_init_runtime(const _PyCoreConfig *core_config) return err; } - if (core_config->allocator != NULL) { - if (_PyMem_SetupAllocators(core_config->allocator) < 0) { - return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator"); - } - } - /* Py_Finalize leaves _Py_Finalizing set in order to help daemon * threads behave a little more gracefully at interpreter shutdown. * We clobber it here so the new interpreter can start with a clean @@ -750,25 +735,20 @@ _Py_InitializeCore(PyInterpreterState **interp_p, { assert(src_config != NULL); - PyMemAllocatorEx old_alloc; - _PyInitError err; + /* Set LC_CTYPE to the user preferred locale */ + _Py_SetLocaleFromEnv(LC_CTYPE); /* Copy the configuration, since _PyCoreConfig_Read() modifies it (and the input configuration is read only). */ _PyCoreConfig config = _PyCoreConfig_INIT; - /* Set LC_CTYPE to the user preferred locale */ - _Py_SetLocaleFromEnv(LC_CTYPE); - - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_PyCoreConfig_Copy(&config, src_config) >= 0) { - err = _PyCoreConfig_Read(&config); - } - else { + _PyInitError err; + if (_PyCoreConfig_Copy(&config, src_config) < 0) { err = _Py_INIT_ERR("failed to copy core config"); + goto done; } - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + err = _PyCoreConfig_Read(&config, NULL); if (_Py_INIT_FAILED(err)) { goto done; } @@ -781,10 +761,7 @@ _Py_InitializeCore(PyInterpreterState **interp_p, } done: - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); _PyCoreConfig_Clear(&config); - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return err; } @@ -959,7 +936,7 @@ Py_InitializeEx(int install_sigs) _PyCoreConfig_Clear(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } } @@ -1431,7 +1408,7 @@ Py_NewInterpreter(void) PyThreadState *tstate; _PyInitError err = new_interpreter(&tstate); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } return tstate; @@ -1551,7 +1528,7 @@ initfsencoding(PyInterpreterState *interp) { _PyCoreConfig *config = &interp->core_config; - char *encoding = get_codec_name(config->filesystem_encoding); + char *encoding = get_codec_name(config->preconfig.filesystem_encoding); if (encoding == NULL) { /* Such error can only occurs in critical situations: no more memory, import a module of the standard library failed, etc. */ @@ -1562,13 +1539,13 @@ initfsencoding(PyInterpreterState *interp) /* Update the filesystem encoding to the normalized Python codec name. For example, replace "ANSI_X3.4-1968" (locale encoding) with "ascii" (Python codec name). */ - PyMem_RawFree(config->filesystem_encoding); - config->filesystem_encoding = encoding; + PyMem_RawFree(config->preconfig.filesystem_encoding); + config->preconfig.filesystem_encoding = encoding; /* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors global configuration variables. */ - if (_Py_SetFileSystemEncoding(config->filesystem_encoding, - config->filesystem_errors) < 0) { + if (_Py_SetFileSystemEncoding(config->preconfig.filesystem_encoding, + config->preconfig.filesystem_errors) < 0) { return _Py_INIT_NO_MEMORY(); } @@ -1770,13 +1747,13 @@ init_sys_streams(PyInterpreterState *interp) } #endif - char *codec_name = get_codec_name(config->stdio_encoding); + char *codec_name = get_codec_name(config->preconfig.stdio_encoding); if (codec_name == NULL) { return _Py_INIT_ERR("failed to get the Python codec name " "of the stdio encoding"); } - PyMem_RawFree(config->stdio_encoding); - config->stdio_encoding = codec_name; + PyMem_RawFree(config->preconfig.stdio_encoding); + config->preconfig.stdio_encoding = codec_name; /* Hack to avoid a nasty recursion issue when Python is invoked in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ @@ -1815,8 +1792,8 @@ init_sys_streams(PyInterpreterState *interp) * GUI apps don't have valid standard streams by default. */ std = create_stdio(config, iomod, fd, 0, "", - config->stdio_encoding, - config->stdio_errors); + config->preconfig.stdio_encoding, + config->preconfig.stdio_errors); if (std == NULL) goto error; PySys_SetObject("__stdin__", std); @@ -1826,8 +1803,8 @@ init_sys_streams(PyInterpreterState *interp) /* Set sys.stdout */ fd = fileno(stdout); std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, - config->stdio_errors); + config->preconfig.stdio_encoding, + config->preconfig.stdio_errors); if (std == NULL) goto error; PySys_SetObject("__stdout__", std); @@ -1838,7 +1815,7 @@ init_sys_streams(PyInterpreterState *interp) /* Set sys.stderr, replaces the preliminary stderr */ fd = fileno(stderr); std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, + config->preconfig.stdio_encoding, "backslashreplace"); if (std == NULL) goto error; @@ -2072,12 +2049,17 @@ Py_FatalError(const char *msg) } void _Py_NO_RETURN -_Py_FatalInitError(_PyInitError err) +_Py_ExitInitError(_PyInitError err) { - /* On "user" error: exit with status 1. - For all other errors, call abort(). */ - int status = err.user_err ? 1 : -1; - fatal_error(err.prefix, err.msg, status); + if (err.exitcode >= 0) { + exit(err.exitcode); + } + else { + /* On "user" error: exit with status 1. + For all other errors, call abort(). */ + int status = err.user_err ? 1 : -1; + fatal_error(err.prefix, err.msg, status); + } } /* Clean up and exit */ diff --git a/Python/pystate.c b/Python/pystate.c index 6fe947d36466e51..a9a425c2f49cb91 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2,6 +2,7 @@ /* Thread and interpreter state structures and their interfaces */ #include "Python.h" +#include "pycore_coreconfig.h" #include "pycore_pymem.h" #include "pycore_pystate.h" diff --git a/Python/sysmodule.c b/Python/sysmodule.c index dd39305f39af6c7..27a0864f7c78e6b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -421,7 +421,7 @@ sys_getfilesystemencoding_impl(PyObject *module) { PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); const _PyCoreConfig *config = &interp->core_config; - return PyUnicode_FromString(config->filesystem_encoding); + return PyUnicode_FromString(config->preconfig.filesystem_encoding); } /*[clinic input] @@ -436,7 +436,7 @@ sys_getfilesystemencodeerrors_impl(PyObject *module) { PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); const _PyCoreConfig *config = &interp->core_config; - return PyUnicode_FromString(config->filesystem_errors); + return PyUnicode_FromString(config->preconfig.filesystem_errors); } /*[clinic input] @@ -1221,13 +1221,13 @@ sys__enablelegacywindowsfsencoding_impl(PyObject *module) return NULL; } - PyMem_RawFree(config->filesystem_encoding); - config->filesystem_encoding = encoding; - PyMem_RawFree(config->filesystem_errors); - config->filesystem_errors = errors; + PyMem_RawFree(config->preconfig.filesystem_encoding); + config->preconfig.filesystem_encoding = encoding; + PyMem_RawFree(config->preconfig.filesystem_errors); + config->preconfig.filesystem_errors = errors; - if (_Py_SetFileSystemEncoding(config->filesystem_encoding, - config->filesystem_errors) < 0) { + if (_Py_SetFileSystemEncoding(config->preconfig.filesystem_encoding, + config->preconfig.filesystem_errors) < 0) { PyErr_NoMemory(); return NULL; } @@ -2172,16 +2172,16 @@ make_flags(void) SetFlag(!config->write_bytecode); SetFlag(!config->user_site_directory); SetFlag(!config->site_import); - SetFlag(!config->use_environment); + SetFlag(!config->preconfig.use_environment); SetFlag(config->verbose); /* SetFlag(saw_unbuffered_flag); */ /* SetFlag(skipfirstline); */ SetFlag(config->bytes_warning); SetFlag(config->quiet); SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0); - SetFlag(config->isolated); - PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode)); - SetFlag(config->utf8_mode); + SetFlag(config->preconfig.isolated); + PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->preconfig.dev_mode)); + SetFlag(config->preconfig.utf8_mode); #undef SetFlag if (PyErr_Occurred()) {