From 5891758bec7ce8ac3ff64a2eb002f7db9b603712 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 21 Nov 2017 10:33:16 -0700
Subject: [PATCH 1/5] Add a pre-init test using Py_DecodeLocale().

---
 Lib/test/test_capi.py | 10 ++++++++++
 Programs/_testembed.c | 23 +++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index bb5b2a3b9f0d734..2fe0feca5a3ca44 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -593,6 +593,16 @@ def test_forced_io_encoding(self):
         self.maxDiff = None
         self.assertEqual(out.strip(), expected_output)
 
+    def test_pre_initialization_api(self):
+        """
+        Checks the few parts of the C-API that work before the runtine
+        is initialized (via Py_Initialize()).
+        """
+        env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
+        out, err = self.run_embedded_interpreter("pre_initialization_api", env=env)
+        self.assertEqual(out, '')
+        self.assertEqual(err, '')
+
 
 class SkipitemTest(unittest.TestCase):
 
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index e68e68de327b129..dc27a40b6e4330d 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -72,6 +72,7 @@ static int test_repeated_init_and_subinterpreters(void)
     return 0;
 }
 
+
 /*****************************************************
  * Test forcing a particular IO encoding
  *****************************************************/
@@ -125,6 +126,27 @@ static int test_forced_io_encoding(void)
     return 0;
 }
 
+/*********************************************************
+ * Test parts of the C-API that work before initialization
+ *********************************************************/
+
+static int test_pre_initialization_api(void)
+{
+    wchar_t *program = Py_DecodeLocale("spam", NULL);
+    if (program == NULL) {
+        fprintf(stderr, "Fatal error: cannot decode program name\n");
+        return 1;
+    }
+    Py_SetProgramName(program);
+
+    Py_Initialize();
+    Py_Finalize();
+
+    PyMem_RawFree(program);
+    return 0;
+}
+
+
 /* *********************************************************
  * List of test cases and the function that implements it.
  *
@@ -146,6 +168,7 @@ struct TestCase
 static struct TestCase TestCases[] = {
     { "forced_io_encoding", test_forced_io_encoding },
     { "repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters },
+    { "pre_initialization_api", test_pre_initialization_api },
     { NULL, NULL }
 };
 

From 3dcd9602a2a509333eed1aed9ae9cb3ab6d11e6a Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Mon, 20 Nov 2017 14:30:02 -0700
Subject: [PATCH 2/5] Statically initialize the default raw allocator.

---
 Include/pymem.h      |  2 ++
 Objects/obmalloc.c   |  4 ++--
 Python/pylifecycle.c | 17 ++++++++++++++++-
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/Include/pymem.h b/Include/pymem.h
index 928851a3d70e459..6d0317efb972270 100644
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -105,6 +105,8 @@ PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
 PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
 #ifndef Py_LIMITED_API
+PyAPI_FUNC(void *) _PyMem_RawMalloc(void *ctx, size_t size);
+PyAPI_FUNC(void) _PyMem_RawFree(void *ctx, void *ptr);
 PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str);
 PyAPI_FUNC(char *) _PyMem_Strdup(const char *str);
 #endif
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index 7c6973ec035f10b..51d3bbc1f84614b 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -63,7 +63,7 @@ static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
 #endif
 
 
-static void *
+void *
 _PyMem_RawMalloc(void *ctx, size_t size)
 {
     /* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL
@@ -97,7 +97,7 @@ _PyMem_RawRealloc(void *ctx, void *ptr, size_t size)
     return realloc(ptr, size);
 }
 
-static void
+void
 _PyMem_RawFree(void *ctx, void *ptr)
 {
     free(ptr);
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 14fe75e0cf7deec..f154c7b5149053b 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -76,7 +76,22 @@ extern void _Py_ReadyTypes(void);
 extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *);
 extern void _PyGILState_Fini(void);
 
-_PyRuntimeState _PyRuntime = _PyRuntimeState_INIT;
+//_PyRuntimeState _PyRuntime = _PyRuntimeState_INIT;
+_PyRuntimeState _PyRuntime = {
+    .initialized = 0,
+    .core_initialized = 0,
+    // Py_DecodeLocale() relies on PyMem_RawMalloc() and PyMem_RawFree()
+    // and may be needed before before runtime initialization.  To allow
+    // for this we pre-initialize the raw allocator statically here.
+    .mem = {
+        .allocators = {
+            .raw = {
+                .malloc = _PyMem_RawMalloc,
+                .free = _PyMem_RawFree
+            }
+        }
+    }
+};
 
 _PyInitError
 _PyRuntime_Initialize(void)

From 9584720c6c7e163de2bf12eaf11d02d69b1d2855 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 21 Nov 2017 09:26:44 -0700
Subject: [PATCH 3/5] Make the _PyMem_* declarations "internal".

---
 Include/internal/mem.h | 2 ++
 Include/pymem.h        | 2 --
 Python/pylifecycle.c   | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Include/internal/mem.h b/Include/internal/mem.h
index 471cdf45df27668..4446b6218d4beaf 100644
--- a/Include/internal/mem.h
+++ b/Include/internal/mem.h
@@ -44,6 +44,8 @@ struct _pymem_runtime_state {
     size_t serialno;     /* incremented on each debug {m,re}alloc */
 };
 
+PyAPI_FUNC(void *) _PyMem_RawMalloc(void *ctx, size_t size);
+PyAPI_FUNC(void) _PyMem_RawFree(void *ctx, void *ptr);
 PyAPI_FUNC(void) _PyMem_Initialize(struct _pymem_runtime_state *);
 
 
diff --git a/Include/pymem.h b/Include/pymem.h
index 6d0317efb972270..928851a3d70e459 100644
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -105,8 +105,6 @@ PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
 PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
 #ifndef Py_LIMITED_API
-PyAPI_FUNC(void *) _PyMem_RawMalloc(void *ctx, size_t size);
-PyAPI_FUNC(void) _PyMem_RawFree(void *ctx, void *ptr);
 PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str);
 PyAPI_FUNC(char *) _PyMem_Strdup(const char *str);
 #endif
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index f154c7b5149053b..799eb4d967351eb 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -5,6 +5,7 @@
 #include "Python-ast.h"
 #undef Yield /* undefine macro conflicting with winbase.h */
 #include "internal/pystate.h"
+#include "internal/mem.h"
 #include "grammar.h"
 #include "node.h"
 #include "token.h"

From 70f4961792bbc11e311741dbc97792443502ccc2 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 21 Nov 2017 12:45:19 -0700
Subject: [PATCH 4/5] Add a Misc/NEWS entry.

---
 .../2017-11-21-12-44-51.bpo-32096.tXX68e.rst                | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-11-21-12-44-51.bpo-32096.tXX68e.rst

diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-21-12-44-51.bpo-32096.tXX68e.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-21-12-44-51.bpo-32096.tXX68e.rst
new file mode 100644
index 000000000000000..67687d3d659386b
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-21-12-44-51.bpo-32096.tXX68e.rst	
@@ -0,0 +1,6 @@
+Since the recent global runtime state consolidation, calls to
+PyMem_RawMalloc() and PyMem_RawFree() made before runtime initialization
+have been crashing.  This is because they rely on the raw memory allocator
+having been initialized already.  Before the runtime state change the
+default raw allocator was initialized statically.  This has now been fixed
+by falling back to the defaults in PyMem_RawMalloc() and PyMem_RawFree().

From b28d3029ae5ade1afc607933e78ab1d7c878a40a Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 21 Nov 2017 13:41:39 -0700
Subject: [PATCH 5/5] Fix a typo.

---
 Python/pylifecycle.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 799eb4d967351eb..df0ba2521583133 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -82,8 +82,8 @@ _PyRuntimeState _PyRuntime = {
     .initialized = 0,
     .core_initialized = 0,
     // Py_DecodeLocale() relies on PyMem_RawMalloc() and PyMem_RawFree()
-    // and may be needed before before runtime initialization.  To allow
-    // for this we pre-initialize the raw allocator statically here.
+    // and may be needed before runtime initialization.  To allow for
+    // this we pre-initialize the raw allocator statically here.
     .mem = {
         .allocators = {
             .raw = {