Massive changes for separate thread state management.

All per-thread globals are moved into a struct which is manipulated
separately.

Author: gvanrossum

Include/Python.h

@@ -95,6 +95,8 @@ PERFORMANCE OF THIS SOFTWARE.
 #include "import.h"
 #include "bltinmodule.h"
 
+#include "pystate.h"
+
 #include "abstract.h"
 
 #define PyArg_GetInt(v, a)	PyArg_Parse((v), "i", (a))

Include/frameobject.h

@@ -52,6 +52,8 @@ typedef struct _frame {
 	PyObject *f_locals;	/* local symbol table (PyDictObject) */
 	PyObject **f_valuestack; /* points after the last local */
 	PyObject *f_trace;	/* Trace function */
+	PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
+	PyThreadState *f_tstate;
 	int f_lasti;		/* Last instruction if called */
 	int f_lineno;		/* Current line number */
 	int f_restricted;	/* Flag set if restricted operations
@@ -71,7 +73,7 @@ extern DL_IMPORT(PyTypeObject) PyFrame_Type;
 #define PyFrame_Check(op) ((op)->ob_type == &PyFrame_Type)
 
 PyFrameObject * PyFrame_New
-	Py_PROTO((PyFrameObject *, PyCodeObject *,
+	Py_PROTO((PyThreadState *, PyCodeObject *,
 		  PyObject *, PyObject *));
 
 

Include/pystate.h

@@ -0,0 +1,134 @@
+#ifndef Py_PYSTATE_H
+#define Py_PYSTATE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/***********************************************************
+Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
+The Netherlands.
+
+                        All Rights Reserved
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the names of Stichting Mathematisch
+Centrum or CWI or Corporation for National Research Initiatives or
+CNRI not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior
+permission.
+
+While CWI is the initial source for this software, a modified version
+is made available by the Corporation for National Research Initiatives
+(CNRI) at the Internet address ftp://ftp.python.org.
+
+STICHTING MATHEMATISCH CENTRUM AND CNRI DISCLAIM ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH
+CENTRUM OR CNRI BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
+DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+
+******************************************************************/
+
+/* Thread and interpreter state structures and their interfaces */
+
+
+/* State shared between threads */
+
+#define NEXITFUNCS 32
+
+typedef struct _is {
+
+	PyObject *import_modules;
+	PyObject *sysdict;
+
+	int nthreads;
+
+	void (*exitfuncs[NEXITFUNCS])();
+	int nexitfuncs;
+
+} PyInterpreterState;
+
+
+/* State unique per thread */
+
+struct _frame; /* Avoid including frameobject.h */
+
+typedef struct _ts {
+
+	PyInterpreterState *interpreter_state;
+
+	struct _frame *frame;
+	int recursion_depth;
+	int ticker;
+	int tracing;
+
+	PyObject *sys_profilefunc;
+	PyObject *sys_tracefunc;
+	int sys_checkinterval;
+
+	PyObject *curexc_type;
+	PyObject *curexc_value;
+	PyObject *curexc_traceback;
+
+	PyObject *exc_type;
+	PyObject *exc_value;
+	PyObject *exc_traceback;
+
+	/* XXX Other state that should be here:
+	   - signal handlers
+	   - low-level "pending calls"
+	   Problem with both is that they may be referenced from
+	   interrupt handlers where there is no clear concept of a
+	   "current thread"???
+	*/
+
+} PyThreadState;
+
+
+PyInterpreterState *PyInterpreterState_New(void);
+void PyInterpreterState_Delete(PyInterpreterState *);
+
+PyThreadState *PyThreadState_New(PyInterpreterState *);
+void PyThreadState_Delete(PyThreadState *);
+
+PyThreadState *PyThreadState_Get(void);
+PyThreadState *PyThreadState_Swap(PyThreadState *);
+
+/* Some background.
+
+   There are lots of issues here.
+
+   First, we can build Python without threads, with threads, or (when
+   Greg Stein's mods are out of beta, on some platforms) with free
+   threading.
+
+   Next, assuming some form of threading is used, there can be several
+   kinds of threads.  Python code can create threads with the thread
+   module.  C code can create threads with the interface defined in
+   python's "thread.h".  Or C code can create threads directly with
+   the OS threads interface (e.g. Solaris threads, SGI threads or
+   pthreads, whatever is being used, as long as it's the same that
+   Python is configured for).
+
+   Next, let's discuss sharing of interpreter state between threads.
+   The exception state (sys.exc_* currently) should never be shared
+   between threads, because it is stack frame specific.  The contents
+   of the sys module, in particular sys.modules and sys.path, are
+   generally shared between threads.  But occasionally it is useful to
+   have separate module collections, e.g. when threads originate in C
+   code and are used to execute unrelated Python scripts.
+   (Traditionally, one would use separate processes for this, but
+   there are lots of reasons why threads are attractive.)
+
+*/
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PYSTATE_H */

Include/pythread.h

@@ -35,6 +35,10 @@ extern "C" {
 #define up_sema PyThread_up_sema
 #define exit_prog PyThread_exit_prog
 #define _exit_prog PyThread__exit_prog
+#define create_key PyThread_create_key
+#define delete_key PyThread_delete_key
+#define get_key_value PyThread_get_key_value
+#define set_key_value PyThread_set_key_value
 
 
 void init_thread Py_PROTO((void));
@@ -62,6 +66,11 @@ void exit_prog Py_PROTO((int));
 void _exit_prog Py_PROTO((int));
 #endif
 
+int create_key Py_PROTO((void));
+void delete_key Py_PROTO((int));
+int set_key_value Py_PROTO((int, void *));
+void * get_key_value Py_PROTO((int));
+
 #ifdef __cplusplus
 }
 #endif

Include/thread.h

@@ -35,6 +35,10 @@ extern "C" {
 #define up_sema PyThread_up_sema
 #define exit_prog PyThread_exit_prog
 #define _exit_prog PyThread__exit_prog
+#define create_key PyThread_create_key
+#define delete_key PyThread_delete_key
+#define get_key_value PyThread_get_key_value
+#define set_key_value PyThread_set_key_value
 
 
 void init_thread Py_PROTO((void));
@@ -62,6 +66,11 @@ void exit_prog Py_PROTO((int));
 void _exit_prog Py_PROTO((int));
 #endif
 
+int create_key Py_PROTO((void));
+void delete_key Py_PROTO((int));
+int set_key_value Py_PROTO((int, void *));
+void * get_key_value Py_PROTO((int));
+
 #ifdef __cplusplus
 }
 #endif

Modules/threadmodule.c

@@ -35,14 +35,13 @@ PERFORMANCE OF THIS SOFTWARE.
 #include "Python.h"
 
 #ifndef WITH_THREAD
-Error!  The rest of Python is not compiled with thread support.
-Rerun configure, adding a --with-thread option.
+#error "Error!  The rest of Python is not compiled with thread support."
+#error "Rerun configure, adding a --with-thread option."
+#error "Then run `make clean' followed by `make'."
 #endif
 
 #include "thread.h"
 
-extern int _PyThread_Started;
-
 static PyObject *ThreadError;
 
 
@@ -192,52 +191,90 @@ static PyTypeObject Locktype = {
 
 /* Module functions */
 
+struct bootstate {
+	PyInterpreterState *interp;
+	PyObject *func;
+	PyObject *args;
+	PyObject *keyw;
+};
+
 static void
-t_bootstrap(args_raw)
-	void *args_raw;
+t_bootstrap(boot_raw)
+	void *boot_raw;
 {
-	PyObject *args = (PyObject *) args_raw;
-	PyObject *func, *arg, *res;
-
-	_PyThread_Started++;
+	struct bootstate *boot = (struct bootstate *) boot_raw;
+	PyThreadState *alttstate, *tstate;
+	PyObject *res;
 
+	tstate = PyThreadState_New(boot->interp);
 	PyEval_RestoreThread((void *)NULL);
-	func = PyTuple_GetItem(args, 0);
-	arg = PyTuple_GetItem(args, 1);
-	res = PyEval_CallObject(func, arg);
-	Py_DECREF(args); /* Matches the INCREF(args) in thread_start_new_thread */
+	alttstate = PyThreadState_Swap(tstate);
+	res = PyEval_CallObjectWithKeywords(
+		boot->func, boot->args, boot->keyw);
+	Py_DECREF(boot->func);
+	Py_DECREF(boot->args);
+	Py_XDECREF(boot->keyw);
+	PyMem_DEL(boot_raw);
 	if (res == NULL) {
 		if (PyErr_Occurred() == PyExc_SystemExit)
 			PyErr_Clear();
 		else {
 			fprintf(stderr, "Unhandled exception in thread:\n");
-			PyErr_Print(); /* From pythonmain.c */
+			PyErr_Print();
 		}
 	}
 	else
 		Py_DECREF(res);
-	(void) PyEval_SaveThread(); /* Should always be NULL */
+	(void) PyThreadState_Swap(alttstate);
+	(void) PyEval_SaveThread();
+	PyThreadState_Delete(tstate);
 	exit_thread();
 }
 
 static PyObject *
-thread_start_new_thread(self, args)
+thread_start_new_thread(self, fargs)
 	PyObject *self; /* Not used */
-	PyObject *args;
+	PyObject *fargs;
 {
-	PyObject *func, *arg;
+	PyObject *func, *args = NULL, *keyw = NULL;
+	struct bootstate *boot;
 
-	if (!PyArg_Parse(args, "(OO)", &func, &arg))
+	if (!PyArg_ParseTuple(fargs, "OO|O", &func, &args, &keyw))
+		return NULL;
+	if (!PyCallable_Check(func)) {
+		PyErr_SetString(PyExc_TypeError,
+				"first arg must be callable");
 		return NULL;
+	}
+	if (!PyTuple_Check(args)) {
+		PyErr_SetString(PyExc_TypeError,
+				"optional 2nd arg must be a tuple");
+		return NULL;
+	}
+	if (keyw != NULL && !PyDict_Check(keyw)) {
+		PyErr_SetString(PyExc_TypeError,
+				"optional 3rd arg must be a dictionary");
+		return NULL;
+	}
+	boot = PyMem_NEW(struct bootstate, 1);
+	if (boot == NULL)
+		return PyErr_NoMemory();
+	boot->interp = PyThreadState_Get()->interpreter_state;
+	boot->func = func;
+	boot->args = args;
+	boot->keyw = keyw;
+	Py_INCREF(func);
 	Py_INCREF(args);
-	/* Initialize the interpreter's stack save/restore mechanism */
-	PyEval_InitThreads();
-	if (!start_new_thread(t_bootstrap, (void*) args)) {
-		Py_DECREF(args);
+	Py_XINCREF(keyw);
+	PyEval_InitThreads(); /* Start the interpreter's thread-awareness */
+	if (!start_new_thread(t_bootstrap, (void*) boot)) {
 		PyErr_SetString(ThreadError, "can't start new thread\n");
+		Py_DECREF(func);
+		Py_DECREF(args);
+		Py_XDECREF(keyw);
+		PyMem_DEL(boot);
 		return NULL;
 	}
-	/* Otherwise the DECREF(args) is done by t_bootstrap */
 	Py_INCREF(Py_None);
 	return Py_None;
 }
@@ -294,8 +331,8 @@ thread_get_ident(self, args)
 }
 
 static PyMethodDef thread_methods[] = {
-	{"start_new_thread",	(PyCFunction)thread_start_new_thread},
-	{"start_new",		(PyCFunction)thread_start_new_thread},
+	{"start_new_thread",	(PyCFunction)thread_start_new_thread, 1},
+	{"start_new",		(PyCFunction)thread_start_new_thread, 1},
 	{"allocate_lock",	(PyCFunction)thread_allocate_lock},
 	{"allocate",		(PyCFunction)thread_allocate_lock},
 	{"exit_thread",		(PyCFunction)thread_exit_thread},

Objects/frameobject.c

@@ -50,6 +50,9 @@ static struct memberlist frame_memberlist[] = {
 	{"f_lineno",	T_INT,		OFF(f_lineno),	RO},
 	{"f_restricted",T_INT,		OFF(f_restricted),RO},
 	{"f_trace",	T_OBJECT,	OFF(f_trace)},
+	{"f_exc_type",	T_OBJECT,	OFF(f_exc_type)},
+	{"f_exc_value",	T_OBJECT,	OFF(f_exc_value)},
+	{"f_exc_traceback", T_OBJECT,	OFF(f_exc_traceback)},
 	{NULL}	/* Sentinel */
 };
 
@@ -112,6 +115,9 @@ frame_dealloc(f)
 	Py_XDECREF(f->f_globals);
 	Py_XDECREF(f->f_locals);
 	Py_XDECREF(f->f_trace);
+	Py_XDECREF(f->f_exc_type);
+	Py_XDECREF(f->f_exc_value);
+	Py_XDECREF(f->f_exc_traceback);
 	f->f_back = free_list;
 	free_list = f;
 }
@@ -134,12 +140,13 @@ PyTypeObject PyFrame_Type = {
 };
 
 PyFrameObject *
-PyFrame_New(back, code, globals, locals)
-	PyFrameObject *back;
+PyFrame_New(tstate, code, globals, locals)
+	PyThreadState *tstate;
 	PyCodeObject *code;
 	PyObject *globals;
 	PyObject *locals;
 {
+	PyFrameObject *back = tstate->frame;
 	static PyObject *builtin_object;
 	PyFrameObject *f;
 	PyObject *builtins;
@@ -214,6 +221,10 @@ PyFrame_New(back, code, globals, locals)
 	}
 	f->f_locals = locals;
 	f->f_trace = NULL;
+	f->f_exc_type = f->f_exc_value = f->f_exc_traceback = NULL;
+	f->f_tstate = PyThreadState_Get();
+	if (f->f_tstate == NULL)
+		Py_FatalError("can't create new frame without thread");
 
 	f->f_lasti = 0;
 	f->f_lineno = code->co_firstlineno;