Skip to content

Commit

Permalink
app_python3: implement script reloading
Browse files Browse the repository at this point in the history 
Copy the method used in app_lua: the version of the script is incremented by RPC.
In python_exec.c:apy_exec() check for a newer version and reload the script.

Set a lock so the script reload only occurs at depth 0 (in the unlikely case
that apy_exec() is called recursively).

This is not thread-safe as we are using a process-wide lock: don't call back
into apy_exec() from a Python extension that uses threads.
  • Loading branch information
@aalba6675 @miconda
aalba6675 authored and miconda committed Feb 26, 2018
1 parent 6568170 commit 7f463b3
Show file tree
Hide file tree
Showing 5 changed files with 191 additions and 190 deletions.
220 changes: 132 additions & 88 deletions src/modules/app_python3/app_python_mod.c
View file
Expand Up @@ -51,7 +51,7 @@ static int mod_init(void);
static int child_init(int rank);
static void mod_destroy(void);

PyObject *_sr_apy_handler_obj;
PyObject *_sr_apy_handler_obj = NULL;

char *dname = NULL, *bname = NULL;

Expand Down Expand Up @@ -197,161 +197,215 @@ static void mod_destroy(void)

#define PY_GIL_ENSURE gstate = PyGILState_Ensure();
#define PY_GIL_RELEASE PyGILState_Release(gstate);

// #define PY_THREADSTATE_SWAP_IN PyThreadState_Swap(myThreadState);
// #define PY_THREADSTATE_SWAP_NULL PyThreadState_Swap(NULL);
#define PY_THREADSTATE_SWAP_IN
#define PY_THREADSTATE_SWAP_NULL

int apy_load_script(void)
int apy_mod_init(PyObject* pModule)
{
PyObject *sys_path, *pDir, *pModule, *pFunc, *pArgs;
PyThreadState *mainThreadState;
PyGILState_STATE gstate;

if (ap_init_modules() != 0) {
return -1;
}

Py_Initialize();
PyEval_InitThreads();
myThreadState = PyThreadState_Get();
/*
* pModule: managed by caller, no need to Py_DECREF
*/
PyObject *pFunc, *pArgs, *pHandler;
PyGILState_STATE gstate;

PY_GIL_ENSURE
format_exc_obj = InitTracebackModule();
pFunc = PyObject_GetAttrString(pModule, mod_init_fname.s);

if (format_exc_obj == NULL || !PyCallable_Check(format_exc_obj))
{
Py_XDECREF(format_exc_obj);
PY_GIL_RELEASE
return -1;
}
/* pFunc is a new reference */

sys_path = PySys_GetObject("path");
/* PySys_GetObject doesn't pass reference! No need to DEREF */
if (sys_path == NULL) {
if (pFunc == NULL) {
if (!PyErr_Occurred())
PyErr_Format(PyExc_AttributeError,
"'module' object 'sys' has no attribute 'path'");
"'module' object '%s' has no attribute '%s'",
bname, mod_init_fname.s);
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
Py_XDECREF(pFunc);
PY_GIL_RELEASE
return -1;
}

pDir = PyUnicode_FromString(dname);
if (pDir == NULL) {
if (!PyCallable_Check(pFunc)) {
if (!PyErr_Occurred())
PyErr_Format(PyExc_AttributeError,
"PyUnicode_FromString() has failed");
"module object '%s' has is not callable attribute '%s'",
bname, mod_init_fname.s);
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
Py_XDECREF(pFunc);
PY_GIL_RELEASE
return -1;
}

PyList_Insert(sys_path, 0, pDir);
Py_DECREF(pDir);

if (python_msgobj_init() != 0) {
if (!PyErr_Occurred())
PyErr_SetString(PyExc_AttributeError,
"python_msgobj_init() has failed");
pArgs = PyTuple_New(0);
if (pArgs == NULL) {
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
Py_DECREF(pFunc);
PY_GIL_RELEASE
return -1;
}

pModule = PyImport_ImportModule(bname);
if (pModule == NULL) {
PyErr_PrintEx(0);
pHandler = PyObject_CallObject(pFunc, pArgs);

Py_XDECREF(pFunc);
Py_XDECREF(pArgs);

if (pHandler == Py_None) {
if (!PyErr_Occurred())
PyErr_Format(PyExc_ImportError, "No module named '%s'", bname);
PyErr_Format(PyExc_TypeError,
"Function '%s' of module '%s' has returned None."
" Should be a class instance.", mod_init_fname.s, bname);
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
PY_GIL_RELEASE

return -1;
}

pFunc = PyObject_GetAttrString(pModule, mod_init_fname.s);
Py_DECREF(pModule);

/* pFunc is a new reference */

if (pFunc == NULL) {
if (!PyErr_Occurred())
PyErr_Format(PyExc_AttributeError,
"'module' object '%s' has no attribute '%s'",
bname, mod_init_fname.s);
if (PyErr_Occurred()) {
python_handle_exception("mod_init");
Py_XDECREF(_sr_apy_handler_obj);
Py_DECREF(format_exc_obj);
Py_XDECREF(pFunc);
PY_GIL_RELEASE
return -1;
}

if (!PyCallable_Check(pFunc)) {
if (pHandler == NULL) {
LM_ERR("PyObject_CallObject() returned NULL but no exception!\n");
if (!PyErr_Occurred())
PyErr_Format(PyExc_AttributeError,
"module object '%s' has is not callable attribute '%s'",
bname, mod_init_fname.s);
PyErr_Format(PyExc_TypeError,
"Function '%s' of module '%s' has returned not returned"
" object. Should be a class instance.",
mod_init_fname.s, bname);
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
Py_XDECREF(pFunc);
PY_GIL_RELEASE
return -1;
}
Py_XDECREF(_sr_apy_handler_obj);
_sr_apy_handler_obj = pHandler;
PY_GIL_RELEASE
return 0;
}


pArgs = PyTuple_New(0);
if (pArgs == NULL) {
/*
* reference to the module to allow reload
*/
static PyObject *_sr_apy_module;

int apy_reload_script(void)
{
PyGILState_STATE gstate;

PY_GIL_ENSURE
PyObject *pModule = PyImport_ReloadModule(_sr_apy_module);
if (!pModule) {
PyErr_PrintEx(0);
if (!PyErr_Occurred())
PyErr_Format(PyExc_ImportError, "Reload module '%s'", bname);
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
Py_DECREF(pFunc);
PY_GIL_RELEASE

return -1;
}
if (apy_mod_init(pModule)) {
LM_ERR("Error calling mod_init on reload\n");
Py_DECREF(pModule);
PY_GIL_RELEASE
return -1;

_sr_apy_handler_obj = PyObject_CallObject(pFunc, pArgs);
}
Py_DECREF(_sr_apy_module);
_sr_apy_module = pModule;

Py_XDECREF(pFunc);
Py_XDECREF(pArgs);
if(apy_init_script(_apy_process_rank)<0) {
LM_ERR("failed to init script\n");
return -1;
}
PY_GIL_RELEASE
return 0;
}

int apy_load_script(void)
{
PyObject *sys_path, *pDir, *pModule;
PyGILState_STATE gstate;

if (ap_init_modules() != 0) {
return -1;
}

Py_Initialize();
PyEval_InitThreads();
myThreadState = PyThreadState_Get();

PY_GIL_ENSURE
format_exc_obj = InitTracebackModule();

if (format_exc_obj == NULL || !PyCallable_Check(format_exc_obj))
{
Py_XDECREF(format_exc_obj);
PY_GIL_RELEASE
return -1;
}

sys_path = PySys_GetObject("path");
/* PySys_GetObject doesn't pass reference! No need to DEREF */
if (sys_path == NULL) {
if (!PyErr_Occurred())
PyErr_Format(PyExc_AttributeError,
"'module' object 'sys' has no attribute 'path'");
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
PY_GIL_RELEASE
return -1;
}

if (_sr_apy_handler_obj == Py_None) {
pDir = PyUnicode_FromString(dname);
if (pDir == NULL) {
if (!PyErr_Occurred())
PyErr_Format(PyExc_TypeError,
"Function '%s' of module '%s' has returned None."
" Should be a class instance.", mod_init_fname.s, bname);
PyErr_Format(PyExc_AttributeError,
"PyUnicode_FromString() has failed");
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
PY_GIL_RELEASE
return -1;
}

if (PyErr_Occurred()) {
PyList_Insert(sys_path, 0, pDir);
Py_DECREF(pDir);

if (python_msgobj_init() != 0) {
if (!PyErr_Occurred())
PyErr_SetString(PyExc_AttributeError,
"python_msgobj_init() has failed");
python_handle_exception("mod_init");
Py_XDECREF(_sr_apy_handler_obj);
Py_DECREF(format_exc_obj);
PY_GIL_RELEASE
return -1;
}

if (_sr_apy_handler_obj == NULL) {
LM_ERR("PyObject_CallObject() returned NULL but no exception!\n");
pModule = PyImport_ImportModule(bname);
if (pModule == NULL) {
PyErr_PrintEx(0);
if (!PyErr_Occurred())
PyErr_Format(PyExc_TypeError,
"Function '%s' of module '%s' has returned not returned"
" object. Should be a class instance.",
mod_init_fname.s, bname);
PyErr_Format(PyExc_ImportError, "No module named '%s'", bname);
python_handle_exception("mod_init");
Py_DECREF(format_exc_obj);
PY_GIL_RELEASE

return -1;
}
if (apy_mod_init(pModule) != 0) {
LM_ERR("Error calling mod_init\n");
Py_DECREF(pModule);
PY_GIL_RELEASE
return -1;

}
_sr_apy_module = pModule;

//myThreadState = PyThreadState_New(mainThreadState->interp);
PY_GIL_RELEASE
return 0;
}
Expand All @@ -365,7 +419,6 @@ int apy_init_script(int rank)


PY_GIL_ENSURE
PY_THREADSTATE_SWAP_IN

// get instance class name
classname = get_instance_class_name(_sr_apy_handler_obj);
Expand All @@ -376,7 +429,6 @@ int apy_init_script(int rank)
"'module' instance has no class name");
python_handle_exception("child_init");
Py_DECREF(format_exc_obj);
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE
return -1;
}
Expand All @@ -387,7 +439,6 @@ int apy_init_script(int rank)
python_handle_exception("child_init");
Py_XDECREF(pFunc);
Py_DECREF(format_exc_obj);
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE
return -1;
}
Expand All @@ -400,7 +451,6 @@ int apy_init_script(int rank)
python_handle_exception("child_init");
Py_DECREF(format_exc_obj);
Py_XDECREF(pFunc);
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE
return -1;
}
Expand All @@ -410,7 +460,6 @@ int apy_init_script(int rank)
python_handle_exception("child_init");
Py_DECREF(format_exc_obj);
Py_DECREF(pFunc);
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE
return -1;
}
Expand All @@ -421,7 +470,6 @@ int apy_init_script(int rank)
Py_DECREF(format_exc_obj);
Py_DECREF(pArgs);
Py_DECREF(pFunc);
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE
return -1;
}
Expand All @@ -436,14 +484,12 @@ int apy_init_script(int rank)
python_handle_exception("child_init");
Py_DECREF(format_exc_obj);
Py_XDECREF(pResult);
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE
return -1;
}

if (pResult == NULL) {
LM_ERR("PyObject_CallObject() returned NULL but no exception!\n");
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE
return -1;
}
Expand All @@ -457,14 +503,12 @@ int apy_init_script(int rank)
python_handle_exception("child_init");
Py_DECREF(format_exc_obj);
Py_XDECREF(pResult);
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE
return -1;
}

rval = PyLong_AsLong(pResult);
Py_DECREF(pResult);
PY_THREADSTATE_SWAP_NULL
PY_GIL_RELEASE

return rval;
Expand Down
2 changes: 1 addition & 1 deletion src/modules/app_python3/app_python_mod.h
View file
Expand Up @@ -27,5 +27,5 @@
extern PyObject *_sr_apy_handler_obj;
extern PyObject *format_exc_obj;
extern PyThreadState *myThreadState;

int apy_reload_script(void);
#endif

0 comments on commit 7f463b3

Please sign in to comment.