patch 8.1.0201: newer Python uses "importlib" instead of "imp"

Problem:    Newer Python uses "importlib" instead of "imp".
Solution:   Use "importlib" for newer Python versions. (closes #3163)

Author: brammool

src/if_py_both.h

@@ -88,8 +88,12 @@ static PyObject *py_getcwd;
 static PyObject *vim_module;
 static PyObject *vim_special_path_object;
 
+#if PY_VERSION_HEX >= 0x030700f0
+static PyObject *py_find_spec;
+#else
 static PyObject *py_find_module;
 static PyObject *py_load_module;
+#endif
 
 static PyObject *VimError;
 
@@ -539,6 +543,7 @@ PythonIO_Init_io(void)
     return 0;
 }
 
+#if PY_VERSION_HEX < 0x030700f0
 typedef struct
 {
     PyObject_HEAD
@@ -567,6 +572,7 @@ static struct PyMethodDef LoaderMethods[] = {
     {"load_module", (PyCFunction)LoaderLoadModule,	METH_VARARGS,	""},
     { NULL,	    NULL,				0,		NULL}
 };
+#endif
 
 /* Check to see whether a Vim error has been reported, or a keyboard
  * interrupt has been detected.
@@ -1163,6 +1169,37 @@ Vim_GetPaths(PyObject *self UNUSED)
     return ret;
 }
 
+#if PY_VERSION_HEX >= 0x030700f0
+    static PyObject *
+FinderFindSpec(PyObject *self, PyObject *args)
+{
+    char	*fullname;
+    PyObject	*paths;
+    PyObject	*target = Py_None;
+    PyObject	*spec;
+
+    if (!PyArg_ParseTuple(args, "s|O", &fullname, &target))
+	return NULL;
+
+    if (!(paths = Vim_GetPaths(self)))
+	return NULL;
+
+    spec = PyObject_CallFunction(py_find_spec, "sNN", fullname, paths, target);
+
+    Py_DECREF(paths);
+
+    if (!spec)
+    {
+	if (PyErr_Occurred())
+	    return NULL;
+
+	Py_INCREF(Py_None);
+	return Py_None;
+    }
+
+    return spec;
+}
+#else
     static PyObject *
 call_load_module(char *name, int len, PyObject *find_module_result)
 {
@@ -1305,6 +1342,7 @@ FinderFindModule(PyObject *self, PyObject *args)
 
     return (PyObject *) loader;
 }
+#endif
 
     static PyObject *
 VimPathHook(PyObject *self UNUSED, PyObject *args)
@@ -1336,7 +1374,11 @@ static struct PyMethodDef VimMethods[] = {
     {"chdir",	    (PyCFunction)VimChdir,	METH_VARARGS|METH_KEYWORDS,	"Change directory"},
     {"fchdir",	    (PyCFunction)VimFchdir,	METH_VARARGS|METH_KEYWORDS,	"Change directory"},
     {"foreach_rtp", VimForeachRTP,		METH_O,				"Call given callable for each path in &rtp"},
+#if PY_VERSION_HEX >= 0x030700f0
+    {"find_spec",   FinderFindSpec,		METH_VARARGS,			"Internal use only, returns spec object for any input it receives"},
+#else
     {"find_module", FinderFindModule,		METH_VARARGS,			"Internal use only, returns loader object for any input it receives"},
+#endif
     {"path_hook",   VimPathHook,		METH_VARARGS,			"Hook function to install in sys.path_hooks"},
     {"_get_paths",  (PyCFunction)Vim_GetPaths,	METH_NOARGS,			"Get &rtp-based additions to sys.path"},
     { NULL,	    NULL,			0,				NULL}
@@ -6545,13 +6587,15 @@ init_structs(void)
     OptionsType.tp_traverse = (traverseproc)OptionsTraverse;
     OptionsType.tp_clear = (inquiry)OptionsClear;
 
+#if PY_VERSION_HEX < 0x030700f0
     vim_memset(&LoaderType, 0, sizeof(LoaderType));
     LoaderType.tp_name = "vim.Loader";
     LoaderType.tp_basicsize = sizeof(LoaderObject);
     LoaderType.tp_flags = Py_TPFLAGS_DEFAULT;
     LoaderType.tp_doc = "vim message object";
     LoaderType.tp_methods = LoaderMethods;
     LoaderType.tp_dealloc = (destructor)LoaderDestructor;
+#endif
 
 #if PY_MAJOR_VERSION >= 3
     vim_memset(&vimmodule, 0, sizeof(vimmodule));
@@ -6583,7 +6627,9 @@ init_types(void)
     PYTYPE_READY(FunctionType);
     PYTYPE_READY(OptionsType);
     PYTYPE_READY(OutputType);
+#if PY_VERSION_HEX < 0x030700f0
     PYTYPE_READY(LoaderType);
+#endif
     return 0;
 }
 
@@ -6707,7 +6753,9 @@ static struct object_constant {
     {"List",       (PyObject *)&ListType},
     {"Function",   (PyObject *)&FunctionType},
     {"Options",    (PyObject *)&OptionsType},
+#if PY_VERSION_HEX < 0x030700f0
     {"_Loader",    (PyObject *)&LoaderType},
+#endif
 };
 
 #define ADD_OBJECT(m, name, obj) \
@@ -6729,6 +6777,10 @@ populate_module(PyObject *m)
     PyObject	*other_module;
     PyObject	*attr;
     PyObject	*imp;
+#if PY_VERSION_HEX >= 0x030700f0
+    PyObject	*dict;
+    PyObject	*cls;
+#endif
 
     for (i = 0; i < (int)(sizeof(numeric_constants)
 					   / sizeof(struct numeric_constant));
@@ -6801,6 +6853,28 @@ populate_module(PyObject *m)
 
     ADD_OBJECT(m, "VIM_SPECIAL_PATH", vim_special_path_object);
 
+#if PY_VERSION_HEX >= 0x030700f0
+    if (!(imp = PyImport_ImportModule("importlib.machinery")))
+	return -1;
+
+    dict = PyModule_GetDict(imp);
+
+    if (!(cls = PyDict_GetItemString(dict, "PathFinder")))
+    {
+	Py_DECREF(imp);
+	return -1;
+    }
+
+    if (!(py_find_spec = PyObject_GetAttrString(cls, "find_spec")))
+    {
+	Py_DECREF(imp);
+	return -1;
+    }
+
+    Py_DECREF(imp);
+
+    ADD_OBJECT(m, "_find_spec", py_find_spec);
+#else
     if (!(imp = PyImport_ImportModule("imp")))
 	return -1;
 
@@ -6821,6 +6895,7 @@ populate_module(PyObject *m)
 
     ADD_OBJECT(m, "_find_module", py_find_module);
     ADD_OBJECT(m, "_load_module", py_load_module);
+#endif
 
     return 0;
 }

src/testdir/test87.in

@@ -219,6 +219,7 @@ import sys
 import re
 
 py33_type_error_pattern = re.compile('^__call__\(\) takes (\d+) positional argument but (\d+) were given$')
+py37_exception_repr = re.compile(r'([^\(\),])(\)+)$')
 
 def ee(expr, g=globals(), l=locals()):
     cb = vim.current.buffer
@@ -227,17 +228,17 @@ def ee(expr, g=globals(), l=locals()):
             exec(expr, g, l)
         except Exception as e:
             if sys.version_info >= (3, 3) and e.__class__ is AttributeError and str(e).find('has no attribute')>=0 and not str(e).startswith("'vim."):
-                cb.append(expr + ':' + repr((e.__class__, AttributeError(str(e)[str(e).rfind(" '") + 2:-1]))))
+                msg = repr((e.__class__, AttributeError(str(e)[str(e).rfind(" '") + 2:-1])))
             elif sys.version_info >= (3, 3) and e.__class__ is ImportError and str(e).find('No module named \'') >= 0:
-                cb.append(expr + ':' + repr((e.__class__, ImportError(str(e).replace("'", '')))))
+                msg = repr((e.__class__, ImportError(str(e).replace("'", ''))))
             elif sys.version_info >= (3, 6) and e.__class__ is ModuleNotFoundError:
                 # Python 3.6 gives ModuleNotFoundError, change it to an ImportError
-                cb.append(expr + ':' + repr((ImportError, ImportError(str(e).replace("'", '')))))
+                msg = repr((ImportError, ImportError(str(e).replace("'", ''))))
             elif sys.version_info >= (3, 3) and e.__class__ is TypeError:
                 m = py33_type_error_pattern.search(str(e))
                 if m:
                     msg = '__call__() takes exactly {0} positional argument ({1} given)'.format(m.group(1), m.group(2))
-                    cb.append(expr + ':' + repr((e.__class__, TypeError(msg))))
+                    msg = repr((e.__class__, TypeError(msg)))
                 else:
                     msg = repr((e.__class__, e))
                     # Messages changed with Python 3.6, change new to old.
@@ -249,9 +250,8 @@ def ee(expr, g=globals(), l=locals()):
                     oldmsg2 = '''"Can't convert 'int' object to str implicitly"'''
                     if msg.find(newmsg2) > -1:
                         msg = msg.replace(newmsg2, oldmsg2)
-                    cb.append(expr + ':' + msg)
             elif sys.version_info >= (3, 5) and e.__class__ is ValueError and str(e) == 'embedded null byte':
-                cb.append(expr + ':' + repr((TypeError, TypeError('expected bytes with no null'))))
+                msg = repr((TypeError, TypeError('expected bytes with no null')))
             else:
                 msg = repr((e.__class__, e))
                 # Some Python versions say can't, others cannot.
@@ -262,11 +262,16 @@ def ee(expr, g=globals(), l=locals()):
                     msg = msg.replace('"cannot ', '\'cannot ')
                 if msg.find(' attributes"') > -1:
                     msg = msg.replace(' attributes"', ' attributes\'')
-                cb.append(expr + ':' + msg)
+            if sys.version_info >= (3, 7):
+                msg = py37_exception_repr.sub(r'\1,\2', msg)
+            cb.append(expr + ':' + msg)
         else:
             cb.append(expr + ':NOT FAILED')
     except Exception as e:
-        cb.append(expr + '::' + repr((e.__class__, e)))
+        msg = repr((e.__class__, e))
+        if sys.version_info >= (3, 7):
+            msg = py37_exception_repr.sub(r'\1,\2', msg)
+        cb.append(expr + '::' + msg)
 EOF
 :fun New(...)
 :   return ['NewStart']+a:000+['NewEnd']

src/version.c

@@ -789,6 +789,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    201,
 /**/
     200,
 /**/