Fix python deprecation warnings

.travis.yml

@@ -135,7 +135,7 @@ script:
     - env LLVM_PROFILE_FILE="$PWD/unittest.profraw" make VERBOSE=1 unittest
     - sudo make install
       # TODO: use DEVEL_COVER_OPTIONS for https://metacpan.org/pod/Devel::Cover
-    - env LLVM_PROFILE_FILE="$PWD/inttest.profraw" ZNC_MODPERL_COVERAGE_OPTS="-db,$PWD/cover_db" make VERBOSE=1 inttest
+    - env LLVM_PROFILE_FILE="$PWD/inttest.profraw" ZNC_MODPERL_COVERAGE_OPTS="-db,$PWD/cover_db" PYTHONWARNINGS=error make VERBOSE=1 inttest
     - /usr/local/bin/znc --version
 after_success:
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ~/perl5/bin/cover --no-gcov --report=clover; fi

CMakeLists.txt

@@ -171,6 +171,7 @@ if(WANT_PERL)
 	find_package(PerlLibs 5.10 REQUIRED)
 endif()
 if (WANT_PYTHON)
+	set (_MIN_PYTHON_VERSION 3.4)
 	find_package(Perl 5.10 REQUIRED)
 	# VERSION_GREATER_EQUAL is available only since 3.7
 	if (CMAKE_VERSION VERSION_LESS 3.12)
@@ -196,8 +197,10 @@ if (WANT_PYTHON)
 				message(FATAL_ERROR "Invalid value of WANT_PYTHON_VERSION")
 			endif()
 		endif()
-		if (Python3_FOUND AND Python3_VERSION VERSION_LESS 3.3)
-			message(STATUS "Python too old, need at least 3.3")
+		if (Python3_FOUND AND Python3_VERSION VERSION_LESS
+				${_MIN_PYTHON_VERSION})
+			message(STATUS
+				"Python too old, need at least ${_MIN_PYTHON_VERSION}")
 			set(Python3_FOUND OFF)
 		else()
 			# Compatibility with pkg-config variables
@@ -206,9 +209,11 @@ if (WANT_PYTHON)
 	endif()
 	if (NOT Python3_FOUND AND WANT_PYTHON_VERSION MATCHES "^python")
 		# Since python 3.8, -embed is required for embedding.
-		pkg_check_modules(Python3 "${WANT_PYTHON_VERSION}-embed >= 3.3")
+		pkg_check_modules(Python3
+			"${WANT_PYTHON_VERSION}-embed >= ${_MIN_PYTHON_VERSION}")
 		if (NOT Python3_FOUND)
-			pkg_check_modules(Python3 "${WANT_PYTHON_VERSION} >= 3.3")
+			pkg_check_modules(Python3
+				"${WANT_PYTHON_VERSION} >= ${_MIN_PYTHON_VERSION}")
 		endif()
 	endif()
 	if (NOT Python3_FOUND)

modules/modpython.cpp

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+#define PY_SSIZE_T_CLEAN
 #include <Python.h>
 
 #include <znc/Chan.h>
@@ -455,7 +456,7 @@ CBSOCK(ConnectionRefused);
 void CPySocket::ReadData(const char* data, size_t len) {
     PyObject* pyRes =
         PyObject_CallMethod(m_pyObj, const_cast<char*>("OnReadData"),
-                            const_cast<char*>("y#"), data, (int)len);
+                            const_cast<char*>("y#"), data, (Py_ssize_t)len);
     CHECKCLEARSOCK("OnReadData");
 }
 

modules/modpython/znc.py

@@ -22,10 +22,13 @@
     _cov.start()
 
 from functools import wraps
-import imp
+import collections.abc
+import importlib.abc
+import importlib.machinery
+import importlib.util
 import re
+import sys
 import traceback
-import collections.abc
 
 from znc_core import *
 
@@ -687,47 +690,85 @@ def make_caller(parent, name, attr):
 make_inherit(Timer, CPyTimer, '_ctimer')
 
 
+class ZNCModuleLoader(importlib.abc.SourceLoader):
+    def __init__(self, modname, pypath):
+        self.pypath = pypath
+
+    def create_module(self, spec):
+        self._datadir = spec.loader_state[0]
+        self._package_dir = spec.loader_state[1]
+        return super().create_module(spec)
+
+    def get_data(self, path):
+        with open(path, 'rb') as f:
+            return f.read()
+
+    def get_filename(self, fullname):
+        return self.pypath
+
+
+class ZNCModuleFinder(importlib.abc.MetaPathFinder):
+    @staticmethod
+    def find_spec(fullname, path, target=None):
+        if fullname == 'znc_modules':
+            spec = importlib.util.spec_from_loader(fullname, None, is_package=True)
+            return spec
+        parts = fullname.split('.')
+        if parts[0] != 'znc_modules':
+            return
+        def dirs():
+            if len(parts) == 2:
+                # common case
+                yield from CModules.GetModDirs()
+            else:
+                # the module is a package and tries to load a submodule of it
+                for libdir in sys.modules['znc_modules.' + parts[1]].__loader__._package_dir:
+                    yield libdir, None
+        for libdir, datadir in dirs():
+            finder = importlib.machinery.FileFinder(libdir,
+                    (ZNCModuleLoader, importlib.machinery.SOURCE_SUFFIXES))
+            spec = finder.find_spec('.'.join(parts[1:]))
+            if spec:
+                spec.name = fullname
+                spec.loader_state = (datadir, spec.submodule_search_locations)
+                # It almost works with original submodule_search_locations,
+                # then python will find submodules of the package itself,
+                # without calling out to ZNCModuleFinder or ZNCModuleLoader.
+                # But updatemod will be flaky for those submodules because as
+                # of py3.8 importlib.invalidate_caches() goes only through
+                # sys.meta_path, but not sys.path_hooks. So we make them load
+                # through ZNCModuleFinder too, but still remember the original
+                # dir so that the whole module comes from a single entry in
+                # CModules.GetModDirs().
+                spec.submodule_search_locations = []
+                return spec
+
+
+sys.meta_path.append(ZNCModuleFinder())
+
+_py_modules = set()
+
 def find_open(modname):
     '''Returns (pymodule, datapath)'''
-    for d in CModules.GetModDirs():
-        # d == (libdir, datadir)
-        try:
-            x = imp.find_module(modname, [d[0]])
-        except ImportError:
-            # no such file in dir d
-            continue
-        # x == (<open file './modules/admin.so', mode 'rb' at 0x7fa2dc748d20>,
-        #       './modules/admin.so', ('.so', 'rb', 3))
-        # x == (<open file './modules/pythontest.py', mode 'U' at
-        #       0x7fa2dc748d20>, './modules/pythontest.py', ('.py', 'U', 1))
-        if x[0] is None and x[2][2] != imp.PKG_DIRECTORY:
-            # the same
-            continue
-        if x[2][0] == '.so':
-            try:
-                pymodule = imp.load_module(modname, *x)
-            except ImportError:
-                # found needed .so but can't load it...
-                # maybe it's normal (non-python) znc module?
-                # another option here could be to "continue"
-                # search of python module in other moddirs.
-                # but... we respect C++ modules ;)
-                return (None, None)
-            finally:
-                x[0].close()
-        else:
-            # this is not .so, so it can be only python module .py or .pyc
-            try:
-                pymodule = imp.load_module(modname, *x)
-            finally:
-                if x[0]:
-                    x[0].close()
-        return (pymodule, d[1]+modname)
+    fullname = 'znc_modules.' + modname
+    for m in _py_modules:
+        if m.GetModName() == modname:
+            break
     else:
-        # nothing found
+        # module is not loaded, clean up previous attempts to load it or even
+        # to list as available modules
+        # This is to to let updatemod work
+        to_remove = []
+        for m in sys.modules:
+            if m == fullname or m.startswith(fullname + '.'):
+                to_remove.append(m)
+        for m in to_remove:
+            del sys.modules[m]
+    try:
+        module = importlib.import_module(fullname)
+    except ImportError:
         return (None, None)
-
-_py_modules = set()
+    return (module, os.path.join(module.__loader__._datadir, modname))
 
 def load_module(modname, args, module_type, user, network, retmsg, modpython):
     '''Returns 0 if not found, 1 on loading error, 2 on success'''

test/integration/tests/scripting.cpp

@@ -243,5 +243,41 @@ TEST_F(ZNCTest, ModperlNV) {
     client.ReadUntil(":a b");
 }
 
+TEST_F(ZNCTest, ModpythonPackage) {
+    if (QProcessEnvironment::systemEnvironment().value(
+            "DISABLED_ZNC_PERL_PYTHON_TEST") == "1") {
+        return;
+    }
+    auto znc = Run();
+    znc->CanLeak();
+
+    QDir dir(m_dir.path());
+    ASSERT_TRUE(dir.mkpath("modules"));
+    ASSERT_TRUE(dir.cd("modules"));
+    ASSERT_TRUE(dir.mkpath("packagetest"));
+    InstallModule("packagetest/__init__.py", R"(
+        import znc
+        from .data import value
+
+        class packagetest(znc.Module):
+            def OnModCommand(self, cmd):
+                self.PutModule('value = ' + value)
+    )");
+    InstallModule("packagetest/data.py", "value = 'a'");
+
+    auto ircd = ConnectIRCd();
+    auto client = LoginClient();
+    client.Write("znc loadmod modpython");
+    client.Write("znc loadmod packagetest");
+    client.Write("PRIVMSG *packagetest :foo");
+    client.ReadUntil("value = a");
+    InstallModule("packagetest/data.py", "value = 'b'");
+    client.Write("PRIVMSG *packagetest :foo");
+    client.ReadUntil("value = a");
+    client.Write("znc updatemod packagetest");
+    client.Write("PRIVMSG *packagetest :foo");
+    client.ReadUntil("value = b");
+}
+
 }  // namespace
 }  // namespace znc_inttest