Merge #509

509: Add the ability to specify 'dependency paths' which are used during .… r=MatthieuDartiailh a=hognala

…dll dependency resolution.

I tested that this works with `visa library` defined, and when it is not. Also, if this section is not present the .pyvisarc file still loads.

The attached file will allow the latest version of Keysight VISA to work with Python 3.8, even without specifying the .dll (it allows it to find its dependency .dll's).

[.pyvisarc.zip](https://github.com/pyvisa/pyvisa/files/4584801/default.pyvisarc.zip)


Co-authored-by: acopelan <alan_copeland@keysight.com>
Co-authored-by: MatthieuDartiailh <marul@laposte.net>

CHANGES

@@ -4,6 +4,8 @@ PyVISA Changelog
 1.11 (unreleased)
 -----------------
 
+- Add support for dll_extra_paths in .pyvisarc to provide a way to specify paths
+  in which to look for dll on Windows and Python >= 3.8  PR #509
 - Drop Python 2 support PR #486
 - Limit support to Python 3.6+ PR #486
 - Improve the test suite and introduce tests relying on Keysight Virtual

docs/source/faq/faq.rst

@@ -152,6 +152,16 @@ or
      You can also create a `virtual environment`_ for this.
 
 
+OSError: Could not open VISA library: function 'viOpen' not found
+-----------------------------------------------------------------
+
+Starting with Python 3.8, the .dll load behavior has changed on Windows (see
+`https://docs.python.org/3/whatsnew/3.8.html#bpo-36085-whatsnew`_). This causes
+some versions of Keysight VISA to fail to load because it cannot find its .dll
+dependencies. You can solve it by creating a configuration file and setting `dll_extra_paths`
+as described in :ref:`intro-configuring`.
+
+
 Where can I get more information about VISA?
 --------------------------------------------
 
@@ -186,3 +196,6 @@ Where can I get more information about VISA?
 .. _`AUTHORS`: https://github.com/pyvisa/pyvisa/blob/master/AUTHORS
 .. _`Issue Tracker`: https://github.com/pyvisa/pyvisa/issues
 .. _`virtual environment`: http://www.virtualenv.org/en/latest/
+
+.. _`https://docs.python.org/3/whatsnew/3.8.html#bpo-36085-whatsnew`:
+       https://docs.python.org/3/whatsnew/3.8.html#bpo-36085-whatsnew

docs/source/introduction/configuring.rst

@@ -69,6 +69,13 @@ contain the following::
 
 Please note that `[Paths]` is treated case-sensitively.
 
+To specify extra .dll search paths on Windows, use a `.pyvisarc` file like the following::
+
+   [Paths]
+
+   dll_extra_paths: C:\Program Files\Keysight\IO Libraries Suite\bin;C:\Program Files (x86)\Keysight\IO Libraries Suite\bin
+
+
 You can define a site-wide configuration file at
 :file:`/usr/share/pyvisa/.pyvisarc` (It may also be
 :file:`/usr/local/...` depending on the location of your Python).

pyvisa/ctwrapper/highlevel.py

@@ -72,7 +72,10 @@ def get_library_paths():
         :rtype: tuple
         """
 
-        from ..util import LibraryPath, read_user_library_path
+        from ..util import LibraryPath, read_user_library_path, add_user_dll_extra_paths
+
+        # Add extra .dll dependency search paths before attempting to load libraries
+        add_user_dll_extra_paths()
 
         # Try to find IVI libraries using known names.
         tmp = [find_library(library_path)

pyvisa/testsuite/test_util.py

@@ -12,6 +12,8 @@
 from configparser import ConfigParser
 from io import StringIO
 from functools import partial
+from unittest.mock import patch
+from unittest.mock import MagicMock
 
 from pyvisa import util, highlevel
 from pyvisa.ctwrapper import IVIVisaLibrary
@@ -31,21 +33,32 @@ class TestConfigFile(BaseTestCase):
 
     def setUp(self):
         # Skip if a real config file exists
-        if any(os.path.isfile(p)
-               for p in [os.path.join(sys.prefix, "share", "pyvisa", ".pyvisarc"),
-                         os.path.join(os.path.expanduser("~"), ".pyvisarc")]
-              ):
+        if any(
+            os.path.isfile(p)
+            for p in [
+                os.path.join(sys.prefix, "share", "pyvisa", ".pyvisarc"),
+                os.path.join(os.path.expanduser("~"), ".pyvisarc"),
+            ]
+        ):
             self.skipTest(".pyvisarc file exists cannot properly test in this case")
         self.temp_dir = tempfile.TemporaryDirectory()
         os.makedirs(os.path.join(self.temp_dir.name, "share", "pyvisa"))
-        self.config_path = os.path.join(self.temp_dir.name, "share", "pyvisa",
-                                        ".pyvisarc")
+        self.config_path = os.path.join(
+            self.temp_dir.name, "share", "pyvisa", ".pyvisarc"
+        )
         self._prefix = sys.prefix
         sys.prefix = self.temp_dir.name
+        self._platform = sys.platform
+        self._add_dll = os.add_dll_directory if self._platform == "win32" else None
 
     def tearDown(self):
         self.temp_dir.cleanup()
         sys.prefix = self._prefix
+        sys.platform = self._platform
+        if self._add_dll:
+            os.add_dll_directory = self._add_dll
+        elif hasattr(os, "add_dll_directory"):
+            del os.add_dll_directory
 
     def test_reading_config_file(self):
         config = ConfigParser()
@@ -77,6 +90,66 @@ def test_no_config_file(self):
             self.assertIsNone(util.read_user_library_path())
         self.assertIn("No user defined", cm.output[0])
 
+    # --- Test reading dll_extra_paths.
+
+    def mock_add_dll(self):
+        """os.add_dll_directory() exists only on windows, mock it on other platform.
+
+        The teardown takes care of restoring the proper value.
+
+        """
+        mock = MagicMock()
+        mock.__str__.return_value = ""
+        os.add_dll_directory = mock
+
+    def test_reading_config_file_not_windows(self):
+        sys.platform = "darwin"
+        with self.assertLogs(level="DEBUG") as cm:
+            self.assertIsNone(util.add_user_dll_extra_paths())
+        self.assertIn(
+            "Not loading dll_extra_paths because it is not Windows", cm.output[0]
+        )
+
+    def test_reading_config_file_for_dll_extra_paths(self):
+        sys.platform = "win32"
+        self.mock_add_dll()
+        config = ConfigParser()
+        config["Paths"] = {}
+        config["Paths"]["dll_extra_paths"] = r"C:\Program Files;C:\Program Files (x86)"
+        with open(self.config_path, "w") as f:
+            config.write(f)
+        self.assertEqual(
+            util.add_user_dll_extra_paths(), r"C:\Program Files;C:\Program Files (x86)"
+        )
+
+    def test_no_section_for_dll_extra_paths(self):
+        sys.platform = "win32"
+        self.mock_add_dll()
+        config = ConfigParser()
+        with open(self.config_path, "w") as f:
+            config.write(f)
+        with self.assertLogs(level="DEBUG") as cm:
+            self.assertIsNone(util.add_user_dll_extra_paths())
+        self.assertIn("NoOptionError or NoSectionError", cm.output[1])
+
+    def test_no_key_for_dll_extra_paths(self):
+        sys.platform = "win32"
+        self.mock_add_dll()
+        config = ConfigParser()
+        config["Paths"] = {}
+        with open(self.config_path, "w") as f:
+            config.write(f)
+        with self.assertLogs(level="DEBUG") as cm:
+            self.assertIsNone(util.add_user_dll_extra_paths())
+        self.assertIn("NoOptionError or NoSectionError", cm.output[1])
+
+    def test_no_config_file_for_dll_extra_paths(self):
+        sys.platform = "win32"
+        self.mock_add_dll()
+        with self.assertLogs(level="DEBUG") as cm:
+            self.assertIsNone(util.add_user_dll_extra_paths())
+        self.assertIn("No user defined", cm.output[0])
+
 class TestParser(BaseTestCase):
 
     def test_parse_binary(self):

pyvisa/util.py

@@ -64,6 +64,7 @@ def read_user_library_path():
 
         [Paths]
         visa library=/my/path/visa.so
+        dll_extra_paths=/my/otherpath/;/my/otherpath2
 
     Return `None` if  configuration files or keys are not present.
 
@@ -87,6 +88,50 @@ def read_user_library_path():
         logger.debug('NoOptionError or NoSectionError while reading config file')
         return None
 
+def add_user_dll_extra_paths():
+    """Add paths to search for .dll dependencies on Windows
+    stored in one of the following configuration files:
+
+        <sys prefix>/share/pyvisa/.pyvisarc
+        ~/.pyvisarc
+
+    <sys prefix> is the site-specific directory prefix where the platform
+    independent Python files are installed.
+
+    Example configuration file:
+
+        [Paths]
+        visa library=/my/path/visa.so
+        dll_extra_paths=/my/otherpath/;/my/otherpath2
+
+    """
+    from configparser import ConfigParser, NoSectionError, NoOptionError
+
+    this_platform = sys.platform
+    if this_platform.startswith('win'):
+        config_parser = ConfigParser()
+        files = config_parser.read([os.path.join(sys.prefix, "share", "pyvisa",
+                                                ".pyvisarc"),
+                                    os.path.join(os.path.expanduser("~"),
+                                                ".pyvisarc")])
+
+        if not files:
+            logger.debug('No user defined configuration')
+            return None
+
+        logger.debug('User defined configuration files: %s' % files)
+
+        try:
+            dll_extra_paths = config_parser.get("Paths", "dll_extra_paths")
+            for path in dll_extra_paths.split(";"):
+                os.add_dll_directory(path)
+            return dll_extra_paths
+        except (NoOptionError, NoSectionError):
+            logger.debug('NoOptionError or NoSectionError while reading config file for dll_extra_paths')
+            return None
+    else:
+        logger.debug('Not loading dll_extra_paths because it is not Windows')
+        return None
 
 class LibraryPath(str):
 
@@ -719,7 +764,6 @@ def get_arch(filename):
 
     out = subprocess.run(["file", filename], capture_output=True)
     out = out.stdout.decode("ascii")
-    print(out)
     ret = []
     if this_platform.startswith('linux'):
         if '32-bit' in out: