From caf9ec647c6128d20bdb426ae13a3d83d47bcd11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 14 Oct 2020 10:22:40 -0500 Subject: [PATCH 1/7] Fix external DLL loading on wheels --- packaging/wheel/relocate.py | 6 ++++++ torchvision/io/_video_opt.py | 5 +++++ torchvision/io/image.py | 5 +++++ 3 files changed, 16 insertions(+) diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index aa2ecb6036a..e9bd07bef97 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -259,6 +259,12 @@ def relocate_elf_library(patchelf, output_dir, output_library, binary): def relocate_dll_library(dumpbin, output_dir, output_library, binary): + """ + Relocate a DLL/PE shared library to be packaged on a wheel. + + Given a shared library, find the transitive closure of its dependencies, + rename and copy them into the wheel. + """ print('Relocating {0}'.format(binary)) binary_path = osp.join(output_library, binary) diff --git a/torchvision/io/_video_opt.py b/torchvision/io/_video_opt.py index ae4b0f7c869..81606fb6e70 100644 --- a/torchvision/io/_video_opt.py +++ b/torchvision/io/_video_opt.py @@ -15,6 +15,11 @@ try: lib_dir = os.path.join(os.path.dirname(__file__), "..") + if os.name == 'nt': + # This is required to load the wheel external binaries + os.environ['PATH'] = os.pathsep.join( + [os.path.abspath(lib_dir)] + [os.environ['PATH']]) + loader_details = ( importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES diff --git a/torchvision/io/image.py b/torchvision/io/image.py index 1e93c0c8d2a..6c78a5f9ddf 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -9,6 +9,11 @@ try: lib_dir = osp.join(osp.dirname(__file__), "..") + if os.name == 'nt': + # This is required to load the wheel external binaries + os.environ['PATH'] = os.pathsep.join( + [osp.abspath(lib_dir)] + [os.environ['PATH']]) + loader_details = ( importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES From c63b347609154ac87827e0bde8b10e215838f765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 14 Oct 2020 10:57:24 -0500 Subject: [PATCH 2/7] Use SetDefaultDllDirectoriess and AddDllDirectory --- torchvision/extension.py | 33 +++++++++++++++++++++++++++++++++ torchvision/io/_video_opt.py | 7 +------ torchvision/io/image.py | 7 +------ 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/torchvision/extension.py b/torchvision/extension.py index 69433be3b0f..07576cffbd6 100644 --- a/torchvision/extension.py +++ b/torchvision/extension.py @@ -12,6 +12,39 @@ def _register_extensions(): # load the custom_op_library and register the custom ops lib_dir = os.path.dirname(__file__) + if os.name == 'nt': + # Register the main torchvision library location on the default DLL path + import ctypes + import sys + + kernel32 = ctypes.WinDLL('kernel32.dll', use_last_error=True) + with_load_library_flags = hasattr(kernel32, 'AddDllDirectory') + prev_error_mode = kernel32.SetErrorMode(0x0001) + + kernel32.LoadLibraryW.restype = ctypes.c_void_p + if with_load_library_flags: + kernel32.SetDefaultDllDirectories.restype = ctypes.c_bool + kernel32.AddDllDirectory.restype = ctypes.c_void_p + kernel32.LoadLibraryExW.restype = ctypes.c_void_p + + if sys.version_info >= (3, 8): + os.add_dll_directory(lib_dir) + elif with_load_library_flags: + res = kernel32.SetDefaultDllDirectories(0x00001000) + if not res: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += (' Error setting LOAD_LIBRARY_SEARCH_DEFAULT_DIRS' + ' when calling SetDefaultDllDirectories.') + raise err + + res = kernel32.AddDllDirectory(lib_dir) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += f' Error adding "{lib_dir}" to the DLL directories.' + raise err + + kernel32.SetErrorMode(prev_error_mode) + loader_details = ( importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES diff --git a/torchvision/io/_video_opt.py b/torchvision/io/_video_opt.py index 81606fb6e70..686c06c5c29 100644 --- a/torchvision/io/_video_opt.py +++ b/torchvision/io/_video_opt.py @@ -13,12 +13,7 @@ _HAS_VIDEO_OPT = False try: - lib_dir = os.path.join(os.path.dirname(__file__), "..") - - if os.name == 'nt': - # This is required to load the wheel external binaries - os.environ['PATH'] = os.pathsep.join( - [os.path.abspath(lib_dir)] + [os.environ['PATH']]) + lib_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) loader_details = ( importlib.machinery.ExtensionFileLoader, diff --git a/torchvision/io/image.py b/torchvision/io/image.py index 6c78a5f9ddf..b793030ac59 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -7,12 +7,7 @@ _HAS_IMAGE_OPT = False try: - lib_dir = osp.join(osp.dirname(__file__), "..") - - if os.name == 'nt': - # This is required to load the wheel external binaries - os.environ['PATH'] = os.pathsep.join( - [osp.abspath(lib_dir)] + [os.environ['PATH']]) + lib_dir = osp.abspath(osp.join(osp.dirname(__file__), "..")) loader_details = ( importlib.machinery.ExtensionFileLoader, From aa91246e2c23c4e87e6f732ac779178bb50420b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 14 Oct 2020 11:21:15 -0500 Subject: [PATCH 3/7] Add previous paths --- torchvision/extension.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/torchvision/extension.py b/torchvision/extension.py index 07576cffbd6..dc4f8b02987 100644 --- a/torchvision/extension.py +++ b/torchvision/extension.py @@ -37,11 +37,13 @@ def _register_extensions(): ' when calling SetDefaultDllDirectories.') raise err - res = kernel32.AddDllDirectory(lib_dir) - if res is None: - err = ctypes.WinError(ctypes.get_last_error()) - err.strerror += f' Error adding "{lib_dir}" to the DLL directories.' - raise err + dll_dirs = os.environ['PATH'].split(os.pathsep) + [lib_dir] + for dll_dir in dll_dirs: + res = kernel32.AddDllDirectory(dll_dir) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += f' Error adding "{dll_dir}" to the DLL directories.' + raise err kernel32.SetErrorMode(prev_error_mode) From 8ad318604cee6ec5d3ab8ac6bacf746cdc2b9f73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 14 Oct 2020 11:47:32 -0500 Subject: [PATCH 4/7] Trigger debug --- torchvision/extension.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/torchvision/extension.py b/torchvision/extension.py index dc4f8b02987..e5332a0cead 100644 --- a/torchvision/extension.py +++ b/torchvision/extension.py @@ -30,21 +30,19 @@ def _register_extensions(): if sys.version_info >= (3, 8): os.add_dll_directory(lib_dir) elif with_load_library_flags: - res = kernel32.SetDefaultDllDirectories(0x00001000) - if not res: + # res = kernel32.SetDefaultDllDirectories(0x00001000) + # if not res: + # err = ctypes.WinError(ctypes.get_last_error()) + # err.strerror += (' Error setting LOAD_LIBRARY_SEARCH_DEFAULT_DIRS' + # ' when calling SetDefaultDllDirectories.') + # raise err + + res = kernel32.AddDllDirectory(lib_dir) + if res is None: err = ctypes.WinError(ctypes.get_last_error()) - err.strerror += (' Error setting LOAD_LIBRARY_SEARCH_DEFAULT_DIRS' - ' when calling SetDefaultDllDirectories.') + err.strerror += f' Error adding "{lib_dir}" to the DLL directories.' raise err - dll_dirs = os.environ['PATH'].split(os.pathsep) + [lib_dir] - for dll_dir in dll_dirs: - res = kernel32.AddDllDirectory(dll_dir) - if res is None: - err = ctypes.WinError(ctypes.get_last_error()) - err.strerror += f' Error adding "{dll_dir}" to the DLL directories.' - raise err - kernel32.SetErrorMode(prev_error_mode) loader_details = ( From 7400f6de5a03ec39fb68bfe63fa67d8f474e9405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 14 Oct 2020 14:54:46 -0500 Subject: [PATCH 5/7] Do not call SetDefaultDllDirectories. --- torchvision/extension.py | 10 ---------- torchvision/io/_video_opt.py | 22 ++++++++++++++++++++++ torchvision/io/image.py | 23 +++++++++++++++++++++++ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/torchvision/extension.py b/torchvision/extension.py index e5332a0cead..265c989a8ce 100644 --- a/torchvision/extension.py +++ b/torchvision/extension.py @@ -21,22 +21,12 @@ def _register_extensions(): with_load_library_flags = hasattr(kernel32, 'AddDllDirectory') prev_error_mode = kernel32.SetErrorMode(0x0001) - kernel32.LoadLibraryW.restype = ctypes.c_void_p if with_load_library_flags: - kernel32.SetDefaultDllDirectories.restype = ctypes.c_bool kernel32.AddDllDirectory.restype = ctypes.c_void_p - kernel32.LoadLibraryExW.restype = ctypes.c_void_p if sys.version_info >= (3, 8): os.add_dll_directory(lib_dir) elif with_load_library_flags: - # res = kernel32.SetDefaultDllDirectories(0x00001000) - # if not res: - # err = ctypes.WinError(ctypes.get_last_error()) - # err.strerror += (' Error setting LOAD_LIBRARY_SEARCH_DEFAULT_DIRS' - # ' when calling SetDefaultDllDirectories.') - # raise err - res = kernel32.AddDllDirectory(lib_dir) if res is None: err = ctypes.WinError(ctypes.get_last_error()) diff --git a/torchvision/io/_video_opt.py b/torchvision/io/_video_opt.py index 686c06c5c29..2e9751d7829 100644 --- a/torchvision/io/_video_opt.py +++ b/torchvision/io/_video_opt.py @@ -22,6 +22,28 @@ extfinder = importlib.machinery.FileFinder(lib_dir, loader_details) ext_specs = extfinder.find_spec("video_reader") + + if os.name == 'nt': + # Load the video_reader extension using LoadLibraryExW + import ctypes + import sys + + kernel32 = ctypes.WinDLL('kernel32.dll', use_last_error=True) + with_load_library_flags = hasattr(kernel32, 'AddDllDirectory') + prev_error_mode = kernel32.SetErrorMode(0x0001) + + if with_load_library_flags: + kernel32.LoadLibraryExW.restype = ctypes.c_void_p + + res = kernel32.LoadLibraryExW(ext_specs.origin, None, 0x00001100) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += (f' Error loading "{ext_specs.origin}" or any or ' + 'its dependencies.') + raise err + + kernel32.SetErrorMode(prev_error_mode) + if ext_specs is not None: torch.ops.load_library(ext_specs.origin) _HAS_VIDEO_OPT = True diff --git a/torchvision/io/image.py b/torchvision/io/image.py index b793030ac59..9c656a7f1c9 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -16,6 +16,29 @@ extfinder = importlib.machinery.FileFinder(lib_dir, loader_details) # type: ignore[arg-type] ext_specs = extfinder.find_spec("image") + + if os.name == 'nt': + # Load the image extension using LoadLibraryExW + import ctypes + import sys + + kernel32 = ctypes.WinDLL('kernel32.dll', use_last_error=True) + with_load_library_flags = hasattr(kernel32, 'AddDllDirectory') + prev_error_mode = kernel32.SetErrorMode(0x0001) + + kernel32.LoadLibraryW.restype = ctypes.c_void_p + if with_load_library_flags: + kernel32.LoadLibraryExW.restype = ctypes.c_void_p + + res = kernel32.LoadLibraryExW(ext_specs.origin, None, 0x00001100) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += (f' Error loading "{ext_specs.origin}" or any or ' + 'its dependencies.') + raise err + + kernel32.SetErrorMode(prev_error_mode) + if ext_specs is not None: torch.ops.load_library(ext_specs.origin) _HAS_IMAGE_OPT = True From 1e930edc65f3623b5ce10b5f1ae8eca53e77b74e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 14 Oct 2020 15:10:16 -0500 Subject: [PATCH 6/7] Do not call loadlibrary if the extensions were not compiled --- torchvision/io/_video_opt.py | 13 +++++++------ torchvision/io/image.py | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/torchvision/io/_video_opt.py b/torchvision/io/_video_opt.py index 2e9751d7829..74afdbbb80b 100644 --- a/torchvision/io/_video_opt.py +++ b/torchvision/io/_video_opt.py @@ -35,12 +35,13 @@ if with_load_library_flags: kernel32.LoadLibraryExW.restype = ctypes.c_void_p - res = kernel32.LoadLibraryExW(ext_specs.origin, None, 0x00001100) - if res is None: - err = ctypes.WinError(ctypes.get_last_error()) - err.strerror += (f' Error loading "{ext_specs.origin}" or any or ' - 'its dependencies.') - raise err + if ext_specs is not None: + res = kernel32.LoadLibraryExW(ext_specs.origin, None, 0x00001100) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += (f' Error loading "{ext_specs.origin}" or any or ' + 'its dependencies.') + raise err kernel32.SetErrorMode(prev_error_mode) diff --git a/torchvision/io/image.py b/torchvision/io/image.py index 9c656a7f1c9..1af3c5ffe72 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -30,12 +30,13 @@ if with_load_library_flags: kernel32.LoadLibraryExW.restype = ctypes.c_void_p - res = kernel32.LoadLibraryExW(ext_specs.origin, None, 0x00001100) - if res is None: - err = ctypes.WinError(ctypes.get_last_error()) - err.strerror += (f' Error loading "{ext_specs.origin}" or any or ' - 'its dependencies.') - raise err + if ext_specs is not None: + res = kernel32.LoadLibraryExW(ext_specs.origin, None, 0x00001100) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += (f' Error loading "{ext_specs.origin}" or any or ' + 'its dependencies.') + raise err kernel32.SetErrorMode(prev_error_mode) From dab1521bec32518b06f9cf23aa020402018556d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 14 Oct 2020 15:12:27 -0500 Subject: [PATCH 7/7] Fix lint issues --- torchvision/io/_video_opt.py | 2 +- torchvision/io/image.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/torchvision/io/_video_opt.py b/torchvision/io/_video_opt.py index 74afdbbb80b..347b367ab27 100644 --- a/torchvision/io/_video_opt.py +++ b/torchvision/io/_video_opt.py @@ -40,7 +40,7 @@ if res is None: err = ctypes.WinError(ctypes.get_last_error()) err.strerror += (f' Error loading "{ext_specs.origin}" or any or ' - 'its dependencies.') + 'its dependencies.') raise err kernel32.SetErrorMode(prev_error_mode) diff --git a/torchvision/io/image.py b/torchvision/io/image.py index 1af3c5ffe72..2279be3ad10 100644 --- a/torchvision/io/image.py +++ b/torchvision/io/image.py @@ -35,7 +35,7 @@ if res is None: err = ctypes.WinError(ctypes.get_last_error()) err.strerror += (f' Error loading "{ext_specs.origin}" or any or ' - 'its dependencies.') + 'its dependencies.') raise err kernel32.SetErrorMode(prev_error_mode)