building: attempt to preserve parent directories for pywin32 extensions

PyInstaller/building/build_main.py

@@ -32,9 +32,9 @@
 from PyInstaller.building.toc_conversion import DependencyProcessor
 from PyInstaller.building.utils import (
     _check_guts_toc_mtime, _should_include_system_binary, format_binaries_and_datas, compile_pymodule,
-    add_suffix_to_extension
+    add_suffix_to_extension, postprocess_binaries_toc_pywin32, postprocess_binaries_toc_pywin32_anaconda
 )
-from PyInstaller.compat import PYDYLIB_NAMES, is_win
+from PyInstaller.compat import PYDYLIB_NAMES, is_win, is_conda
 from PyInstaller.depend import bindepend
 from PyInstaller.depend.analysis import initialize_modgraph
 from PyInstaller.depend.utils import create_py3_base_library, scan_code_for_ctypes
@@ -736,6 +736,13 @@ def assemble(self):
             isolated.call(find_binary_dependencies, self.binaries, self.binding_redirects, collected_packages)
         )
 
+        # Apply work-around for (potential) binaries collected from `pywin32` package...
+        if is_win:
+            self.binaries = postprocess_binaries_toc_pywin32(self.binaries)
+            # With anaconda, we need additional work-around...
+            if is_conda:
+                self.binaries = postprocess_binaries_toc_pywin32_anaconda(self.binaries)
+
         # Include zipped Python eggs.
         logger.info('Looking for eggs')
         self.zipfiles = deps_proc.make_zipfiles_toc()  # Already normalized

PyInstaller/building/utils.py

@@ -738,3 +738,68 @@ def compile_pymodule(name, src_path, workpath, code_cache=None):
 
     # Return output path
     return pyc_path
+
+
+def postprocess_binaries_toc_pywin32(binaries):
+    """
+    Process the given `binaries` TOC list to apply work around for `pywin32` package, fixing the target directory
+    for collected extensions.
+    """
+    # Ensure that all files collected from `win32`  or `pythonwin` into top-level directory are put back into
+    # their corresponding directories. They end up in top-level directory because `pywin32.pth` adds both
+    # directories to the `sys.path`, so they end up visible as top-level directories. But these extensions
+    # might in fact be linked against each other, so we should preserve the directory layout for consistency
+    # between modulegraph-discovered extensions and linked binaries discovered by link-time dependency analysis.
+    # Within the same framework, also consider `pywin32_system32`, just in case.
+    PYWIN32_SUBDIRS = {'win32', 'pythonwin', 'pywin32_system32'}
+
+    processed_binaries = []
+    for dest_name, src_name, typecode in binaries:
+        dest_path = pathlib.PurePath(dest_name)
+        src_path = pathlib.PurePath(src_name)
+
+        if dest_path.parent == pathlib.PurePath('.') and src_path.parent.name.lower() in PYWIN32_SUBDIRS:
+            dest_path = pathlib.PurePath(src_path.parent.name) / dest_path
+            dest_name = str(dest_path)
+
+        processed_binaries.append((dest_name, src_name, typecode))
+
+    return processed_binaries
+
+
+def postprocess_binaries_toc_pywin32_anaconda(binaries):
+    """
+    Process the given `binaries` TOC list to apply work around for Anaconda `pywin32` package, fixing the location
+    of collected `pywintypes3X.dll` and `pythoncom3X.dll`.
+    """
+    # The Anaconda-provided `pywin32` package installs three copies of `pywintypes3X.dll` and `pythoncom3X.dll`,
+    # located in the following directories (relative to the environment):
+    # - Library/bin
+    # - Lib/site-packages/pywin32_system32
+    # - Lib/site-packages/win32
+    #
+    # This turns our dependency scanner and directory layout preservation mechanism into a lottery based on what
+    # `pywin32` modules are imported and in what order. To keep things simple, we deal with this insanity by
+    # post-processing the `binaries` list, modifying the destination of offending copies, and let the final TOC
+    # list normalization deal with potential duplicates.
+    DLL_CANDIDATES = {
+        f"pywintypes{sys.version_info[0]}{sys.version_info[1]}.dll",
+        f"pythoncom{sys.version_info[0]}{sys.version_info[1]}.dll",
+    }
+
+    DUPLICATE_DIRS = {
+        pathlib.PurePath('.'),
+        pathlib.PurePath('win32'),
+    }
+
+    processed_binaries = []
+    for dest_name, src_name, typecode in binaries:
+        # Check if we need to divert - based on the destination base name and destination parent directory.
+        dest_path = pathlib.PurePath(dest_name)
+        if dest_path.name.lower() in DLL_CANDIDATES and dest_path.parent in DUPLICATE_DIRS:
+            dest_path = pathlib.PurePath("pywin32_system32") / dest_path.name
+            dest_name = str(dest_path)
+
+        processed_binaries.append((dest_name, src_name, typecode))
+
+    return processed_binaries

PyInstaller/depend/bindepend.py

@@ -232,18 +232,6 @@ def _get_paths_for_parent_directory_preservation():
 
 
 def _select_destination_directory(src_filename, parent_dir_preservation_paths):
-    # Special handling for pywin32 on Windows, because its .pyd extensions end up linking each other, but, due to
-    # sys.path modifications the packages perform, they all end up as top-modules and should be collected into
-    # top-level directory... i.e., we must NOT preserve the directory layout in this case.
-    if compat.is_win:
-        # match <...>/site-packages/pythonwin
-        # The caller might not have resolved the src_filename, so we need to explicitly lower-case the parent_dir.name
-        # before comparing it to account for case variations.
-        parent_dir = src_filename.parent
-        if parent_dir.name.lower() == "pythonwin" and parent_dir.parent in parent_dir_preservation_paths:
-            # Collect into top-level directory.
-            return src_filename.name
-
     # Check parent directory preservation paths
     for parent_dir_preservation_path in parent_dir_preservation_paths:
         if parent_dir_preservation_path in src_filename.parents:

PyInstaller/loader/pyimod04_pywin32.py

@@ -16,10 +16,20 @@
 
 
 def install():
+    # Sub-directories containing extensions. In original python environment, these are added to `sys.path` by the
+    # `pywin32.pth` so the extensions end up treated as top-level modules. We attempt to preserve the directory
+    # layout, so we need to add these directories to `sys.path` ourselves.
+    pywin32_ext_paths = ('win32', 'pythonwin')
+    pywin32_ext_paths = [os.path.join(sys._MEIPASS, pywin32_ext_path) for pywin32_ext_path in pywin32_ext_paths]
+    pywin32_ext_paths = [path for path in pywin32_ext_paths if os.path.isdir(path)]
+    sys.path.extend(pywin32_ext_paths)
+
+    # Additional handling of `pywin32_system32` DLL directory
     pywin32_system32_path = os.path.join(sys._MEIPASS, 'pywin32_system32')
+
     if not os.path.isdir(pywin32_system32_path):
-        # Either pywin32 is not collected, or we are dealing with Anaconda-packaged version that does not use the
-        # pywin32_system32 sub-directory. In the latter case, the pywin32 DLLs should be in `sys._MEIPASS`, and nothing
+        # Either pywin32 is not collected, or we are dealing with version that does not use the pywin32_system32
+        # sub-directory. In the latter case, the pywin32 DLLs should be in `sys._MEIPASS`, and nothing
         # else needs to be done here.
         return
 

news/7627.bugfix.rst

@@ -0,0 +1,3 @@
+Attempt to mitigate issues with Anaconda ``pywin32`` package that
+result from the package installing three copies of ``pywintypes3X.dll``
+and ``pythoncom3X.dll`` in different locations.

news/7627.feature.rst

@@ -0,0 +1,3 @@
+Attempt to preserve the parent directory layout for ``pywin32``
+extensions that originate from ``win32`` and ``pythonwin`` directories,
+instead of collecting those extensions to top-level application directory.