From 36714ee3606bd66b11a91118046f52ad764d5480 Mon Sep 17 00:00:00 2001 From: Christoph Heine Date: Sat, 18 May 2024 00:12:57 +0200 Subject: [PATCH 1/7] main: Add several default DLL search paths for Python. --- openage/__main__.py | 30 +++++------- openage/util/CMakeLists.txt | 1 + openage/util/dll.py | 95 +++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 19 deletions(-) create mode 100644 openage/util/dll.py diff --git a/openage/__main__.py b/openage/__main__.py index 4475f46518..c133ed05cb 100644 --- a/openage/__main__.py +++ b/openage/__main__.py @@ -31,27 +31,15 @@ def print_version(): sys.exit(0) -def add_dll_search_paths(dll_paths): +def add_dll_search_paths(dll_paths: list[str]): """ This function adds DLL search paths. - This function does nothing if current OS is not Windows. """ + from .util.dll import DllDirectoryManager - def close_windows_dll_path_handles(dll_path_handles): - """ - This function calls close() method on each of the handles. - """ - for handle in dll_path_handles: - handle.close() + manager = DllDirectoryManager(dll_paths) - if sys.platform != 'win32' or dll_paths is None: - return - - import atexit - win_dll_path_handles = [] - for addtional_path in dll_paths: - win_dll_path_handles.append(os.add_dll_directory(addtional_path)) - atexit.register(close_windows_dll_path_handles, win_dll_path_handles) + return manager def main(argv=None): @@ -62,11 +50,11 @@ def main(argv=None): ) if sys.platform == 'win32': - import inspect + from .util.dll import default_paths cli.add_argument( "--add-dll-search-path", action='append', dest='dll_paths', # use path of current openage executable as default - default=[os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: 0)))], + default=default_paths(), help="(Windows only) provide additional DLL search path") cli.add_argument("--version", "-V", action='store_true', dest='print_version', @@ -142,7 +130,11 @@ def main(argv=None): args = cli.parse_args(argv) if sys.platform == 'win32': - add_dll_search_paths(args.dll_paths) + args.dll_manager = add_dll_search_paths(args.dll_paths) + args.dll_manager.add_directories() + + else: + args.dll_manager = None if args.print_version: print_version() diff --git a/openage/util/CMakeLists.txt b/openage/util/CMakeLists.txt index 3d58384dee..79e7309cb3 100644 --- a/openage/util/CMakeLists.txt +++ b/openage/util/CMakeLists.txt @@ -3,6 +3,7 @@ add_py_modules( bytequeue.py context.py decorators.py + dll.py files.py fsprinting.py hash.py diff --git a/openage/util/dll.py b/openage/util/dll.py new file mode 100644 index 0000000000..bc06e88dd9 --- /dev/null +++ b/openage/util/dll.py @@ -0,0 +1,95 @@ +# Copyright 2024-2024 the openage authors. See copying.md for legal info. + +""" +Windows-specific loading of compiled Python modules and DLLs. +""" + +import inspect +import os +import sys + +# python.dll location +DEFAULT_PYTHON_DLL_DIR = os.path.dirname(sys.executable) + +# openage.dll locations (relative to this file) +DEFAULT_OPENAGE_DLL_DIRs = [ + "../../libopenage/RelWithDebInfo", +] + +# nyan.dll locations (relative to this file) +DEFAULT_NYAN_DLL_DIRS = [ + "../../../../nyan/build/nyan/RelWithDebInfo", + "../../nyan-external/bin/nyan/RelWithDebInfo", +] + + +class DllDirectoryManager: + """ + Manages directories that should be added to/removed from Python's DLL search path. + + All dependent DLLs or compiled cython modules that are not in Python's default search path + mst be added manually at runtime. Basically, this applies to all openage-specific libraries. + """ + + def __init__(self, directory_paths: list[str]): + """ + Create a new DLL directory manager. + + :param directory_paths: Absolute paths to the directories that are added. + """ + # Directory paths + self.directories = directory_paths + + # Store handles for added directories + self.handles = [] + + def add_directories(self): + """ + Add the manager's directories to Python's DLL search path. + """ + for directory in self.directories: + handle = os.add_dll_directory(directory) + self.handles.append(handle) + + def remove_directories(self): + """ + Remove the manager's directories from Python's DLL search path. + """ + for handle in self.handles: + handle.close() + + self.handles = [] + + def __del__(self): + """ + Ensure that DLL paths are removed when the object is deleted. + """ + self.remove_directories() + + def __enter__(self): + self.add_directories() + + def __exit__(self, exc_type, exc_value, traceback): + self.remove_directories() + + +def default_paths() -> list[str]: + """ + Create a list of default paths. + """ + directory_paths = [] + + # Add Python DLL search path + directory_paths.append(DEFAULT_PYTHON_DLL_DIR) + + file_dir = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: 0))) + + # Add openage DLL search paths + for path in DEFAULT_OPENAGE_DLL_DIRs: + directory_paths.append(os.path.join(file_dir, path)) + + # Add nyan DLL search paths + for path in DEFAULT_NYAN_DLL_DIRS: + directory_paths.append(os.path.join(file_dir, path)) + + return directory_paths From 42d28490d2e96530bcc62d3b8f6095d89bdd957e Mon Sep 17 00:00:00 2001 From: Christoph Heine Date: Sat, 18 May 2024 00:43:24 +0200 Subject: [PATCH 2/7] util: Do not pickle DLL directory handles. --- openage/util/dll.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/openage/util/dll.py b/openage/util/dll.py index bc06e88dd9..bfc365c781 100644 --- a/openage/util/dll.py +++ b/openage/util/dll.py @@ -67,11 +67,25 @@ def __del__(self): self.remove_directories() def __enter__(self): + """ + Enter a context guard. + """ self.add_directories() def __exit__(self, exc_type, exc_value, traceback): + """ + Exit a context guard. + """ self.remove_directories() + def __getstate__(self): + """ + Change pickling behavior so that directory handles are not serialized. + """ + content = self.__dict__ + content["handles"] = [] + return content + def default_paths() -> list[str]: """ From 5774f9f549248c2fd019587403b45d84e6d0d511 Mon Sep 17 00:00:00 2001 From: Christoph Heine Date: Sat, 18 May 2024 00:43:48 +0200 Subject: [PATCH 3/7] convert: Load DLLs in subprocesses. --- .../processor/export/media_exporter.py | 69 +++++++++---------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/openage/convert/processor/export/media_exporter.py b/openage/convert/processor/export/media_exporter.py index 966ebe8da6..ace7e1f83d 100644 --- a/openage/convert/processor/export/media_exporter.py +++ b/openage/convert/processor/export/media_exporter.py @@ -11,6 +11,7 @@ import os import multiprocessing import queue +import sys from openage.convert.entity_object.export.texture import Texture from openage.convert.service import debug_info @@ -27,6 +28,7 @@ from openage.convert.value_object.read.media.colortable import ColorTable from openage.convert.value_object.init.game_version import GameVersion from openage.util.fslike.path import Path + from openage.util.dll import DllDirectoryManager class MediaExporter: @@ -126,7 +128,8 @@ def export( handle_outqueue_func, itargs, kwargs, - args.jobs + args.jobs, + args.dll_manager, ) if args.debug_info > 5: @@ -198,6 +201,7 @@ def _export_singlethreaded( idx, source_data, single_queue, + None, request.source_filename, target_path, *itargs, @@ -225,7 +229,8 @@ def _export_multithreaded( handle_outqueue_func: typing.Callable | None, itargs: tuple, kwargs: dict, - job_count: int = None + job_count: int = None, + dll_manager: DllDirectoryManager = None, ): """ Export media files in multiple threads. @@ -241,15 +246,7 @@ def _export_multithreaded( :param itargs: Arguments for the export function. :param kwargs: Keyword arguments for the export function. :param job_count: Number of worker processes to use. - :type requests: list[MediaExportRequest] - :type sourcedir: Path - :type exportdir: Path - :type read_data_func: typing.Callable - :type export_func: typing.Callable - :type handle_outqueue_func: typing.Callable - :type itargs: tuple - :type kwargs: dict - :type job_count: int + :param dll_manager: Adds DLL search paths for the subrocesses (Windows-only). """ worker_count = job_count if worker_count is None: @@ -297,6 +294,7 @@ def error_callback(exception: Exception): idx, source_data, outqueue, + dll_manager, request.source_filename, target_path, *itargs @@ -625,6 +623,7 @@ def _export_blend( request_id: int, blendfile_data: bytes, outqueue: multiprocessing.Queue, + dll_manager: DllDirectoryManager, source_filename: str, # pylint: disable=unused-argument targetdir: Path, target_filename: str, @@ -633,15 +632,16 @@ def _export_blend( """ Convert and export a blending mode. + :param request_id: ID of the export request. :param blendfile_data: Raw file data of the blending mask. :param outqueue: Queue for passing metadata to the main process. + :param dll_manager: Adds DLL search paths for the subrocesses (Windows-only). :param target_path: Path to the resulting image file. :param blend_mode_count: Number of blending modes extracted from the source file. - :type blendfile_data: bytes - :type outqueue: multiprocessing.Queue - :type target_path: openage.util.fslike.path.Path - :type blend_mode_count: int """ + if sys.platform == "win32" and dll_manager is not None: + dll_manager.add_directories() + blend_data = Blendomatic(blendfile_data, blend_mode_count) from .texture_merge import merge_frames @@ -661,6 +661,7 @@ def _export_sound( request_id: int, sound_data: bytes, outqueue: multiprocessing.Queue, + dll_manager: DllDirectoryManager, source_filename: str, # pylint: disable=unused-argument target_path: Path, **kwargs # pylint: disable=unused-argument @@ -668,13 +669,15 @@ def _export_sound( """ Convert and export a sound file. + :param request_id: ID of the export request. :param sound_data: Raw file data of the sound file. :param outqueue: Queue for passing metadata to the main process. + :param dll_manager: Adds DLL search paths for the subrocesses (Windows-only). :param target_path: Path to the resulting sound file. - :type sound_data: bytes - :type outqueue: multiprocessing.Queue - :type target_path: openage.util.fslike.path.Path """ + if sys.platform == "win32" and dll_manager is not None: + dll_manager.add_directories() + from ...service.export.opus.opusenc import encode encoded = encode(sound_data) @@ -691,6 +694,7 @@ def _export_terrain( request_id: int, graphics_data: bytes, outqueue: multiprocessing.Queue, + dll_manager: DllDirectoryManager, source_filename: str, target_path: Path, palettes: dict[int, ColorTable], @@ -700,21 +704,19 @@ def _export_terrain( """ Convert and export a terrain graphics file. + :param request_id: ID of the export request. :param graphics_data: Raw file data of the graphics file. :param outqueue: Queue for passing the image metadata to the main process. + :param dll_manager: Adds DLL search paths for the subrocesses (Windows-only). :param source_filename: Filename of the source file. :param target_path: Path to the resulting image file. :param palettes: Palettes used by the game. :param compression_level: PNG compression level for the resulting image file. :param game_version: Game edition and expansion info. - :type graphics_data: bytes - :type outqueue: multiprocessing.Queue - :type source_filename: str - :type target_path: openage.util.fslike.path.Path - :type palettes: dict - :type compression_level: int - :type game_version: GameVersion """ + if sys.platform == "win32" and dll_manager is not None: + dll_manager.add_directories() + file_ext = source_filename.split('.')[-1].lower() if file_ext == "slp": from ...value_object.read.media.slp import SLP @@ -758,6 +760,7 @@ def _export_texture( request_id: int, graphics_data: bytes, outqueue: multiprocessing.Queue, + dll_manager: DllDirectoryManager, source_filename: str, target_path: Path, palettes: dict[int, ColorTable], @@ -770,20 +773,16 @@ def _export_texture( :param request_id: ID of the export request. :param graphics_data: Raw file data of the graphics file. :param outqueue: Queue for passing the image metadata to the main process. + :param dll_manager: Adds DLL search paths for the subrocesses (Windows-only). :param source_filename: Filename of the source file. :param target_path: Path to the resulting image file. :param palettes: Palettes used by the game. :param compression_level: PNG compression level for the resulting image file. :param cache_info: Media cache information with compression parameters from a previous run. - :type request_id: int - :type graphics_data: bytes - :type outqueue: multiprocessing.Queue - :type source_filename: str - :type target_path: openage.util.fslike.path.Path - :type palettes: dict - :type compression_level: int - :type cache_info: tuple """ + if sys.platform == "win32" and dll_manager is not None: + dll_manager.add_directories() + file_ext = source_filename.split('.')[-1].lower() if file_ext == "slp": from ...value_object.read.media.slp import SLP @@ -844,10 +843,6 @@ def _save_png( :param target_path: Path to the resulting image file. :param compression_level: PNG compression level used for the resulting image file. :param dry_run: If True, create the PNG but don't save it as a file. - :type texture: Texture - :type target_path: openage.util.fslike.path.Path - :type compression_level: int - :type dry_run: bool """ from ...service.export.png import png_create From 50522776b53a61f8a64e8cbc2a97dc87df354488 Mon Sep 17 00:00:00 2001 From: Christoph Heine Date: Sat, 18 May 2024 00:48:08 +0200 Subject: [PATCH 4/7] convert: Add default DLL paths to singlefile converter, --- openage/convert/tool/singlefile.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openage/convert/tool/singlefile.py b/openage/convert/tool/singlefile.py index 78b67380b0..9d7f1b4583 100644 --- a/openage/convert/tool/singlefile.py +++ b/openage/convert/tool/singlefile.py @@ -1,10 +1,11 @@ -# Copyright 2015-2023 the openage authors. See copying.md for legal info. +# Copyright 2015-2024 the openage authors. See copying.md for legal info. """ Convert a single slp/wav file from some drs archive to a png/opus file. """ from __future__ import annotations +import sys from pathlib import Path @@ -59,6 +60,11 @@ def main(args, error): file_path = Path(args.filename) file_extension = file_path.suffix[1:].lower() + if sys.platform == "win32": + from openage.util.dll import DllDirectoryManager, default_paths + dll_manager = DllDirectoryManager(default_paths()) + dll_manager.add_directories() + if not (args.mode in ("sld", "drs-wav", "wav") or file_extension in ("sld", "wav")): if not args.palettes_path: raise RuntimeError("palettes-path needs to be specified for " From 1ad51ba4c1579e0b4fb5206845626b4cea6ee667 Mon Sep 17 00:00:00 2001 From: Christoph Heine Date: Sat, 18 May 2024 02:36:19 +0200 Subject: [PATCH 5/7] cli: Store DLL manager for fallback subparser. --- openage/__main__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openage/__main__.py b/openage/__main__.py index c133ed05cb..970643bb51 100644 --- a/openage/__main__.py +++ b/openage/__main__.py @@ -129,12 +129,10 @@ def main(argv=None): args = cli.parse_args(argv) + dll_manager = None if sys.platform == 'win32': - args.dll_manager = add_dll_search_paths(args.dll_paths) - args.dll_manager.add_directories() - - else: - args.dll_manager = None + dll_manager = add_dll_search_paths(args.dll_paths) + dll_manager.add_directories() if args.print_version: print_version() @@ -143,6 +141,8 @@ def main(argv=None): # the user didn't specify a subcommand. default to 'main'. args = main_cli.parse_args(argv) + args.dll_manager = dll_manager + # process the shared args set_loglevel(verbosity_to_level(args.verbose - args.quiet)) From 74457691107656d52097dc66272604c8f43665cc Mon Sep 17 00:00:00 2001 From: heinezen Date: Sat, 18 May 2024 22:20:46 +0200 Subject: [PATCH 6/7] util: Check if paths exist before adding them as defaults. --- openage/util/dll.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/openage/util/dll.py b/openage/util/dll.py index bfc365c781..748e503bf0 100644 --- a/openage/util/dll.py +++ b/openage/util/dll.py @@ -99,11 +99,15 @@ def default_paths() -> list[str]: file_dir = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: 0))) # Add openage DLL search paths - for path in DEFAULT_OPENAGE_DLL_DIRs: - directory_paths.append(os.path.join(file_dir, path)) + for candidate in DEFAULT_OPENAGE_DLL_DIRs: + path = os.path.join(file_dir, candidate) + if os.path.exists(path): + directory_paths.append(path) # Add nyan DLL search paths - for path in DEFAULT_NYAN_DLL_DIRS: - directory_paths.append(os.path.join(file_dir, path)) + for candidate in DEFAULT_NYAN_DLL_DIRS: + path = os.path.join(file_dir, candidate) + if os.path.exists(path): + directory_paths.append(path) return directory_paths From e5311c1f9b917810c1d3e5835e64ac1526198734 Mon Sep 17 00:00:00 2001 From: heinezen Date: Sat, 18 May 2024 22:23:11 +0200 Subject: [PATCH 7/7] util: Add more path locations for different build types. --- openage/util/dll.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/openage/util/dll.py b/openage/util/dll.py index 748e503bf0..a3d23be7f1 100644 --- a/openage/util/dll.py +++ b/openage/util/dll.py @@ -13,13 +13,22 @@ # openage.dll locations (relative to this file) DEFAULT_OPENAGE_DLL_DIRs = [ + "../../libopenage/Debug", + "../../libopenage/Release", "../../libopenage/RelWithDebInfo", + "../../libopenage/MinSizeRel", ] # nyan.dll locations (relative to this file) DEFAULT_NYAN_DLL_DIRS = [ + "../../../../nyan/build/nyan/Debug", + "../../../../nyan/build/nyan/Release", "../../../../nyan/build/nyan/RelWithDebInfo", + "../../../../nyan/build/nyan/MinSizeRel", + "../../nyan-external/bin/nyan/Debug", + "../../nyan-external/bin/nyan/Release", "../../nyan-external/bin/nyan/RelWithDebInfo", + "../../nyan-external/bin/nyan/MinSizeRel", ]