diff --git a/Documentation/docs/migration_guides/itk_6_migration_guide.md b/Documentation/docs/migration_guides/itk_6_migration_guide.md index 2ef423b5138..2f479ba9bf5 100644 --- a/Documentation/docs/migration_guides/itk_6_migration_guide.md +++ b/Documentation/docs/migration_guides/itk_6_migration_guide.md @@ -394,6 +394,74 @@ for thread in threads: issues with callbacks, you can disable GIL release by setting `-DITK_PYTHON_RELEASE_GIL=OFF` when building ITK. +Python lazy loading is always on +-------------------------------- + +The Python wrapping layer's lazy-loading mechanism has been rewritten on top +of the standard-library [PEP 562](https://peps.python.org/pep-0562/) +module-level `__getattr__` / `__dir__` protocol. Lazy first-touch resolution +is now the only mode of operation: there is no eager-import alternative. + +### Removed + +- `itkConfig.LazyLoading` — previously a boolean attribute on `itkConfig` + that, when set to `False`, forced every wrapped submodule to be imported + at `import itk` time. +- `ITK_PYTHON_LAZYLOADING` — the environment variable that seeded the value + of `itkConfig.LazyLoading` at startup. +- The custom `itk.support.lazy.LazyITKModule` `types.ModuleType` subclass + (and its `ITKLazyLoadLock`, `_lazy_itk_module_reconstructor`, and + `not_loaded` sentinel) that the previous implementation relied on. + +### What you need to do + +If your code or test suite contains either of the following, remove +them: + +- `itkConfig.LazyLoading = False` — assignment alone is silently ignored + (module attributes are mutable, but no ITK code reads the value + anymore). On a fresh `import itkConfig` with no prior assignment, a + *read* of `itkConfig.LazyLoading` raises `AttributeError` because the + attribute is no longer defined. Delete both the writes and any reads: + + ```python + import itkConfig + + # Remove any assignments — they are silently no-ops: + itkConfig.LazyLoading = False + + # Remove any reads — without a prior assignment they raise + # AttributeError, since itkConfig no longer defines this attribute: + if itkConfig.LazyLoading: + ... + ``` + +- `ITK_PYTHON_LAZYLOADING` — the environment variable is no longer + consulted. Any export is silently ignored; remove it from launch + scripts and CI configurations: + + ```bash + ITK_PYTHON_LAZYLOADING=0 python my_script.py # drop the env var + ``` + +The observable contract from a caller's point of view is otherwise +unchanged: `import itk` returns the package, attribute access on `itk` +or any `itk.` resolves on first touch, `dir(itk)` lists the +full set of lazily-discoverable attributes, and pickle / cloudpickle +round-trips through `sys.modules` continue to work. + +### Why this changed + +The previous `LazyITKModule.__getattribute__` intercepted *every* attribute +read on the `itk` package just to check a sentinel; the PEP 562 hook only +fires on a miss, so steady-state attribute access now hits the standard +module fast path. The legacy `ITKLazyLoadLock` also combined +`threading.RLock` and `multiprocessing.RLock`, and constructing the +multiprocessing half pinned the multiprocessing start method at +`import itk` time — breaking downstream callers that wanted to call +`multiprocessing.set_start_method(...)` after import. The replacement uses +only `threading.RLock`. + Modern CMake Interface Libraries --------------------------------- diff --git a/Modules/Filtering/ImageIntensity/wrapping/test/itkImageFilterNumPyInputsTest.py b/Modules/Filtering/ImageIntensity/wrapping/test/itkImageFilterNumPyInputsTest.py index a1132a67a47..a8c6d9b020d 100644 --- a/Modules/Filtering/ImageIntensity/wrapping/test/itkImageFilterNumPyInputsTest.py +++ b/Modules/Filtering/ImageIntensity/wrapping/test/itkImageFilterNumPyInputsTest.py @@ -15,9 +15,6 @@ # limitations under the License. # # ========================================================================== -import itkConfig - -itkConfig.LazyLoading = False import itk import numpy as np diff --git a/Wrapping/Generators/Python/CMakeLists.txt b/Wrapping/Generators/Python/CMakeLists.txt index 302e0f31d9b..e4fd7bad312 100644 --- a/Wrapping/Generators/Python/CMakeLists.txt +++ b/Wrapping/Generators/Python/CMakeLists.txt @@ -10,7 +10,7 @@ ## - itk/ ## - __init__.py # cmake copied file ## - support/ -## - itk(Extras|Template|Base|LazyLoading|...).py # static python files cmake copied +## - (extras|template_class|base|_lazy_submodule|...).py # static python files cmake copied ## - Configuration/ ## -- ITK${MODULE_ITEM}Config.py # igenerator.py output config database index files for .so ## -- ITK${MODULE_ITEM}_snake_case.py # igenerator.py output config database index files for .so @@ -185,7 +185,7 @@ if(NOT EXTERNAL_WRAP_ITK_PROJECT) support/types support/extras support/xarray - support/lazy + support/_lazy_submodule support/helpers support/init_helpers support/build_options diff --git a/Wrapping/Generators/Python/Tests/CMakeLists.txt b/Wrapping/Generators/Python/Tests/CMakeLists.txt index 58057bb19ef..958f5b2e426 100644 --- a/Wrapping/Generators/Python/Tests/CMakeLists.txt +++ b/Wrapping/Generators/Python/Tests/CMakeLists.txt @@ -110,11 +110,6 @@ itk_python_add_test( COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lazy.py ) -itk_python_add_test( - NAME PythonNoLazyModule - COMMAND - ${CMAKE_CURRENT_SOURCE_DIR}/nolazy.py -) itk_python_add_test( NAME PythonNoDefaultFactories COMMAND diff --git a/Wrapping/Generators/Python/Tests/lazy.py b/Wrapping/Generators/Python/Tests/lazy.py index 0f0dd9c815b..3bc75f9dd1c 100644 --- a/Wrapping/Generators/Python/Tests/lazy.py +++ b/Wrapping/Generators/Python/Tests/lazy.py @@ -15,18 +15,45 @@ # limitations under the License. # # ========================================================================== -import itkConfig - -# Override environmental variable default to force LazyLoading -itkConfig.LazyLoading = True import itk -# Test PEP 366 compliance of LazyITKModule +# Test PEP 366 compliance of itk and per-submodule namespaces assert itk.__package__ == "itk" from itk import ITKCommon assert ITKCommon.__package__ == "itk" +# PEP 562 __dir__: lazy attributes are visible to tab-completion and +# tooling without forcing a SWIG load. Only dunder attributes (which +# the lazy __getattr__ explicitly skips) have been touched up to this +# point, so the underlying SWIG modules have not been loaded yet -- +# "Image" must show up in dir() but stay absent from the module's +# __dict__ until first access. +assert "Image" in dir(itk) +assert "Image" not in vars(itk) +assert "Image" in dir(ITKCommon) +assert "Image" not in vars(ITKCommon) + +# Factory hook: under the default configuration, accessing a class +# whose module declares a needed factory (ITKIOImageBase -> ImageIO) +# must trigger the corresponding factory registration as a side effect +# of the lazy load. nodefaultfactories.py covers the disabled path. +import itkConfig + +if itkConfig.DefaultFactoryLoading: + factories_before = len(itk.ObjectFactoryBase.GetRegisteredFactories()) + _ = itk.ImageFileReader + factories_after = len(itk.ObjectFactoryBase.GetRegisteredFactories()) + assert factories_after > factories_before + +# stdlib pickle round-trip on a per-submodule namespace: the lazy +# submodule is registered in sys.modules and its instance __reduce_ex__ +# resolves through importlib.import_module, so plain pickle (not just +# cloudpickle) must round-trip to the same instance. +import pickle + +assert pickle.loads(pickle.dumps(itk.ITKCommon)) is itk.ITKCommon + # Test pickling used bash Dask _has_cloudpickle: bool = False try: diff --git a/Wrapping/Generators/Python/Tests/multiprocess_lazy_loading.py b/Wrapping/Generators/Python/Tests/multiprocess_lazy_loading.py index 6246d9dc5e2..753ae50867f 100755 --- a/Wrapping/Generators/Python/Tests/multiprocess_lazy_loading.py +++ b/Wrapping/Generators/Python/Tests/multiprocess_lazy_loading.py @@ -1,35 +1,27 @@ #!/usr/bin/env python -# LazyLoading must be threadsafe +# Lazy module loading must be threadsafe # # The loading of modules in python *must* occur as a # single atomic transaction in multiprocessing environments (i.e. # the module should only be loaded by one thread). # -# The LazyLoading of ITK did not treat the loading of +# Historically ITK's lazy loader did not treat the loading of # modules as an atomic transaction, and multiple threads # would attempt to load the cascading dependencies out of # order. # -# The `getattr` override that allows LazyLoading to work -# in the case where the module is *not* loaded now blocks -# while the first thread completes the delayed module loading. +# The PEP 562 ``__getattr__`` hook that triggers a delayed +# load now blocks while the first thread completes the load. # After the first thread completes module load as an atomic # transaction, the other threads fall through (skip loading) # and return the value requested. # # Need to use a recursive lock for thread ownership so that the # first thread can can acquire a RLock as often as needed while -# recursively processing dependent modules lazy loads. Other threads need +# recursively processing dependent module loads. Other threads need # to wait until this first thread releases the RLock. -# NOTE: This test requires itkConfig.LazyLoading=True -# Explicitly set to override potential environmental -# variable settings. -import itkConfig - -itkConfig.LazyLoading = True - from multiprocessing.pool import ThreadPool from multiprocessing import cpu_count diff --git a/Wrapping/Generators/Python/Tests/nolazy.py b/Wrapping/Generators/Python/Tests/nolazy.py deleted file mode 100644 index b052333ff4a..00000000000 --- a/Wrapping/Generators/Python/Tests/nolazy.py +++ /dev/null @@ -1,46 +0,0 @@ -# ========================================================================== -# -# Copyright NumFOCUS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ========================================================================== -import itkConfig - -# Override environmental variable default to force non-LazyLoading -itkConfig.LazyLoading = False -import itk - -# Test PEP 366 compliance of LazyITKModule -assert itk.__package__ == "itk" -from itk import ITKCommon - -assert ITKCommon.__package__ == "itk" - -# Test pickling used bash Dask -_has_cloudpickle: bool = False -try: - import cloudpickle - - _has_cloudpickle = True -except ImportError: - _has_cloudpickle = False - pass - -if _has_cloudpickle: - print("Using cloudpickle to test dumping and loading itk.") - itkpickled = cloudpickle.dumps(itk) - cloudpickle.loads(itkpickled) -else: - print("cloudpickle module not available for testing.") - pass diff --git a/Wrapping/Generators/Python/itk/__init__.py b/Wrapping/Generators/Python/itk/__init__.py index 0476d3d2fee..865c06d8af5 100644 --- a/Wrapping/Generators/Python/itk/__init__.py +++ b/Wrapping/Generators/Python/itk/__init__.py @@ -22,9 +22,13 @@ # in order to maintain the singular value of the module values. # `import .conf.itkConfig` is a different context than # `import itkConfig`, even if they are the same file. The -# LazyLoading and other values may be different in the two contexts. +# DefaultFactoryLoading and other values may be different in the +# two contexts. from itkConfig import ITK_GLOBAL_VERSION_STRING as __version__ +# Tests/lazy.py asserts itk.__package__ == "itk"; guard the invariant. +assert __package__ == "itk" + from itk.support.extras import * from itk.support.init_helpers import * from itk.support.types import ( @@ -47,96 +51,118 @@ OT, ) +import threading as _threading + +import itkConfig as _itkConfig +from itk.support import base as _base + +# Recursive lock so transitive dependency loads from the same thread +# don't self-deadlock. Deliberately threading.RLock — a +# multiprocessing.RLock would pin the multiprocessing start method +# and break spawn/forkserver after `import itk`. +_lazy_load_lock = _threading.RLock() +_lazy_attribute_to_module: dict[str, str] = {} +_MISSING = object() + + +def __getattr__(name: str): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + try: + module_name = _lazy_attribute_to_module[name] + except KeyError: + raise AttributeError(f"module 'itk' has no attribute {name!r}") from None + g = globals() + with _lazy_load_lock: + cached = g.get(name, _MISSING) + if cached is not _MISSING: + return cached + namespace: dict = {} + _base.itk_load_swig_module(module_name, namespace) + g.update(namespace) + if _itkConfig.DefaultFactoryLoading: + _base.load_module_needed_factories(module_name) + try: + return g[name] + except KeyError: + raise AttributeError(f"module 'itk' has no attribute {name!r}") from None + + +def __dir__(): + # Set literal — avoids resolving the bare name `set`, which is + # shadowed in module globals by `itk.set` (itkTemplate std::set) + # once any SWIG module is loaded. + return sorted({*globals().keys(), *_lazy_attribute_to_module.keys()}) + def _initialize_module(): """ A function to explicitly avoid polluting the global namespace """ - from .support.base import ITKModuleInfo, ITKTemplateFeatures + import importlib + import os + import sys # Needed to avoid problem with aliasing of itk.set (itkTemplate) # inside the itk namespace. We need to explicitly specify the # use of the builtin set from builtins import set as _builtin_set - def _get_lazy_attributes(local_lazy_attributes, l_module, l_data: ITKModuleInfo): - """ - Set up lazy attribute relationships - """ - for template_feature in l_data._template_feature_tuples: + from .support._lazy_submodule import _make_itk_lazy_submodule + + # Build the flat first-owner-wins map driving module-level + # __getattr__, and populate base.itk_base_global_lazy_attributes + # with the full set of owners per attribute (consumed by + # support/template_class.py:_LoadModules). + # + # Precedence rules (mirroring the legacy _get_lazy_attributes + + # __belong_lazy_attributes pipeline): + # - template_feature with _class_in_module=True: that module + # wins, even over a previously recorded owner. + # - template_feature with _class_in_module=False: first walk + # wins; later writers do not override. + # - snake_case_functions: first walk wins; never override. + for module, data in _base.itk_base_global_module_data.items(): + for template_feature in data._template_feature_tuples: + attr = template_feature._py_class_name + _base.itk_base_global_lazy_attributes.setdefault(attr, _builtin_set()).add( + module + ) if template_feature._class_in_module: - # insert in front front if in library - local_lazy_attributes.setdefault( - template_feature._py_class_name, [] - ).insert(0, l_module) + _lazy_attribute_to_module[attr] = module else: - # append to end - local_lazy_attributes.setdefault( - template_feature._py_class_name, [] - ).append(l_module) - - for function in l_data._snake_case_functions: - # snake case always appended to end - local_lazy_attributes.setdefault(function, []).append(l_module) - - # Remove duplicates in attributes, preserving only the first - def _dedup(seq): + _lazy_attribute_to_module.setdefault(attr, module) + for function in data._snake_case_functions: + _base.itk_base_global_lazy_attributes.setdefault( + function, _builtin_set() + ).add(module) + _lazy_attribute_to_module.setdefault(function, module) + + # Per-submodule lazy namespaces (itk.ITKCommon, ...). Plain + # types.ModuleType instances wired with PEP 562 __getattr__ / + # __dir__ closures by _make_itk_lazy_submodule, registered in + # sys.modules['itk.'] so cloudpickle round-trips by + # dotted name (Tests/lazy.py). + this_module = sys.modules[__name__] + for module, data in _base.itk_base_global_module_data.items(): + attributes: dict[str, list[str]] = {} + for template_feature in data._template_feature_tuples: + if template_feature._class_in_module: + attributes.setdefault(template_feature._py_class_name, []).insert( + 0, module + ) + else: + attributes.setdefault(template_feature._py_class_name, []).append( + module + ) + for function in data._snake_case_functions: + attributes.setdefault(function, []).append(module) + for k, v in attributes.items(): seen = _builtin_set() - seen_add = seen.add - return [x for x in seq if not (x in seen or seen_add(x))] - - for k, v in local_lazy_attributes.items(): - local_lazy_attributes[k] = _dedup(v) - - from .support import base as _base - from .support import lazy as _lazy - from itkConfig import LazyLoading as _LazyLoading - import sys - import os - import importlib + attributes[k] = [m for m in v if not (m in seen or seen.add(m))] - if _LazyLoading: - # If we are loading lazily (on-demand), make a dict mapping the available - # classes/functions/etc. (read from the configuration modules) to the - # modules they are declared in. Then pass that dict to a LazyITKModule - # instance and (later) do some surgery on sys.modules so that the 'itk' - # module becomes that new instance instead of what is executed from this - # file. - lazy_attributes = {} - for module, data in _base.itk_base_global_module_data.items(): - _get_lazy_attributes(lazy_attributes, module, data) - - if isinstance(sys.modules[__name__], _lazy.LazyITKModule): - # Handle reload case where we've already done this once. - # If we made a new module every time, multiple reload()s would fail - # because the identity of sys.modules['itk'] would always be changing. - sys.modules[__name__].__init__(__name__, lazy_attributes) - del lazy_attributes - else: - # Create a new LazyITKModule - lzy_module = _lazy.LazyITKModule(__name__, lazy_attributes) - - # Pre-existing attributes need to be propagated too! - # except for the lazy overridden elements - exclusion_copy_list = ["__name__", "__loader__", "__builtins__"] - for k, v in sys.modules[__name__].__dict__.items(): - if k not in exclusion_copy_list: - setattr(lzy_module, k, v) - - # Now override the default sys.modules[__name__] (__name__ == 'itk' ) - sys.modules[__name__] = lzy_module - else: - # We're not lazy-loading. Just load the modules in the order specified in - # the known_modules list for consistency. - for module in _base.itk_base_global_module_data.keys(): - _base.itk_load_swig_module(module, sys.modules[__name__].__dict__) - - # Populate itk.ITKModuleName - for module, data in _base.itk_base_global_module_data.items(): - attributes = {} - _get_lazy_attributes(attributes, module, data) - itk_module = _lazy.LazyITKModule(module, attributes) - setattr(sys.modules[__name__], module, itk_module) + itk_module = _make_itk_lazy_submodule(module, attributes, _lazy_load_lock) + setattr(this_module, module, itk_module) # Check if the module installed its own init file and load it. # ITK Modules __init__.py must be renamed to __init_{module_name}__.py before packaging @@ -157,6 +183,8 @@ def _dedup(seq): spec.loader.exec_module(loaded_module) -# After 'lifting' external symbols into this itk namespace, -# Now do the initialization, and conversion to LazyLoading if necessary +# Build the lazy-attribute map and materialise per-submodule lazy +# namespaces. This must run before any consumer attribute access; +# module-level __getattr__ depends on _lazy_attribute_to_module being +# populated. _initialize_module() diff --git a/Wrapping/Generators/Python/itk/support/_lazy_submodule.py b/Wrapping/Generators/Python/itk/support/_lazy_submodule.py new file mode 100644 index 00000000000..7a41143faee --- /dev/null +++ b/Wrapping/Generators/Python/itk/support/_lazy_submodule.py @@ -0,0 +1,117 @@ +# ========================================================================== +# +# Copyright NumFOCUS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ========================================================================== + +"""Builder for per-submodule lazy namespaces (``itk.ITKCommon``, ...). + +Each submodule is a plain ``types.ModuleType`` instance wired with PEP 562 +``__getattr__`` / ``__dir__`` callables, registered in +``sys.modules['itk.']`` so ``cloudpickle`` (used by the Dask +worker round-trip exercised in ``Tests/lazy.py``) can re-import it by +its dotted name. +""" + +import importlib +import sys +import types + +import itkConfig as _itkConfig +from itk.support import base as _base + + +_MISSING = object() + + +def _make_itk_lazy_submodule( + module_name: str, + lazy_attributes: dict[str, list[str]], + lazy_load_lock, +) -> types.ModuleType: + """Build the lazy ``itk.`` namespace. + + Parameters + ---------- + module_name + Bare submodule name (e.g. ``"ITKCommon"``); the resulting + module is registered as ``itk.``. + lazy_attributes + Mapping from attribute name to the list of owning submodule + names. The first element of each list wins (matching the + legacy ``__belong_lazy_attributes`` first-owner-wins rule). + lazy_load_lock + The shared recursive lock from ``itk/__init__.py`` used to + serialise SWIG module loads across both top-level and + per-submodule lazy paths. + """ + m = types.ModuleType(f"itk.{module_name}") + m.__package__ = "itk" # PEP 366 + m.__loader__ = None + + belong: dict[str, str] = {k: v[0] for k, v in lazy_attributes.items() if v} + + def __getattr__(name: str): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + try: + target = belong[name] + except KeyError: + raise AttributeError( + f"module {m.__name__!r} has no attribute {name!r}" + ) from None + d = m.__dict__ + with lazy_load_lock: + cached = d.get(name, _MISSING) + if cached is not _MISSING: + return cached + namespace: dict = {} + _base.itk_load_swig_module(target, namespace) + d.update(namespace) + if _itkConfig.DefaultFactoryLoading: + _base.load_module_needed_factories(target) + try: + return d[name] + except KeyError: + raise AttributeError( + f"module {m.__name__!r} has no attribute {name!r}" + ) from None + + def __dir__(): + # Set literal — avoids resolving the bare name `set`, which + # gets shadowed in this submodule's __dict__ by `itk.set` + # (itkTemplate std::set) once any SWIG module loads into it. + return sorted({*m.__dict__.keys(), *belong.keys()}) + + m.__getattr__ = __getattr__ + m.__dir__ = __dir__ + + # Pickle shim: a bare ``types.ModuleType`` has no reducer, so + # ``pickle.dumps`` raises ``TypeError: cannot pickle 'module' + # object`` on CPython 3.12 even when the instance is registered in + # ``sys.modules``. A per-instance ``__reduce_ex__`` makes pickle + # round-trip via ``importlib.import_module(name)``, which then + # resolves through ``sys.modules`` to the same instance. Set as + # an instance attribute (not on ``types.ModuleType``) so unrelated + # modules in the process are unaffected. + submodule_dotted_name = m.__name__ + + def __reduce_ex__(_protocol: int): + return (importlib.import_module, (submodule_dotted_name,)) + + m.__reduce_ex__ = __reduce_ex__ + + sys.modules[submodule_dotted_name] = m + return m diff --git a/Wrapping/Generators/Python/itk/support/lazy.py b/Wrapping/Generators/Python/itk/support/lazy.py deleted file mode 100644 index a8fbf21d94f..00000000000 --- a/Wrapping/Generators/Python/itk/support/lazy.py +++ /dev/null @@ -1,181 +0,0 @@ -# ========================================================================== -# -# Copyright NumFOCUS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ========================================================================== -import types -from itk.support import base -from itkConfig import DefaultFactoryLoading as _DefaultFactoryLoading - -# Needed to avoid problem with aliasing of itk.set (itkTemplate) -# inside the itk namespace. We need to explicitly specify the -# use of the builtin set -from builtins import set as _builtin_set - -not_loaded: str = "not loaded" - - -def _lazy_itk_module_reconstructor(module_name, state): - # Similar to copyreg._reconstructor - lazy_module = types.ModuleType.__new__(LazyITKModule, state) - types.ModuleType.__init__(lazy_module, module_name) - - return lazy_module - - -def _ThreadingRLock(*args, **kwargs): - """threading RLock""" - try: - from threading import RLock - - return RLock(*args, **kwargs) - except (ImportError, OSError): - pass - - -class ITKLazyLoadLock: - """Need to use a recursive lock for thread ownership - within the given thread you can acquire a RLock as often as you like. - Other threads need to wait until this thread releases the resource again. - - A single lock is needed for all lazy loading. This lock blocks - across all threads until this thread has completed all its imports - and dependencies. The complex inter-relationship, and the recursive - nature of imports, makes a more fine-grained locking very difficult - to implement robustly.""" - - # global thread lock so no setup required for multithreading. - # NB: Do not create multiprocessing lock as it sets the multiprocessing - # context, disallowing `spawn()`/`forkserver()` - th_lock = _ThreadingRLock() - - def __init__(self): - cls = type(self) - root_lock = cls.th_lock - if root_lock is not None: - root_lock.acquire() - cls.create_mp_lock() - self.locks = [lk for lk in [cls.mp_lock, cls.th_lock] if lk is not None] - if root_lock is not None: - root_lock.release() - - def acquire(self, *a, **k): - for lock in self.locks: - lock.acquire(*a, **k) - - def release(self): - for lock in self.locks[::-1]: # Release in inverse order of acquisition - lock.release() - - def __enter__(self): - self.acquire() - - def __exit__(self, *exc): - self.release() - - @classmethod - def create_mp_lock(cls): - if not hasattr(cls, "mp_lock"): - try: - from multiprocessing import RLock - - cls.mp_lock = RLock() - except (ImportError, OSError): - cls.mp_lock = None - - -class LazyITKModule(types.ModuleType): - """Subclass of ModuleType that implements a custom __getattribute__ method - to allow lazy-loading of attributes from ITK sub-modules.""" - - def __init__(self, name, lazy_attributes): - types.ModuleType.__init__(self, name) - for k, v in lazy_attributes.items(): - base.itk_base_global_lazy_attributes.setdefault(k, _builtin_set()).update(v) - self.__belong_lazy_attributes = { - k: v[0] for k, v in lazy_attributes.items() if len(v) > 0 - } - for k in lazy_attributes: - setattr(self, k, not_loaded) # use default known value - # For PEP 366 - setattr(self, "__package__", "itk") - setattr(self, "itk_base_global_lazy_attributes", lazy_attributes) - setattr(self, "loaded_lazy_modules", _builtin_set()) - - @classmethod - def set_lock(cls, lock): - """Set the global lock.""" - cls._lock = lock - - @classmethod - def get_lock(cls): - """Get the global lock. Construct it if it does not exist.""" - if not hasattr(cls, "_lock"): - cls._lock = ITKLazyLoadLock() - return cls._lock - - def __getattribute__(self, attr): - value = types.ModuleType.__getattribute__(self, attr) - if value is not_loaded: - with type(self).get_lock(): # All but one thread will block here. - if value is not_loaded: - # Only the first thread needs to run this code, all other blocked threads skip - module = self.__belong_lazy_attributes[attr] - namespace = {} - base.itk_load_swig_module(module, namespace) - self.loaded_lazy_modules.add(module) - for k, v in namespace.items(): - setattr(self, k, v) - value = namespace[attr] - if _DefaultFactoryLoading: - base.load_module_needed_factories(module) - else: # one of the other threads that had been blocking - # waiting for first thread to complete. Now the - # attribute is REQUIRED to be available - # can just fall through now. - value = types.ModuleType.__getattribute__(self, attr) - assert value is not not_loaded - return value - - # For pickle support - def __reduce_ex__(self, proto): - state = self.__getstate__() - return _lazy_itk_module_reconstructor, (self.__name__, state), state - - # For pickle support - def __getstate__(self): - state = self.__dict__.copy() - lazy_modules = list() - for key in self.itk_base_global_lazy_attributes: - if isinstance(state[key], LazyITKModule): - lazy_modules.append((key, state[key].itk_base_global_lazy_attributes)) - state[key] = not_loaded - state["lazy_modules"] = lazy_modules - - return state - - # For pickle support - def __setstate__(self, state): - self.__dict__.update(state) - for module_name, lazy_attributes in state["lazy_modules"]: - self.__dict__.update( - {module_name: LazyITKModule(module_name, lazy_attributes)} - ) - for module in state["loaded_lazy_modules"]: - namespace = {} - base.itk_load_swig_module(module, namespace) - for k, v in namespace.items(): - setattr(self, k, v) - base.load_module_needed_factories(module) diff --git a/Wrapping/Generators/Python/itkConfig.template.in.py b/Wrapping/Generators/Python/itkConfig.template.in.py index 1e68019fa54..8b235c8095e 100644 --- a/Wrapping/Generators/Python/itkConfig.template.in.py +++ b/Wrapping/Generators/Python/itkConfig.template.in.py @@ -30,9 +30,7 @@ be called when each new library is imported in the import process. ImportCallback must be a function that takes two parameters: the name of the library being imported, and a float (between 0 and 1) reflecting the - fraction of the import that is completed. - LazyLoading: Only load an itk library when needed. Before the library is - loaded, the namespace will be inhabited with dummy objects.""" + fraction of the import that is completed.""" # User options @@ -79,7 +77,6 @@ def _strtobool(val: str) -> bool: DefaultFactoryLoading: bool = _get_environment_boolean( "ITK_PYTHON_DEFAULTFACTORYLOADING", "True" ) -LazyLoading: bool = _get_environment_boolean("ITK_PYTHON_LAZYLOADING", "True") NotInPlace: bool = _get_environment_boolean("ITK_PYTHON_NOTINPLACE", "False") del _get_environment_boolean