Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ENH: Add require argument to load() to accept version specifiers #48

Merged
merged 9 commits into from
Jan 30, 2024
25 changes: 24 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,29 @@ discouraged._

You can ask `lazy.load` to raise import errors as soon as it is called:

```
```python
linalg = lazy.load('scipy.linalg', error_on_import=True)
```

#### Optional requirements

One use for lazy loading is for loading optional dependencies, with
`ImportErrors` only arising when optional functionality is accessed. If optional
functionality depends on a specific version, a version requirement can
be set:

```python
np = lazy.load("numpy", require="numpy >=1.24")
```

In this case, if `numpy` is installed, but the version is less than 1.24,
the `np` module returned will raise an error on attribute access. Using
this feature is not all-or-nothing: One module may rely on one version of
numpy, while another module may not set any requirement.

_Note that the requirement must use the package [distribution name][] instead
of the module [import name][]. For example, the `pyyaml` distribution provides
the `yaml` module for import._

[distribution name]: https://packaging.python.org/en/latest/glossary/#term-Distribution-Package
[import name]: https://packaging.python.org/en/latest/glossary/#term-Import-Package
125 changes: 94 additions & 31 deletions lazy_loader/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import ast
import importlib
import importlib.util
import inspect
import os
import sys
import types
Expand Down Expand Up @@ -99,24 +98,25 @@ def __dir__():


class DelayedImportErrorModule(types.ModuleType):
def __init__(self, frame_data, *args, **kwargs):
def __init__(self, frame_data, *args, message, **kwargs):
self.__frame_data = frame_data
self.__message = message
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want a default message? Like the old value: f"No module named '{fd['spec']}'\n\n""?
Also, why is message after *args in the function sig instead of before? For backward compat?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want a default message?

I found it was easier to be more explicit about setting the message in the branching logic. I can rework with a default, if preferred.

Also, why is message after *args in the function sig instead of before? For backward compat?

I figured keyword-only would be clearer, but I'm okay with any signature. Let me know if you'd like me to change this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I see what you mean and agree that dealing with the message is better handled in the branching logic.

super().__init__(*args, **kwargs)

def __getattr__(self, x):
if x in ("__class__", "__file__", "__frame_data"):
if x in ("__class__", "__file__", "__frame_data", "__message"):
super().__getattr__(x)
else:
fd = self.__frame_data
raise ModuleNotFoundError(
f"No module named '{fd['spec']}'\n\n"
f"{self.__message}\n\n"
"This error is lazily reported, having originally occured in\n"
f' File {fd["filename"]}, line {fd["lineno"]}, in {fd["function"]}\n\n'
f'----> {"".join(fd["code_context"] or "").strip()}'
)


def load(fullname, error_on_import=False):
def load(fullname, *, require=None, error_on_import=False):
"""Return a lazily imported proxy for a module.

We often see the following pattern::
Expand Down Expand Up @@ -160,6 +160,14 @@ def myfunc():

sp = lazy.load('scipy') # import scipy as sp

require : str
A dependency requirement as defined in PEP-508. For example::

"numpy >=1.24"

If defined, the proxy module will raise an error if the installed
version does not satisfy the requirement.

error_on_import : bool
Whether to postpone raising import errors until the module is accessed.
If set to `True`, import errors are raised as soon as `load` is called.
Expand All @@ -171,10 +179,12 @@ def myfunc():
Actual loading of the module occurs upon first attribute request.

"""
try:
return sys.modules[fullname]
except KeyError:
pass
module = sys.modules.get(fullname)
have_module = module is not None

# Most common, short-circuit
if have_module and require is None:
return module

if "." in fullname:
msg = (
Expand All @@ -184,33 +194,86 @@ def myfunc():
)
warnings.warn(msg, RuntimeWarning)

spec = importlib.util.find_spec(fullname)
if spec is None:
spec = None
if not have_module:
spec = importlib.util.find_spec(fullname)
have_module = spec is not None

if not have_module:
not_found_message = f"No module named '{fullname}'"
elif require is not None:
try:
have_module = _check_requirement(require)
except ModuleNotFoundError as e:
raise ValueError(
f"Found module '{fullname}' but cannot test requirement '{require}'. "
"Requirements must match distribution name, not module name."
effigies marked this conversation as resolved.
Show resolved Hide resolved
) from e

not_found_message = f"No distribution can be found matching '{require}'"

if not have_module:
if error_on_import:
raise ModuleNotFoundError(f"No module named '{fullname}'")
else:
try:
parent = inspect.stack()[1]
frame_data = {
"spec": fullname,
"filename": parent.filename,
"lineno": parent.lineno,
"function": parent.function,
"code_context": parent.code_context,
}
return DelayedImportErrorModule(frame_data, "DelayedImportErrorModule")
finally:
del parent

module = importlib.util.module_from_spec(spec)
sys.modules[fullname] = module

loader = importlib.util.LazyLoader(spec.loader)
loader.exec_module(module)
raise ModuleNotFoundError(not_found_message)
import inspect

try:
parent = inspect.stack()[1]
frame_data = {
"filename": parent.filename,
"lineno": parent.lineno,
"function": parent.function,
"code_context": parent.code_context,
}
return DelayedImportErrorModule(
frame_data,
"DelayedImportErrorModule",
message=not_found_message,
)
finally:
del parent

if spec is not None:
module = importlib.util.module_from_spec(spec)
sys.modules[fullname] = module

loader = importlib.util.LazyLoader(spec.loader)
loader.exec_module(module)

return module


def _check_requirement(require: str) -> bool:
"""Verify that a package requirement is satisfied

If the package is required, a ``ModuleNotFoundError`` is raised
by ``importlib.metadata``.

Parameters
----------
require : str
A dependency requirement as defined in PEP-508

Returns
-------
satisfied : bool
True if the installed version of the dependency matches
the specified version, False otherwise.
"""
import packaging.requirements

try:
import importlib.metadata as importlib_metadata
except ImportError: # PY37
import importlib_metadata

req = packaging.requirements.Requirement(require)
return req.specifier.contains(
importlib_metadata.version(req.name),
prereleases=True,
)


class _StubVisitor(ast.NodeVisitor):
"""AST visitor to parse a stub file for submodules and submod_attrs."""

Expand Down
23 changes: 23 additions & 0 deletions lazy_loader/tests/test_lazy_loader.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import importlib
import sys
import types
from unittest import mock

import pytest

Expand Down Expand Up @@ -149,3 +150,25 @@ def test_stub_loading_errors(tmp_path):

with pytest.raises(ValueError, match="Cannot load imports from non-existent stub"):
lazy.attach_stub("name", "not a file")


def test_require_kwarg():
have_importlib_metadata = importlib.util.find_spec("importlib.metadata") is not None
dot = "." if have_importlib_metadata else "_"
# Test with a module that definitely exists, behavior hinges on requirement
with mock.patch(f"importlib{dot}metadata.version") as version:
version.return_value = "1.0.0"
math = lazy.load("math", require="somepkg >= 2.0")
assert isinstance(math, lazy.DelayedImportErrorModule)

math = lazy.load("math", require="somepkg >= 1.0")
assert math.sin(math.pi) == pytest.approx(0, 1e-6)

# We can fail even after a successful import
math = lazy.load("math", require="somepkg >= 2.0")
assert isinstance(math, lazy.DelayedImportErrorModule)

# When a module can be loaded but the version can't be checked,
# raise a ValueError
with pytest.raises(ValueError):
lazy.load("math", require="somepkg >= 1.0")
4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ classifiers = [
"Programming Language :: Python :: 3.12",
]
description = "Makes it easy to load subpackages and functions on demand."
dependencies = [
"packaging",
"importlib_metadata; python_version < '3.8'",
]

[project.optional-dependencies]
test = ["pytest >= 7.4", "pytest-cov >= 4.1"]
Expand Down
Loading