diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aa6ea4..5d7055f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,17 @@ page. ### Added +- **`PackageData.from_strings(name, version, requires=None, variants=None)`** — + classmethod constructor for raw-string callers, symmetric with + `from_rez(pkg)`. Skips rez's `AttributeForwardMeta` chain, the + `Requirement` parse, and the `str(Requirement)` round-trip — the + latter being a measurable fraction of integration overhead on + rez-shim hot paths (per-package, every package, every resolve). + Functionally equivalent to the four-arg constructor; the + classmethod form exists so callers wiring `pkg.resource.data` + through pyrer have a named, documented contract to reach for. + Falls back to `from_rez` for `@early` / `@late`-bound attributes. + Closes #88. - **`load_family` callback** on `pyrer.solve()` — opt-in lazy package discovery: pass `load_family: Callable[[str], list[PackageData]]` and the solver calls it on demand the first time it needs a family it hasn't seen. diff --git a/Cargo.toml b/Cargo.toml index 4e3b845..e1c771b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["crates/*"] resolver = "2" [workspace.package] -version = "0.1.0-rc.8" +version = "0.1.0-rc.9" authors = [ "Lorenzo Montant ", "Maxim Doucet ", @@ -23,8 +23,8 @@ lazy_static = "1.5.0" rand = "0.8.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -rer-version = { path = "crates/rer-version", version = "0.1.0-rc.8" } -rer-resolver = { path = "crates/rer-resolver", version = "0.1.0-rc.8" } +rer-version = { path = "crates/rer-version", version = "0.1.0-rc.9" } +rer-resolver = { path = "crates/rer-resolver", version = "0.1.0-rc.9" } pyo3 = { version = "0.23.5", features = ["extension-module"] } # `mimalloc` is wired into the bench binary as a `#[global_allocator]`. # Callgrind shows ~33 % of cycles in libc malloc/free; mimalloc has measurably diff --git a/crates/rer-python/src/lib.rs b/crates/rer-python/src/lib.rs index 75952d8..c6792be 100644 --- a/crates/rer-python/src/lib.rs +++ b/crates/rer-python/src/lib.rs @@ -85,6 +85,16 @@ impl PackageData { /// `pyrer` so every integration site doesn't have to write the same /// extraction loop. `pyrer` itself does not import rez; this method is /// duck-typed and works against any object with the four attributes. + /// + /// **Faster alternative for raw-data callers:** if you already have the + /// raw strings (typically from `pkg.resource.data` on a rez `Package`, + /// which stores `requires` / `variants` as raw `list[str]` / + /// `list[list[str]]` in the common non-late-bound case), prefer + /// [`Self::from_strings`]. It skips the per-attribute + /// `AttributeForwardMeta` lookup, the late-bound wrapping, the + /// `Requirement` parse, and the `str(Requirement)` round-trip — none + /// of which produce a different `PackageData` for the common case, + /// but all of which take real time per package. #[classmethod] fn from_rez(_cls: &Bound<'_, PyType>, pkg: &Bound<'_, PyAny>) -> PyResult { let name: String = pkg.getattr("name")?.extract()?; @@ -98,6 +108,58 @@ impl PackageData { variants, }) } + + /// Build a [`PackageData`] from raw strings, skipping any rez + /// wrapper-object resolution. Use this when you already have raw + /// `(name, version, requires, variants)` data — typically pulled from + /// `pkg.resource.data` on a rez `Package`: + /// + /// ```python + /// data = pkg.resource.data + /// pd = pyrer.PackageData.from_strings( + /// data["name"], + /// data["version"], + /// data.get("requires"), # may be None / list[str] + /// data.get("variants"), # may be None / list[list[str]] + /// ) + /// ``` + /// + /// Faster than [`Self::from_rez`] on rez-integration hot paths because + /// it does not trigger rez's `AttributeForwardMeta` per attribute, does + /// not parse each requirement string into a `Requirement` object, and + /// does not round-trip each `Requirement` back through `__str__`. + /// + /// `requires` and `variants` accept `None` (interpreted as empty), + /// matching `dict.get(...)` ergonomics. + /// + /// Functionally equivalent to the four-arg constructor + /// `PackageData(name, version, requires, variants)` — both take the + /// same fast PyO3 extraction path. The classmethod form exists to make + /// the fast path discoverable alongside [`Self::from_rez`] and to give + /// the contract a name in callers' code. Closes #88. + /// + /// **Caveat — late-bound requirements:** for packages where rez stores + /// `requires` or `variants` as a `SourceCode` instance (`@early` / + /// `@late` binding), `pkg.resource.data["requires"]` is *not* a + /// `list[str]` and this method will raise. Fall back to + /// [`Self::from_rez`] for those packages — it walks rez's lazy + /// attribute path which evaluates the source code. + #[classmethod] + #[pyo3(signature = (name, version, requires=None, variants=None))] + fn from_strings( + _cls: &Bound<'_, PyType>, + name: String, + version: String, + requires: Option>, + variants: Option>>, + ) -> Self { + PackageData { + name, + version, + requires: requires.unwrap_or_default(), + variants: variants.unwrap_or_default(), + } + } } /// Pull a flat list of requirement strings from a Python object that is diff --git a/docs/config.toml b/docs/config.toml index f1cd0b6..e3e16c9 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -125,7 +125,7 @@ weight = 10 name = "GitHub" pre = '' url = "https://github.com/doubleailes/rer" -post = "v0.1.0-rc.8" +post = "v0.1.0-rc.9" weight = 20 # Footer contents diff --git a/docs/content/_index.md b/docs/content/_index.md index f16bd1e..46882bd 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -7,7 +7,7 @@ title = "rer — Rez En Rust" lead = "A faithful Rust port of rez's package solver — callable from Python via PyO3, resolves match rez 1:1." url = "/docs/getting-started/introduction/" url_button = "Get started" -repo_version = "GitHub v0.1.0-rc.8" +repo_version = "GitHub v0.1.0-rc.9" repo_license = "MIT-licensed." repo_url = "https://github.com/doubleailes/rer" diff --git a/docs/content/docs/getting-started/quick-start.md b/docs/content/docs/getting-started/quick-start.md index c04fdcc..941ca27 100644 --- a/docs/content/docs/getting-started/quick-start.md +++ b/docs/content/docs/getting-started/quick-start.md @@ -95,7 +95,7 @@ Add the resolver crate to your `Cargo.toml`: ```toml [dependencies] -rer-resolver = "0.1.0-rc.8" +rer-resolver = "0.1.0-rc.9" ``` Then call the solver against an in-memory repository: diff --git a/docs/content/docs/getting-started/rez-integration.md b/docs/content/docs/getting-started/rez-integration.md index d633c33..89f79e7 100644 --- a/docs/content/docs/getting-started/rez-integration.md +++ b/docs/content/docs/getting-started/rez-integration.md @@ -79,6 +79,50 @@ is duck-typed — `pyrer` itself does not import rez — so you can also pass any object exposing the same four attributes (e.g. a test fixture). +### Faster construction with `from_strings` + +`from_rez(pkg)` triggers rez's `AttributeForwardMeta` chain on every +attribute and parses each requirement string into a `Requirement` +object only to immediately turn it back into a string. When you +already have the raw strings, prefer +`PackageData.from_strings(name, version, requires, variants)` — +it skips the wrapper round-trip entirely: + +```python +def build_pyrer_packages_fast(package_paths): + for family in iter_package_families(paths=package_paths): + for pkg in family.iter_packages(): + data = pkg.resource.data + # `data["requires"]` is a raw list[str] in the common + # (non-late-bound) case; fall back to from_rez otherwise. + if isinstance(data.get("requires", []), list) and \ + isinstance(data.get("variants", []), list): + yield pyrer.PackageData.from_strings( + data["name"], + data["version"], + data.get("requires"), + data.get("variants"), + ) + else: + # @early / @late bindings — let rez evaluate them. + yield pyrer.PackageData.from_rez(pkg) +``` + +The `from_strings` method: + +- Skips the per-attribute `AttributeForwardMeta` lookup. +- Skips the `Requirement` parse (no `Version` / `VersionRange` AST + is built then discarded). +- Skips the `str(Requirement)` round-trip per requirement. +- Accepts `None` for `requires` / `variants` (matches + `dict.get(...)` ergonomics — no `or ()` boilerplate needed). + +Functionally equivalent to the four-arg constructor; the +classmethod form exists so the contract has a name. **Always fall +back to `from_rez` for packages with `@early` or `@late` binding** — +in those cases `resource.data["requires"]` is a `SourceCode` +instance, not a `list[str]`, and `from_strings` will raise. + Two notes on this step: - It is **eager** — every package on every path is loaded before the diff --git a/tests/test_rich_api.py b/tests/test_rich_api.py index 3370e8b..c20d2ba 100644 --- a/tests/test_rich_api.py +++ b/tests/test_rich_api.py @@ -265,6 +265,119 @@ class NotAPackage: pyrer.PackageData.from_rez(NotAPackage()) +# --------------------------------------------------------------------------- +# PackageData.from_strings — raw-string fast path (issue #88) +# --------------------------------------------------------------------------- + + +def test_from_strings_basic(): + """All four args supplied as raw strings — no wrapper objects involved.""" + pd = pyrer.PackageData.from_strings( + "maya", + "2024.0", + ["python-3"], + [["python-3.10"], ["python-3.11"]], + ) + assert pd.name == "maya" + assert pd.version == "2024.0" + assert pd.requires == ["python-3"] + assert pd.variants == [["python-3.10"], ["python-3.11"]] + + +def test_from_strings_defaults_to_empty(): + """requires=None and variants=None default to empty lists.""" + pd = pyrer.PackageData.from_strings("foo", "1.0") + assert pd.requires == [] + assert pd.variants == [] + + +def test_from_strings_accepts_none_for_collections(): + """`dict.get("requires")` returns None for a missing key — must accept it.""" + pd = pyrer.PackageData.from_strings("foo", "1.0", None, None) + assert pd.requires == [] + assert pd.variants == [] + + +def test_from_strings_accepts_tuples_and_iterables(): + """PyO3 extracts Vec from any iterable, not just list.""" + pd = pyrer.PackageData.from_strings( + "tool", + "1.0", + ("python-3", "qt-5"), + (("linux", "python-3.10"),), + ) + assert pd.requires == ["python-3", "qt-5"] + assert pd.variants == [["linux", "python-3.10"]] + + +def test_from_strings_matches_constructor(): + """`from_strings` must produce the same PackageData as the four-arg + constructor — same fast PyO3 extraction path, classmethod is just a + named alias for callers wiring rez's resource.data through pyrer.""" + args = ("maya", "2024.0", ["python-3"], [["python-3.10"], ["python-3.11"]]) + via_classmethod = pyrer.PackageData.from_strings(*args) + via_constructor = pyrer.PackageData(*args) + assert via_classmethod.name == via_constructor.name + assert via_classmethod.version == via_constructor.version + assert via_classmethod.requires == via_constructor.requires + assert via_classmethod.variants == via_constructor.variants + + +def test_from_strings_drives_solve_like_from_rez(): + """End-to-end: a solve fed via from_strings produces the same result as + one fed via from_rez against an equivalent fake-rez Package.""" + + class FakeReq: + def __init__(self, s): + self._s = s + + def __str__(self): + return self._s + + class FakePkg: + def __init__(self, name, version, requires=None, variants=None): + self.name = name + self.version = version + self.requires = ( + [FakeReq(r) for r in requires] if requires else None + ) + self.variants = ( + [[FakeReq(r) for r in v] for v in variants] if variants else None + ) + + fakes = [ + FakePkg("app", "1.0.0", requires=["lib-2"]), + FakePkg("lib", "1.0.0"), + FakePkg("lib", "2.0.0"), + ] + via_from_rez = [pyrer.PackageData.from_rez(p) for p in fakes] + + via_from_strings = [ + pyrer.PackageData.from_strings("app", "1.0.0", ["lib-2"]), + pyrer.PackageData.from_strings("lib", "1.0.0"), + pyrer.PackageData.from_strings("lib", "2.0.0"), + ] + + result_a = pyrer.solve(["app"], via_from_rez) + result_b = pyrer.solve(["app"], via_from_strings) + assert result_a.resolved == result_b.resolved + assert result_a.status == result_b.status == "solved" + + +def test_from_strings_rejects_non_string_requires(): + """from_strings is the contract-strict fast path — pass it a non-string + in `requires` and it raises rather than silently stringifying. Use + `from_rez` (or pre-stringify) for object inputs.""" + import pytest + + class NotAString: + def __str__(self): + return "python-3" + + with pytest.raises(TypeError): + pyrer.PackageData.from_strings("foo", "1.0", [NotAString()]) + + # --------------------------------------------------------------------------- # variant_select_mode — rez's intersection_priority vs version_priority # ---------------------------------------------------------------------------