From efef8f987cf76196b685734a42d43ed770779f3b Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Thu, 3 Sep 2026 15:15:19 -0700 Subject: [PATCH] feat(agent): --distill flag recording LLM calls as a ChatML NDJSON fine-tuning dataset Add a boolean --distill (-d, -dt) flag to tablassert agent that appends every LLM call of the run (inner agent, judge, reflexion) as one ChatML record per line to /distill/records.ndjson. The file is append-only across invocations so a corpus accumulates run over run, and loads directly in Unsloth Studio / datasets.load_dataset("json"). - tablassert.distill: zero-dependency DistillRecorder + ChatML serializers; recording never raises into a batch - agent.make_distilling_model: Model wrapper around generate() (the single capture seam), tagged per-article with pmc_id for state.json filtering - tablassert distill-export: convert the NDJSON to an on-disk HF dataset (new [distill] extra shipping datasets) - docs (cli.md, agent.md) + docs-coverage mapping; CLI/recorder/wrapper/ supervisor end-to-end tests --- docs/agent.md | 26 ++++ docs/cli.md | 23 +++- pyproject.toml | 3 + src/tablassert/agent.py | 67 +++++++++- src/tablassert/cli.py | 64 +++++++++- src/tablassert/distill.py | 108 ++++++++++++++++ src/tablassert/extras.py | 2 + tests/test_agent_cli.py | 51 ++++++++ tests/test_agent_supervisor.py | 40 +++++- tests/test_distill.py | 218 ++++++++++++++++++++++++++++++++ tests/test_docs_cli_coverage.py | 2 + uv.lock | 201 ++++++++++++++++++++++++++++- 12 files changed, 791 insertions(+), 14 deletions(-) create mode 100644 src/tablassert/distill.py create mode 100644 tests/test_distill.py diff --git a/docs/agent.md b/docs/agent.md index 82003c2..ed513d8 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -217,6 +217,32 @@ endpoint; neither is required): the threshold turns that measurement into a terminal gate. See [Biolink validity](#biolink-validity) below. +### Distilling a fine-tuning dataset (`--distill`) + +`--distill` (short: `-d`, `-dt`) records **every LLM call of the run** — the inner agent's +multi-turn conversations, plus the judge and reflexion calls when those gates are enabled — as one +ChatML JSON object per line, appended to `/distill/records.ndjson`: + +```json +{"messages": [{"role": "system", "content": "..."}, {"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}], "purpose": "agent", "model_id": "...", "pmc_id": "PMC11708054", "call_index": 0, "timestamp": "...", "token_usage": {"input_tokens": 0, "output_tokens": 0}} +``` + +The file is **append-only**: every `--distill` invocation keeps adding to the same dataset, so a +corpus accumulates over many batches. The `messages` column is plain ChatML, which Unsloth Studio +auto-detects on JSONL upload (no column mapping needed); the metadata columns (`purpose`, +`pmc_id`, `call_index`, `timestamp`, `token_usage`) ride along for filtering — e.g. join on +`pmc_id` against `state.json` to keep only `MAPPED` runs, or keep each run's highest `call_index` +for the most complete conversation. Recording is zero-dependency and never breaks a run: a failed +write is logged, not raised. `--distill` is not supported with `--optimize` (the GEPA path +bypasses the recording seam). + +To convert the NDJSON into an on-disk Hugging Face dataset, use +[`tablassert distill-export`](cli.md#distill-export) (requires the `[distill]` extra): + +```bash +tablassert distill-export --distill-dir .tablassert/agent/distill --out ./hf-dataset +``` + ### Biolink validity Coverage answers *did the terms resolve?* It says nothing about whether the resulting records are diff --git a/docs/cli.md b/docs/cli.md index 89602ad..4e9ca6f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,7 +1,7 @@ # CLI Reference Tablassert extracts knowledge assertions from tabular data into KGX NDJSON. The `tablassert` app -exposes **five subcommands**: `agent`, `build-fullmap`, `build-kg`, `validate`, +exposes **six subcommands**: `agent`, `build-fullmap`, `build-kg`, `distill-export`, `validate`, and `validate-kgx`, plus an app-level `--version` flag. Run `tablassert --help` (or ` --help`) for the live surface. @@ -12,6 +12,7 @@ for the live surface. | [`agent`](#agent) | Autonomously derive, build, audit, and improve KG configs from PMC articles | | [`build-fullmap`](#build-fullmap) | Build the embedded fullmap redb used for entity resolution | | [`build-kg`](#build-kg) | Build a KGX NDJSON knowledge graph from a YAML configuration | +| [`distill-export`](#distill-export) | Export a recorded distillation NDJSON dataset to an on-disk Hugging Face dataset | | [`validate`](#validate) | Validate a graph or table configuration without executing it | | [`validate-kgx`](#validate-kgx) | Validate built KGX NDJSON against the Biolink Model | @@ -68,6 +69,7 @@ page lists the flags; see | `--biolink-threshold` | float | No | `0.0` | Minimum Biolink pass rate of the built KGX for MAPPED; `0.0` reports the rate without gating | | `--local`, `-l` | list[str] | No | `None` | Local payload: one DIR for all ids, or `PMCid=DIR` mappings; skips the PMC-AWS fetch (exit 2 on a missing DIR) | | `--optimize`, `-o` | bool | No | `False` | Run GEPA prompt optimization and persist optimized instructions instead of running the supervisor | +| `--distill`, `-d`, `-dt` | bool | No | `False` | Record every LLM call (agent, judge, reflexion) as ChatML NDJSON under `/distill/records.ndjson` for fine-tuning; not supported with `--optimize` | | `--instructions-file` | Path | No | `None` | Load GEPA-optimized instructions from a prior `--optimize` run | | `--instructions-out` | Path | No | `None` | Where `--optimize` writes optimized instructions (default `/optimized_instructions.yaml`) | | `--max-metric-calls` | int | No | `8` | GEPA metric-call budget for `--optimize` | @@ -88,6 +90,25 @@ tablassert agent PMC11708054 -f ./graph.yaml --- +## distill-export + +Use this to convert a distillation dataset recorded with +[`agent --distill`](#agent) into an on-disk Hugging Face dataset (`save_to_disk`). Requires the +`[distill]` extra (`pip install "tablassert[distill]"`, pulls `datasets`). The raw NDJSON already +loads directly in Unsloth Studio and via `datasets.load_dataset("json", ...)` — this export is +only needed for `datasets`-native workflows. + +```bash +tablassert distill-export --distill-dir .tablassert/agent/distill --out ./hf-dataset +``` + +| Option | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `--distill-dir`, `-dd` | Path | Yes | n/a | Directory holding the recorded `*.ndjson` files (exit 2 when empty) | +| `--out`, `-o` | Path | Yes | n/a | Destination directory for the `save_to_disk` dataset | + +--- + ## build-fullmap Use this to obtain the embedded `fullmap.redb` entity-resolution database. By default it first tries diff --git a/pyproject.toml b/pyproject.toml index 9ecd85c..e30c19c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,9 @@ agent = [ optimize = [ "dspy>=3.2.1", ] +distill = [ + "datasets>=3.0.0", +] log = [ "loguru>=0.7.3", ] diff --git a/src/tablassert/agent.py b/src/tablassert/agent.py index b50bb43..7d2f1cd 100644 --- a/src/tablassert/agent.py +++ b/src/tablassert/agent.py @@ -3232,6 +3232,56 @@ def generate( return FakeModel() +def make_distilling_model(model: object, recorder: object, *, purpose: str, meta: dict[str, object] | None = None) -> object: + """Wrap a smolagents model so every ``generate`` call is recorded by ``recorder``. + + Returns a ``Model`` subclass (same shape as :func:`make_fake_model`) that delegates + ``generate`` to the wrapped model and appends one ChatML NDJSON record per call, tagged with + ``purpose`` (``"agent"``/``"judge"``/``"reflexion"``) plus ``meta`` (e.g. ``pmc_id``). The + wrapped model's own ``model_id`` is added to the record when recoverable. Attribute access + falls through to the wrapped model (``__getattr__``) so smolagents sees the real model's + metadata. Recording failures never propagate — the response is returned untouched. + """ + _require("smolagents") + from smolagents.models import ChatMessage, Model # local import keeps module import lazy # pyright: ignore[reportMissingImports] + + extra_meta: dict[str, object] = dict(meta or {}) + if "model_id" not in extra_meta: + model_id: object = getattr(model, "model_id", None) + if model_id is not None: + extra_meta["model_id"] = str(model_id) + + class DistillingModel(Model): # pyright: ignore[reportMissingImports] + def __init__(self) -> None: + with contextlib.suppress(Exception): + super().__init__() + self._wrapped: object = model + + def __getattr__(self, name: str) -> object: + # Only fires for attributes Model does not define; delegates model_id & friends. + return getattr(self._wrapped, name) + + def generate( + self, + messages: list[ChatMessage], + stop_sequences: list[str] | None = None, + response_format: dict[str, str] | None = None, + tools_to_call_from: object = None, + **kwargs: Any, + ) -> ChatMessage: + response: ChatMessage = cast( + ChatMessage, + self._wrapped.generate( # pyright: ignore[reportAttributeAccessIssue] + messages, stop_sequences=stop_sequences, response_format=response_format, tools_to_call_from=tools_to_call_from, **kwargs + ), + ) + with contextlib.suppress(Exception): # recording must never break the run + recorder.record(purpose, messages, response, **extra_meta) # pyright: ignore[reportAttributeAccessIssue] + return response + + return DistillingModel() + + # --------------------------------------------------------------------------- # # US-009: outer DETERMINISTIC supervisor + monotonic improve loop + checkpoint/resume # @@ -3433,6 +3483,11 @@ def pmc_build_dir(root: Path, pmc_id: str) -> Path: return root / "builds" / pmc_id +def distill_dir(root: Path) -> Path: + """Return ``/distill`` (the distillation dataset dir); pure, no mkdir.""" + return root / "distill" + + @dataclass class ConfigRecord: """Per-PMC supervisor record: status, derived/best config paths, and coverage history. @@ -3616,6 +3671,7 @@ def run_supervisor( instructions: str | None = None, derive_mode: DeriveMode = "full", min_rows: int = MIN_TABLE_ROWS, + distill_recorder: object | None = None, ) -> dict[str, object]: """Run the deterministic supervisor over a batch of PMC ids with checkpoint/resume. @@ -3652,7 +3708,9 @@ def run_supervisor( The whole per-pmc body is wrapped in try/except: ANY failure marks that record SKIPPED with the reason and advances (one bad pmc never aborts the batch). ``build_model_factory`` is a zero-arg callable returning a configured model so tests inject a FakeModel and the real CLI keeps secrets - out of this signature. Returns ``{"state", "records", "metrics"}`` after a final checkpoint. + out of this signature. ``distill_recorder`` (optional) wraps each article's model via + :func:`make_distilling_model` so every LLM call is appended to the distillation NDJSON dataset. + Returns ``{"state", "records", "metrics"}`` after a final checkpoint. """ if min_rows < 0: raise ValueError("min_rows must be non-negative") @@ -3749,8 +3807,13 @@ def audit_config(config_yaml: str, **kwargs: Any) -> dict[str, object]: article_xml: Path | None = next((path for path in files if path.suffix.lower() in {".xml", ".nxml"}), None) metrics: dict[str, object] = {} + model: object = build_model_factory() + if distill_recorder is not None: + # Distillation capture: wrap so every generate() call lands in the NDJSON dataset, + # tagged with this article's id for later filtering against state.json status. + model = make_distilling_model(model, distill_recorder, purpose="agent", meta={"pmc_id": pmc_id}) agent: object = build_agent( - model=build_model_factory(), + model=model, tools=make_tools( graph=target_graph, fullmap=effective_fullmap, diff --git a/src/tablassert/cli.py b/src/tablassert/cli.py index e261dd8..386eac8 100644 --- a/src/tablassert/cli.py +++ b/src/tablassert/cli.py @@ -776,6 +776,7 @@ def agent( biolink_threshold: Annotated[float, cyclopts.Parameter(name=["--biolink-threshold"])] = 0.0, local: Annotated[list[str] | None, cyclopts.Parameter(name=["--local", "-l"])] = None, optimize: Annotated[bool, cyclopts.Parameter(name=["--optimize", "-o"], negative="")] = False, + distill: Annotated[bool, cyclopts.Parameter(name=["--distill", "-d", "-dt"], negative="")] = False, instructions_file: Annotated[Path | None, cyclopts.Parameter(name=["--instructions-file"])] = None, instructions_out: Annotated[Path | None, cyclopts.Parameter(name=["--instructions-out"])] = None, max_metric_calls: Annotated[int, cyclopts.Parameter(name=["--max-metric-calls"])] = 8, @@ -825,6 +826,9 @@ def agent( one or more ``PMCid=DIR`` mappings (per-article). Fails loud (exit 2) if a DIR does not exist. optimize: Run GEPA prompt optimization over the model config and persist optimized instructions (instead of running the supervisor); use ``--instructions-out`` to choose the output file. + distill: Record every LLM call of the run (inner agent, judge, reflexion) as ChatML NDJSON + under ``/distill/records.ndjson`` for fine-tuning (Unsloth Studio / QLoRA). + Zero extra dependencies; ``tablassert distill-export`` converts it to an on-disk HF dataset. instructions_file: Load GEPA-optimized instructions (from a prior ``--optimize`` run) for this run. instructions_out: Where ``--optimize`` writes optimized instructions (default ``/optimized_instructions.yaml``). @@ -877,6 +881,12 @@ def agent( print("tablassert agent: --gepa-threads must be a positive integer.", file=sys.stderr) raise SystemExit(2) + # --distill records the SUPERVISOR's model calls; the --optimize path returns early below and + # GEPA's dspy LM bypasses the recording seam, so the combination would silently record nothing. + if distill and optimize: + print("tablassert agent: --distill records supervisor LLM calls and is not supported with --optimize.", file=sys.stderr) + raise SystemExit(2) + # Preflight the extras once the flags are known to be valid and BEFORE any model is # built or any article fetched. smolagents is otherwise only required per-article # (inside build_agent) and dspy only once GEPA starts, so an absent extra would @@ -885,6 +895,17 @@ def agent( if optimize: extras.require("optimize", required_by="tablassert agent --optimize") + # Distillation capture (optional, zero-dep): every LLM call is appended as one ChatML NDJSON + # record. The inner agent's model is wrapped per-article inside run_supervisor (which knows the + # pmc_id); the judge and reflexion models are wrapped at their construction sites below. + distill_recorder: object | None = None + if distill: + from tablassert import distill as distill_mod + + distill_path: Path = agent_mod.distill_dir(state_dir) / distill_mod.RECORDS_FILENAME + distill_recorder = distill_mod.DistillRecorder(distill_path) + print(f"tablassert agent: distilling LLM calls -> {distill_path}") + def build_model_factory() -> object: return agent_mod.build_model(resolved_id, resolved_base, resolved_key, backend=backend) @@ -893,14 +914,20 @@ def build_model_factory() -> object: if reflexion: def _make_reflexion() -> object: - return agent_mod.make_prompt_callable(agent_mod.build_model(resolved_id, resolved_base, resolved_key, backend=backend)) + reflexion_model: object = agent_mod.build_model(resolved_id, resolved_base, resolved_key, backend=backend) + if distill_recorder is not None: + reflexion_model = agent_mod.make_distilling_model(reflexion_model, distill_recorder, purpose="reflexion") + return agent_mod.make_prompt_callable(reflexion_model) reflexion_factory = _make_reflexion # Semantic judge (optional): a prompt-callable over the judge model (same api_base/api_key). judge: object | None = None if judge_model is not None: - judge = agent_mod.make_prompt_callable(agent_mod.build_model(judge_model, resolved_base, resolved_key, backend=backend)) + judge_base_model: object = agent_mod.build_model(judge_model, resolved_base, resolved_key, backend=backend) + if distill_recorder is not None: + judge_base_model = agent_mod.make_distilling_model(judge_base_model, distill_recorder, purpose="judge") + judge = agent_mod.make_prompt_callable(judge_base_model) # Local payload (optional, W4): a DIR for all ids, or PMCid=DIR mappings; fail loud on a missing dir. def parse_local(specs: list[str] | None) -> dict[str, Path] | Path | None: @@ -990,6 +1017,7 @@ def parse_local(specs: list[str] | None) -> dict[str, Path] | Path | None: biolink_threshold=biolink_threshold, local=local_payload, instructions=run_instructions, + distill_recorder=distill_recorder, ) metrics_raw: object = result.get("metrics") @@ -1012,6 +1040,38 @@ def metric(key: str, default: float) -> float: ) +@APP.command(name="distill-export") +def distill_export( + *, distill_dir: Annotated[Path, cyclopts.Parameter(name=["--distill-dir", "-dd"])], out: Annotated[Path, cyclopts.Parameter(name=["--out", "-o"])] +) -> None: + """Export a recorded distillation NDJSON dataset to an on-disk Hugging Face dataset. + + Loads every ``*.ndjson`` under ``--distill-dir`` (the ChatML records written by ``tablassert + agent --distill``) with ``datasets.load_dataset("json", ...)`` and writes the result with + ``save_to_disk`` to ``--out``. Requires the ``distill`` extra (``pip install + "tablassert[distill]"``). The raw NDJSON also loads directly in Unsloth Studio — this export + is only needed for ``datasets``-native workflows. + + Args: + distill_dir: Directory holding the recorded ``*.ndjson`` files (default output of + ``tablassert agent --distill`` is ``/distill``). + out: Destination directory for the ``save_to_disk`` dataset. + """ + # Input validation precedes the extras preflight: a missing directory is the user's typo, an + # absent extra is their environment, and the typo is the faster loop to close first. + files: list[Path] = sorted(distill_dir.glob("*.ndjson")) + if not files: + print(f"tablassert distill-export: no .ndjson records under {distill_dir} — run tablassert agent --distill first.", file=sys.stderr) + raise SystemExit(2) + extras.require("distill", required_by="tablassert distill-export") + from datasets import load_dataset # local import keeps the CLI import-light # pyright: ignore[reportMissingImports] + + dataset: object = load_dataset("json", data_files=[str(path) for path in files], split="train") + out.parent.mkdir(parents=True, exist_ok=True) + dataset.save_to_disk(str(out)) # pyright: ignore[reportAttributeAccessIssue] + print(f"tablassert distill-export: {len(dataset)} record(s) from {len(files)} file(s) -> {out}") # pyright: ignore[reportArgumentType] + + class PrebuiltFullmapUnavailable(Exception): """A prebuilt fullmap could not be fetched or extracted. diff --git a/src/tablassert/distill.py b/src/tablassert/distill.py new file mode 100644 index 0000000..ee8ba02 --- /dev/null +++ b/src/tablassert/distill.py @@ -0,0 +1,108 @@ +"""Distillation capture: record agent LLM traffic as an HF-ready ChatML NDJSON dataset. + +When ``tablassert agent --distill`` is set, every ``model.generate`` call made during the run +(inner CodeAgent turns, the semantic judge, tier-2 reflexion) is appended as ONE JSON object per +line to ``/distill/records.ndjson``. The schema is ChatML — a top-level ``messages`` +list of ``{"role", "content"}`` dicts — which Unsloth Studio auto-detects on JSONL upload and +``datasets.load_dataset("json", ...)`` loads directly; metadata keys (``purpose``, ``pmc_id``, +``model_id``, ``call_index``, ``timestamp``, ``token_usage``) ride alongside as extra columns for +filtering (e.g. keeping only MAPPED runs). Strict JSONL: no outer array, no commas between lines. + +This module is ZERO-dependency by design — recording must work in any install that can run the +agent, and must NEVER break a batch: every write is guarded and failures only log. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Final + +from tablassert.log import cat + +logger = cat("AGENT") + +#: Default dataset filename inside ``distill_dir(state_dir)``; append-only across runs so a +#: fine-tuning corpus accumulates over many invocations. +RECORDS_FILENAME: Final[str] = "records.ndjson" + + +def _role_name(role: object) -> str: + """Coerce a message role (``MessageRole`` enum or plain string) to its plain name.""" + value: object = getattr(role, "value", role) + return str(value) + + +def serialize_message(message: object) -> dict[str, str]: + """Serialize one smolagents ``ChatMessage`` (or ``{"role", "content"}`` dict) to ChatML. + + All access is guarded so an unexpected message shape degrades to a best-effort dict rather + than raising into the run. Non-string content (e.g. multimodal part lists) is stringified. + """ + if isinstance(message, dict): + role: object = message.get("role", "user") + content: object = message.get("content", "") + else: + role = getattr(message, "role", "user") + content = getattr(message, "content", "") + return {"role": _role_name(role), "content": content if isinstance(content, str) else str(content)} + + +def serialize_messages(messages: object) -> list[dict[str, str]]: + """Serialize a ``model.generate`` message list to a ChatML ``messages`` column.""" + if not isinstance(messages, (list, tuple)): + return [] + return [serialize_message(message) for message in messages] + + +def serialize_token_usage(response: object) -> dict[str, int] | None: + """Extract ``{"input_tokens", "output_tokens"}`` from a response's ``token_usage``, if any.""" + usage: object = getattr(response, "token_usage", None) + if usage is None: + return None + input_tokens: object = getattr(usage, "input_tokens", None) + output_tokens: object = getattr(usage, "output_tokens", None) + if not isinstance(input_tokens, (int, float)) and not isinstance(output_tokens, (int, float)): + return None + return { + "input_tokens": int(input_tokens) if isinstance(input_tokens, (int, float)) else 0, + "output_tokens": int(output_tokens) if isinstance(output_tokens, (int, float)) else 0, + } + + +class DistillRecorder: + """Append-only NDJSON sink for distillation records; never raises into the run. + + ``record(purpose, messages, response, **meta)`` serializes the conversation, APPENDS the + assistant response as the final message (so each line is a complete ChatML training + example), and writes one JSON line. The parent directory is created lazily on first write. + ``call_index`` counts records written by THIS recorder (per invocation), letting downstream + filtering keep only each run's final (most complete) record. + """ + + def __init__(self, path: Path) -> None: + self.path: Path = Path(path) + self.call_index: int = 0 + + def record(self, purpose: str, messages: object, response: object = None, **meta: Any) -> None: + """Append one record; any serialization or I/O failure is logged and swallowed.""" + try: + conversation: list[dict[str, str]] = serialize_messages(messages) + if response is not None: + conversation.append(serialize_message(response)) + record: dict[str, object] = { + "messages": conversation, + "purpose": purpose, + "call_index": self.call_index, + "timestamp": datetime.now(UTC).isoformat(), + "token_usage": serialize_token_usage(response), + } + record.update(meta) + line: str = json.dumps(record, ensure_ascii=False, default=str) + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + self.call_index += 1 + except Exception as exc: # recording must never break a batch + logger.warning(f"distill: failed to record {purpose} call to {self.path}: {exc}") diff --git a/src/tablassert/extras.py b/src/tablassert/extras.py index cffa691..53c11d1 100644 --- a/src/tablassert/extras.py +++ b/src/tablassert/extras.py @@ -35,6 +35,7 @@ "qc": {"sklearn": "scikit-learn", "sentence_transformers": "sentence-transformers"}, "agent": {"smolagents": "smolagents", "litellm": "litellm"}, "optimize": {"dspy": "dspy"}, + "distill": {"datasets": "datasets"}, "log": {"loguru": "loguru"}, } @@ -48,6 +49,7 @@ "qc": "the QC audit", "agent": "the tablassert agent", "optimize": "GEPA prompt optimization (tablassert agent --optimize)", + "distill": "the distillation dataset export (tablassert distill-export)", "log": "file and progress logging", } diff --git a/tests/test_agent_cli.py b/tests/test_agent_cli.py index 39c73db..444ffec 100644 --- a/tests/test_agent_cli.py +++ b/tests/test_agent_cli.py @@ -260,6 +260,57 @@ def test_agent_optimize_flag_parses() -> None: assert bound.kwargs["optimize"] is True +def test_agent_distill_flag_parses() -> None: + """``--distill`` and both shorthands parse to distill=True without executing the body.""" + for flag in ("--distill", "-d", "-dt"): + fn, bound, _ = APP.parse_args(["agent", "PMC9", "--configuration-file", str(_graph_path()), flag], exit_on_error=False) + assert fn is agent + assert bound.kwargs["distill"] is True, flag + + +def test_agent_distill_forwards_a_recorder(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """``--distill`` builds a DistillRecorder under /distill and forwards it. + + Why: recording is off by default (``None`` reaches the supervisor untouched); opting in must + thread a recorder pointing at the state dir's ``distill/records.ndjson`` so the dataset + accumulates in one append-only file across invocations. + """ + _set_model_env(monkeypatch) + captured: dict[str, object] = {} + + def fake_run_supervisor(pmc_ids: list[str], **kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return {"records": {}, "metrics": {}} + + monkeypatch.setattr("tablassert.agent.run_supervisor", fake_run_supervisor) + + agent(["PMC1"], graph_configuration_file=_graph_path(), state_dir=tmp_path) + assert captured["distill_recorder"] is None # off by default + + agent(["PMC1"], graph_configuration_file=_graph_path(), state_dir=tmp_path, distill=True) + recorder: object = captured["distill_recorder"] + assert recorder is not None + assert recorder.path == tmp_path / "distill" / "records.ndjson" # pyright: ignore[reportAttributeAccessIssue] + assert "distilling LLM calls" in capsys.readouterr().out + + +def test_agent_distill_rejects_optimize(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + """``--distill --optimize`` exits 2: the GEPA path bypasses the recording seam. + + Why: ``--optimize`` returns before the supervisor runs and its dspy LM never routes through + the wrapped ``generate``, so the combination would silently record nothing. + """ + _set_model_env(monkeypatch) + monkeypatch.setattr("tablassert.agent.run_supervisor", lambda *a, **k: pytest.fail("supervisor must not run")) + monkeypatch.setattr("tablassert.agent.run_gepa", lambda *a, **k: pytest.fail("GEPA must not run")) + + with pytest.raises(SystemExit) as exc_info: + agent(["PMC1"], graph_configuration_file=_graph_path(), distill=True, optimize=True) + + assert exc_info.value.code == 2 + assert "--distill" in capsys.readouterr().err + + def test_agent_optimize_persists_instructions(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: """W6: ``--optimize`` runs GEPA (stubbed) and persists optimized instructions; the supervisor is NOT run.""" monkeypatch.setenv(ENV_MODEL_ID, "m") diff --git a/tests/test_agent_supervisor.py b/tests/test_agent_supervisor.py index 7777466..d57e727 100644 --- a/tests/test_agent_supervisor.py +++ b/tests/test_agent_supervisor.py @@ -19,8 +19,8 @@ import pytest import yaml -from tablassert import rs -from tablassert.agent import ConfigRecord, SupervisorState, load_state, make_fake_model, run_supervisor, save_state +from tablassert import distill, rs +from tablassert.agent import ConfigRecord, SupervisorState, distill_dir, load_state, make_fake_model, run_supervisor, save_state pytest.importorskip("smolagents") @@ -138,6 +138,42 @@ def test_supervisor_happy_path_mapped(tmp_path: Path, fullmap_db: Path, monkeypa assert reloaded.records["PMC1"].status == "MAPPED" +def test_supervisor_distill_records_every_generate_call(tmp_path: Path, fullmap_db: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """``distill_recorder`` end-to-end: every inner-agent ``generate`` lands in the NDJSON dataset. + + Why: the supervisor owns the per-article model construction, so it is the seam that tags each + record with its ``pmc_id`` — the field downstream filtering joins against ``state.json`` to keep + only MAPPED runs as training data. + """ + table: Path = _write_table(tmp_path, "good.tsv", "brca1\tmapk1\nbrca1\tmapk1\n") + _patch_fetch(monkeypatch, table) + good_yaml: str = yaml.safe_dump(_column_cfg(table), sort_keys=False) + state_dir: Path = tmp_path / "state" + recorder = distill.DistillRecorder(distill_dir(state_dir) / distill.RECORDS_FILENAME) + + result = run_supervisor( + ["PMC1"], + fullmap=fullmap_db, + build_model_factory=lambda: make_fake_model(final_yaml=good_yaml), + map_threshold=0.8, + state_dir=state_dir, + workdir=tmp_path / "w", + min_rows=0, + distill_recorder=recorder, + ) + + assert result["records"]["PMC1"].status == "MAPPED" # pyright: ignore[reportIndexIssue] + records: list[dict[str, object]] = [json.loads(line) for line in recorder.path.read_text(encoding="utf-8").splitlines() if line.strip()] + assert records, "the wrapped model must record at least one call" + for index, record in enumerate(records): + assert record["purpose"] == "agent" + assert record["pmc_id"] == "PMC1" + assert record["call_index"] == index + assert record["messages"][-1]["role"] == "assistant" # pyright: ignore[reportAttributeAccessIssue] + # The final (most complete) record's assistant turn carries the FakeModel's final-answer config. + assert "final_answer" in records[-1]["messages"][-1]["content"] # pyright: ignore[reportAttributeAccessIssue] + + def test_supervisor_improve_loop_accepts_better(tmp_path: Path, fullmap_db: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A first config below threshold is genuinely improved by propose_config_edit and accepted. diff --git a/tests/test_distill.py b/tests/test_distill.py new file mode 100644 index 0000000..2267191 --- /dev/null +++ b/tests/test_distill.py @@ -0,0 +1,218 @@ +"""``--distill`` capture: the ChatML NDJSON recorder, the generate-wrapping model, and export. + +The recorder tests are PURE Python (base env, no ``importorskip``): ``tablassert.distill`` is +zero-dependency by design and serializes duck-typed messages. The wrapping-model tests drive the +real ``make_distilling_model`` over ``make_fake_model`` and so skip cleanly without the ``[agent]`` +extra; the export test skips without ``[distill]``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tablassert import distill +from tablassert.agent import distill_dir, make_distilling_model, make_fake_model + + +class _Msg: + """ChatMessage-shaped duck type (``role``/``content``/optional ``token_usage``).""" + + def __init__(self, role: object, content: object, token_usage: object = None) -> None: + self.role = role + self.content = content + self.token_usage = token_usage + + +class _Role: + """Enum-shaped role (MessageRole carries a ``value``).""" + + def __init__(self, value: str) -> None: + self.value = value + + +class _Usage: + def __init__(self, input_tokens: int, output_tokens: int) -> None: + self.input_tokens = input_tokens + self.output_tokens = output_tokens + + +def _read_records(path: Path) -> list[dict[str, object]]: + """Parse an NDJSON file into a list of records, asserting strict one-object-per-line.""" + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def test_serialize_message_accepts_dicts_and_chatmessage_shapes() -> None: + """Both plain dicts and ChatMessage-shaped objects serialize to ChatML role/content. + + Why: the wrapper sees smolagents ``ChatMessage`` objects, but tests and defensive callers may + hand in dicts; both must land in the same ``{"role", "content"}`` shape Studio auto-detects. + """ + assert distill.serialize_message({"role": "user", "content": "hi"}) == {"role": "user", "content": "hi"} + assert distill.serialize_message(_Msg("assistant", "...")) == {"role": "assistant", "content": "..."} + + +def test_serialize_message_unwraps_enum_roles_and_stringifies_content() -> None: + """A ``MessageRole`` enum serializes to its plain value; non-string content is stringified. + + Why: ``str(MessageRole.USER)`` would leak ``"MessageRole.USER"`` into the training corpus, + and multimodal part-lists would serialize as Python reprs — both poison a chat template. + """ + record: dict[str, str] = distill.serialize_message(_Msg(_Role("system"), ["part-a", "part-b"])) + assert record["role"] == "system" + assert isinstance(record["content"], str) + + +def test_serialize_token_usage_extracts_counts_or_none() -> None: + """Token usage rides along when the response carries it, else the field is ``None``.""" + assert distill.serialize_token_usage(_Msg("assistant", "x", _Usage(10, 5))) == {"input_tokens": 10, "output_tokens": 5} + assert distill.serialize_token_usage(_Msg("assistant", "x")) is None + assert distill.serialize_token_usage(None) is None + + +def test_recorder_writes_complete_chatml_records(tmp_path: Path) -> None: + """Each record is one JSON line: full messages + appended assistant response + metadata. + + Why: Unsloth Studio auto-maps a top-level ``messages`` column on JSONL upload ONLY when every + line is a standalone object (no outer array), and the assistant reply must be the final + message for the line to be a complete SFT example. + """ + path: Path = tmp_path / "nested" / distill.RECORDS_FILENAME # parent created lazily + recorder = distill.DistillRecorder(path) + recorder.record( + "agent", + [_Msg(_Role("system"), "You derive configs."), _Msg("user", "Derive PMC1.")], + _Msg(_Role("assistant"), "final_answer(...)", _Usage(100, 20)), + pmc_id="PMC1", + model_id="big-model", + ) + + records = _read_records(path) + assert len(records) == 1 + record = records[0] + assert [m["role"] for m in record["messages"]] == ["system", "user", "assistant"] # pyright: ignore[reportAttributeAccessIssue] + assert record["messages"][-1]["content"] == "final_answer(...)" # pyright: ignore[reportAttributeAccessIssue] + assert record["purpose"] == "agent" + assert record["pmc_id"] == "PMC1" + assert record["model_id"] == "big-model" + assert record["call_index"] == 0 + assert record["token_usage"] == {"input_tokens": 100, "output_tokens": 20} + assert isinstance(record["timestamp"], str) + + +def test_recorder_appends_across_invocations(tmp_path: Path) -> None: + """A SECOND recorder over the same file appends — the dataset accumulates run over run. + + Why: the fine-tuning corpus is built by repeated ``tablassert agent --distill`` invocations; + each constructs a fresh ``DistillRecorder``, so append (never truncate) is the load-bearing + behavior. ``call_index`` restarts per invocation; ``timestamp`` disambiguates runs. + """ + path: Path = tmp_path / distill.RECORDS_FILENAME + first = distill.DistillRecorder(path) + first.record("agent", [_Msg("user", "run one")], _Msg("assistant", "a"), pmc_id="PMC1") + second = distill.DistillRecorder(path) # a later invocation + second.record("judge", [_Msg("user", "run two")], _Msg("assistant", "b")) + + records = _read_records(path) + assert len(records) == 2 + assert [r["purpose"] for r in records] == ["agent", "judge"] + assert [r["call_index"] for r in records] == [0, 0] # per-invocation counter + + +def test_recorder_without_response_still_records(tmp_path: Path) -> None: + """A call with no response object records the prompt messages alone (best-effort).""" + path: Path = tmp_path / distill.RECORDS_FILENAME + distill.DistillRecorder(path).record("reflexion", [_Msg("user", "propose an edit")]) + (record,) = _read_records(path) + assert [m["role"] for m in record["messages"]] == ["user"] # pyright: ignore[reportAttributeAccessIssue] + assert record["token_usage"] is None + + +def test_recorder_never_raises_into_the_run(tmp_path: Path) -> None: + """An unwritable sink (path IS a directory) logs and swallows instead of breaking a batch. + + Why: recording is observability, not pipeline logic — a distillation failure must never skip + an article mid-supervisor-run. + """ + recorder = distill.DistillRecorder(tmp_path) # opening a directory for append fails + recorder.record("agent", [_Msg("user", "x")], _Msg("assistant", "y")) # must not raise + + +def test_distill_dir_is_a_pure_path_helper() -> None: + """``distill_dir(root)`` is ``root/distill`` with no I/O, matching the sibling helpers.""" + root: Path = Path(".tablassert") / "agent" + assert distill_dir(root) == root / "distill" + assert not distill_dir(root).exists() # pure: nothing created + + +def test_distilling_model_records_every_generate_call(tmp_path: Path) -> None: + """The wrapper delegates to the real model and appends one tagged record per call. + + Why: wrapping ``generate`` is the single capture seam — the inner agent, judge, and reflexion + all route through it. The wrapped model's ``model_id`` is recovered for the record, and the + response passes through untouched (the run must behave exactly as if unwrapped). + """ + pytest.importorskip("smolagents") + recorder = distill.DistillRecorder(tmp_path / distill.RECORDS_FILENAME) + wrapped = make_distilling_model(make_fake_model(), recorder, purpose="agent", meta={"pmc_id": "PMC7"}) + + response = wrapped.generate([_Msg(_Role("user"), "derive a config")]) # pyright: ignore[reportAttributeAccessIssue] + assert "final_answer" in str(getattr(response, "content", "")) # the fake's answer passes through + wrapped.generate([_Msg(_Role("user"), "second call")]) # pyright: ignore[reportAttributeAccessIssue] + + records = _read_records(recorder.path) + assert len(records) == 2 + for index, record in enumerate(records): + assert record["purpose"] == "agent" + assert record["pmc_id"] == "PMC7" + assert record["call_index"] == index + assert record["messages"][-1]["role"] == "assistant" # pyright: ignore[reportAttributeAccessIssue] + assert "final_answer" in records[0]["messages"][-1]["content"] # pyright: ignore[reportAttributeAccessIssue] + + +def test_distilling_model_survives_a_raising_recorder(tmp_path: Path) -> None: + """A recorder that raises does not change the wrapped model's behavior.""" + + class ExplodingRecorder: + def record(self, *args: object, **kwargs: object) -> None: + raise OSError("disk full") + + pytest.importorskip("smolagents") + wrapped = make_distilling_model(make_fake_model(), ExplodingRecorder(), purpose="agent") + response = wrapped.generate([_Msg("user", "still works")]) # pyright: ignore[reportAttributeAccessIssue] + assert "final_answer" in str(getattr(response, "content", "")) + + +def test_distill_export_writes_an_hf_dataset(tmp_path: Path) -> None: + """``distill-export`` turns recorded NDJSON into a ``save_to_disk`` dataset. + + Why: the raw NDJSON already loads in Studio; this command exists for ``datasets``-native + workflows, and its output must round-trip through ``load_from_disk``. + """ + pytest.importorskip("datasets") + from datasets import load_from_disk # pyright: ignore[reportMissingImports] + + from tablassert.cli import distill_export + + ndjson_dir: Path = tmp_path / "distill" + recorder = distill.DistillRecorder(ndjson_dir / distill.RECORDS_FILENAME) + recorder.record("agent", [_Msg("user", "derive"), _Msg("assistant", "thinking")], _Msg("assistant", "done"), pmc_id="PMC1") + out: Path = tmp_path / "hf-dataset" + + distill_export(distill_dir=ndjson_dir, out=out) + + dataset = load_from_disk(str(out)) + assert len(dataset) == 1 + assert dataset[0]["purpose"] == "agent" + assert dataset[0]["messages"][-1]["content"] == "done" + + +def test_distill_export_fails_loud_on_an_empty_dir(tmp_path: Path) -> None: + """No recorded NDJSON means exit 2 naming the fix — not a cryptic datasets error.""" + from tablassert.cli import distill_export + + with pytest.raises(SystemExit) as exc_info: + distill_export(distill_dir=tmp_path / "empty", out=tmp_path / "out") + assert exc_info.value.code == 2 diff --git a/tests/test_docs_cli_coverage.py b/tests/test_docs_cli_coverage.py index b92e5fe..758b57c 100644 --- a/tests/test_docs_cli_coverage.py +++ b/tests/test_docs_cli_coverage.py @@ -25,6 +25,7 @@ "agent": ("agent.md",), "build-fullmap": ("cli.md", "fullmap.md"), "build-kg": ("cli.md",), + "distill-export": ("cli.md",), "validate": ("cli.md",), "validate-kgx": ("cli.md",), } @@ -32,6 +33,7 @@ "agent": ("agent.md", "cli.md"), "build-fullmap": ("cli.md", "fullmap.md"), "build-kg": ("cli.md",), + "distill-export": ("cli.md",), "validate": ("cli.md",), "validate-kgx": ("cli.md",), } diff --git a/uv.lock b/uv.lock index ca434a6..665d215 100644 --- a/uv.lock +++ b/uv.lock @@ -2,12 +2,30 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.13' and platform_machine != 's390x'", - "python_full_version >= '3.13' and platform_machine == 's390x'", - "python_full_version == '3.12.*' and platform_machine != 's390x'", - "python_full_version == '3.12.*' and platform_machine == 's390x'", - "python_full_version < '3.12' and platform_machine != 's390x'", - "python_full_version < '3.12' and platform_machine == 's390x'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'win32'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.12' and platform_machine != 's390x' and sys_platform == 'win32'", + "python_full_version < '3.12' and platform_machine != 's390x' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and platform_machine != 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.12' and platform_machine == 's390x' and sys_platform == 'win32'", + "python_full_version < '3.12' and platform_machine == 's390x' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] @@ -647,6 +665,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/18/4cedda786e7da429e7489549a9e5461530d4133130e541f25fb94f015776/cyclopts-4.11.2-py3-none-any.whl", hash = "sha256:838020120b939549ff7c8423aca29c86764b5dd1d8a5d7f3753a6327861f537b", size = 213537, upload-time = "2026-05-04T00:11:56.103Z" }, ] +[[package]] +name = "datasets" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, + { name = "filelock" }, + { name = "fsspec", extra = ["http"] }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/5b/836516269d4f618efe621661cfb6f9acc57e6f95265db3efaee48a5ffe04/datasets-5.0.1.tar.gz", hash = "sha256:ce22bb851efd7494f08aad33b940803784434f6e77763d00679a0dc45fcf686a", size = 641498, upload-time = "2026-07-28T11:09:12.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/0b/98fc6eb83333508ca5f44c52b3e287ea8137a0ad582714e2cbc67a02154b/datasets-5.0.1-py3-none-any.whl", hash = "sha256:9fbf73688f8c18f7529b4fe592abd04015f81d1e58001e4bac73ffb2b39d7cc4", size = 559079, upload-time = "2026-07-28T11:09:10.266Z" }, +] + [[package]] name = "deprecated" version = "1.3.1" @@ -659,6 +702,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "dill" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, +] + [[package]] name = "diskcache" version = "5.6.3" @@ -949,6 +1001,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, +] + [[package]] name = "gepa" version = "0.0.27" @@ -1670,6 +1727,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] +[[package]] +name = "multiprocess" +version = "0.70.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/f2/e783ac7f2aeeed14e9e12801f22529cc7e6b7ab80928d6dcce4e9f00922d/multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897", size = 2079989, upload-time = "2026-01-19T06:47:39.744Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/aa/714635c727dbfc251139226fa4eaf1b07f00dc12d9cd2eb25f931adaf873/multiprocess-0.70.19-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1bbf1b69af1cf64cd05f65337d9215b88079ec819cd0ea7bac4dab84e162efe7", size = 144743, upload-time = "2026-01-19T06:47:24.562Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/155f6abf5e6b5d9cef29b6d0167c180846157a4aca9b9bee1a217f67c959/multiprocess-0.70.19-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5be9ec7f0c1c49a4f4a6fd20d5dda4aeabc2d39a50f4ad53720f1cd02b3a7c2e", size = 144738, upload-time = "2026-01-19T06:47:26.636Z" }, + { url = "https://files.pythonhosted.org/packages/af/cb/f421c2869d75750a4f32301cc20c4b63fab6376e9a75c8e5e655bdeb3d9b/multiprocess-0.70.19-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1c3dce098845a0db43b32a0b76a228ca059a668071cfeaa0f40c36c0b1585d45", size = 144741, upload-time = "2026-01-19T06:47:27.985Z" }, + { url = "https://files.pythonhosted.org/packages/e3/45/8004d1e6b9185c1a444d6b55ac5682acf9d98035e54386d967366035a03a/multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87", size = 134948, upload-time = "2026-01-19T06:47:32.325Z" }, + { url = "https://files.pythonhosted.org/packages/86/c2/dec9722dc3474c164a0b6bcd9a7ed7da542c98af8cabce05374abab35edd/multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c", size = 144457, upload-time = "2026-01-19T06:47:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/71/70/38998b950a97ea279e6bd657575d22d1a2047256caf707d9a10fbce4f065/multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28", size = 150281, upload-time = "2026-01-19T06:47:35.037Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/d2c27e03cb84251dfe7249b8e82923643c6d48fa4883b9476b025e7dc7eb/multiprocess-0.70.19-py313-none-any.whl", hash = "sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952", size = 156414, upload-time = "2026-01-19T06:47:35.915Z" }, + { url = "https://files.pythonhosted.org/packages/a0/61/af9115673a5870fd885247e2f1b68c4f1197737da315b520a91c757a861a/multiprocess-0.70.19-py314-none-any.whl", hash = "sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f", size = 160318, upload-time = "2026-01-19T06:47:37.497Z" }, + { url = "https://files.pythonhosted.org/packages/7e/82/69e539c4c2027f1e1697e09aaa2449243085a0edf81ae2c6341e84d769b6/multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5", size = 133477, upload-time = "2026-01-19T06:47:38.619Z" }, +] + [[package]] name = "networkx" version = "3.6.1" @@ -2033,6 +2110,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, ] +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/ef/f1fd7431d635bf20015489bf0bd69c17fff1018de773540f651455a3916b/pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282", size = 10397178, upload-time = "2026-07-22T22:17:48.274Z" }, + { url = "https://files.pythonhosted.org/packages/31/b4/0eafac990a431561187694126de01f9b12559549b4d86360c0c4bd870fde/pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298", size = 9990736, upload-time = "2026-07-22T22:17:52.388Z" }, + { url = "https://files.pythonhosted.org/packages/de/21/359880af3ea9b7cb23bea5b51e8e70ef3866c03be09da9a2787e18e330a8/pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899", size = 10814438, upload-time = "2026-07-22T22:17:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/d1/50/d6cc4d7e508bbccf5d6027314a8312bc7ac73d0ec7f195f53838daafab40/pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a", size = 11323634, upload-time = "2026-07-22T22:17:56.858Z" }, + { url = "https://files.pythonhosted.org/packages/70/2b/d5f0a8c90dd0ae04e64ba53b871afb796ec026b615086d382ddc2ade729b/pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928", size = 11850860, upload-time = "2026-07-22T22:17:59.1Z" }, + { url = "https://files.pythonhosted.org/packages/5c/30/183aec2e19adf778a98d29b5729a0a68f4cc4ebf9b9c3b70d0297355bcb1/pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d", size = 12411100, upload-time = "2026-07-22T22:18:01.485Z" }, + { url = "https://files.pythonhosted.org/packages/fa/9a/31f4983f191af51ab2a8f2d0c7b33dff3a84da26533f982fff02c2f9e28b/pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a", size = 9968804, upload-time = "2026-07-22T22:18:03.903Z" }, + { url = "https://files.pythonhosted.org/packages/49/97/7886c89a39045c69ad82cbceaf3343810480c8ef49a216319ce8183860a6/pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a", size = 9205447, upload-time = "2026-07-22T22:18:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + [[package]] name = "pathspec" version = "1.1.1" @@ -2349,6 +2480,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "pyarrow" +version = "25.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/e3/27f57f80141379d60defe6703eb50a707325706f07fedfd1312c7a751995/pyarrow-25.0.1.tar.gz", hash = "sha256:9150a83248bfed9813ea3c3af74c3856c1984d444aa28e58bf7733b9750ddf6a", size = 1201653, upload-time = "2026-08-10T12:40:53.904Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/8b/0d23b47702fcfe8b3618d5292035099675c5a1c48258932350c08020f7b5/pyarrow-25.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:51093dd9e10325fbdb3c10a2ae7c4806e5c822d94e74ae4938b26524a3323fee", size = 35946180, upload-time = "2026-08-10T12:37:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/707d17a5476c55a9541fde0db8213ac30979a792864d72415f176ba50c45/pyarrow-25.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:eb6203482ff3746a5632303a7279ae0b5a304c46985b49ed1378cb350ea6728d", size = 37644787, upload-time = "2026-08-10T12:37:25.795Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b2/cdc98ecf1a6408280bc3a6a07054cdd99a3f4670acc0545d383ce113e87d/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:880523be3d29efcf83d3998835d206118ccf35e3871dbd2fb60408cf6b007a80", size = 46834633, upload-time = "2026-08-10T12:37:33.604Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6e/d3fafc41f378b2c65be43b827798c0fae42049a641c8526633ed3eb573e2/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:25f8720bf6387d5dc2ebd2622112de630760419e4b66134405dd24110d15f37e", size = 50065507, upload-time = "2026-08-10T12:37:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/d5/12/8d0698954b8c3001844a898e0a6900bebe83d7ee40c11195174c5122f324/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4facd65742a024a4a366328a1d2292062d72d6e023c1b7dda8d4c37544933a25", size = 49955690, upload-time = "2026-08-10T12:37:46.644Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0b/1ecb936ac6409e90a34d58eea1c7cec09a9ae6d2141b9e49ad01a2b1ea47/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa0559502e1cd6254d6814614085dd9c5a3dd0419362978a936a3f68a9e5c3df", size = 53128198, upload-time = "2026-08-10T12:37:52.531Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/5236033550633c9b7377b2a53660b2bbb06cb06dc09c4356332d67643ca1/pyarrow-25.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:62cd0d785b8aa6675ee355f9fc02252a340f4441257c42674937826fd7594325", size = 27857263, upload-time = "2026-08-10T12:37:56.943Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e2/9ab15b88cbfac28e16419ce5439ec29234c5172cb8259301b4ba639bdec0/pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:df961f2e7ae9cf496459259d798652c70625f6c080650d6952f8c04053c58ee9", size = 35861559, upload-time = "2026-08-10T12:38:02.567Z" }, + { url = "https://files.pythonhosted.org/packages/58/79/a0036dbe1eabe1f73127427342f1d99982584c4a2cde2651d6c93499c6f6/pyarrow-25.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:cc4aa407fde9fc660be3939e49ea31f50f3e9fec17c0ec63159f7711edd3efc9", size = 37628383, upload-time = "2026-08-10T12:38:09.083Z" }, + { url = "https://files.pythonhosted.org/packages/13/49/d93a57d375f4bf0cf82913dd6bb54acafde83dd993be2282c81ac5616cad/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:4340f0ba6c1d2e13f21658de1d7c662ca2545018568d0030a1e9afca159d87e3", size = 46820190, upload-time = "2026-08-10T12:38:15.458Z" }, + { url = "https://files.pythonhosted.org/packages/60/c9/711ca85d79f1ec98f29a5eae2b051e25b4ecec5de3e3c0e2d5c5dcb15664/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5389cdf79447ed1515c9e31620e6e1e2302249564d603f2ad727d4f6d313e4c3", size = 50102437, upload-time = "2026-08-10T12:38:22.487Z" }, + { url = "https://files.pythonhosted.org/packages/80/53/8fb8359ff17cfb6263a1cf3ebf7caec9fe197de118719e84fcb1d0618026/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d51592cb7561e87877c506113e7adbf1342ab579e6c21f0ef44b8ba41cb74c80", size = 49942424, upload-time = "2026-08-10T12:38:28.755Z" }, + { url = "https://files.pythonhosted.org/packages/e8/83/4e5ae02a9341571b18a6fca380ac7a58ce6ddae7ab3c060208c0a1e79f02/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6109c94d8b9f3b17a041daca16cacb2f651ad8f1ef70a4232c2c0f37a23da2a8", size = 53144206, upload-time = "2026-08-10T12:38:34.862Z" }, + { url = "https://files.pythonhosted.org/packages/65/ee/197cbf47e49f83e6ebeb946a5259a48a638dea27ac774db42fe78022179d/pyarrow-25.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:8858d7bfc22e3f51529aeaa4077225029724623e4595dc9eff8c793935c34140", size = 27953934, upload-time = "2026-08-10T12:38:39.808Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8d/8f271a7a034c834910ec925d56fa4b29733b1380f5289419f5aaa3b02777/pyarrow-25.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c7c534ec03c358a76ea3e505e74c1b6aef290af90c444dfd092dbfe23e755b85", size = 35855328, upload-time = "2026-08-10T12:38:45.489Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cd/5bac242f4e841b9971d5eb94fdfe2577e2b70be983e27401e72055786037/pyarrow-25.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:dda9470024204d7bbf2042b47c6e8a0e47a3eeb8e34405882dfaea6577e0c153", size = 37622415, upload-time = "2026-08-10T12:38:51.107Z" }, + { url = "https://files.pythonhosted.org/packages/63/1f/96d03b4e1506524f7087adb0fd6b2f69f0c9c7aaff1ec36d8030082e15a5/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:44a9120ce5bd81936b8ab9a88076e3fd47c2c6838e0e43630fed83626aca81d9", size = 46813813, upload-time = "2026-08-10T12:38:57.773Z" }, + { url = "https://files.pythonhosted.org/packages/98/d6/33a411115b61dbfc16ad6ad73e71730f6fea654ee3667673bc53ab0e2fe7/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0befcf816e45a1af33ac775a9970b749e4868a230c7372f0ae5e932bee27039f", size = 50104452, upload-time = "2026-08-10T12:39:04.579Z" }, + { url = "https://files.pythonhosted.org/packages/33/ae/b1b97c9ca87f9f9ddbb5230c798df94eccce61bd79b9b45458c69a478588/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f89685964f46e4216103c75483aac0c0692a5f72212d7ca835adba5ede56ce3", size = 49951343, upload-time = "2026-08-10T12:39:11.8Z" }, + { url = "https://files.pythonhosted.org/packages/98/9e/a112df5cfd5a68cb1d9fc31cfe38c28d5aec9f10865ce37ecef2e4450873/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6943e2fe7954d29d84de45d29d34c8dc36ce96570e67d89aa9976e650a4a9138", size = 53144784, upload-time = "2026-08-10T12:39:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/31/24/97e8bd98f1e3b07e2ba08bcdff690674fbe16d69a7d2712cc3884665e615/pyarrow-25.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:31e49a7888fcdf3a835da33ae777f6bb9a866334e5a789282fc26dcf426f7f15", size = 27870159, upload-time = "2026-08-10T12:39:26.161Z" }, + { url = "https://files.pythonhosted.org/packages/36/4c/b525824ad3094076919273cd97db61fb3d78252dee76fa3b8dc8f76774aa/pyarrow-25.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:bf0b672390cdcb640d7288f96b826d71ff4e9abb254a86c89890baf51a29cee6", size = 35885255, upload-time = "2026-08-10T12:39:32.366Z" }, + { url = "https://files.pythonhosted.org/packages/08/62/448bb0e940de41aec31d1a956e63ad9c54afdf122a103cc3ab20c2a3ce33/pyarrow-25.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:38a9a4b4b9613380e200641891495a56c3d5a98a092db4a870af9975e220471d", size = 37644461, upload-time = "2026-08-10T12:39:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/6e/9a/13587e38bd4806fd218f50fd13b8903fab60588a699ff0c406372e5b4043/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b726ad7e7b669be982b0c71c07fe4b037d654354130da79a7902a669e93a66b", size = 46877146, upload-time = "2026-08-10T12:39:43.722Z" }, + { url = "https://files.pythonhosted.org/packages/8d/61/1c5d1229fa21da4cff5365e41e57177aaac57c563c727f35419b8513d1c1/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:9171748cdf796972d85a4b60157c279913e242992e350c90c7450182a9838b2a", size = 50131616, upload-time = "2026-08-10T12:39:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/43/20/291e1d65cc0b09aa19f03cf25cf51a2f5fa94b5db315178f2d254ed5cad4/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b7a296aac7a71fa0886c08e155ddb6c636a50013f801f6178daafa0f9e726188", size = 50008879, upload-time = "2026-08-10T12:39:56.891Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7c/1b7c9ec28e76576337e4f97b31141c9a181b89b6d1d6221e9d8205621a58/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0fe7c8b6c03969b49c8c66182e4a18e3819ab92d07cfab5d8370c531b9369ef0", size = 53170864, upload-time = "2026-08-10T12:40:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/75/f3d789dc06011a765d14d86bda799cf72ac1d715b6a6edecaa0d73d95062/pyarrow-25.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:f729cfdbd36fd99d543b67a914d2de044c84ebe45be8b34902b299b608c15c8f", size = 28620729, upload-time = "2026-08-10T12:40:51.41Z" }, + { url = "https://files.pythonhosted.org/packages/fc/05/647a8ee6f7c2662feb6921315617bc04dcd6034763fb61b1199720bf6162/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:59a2de54c0cbd954da861eee4d1d330f8e909c45b53455baef696380f2c55033", size = 36130288, upload-time = "2026-08-10T12:40:11.014Z" }, + { url = "https://files.pythonhosted.org/packages/93/f8/c9ee997554d7bea94520667dd1933f109ac1da3ee3556d2b49381e023484/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:35935cd5de130aa5cf4dea052a63e6bf2e17006c35c3a468194242b9b2bf5956", size = 37762187, upload-time = "2026-08-10T12:40:16.592Z" }, + { url = "https://files.pythonhosted.org/packages/a2/08/a28c01c7fe9e96e8233ce2d13df1d402f4f999f848f51d2daacd6bb4c036/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f3831aaa25c67a99f99dc8b05873cb9d64560390372e2aa197ce9dd4a3f06a44", size = 46888003, upload-time = "2026-08-10T12:40:23.242Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b9/58612e977d28dc58c878448866838369ee8da2f1e7cc8ed2c84b952aafee/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6a1fdfc6659b6b19022f2e50627fb5cf7156a66c46bf4299379955cbe742382a", size = 50079036, upload-time = "2026-08-10T12:40:29.169Z" }, + { url = "https://files.pythonhosted.org/packages/72/13/66e1402dcc860e1dc2760b1e0292c9a569b62b3bccab69def1b3e907d006/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:169d3429d5be7c752125890620f75a60776d38b0035eddae939651640822332e", size = 50040226, upload-time = "2026-08-10T12:40:35.186Z" }, + { url = "https://files.pythonhosted.org/packages/78/10/3f1a5497a7ef732ab0f03ecca3e66d89d9c0f57fdc61b4794c456b781f01/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:119297a6dc197e45d9c6d4415f7814a67ffa36c180d26f68c154c58067ae782d", size = 53149035, upload-time = "2026-08-10T12:40:41.454Z" }, + { url = "https://files.pythonhosted.org/packages/93/c0/37d4a7e8e2f7a6076283673d5298018ca26478b934c6ee369e10505ab32c/pyarrow-25.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4288f27577352d608ca08553b0865e4a9b3aa14820c5d95b53337218d609835b", size = 28753071, upload-time = "2026-08-10T12:40:46.623Z" }, +] + [[package]] name = "pydantic" version = "2.13.3" @@ -3316,6 +3490,9 @@ agent = [ aria2 = [ { name = "aria2" }, ] +distill = [ + { name = "datasets" }, +] log = [ { name = "loguru" }, ] @@ -3371,6 +3548,7 @@ requires-dist = [ { name = "aria2", marker = "extra == 'aria2'", specifier = "==0.0.1b0" }, { name = "biolink-model", specifier = ">=4.4.4" }, { name = "cyclopts", specifier = ">=1.0.0" }, + { name = "datasets", marker = "extra == 'distill'", specifier = ">=3.0.0" }, { name = "dspy", marker = "extra == 'optimize'", specifier = ">=3.2.1" }, { name = "fastexcel", specifier = ">=0.20.2" }, { name = "litellm", marker = "extra == 'agent'", specifier = ">=1.93.0" }, @@ -3385,7 +3563,7 @@ requires-dist = [ { name = "sentence-transformers", marker = "extra == 'qc'", specifier = ">=5.3.0" }, { name = "smolagents", marker = "extra == 'agent'", specifier = ">=1.26.0" }, ] -provides-extras = ["rt", "aria2", "qc", "agent", "optimize", "log"] +provides-extras = ["rt", "aria2", "qc", "agent", "optimize", "distill", "log"] [package.metadata.requires-dev] ci = [ @@ -3705,6 +3883,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +] + [[package]] name = "urllib3" version = "2.6.3"