diff --git a/.claude/commands/bump-version.md b/.claude/commands/bump-version.md index 47213c73..d053075f 100644 --- a/.claude/commands/bump-version.md +++ b/.claude/commands/bump-version.md @@ -24,7 +24,7 @@ Files that need updating: | `pyproject.toml` | `version = "X.Y.Z"` | ~7 | | `rust/Cargo.toml` | `version = "X.Y.Z"` | ~3 | | `CHANGELOG.md` | Section header + comparison link | Top + bottom | -| `docs/llms-full.txt` | `- Version: X.Y.Z` | ~5 | +| `diff_diff/guides/llms-full.txt` | `- Version: X.Y.Z` | ~5 | ## Instructions @@ -80,7 +80,7 @@ Files that need updating: Replace `version = "OLD_VERSION"` (the first version line under [package]) with `version = "NEW_VERSION"` Note: Rust version may differ from Python version; always sync to the new version - - `docs/llms-full.txt`: + - `diff_diff/guides/llms-full.txt`: Replace `- Version: OLD_VERSION` with `- Version: NEW_VERSION` 6. **Update CHANGELOG comparison links**: @@ -101,7 +101,7 @@ Files that need updating: - diff_diff/__init__.py: __version__ = "NEW_VERSION" - pyproject.toml: version = "NEW_VERSION" - rust/Cargo.toml: version = "NEW_VERSION" - - docs/llms-full.txt: Version: NEW_VERSION + - diff_diff/guides/llms-full.txt: Version: NEW_VERSION - CHANGELOG.md: Added/verified [NEW_VERSION] entry Next steps: diff --git a/CLAUDE.md b/CLAUDE.md index d7bf561d..ec5f9b83 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -139,7 +139,7 @@ category (`Methodology/Correctness`, `Performance`, or `Testing/Docs`): | `CONTRIBUTING.md` | Documentation requirements, test writing guidelines | | `.claude/commands/dev-checklists.md` | Checklists for params, methodology, warnings, reviews, bugs (run `/dev-checklists`) | | `.claude/memory.md` | Debugging patterns, tolerances, API conventions (git-tracked) | -| `docs/llms-practitioner.txt` | Baker et al. (2025) 8-step practitioner workflow for AI agents | +| `diff_diff/guides/llms-practitioner.txt` | Baker et al. (2025) 8-step practitioner workflow for AI agents (accessible at runtime via `diff_diff.get_llm_guide("practitioner")`) | | `docs/performance-plan.md` | Performance optimization details | | `docs/benchmarks.rst` | Validation results vs R | diff --git a/README.md b/README.md index 724a07e4..0095bf0e 100644 --- a/README.md +++ b/README.md @@ -69,11 +69,19 @@ Signif. codes: '***' 0.001, '**' 0.01, '*' 0.05, '.' 0.1 ## For AI Agents -If you are an AI agent or LLM using this library, read [`docs/llms.txt`](docs/llms.txt) for a concise API reference with an 8-step practitioner workflow (based on Baker et al. 2025). The workflow ensures rigorous DiD analysis — not just calling `fit()`, but testing assumptions, running sensitivity analysis, and checking robustness. +If you are an AI agent or LLM using this library, call `diff_diff.get_llm_guide()` for a concise API reference with an 8-step practitioner workflow (based on Baker et al. 2025). The workflow ensures rigorous DiD analysis — not just calling `fit()`, but testing assumptions, running sensitivity analysis, and checking robustness. -After estimation, call `practitioner_next_steps(results)` for context-aware guidance on remaining diagnostic steps. +```python +from diff_diff import get_llm_guide + +get_llm_guide() # concise API reference +get_llm_guide("practitioner") # 8-step workflow (Baker et al. 2025) +get_llm_guide("full") # comprehensive documentation +``` + +The guides are bundled in the wheel, so they are accessible from a `pip install` with no network access required. -Detailed guide: [`docs/llms-practitioner.txt`](docs/llms-practitioner.txt) +After estimation, call `practitioner_next_steps(results)` for context-aware guidance on remaining diagnostic steps. ## For Data Scientists diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 339b5dc5..bc1b88de 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -4,12 +4,14 @@ This library provides sklearn-like estimators for causal inference using the difference-in-differences methodology. -For rigorous analysis, follow the 8-step practitioner workflow in -docs/llms-practitioner.txt (based on Baker et al. 2025). After -estimation, call ``practitioner_next_steps(results)`` for context-aware -guidance on remaining diagnostic steps. +For rigorous analysis, follow the 8-step practitioner workflow based +on Baker et al. (2025). After estimation, call +``practitioner_next_steps(results)`` for context-aware guidance on +remaining diagnostic steps. -AI agent reference: docs/llms.txt +AI agents: call ``diff_diff.get_llm_guide()`` for a complete API reference. +Use ``get_llm_guide("practitioner")`` for the 8-step workflow or +``get_llm_guide("full")`` for comprehensive documentation. """ # Import backend detection from dedicated module (avoids circular imports) @@ -200,6 +202,7 @@ plot_synth_weights, ) from diff_diff.practitioner import practitioner_next_steps +from diff_diff._guides_api import get_llm_guide from diff_diff.datasets import ( clear_cache, list_datasets, @@ -402,4 +405,6 @@ "clear_cache", # Practitioner guidance "practitioner_next_steps", + # LLM guide accessor + "get_llm_guide", ] diff --git a/diff_diff/_guides_api.py b/diff_diff/_guides_api.py new file mode 100644 index 00000000..5a00ed77 --- /dev/null +++ b/diff_diff/_guides_api.py @@ -0,0 +1,48 @@ +"""Runtime accessor for bundled LLM guide files.""" +from __future__ import annotations + +from importlib.resources import files + +_VARIANT_TO_FILE = { + "concise": "llms.txt", + "full": "llms-full.txt", + "practitioner": "llms-practitioner.txt", +} + + +def get_llm_guide(variant: str = "concise") -> str: + """Return the contents of a bundled LLM guide. + + Parameters + ---------- + variant : str, default "concise" + Which guide to load. Names are case-sensitive. One of: + + - ``"concise"`` -- compact API reference (llms.txt) + - ``"full"`` -- complete API documentation (llms-full.txt) + - ``"practitioner"`` -- 8-step practitioner workflow (llms-practitioner.txt) + + Returns + ------- + str + The full text of the requested guide. + + Raises + ------ + ValueError + If ``variant`` is not one of the known guide names. + + Examples + -------- + >>> from diff_diff import get_llm_guide + >>> concise = get_llm_guide() + >>> workflow = get_llm_guide("practitioner") + """ + try: + filename = _VARIANT_TO_FILE[variant] + except (KeyError, TypeError): + valid = ", ".join(repr(k) for k in _VARIANT_TO_FILE) + raise ValueError( + f"Unknown guide variant {variant!r}. Valid options: {valid}." + ) from None + return files("diff_diff.guides").joinpath(filename).read_text(encoding="utf-8") diff --git a/diff_diff/guides/__init__.py b/diff_diff/guides/__init__.py new file mode 100644 index 00000000..b7bb9954 --- /dev/null +++ b/diff_diff/guides/__init__.py @@ -0,0 +1 @@ +"""LLM guide files bundled with diff-diff.""" diff --git a/docs/llms-full.txt b/diff_diff/guides/llms-full.txt similarity index 98% rename from docs/llms-full.txt rename to diff_diff/guides/llms-full.txt index 6569316c..1b794d66 100644 --- a/docs/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -33,7 +33,7 @@ print(f"ATT: {results.att:.3f} (SE: {results.se:.3f})") ## Practitioner Workflow (based on Baker et al. 2025) -For rigorous DiD analysis, follow the 8-step framework in docs/llms-practitioner.txt. +For rigorous DiD analysis, follow the 8-step framework (call `diff_diff.get_llm_guide("practitioner")`). After estimation, call: ```python @@ -1029,6 +1029,12 @@ Returned by `SyntheticDiD.fit()`. **Methods:** `summary()`, `print_summary()`, `to_dict()`, `to_dataframe()`, `get_unit_weights_df()`, `get_time_weights_df()` +**Validation diagnostics** (call after `fit()`): +- `get_weight_concentration(top_k=5)` - effective N and top-k weight share; flags fragile synthetic controls dominated by a few donor units +- `get_loo_effects_df()` - per-unit leave-one-out influence from the jackknife pass (DataFrame includes both control and treated rows). Requires `variance_method="jackknife"`; raises `ValueError` if LOO is unavailable (see the method docstring for the full set of conditions, e.g. single treated unit or only one control with nonzero effective weight) +- `in_time_placebo()` - re-estimate on shifted fake treatment dates in the pre-period; near-zero placebo ATTs indicate a credible design +- `sensitivity_to_zeta_omega()` - re-estimate across a grid of unit-weight regularization values; checks ATT robustness to the auto-selected zeta_omega + ### TripleDifferenceResults Returned by `TripleDifference.fit()`. diff --git a/docs/llms-practitioner.txt b/diff_diff/guides/llms-practitioner.txt similarity index 93% rename from docs/llms-practitioner.txt rename to diff_diff/guides/llms-practitioner.txt index 194b68f7..8d487fcc 100644 --- a/docs/llms-practitioner.txt +++ b/diff_diff/guides/llms-practitioner.txt @@ -3,7 +3,21 @@ > An 8-step workflow for rigorous Difference-in-Differences analysis, based on > Baker et al. (2025) "Difference-in-Differences Designs: A Practitioner's > Guide" and adapted for the diff-diff library. Some steps are reorganized or -> extended relative to the paper (see docs/methodology/REGISTRY.md for details). +> extended relative to the paper: +> +> - **Numbering**: diff-diff uses 1-Define, 2-Assumptions, 3-Test PT, +> 4-Choose estimator, 5-Estimate, 6-Sensitivity, 7-Heterogeneity, +> 8-Robustness. The paper uses 1-Define, 2-Assumptions, 3-Estimation method, +> 4-Uncertainty, 5-Estimate, 6-Sensitivity, 7-Heterogeneity, 8-Keep learning. +> - **Parallel trends testing** is a separate Step 3 (the paper embeds it in +> Step 2), to ensure AI agents execute it as a distinct action. +> - **Sources of uncertainty** (paper's Step 4) are folded into Step 5 +> (Estimate) with an explicit cluster-count check directive: >= 50 clusters +> for asymptotic SEs, otherwise wild bootstrap. The 50-cluster threshold is +> a diff-diff convention. +> - **Step 8** is "Robustness & Reporting" (compare estimators, report with +> and without covariates). The paper's Step 8 is "Keep learning." The +> mandatory with/without covariate comparison is a diff-diff convention. ## Instructions for AI Agents @@ -282,7 +296,8 @@ to your estimator's API. Examples: - **StackedDiD**: vary `clean_control` definition - **EfficientDiD**: compare `control_group='never_treated'` vs `'last_cohort'` - **ImputationDiD/TwoStageDiD**: leave-one-cohort-out, cross-estimator comparison -- **SyntheticDiD/TROP**: in-time or in-space placebo (fake treatment date, leave-one-unit-out) +- **SyntheticDiD**: built-in diagnostics on the results object - `results.in_time_placebo()`, `results.get_loo_effects_df()` (requires `variance_method="jackknife"` at fit time), `results.sensitivity_to_zeta_omega()`, and `results.get_weight_concentration()` +- **TROP**: in-time or in-space placebo (fake treatment date, leave-one-unit-out) ```python from diff_diff import run_all_placebo_tests diff --git a/docs/llms.txt b/diff_diff/guides/llms.txt similarity index 98% rename from docs/llms.txt rename to diff_diff/guides/llms.txt index cb59f16a..996a440d 100644 --- a/docs/llms.txt +++ b/diff_diff/guides/llms.txt @@ -27,13 +27,13 @@ diagnostic steps produces unreliable results. After estimation, call `practitioner_next_steps(results)` for context-aware guidance on remaining steps. -Full practitioner guide: docs/llms-practitioner.txt +Full practitioner guide: call `diff_diff.get_llm_guide("practitioner")` ## Documentation ### Getting Started -- [Practitioner Guide](docs/llms-practitioner.txt): 8-step workflow for rigorous DiD analysis (Baker et al. 2025) — **start here** +- **Practitioner Guide** (call `diff_diff.get_llm_guide("practitioner")`): 8-step workflow for rigorous DiD analysis (Baker et al. 2025) — **start here** - [Quickstart](https://diff-diff.readthedocs.io/en/stable/quickstart.html): Installation, basic 2x2 DiD — column-name and formula interfaces, covariates, fixed effects, cluster-robust SEs - [Choosing an Estimator](https://diff-diff.readthedocs.io/en/stable/choosing_estimator.html): Decision flowchart for selecting the right estimator for your research design - [Troubleshooting](https://diff-diff.readthedocs.io/en/stable/troubleshooting.html): Common issues and solutions diff --git a/docs/conf.py b/docs/conf.py index 3ef40e39..cd68e2a7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,7 +33,7 @@ ] templates_path = ["_templates"] -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "llms.txt", "llms-full.txt"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for autodoc ----------------------------------------------------- autodoc_default_options = { @@ -71,7 +71,11 @@ "https://diff-diff.readthedocs.io/en/stable/", ) html_baseurl = _canonical_url -html_extra_path = ["llms.txt", "llms-full.txt"] +html_extra_path = [ + "../diff_diff/guides/llms.txt", + "../diff_diff/guides/llms-full.txt", + "../diff_diff/guides/llms-practitioner.txt", +] sitemap_url_scheme = "{link}" html_theme_options = { diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index b0732453..aee0b9d0 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -96,10 +96,10 @@ sources: - path: README.md section: "DifferenceInDifferences" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "DifferenceInDifferences" type: user_guide - - path: docs/llms.txt + - path: diff_diff/guides/llms.txt type: user_guide - path: docs/choosing_estimator.rst type: user_guide @@ -137,10 +137,10 @@ sources: - path: README.md section: "CallawaySantAnna" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "CallawaySantAnna" type: user_guide - - path: docs/llms.txt + - path: diff_diff/guides/llms.txt type: user_guide - path: docs/choosing_estimator.rst type: user_guide @@ -183,7 +183,7 @@ sources: - path: README.md section: "SunAbraham" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "SunAbraham" type: user_guide - path: docs/choosing_estimator.rst @@ -206,7 +206,7 @@ sources: - path: README.md section: "ImputationDiD" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "ImputationDiD" type: user_guide - path: docs/choosing_estimator.rst @@ -227,7 +227,7 @@ sources: - path: README.md section: "TwoStageDiD" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "TwoStageDiD" type: user_guide - path: docs/choosing_estimator.rst @@ -248,7 +248,7 @@ sources: - path: README.md section: "EfficientDiD" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "EfficientDiD" type: user_guide - path: docs/choosing_estimator.rst @@ -270,10 +270,10 @@ sources: - path: README.md section: "ChaisemartinDHaultfoeuille" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "ChaisemartinDHaultfoeuille" type: user_guide - - path: docs/llms.txt + - path: diff_diff/guides/llms.txt type: user_guide - path: docs/choosing_estimator.rst type: user_guide @@ -302,7 +302,7 @@ sources: - path: README.md section: "ContinuousDiD" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "ContinuousDiD" type: user_guide - path: docs/choosing_estimator.rst @@ -326,7 +326,7 @@ sources: - path: README.md section: "SyntheticDiD" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "SyntheticDiD" type: user_guide - path: docs/practitioner_decision_tree.rst @@ -352,7 +352,7 @@ sources: - path: README.md section: "TripleDifference" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "TripleDifference" type: user_guide - path: docs/choosing_estimator.rst @@ -373,7 +373,7 @@ sources: - path: README.md section: "StackedDiD" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "StackedDiD" type: user_guide - path: docs/choosing_estimator.rst @@ -394,7 +394,7 @@ sources: - path: README.md section: "WooldridgeDiD" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "WooldridgeDiD" type: user_guide - path: docs/choosing_estimator.rst @@ -415,7 +415,7 @@ sources: - path: README.md section: "TROP" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "TROP" type: user_guide - path: docs/choosing_estimator.rst @@ -438,7 +438,7 @@ sources: - path: README.md section: "HonestDiD" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "HonestDiD" type: user_guide @@ -513,7 +513,7 @@ sources: - path: README.md section: "Survey" type: user_guide - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "Survey" type: user_guide - path: docs/choosing_estimator.rst @@ -561,10 +561,10 @@ sources: docs: - path: docs/api/results.rst type: api_reference - - path: docs/llms-full.txt + - path: diff_diff/guides/llms-full.txt section: "Results API" type: user_guide - - path: docs/llms.txt + - path: diff_diff/guides/llms.txt type: user_guide - path: docs/methodology/REGISTRY.md section: "SyntheticDiD" @@ -616,7 +616,7 @@ sources: diff_diff/practitioner.py: drift_risk: low docs: - - path: docs/llms-practitioner.txt + - path: diff_diff/guides/llms-practitioner.txt type: user_guide # ── Visualization (visualization group) ──────────────────────────── @@ -632,7 +632,7 @@ sources: diff_diff/__init__.py: drift_risk: low docs: - - path: docs/llms.txt + - path: diff_diff/guides/llms.txt type: user_guide note: "Public API surface" diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index e14513ed..5bfd6ec1 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -2779,7 +2779,7 @@ Domain estimation preserving full design structure. # Practitioner Guide -The 8-step workflow in `docs/llms-practitioner.txt` is adapted from Baker et al. (2025) +The 8-step workflow in `diff_diff/guides/llms-practitioner.txt` is adapted from Baker et al. (2025) "Difference-in-Differences Designs: A Practitioner's Guide" (arXiv:2503.13323), not a 1:1 mapping of the paper's forward-engineering framework. diff --git a/pyproject.toml b/pyproject.toml index 48441903..395b7ef7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ Homepage = "https://github.com/igerber/diff-diff" Documentation = "https://diff-diff.readthedocs.io" Repository = "https://github.com/igerber/diff-diff" Issues = "https://github.com/igerber/diff-diff/issues" -"Practitioner Guide" = "https://github.com/igerber/diff-diff/blob/main/docs/llms-practitioner.txt" +"Practitioner Guide" = "https://diff-diff.readthedocs.io/en/stable/llms-practitioner.txt" [tool.maturin] # Build the Rust extension module @@ -93,6 +93,9 @@ module-name = "diff_diff._rust_backend" manifest-path = "rust/Cargo.toml" # Include Python packages python-packages = ["diff_diff"] +# Bundle LLM guide files (auto-inclusion varies across maturin 1.4-2.0; pin explicitly). +# Bare-string glob form includes in both sdist and wheel. +include = ["diff_diff/guides/*.txt"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/test_guides.py b/tests/test_guides.py new file mode 100644 index 00000000..e6c321ce --- /dev/null +++ b/tests/test_guides.py @@ -0,0 +1,67 @@ +"""Tests for the bundled LLM guide accessor.""" +import importlib.resources + +import pytest + +from diff_diff import get_llm_guide +from diff_diff._guides_api import _VARIANT_TO_FILE + + +@pytest.mark.parametrize("variant", ["concise", "full", "practitioner"]) +def test_all_variants_load(variant): + text = get_llm_guide(variant) + assert isinstance(text, str) + assert len(text) > 1000 + + +def test_default_is_concise(): + assert get_llm_guide() == get_llm_guide("concise") + + +def test_full_is_largest(): + lengths = {v: len(get_llm_guide(v)) for v in ("concise", "full", "practitioner")} + assert lengths["full"] > lengths["concise"] + assert lengths["full"] > lengths["practitioner"] + + +def test_content_stability_practitioner_workflow(): + assert "8-step" in get_llm_guide("practitioner").lower() + + +def test_content_stability_self_reference_after_rewrite(): + assert "get_llm_guide" in get_llm_guide("concise") + + +def test_wheel_content_matches_package_resource(): + for variant, filename in _VARIANT_TO_FILE.items(): + on_disk = ( + importlib.resources.files("diff_diff.guides") + .joinpath(filename) + .read_text(encoding="utf-8") + ) + assert get_llm_guide(variant) == on_disk + + +def test_utf8_encoding_preserved(): + # llms-full.txt contains the em-dash '\u2014'; verify it roundtrips. + text = get_llm_guide("full") + assert "\u2014" in text + + +@pytest.mark.parametrize("bad", ["bogus", "", "CONCISE", None, 0, True, ["x"]]) +def test_unknown_variant_raises(bad): + with pytest.raises(ValueError, match="Unknown guide variant"): + get_llm_guide(bad) + + +def test_exported_in_namespace(): + import diff_diff + + assert "get_llm_guide" in diff_diff.__all__ + assert callable(diff_diff.get_llm_guide) + + +def test_module_docstring_mentions_helper(): + import diff_diff + + assert "get_llm_guide" in diff_diff.__doc__