From f0a5d08e6c6b6e774523e5ac61422c9742308899 Mon Sep 17 00:00:00 2001 From: saccharin98 Date: Tue, 28 Apr 2026 19:26:32 +0800 Subject: [PATCH 1/3] feat: correction mechanism introduced --- openkb/agent/correction.py | 332 +++++++++++++++++++++++++++++++++++ openkb/cli.py | 104 +++++++++++ tests/test_correction.py | 185 +++++++++++++++++++ tests/test_correction_cli.py | 150 ++++++++++++++++ 4 files changed, 771 insertions(+) create mode 100644 openkb/agent/correction.py create mode 100644 tests/test_correction.py create mode 100644 tests/test_correction_cli.py diff --git a/openkb/agent/correction.py b/openkb/agent/correction.py new file mode 100644 index 00000000..3892a217 --- /dev/null +++ b/openkb/agent/correction.py @@ -0,0 +1,332 @@ +"""User-requested fact correction agent for wiki pages.""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml +from agents import Agent, Runner, function_tool +from agents.model_settings import ModelSettings + +from openkb.agent.tools import get_wiki_page_content, list_wiki_files, read_wiki_file +from openkb.schema import get_agents_md + +MAX_TURNS = 50 + + +@dataclass +class CorrectionRunResult: + """Result text plus whether the target page was actually modified.""" + + output: str + applied: bool = False + + def __str__(self) -> str: + return self.output + + def __contains__(self, item: str) -> bool: + return item in self.output + + +@dataclass +class _CorrectionWriteState: + applied: bool = False + + +_CORRECTION_INSTRUCTIONS_TEMPLATE = """\ +You are OpenKB's wiki correction agent. A user has challenged a specific claim +in one wiki page. Your job is to verify the challenged claim against the +available source material and, only when allowed, make the smallest faithful +correction. + +{schema_md} + +## Rules +1. Treat files under sources/ as evidence. Treat summaries/ and concepts/ as + derived wiki content that may contain mistakes. +2. Do not use outside knowledge. If the available files do not prove the + challenged claim wrong or right, say that it is uncertain. +3. Read the target page first. Then inspect the suggested related files and any + other clearly relevant wiki files. +4. If applying a correction, preserve the page's frontmatter and overall style. + Change only the smallest section needed to fix the unsupported or incorrect + statement. +5. Never weaken a correct statement simply because the user disagrees with it. +6. If there is not enough evidence, do not modify the page. + +## Output format +Return Markdown with these sections: +- Verdict: Supported, Incorrect, Partially supported, or Uncertain. +- Evidence checked: concise list of files/pages you inspected. +- Reasoning: short explanation grounded in the checked files. +- Proposed correction: the exact replacement or "None". +- Applied: Yes or No. +""" + + +def _split_frontmatter(text: str) -> tuple[dict[str, Any], str]: + """Return YAML frontmatter dict and body for a Markdown document.""" + frontmatter, body = _split_frontmatter_block(text) + if frontmatter is None: + return {}, text + + raw = frontmatter.removeprefix("---\n").removesuffix("\n---") + try: + data = yaml.safe_load(raw) or {} + except yaml.YAMLError: + data = {} + if not isinstance(data, dict): + data = {} + return data, body + + +def _split_frontmatter_block(text: str) -> tuple[str | None, str]: + """Return raw YAML frontmatter block and Markdown body.""" + if not text.startswith("---\n"): + return None, text + + end = text.find("\n---", 4) + if end == -1: + return None, text + + body_start = end + len("\n---") + return text[:body_start], text[body_start:].lstrip("\n") + + +def _preserve_existing_frontmatter(original: str, corrected: str) -> str: + """Keep the target page's metadata when writing corrected Markdown.""" + original_frontmatter, _ = _split_frontmatter_block(original) + if original_frontmatter is None: + return corrected + + _, corrected_body = _split_frontmatter_block(corrected) + corrected_body = corrected_body.lstrip("\n") + return f"{original_frontmatter}\n\n{corrected_body}" + + +def _ensure_md(path: str) -> str: + return path if Path(path).suffix else f"{path}.md" + + +def _normalize_wiki_path(path: str, wiki_root: Path) -> str: + """Validate a user-supplied wiki path and return a normalized relative path.""" + rel = Path(path) + if rel.is_absolute(): + raise ValueError("Wiki page must be a path relative to wiki/.") + + root = wiki_root.resolve() + full_path = (root / rel).resolve() + if not full_path.is_relative_to(root): + raise ValueError("Wiki page path escapes wiki root.") + if full_path.suffix != ".md": + raise ValueError("Wiki page must be a Markdown file.") + if not full_path.exists(): + raise FileNotFoundError(f"Wiki page not found: {path}") + + normalized = str(full_path.relative_to(root)).replace("\\", "/") + allowed = ( + normalized == "index.md" + or normalized.startswith("summaries/") + or normalized.startswith("concepts/") + or normalized.startswith("explorations/") + ) + if not allowed: + raise ValueError( + "Corrections can only target index.md, summaries/, concepts/, or explorations/ pages." + ) + return normalized + + +def _coerce_source_paths(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [str(item) for item in value if item] + return [] + + +def collect_related_files(wiki_root: Path, target_path: str) -> list[str]: + """Collect likely source/summary files for a correction request. + + This intentionally stays lightweight for the MVP. It follows common + frontmatter fields produced by the compiler instead of building a full + provenance graph. + """ + root = wiki_root.resolve() + target = (root / target_path).resolve() + if not target.is_relative_to(root) or not target.exists(): + return [] + + related: list[str] = [] + + def add(path: str) -> None: + candidate = _ensure_md(path.strip()) + full = (root / candidate).resolve() + if full.is_relative_to(root) and full.exists(): + normalized = str(full.relative_to(root)).replace("\\", "/") + if normalized not in related and normalized != target_path: + related.append(normalized) + + def add_full_text(path: str) -> None: + full = (root / path).resolve() + if full.suffix == ".json": + return + if full.is_relative_to(root) and full.exists(): + normalized = str(full.relative_to(root)).replace("\\", "/") + if normalized not in related and normalized != target_path: + related.append(normalized) + + fm, _ = _split_frontmatter(target.read_text(encoding="utf-8")) + + full_text = fm.get("full_text") + if isinstance(full_text, str): + add_full_text(full_text) + + for source in _coerce_source_paths(fm.get("sources")): + add(source) + + # Concept pages usually point to summaries; follow those summaries to their + # original full_text source when present. + for path in list(related): + if not path.startswith("summaries/"): + continue + summary = root / path + summary_fm, _ = _split_frontmatter(summary.read_text(encoding="utf-8")) + summary_full_text = summary_fm.get("full_text") + if isinstance(summary_full_text, str): + add_full_text(summary_full_text) + + return related[:12] + + +def build_correction_agent( + wiki_root: str, + model: str, + target_path: str, + apply: bool = False, + language: str = "en", + write_state: _CorrectionWriteState | None = None, +) -> Agent: + """Build the fact correction agent. + + In review-only mode the agent receives read tools only. In apply mode it + receives a single write tool that can only overwrite the target page. + """ + root = Path(wiki_root) + schema_md = get_agents_md(root) + instructions = _CORRECTION_INSTRUCTIONS_TEMPLATE.format(schema_md=schema_md) + instructions += f"\n\nIMPORTANT: Write the correction report in {language} language." + if apply: + instructions += ( + "\nYou may call write_target_file only after you have verified that the " + "challenged wiki content is incorrect or unsupported by the evidence. " + "When writing corrected Markdown, keep all existing YAML frontmatter " + "fields exactly as they are and change only the Markdown body." + ) + else: + instructions += "\nYou are in review-only mode. Do not modify files." + + @function_tool + def list_files(directory: str) -> str: + """List Markdown files in a wiki subdirectory such as summaries or concepts.""" + return list_wiki_files(directory, wiki_root) + + @function_tool + def read_file(path: str) -> str: + """Read a Markdown file from the wiki.""" + return read_wiki_file(path, wiki_root) + + @function_tool + def get_page_content(doc_name: str, pages: str) -> str: + """Read specific pages from a PageIndex source document.""" + return get_wiki_page_content(doc_name, pages, wiki_root) + + tools = [list_files, read_file, get_page_content] + + if apply: + + @function_tool + def write_target_file(content: str) -> str: + """Overwrite only the challenged target wiki page with corrected Markdown.""" + full_path = (root.resolve() / target_path).resolve() + if not full_path.is_relative_to(root.resolve()): + return "Access denied: target path escapes wiki root." + original = full_path.read_text(encoding="utf-8") + full_path.write_text( + _preserve_existing_frontmatter(original, content), + encoding="utf-8", + ) + if write_state is not None: + write_state.applied = True + return f"Written: {target_path}" + + tools.append(write_target_file) + + return Agent( + name="wiki-correction", + instructions=instructions, + tools=tools, + model=f"litellm/{model}", + model_settings=ModelSettings(parallel_tool_calls=False), + ) + + +async def run_correction( + kb_dir: Path, + target_path: str, + claim: str, + model: str, + note: str | None = None, + apply: bool = False, +) -> CorrectionRunResult: + """Verify a user challenge and optionally apply a correction.""" + from openkb.config import load_config + + wiki_root = kb_dir / "wiki" + normalized_target = _normalize_wiki_path(target_path, wiki_root) + + config = load_config(kb_dir / ".openkb" / "config.yaml") + language: str = config.get("language", "en") + related = collect_related_files(wiki_root, normalized_target) + related_text = "\n".join(f"- {path}" for path in related) or "- None found automatically" + + mode = "apply correction if verified" if apply else "review only" + user_note = note or "None" + prompt = f"""\ +Mode: {mode} + +Target wiki page: {normalized_target} + +Challenged claim: +{claim} + +User note: +{user_note} + +Suggested related files to inspect: +{related_text} + +Please verify whether the challenged claim is faithful to the source material. +Start by reading the target page. Then read the suggested related files and any +other clearly relevant files needed to decide. If PageIndex source content is +needed, use get_page_content with narrow page ranges based on the summary tree. +""" + + write_state = _CorrectionWriteState() + agent = build_correction_agent( + str(wiki_root), + model, + normalized_target, + apply=apply, + language=language, + write_state=write_state, + ) + result = await Runner.run(agent, prompt, max_turns=MAX_TURNS) + output = result.final_output or "Correction completed. No output produced." + return CorrectionRunResult( + output=output, + applied=write_state.applied, + ) diff --git a/openkb/cli.py b/openkb/cli.py index 658b3781..9db32f14 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -87,6 +87,7 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None: } _SHORT_DOC_TYPES = {"pdf", "docx", "md", "markdown", "html", "htm", "txt", "csv", "pptx", "xlsx"} +DEFAULT_CORRECTION_MODEL = "gpt-5.4" def _display_type(raw_type: str) -> str: @@ -395,6 +396,109 @@ def query(ctx, question, save): click.echo(f"\nSaved to {explore_path}") +def _write_correction_report( + kb_dir: Path, + page: str, + claim: str, + note: str | None, + applied: bool, + model: str, + result: str, + apply_requested: bool | None = None, +) -> Path: + """Persist a correction run report under wiki/reports/corrections/.""" + import datetime + import re + + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3] + slug_base = Path(page).stem or "correction" + slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", slug_base).strip("-")[:60] or "correction" + reports_dir = kb_dir / "wiki" / "reports" / "corrections" + reports_dir.mkdir(parents=True, exist_ok=True) + report_path = reports_dir / f"{timestamp}_{slug}.md" + suffix = 1 + while report_path.exists(): + report_path = reports_dir / f"{timestamp}_{slug}_{suffix}.md" + suffix += 1 + note_text = note or "" + apply_requested_text = ( + f"- Apply requested: `{apply_requested}`\n" if apply_requested is not None else "" + ) + report_path.write_text( + f"# Correction Report — {timestamp}\n\n" + f"- Page: `{page}`\n" + f"{apply_requested_text}" + f"- Applied: `{applied}`\n" + f"- Model: `{model}`\n\n" + f"## Challenged Claim\n\n{claim}\n\n" + f"## User Note\n\n{note_text}\n\n" + f"## Agent Result\n\n{result}\n", + encoding="utf-8", + ) + return report_path + + +@cli.command() +@click.argument("page") +@click.argument("claim") +@click.option("--note", default=None, help="Optional user note explaining the suspected issue.") +@click.option("--apply", "apply_fix", is_flag=True, default=False, help="Apply the correction if the agent verifies an error.") +@click.option("--model", "model_override", default=None, help="Override the correction model for this run.") +@click.pass_context +def correct(ctx, page, claim, note, apply_fix, model_override): + """Challenge a wiki claim and optionally apply a source-grounded correction.""" + kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override")) + if kb_dir is None: + click.echo("No knowledge base found. Run `openkb init` first.") + return + + from openkb.agent.correction import run_correction + + openkb_dir = kb_dir / ".openkb" + config = load_config(openkb_dir / "config.yaml") + _setup_llm_key(kb_dir) + model: str = ( + model_override + or config.get("correction_model") + or DEFAULT_CORRECTION_MODEL + ) + + mode = "apply" if apply_fix else "review" + click.echo(f"Running correction in {mode} mode with model: {model}") + + try: + correction_result = asyncio.run( + run_correction( + kb_dir, + page, + claim, + model, + note=note, + apply=apply_fix, + ) + ) + except Exception as exc: + click.echo(f"[ERROR] Correction failed: {exc}") + return + + result = getattr(correction_result, "output", str(correction_result)) + applied = bool(getattr(correction_result, "applied", False)) + click.echo(result) + + report_path = _write_correction_report( + kb_dir, + page, + claim, + note, + applied, + model, + result, + apply_requested=apply_fix, + ) + append_log(kb_dir / "wiki", "correct", f"{page} → {report_path.name}") + click.echo(f"\nReport written to {report_path}") + + @cli.command() @click.option( "--resume", "-r", "resume", diff --git a/tests/test_correction.py b/tests/test_correction.py new file mode 100644 index 00000000..a041aaaf --- /dev/null +++ b/tests/test_correction.py @@ -0,0 +1,185 @@ +"""Tests for openkb.agent.correction.""" +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from openkb.agent.correction import ( + CorrectionRunResult, + build_correction_agent, + collect_related_files, + run_correction, + _preserve_existing_frontmatter, +) +from openkb.schema import SCHEMA_MD + + +class TestBuildCorrectionAgent: + def test_agent_name(self, tmp_path): + agent = build_correction_agent(str(tmp_path), "strong-model", "concepts/topic.md") + assert agent.name == "wiki-correction" + + def test_review_mode_has_read_only_tools(self, tmp_path): + agent = build_correction_agent(str(tmp_path), "strong-model", "concepts/topic.md") + names = {t.name for t in agent.tools} + assert names == {"list_files", "read_file", "get_page_content"} + + def test_apply_mode_has_target_write_tool(self, tmp_path): + agent = build_correction_agent( + str(tmp_path), "strong-model", "concepts/topic.md", apply=True + ) + names = {t.name for t in agent.tools} + assert "write_target_file" in names + + def test_apply_mode_instructions_preserve_frontmatter(self, tmp_path): + agent = build_correction_agent( + str(tmp_path), "strong-model", "concepts/topic.md", apply=True + ) + + assert "keep all existing YAML frontmatter fields exactly as they are" in agent.instructions + + def test_schema_in_instructions(self, tmp_path): + agent = build_correction_agent(str(tmp_path), "strong-model", "concepts/topic.md") + assert SCHEMA_MD in agent.instructions + + def test_agent_model(self, tmp_path): + agent = build_correction_agent(str(tmp_path), "custom-model", "concepts/topic.md") + assert agent.model == "litellm/custom-model" + + +def test_collect_related_files_follows_concept_sources_to_full_text(tmp_path): + wiki = tmp_path / "wiki" + (wiki / "concepts").mkdir(parents=True) + (wiki / "summaries").mkdir() + (wiki / "sources").mkdir() + + (wiki / "concepts" / "topic.md").write_text( + "---\nsources: [summaries/doc]\n---\n\nClaim text.", + encoding="utf-8", + ) + (wiki / "summaries" / "doc.md").write_text( + "---\nfull_text: sources/doc.md\n---\n\nSummary.", + encoding="utf-8", + ) + (wiki / "sources" / "doc.md").write_text("Original source.", encoding="utf-8") + + related = collect_related_files(wiki, "concepts/topic.md") + + assert related == ["summaries/doc.md", "sources/doc.md"] + + +def test_collect_related_files_skips_pageindex_json_full_text(tmp_path): + wiki = tmp_path / "wiki" + (wiki / "concepts").mkdir(parents=True) + (wiki / "summaries").mkdir() + (wiki / "sources").mkdir() + + (wiki / "concepts" / "topic.md").write_text( + "---\nsources: [summaries/doc]\n---\n\nClaim text.", + encoding="utf-8", + ) + (wiki / "summaries" / "doc.md").write_text( + "---\ndoc_type: pageindex\nfull_text: sources/doc.json\n---\n\n# Doc\n\nPages 1-10.", + encoding="utf-8", + ) + (wiki / "sources" / "doc.json").write_text("[]", encoding="utf-8") + + related = collect_related_files(wiki, "concepts/topic.md") + + assert related == ["summaries/doc.md"] + + +def test_preserve_existing_frontmatter_for_corrected_body(): + original = "---\nsources: [summaries/doc]\nbrief: Old brief\n---\n\n# Old\n\nBad claim." + corrected = "# New\n\nCorrected claim." + + written = _preserve_existing_frontmatter(original, corrected) + + assert written == "---\nsources: [summaries/doc]\nbrief: Old brief\n---\n\n# New\n\nCorrected claim." + + +def test_preserve_existing_frontmatter_replaces_agent_frontmatter(): + original = "---\nsources: [summaries/doc]\nbrief: Old brief\n---\n\n# Old" + corrected = "---\nsources: []\nbrief: Changed\n---\n\n# New" + + written = _preserve_existing_frontmatter(original, corrected) + + assert "sources: [summaries/doc]" in written + assert "brief: Old brief" in written + assert "brief: Changed" not in written + assert written.endswith("# New") + + +class TestRunCorrection: + @pytest.mark.asyncio + async def test_returns_final_output_and_passes_prompt_context(self, tmp_path): + wiki = tmp_path / "wiki" + (tmp_path / ".openkb").mkdir() + (wiki / "concepts").mkdir(parents=True) + (wiki / "concepts" / "topic.md").write_text("# Topic\n\nBad claim.") + + captured = {} + + async def fake_run(agent, message, **kwargs): + captured["agent"] = agent + captured["message"] = message + captured["kwargs"] = kwargs + return MagicMock(final_output="## Verdict\n\nIncorrect.") + + with patch("openkb.agent.correction.Runner.run", side_effect=fake_run): + result = await run_correction( + tmp_path, "concepts/topic.md", "Bad claim.", "strong-model" + ) + + assert isinstance(result, CorrectionRunResult) + assert "Incorrect" in result + assert result.applied is False + assert captured["agent"].name == "wiki-correction" + assert "Target wiki page: concepts/topic.md" in captured["message"] + assert "Bad claim." in captured["message"] + assert captured["kwargs"]["max_turns"] > 0 + + @pytest.mark.asyncio + async def test_rejects_sources_as_target(self, tmp_path): + wiki = tmp_path / "wiki" + (tmp_path / ".openkb").mkdir() + (wiki / "sources").mkdir(parents=True) + (wiki / "sources" / "doc.md").write_text("Evidence.") + + with pytest.raises(ValueError): + await run_correction(tmp_path, "sources/doc.md", "Claim", "strong-model") + + @pytest.mark.asyncio + async def test_apply_mode_builds_agent_with_write_tool(self, tmp_path): + wiki = tmp_path / "wiki" + (tmp_path / ".openkb").mkdir() + (wiki / "summaries").mkdir(parents=True) + (wiki / "summaries" / "doc.md").write_text("# Doc\n\nBad claim.") + + with patch("openkb.agent.correction.Runner.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = MagicMock(final_output="## Applied\n\nYes.") + await run_correction( + tmp_path, "summaries/doc.md", "Bad claim.", "strong-model", apply=True + ) + + agent = mock_run.call_args.args[0] + names = {t.name for t in agent.tools} + assert "write_target_file" in names + + @pytest.mark.asyncio + async def test_apply_mode_reports_not_applied_when_agent_does_not_write(self, tmp_path): + wiki = tmp_path / "wiki" + (tmp_path / ".openkb").mkdir() + (wiki / "summaries").mkdir(parents=True) + (wiki / "summaries" / "doc.md").write_text("# Doc\n\nAccurate claim.") + + with patch("openkb.agent.correction.Runner.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = MagicMock( + final_output="## Verdict\n\nSupported.\n\n## Applied\n\nNo." + ) + result = await run_correction( + tmp_path, "summaries/doc.md", "Accurate claim.", "strong-model", apply=True + ) + + assert result.applied is False diff --git a/tests/test_correction_cli.py b/tests/test_correction_cli.py new file mode 100644 index 00000000..6682e595 --- /dev/null +++ b/tests/test_correction_cli.py @@ -0,0 +1,150 @@ +"""Tests for the openkb correct CLI command.""" +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +from click.testing import CliRunner + +from openkb.agent.correction import CorrectionRunResult +from openkb.cli import cli + + +def _make_kb(tmp_path): + kb = tmp_path / "kb" + (kb / ".openkb").mkdir(parents=True) + (kb / "wiki" / "concepts").mkdir(parents=True) + (kb / "wiki" / "reports").mkdir() + (kb / "wiki" / "log.md").write_text("# Operations Log\n\n", encoding="utf-8") + (kb / ".openkb" / "config.yaml").write_text( + "model: weak-model\ncorrection_model: strong-model\n", + encoding="utf-8", + ) + (kb / "wiki" / "concepts" / "topic.md").write_text( + "# Topic\n\nQuestionable claim.", + encoding="utf-8", + ) + return kb + + +def test_correct_cli_runs_review_and_writes_report(tmp_path): + kb = _make_kb(tmp_path) + runner = CliRunner() + + with patch("openkb.cli._setup_llm_key"), \ + patch("openkb.agent.correction.run_correction", new_callable=AsyncMock) as mock_run: + mock_run.return_value = "## Verdict\n\nIncorrect." + result = runner.invoke( + cli, + [ + "--kb-dir", + str(kb), + "correct", + "concepts/topic.md", + "Questionable claim.", + "--note", + "Please verify this.", + ], + ) + + assert result.exit_code == 0 + assert "review mode" in result.output + assert "strong-model" in result.output + assert "Incorrect" in result.output + mock_run.assert_awaited_once() + args = mock_run.call_args.args + kwargs = mock_run.call_args.kwargs + assert args[:4] == (kb, "concepts/topic.md", "Questionable claim.", "strong-model") + assert kwargs["note"] == "Please verify this." + assert kwargs["apply"] is False + + reports = list((kb / "wiki" / "reports" / "corrections").glob("*.md")) + assert len(reports) == 1 + report = reports[0].read_text(encoding="utf-8") + assert "Questionable claim." in report + assert "Incorrect" in report + + +def test_correct_cli_apply_and_model_override(tmp_path): + kb = _make_kb(tmp_path) + runner = CliRunner() + + with patch("openkb.cli._setup_llm_key"), \ + patch("openkb.agent.correction.run_correction", new_callable=AsyncMock) as mock_run: + mock_run.return_value = CorrectionRunResult( + output="## Applied\n\nYes.", + applied=True, + ) + result = runner.invoke( + cli, + [ + "--kb-dir", + str(kb), + "correct", + "concepts/topic.md", + "Questionable claim.", + "--apply", + "--model", + "override-strong", + ], + ) + + assert result.exit_code == 0 + assert "apply mode" in result.output + mock_run.assert_awaited_once() + assert mock_run.call_args.args[3] == "override-strong" + assert mock_run.call_args.kwargs["apply"] is True + + reports = list((kb / "wiki" / "reports" / "corrections").glob("*.md")) + assert len(reports) == 1 + report = reports[0].read_text(encoding="utf-8") + assert "- Apply requested: `True`" in report + assert "- Applied: `True`" in report + + +def test_correct_cli_reports_actual_not_applied_status(tmp_path): + kb = _make_kb(tmp_path) + runner = CliRunner() + + with patch("openkb.cli._setup_llm_key"), \ + patch("openkb.agent.correction.run_correction", new_callable=AsyncMock) as mock_run: + mock_run.return_value = CorrectionRunResult( + output="## Verdict\n\nSupported.\n\n## Applied\n\nNo.", + applied=False, + ) + result = runner.invoke( + cli, + [ + "--kb-dir", + str(kb), + "correct", + "concepts/topic.md", + "Questionable claim.", + "--apply", + ], + ) + + assert result.exit_code == 0 + reports = list((kb / "wiki" / "reports" / "corrections").glob("*.md")) + assert len(reports) == 1 + report = reports[0].read_text(encoding="utf-8") + assert "- Apply requested: `True`" in report + assert "- Applied: `False`" in report + + +def test_write_correction_report_uses_millisecond_timestamp_and_unique_suffix(tmp_path): + from openkb.cli import _write_correction_report + + kb = _make_kb(tmp_path) + + first = _write_correction_report( + kb, "concepts/topic.md", "Claim.", None, False, "model", "Result." + ) + second = _write_correction_report( + kb, "concepts/topic.md", "Claim.", None, False, "model", "Result." + ) + + assert first != second + assert first.exists() + assert second.exists() + timestamp = first.name.split("_topic.md")[0] + assert len(timestamp.split("_")[-1]) == 3 From 9a6f5318e959a1ff4cf0a6888d12b1660f99c5a7 Mon Sep 17 00:00:00 2001 From: saccharin98 Date: Wed, 29 Apr 2026 17:12:11 +0800 Subject: [PATCH 2/3] integrate correction mechanism with lint system --- openkb/agent/{correction.py => lint_fix.py} | 68 ++--- openkb/cli.py | 252 ++++++++++-------- openkb/lint.py | 37 +++ tests/test_correction_cli.py | 150 ----------- tests/test_lint.py | 15 ++ tests/test_lint_cli.py | 100 ++++++- .../{test_correction.py => test_lint_fix.py} | 46 ++-- 7 files changed, 346 insertions(+), 322 deletions(-) rename openkb/agent/{correction.py => lint_fix.py} (83%) delete mode 100644 tests/test_correction_cli.py rename tests/{test_correction.py => test_lint_fix.py} (81%) diff --git a/openkb/agent/correction.py b/openkb/agent/lint_fix.py similarity index 83% rename from openkb/agent/correction.py rename to openkb/agent/lint_fix.py index 3892a217..4c1b21db 100644 --- a/openkb/agent/correction.py +++ b/openkb/agent/lint_fix.py @@ -1,4 +1,4 @@ -"""User-requested fact correction agent for wiki pages.""" +"""Knowledge issue fixer for OpenKB lint.""" from __future__ import annotations from dataclasses import dataclass @@ -16,7 +16,7 @@ @dataclass -class CorrectionRunResult: +class LintFixRunResult: """Result text plus whether the target page was actually modified.""" output: str @@ -30,15 +30,15 @@ def __contains__(self, item: str) -> bool: @dataclass -class _CorrectionWriteState: +class _LintFixWriteState: applied: bool = False -_CORRECTION_INSTRUCTIONS_TEMPLATE = """\ -You are OpenKB's wiki correction agent. A user has challenged a specific claim -in one wiki page. Your job is to verify the challenged claim against the -available source material and, only when allowed, make the smallest faithful -correction. +_KNOWLEDGE_FIX_INSTRUCTIONS_TEMPLATE = """\ +You are OpenKB's knowledge issue fix agent. A lint issue or user feedback has +challenged a specific claim in one wiki page. Your job is to verify the claim +against the available source material and, only when allowed, make the smallest +faithful fix. {schema_md} @@ -49,7 +49,7 @@ class _CorrectionWriteState: challenged claim wrong or right, say that it is uncertain. 3. Read the target page first. Then inspect the suggested related files and any other clearly relevant wiki files. -4. If applying a correction, preserve the page's frontmatter and overall style. +4. If applying a fix, preserve the page's frontmatter and overall style. Change only the smallest section needed to fix the unsupported or incorrect statement. 5. Never weaken a correct statement simply because the user disagrees with it. @@ -60,7 +60,7 @@ class _CorrectionWriteState: - Verdict: Supported, Incorrect, Partially supported, or Uncertain. - Evidence checked: concise list of files/pages you inspected. - Reasoning: short explanation grounded in the checked files. -- Proposed correction: the exact replacement or "None". +- Proposed fix: the exact replacement or "None". - Applied: Yes or No. """ @@ -94,15 +94,15 @@ def _split_frontmatter_block(text: str) -> tuple[str | None, str]: return text[:body_start], text[body_start:].lstrip("\n") -def _preserve_existing_frontmatter(original: str, corrected: str) -> str: - """Keep the target page's metadata when writing corrected Markdown.""" +def _preserve_existing_frontmatter(original: str, fixed: str) -> str: + """Keep the target page's metadata when writing fixed Markdown.""" original_frontmatter, _ = _split_frontmatter_block(original) if original_frontmatter is None: - return corrected + return fixed - _, corrected_body = _split_frontmatter_block(corrected) - corrected_body = corrected_body.lstrip("\n") - return f"{original_frontmatter}\n\n{corrected_body}" + _, fixed_body = _split_frontmatter_block(fixed) + fixed_body = fixed_body.lstrip("\n") + return f"{original_frontmatter}\n\n{fixed_body}" def _ensure_md(path: str) -> str: @@ -133,7 +133,7 @@ def _normalize_wiki_path(path: str, wiki_root: Path) -> str: ) if not allowed: raise ValueError( - "Corrections can only target index.md, summaries/, concepts/, or explorations/ pages." + "Lint fixes can only target index.md, summaries/, concepts/, or explorations/ pages." ) return normalized @@ -149,7 +149,7 @@ def _coerce_source_paths(value: Any) -> list[str]: def collect_related_files(wiki_root: Path, target_path: str) -> list[str]: - """Collect likely source/summary files for a correction request. + """Collect likely source/summary files for a knowledge fix request. This intentionally stays lightweight for the MVP. It follows common frontmatter fields produced by the compiler instead of building a full @@ -202,28 +202,28 @@ def add_full_text(path: str) -> None: return related[:12] -def build_correction_agent( +def build_knowledge_fix_agent( wiki_root: str, model: str, target_path: str, apply: bool = False, language: str = "en", - write_state: _CorrectionWriteState | None = None, + write_state: _LintFixWriteState | None = None, ) -> Agent: - """Build the fact correction agent. + """Build the source-grounded knowledge fix agent. In review-only mode the agent receives read tools only. In apply mode it receives a single write tool that can only overwrite the target page. """ root = Path(wiki_root) schema_md = get_agents_md(root) - instructions = _CORRECTION_INSTRUCTIONS_TEMPLATE.format(schema_md=schema_md) - instructions += f"\n\nIMPORTANT: Write the correction report in {language} language." + instructions = _KNOWLEDGE_FIX_INSTRUCTIONS_TEMPLATE.format(schema_md=schema_md) + instructions += f"\n\nIMPORTANT: Write the fix report in {language} language." if apply: instructions += ( "\nYou may call write_target_file only after you have verified that the " "challenged wiki content is incorrect or unsupported by the evidence. " - "When writing corrected Markdown, keep all existing YAML frontmatter " + "When writing fixed Markdown, keep all existing YAML frontmatter " "fields exactly as they are and change only the Markdown body." ) else: @@ -250,7 +250,7 @@ def get_page_content(doc_name: str, pages: str) -> str: @function_tool def write_target_file(content: str) -> str: - """Overwrite only the challenged target wiki page with corrected Markdown.""" + """Overwrite only the challenged target wiki page with fixed Markdown.""" full_path = (root.resolve() / target_path).resolve() if not full_path.is_relative_to(root.resolve()): return "Access denied: target path escapes wiki root." @@ -266,7 +266,7 @@ def write_target_file(content: str) -> str: tools.append(write_target_file) return Agent( - name="wiki-correction", + name="wiki-knowledge-fixer", instructions=instructions, tools=tools, model=f"litellm/{model}", @@ -274,15 +274,15 @@ def write_target_file(content: str) -> str: ) -async def run_correction( +async def run_knowledge_fix( kb_dir: Path, target_path: str, claim: str, model: str, note: str | None = None, apply: bool = False, -) -> CorrectionRunResult: - """Verify a user challenge and optionally apply a correction.""" +) -> LintFixRunResult: + """Verify a knowledge issue and optionally apply a source-grounded fix.""" from openkb.config import load_config wiki_root = kb_dir / "wiki" @@ -293,7 +293,7 @@ async def run_correction( related = collect_related_files(wiki_root, normalized_target) related_text = "\n".join(f"- {path}" for path in related) or "- None found automatically" - mode = "apply correction if verified" if apply else "review only" + mode = "apply fix if verified" if apply else "review only" user_note = note or "None" prompt = f"""\ Mode: {mode} @@ -315,8 +315,8 @@ async def run_correction( needed, use get_page_content with narrow page ranges based on the summary tree. """ - write_state = _CorrectionWriteState() - agent = build_correction_agent( + write_state = _LintFixWriteState() + agent = build_knowledge_fix_agent( str(wiki_root), model, normalized_target, @@ -325,8 +325,8 @@ async def run_correction( write_state=write_state, ) result = await Runner.run(agent, prompt, max_turns=MAX_TURNS) - output = result.final_output or "Correction completed. No output produced." - return CorrectionRunResult( + output = result.final_output or "Knowledge fix completed. No output produced." + return LintFixRunResult( output=output, applied=write_state.applied, ) diff --git a/openkb/cli.py b/openkb/cli.py index 9db32f14..dabb1e71 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -87,7 +87,7 @@ def _setup_llm_key(kb_dir: Path | None = None) -> None: } _SHORT_DOC_TYPES = {"pdf", "docx", "md", "markdown", "html", "htm", "txt", "csv", "pptx", "xlsx"} -DEFAULT_CORRECTION_MODEL = "gpt-5.4" +DEFAULT_LINT_FIX_MODEL = "gpt-5.4" def _display_type(raw_type: str) -> str: @@ -396,109 +396,6 @@ def query(ctx, question, save): click.echo(f"\nSaved to {explore_path}") -def _write_correction_report( - kb_dir: Path, - page: str, - claim: str, - note: str | None, - applied: bool, - model: str, - result: str, - apply_requested: bool | None = None, -) -> Path: - """Persist a correction run report under wiki/reports/corrections/.""" - import datetime - import re - - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3] - slug_base = Path(page).stem or "correction" - slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", slug_base).strip("-")[:60] or "correction" - reports_dir = kb_dir / "wiki" / "reports" / "corrections" - reports_dir.mkdir(parents=True, exist_ok=True) - report_path = reports_dir / f"{timestamp}_{slug}.md" - suffix = 1 - while report_path.exists(): - report_path = reports_dir / f"{timestamp}_{slug}_{suffix}.md" - suffix += 1 - note_text = note or "" - apply_requested_text = ( - f"- Apply requested: `{apply_requested}`\n" if apply_requested is not None else "" - ) - report_path.write_text( - f"# Correction Report — {timestamp}\n\n" - f"- Page: `{page}`\n" - f"{apply_requested_text}" - f"- Applied: `{applied}`\n" - f"- Model: `{model}`\n\n" - f"## Challenged Claim\n\n{claim}\n\n" - f"## User Note\n\n{note_text}\n\n" - f"## Agent Result\n\n{result}\n", - encoding="utf-8", - ) - return report_path - - -@cli.command() -@click.argument("page") -@click.argument("claim") -@click.option("--note", default=None, help="Optional user note explaining the suspected issue.") -@click.option("--apply", "apply_fix", is_flag=True, default=False, help="Apply the correction if the agent verifies an error.") -@click.option("--model", "model_override", default=None, help="Override the correction model for this run.") -@click.pass_context -def correct(ctx, page, claim, note, apply_fix, model_override): - """Challenge a wiki claim and optionally apply a source-grounded correction.""" - kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override")) - if kb_dir is None: - click.echo("No knowledge base found. Run `openkb init` first.") - return - - from openkb.agent.correction import run_correction - - openkb_dir = kb_dir / ".openkb" - config = load_config(openkb_dir / "config.yaml") - _setup_llm_key(kb_dir) - model: str = ( - model_override - or config.get("correction_model") - or DEFAULT_CORRECTION_MODEL - ) - - mode = "apply" if apply_fix else "review" - click.echo(f"Running correction in {mode} mode with model: {model}") - - try: - correction_result = asyncio.run( - run_correction( - kb_dir, - page, - claim, - model, - note=note, - apply=apply_fix, - ) - ) - except Exception as exc: - click.echo(f"[ERROR] Correction failed: {exc}") - return - - result = getattr(correction_result, "output", str(correction_result)) - applied = bool(getattr(correction_result, "applied", False)) - click.echo(result) - - report_path = _write_correction_report( - kb_dir, - page, - claim, - note, - applied, - model, - result, - apply_requested=apply_fix, - ) - append_log(kb_dir / "wiki", "correct", f"{page} → {report_path.name}") - click.echo(f"\nReport written to {report_path}") - - @cli.command() @click.option( "--resume", "-r", "resume", @@ -629,17 +526,53 @@ def on_new_files(paths): watch_directory(raw_dir, on_new_files) -async def run_lint(kb_dir: Path) -> Path | None: +def _assign_issue_ids(issues: list[dict]) -> list[dict]: + """Return a copy of issues with stable per-report ids.""" + numbered: list[dict] = [] + for idx, issue in enumerate(issues, 1): + item = dict(issue) + item.setdefault("id", f"issue-{idx:03d}") + numbered.append(item) + return numbered + + +def _format_fix_results(results: list[dict]) -> str: + if not results: + return "No fixes were attempted." + + lines = ["## Fixes\n"] + for result in results: + status = result.get("status", "unknown") + issue_id = result.get("issue_id", "unknown") + lines.append(f"### {issue_id} - {status}") + if result.get("page"): + lines.append(f"- Page: `{result['page']}`") + if result.get("message"): + lines.append(f"- Message: {result['message']}") + if result.get("output"): + lines.append("") + lines.append(str(result["output"])) + lines.append("") + return "\n".join(lines).rstrip() + + +async def run_lint( + kb_dir: Path, + fix: bool = False, + feedback_page: str | None = None, + feedback: str | None = None, +) -> Path | None: """Run structural + knowledge lint, write report, return report path. Returns ``None`` if the KB has no indexed documents (nothing to lint). Async because knowledge lint uses an LLM agent. Usable from CLI (via ``asyncio.run``) and directly from the chat REPL. """ - from openkb.lint import run_structural_lint + from openkb.lint import collect_structural_issues, run_structural_lint from openkb.agent.linter import run_knowledge_lint openkb_dir = kb_dir / ".openkb" + has_user_feedback = bool(feedback_page and feedback) # Skip lint entirely when the KB has no indexed documents hashes_file = openkb_dir / "hashes.json" @@ -647,16 +580,31 @@ async def run_lint(kb_dir: Path) -> Path | None: hashes = json.loads(hashes_file.read_text(encoding="utf-8")) else: hashes = {} - if not hashes: + if not hashes and not has_user_feedback: click.echo("Nothing to lint — no documents indexed yet. Run `openkb add` first.") return config = load_config(openkb_dir / "config.yaml") _setup_llm_key(kb_dir) model: str = config.get("model", DEFAULT_CONFIG["model"]) + lint_fix_model: str = ( + config.get("lint_fix_model") + or DEFAULT_LINT_FIX_MODEL + ) click.echo("Running structural lint...") structural_report = run_structural_lint(kb_dir) + issues = collect_structural_issues(kb_dir) + if has_user_feedback: + issues.append({ + "type": "knowledge", + "source": "user_feedback", + "page": feedback_page, + "description": feedback, + "message": "User-reported knowledge issue.", + "fixable": True, + }) + issues = _assign_issue_ids(issues) click.echo(structural_report) click.echo("Running knowledge lint...") @@ -666,31 +614,107 @@ async def run_lint(kb_dir: Path) -> Path | None: knowledge_report = f"Knowledge lint failed: {exc}" click.echo(knowledge_report) - # Write combined report + fix_results: list[dict] = [] + if fix: + feedback_issue = next( + ( + issue for issue in issues + if issue.get("type") == "knowledge" + and issue.get("source") == "user_feedback" + and issue.get("fixable") + ), + None, + ) + if feedback_issue is None: + fix_results.append({ + "issue_id": "none", + "status": "skipped", + "message": ( + "No fixable user feedback issue was provided. " + "First-version lint --fix does not auto-repair structural or schema issues." + ), + }) + else: + click.echo("Running knowledge fix for user feedback...") + from openkb.agent.lint_fix import run_knowledge_fix + + try: + fix_result = await run_knowledge_fix( + kb_dir, + str(feedback_issue["page"]), + str(feedback_issue["description"]), + lint_fix_model, + note="Created from openkb lint --fix user feedback.", + apply=True, + ) + applied = bool(getattr(fix_result, "applied", False)) + output = getattr(fix_result, "output", str(fix_result)) + status = "applied" if applied else "not_applied" + fix_results.append({ + "issue_id": feedback_issue["id"], + "status": status, + "page": feedback_issue["page"], + "message": "Knowledge fix completed.", + "output": output, + }) + click.echo(output) + except Exception as exc: + fix_results.append({ + "issue_id": feedback_issue["id"], + "status": "failed", + "page": feedback_issue["page"], + "message": str(exc), + }) + click.echo(f"[ERROR] Knowledge fix failed: {exc}") + + # Write combined report and machine-readable issue report reports_dir = kb_dir / "wiki" / "reports" reports_dir.mkdir(parents=True, exist_ok=True) import datetime timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") report_path = reports_dir / f"lint_{timestamp}.md" - report_content = f"# Lint Report — {timestamp}\n\n## Structural\n\n{structural_report}\n\n## Semantic\n\n{knowledge_report}\n" + issue_report_path = reports_dir / f"lint_{timestamp}.issues.json" + issue_report = { + "version": 1, + "issue_types": ["structural", "schema", "knowledge"], + "issues": issues, + "fix_requested": fix, + "fix_results": fix_results, + } + issue_report_path.write_text( + json.dumps(issue_report, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + fixes_section = f"\n\n{_format_fix_results(fix_results)}" if fix else "" + report_content = ( + f"# Lint Report — {timestamp}\n\n" + f"- Issue JSON: `{issue_report_path.name}`\n\n" + f"## Structural\n\n{structural_report}\n\n" + f"## Semantic\n\n{knowledge_report}" + f"{fixes_section}\n" + ) report_path.write_text(report_content, encoding="utf-8") append_log(kb_dir / "wiki", "lint", f"report → {report_path.name}") click.echo(f"\nReport written to {report_path}") + click.echo(f"Issue JSON written to {issue_report_path}") return report_path @cli.command() -@click.option("--fix", is_flag=True, default=False, help="Automatically fix lint issues (not yet implemented).") +@click.option("--fix", is_flag=True, default=False, help="Apply safe fixes for supported lint issues.") +@click.option("--page", default=None, help="Wiki page related to user feedback, e.g. concepts/topic.md.") +@click.option("--feedback", default=None, help="User-described wiki issue to record or fix.") @click.pass_context -def lint(ctx, fix): +def lint(ctx, fix, page, feedback): """Lint the knowledge base for structural and semantic inconsistencies.""" - if fix: - click.echo("Warning: --fix is not yet implemented. Running lint in report-only mode.") + if bool(page) != bool(feedback): + click.echo("Use --page and --feedback together.") + return kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override")) if kb_dir is None: click.echo("No knowledge base found. Run `openkb init` first.") return - asyncio.run(run_lint(kb_dir)) + asyncio.run(run_lint(kb_dir, fix=fix, feedback_page=page, feedback=feedback)) def print_list(kb_dir: Path) -> None: diff --git a/openkb/lint.py b/openkb/lint.py index 78b22e5f..30cfdcd0 100644 --- a/openkb/lint.py +++ b/openkb/lint.py @@ -10,6 +10,7 @@ import re from pathlib import Path +from typing import Any # Matches [[wikilink]] or [[subdir/link]] _WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]") @@ -207,6 +208,42 @@ def check_index_sync(wiki: Path) -> list[str]: return sorted(issues) +def collect_structural_issues(kb_dir: Path) -> list[dict[str, Any]]: + """Return structural lint findings as machine-readable issues.""" + wiki = kb_dir / "wiki" + raw = kb_dir / "raw" + + issues: list[dict[str, Any]] = [] + for message in find_broken_links(wiki): + issues.append({ + "type": "structural", + "message": message, + "fixable": False, + }) + for page in find_orphans(wiki): + issues.append({ + "type": "structural", + "page": f"{page}.md", + "message": f"Orphaned page: {page}", + "fixable": False, + }) + for name in find_missing_entries(raw, wiki): + issues.append({ + "type": "structural", + "message": f"Raw file has no wiki entry: {name}", + "raw_file": name, + "fixable": False, + }) + for message in check_index_sync(wiki): + issues.append({ + "type": "structural", + "message": message, + "fixable": False, + }) + + return issues + + def run_structural_lint(kb_dir: Path) -> str: """Run all structural lint checks and return a formatted Markdown report. diff --git a/tests/test_correction_cli.py b/tests/test_correction_cli.py deleted file mode 100644 index 6682e595..00000000 --- a/tests/test_correction_cli.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Tests for the openkb correct CLI command.""" -from __future__ import annotations - -from unittest.mock import AsyncMock, patch - -from click.testing import CliRunner - -from openkb.agent.correction import CorrectionRunResult -from openkb.cli import cli - - -def _make_kb(tmp_path): - kb = tmp_path / "kb" - (kb / ".openkb").mkdir(parents=True) - (kb / "wiki" / "concepts").mkdir(parents=True) - (kb / "wiki" / "reports").mkdir() - (kb / "wiki" / "log.md").write_text("# Operations Log\n\n", encoding="utf-8") - (kb / ".openkb" / "config.yaml").write_text( - "model: weak-model\ncorrection_model: strong-model\n", - encoding="utf-8", - ) - (kb / "wiki" / "concepts" / "topic.md").write_text( - "# Topic\n\nQuestionable claim.", - encoding="utf-8", - ) - return kb - - -def test_correct_cli_runs_review_and_writes_report(tmp_path): - kb = _make_kb(tmp_path) - runner = CliRunner() - - with patch("openkb.cli._setup_llm_key"), \ - patch("openkb.agent.correction.run_correction", new_callable=AsyncMock) as mock_run: - mock_run.return_value = "## Verdict\n\nIncorrect." - result = runner.invoke( - cli, - [ - "--kb-dir", - str(kb), - "correct", - "concepts/topic.md", - "Questionable claim.", - "--note", - "Please verify this.", - ], - ) - - assert result.exit_code == 0 - assert "review mode" in result.output - assert "strong-model" in result.output - assert "Incorrect" in result.output - mock_run.assert_awaited_once() - args = mock_run.call_args.args - kwargs = mock_run.call_args.kwargs - assert args[:4] == (kb, "concepts/topic.md", "Questionable claim.", "strong-model") - assert kwargs["note"] == "Please verify this." - assert kwargs["apply"] is False - - reports = list((kb / "wiki" / "reports" / "corrections").glob("*.md")) - assert len(reports) == 1 - report = reports[0].read_text(encoding="utf-8") - assert "Questionable claim." in report - assert "Incorrect" in report - - -def test_correct_cli_apply_and_model_override(tmp_path): - kb = _make_kb(tmp_path) - runner = CliRunner() - - with patch("openkb.cli._setup_llm_key"), \ - patch("openkb.agent.correction.run_correction", new_callable=AsyncMock) as mock_run: - mock_run.return_value = CorrectionRunResult( - output="## Applied\n\nYes.", - applied=True, - ) - result = runner.invoke( - cli, - [ - "--kb-dir", - str(kb), - "correct", - "concepts/topic.md", - "Questionable claim.", - "--apply", - "--model", - "override-strong", - ], - ) - - assert result.exit_code == 0 - assert "apply mode" in result.output - mock_run.assert_awaited_once() - assert mock_run.call_args.args[3] == "override-strong" - assert mock_run.call_args.kwargs["apply"] is True - - reports = list((kb / "wiki" / "reports" / "corrections").glob("*.md")) - assert len(reports) == 1 - report = reports[0].read_text(encoding="utf-8") - assert "- Apply requested: `True`" in report - assert "- Applied: `True`" in report - - -def test_correct_cli_reports_actual_not_applied_status(tmp_path): - kb = _make_kb(tmp_path) - runner = CliRunner() - - with patch("openkb.cli._setup_llm_key"), \ - patch("openkb.agent.correction.run_correction", new_callable=AsyncMock) as mock_run: - mock_run.return_value = CorrectionRunResult( - output="## Verdict\n\nSupported.\n\n## Applied\n\nNo.", - applied=False, - ) - result = runner.invoke( - cli, - [ - "--kb-dir", - str(kb), - "correct", - "concepts/topic.md", - "Questionable claim.", - "--apply", - ], - ) - - assert result.exit_code == 0 - reports = list((kb / "wiki" / "reports" / "corrections").glob("*.md")) - assert len(reports) == 1 - report = reports[0].read_text(encoding="utf-8") - assert "- Apply requested: `True`" in report - assert "- Applied: `False`" in report - - -def test_write_correction_report_uses_millisecond_timestamp_and_unique_suffix(tmp_path): - from openkb.cli import _write_correction_report - - kb = _make_kb(tmp_path) - - first = _write_correction_report( - kb, "concepts/topic.md", "Claim.", None, False, "model", "Result." - ) - second = _write_correction_report( - kb, "concepts/topic.md", "Claim.", None, False, "model", "Result." - ) - - assert first != second - assert first.exists() - assert second.exists() - timestamp = first.name.split("_topic.md")[0] - assert len(timestamp.split("_")[-1]) == 3 diff --git a/tests/test_lint.py b/tests/test_lint.py index 526e4a9e..c48dc703 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -7,6 +7,7 @@ from openkb.lint import ( check_index_sync, + collect_structural_issues, find_broken_links, find_missing_entries, find_orphans, @@ -224,3 +225,17 @@ def test_report_includes_broken_link_details(self, tmp_path): report = run_structural_lint(tmp_path) assert "missing" in report + + +def test_collect_structural_issues_returns_json_ready_items(tmp_path): + wiki = _make_wiki(tmp_path) + raw = tmp_path / "raw" + raw.mkdir() + (wiki / "summaries" / "doc.md").write_text("See [[concepts/missing]]") + + issues = collect_structural_issues(tmp_path) + + assert issues + assert issues[0]["type"] == "structural" + assert issues[0]["fixable"] is False + assert "message" in issues[0] diff --git a/tests/test_lint_cli.py b/tests/test_lint_cli.py index bc207f08..fec3a1bd 100644 --- a/tests/test_lint_cli.py +++ b/tests/test_lint_cli.py @@ -3,10 +3,11 @@ import json from pathlib import Path -from unittest.mock import patch +from unittest.mock import AsyncMock, patch from click.testing import CliRunner +from openkb.agent.lint_fix import LintFixRunResult from openkb.cli import cli @@ -73,3 +74,100 @@ def test_lint_runs_when_docs_exist(self, tmp_path): assert "Running structural lint" in result.output assert "Running knowledge lint" in result.output assert "Report written to" in result.output + + issue_reports = list((kb_dir / "wiki" / "reports").glob("*.issues.json")) + assert len(issue_reports) == 1 + data = json.loads(issue_reports[0].read_text(encoding="utf-8")) + assert data["issue_types"] == ["structural", "schema", "knowledge"] + assert data["fix_requested"] is False + + def test_lint_records_user_feedback_as_knowledge_issue(self, tmp_path): + kb_dir = _setup_kb(tmp_path) + hashes = {"abc": {"name": "paper.pdf", "type": "pdf"}} + (kb_dir / ".openkb" / "hashes.json").write_text(json.dumps(hashes)) + (kb_dir / "wiki" / "concepts" / "topic.md").write_text("Questionable claim.") + runner = CliRunner() + + with patch("openkb.cli._find_kb_dir", return_value=kb_dir), \ + patch("openkb.cli._setup_llm_key"), \ + patch("openkb.agent.linter.run_knowledge_lint", return_value="No issues."): + result = runner.invoke( + cli, + [ + "lint", + "--page", + "concepts/topic.md", + "--feedback", + "This claim looks wrong.", + ], + ) + + assert result.exit_code == 0 + issue_reports = list((kb_dir / "wiki" / "reports").glob("*.issues.json")) + data = json.loads(issue_reports[0].read_text(encoding="utf-8")) + feedback_issues = [ + issue for issue in data["issues"] + if issue["type"] == "knowledge" and issue["source"] == "user_feedback" + ] + assert len(feedback_issues) == 1 + assert feedback_issues[0]["page"] == "concepts/topic.md" + assert feedback_issues[0]["description"] == "This claim looks wrong." + assert feedback_issues[0]["fixable"] is True + + def test_lint_fix_runs_knowledge_fix_for_user_feedback(self, tmp_path): + kb_dir = _setup_kb(tmp_path) + hashes = {"abc": {"name": "paper.pdf", "type": "pdf"}} + (kb_dir / ".openkb" / "hashes.json").write_text(json.dumps(hashes)) + (kb_dir / ".openkb" / "config.yaml").write_text( + "model: weak-model\nlint_fix_model: strong-model\n" + ) + (kb_dir / "wiki" / "concepts" / "topic.md").write_text("Questionable claim.") + runner = CliRunner() + + with patch("openkb.cli._find_kb_dir", return_value=kb_dir), \ + patch("openkb.cli._setup_llm_key"), \ + patch("openkb.agent.linter.run_knowledge_lint", return_value="No issues."), \ + patch("openkb.agent.lint_fix.run_knowledge_fix", new_callable=AsyncMock) as mock_fix: + mock_fix.return_value = LintFixRunResult( + output="## Applied\n\nYes.", + applied=True, + ) + result = runner.invoke( + cli, + [ + "lint", + "--fix", + "--page", + "concepts/topic.md", + "--feedback", + "This claim looks wrong.", + ], + ) + + assert result.exit_code == 0 + assert "Running knowledge fix for user feedback" in result.output + mock_fix.assert_awaited_once() + args = mock_fix.call_args.args + kwargs = mock_fix.call_args.kwargs + assert args[:4] == ( + kb_dir, + "concepts/topic.md", + "This claim looks wrong.", + "strong-model", + ) + assert kwargs["apply"] is True + + issue_reports = list((kb_dir / "wiki" / "reports").glob("*.issues.json")) + data = json.loads(issue_reports[0].read_text(encoding="utf-8")) + assert data["fix_requested"] is True + assert data["fix_results"][0]["status"] == "applied" + + def test_lint_page_requires_feedback(self, tmp_path): + kb_dir = _setup_kb(tmp_path) + runner = CliRunner() + + with patch("openkb.cli._find_kb_dir", return_value=kb_dir): + result = runner.invoke(cli, ["lint", "--page", "concepts/topic.md"]) + + assert result.exit_code == 0 + assert "Use --page and --feedback together" in result.output diff --git a/tests/test_correction.py b/tests/test_lint_fix.py similarity index 81% rename from tests/test_correction.py rename to tests/test_lint_fix.py index a041aaaf..fb660371 100644 --- a/tests/test_correction.py +++ b/tests/test_lint_fix.py @@ -1,50 +1,50 @@ -"""Tests for openkb.agent.correction.""" +"""Tests for openkb.agent.lint_fix.""" from __future__ import annotations from unittest.mock import AsyncMock, MagicMock, patch import pytest -from openkb.agent.correction import ( - CorrectionRunResult, - build_correction_agent, +from openkb.agent.lint_fix import ( + LintFixRunResult, + build_knowledge_fix_agent, collect_related_files, - run_correction, + run_knowledge_fix, _preserve_existing_frontmatter, ) from openkb.schema import SCHEMA_MD -class TestBuildCorrectionAgent: +class TestBuildKnowledgeFixAgent: def test_agent_name(self, tmp_path): - agent = build_correction_agent(str(tmp_path), "strong-model", "concepts/topic.md") - assert agent.name == "wiki-correction" + agent = build_knowledge_fix_agent(str(tmp_path), "strong-model", "concepts/topic.md") + assert agent.name == "wiki-knowledge-fixer" def test_review_mode_has_read_only_tools(self, tmp_path): - agent = build_correction_agent(str(tmp_path), "strong-model", "concepts/topic.md") + agent = build_knowledge_fix_agent(str(tmp_path), "strong-model", "concepts/topic.md") names = {t.name for t in agent.tools} assert names == {"list_files", "read_file", "get_page_content"} def test_apply_mode_has_target_write_tool(self, tmp_path): - agent = build_correction_agent( + agent = build_knowledge_fix_agent( str(tmp_path), "strong-model", "concepts/topic.md", apply=True ) names = {t.name for t in agent.tools} assert "write_target_file" in names def test_apply_mode_instructions_preserve_frontmatter(self, tmp_path): - agent = build_correction_agent( + agent = build_knowledge_fix_agent( str(tmp_path), "strong-model", "concepts/topic.md", apply=True ) assert "keep all existing YAML frontmatter fields exactly as they are" in agent.instructions def test_schema_in_instructions(self, tmp_path): - agent = build_correction_agent(str(tmp_path), "strong-model", "concepts/topic.md") + agent = build_knowledge_fix_agent(str(tmp_path), "strong-model", "concepts/topic.md") assert SCHEMA_MD in agent.instructions def test_agent_model(self, tmp_path): - agent = build_correction_agent(str(tmp_path), "custom-model", "concepts/topic.md") + agent = build_knowledge_fix_agent(str(tmp_path), "custom-model", "concepts/topic.md") assert agent.model == "litellm/custom-model" @@ -111,7 +111,7 @@ def test_preserve_existing_frontmatter_replaces_agent_frontmatter(): assert written.endswith("# New") -class TestRunCorrection: +class TestRunKnowledgeFix: @pytest.mark.asyncio async def test_returns_final_output_and_passes_prompt_context(self, tmp_path): wiki = tmp_path / "wiki" @@ -127,15 +127,15 @@ async def fake_run(agent, message, **kwargs): captured["kwargs"] = kwargs return MagicMock(final_output="## Verdict\n\nIncorrect.") - with patch("openkb.agent.correction.Runner.run", side_effect=fake_run): - result = await run_correction( + with patch("openkb.agent.lint_fix.Runner.run", side_effect=fake_run): + result = await run_knowledge_fix( tmp_path, "concepts/topic.md", "Bad claim.", "strong-model" ) - assert isinstance(result, CorrectionRunResult) + assert isinstance(result, LintFixRunResult) assert "Incorrect" in result assert result.applied is False - assert captured["agent"].name == "wiki-correction" + assert captured["agent"].name == "wiki-knowledge-fixer" assert "Target wiki page: concepts/topic.md" in captured["message"] assert "Bad claim." in captured["message"] assert captured["kwargs"]["max_turns"] > 0 @@ -148,7 +148,7 @@ async def test_rejects_sources_as_target(self, tmp_path): (wiki / "sources" / "doc.md").write_text("Evidence.") with pytest.raises(ValueError): - await run_correction(tmp_path, "sources/doc.md", "Claim", "strong-model") + await run_knowledge_fix(tmp_path, "sources/doc.md", "Claim", "strong-model") @pytest.mark.asyncio async def test_apply_mode_builds_agent_with_write_tool(self, tmp_path): @@ -157,9 +157,9 @@ async def test_apply_mode_builds_agent_with_write_tool(self, tmp_path): (wiki / "summaries").mkdir(parents=True) (wiki / "summaries" / "doc.md").write_text("# Doc\n\nBad claim.") - with patch("openkb.agent.correction.Runner.run", new_callable=AsyncMock) as mock_run: + with patch("openkb.agent.lint_fix.Runner.run", new_callable=AsyncMock) as mock_run: mock_run.return_value = MagicMock(final_output="## Applied\n\nYes.") - await run_correction( + await run_knowledge_fix( tmp_path, "summaries/doc.md", "Bad claim.", "strong-model", apply=True ) @@ -174,11 +174,11 @@ async def test_apply_mode_reports_not_applied_when_agent_does_not_write(self, tm (wiki / "summaries").mkdir(parents=True) (wiki / "summaries" / "doc.md").write_text("# Doc\n\nAccurate claim.") - with patch("openkb.agent.correction.Runner.run", new_callable=AsyncMock) as mock_run: + with patch("openkb.agent.lint_fix.Runner.run", new_callable=AsyncMock) as mock_run: mock_run.return_value = MagicMock( final_output="## Verdict\n\nSupported.\n\n## Applied\n\nNo." ) - result = await run_correction( + result = await run_knowledge_fix( tmp_path, "summaries/doc.md", "Accurate claim.", "strong-model", apply=True ) From 71a7b1211825fff9077ae655a52d972a02e05b99 Mon Sep 17 00:00:00 2001 From: saccharin98 Date: Wed, 29 Apr 2026 17:40:22 +0800 Subject: [PATCH 3/3] support non OpenAI model for lint fix --- openkb/cli.py | 1 + tests/test_lint_cli.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/openkb/cli.py b/openkb/cli.py index dabb1e71..ee9f5e95 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -589,6 +589,7 @@ async def run_lint( model: str = config.get("model", DEFAULT_CONFIG["model"]) lint_fix_model: str = ( config.get("lint_fix_model") + or model or DEFAULT_LINT_FIX_MODEL ) diff --git a/tests/test_lint_cli.py b/tests/test_lint_cli.py index fec3a1bd..555cda30 100644 --- a/tests/test_lint_cli.py +++ b/tests/test_lint_cli.py @@ -162,6 +162,36 @@ def test_lint_fix_runs_knowledge_fix_for_user_feedback(self, tmp_path): assert data["fix_requested"] is True assert data["fix_results"][0]["status"] == "applied" + def test_lint_fix_falls_back_to_configured_model(self, tmp_path): + kb_dir = _setup_kb(tmp_path) + hashes = {"abc": {"name": "paper.pdf", "type": "pdf"}} + (kb_dir / ".openkb" / "hashes.json").write_text(json.dumps(hashes)) + (kb_dir / ".openkb" / "config.yaml").write_text( + "model: anthropic/claude-sonnet-4-6\n" + ) + (kb_dir / "wiki" / "concepts" / "topic.md").write_text("Questionable claim.") + runner = CliRunner() + + with patch("openkb.cli._find_kb_dir", return_value=kb_dir), \ + patch("openkb.cli._setup_llm_key"), \ + patch("openkb.agent.linter.run_knowledge_lint", return_value="No issues."), \ + patch("openkb.agent.lint_fix.run_knowledge_fix", new_callable=AsyncMock) as mock_fix: + mock_fix.return_value = LintFixRunResult(output="No change.", applied=False) + result = runner.invoke( + cli, + [ + "lint", + "--fix", + "--page", + "concepts/topic.md", + "--feedback", + "This claim looks wrong.", + ], + ) + + assert result.exit_code == 0 + assert mock_fix.call_args.args[3] == "anthropic/claude-sonnet-4-6" + def test_lint_page_requires_feedback(self, tmp_path): kb_dir = _setup_kb(tmp_path) runner = CliRunner()