From 73964f5bf234f9eedce6a82bd90d093a0e55283f Mon Sep 17 00:00:00 2001 From: xrendan Date: Fri, 27 Mar 2026 11:21:14 -0600 Subject: [PATCH 1/4] Replace Python agent with claude -p CLI Eliminates the Python agent entirely (claude-agent-sdk, psycopg2, MCP server) in favour of spawning `claude -p` directly from Rails jobs. Claude uses curl via Bash to call Rails API read/write endpoints, and WebFetch (restricted to canada.ca / gc.ca / parl.ca) for government pages. Key changes: - Delete agent/src/ and pyproject.toml - Add GET read endpoints: /api/agent/commitments, /api/agent/bills, /api/agent/entries (with full detail and list/filter support) - Add PATCH /api/agent/commitments/:id/touch_assessed and /api/agent/entries/:id/mark_processed for Stop hooks - Migration: add agent_processed_at to entries - AgentEvaluateCommitmentJob + AgentProcessEntryJob rewritten to spawn claude -p with streamed output, per-job Stop hooks, and explicit RAILS_MASTER_KEY unset - agent/.claude/hooks/on_stop_commitment.sh and on_stop_entry.sh guarantee last_assessed_at + agent_processed_at are updated even if the agent skips record_evaluation_run - agent/CLAUDE.md rewritten as curl-based API reference - CORS: allow all localhost origins for development - Dockerfiles: install claude CLI binary Co-Authored-By: Claude Sonnet 4.6 --- .devcontainer/Dockerfile | 5 +- Dockerfile | 4 + agent/.claude/hooks/on_stop_commitment.sh | 9 + agent/.claude/hooks/on_stop_entry.sh | 8 + agent/.claude/settings.json | 12 + agent/CLAUDE.md | 213 +++++++---- agent/pyproject.toml | 27 -- agent/src/agent/__init__.py | 0 agent/src/agent/db.py | 37 -- agent/src/agent/domain/__init__.py | 0 agent/src/agent/domain/validators.py | 28 -- agent/src/agent/evaluator.py | 351 ------------------ agent/src/agent/main.py | 98 ----- agent/src/agent/prompts.py | 219 ----------- agent/src/agent/tools/__init__.py | 0 agent/src/agent/tools/db_read.py | 310 ---------------- agent/src/agent/tools/rails_write.py | 220 ----------- agent/src/agent/tools/web_search.py | 79 ---- app/controllers/api/agent/bills_controller.rb | 56 +++ .../api/agent/commitments_controller.rb | 164 ++++++++ .../api/agent/entries_controller.rb | 57 +++ app/jobs/agent_evaluate_commitment_job.rb | 98 +++-- app/jobs/agent_process_entry_job.rb | 98 +++-- app/services/agent_prompts.rb | 205 ++++++++++ config/initializers/cors.rb | 2 +- config/routes.rb | 8 +- ...61901_add_agent_processed_at_to_entries.rb | 5 + db/schema.rb | 3 +- lib/tasks/database_restore.rake | 7 +- 29 files changed, 827 insertions(+), 1496 deletions(-) create mode 100755 agent/.claude/hooks/on_stop_commitment.sh create mode 100755 agent/.claude/hooks/on_stop_entry.sh create mode 100644 agent/.claude/settings.json delete mode 100644 agent/pyproject.toml delete mode 100644 agent/src/agent/__init__.py delete mode 100644 agent/src/agent/db.py delete mode 100644 agent/src/agent/domain/__init__.py delete mode 100644 agent/src/agent/domain/validators.py delete mode 100644 agent/src/agent/evaluator.py delete mode 100644 agent/src/agent/main.py delete mode 100644 agent/src/agent/prompts.py delete mode 100644 agent/src/agent/tools/__init__.py delete mode 100644 agent/src/agent/tools/db_read.py delete mode 100644 agent/src/agent/tools/rails_write.py delete mode 100644 agent/src/agent/tools/web_search.py create mode 100644 app/controllers/api/agent/bills_controller.rb create mode 100644 app/controllers/api/agent/entries_controller.rb create mode 100644 app/services/agent_prompts.rb create mode 100644 db/migrate/20260327161901_add_agent_processed_at_to_entries.rb diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 6df5bee..f6877c9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -43,6 +43,9 @@ RUN chown -R $USERNAME:$USERNAME /workspaces # Switch to the non-root user USER $USERNAME +# Install Claude Code (native binary) +RUN curl -fsSL https://claude.ai/install.sh | bash + # Install bundler RUN gem install bundler @@ -50,7 +53,7 @@ RUN gem install bundler ENV BUNDLE_PATH=/usr/local/bundle \ BUNDLE_BIN=/usr/local/bundle/bin \ GEM_HOME=/usr/local/bundle -ENV PATH=$BUNDLE_BIN:$PATH +ENV PATH=$BUNDLE_BIN:/home/$USERNAME/.local/bin:$PATH # Default command CMD ["/bin/bash"] diff --git a/Dockerfile b/Dockerfile index e10bd25..f238f9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,6 +77,10 @@ FROM base COPY --from=build "${BUNDLE_PATH}" "${BUNDLE_PATH}" COPY --from=build /rails /rails +# Install Claude Code (native binary) +RUN curl -fsSL https://claude.ai/install.sh | bash && \ + mv /root/.local/bin/claude /usr/local/bin/claude + # Run and own only the runtime files as a non-root user for security RUN groupadd --system --gid 1000 rails && \ useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \ diff --git a/agent/.claude/hooks/on_stop_commitment.sh b/agent/.claude/hooks/on_stop_commitment.sh new file mode 100755 index 0000000..4c442a6 --- /dev/null +++ b/agent/.claude/hooks/on_stop_commitment.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Stop hook for commitment evaluation runs. +# No-ops if the agent already called record_evaluation_run this session. + +curl -s -X PATCH "${RAILS_API_URL}/api/agent/commitments/${COMMITMENT_ID}/touch_assessed" \ + -H "Authorization: Bearer ${RAILS_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"reasoning\": \"Session ended without evaluation run — fallback timestamp update\"}" \ + > /dev/null 2>&1 || true diff --git a/agent/.claude/hooks/on_stop_entry.sh b/agent/.claude/hooks/on_stop_entry.sh new file mode 100755 index 0000000..90a852d --- /dev/null +++ b/agent/.claude/hooks/on_stop_entry.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Stop hook for entry processing runs. +# No-ops if the entry was already marked processed. + +curl -s -X PATCH "${RAILS_API_URL}/api/agent/entries/${ENTRY_ID}/mark_processed" \ + -H "Authorization: Bearer ${RAILS_API_KEY}" \ + -H "Content-Type: application/json" \ + > /dev/null 2>&1 || true diff --git a/agent/.claude/settings.json b/agent/.claude/settings.json new file mode 100644 index 0000000..57adcc5 --- /dev/null +++ b/agent/.claude/settings.json @@ -0,0 +1,12 @@ +{ + "permissions": { + "allow": [ + "Bash(curl *)", + "WebFetch(https://*.canada.ca/*)", + "WebFetch(https://*.gc.ca/*)", + "WebFetch(https://www.parl.ca/*)", + "WebSearch" + ], + "deny": [] + } +} diff --git a/agent/CLAUDE.md b/agent/CLAUDE.md index 6a29adb..60f6573 100644 --- a/agent/CLAUDE.md +++ b/agent/CLAUDE.md @@ -11,7 +11,12 @@ These tools proxy to the existing REST API endpoints and return JSON. Tool class ## Rails API Reference -Base URL: provided in system prompt. Auth: `Authorization: Bearer ` (also in system prompt). +Base URL: provided in system prompt as `$RAILS_API_URL`. Auth: `Authorization: Bearer $RAILS_API_KEY` (also in system prompt). + +Use `curl -s` for all API calls. Example pattern: +```bash +curl -s -H "Authorization: Bearer $RAILS_API_KEY" "$RAILS_API_URL/api/agent/commitments/1" +``` ### Enum Values @@ -30,96 +35,178 @@ Base URL: provided in system prompt. Auth: `Authorization: Bearer ` (also i **commitment_event.event_type** (integer enum): - `promised` (0), `mentioned` (1), `legislative_action` (2), `funding_allocated` (3), `status_change` (4), `criterion_assessed` (5) -**commitment_event.action_type** (integer enum, optional, prefix: `action_type_`): +**commitment_event.action_type** (integer enum, optional): - `announcement` (0), `concrete_action` (1) **source.source_type** (integer enum): - `platform_document` (0), `speech_from_throne` (1), `budget` (2), `press_conference` (3), `mandate_letter` (4), `debate` (5), `other` (6), `order_in_council` (7), `treasury_board_submission` (8), `gazette_notice` (9), `committee_report` (10), `departmental_results_report` (11) -### API Endpoints (all require source_url/source_urls) - -**Create commitment event** — `POST /api/agent/commitment_events` -```json -{ - "commitment_id": 2354, - "event_type": "legislative_action", - "title": "Short title", - "description": "1-3 sentence blurb", - "occurred_at": "2025-07-09", - "source_url": "https://www.canada.ca/...", - "action_type": "concrete_action" -} +--- + +## Read Endpoints (GET) + +**Commitment (full detail)** — returns criteria, matches, events, sources, departments, status_changes: +```bash +curl -s -H "Authorization: Bearer $RAILS_API_KEY" "$RAILS_API_URL/api/agent/commitments/:id" +``` + +**List commitments** — params: `status`, `policy_area`, `commitment_type`, `stale_days`, `government_id`, `limit`, `offset`: +```bash +curl -s -H "Authorization: Bearer $RAILS_API_KEY" "$RAILS_API_URL/api/agent/commitments?stale_days=7&limit=50" +``` + +**Commitment source documents** (platform, SFT, budget — use to check Budget Evidence Rule): +```bash +curl -s -H "Authorization: Bearer $RAILS_API_KEY" "$RAILS_API_URL/api/agent/commitments/:id/sources" ``` -**Assess criterion** — `PATCH /api/agent/criteria/:id` -```json -{ - "new_status": "met", - "evidence_notes": "Explanation with citations", - "source_url": "https://www.canada.ca/..." -} +**Bill (with stage dates + linked commitments)**: +```bash +curl -s -H "Authorization: Bearer $RAILS_API_KEY" "$RAILS_API_URL/api/agent/bills/:id" ``` -**Update commitment status** — `PATCH /api/agent/commitments/:id/status` -```json -{ - "new_status": "in_progress", - "reasoning": "Clear explanation based on evidence — this is shown in the UI", - "source_urls": ["https://www.canada.ca/...", "https://gazette.gc.ca/..."], - "effective_date": "2025-07-09" -} +**List bills** — param: `parliament_number` (default 45, returns government bills only): +```bash +curl -s -H "Authorization: Bearer $RAILS_API_KEY" "$RAILS_API_URL/api/agent/bills?parliament_number=45" ``` -- `reasoning` is displayed in the UI as the status change reason — make it clear and concise (1-3 sentences) -- `effective_date` **(required)** is the date the status actually changed based on evidence (e.g., when the bill was introduced, when the program launched). Use the date of the earliest source that justifies this status. Must be YYYY-MM-DD format. - -**Link bill to commitment** — `POST /api/agent/commitment_matches` -```json -{ - "commitment_id": 2354, - "matchable_type": "Bill", - "matchable_id": 40, - "relevance_score": 0.9, - "relevance_reasoning": "Why this bill implements this commitment" -} + +**Entry (with parsed_markdown)**: +```bash +curl -s -H "Authorization: Bearer $RAILS_API_KEY" "$RAILS_API_URL/api/agent/entries/:id" +``` + +**List unprocessed entries** — params: `unprocessed=true`, `government_id`, `limit`: +```bash +curl -s -H "Authorization: Bearer $RAILS_API_KEY" "$RAILS_API_URL/api/agent/entries?unprocessed=true&limit=50" +``` + +--- + +## Write Endpoints + +**Fetch + register government page** — fetches URL, converts to markdown, saves as Source. Returns `source_id` and `url`. Call this BEFORE using a URL in any judgement: +```bash +curl -s -X POST "$RAILS_API_URL/api/agent/pages/fetch" \ + -H "Authorization: Bearer $RAILS_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"url": "https://www.canada.ca/...", "government_id": 1}' ``` -**Record evaluation run** — `POST /api/agent/evaluation_runs` -```json -{ - "commitment_id": 2354, - "trigger_type": "manual", - "reasoning": "Summary of evaluation", - "previous_status": "not_started", - "new_status": "in_progress", - "criteria_assessed": 8, - "evidence_found": 5 -} +**Create commitment event**: +```bash +curl -s -X POST "$RAILS_API_URL/api/agent/commitment_events" \ + -H "Authorization: Bearer $RAILS_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "commitment_id": 2354, + "event_type": "legislative_action", + "title": "Short title", + "description": "1-3 sentence blurb", + "occurred_at": "2025-07-09", + "source_url": "https://www.canada.ca/...", + "action_type": "concrete_action" + }' ``` +**Assess criterion**: +```bash +curl -s -X PATCH "$RAILS_API_URL/api/agent/criteria/:id" \ + -H "Authorization: Bearer $RAILS_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "new_status": "met", + "evidence_notes": "Explanation with citations", + "source_url": "https://www.canada.ca/..." + }' +``` + +**Update commitment status**: +```bash +curl -s -X PATCH "$RAILS_API_URL/api/agent/commitments/:id/status" \ + -H "Authorization: Bearer $RAILS_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "new_status": "in_progress", + "reasoning": "Clear explanation based on evidence — shown in the UI", + "source_urls": ["https://www.canada.ca/..."], + "effective_date": "2025-07-09" + }' +``` +- `reasoning` is displayed to users — make it clear and concise (1-3 sentences) +- `effective_date` **(required)** — the date the status actually changed based on evidence (YYYY-MM-DD) + +**Link bill to commitment**: +```bash +curl -s -X POST "$RAILS_API_URL/api/agent/commitment_matches" \ + -H "Authorization: Bearer $RAILS_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "commitment_id": 2354, + "matchable_type": "Bill", + "matchable_id": 40, + "relevance_score": 0.9, + "relevance_reasoning": "Why this bill implements this commitment" + }' +``` + +**Record evaluation run** (required at end of every evaluation — also updates `last_assessed_at`): +```bash +curl -s -X POST "$RAILS_API_URL/api/agent/evaluation_runs" \ + -H "Authorization: Bearer $RAILS_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "commitment_id": 2354, + "trigger_type": "manual", + "reasoning": "Summary of what was evaluated and found", + "previous_status": "not_started", + "new_status": "in_progress", + "criteria_assessed": 8, + "evidence_found": 5 + }' +``` + +--- + +## Fetching Government Pages + +Use the **WebFetch tool** to read official page content (allowed domains: `*.canada.ca`, `*.gc.ca`, `www.parl.ca`). + +Then register the page as a Source: +```bash +curl -s -X POST "$RAILS_API_URL/api/agent/pages/fetch" \ + -H "Authorization: Bearer $RAILS_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"url": "https://...", "government_id": 1}' +``` + +You MUST register a page before referencing its URL in any judgement (assess_criterion, create_commitment_event, update_commitment_status). + +--- + ## Database Schema (key tables) -**commitments**: id, government_id, title, description, original_text, commitment_type (enum), status (enum), date_promised, target_date, last_assessed_at, policy_area_id, metadata (jsonb) +**commitments**: id, government_id, title, description, original_text, commitment_type (enum), status (enum), date_promised, target_date, last_assessed_at, policy_area_id **criteria**: id, commitment_id, category (enum), description, verification_method, status (enum), evidence_notes, assessed_at, position -**criterion_assessments**: id, criterion_id, previous_status, new_status, source_id, evidence_notes, assessed_at - **commitment_matches**: id, commitment_id, matchable_type, matchable_id, relevance_score, relevance_reasoning, matched_at, assessed -**commitment_events**: id, commitment_id, source_id, event_type (enum), action_type (enum), title, description, occurred_at, metadata (jsonb) +**commitment_events**: id, commitment_id, source_id, event_type (enum), action_type (enum), title, description, occurred_at **bills**: id, bill_id, bill_number_formatted, parliament_number, short_title, long_title, latest_activity, passed_house_first_reading_at, passed_house_second_reading_at, passed_house_third_reading_at, passed_senate_first_reading_at, passed_senate_second_reading_at, passed_senate_third_reading_at, received_royal_assent_at **sources**: id, government_id, source_type (enum), title, url, date -**entries**: id, feed_id, title, published_at, url, scraped_at, parsed_markdown, activities_extracted_at, government_id +**entries**: id, feed_id, title, published_at, url, scraped_at, parsed_markdown, agent_processed_at, government_id **governments**: id=1 is the current Carney government (45th Parliament) +--- + ## Rules -- Do NOT use Read, Glob, Grep, or filesystem tools to explore the Rails codebase. Everything you need is above. -- Use the MCP tools (get_commitment, get_bill, etc.) for reading data. -- Use curl via Bash for ALL write operations. -- Every judgement (assess_criterion, create_commitment_event, update_commitment_status) MUST include a source_url that was previously fetched via fetch_government_page. -- Fetch pages BEFORE referencing them. The fetch auto-registers them as Sources in the DB. +- Do NOT use Read, Glob, Grep, or filesystem tools. Everything you need is above. +- Use `curl -s` via Bash for ALL API calls (reads and writes). +- Use WebFetch for reading government page content (canada.ca / gc.ca / parl.ca only). +- Every judgement (assess_criterion, create_commitment_event, update_commitment_status) MUST include a source_url that was first registered via `POST /api/agent/pages/fetch`. +- Always call `POST /api/agent/evaluation_runs` at the end of every commitment evaluation with a reasoning summary. diff --git a/agent/pyproject.toml b/agent/pyproject.toml deleted file mode 100644 index 8ca645c..0000000 --- a/agent/pyproject.toml +++ /dev/null @@ -1,27 +0,0 @@ -[project] -name = "commitment-agent" -version = "0.1.0" -description = "Claude agent for end-to-end commitment evaluation" -requires-python = ">=3.11" -dependencies = [ - "claude-agent-sdk", - "psycopg2-binary>=2.9.9", - "httpx>=0.27.0", - "pydantic>=2.0.0", - "click>=8.1.0", - "beautifulsoup4>=4.12.0", - "markdownify>=0.13.0", -] - -[project.optional-dependencies] -dev = [ - "pytest>=8.0.0", - "pytest-asyncio>=0.23.0", -] - -[build-system] -requires = ["setuptools>=68.0"] -build-backend = "setuptools.build_meta" - -[tool.setuptools.packages.find] -where = ["src"] diff --git a/agent/src/agent/__init__.py b/agent/src/agent/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/agent/src/agent/db.py b/agent/src/agent/db.py deleted file mode 100644 index c542daf..0000000 --- a/agent/src/agent/db.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -from contextlib import contextmanager - -import psycopg2 -import psycopg2.extras - - -def get_connection_string() -> str: - return os.environ["AGENT_DATABASE_URL"] - - -@contextmanager -def get_db(): - """Context manager for read-only database connection.""" - conn = psycopg2.connect( - get_connection_string(), - cursor_factory=psycopg2.extras.RealDictCursor, - ) - conn.set_session(readonly=True, autocommit=True) - try: - yield conn - finally: - conn.close() - - -def query(sql: str, params: tuple | None = None) -> list[dict]: - """Execute a read-only query and return results as list of dicts.""" - with get_db() as conn: - with conn.cursor() as cur: - cur.execute(sql, params) - return [dict(row) for row in cur.fetchall()] - - -def query_one(sql: str, params: tuple | None = None) -> dict | None: - """Execute a read-only query and return a single result.""" - results = query(sql, params) - return results[0] if results else None diff --git a/agent/src/agent/domain/__init__.py b/agent/src/agent/domain/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/agent/src/agent/domain/validators.py b/agent/src/agent/domain/validators.py deleted file mode 100644 index 19c8c00..0000000 --- a/agent/src/agent/domain/validators.py +++ /dev/null @@ -1,28 +0,0 @@ -from urllib.parse import urlparse - -ALLOWED_SUFFIXES = (".canada.ca", ".gc.ca") - - -def is_government_url(url: str) -> bool: - """Check if a URL belongs to an allowed Canadian government domain.""" - try: - parsed = urlparse(url) - hostname = parsed.hostname - if hostname is None: - return False - return any( - hostname == suffix.lstrip(".") or hostname.endswith(suffix) - for suffix in ALLOWED_SUFFIXES - ) - except Exception: - return False - - -def validate_government_url(url: str) -> str: - """Validate and return the URL, or raise ValueError if not a government domain.""" - if not is_government_url(url): - raise ValueError( - f"URL rejected: {url} is not on an allowed government domain " - f"(must end with {' or '.join(ALLOWED_SUFFIXES)})" - ) - return url diff --git a/agent/src/agent/evaluator.py b/agent/src/agent/evaluator.py deleted file mode 100644 index e35d6d3..0000000 --- a/agent/src/agent/evaluator.py +++ /dev/null @@ -1,351 +0,0 @@ -"""Core agent evaluator — drives commitment evaluation using the Claude Agent SDK.""" - -import asyncio -import json -import os -import pathlib -import sys -import traceback -from datetime import date -from typing import Any - -from claude_agent_sdk import ( - AssistantMessage, - ClaudeAgentOptions, - ResultMessage, - ToolAnnotations, - create_sdk_mcp_server, - query, - tool, -) - -from agent.prompts import ( - EVALUATE_COMMITMENT_PROMPT, - PROCESS_BILL_CHANGE_PROMPT, - PROCESS_ENTRY_PROMPT, - SYSTEM_PROMPT, - WEEKLY_SCAN_PROMPT, -) -from agent.tools.db_read import ( - get_bill, - get_bills_for_parliament, - get_commitment, - get_commitment_sources, - get_entry, - list_commitments, - list_unprocessed_entries, -) -from agent.tools.web_search import fetch_government_page -from agent.tools.rails_write import register_source - - -def _tool_log(tool_name: str, msg: str) -> None: - print(f" [{tool_name}] {msg}", file=sys.stderr, flush=True) - - -def _tool_result(data: dict, tool_name: str) -> dict[str, Any]: - text = json.dumps(data, default=str) - _tool_log(tool_name, f"OK ({len(text)} chars)") - return {"content": [{"type": "text", "text": text}]} - - -def _tool_error(e: Exception, tool_name: str, args: dict) -> dict[str, Any]: - tb = traceback.format_exc() - args_preview = json.dumps(args, default=str)[:300] - tb_preview = tb[-500:] - _tool_log(tool_name, f"ERROR: {type(e).__name__}: {e}") - _tool_log(tool_name, f" args: {args_preview}") - _tool_log(tool_name, f" traceback:\n{tb_preview}") - error_detail = f"Error calling {tool_name}: {type(e).__name__}: {e}\nArgs: {args_preview}" - return { - "content": [{"type": "text", "text": error_detail}], - "is_error": True, - } - - -# ── Read-only DB tools (via MCP) ─────────────────────────────────────────── - -@tool( - "get_commitment", - "Fetch a commitment with its criteria, matches, events, linked bills, departments, and source documents.", - {"commitment_id": int}, - annotations=ToolAnnotations(readOnlyHint=True), -) -async def get_commitment_tool(args: dict[str, Any]) -> dict[str, Any]: - _tool_log("get_commitment", f"Loading commitment {args['commitment_id']}") - try: - return _tool_result(get_commitment(args["commitment_id"]), "get_commitment") - except Exception as e: - return _tool_error(e, "get_commitment", args) - - -@tool( - "list_commitments", - "List commitments with optional filters. Params: status, policy_area, commitment_type, stale_days, limit.", - { - "type": "object", - "properties": { - "status": {"type": "string", "enum": ["not_started", "in_progress", "completed", "broken"]}, - "policy_area": {"type": "string", "description": "Policy area slug"}, - "commitment_type": {"type": "string"}, - "stale_days": {"type": "integer"}, - "limit": {"type": "integer"}, - }, - }, - annotations=ToolAnnotations(readOnlyHint=True), -) -async def list_commitments_tool(args: dict[str, Any]) -> dict[str, Any]: - _tool_log("list_commitments", f"filters: {args}") - try: - return _tool_result(list_commitments(**args), "list_commitments") - except Exception as e: - return _tool_error(e, "list_commitments", args) - - -@tool( - "get_bill", - "Fetch a bill with all stage dates (House/Senate readings, Royal Assent) and linked commitments.", - {"bill_id": int}, - annotations=ToolAnnotations(readOnlyHint=True), -) -async def get_bill_tool(args: dict[str, Any]) -> dict[str, Any]: - _tool_log("get_bill", f"Loading bill {args['bill_id']}") - try: - return _tool_result(get_bill(args["bill_id"]), "get_bill") - except Exception as e: - return _tool_error(e, "get_bill", args) - - -@tool( - "get_entry", - "Fetch a scraped entry (news article, gazette item) with parsed content.", - {"entry_id": int}, - annotations=ToolAnnotations(readOnlyHint=True), -) -async def get_entry_tool(args: dict[str, Any]) -> dict[str, Any]: - _tool_log("get_entry", f"Loading entry {args['entry_id']}") - try: - return _tool_result(get_entry(args["entry_id"]), "get_entry") - except Exception as e: - return _tool_error(e, "get_entry", args) - - -@tool( - "list_unprocessed_entries", - "List entries that have been scraped but not yet evaluated by the agent.", - {"type": "object", "properties": {"limit": {"type": "integer"}}}, - annotations=ToolAnnotations(readOnlyHint=True), -) -async def list_unprocessed_entries_tool(args: dict[str, Any]) -> dict[str, Any]: - _tool_log("list_unprocessed_entries", f"args: {args}") - try: - return _tool_result(list_unprocessed_entries(**args), "list_unprocessed_entries") - except Exception as e: - return _tool_error(e, "list_unprocessed_entries", args) - - -@tool( - "get_commitment_sources", - "Get the source documents (platform, Speech from the Throne, budget) for a commitment. Use this to determine where a commitment originated for the budget evidence rule.", - {"commitment_id": int}, - annotations=ToolAnnotations(readOnlyHint=True), -) -async def get_commitment_sources_tool(args: dict[str, Any]) -> dict[str, Any]: - _tool_log("get_commitment_sources", f"commitment {args['commitment_id']}") - try: - return _tool_result(get_commitment_sources(args["commitment_id"]), "get_commitment_sources") - except Exception as e: - return _tool_error(e, "get_commitment_sources", args) - - -@tool( - "get_bills_for_parliament", - "Get all government bills for a parliament session with their stage dates.", - {"type": "object", "properties": {"parliament_number": {"type": "integer"}}}, - annotations=ToolAnnotations(readOnlyHint=True), -) -async def get_bills_for_parliament_tool(args: dict[str, Any]) -> dict[str, Any]: - pn = args.get("parliament_number", 45) - _tool_log("get_bills_for_parliament", f"parliament {pn}") - try: - return _tool_result(get_bills_for_parliament(pn), "get_bills_for_parliament") - except Exception as e: - return _tool_error(e, "get_bills_for_parliament", args) - - -@tool( - "fetch_government_page", - "Fetch and parse content from an official Canadian government webpage (*.canada.ca / *.gc.ca only). " - "The page is automatically saved as a Source in the database. You MUST fetch pages before using their " - "URLs in write operations (assess_criterion, update_commitment_status, create_commitment_event).", - { - "type": "object", - "properties": { - "url": {"type": "string", "description": "The government page URL to fetch"}, - "government_id": {"type": "integer", "description": "Government ID (usually 1)"}, - }, - "required": ["url", "government_id"], - }, - annotations=ToolAnnotations(openWorldHint=True), -) -async def fetch_government_page_tool(args: dict[str, Any]) -> dict[str, Any]: - url = args.get("url", "") - gov_id = args.get("government_id", 1) - _tool_log("fetch", f"GET {url[:100]}") - try: - result = fetch_government_page(url) - if "error" in result: - _tool_log("fetch", f"FAILED: {result['error']}") - return { - "content": [{"type": "text", "text": f"Fetch failed for {url}: {result['error']}"}], - "is_error": True, - } - - _tool_log("fetch", f"OK {len(result.get('content_markdown', ''))} chars — registering source...") - - source_result = register_source( - government_id=gov_id, - url=result.get("url", url), - title=result.get("title", ""), - date=result.get("published_date"), - ) - if "error" in source_result: - _tool_log("fetch", f"SOURCE REGISTRATION FAILED: {source_result['error']}") - # Still return the content even if source registration failed - result["source_id"] = None - result["source_error"] = source_result["error"] - else: - source_id = source_result.get("id") - existed = source_result.get("existed", False) - _tool_log("fetch", f"source_id={source_id} {'(existed)' if existed else '(created)'}") - result["source_id"] = source_id - - return {"content": [{"type": "text", "text": json.dumps(result, default=str)}]} - except Exception as e: - return _tool_error(e, "fetch_government_page", args) - - -# ── MCP Server (read tools + fetch only) ─────────────────────────────────── - -ALL_TOOLS = [ - get_commitment_tool, - list_commitments_tool, - get_bill_tool, - get_entry_tool, - list_unprocessed_entries_tool, - get_commitment_sources_tool, - get_bills_for_parliament_tool, - fetch_government_page_tool, -] - -tracker_server = create_sdk_mcp_server( - name="tracker", - version="1.0.0", - tools=ALL_TOOLS, -) - -ALLOWED_TOOLS = [f"mcp__tracker__{t.name}" for t in ALL_TOOLS] + ["Bash", "WebSearch"] - - -# ── Agent runner ──────────────────────────────────────────────────────────── - -def _build_options() -> ClaudeAgentOptions: - rails_url = os.environ.get("RAILS_API_URL", "http://localhost:3000") - rails_key = os.environ.get("RAILS_API_KEY", "") - - api_context = ( - f"\n\n## Rails API Connection\n" - f"Base URL: `{rails_url}`\n" - f"Auth header: `Authorization: Bearer {rails_key}`\n" - f"Use `curl -s` via Bash for all write operations. " - f"See CLAUDE.md for endpoint details and enum values.\n" - ) - - model = os.environ.get("AGENT_MODEL", "claude-sonnet-4-6") - - return ClaudeAgentOptions( - model=model, - system_prompt=SYSTEM_PROMPT + api_context, - mcp_servers={"tracker": tracker_server}, - allowed_tools=ALLOWED_TOOLS, - permission_mode="bypassPermissions", - cwd=str(pathlib.Path(__file__).resolve().parent.parent.parent), # agent/ dir where CLAUDE.md lives - setting_sources=["project"], - ) - - -async def _run_agent(prompt: str, as_of_date: str | None = None) -> str: - """Run the agent loop and return the final result text.""" - import time as _time - - current_date = as_of_date or date.today().isoformat() - formatted_prompt = prompt.format(current_date=current_date) - options = _build_options() - - start = _time.time() - tool_count = 0 - result_text = "" - - def _log(msg: str) -> None: - elapsed = _time.time() - start - print(f"[{elapsed:6.1f}s] {msg}", flush=True) - - _log("Starting agent...") - _log(f"Prompt: {formatted_prompt[:120]}...") - - async for message in query(prompt=formatted_prompt, options=options): - elapsed = _time.time() - start - msg_type = type(message).__name__ - - if isinstance(message, AssistantMessage): - for block in message.content: - if hasattr(block, "text") and block.text: - preview = block.text[:200].replace("\n", " ") - _log(f"💬 {preview}{'...' if len(block.text) > 200 else ''}") - elif hasattr(block, "name"): - tool_count += 1 - tool_input = getattr(block, "input", {}) - input_preview = json.dumps(tool_input, default=str)[:100] - _log(f"🔧 [{tool_count}] {block.name}({input_preview})") - elif isinstance(message, ResultMessage): - if message.subtype == "success": - result_text = message.result or "" - _log(f"✅ Done — {tool_count} tool calls, {elapsed:.1f}s total") - else: - result_text = f"Agent ended with: {message.subtype}" - _log(f"⚠️ Ended: {message.subtype}") - else: - subtype = getattr(message, "subtype", "") - if subtype == "init": - session_id = getattr(message, "session_id", "?") - _log(f"🚀 Session initialized: {session_id}") - - return result_text - - -def evaluate_commitment(commitment_id: int, as_of_date: str | None = None) -> str: - prompt = EVALUATE_COMMITMENT_PROMPT.format( - commitment_id=commitment_id, current_date="{current_date}", - ) - return asyncio.run(_run_agent(prompt, as_of_date)) - - -def process_entry(entry_id: int, as_of_date: str | None = None) -> str: - prompt = PROCESS_ENTRY_PROMPT.format( - entry_id=entry_id, current_date="{current_date}", - ) - return asyncio.run(_run_agent(prompt, as_of_date)) - - -def process_bill_change(bill_id: int, as_of_date: str | None = None) -> str: - prompt = PROCESS_BILL_CHANGE_PROMPT.format( - bill_id=bill_id, current_date="{current_date}", - ) - return asyncio.run(_run_agent(prompt, as_of_date)) - - -def weekly_scan_commitment(commitment_id: int, as_of_date: str | None = None) -> str: - prompt = WEEKLY_SCAN_PROMPT.format( - commitment_id=commitment_id, current_date="{current_date}", - ) - return asyncio.run(_run_agent(prompt, as_of_date)) diff --git a/agent/src/agent/main.py b/agent/src/agent/main.py deleted file mode 100644 index 1edb4df..0000000 --- a/agent/src/agent/main.py +++ /dev/null @@ -1,98 +0,0 @@ -"""CLI entry point for the commitment evaluation agent.""" - -import sys -import time -import uuid - -import click - -from agent.evaluator import ( - evaluate_commitment, - process_bill_change, - process_entry, - weekly_scan_commitment, -) -from agent.tools.db_read import list_commitments - - -@click.group() -def cli(): - """Build Canada Commitment Evaluation Agent.""" - pass - - -@cli.command() -@click.option("--commitment-id", required=True, type=int, help="Commitment ID to evaluate") -@click.option("--as-of", default=None, help="Evaluate as of this date (YYYY-MM-DD) for backfilling") -def evaluate(commitment_id: int, as_of: str | None): - """Evaluate a single commitment end-to-end.""" - click.echo(f"Evaluating commitment {commitment_id}...") - start = time.time() - result = evaluate_commitment(commitment_id, as_of_date=as_of) - elapsed = time.time() - start - click.echo(f"\n{result}") - click.echo(f"\nCompleted in {elapsed:.1f}s") - - -@cli.command("process-entry") -@click.option("--entry-id", required=True, type=int, help="Entry ID to process") -@click.option("--as-of", default=None, help="Process as of this date (YYYY-MM-DD)") -def process_entry_cmd(entry_id: int, as_of: str | None): - """Process a new scraped entry and match to commitments.""" - click.echo(f"Processing entry {entry_id}...") - start = time.time() - result = process_entry(entry_id, as_of_date=as_of) - elapsed = time.time() - start - click.echo(f"\n{result}") - click.echo(f"\nCompleted in {elapsed:.1f}s") - - -@cli.command("process-bill-change") -@click.option("--bill-id", required=True, type=int, help="Bill database ID") -@click.option("--as-of", default=None, help="Process as of this date (YYYY-MM-DD)") -def process_bill_change_cmd(bill_id: int, as_of: str | None): - """Process a bill stage change and re-evaluate linked commitments.""" - click.echo(f"Processing bill change for bill {bill_id}...") - start = time.time() - result = process_bill_change(bill_id, as_of_date=as_of) - elapsed = time.time() - start - click.echo(f"\n{result}") - click.echo(f"\nCompleted in {elapsed:.1f}s") - - -@cli.command("scan-all") -@click.option("--limit", default=100, help="Max commitments to evaluate per run") -@click.option("--status", default=None, help="Filter by status") -@click.option("--policy-area", default=None, help="Filter by policy area slug") -@click.option("--as-of", default=None, help="Evaluate as of this date (YYYY-MM-DD)") -def scan_all(limit: int, status: str | None, policy_area: str | None, as_of: str | None): - """Weekly proactive scan — evaluate all commitments.""" - click.echo("Starting weekly scan...") - run_id = str(uuid.uuid4()) - - commitments = list_commitments( - status=status, - policy_area=policy_area, - limit=limit, - ) - - click.echo(f"Found {len(commitments)} commitments to evaluate (run {run_id})") - - for i, c in enumerate(commitments, 1): - cid = c["id"] - title = c.get("title", "")[:60] - click.echo(f"\n[{i}/{len(commitments)}] Commitment {cid}: {title}") - - try: - start = time.time() - result = weekly_scan_commitment(cid, as_of_date=as_of) - elapsed = time.time() - start - click.echo(f" Done in {elapsed:.1f}s") - except Exception as e: - click.echo(f" ERROR: {e}", err=True) - - click.echo(f"\nWeekly scan complete. Evaluated {len(commitments)} commitments.") - - -if __name__ == "__main__": - cli() diff --git a/agent/src/agent/prompts.py b/agent/src/agent/prompts.py deleted file mode 100644 index 44050cd..0000000 --- a/agent/src/agent/prompts.py +++ /dev/null @@ -1,219 +0,0 @@ -"""System prompts for the commitment evaluation agent.""" - -SYSTEM_PROMPT = """\ -You are a government accountability analyst for Build Canada. You are the single -source of truth for evaluating the Canadian federal government's progress on its -commitments from the 2025 Liberal platform, Speech from the Throne, and Budget 2025. - -## Your Responsibilities - -1. Review existing evidence linked to each commitment -2. Proactively search official government sources (*.canada.ca, *.gc.ca) for new evidence -3. Link bills to commitments when they implement or affect a commitment -4. Assess each criterion against available evidence -5. Derive commitment status using the evidence hierarchy -6. Create events explaining how each piece of evidence affects the commitment - -## Status Definitions - -- **not_started**: No evidence of meaningful government action -- **in_progress**: Concrete steps underway (legislation introduced, funding allocated, \ -program design started) -- **completed**: Commitment fulfilled (legislation enacted, program operational, \ -objectives substantially achieved) -- **broken**: Government took a policy position counter to the commitment, OR the \ -commitment had a specific deadline that has passed without completion - -## Evidence Hierarchy (strictly enforced) - -**"completed" requires one of:** -- Bill with Royal Assent that implements the commitment -- Canada Gazette Part II/III entry (enacted regulation) -- Departmental evidence confirming a program is operational - -**"in_progress" requires one of:** -- Bill introduced and progressing in Parliament (no Royal Assent yet) -- Canada Gazette Part I entry (proposed regulation) -- Appropriation voted with program implementation evidence -- Budget allocation for a **pre-existing** commitment (see Budget Rule below) - -**"not_started":** -- No evidence of action, OR only announcements without concrete follow-through - -## Budget Evidence Rule - -This rule is critical for accurate assessment: - -- The Budget **CANNOT** be used as evidence that a commitment **made in the budget \ -itself** is in progress. Including something in the budget is announcing it, not \ -implementing it. This is circular reasoning. -- The Budget **CAN** be used as evidence for commitments from the **platform or \ -Speech from the Throne** that pre-date the budget. If the platform promised X and \ -Budget 2025 allocates funding for X, that IS evidence of progress. -- Check the commitment's source documents to determine where it originated. - -## Criteria Assessment - -Each commitment has criteria across four categories: -- **Completion**: Did the government literally do what it said? ("the letter") -- **Success**: Did the real-world outcome materialize? ("the spirit") -- **Progress**: Are they actively working toward it? -- **Failure**: Has the commitment been broken or contradicted? - -**Important:** Criteria are a structured guide, NOT a rigid checklist: -- A commitment can be marked "completed" even if the criteria don't exactly match \ -the commitment text — what matters is whether the government fulfilled the spirit \ -of the commitment -- If evidence shows the commitment was fulfilled through a different mechanism than \ -the criteria anticipated, that's still "completed" -- Assess criteria based on available evidence; mark as "not_assessed" if insufficient \ -evidence exists - -Criterion statuses: not_assessed, met, not_met, no_longer_applicable - -## Bill Tracking - -- When a bill progresses through stages (readings, Royal Assent), evaluate its impact \ -on linked commitments -- Track stages: House 1R → 2R → 3R → Senate 1R → 2R → 3R → Royal Assent -- If enacted legislation text diverges from what was promised, note this in the \ -commitment event description -- A bill at Royal Assent that implements the commitment = strong completion evidence - -## Date Awareness - -You must always consider dates: -- Only use evidence that existed at the evaluation date -- When backfilling, don't use future evidence to justify past status -- Respect commitment target_date — if the date has passed without completion, \ -evaluate for "broken" status -- More recent evidence should be weighted more heavily -- Note the publication date of every source you cite - -## Source Requirement (CRITICAL) - -Every judgement you make MUST be backed by a source. The workflow is: - -1. **Search** for evidence using WebSearch (site:canada.ca OR site:gc.ca) -2. **Fetch** relevant pages using fetch_government_page — this automatically \ -saves the page as a Source in the database and returns a source_id -3. **Use the fetched URL** when making judgements: - - assess_criterion requires source_url - - create_commitment_event requires source_url - - update_commitment_status requires source_urls (array) - -You MUST fetch a page before referencing its URL in a judgement. The system \ -will reject judgements that reference URLs not in the database. - -## When You Find Evidence - -For every piece of evidence that tangibly affects a commitment's implementation: -1. **Fetch the page** using fetch_government_page (this auto-saves the source) -2. Create a CommitmentEvent with a blurb explaining WHY this evidence \ -moves the commitment forward, backward, or is neutral — include the source_url -3. Assess the relevant criteria against the new evidence — include the source_url -4. Re-evaluate the overall commitment status if warranted — include all source_urls - -## Search Strategy - -When evaluating a commitment, consider searching for: -- The commitment's keywords on canada.ca -- Related bills on parl.ca -- Budget 2025 provisions (for platform commitments only) -- Canada Gazette publications for regulatory changes -- Departmental plans and reports from the responsible department -- News releases from the responsible minister - -Use the WebSearch tool with "site:canada.ca" or "site:gc.ca" to restrict results \ -to official government sources. Then fetch the relevant pages to create sources. - -## Status Change Requirements - -When updating a commitment's status, you MUST provide: -- **reasoning**: A clear 1-3 sentence explanation shown in the UI to users. \ -Cite the specific evidence (bill number, program name, gazette reference). -- **effective_date**: The real-world date the status actually changed — NOT today's \ -date. Use the date of the earliest evidence that justifies the new status. \ -For example, if a bill was introduced on 2025-06-15 that moves a commitment to \ -"in_progress", the effective_date is 2025-06-15. -- **source_urls**: All source URLs that justify the status change. - -## Output Guidelines - -- Be factual and evidence-based -- Cite specific evidence (bill numbers, gazette references, program names, URLs) -- Note evidence gaps that could change the assessment -- Keep event descriptions concise but informative (1-3 sentences) -- If evidence is ambiguous, explain the ambiguity -""" - -EVALUATE_COMMITMENT_PROMPT = """\ -Evaluate commitment #{commitment_id}. - -Use the get_commitment tool to load the full commitment details, criteria, existing \ -evidence matches, and events. Then: - -1. Review the existing evidence and criteria assessments -2. Search for any new evidence on government sources that may have been missed -3. For any new evidence found, create commitment events explaining its impact -4. Assess each criterion that can be assessed with available evidence -5. Determine the correct status based on the evidence hierarchy -6. If the status should change, update it with clear reasoning -7. Record an evaluation run for audit purposes - -Current date: {current_date} -""" - -PROCESS_ENTRY_PROMPT = """\ -A new entry has been scraped from an RSS feed. - -Use the get_entry tool to read entry #{entry_id}. Then: - -1. Read the entry content carefully -2. Determine which commitments this entry is relevant to -3. For each relevant commitment: - a. Link the entry via a CommitmentMatch if it provides evidence - b. Create a CommitmentEvent explaining how this evidence affects the commitment - c. Assess any criteria that this evidence helps evaluate - d. Update the commitment status if warranted -4. Record evaluation runs for each affected commitment - -Current date: {current_date} -""" - -PROCESS_BILL_CHANGE_PROMPT = """\ -Bill #{bill_id} has had a stage change. - -Use the get_bill tool to read the bill details and its linked commitments. Then: - -1. Review what stage the bill has reached -2. For each commitment linked to this bill: - a. Create a CommitmentEvent noting the bill's progress - b. Evaluate if this stage change affects the commitment's status - c. If the bill has received Royal Assent, strongly consider if the commitment \ -is now "completed" - d. Assess relevant criteria - e. Update status if warranted -3. If the bill is a government bill but NOT yet linked to any commitments, search \ -for commitments it may implement and create links -4. Record evaluation runs for each affected commitment - -Current date: {current_date} -""" - -WEEKLY_SCAN_PROMPT = """\ -Perform a weekly proactive scan of commitment #{commitment_id}. - -Use the get_commitment tool to load the full details. Then: - -1. Review the current status and when it was last assessed -2. Search government sources for any new evidence since the last assessment -3. Check if any linked bills have progressed -4. For any new evidence found, create commitment events and assess criteria -5. Check if the target_date has passed — if so and the commitment is not completed, \ -evaluate for "broken" status -6. Update status if evidence warrants it -7. Record an evaluation run - -Current date: {current_date} -""" diff --git a/agent/src/agent/tools/__init__.py b/agent/src/agent/tools/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/agent/src/agent/tools/db_read.py b/agent/src/agent/tools/db_read.py deleted file mode 100644 index 251d63f..0000000 --- a/agent/src/agent/tools/db_read.py +++ /dev/null @@ -1,310 +0,0 @@ -"""Database read tools — direct read-only Postgres queries.""" - -import json - -from agent.db import query, query_one - - -def get_commitment(commitment_id: int) -> dict: - """Fetch a commitment with its criteria, matches, events, linked bills, and sources.""" - commitment = query_one( - """ - SELECT c.*, pa.name AS policy_area_name, pa.slug AS policy_area_slug, - g.name AS government_name - FROM commitments c - LEFT JOIN policy_areas pa ON pa.id = c.policy_area_id - LEFT JOIN governments g ON g.id = c.government_id - WHERE c.id = %s - """, - (commitment_id,), - ) - if not commitment: - return {"error": f"Commitment {commitment_id} not found"} - - criteria = query( - """ - SELECT id, category, description, verification_method, status, - evidence_notes, assessed_at, position - FROM criteria - WHERE commitment_id = %s - ORDER BY category, position - """, - (commitment_id,), - ) - - matches = query( - """ - SELECT cm.id, cm.matchable_type, cm.matchable_id, cm.relevance_score, - cm.relevance_reasoning, cm.matched_at, cm.assessed, - CASE - WHEN cm.matchable_type = 'Bill' THEN b.bill_number_formatted - WHEN cm.matchable_type = 'Entry' THEN e.title - ELSE NULL - END AS matchable_title, - CASE - WHEN cm.matchable_type = 'Bill' THEN b.short_title - WHEN cm.matchable_type = 'Entry' THEN e.url - ELSE NULL - END AS matchable_detail - FROM commitment_matches cm - LEFT JOIN bills b ON cm.matchable_type = 'Bill' AND b.id = cm.matchable_id - LEFT JOIN entries e ON cm.matchable_type = 'Entry' AND e.id = cm.matchable_id - WHERE cm.commitment_id = %s - ORDER BY cm.relevance_score DESC - """, - (commitment_id,), - ) - - events = query( - """ - SELECT id, event_type, action_type, title, description, occurred_at, metadata - FROM commitment_events - WHERE commitment_id = %s - ORDER BY occurred_at DESC - LIMIT 50 - """, - (commitment_id,), - ) - - sources = query( - """ - SELECT cs.id, cs.section, cs.reference, cs.excerpt, cs.relevance_note, - s.title AS source_title, s.source_type, s.url AS source_url, s.date AS source_date - FROM commitment_sources cs - JOIN sources s ON s.id = cs.source_id - WHERE cs.commitment_id = %s - """, - (commitment_id,), - ) - - departments = query( - """ - SELECT d.id, d.slug, d.display_name, d.official_name, cd.is_lead - FROM commitment_departments cd - JOIN departments d ON d.id = cd.department_id - WHERE cd.commitment_id = %s - ORDER BY cd.is_lead DESC - """, - (commitment_id,), - ) - - status_changes = query( - """ - SELECT previous_status, new_status, changed_at, reason - FROM commitment_status_changes - WHERE commitment_id = %s - ORDER BY changed_at DESC - LIMIT 10 - """, - (commitment_id,), - ) - - commitment["criteria"] = criteria - commitment["matches"] = matches - commitment["events"] = events - commitment["sources"] = sources - commitment["departments"] = departments - commitment["status_changes"] = status_changes - - return _serialize(commitment) - - -def list_commitments( - status: str | None = None, - policy_area: str | None = None, - commitment_type: str | None = None, - stale_days: int | None = None, - government_id: int | None = None, - limit: int = 50, - offset: int = 0, -) -> list[dict]: - """List commitments with optional filters.""" - conditions = [] - params = [] - - if status: - # Map status name to integer - status_map = {"not_started": 0, "in_progress": 1, "completed": 2, "broken": 4} - if status in status_map: - conditions.append("c.status = %s") - params.append(status_map[status]) - - if policy_area: - conditions.append("pa.slug = %s") - params.append(policy_area) - - if commitment_type: - type_map = { - "legislative": 0, "spending": 1, "procedural": 2, - "institutional": 3, "diplomatic": 4, "aspirational": 5, "outcome": 6, - } - if commitment_type in type_map: - conditions.append("c.commitment_type = %s") - params.append(type_map[commitment_type]) - - if stale_days: - conditions.append( - "(c.last_assessed_at IS NULL OR c.last_assessed_at < NOW() - INTERVAL '%s days')" - ) - params.append(stale_days) - - if government_id: - conditions.append("c.government_id = %s") - params.append(government_id) - - where = "WHERE " + " AND ".join(conditions) if conditions else "" - - params.extend([limit, offset]) - - results = query( - f""" - SELECT c.id, c.title, c.description, c.commitment_type, c.status, - c.target_date, c.date_promised, c.last_assessed_at, - pa.name AS policy_area_name, pa.slug AS policy_area_slug, - (SELECT COUNT(*) FROM criteria cr WHERE cr.commitment_id = c.id) AS criteria_count, - (SELECT COUNT(*) FROM commitment_matches cm WHERE cm.commitment_id = c.id) AS matches_count - FROM commitments c - LEFT JOIN policy_areas pa ON pa.id = c.policy_area_id - {where} - ORDER BY c.last_assessed_at ASC NULLS FIRST, c.id - LIMIT %s OFFSET %s - """, - tuple(params), - ) - return [_serialize(r) for r in results] - - -def get_bill(bill_id: int) -> dict: - """Fetch a bill with stage dates and linked commitments.""" - bill = query_one( - """ - SELECT id, bill_id, bill_number_formatted, parliament_number, - short_title, long_title, latest_activity, - passed_house_first_reading_at, passed_house_second_reading_at, - passed_house_third_reading_at, - passed_senate_first_reading_at, passed_senate_second_reading_at, - passed_senate_third_reading_at, - received_royal_assent_at, latest_activity_at - FROM bills - WHERE id = %s - """, - (bill_id,), - ) - if not bill: - return {"error": f"Bill {bill_id} not found"} - - linked_commitments = query( - """ - SELECT cm.commitment_id, cm.relevance_score, cm.relevance_reasoning, - c.title AS commitment_title, c.status AS commitment_status - FROM commitment_matches cm - JOIN commitments c ON c.id = cm.commitment_id - WHERE cm.matchable_type = 'Bill' AND cm.matchable_id = %s - ORDER BY cm.relevance_score DESC - """, - (bill_id,), - ) - bill["linked_commitments"] = linked_commitments - return _serialize(bill) - - -def get_entry(entry_id: int) -> dict: - """Fetch an entry with its parsed content and feed info.""" - entry = query_one( - """ - SELECT e.id, e.title, e.url, e.published_at, e.scraped_at, - e.summary, e.parsed_markdown, e.activities_extracted_at, - f.title AS feed_title, f.source_url AS feed_source_url - FROM entries e - JOIN feeds f ON f.id = e.feed_id - WHERE e.id = %s - """, - (entry_id,), - ) - if not entry: - return {"error": f"Entry {entry_id} not found"} - return _serialize(entry) - - -def list_unprocessed_entries(government_id: int | None = None, limit: int = 50) -> list[dict]: - """Entries that have been scraped but not yet processed by the agent.""" - conditions = ["e.scraped_at IS NOT NULL", "e.skipped_at IS NULL"] - params = [] - - if government_id: - conditions.append("e.government_id = %s") - params.append(government_id) - - where = "WHERE " + " AND ".join(conditions) - params.append(limit) - - results = query( - f""" - SELECT e.id, e.title, e.url, e.published_at, e.scraped_at, - e.activities_extracted_at, - f.title AS feed_title - FROM entries e - JOIN feeds f ON f.id = e.feed_id - {where} - ORDER BY e.published_at DESC - LIMIT %s - """, - tuple(params), - ) - return [_serialize(r) for r in results] - - -def get_commitment_sources(commitment_id: int) -> list[dict]: - """Get the source documents (platform, SFT, budget) for a commitment.""" - results = query( - """ - SELECT cs.section, cs.reference, cs.excerpt, cs.relevance_note, - s.title, s.source_type, s.url, s.date - FROM commitment_sources cs - JOIN sources s ON s.id = cs.source_id - WHERE cs.commitment_id = %s - """, - (commitment_id,), - ) - return [_serialize(r) for r in results] - - -def get_bills_for_parliament(parliament_number: int = 45) -> list[dict]: - """Get all government bills for a parliament session.""" - results = query( - """ - SELECT id, bill_id, bill_number_formatted, short_title, long_title, - latest_activity, latest_activity_at, - passed_house_first_reading_at, passed_house_second_reading_at, - passed_house_third_reading_at, - passed_senate_first_reading_at, passed_senate_second_reading_at, - passed_senate_third_reading_at, - received_royal_assent_at - FROM bills - WHERE parliament_number = %s - AND data->>'BillTypeEn' IN ('House Government Bill', 'Senate Government Bill') - ORDER BY latest_activity_at DESC NULLS LAST - """, - (parliament_number,), - ) - return [_serialize(r) for r in results] - - -def _serialize(obj: dict) -> dict: - """Serialize datetime and other non-JSON-serializable types.""" - from datetime import date, datetime - from decimal import Decimal - - result = {} - for key, value in obj.items(): - if isinstance(value, (datetime, date)): - result[key] = value.isoformat() - elif isinstance(value, Decimal): - result[key] = float(value) - elif isinstance(value, list): - result[key] = [_serialize(v) if isinstance(v, dict) else v for v in value] - elif isinstance(value, dict): - result[key] = _serialize(value) - else: - result[key] = value - return result diff --git a/agent/src/agent/tools/rails_write.py b/agent/src/agent/tools/rails_write.py deleted file mode 100644 index e567e4d..0000000 --- a/agent/src/agent/tools/rails_write.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Rails API write tools — HTTP calls to /api/agent/* for mutations.""" - -import os -import sys - -import httpx - - -def _api_url() -> str: - return os.environ.get("RAILS_API_URL", "http://localhost:3000") - - -def _api_key() -> str: - return os.environ["RAILS_API_KEY"] - - -def _headers() -> dict: - return { - "Authorization": f"Bearer {_api_key()}", - "Content-Type": "application/json", - "Accept": "application/json", - } - - -def _post(path: str, data: dict) -> dict: - url = f"{_api_url()}{path}" - print(f" [rails] POST {url}", file=sys.stderr, flush=True) - try: - resp = httpx.post(url, json=data, headers=_headers(), timeout=60.0) - print(f" [rails] → {resp.status_code}", file=sys.stderr, flush=True) - if resp.status_code >= 400: - err = f"API error {resp.status_code}: {resp.text[:300]}" - print(f" [rails] ERROR: {err}", file=sys.stderr, flush=True) - return {"error": err} - return resp.json() - except Exception as e: - print(f" [rails] EXCEPTION: {e}", file=sys.stderr, flush=True) - return {"error": str(e)} - - -def _patch(path: str, data: dict) -> dict: - url = f"{_api_url()}{path}" - print(f" [rails] PATCH {url}", file=sys.stderr, flush=True) - try: - resp = httpx.patch(url, json=data, headers=_headers(), timeout=60.0) - print(f" [rails] → {resp.status_code}", file=sys.stderr, flush=True) - if resp.status_code >= 400: - err = f"API error {resp.status_code}: {resp.text[:300]}" - print(f" [rails] ERROR: {err}", file=sys.stderr, flush=True) - return {"error": err} - return resp.json() - except Exception as e: - print(f" [rails] EXCEPTION: {e}", file=sys.stderr, flush=True) - return {"error": str(e)} - - -def fetch_page(url: str, government_id: int) -> dict: - """Fetch a government page through Rails — auto-creates Source record.""" - return _post( - "/api/agent/pages/fetch", - { - "url": url, - "government_id": government_id, - }, - ) - - -def assess_criterion( - criterion_id: int, - new_status: str, - evidence_notes: str, - source_url: str, -) -> dict: - """Assess a criterion — requires source_url. Creates a CriterionAssessment audit record.""" - return _patch( - f"/api/agent/criteria/{criterion_id}", - { - "new_status": new_status, - "evidence_notes": evidence_notes, - "source_url": source_url, - }, - ) - - -def update_commitment_status( - commitment_id: int, - new_status: str, - reasoning: str, - source_urls: list[str], -) -> dict: - """Update commitment status — requires source_urls. Creates a CommitmentStatusChange audit record.""" - return _patch( - f"/api/agent/commitments/{commitment_id}/status", - { - "new_status": new_status, - "reasoning": reasoning, - "source_urls": source_urls, - }, - ) - - -def link_bill_to_commitment( - bill_id: int, - commitment_id: int, - relevance_score: float, - relevance_reasoning: str, -) -> dict: - """Link a bill to a commitment via CommitmentMatch.""" - return _post( - "/api/agent/commitment_matches", - { - "commitment_id": commitment_id, - "matchable_type": "Bill", - "matchable_id": bill_id, - "relevance_score": relevance_score, - "relevance_reasoning": relevance_reasoning, - }, - ) - - -def create_commitment_event( - commitment_id: int, - event_type: str, - title: str, - description: str, - occurred_at: str, - source_url: str, - action_type: str | None = None, - metadata: dict | None = None, -) -> dict: - """Create a CommitmentEvent — requires source_url. Auto-creates FeedItem via Rails callback.""" - return _post( - "/api/agent/commitment_events", - { - "commitment_id": commitment_id, - "event_type": event_type, - "action_type": action_type, - "title": title, - "description": description, - "occurred_at": occurred_at, - "source_url": source_url, - "metadata": metadata or {}, - }, - ) - - -def register_source( - government_id: int, - url: str, - title: str, - date: str | None = None, -) -> dict: - """Register a fetched page as a Source in Rails. Infers source_type from URL.""" - source_type = "other" - if "gazette.gc.ca" in url: - source_type = "gazette_notice" - elif "budget" in url or "finance" in url: - source_type = "budget" - - return _post( - "/api/agent/sources", - { - "government_id": government_id, - "url": url, - "title": title or f"Government page: {url[:80]}", - "source_type": source_type, - "source_type_other": "government_webpage" if source_type == "other" else None, - "date": date, - }, - ) - - -def add_source( - government_id: int, - url: str, - title: str, - source_type: str, - date: str | None = None, -) -> dict: - """Create a Source record if not already in the database.""" - return _post( - "/api/agent/sources", - { - "government_id": government_id, - "url": url, - "title": title, - "source_type": source_type, - "date": date, - }, - ) - - -def record_evaluation_run( - commitment_id: int, - trigger_type: str, - reasoning: str, - previous_status: str | None = None, - new_status: str | None = None, - criteria_assessed: int = 0, - evidence_found: int = 0, - search_queries: list[str] | None = None, - duration_seconds: float | None = None, - agent_run_id: str | None = None, -) -> dict: - """Record an audit trail for this evaluation run.""" - return _post( - "/api/agent/evaluation_runs", - { - "commitment_id": commitment_id, - "agent_run_id": agent_run_id, - "trigger_type": trigger_type, - "reasoning": reasoning, - "previous_status": previous_status, - "new_status": new_status, - "criteria_assessed": criteria_assessed, - "evidence_found": evidence_found, - "search_queries": search_queries or [], - "duration_seconds": duration_seconds, - }, - ) diff --git a/agent/src/agent/tools/web_search.py b/agent/src/agent/tools/web_search.py deleted file mode 100644 index 8a80ebb..0000000 --- a/agent/src/agent/tools/web_search.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Domain-restricted web search and fetch tools using Claude's built-in capabilities.""" - -from urllib.parse import urlparse - -import httpx -from bs4 import BeautifulSoup -from markdownify import markdownify - -from agent.domain.validators import is_government_url, validate_government_url - - -def fetch_government_page(url: str) -> dict: - """ - Fetch and parse content from an official Canadian government webpage. - Only URLs matching *.canada.ca or *.gc.ca are allowed. - """ - try: - validate_government_url(url) - except ValueError as e: - return {"error": str(e)} - - try: - resp = httpx.get( - url, - follow_redirects=True, - timeout=30.0, - headers={"User-Agent": "BuildCanada-Tracker/1.0"}, - ) - resp.raise_for_status() - except httpx.HTTPError as e: - return {"error": f"Failed to fetch {url}: {e}"} - - # Verify the final URL (after redirects) is still on a government domain - if not is_government_url(str(resp.url)): - return {"error": f"Redirect led to non-government domain: {resp.url}"} - - soup = BeautifulSoup(resp.text, "html.parser") - - # Remove nav, footer, scripts, styles - for tag in soup.find_all(["nav", "footer", "script", "style", "header"]): - tag.decompose() - - # Try to find the main content area (canada.ca uses specific IDs) - main = ( - soup.find("main") - or soup.find(id="wb-cont") - or soup.find(class_="mwsgeneric-base-html") - or soup.find("article") - or soup.body - ) - - content_html = str(main) if main else str(soup) - content_md = markdownify(content_html, strip=["img", "a"]).strip() - - # Truncate very long pages - if len(content_md) > 15000: - content_md = content_md[:15000] + "\n\n[Content truncated at 15,000 characters]" - - title = soup.title.string.strip() if soup.title and soup.title.string else "" - - # Try to find publication date - published_date = None - date_meta = soup.find("meta", {"name": "dcterms.modified"}) or soup.find( - "meta", {"name": "dcterms.issued"} - ) - if date_meta: - published_date = date_meta.get("content") - - return { - "url": str(resp.url), - "title": title, - "content_markdown": content_md, - "published_date": published_date, - } - - -def filter_search_results(results: list[dict]) -> list[dict]: - """Post-filter search results to only include government domains.""" - return [r for r in results if is_government_url(r.get("url", ""))] diff --git a/app/controllers/api/agent/bills_controller.rb b/app/controllers/api/agent/bills_controller.rb new file mode 100644 index 0000000..b02db30 --- /dev/null +++ b/app/controllers/api/agent/bills_controller.rb @@ -0,0 +1,56 @@ +module Api + module Agent + class BillsController < BaseController + def index + parliament_number = (params[:parliament_number] || 45).to_i + bills = Bill.where(parliament_number: parliament_number) + .where("data->>'BillTypeEn' IN ('House Government Bill', 'Senate Government Bill')") + .order("latest_activity_at DESC NULLS LAST") + + render json: bills.map { |b| serialize_bill_summary(b) } + end + + def show + bill = Bill.find(params[:id]) + + linked_commitments = CommitmentMatch + .includes(:commitment) + .where(matchable_type: "Bill", matchable_id: bill.id) + .order(relevance_score: :desc) + .map do |m| + { + commitment_id: m.commitment_id, + relevance_score: m.relevance_score, + relevance_reasoning: m.relevance_reasoning, + commitment_title: m.commitment&.title, + commitment_status: m.commitment&.status, + } + end + + render json: serialize_bill_summary(bill).merge(linked_commitments: linked_commitments) + end + + private + + def serialize_bill_summary(b) + { + id: b.id, + bill_id: b.bill_id, + bill_number_formatted: b.bill_number_formatted, + parliament_number: b.parliament_number, + short_title: b.short_title, + long_title: b.long_title, + latest_activity: b.latest_activity, + latest_activity_at: b.latest_activity_at, + passed_house_first_reading_at: b.passed_house_first_reading_at, + passed_house_second_reading_at: b.passed_house_second_reading_at, + passed_house_third_reading_at: b.passed_house_third_reading_at, + passed_senate_first_reading_at: b.passed_senate_first_reading_at, + passed_senate_second_reading_at: b.passed_senate_second_reading_at, + passed_senate_third_reading_at: b.passed_senate_third_reading_at, + received_royal_assent_at: b.received_royal_assent_at, + } + end + end + end +end diff --git a/app/controllers/api/agent/commitments_controller.rb b/app/controllers/api/agent/commitments_controller.rb index 029bed5..9b60528 100644 --- a/app/controllers/api/agent/commitments_controller.rb +++ b/app/controllers/api/agent/commitments_controller.rb @@ -1,6 +1,170 @@ module Api module Agent class CommitmentsController < BaseController + def index + scope = Commitment.includes(:policy_area) + + scope = scope.where(status: params[:status]) if params[:status].present? + scope = scope.joins(:policy_area).where(policy_areas: { slug: params[:policy_area] }) if params[:policy_area].present? + scope = scope.where(commitment_type: params[:commitment_type]) if params[:commitment_type].present? + if params[:government_id].present? + scope = scope.where(government_id: params[:government_id]) + end + if params[:stale_days].present? + cutoff = params[:stale_days].to_i.days.ago + scope = scope.where("last_assessed_at IS NULL OR last_assessed_at < ?", cutoff) + end + + limit = (params[:limit] || 50).to_i + offset = (params[:offset] || 0).to_i + commitments = scope.order("last_assessed_at ASC NULLS FIRST, id ASC").limit(limit).offset(offset) + + render json: commitments.map { |c| serialize_commitment_summary(c) } + end + + def show + commitment = Commitment.find(params[:id]) + + criteria = commitment.criteria.order(:category, :position).map do |cr| + { + id: cr.id, category: cr.category, description: cr.description, + verification_method: cr.verification_method, status: cr.status, + evidence_notes: cr.evidence_notes, assessed_at: cr.assessed_at, position: cr.position, + } + end + + matches = CommitmentMatch.includes(:matchable) + .where(commitment_id: commitment.id) + .order(relevance_score: :desc) + .map do |m| + matchable_title = m.matchable_type == "Bill" ? m.matchable&.bill_number_formatted : m.matchable&.title + matchable_detail = m.matchable_type == "Bill" ? m.matchable&.short_title : m.matchable&.url + { + id: m.id, matchable_type: m.matchable_type, matchable_id: m.matchable_id, + relevance_score: m.relevance_score, relevance_reasoning: m.relevance_reasoning, + matched_at: m.matched_at, assessed: m.assessed, + matchable_title: matchable_title, matchable_detail: matchable_detail, + } + end + + events = commitment.events.order(occurred_at: :desc).limit(50).map do |e| + { + id: e.id, event_type: e.event_type, action_type: e.action_type, + title: e.title, description: e.description, occurred_at: e.occurred_at, + metadata: e.metadata, + } + end + + sources = CommitmentSource.includes(:source) + .where(commitment_id: commitment.id) + .map do |cs| + { + id: cs.id, section: cs.section, reference: cs.reference, + excerpt: cs.excerpt, relevance_note: cs.relevance_note, + source_title: cs.source&.title, source_type: cs.source&.source_type, + source_url: cs.source&.url, source_date: cs.source&.date, + } + end + + departments = CommitmentDepartment.includes(:department) + .where(commitment_id: commitment.id) + .order(is_lead: :desc) + .map do |cd| + { + id: cd.department_id, slug: cd.department&.slug, + display_name: cd.department&.display_name, + official_name: cd.department&.official_name, + is_lead: cd.is_lead, + } + end + + status_changes = CommitmentStatusChange.where(commitment_id: commitment.id) + .order(changed_at: :desc).limit(10) + .map do |sc| + { + previous_status: sc.previous_status, new_status: sc.new_status, + changed_at: sc.changed_at, reason: sc.reason, + } + end + + render json: { + id: commitment.id, + title: commitment.title, + description: commitment.description, + original_text: commitment.original_text, + commitment_type: commitment.commitment_type, + status: commitment.status, + date_promised: commitment.date_promised, + target_date: commitment.target_date, + last_assessed_at: commitment.last_assessed_at, + government_id: commitment.government_id, + policy_area_name: commitment.policy_area&.name, + policy_area_slug: commitment.policy_area&.slug, + criteria: criteria, + matches: matches, + events: events, + sources: sources, + departments: departments, + status_changes: status_changes, + } + end + + def sources + commitment = Commitment.find(params[:id]) + result = CommitmentSource.includes(:source) + .where(commitment_id: commitment.id) + .map do |cs| + { + section: cs.section, reference: cs.reference, + excerpt: cs.excerpt, relevance_note: cs.relevance_note, + title: cs.source&.title, source_type: cs.source&.source_type, + url: cs.source&.url, date: cs.source&.date, + } + end + render json: result + end + + def touch_assessed + commitment = Commitment.find(params[:id]) + reasoning = params[:reasoning].presence || "Session ended — last assessed timestamp updated" + + # Only update if the agent didn't already record an evaluation run this session + # (record_evaluation_run also sets last_assessed_at, so skip the fallback if it was called) + recent_run = EvaluationRun.where(commitment_id: commitment.id) + .where("created_at > ?", 15.minutes.ago) + .exists? + + if recent_run + render json: { id: commitment.id, last_assessed_at: commitment.last_assessed_at, skipped: true, reason: "evaluation_run already recorded" } + return + end + + commitment.update!(last_assessed_at: Time.current) + Rails.logger.info("AgentHook: Updated last_assessed_at for commitment #{commitment.id} — #{reasoning}") + render json: { id: commitment.id, last_assessed_at: commitment.last_assessed_at, skipped: false } + end + + private + + def serialize_commitment_summary(c) + { + id: c.id, + title: c.title, + description: c.description, + commitment_type: c.commitment_type, + status: c.status, + target_date: c.target_date, + date_promised: c.date_promised, + last_assessed_at: c.last_assessed_at, + policy_area_name: c.policy_area&.name, + policy_area_slug: c.policy_area&.slug, + criteria_count: c.criteria.size, + matches_count: c.commitment_matches.size, + } + end + + public + def status commitment = Commitment.find(params[:id]) previous_status = commitment.status diff --git a/app/controllers/api/agent/entries_controller.rb b/app/controllers/api/agent/entries_controller.rb new file mode 100644 index 0000000..ce8b592 --- /dev/null +++ b/app/controllers/api/agent/entries_controller.rb @@ -0,0 +1,57 @@ +module Api + module Agent + class EntriesController < BaseController + def index + scope = Entry.includes(:feed).where.not(scraped_at: nil).where(skipped_at: nil, is_index: [ false, nil ]) + + if params[:unprocessed] == "true" + scope = scope.where(agent_processed_at: nil) + end + + if params[:government_id].present? + scope = scope.where(government_id: params[:government_id]) + end + + limit = (params[:limit] || 50).to_i + entries = scope.order(published_at: :desc).limit(limit) + + render json: entries.map { |e| + { + id: e.id, title: e.title, url: e.url, + published_at: e.published_at, scraped_at: e.scraped_at, + activities_extracted_at: e.activities_extracted_at, + feed_title: e.feed&.title, + } + } + end + + def show + entry = Entry.includes(:feed).find(params[:id]) + render json: { + id: entry.id, + title: entry.title, + url: entry.url, + published_at: entry.published_at, + scraped_at: entry.scraped_at, + summary: entry.summary, + parsed_markdown: entry.parsed_markdown, + activities_extracted_at: entry.activities_extracted_at, + feed_title: entry.feed&.title, + feed_source_url: entry.feed&.source_url, + } + end + + def mark_processed + entry = Entry.find(params[:id]) + + if entry.agent_processed_at.present? + render json: { id: entry.id, agent_processed_at: entry.agent_processed_at, skipped: true } + return + end + + entry.update!(agent_processed_at: Time.current) + render json: { id: entry.id, agent_processed_at: entry.agent_processed_at, skipped: false } + end + end + end +end diff --git a/app/jobs/agent_evaluate_commitment_job.rb b/app/jobs/agent_evaluate_commitment_job.rb index 5b2b06f..44edf81 100644 --- a/app/jobs/agent_evaluate_commitment_job.rb +++ b/app/jobs/agent_evaluate_commitment_job.rb @@ -3,7 +3,6 @@ class AgentEvaluateCommitmentJob < ApplicationJob queue_as :default - # Limit to 5 concurrent agent evaluations to avoid API rate limits include GoodJob::ActiveJobExtensions::Concurrency good_job_control_concurrency_with( perform_limit: 5, @@ -16,45 +15,84 @@ class AgentEvaluateCommitmentJob < ApplicationJob def perform(commitment, trigger_type: "manual", as_of_date: nil) agent_dir = Rails.root.join("agent") - - args = [ - "python", "-m", "agent.main", "evaluate", - "--commitment-id", commitment.id.to_s - ] - args.push("--as-of", as_of_date) if as_of_date.present? - - env = agent_env + current_date = as_of_date || Date.today.iso8601 + prompt = format(AgentPrompts::EVALUATE_COMMITMENT_PROMPT, commitment_id: commitment.id, current_date: current_date) + hook_script = agent_dir.join(".claude/hooks/on_stop_commitment.sh").to_s Rails.logger.info("AgentEvaluateCommitmentJob: Evaluating commitment #{commitment.id} (#{trigger_type})") - stdout, stderr, status = Open3.capture3(env, *args, chdir: agent_dir.to_s) + exit_status = stream_agent( + agent_env(commitment_id: commitment.id), + build_cmd(prompt, hook_script: hook_script), + chdir: agent_dir.to_s + ) - if status.success? - Rails.logger.info("AgentEvaluateCommitmentJob: Success for commitment #{commitment.id}") - Rails.logger.debug(stdout) if stdout.present? - else - Rails.logger.error("AgentEvaluateCommitmentJob: Failed for commitment #{commitment.id}") - Rails.logger.error(stderr) if stderr.present? - raise "Agent evaluation failed for commitment #{commitment.id}: #{stderr.first(500)}" + unless exit_status.success? + raise "Agent evaluation failed for commitment #{commitment.id} (exit #{exit_status.exitstatus})" end + + Rails.logger.info("AgentEvaluateCommitmentJob: Success for commitment #{commitment.id}") end private - def agent_env - { - "ANTHROPIC_API_KEY" => Rails.application.credentials.dig(:anthropic, :api_key) || ENV["ANTHROPIC_API_KEY"], - "AGENT_DATABASE_URL" => agent_database_url, - "RAILS_API_URL" => ENV.fetch("RAILS_API_URL", "http://localhost:3000"), - "RAILS_API_KEY" => Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"], - "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-sonnet-4-6"), - }.compact + def stream_agent(env, cmd, chdir:) + Open3.popen2e(env, *cmd, chdir: chdir) do |stdin, output, thread| + stdin.close + output.each_line { |line| $stderr.print(line) } + thread.value + end end - def agent_database_url - ENV["AGENT_DATABASE_URL"] || begin - config = ActiveRecord::Base.connection_db_config.configuration_hash - "postgresql://agent_reader:#{config[:password]}@#{config[:host] || 'localhost'}:#{config[:port] || 5432}/#{config[:database]}" - end + def build_cmd(prompt, hook_script:) + hook_settings = { + "hooks" => { + "Stop" => [ { "hooks" => [ { "type" => "command", "command" => hook_script, "async" => true, "timeout" => 10 } ] } ] + } + }.to_json + + [ + "claude", "-p", prompt, + "--system-prompt", system_prompt, + "--allowedTools", allowed_tools.join(","), + "--permission-mode", "bypassPermissions", + "--model", ENV.fetch("AGENT_MODEL", "claude-sonnet-4-6"), + "--output-format", "text", + "--settings", hook_settings, + ] + end + + def allowed_tools + [ + "Bash(curl *)", + "WebFetch(https://*.canada.ca/*)", + "WebFetch(https://*.gc.ca/*)", + "WebFetch(https://www.parl.ca/*)", + "WebSearch", + ] + end + + def system_prompt + AgentPrompts::SYSTEM_PROMPT + api_context + end + + def api_context + url = ENV.fetch("RAILS_API_URL", "http://localhost:3000") + key = Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"] + "\n\n## Rails API Connection\nBase URL: `#{url}`\nAuth header: `Authorization: Bearer #{key}`\nSee CLAUDE.md for endpoint details and enum values.\n" + end + + def agent_env(commitment_id: nil, entry_id: nil) + { + "CLAUDE_CODE_OAUTH_TOKEN" => ENV["CLAUDE_CODE_OAUTH_TOKEN"], + "RAILS_API_URL" => ENV.fetch("RAILS_API_URL", "http://localhost:3000"), + "RAILS_API_KEY" => Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"], + "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-sonnet-4-6"), + "COMMITMENT_ID" => commitment_id&.to_s, + "ENTRY_ID" => entry_id&.to_s, + # Explicitly unset — subprocess must not access Rails credentials + "RAILS_MASTER_KEY" => nil, + "SECRET_KEY_BASE" => nil, + }.compact end end diff --git a/app/jobs/agent_process_entry_job.rb b/app/jobs/agent_process_entry_job.rb index 01be0a2..080c941 100644 --- a/app/jobs/agent_process_entry_job.rb +++ b/app/jobs/agent_process_entry_job.rb @@ -3,46 +3,88 @@ class AgentProcessEntryJob < ApplicationJob queue_as :default + retry_on StandardError, wait: 30.seconds, attempts: 3 + def perform(entry) agent_dir = Rails.root.join("agent") - - args = [ - "python", "-m", "agent.main", "process-entry", - "--entry-id", entry.id.to_s - ] - - env = agent_env + current_date = Date.today.iso8601 + prompt = format(AgentPrompts::PROCESS_ENTRY_PROMPT, entry_id: entry.id, current_date: current_date) + hook_script = agent_dir.join(".claude/hooks/on_stop_entry.sh").to_s Rails.logger.info("AgentProcessEntryJob: Processing entry #{entry.id} (#{entry.title})") - stdout, stderr, status = Open3.capture3(env, *args, chdir: agent_dir.to_s) + exit_status = stream_agent( + agent_env(entry_id: entry.id), + build_cmd(prompt, hook_script: hook_script), + chdir: agent_dir.to_s + ) - if status.success? - Rails.logger.info("AgentProcessEntryJob: Success for entry #{entry.id}") - Rails.logger.debug(stdout) if stdout.present? - else - Rails.logger.error("AgentProcessEntryJob: Failed for entry #{entry.id}") - Rails.logger.error(stderr) if stderr.present? - raise "Agent processing failed for entry #{entry.id}: #{stderr.first(500)}" + unless exit_status.success? + raise "Agent processing failed for entry #{entry.id} (exit #{exit_status.exitstatus})" end + + Rails.logger.info("AgentProcessEntryJob: Success for entry #{entry.id}") end private - def agent_env - { - "ANTHROPIC_API_KEY" => Rails.application.credentials.dig(:anthropic, :api_key) || ENV["ANTHROPIC_API_KEY"], - "AGENT_DATABASE_URL" => agent_database_url, - "RAILS_API_URL" => ENV.fetch("RAILS_API_URL", "http://localhost:3000"), - "RAILS_API_KEY" => Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"], - "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-opus-4-6"), - }.compact + def stream_agent(env, cmd, chdir:) + Open3.popen2e(env, *cmd, chdir: chdir) do |stdin, output, thread| + stdin.close + output.each_line { |line| $stderr.print(line) } + thread.value + end end - def agent_database_url - ENV["AGENT_DATABASE_URL"] || begin - config = ActiveRecord::Base.connection_db_config.configuration_hash - "postgresql://agent_reader:#{config[:password]}@#{config[:host] || 'localhost'}:#{config[:port] || 5432}/#{config[:database]}" - end + def build_cmd(prompt, hook_script:) + hook_settings = { + "hooks" => { + "Stop" => [ { "hooks" => [ { "type" => "command", "command" => hook_script, "async" => true, "timeout" => 10 } ] } ] + } + }.to_json + + [ + "claude", "-p", prompt, + "--system-prompt", system_prompt, + "--allowedTools", allowed_tools.join(","), + "--permission-mode", "bypassPermissions", + "--model", ENV.fetch("AGENT_MODEL", "claude-opus-4-6"), + "--output-format", "text", + "--settings", hook_settings, + ] + end + + def allowed_tools + [ + "Bash(curl *)", + "WebFetch(https://*.canada.ca/*)", + "WebFetch(https://*.gc.ca/*)", + "WebFetch(https://www.parl.ca/*)", + "WebSearch", + ] + end + + def system_prompt + AgentPrompts::SYSTEM_PROMPT + api_context + end + + def api_context + url = ENV.fetch("RAILS_API_URL", "http://localhost:3000") + key = Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"] + "\n\n## Rails API Connection\nBase URL: `#{url}`\nAuth header: `Authorization: Bearer #{key}`\nSee CLAUDE.md for endpoint details and enum values.\n" + end + + def agent_env(commitment_id: nil, entry_id: nil) + { + "CLAUDE_CODE_OAUTH_TOKEN" => ENV["CLAUDE_CODE_OAUTH_TOKEN"], + "RAILS_API_URL" => ENV.fetch("RAILS_API_URL", "http://localhost:3000"), + "RAILS_API_KEY" => Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"], + "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-opus-4-6"), + "COMMITMENT_ID" => commitment_id&.to_s, + "ENTRY_ID" => entry_id&.to_s, + # Explicitly unset — subprocess must not access Rails credentials + "RAILS_MASTER_KEY" => nil, + "SECRET_KEY_BASE" => nil, + }.compact end end diff --git a/app/services/agent_prompts.rb b/app/services/agent_prompts.rb new file mode 100644 index 0000000..9ef25e9 --- /dev/null +++ b/app/services/agent_prompts.rb @@ -0,0 +1,205 @@ +module AgentPrompts + SYSTEM_PROMPT = <<~PROMPT + You are a government accountability analyst for Build Canada. You are the single + source of truth for evaluating the Canadian federal government's progress on its + commitments from the 2025 Liberal platform, Speech from the Throne, and Budget 2025. + + ## Your Responsibilities + + 1. Review existing evidence linked to each commitment + 2. Proactively search official government sources (*.canada.ca, *.gc.ca) for new evidence + 3. Link bills to commitments when they implement or affect a commitment + 4. Assess each criterion against available evidence + 5. Derive commitment status using the evidence hierarchy + 6. Create events explaining how each piece of evidence affects the commitment + + ## Status Definitions + + - **not_started**: No evidence of meaningful government action + - **in_progress**: Concrete steps underway (legislation introduced, funding allocated, program design started) + - **completed**: Commitment fulfilled (legislation enacted, program operational, objectives substantially achieved) + - **broken**: Government took a policy position counter to the commitment, OR the commitment had a specific deadline that has passed without completion + + ## Evidence Hierarchy (strictly enforced) + + **"completed" requires one of:** + - Bill with Royal Assent that implements the commitment + - Canada Gazette Part II/III entry (enacted regulation) + - Departmental evidence confirming a program is operational + + **"in_progress" requires one of:** + - Bill introduced and progressing in Parliament (no Royal Assent yet) + - Canada Gazette Part I entry (proposed regulation) + - Appropriation voted with program implementation evidence + - Budget allocation for a **pre-existing** commitment (see Budget Rule below) + + **"not_started":** + - No evidence of action, OR only announcements without concrete follow-through + + ## Budget Evidence Rule + + This rule is critical for accurate assessment: + + - The Budget **CANNOT** be used as evidence that a commitment **made in the budget itself** is in progress. Including something in the budget is announcing it, not implementing it. This is circular reasoning. + - The Budget **CAN** be used as evidence for commitments from the **platform or Speech from the Throne** that pre-date the budget. If the platform promised X and Budget 2025 allocates funding for X, that IS evidence of progress. + - Check the commitment's source documents to determine where it originated. + + ## Criteria Assessment + + Each commitment has criteria across four categories: + - **Completion**: Did the government literally do what it said? ("the letter") + - **Success**: Did the real-world outcome materialize? ("the spirit") + - **Progress**: Are they actively working toward it? + - **Failure**: Has the commitment been broken or contradicted? + + **Important:** Criteria are a structured guide, NOT a rigid checklist: + - A commitment can be marked "completed" even if the criteria don't exactly match the commitment text — what matters is whether the government fulfilled the spirit of the commitment + - If evidence shows the commitment was fulfilled through a different mechanism than the criteria anticipated, that's still "completed" + - Assess criteria based on available evidence; mark as "not_assessed" if insufficient evidence exists + + Criterion statuses: not_assessed, met, not_met, no_longer_applicable + + ## Bill Tracking + + - When a bill progresses through stages (readings, Royal Assent), evaluate its impact on linked commitments + - Track stages: House 1R → 2R → 3R → Senate 1R → 2R → 3R → Royal Assent + - If enacted legislation text diverges from what was promised, note this in the commitment event description + - A bill at Royal Assent that implements the commitment = strong completion evidence + + ## Date Awareness + + You must always consider dates: + - Only use evidence that existed at the evaluation date + - When backfilling, don't use future evidence to justify past status + - Respect commitment target_date — if the date has passed without completion, evaluate for "broken" status + - More recent evidence should be weighted more heavily + - Note the publication date of every source you cite + + ## Reading Data + + Use `curl` via Bash to read commitment and related data from the Rails API. All auth via `Authorization: Bearer $RAILS_API_KEY`. See CLAUDE.md for endpoint reference. + + ## Fetching Government Pages + + Use the `WebFetch` tool to read official government page content directly (restricted to canada.ca, gc.ca, parl.ca). + After reading a page you intend to cite as evidence, register it as a Source by calling: + `POST $RAILS_API_URL/api/agent/pages/fetch` with `{ "url": "...", "government_id": 1 }` via curl. + This returns a `source_id`. You MUST register pages before using their URLs in write operations. + + ## Source Requirement (CRITICAL) + + Every judgement you make MUST be backed by a source. The workflow is: + + 1. **Search** for evidence using WebSearch (site:canada.ca OR site:gc.ca) + 2. **Fetch** the page with WebFetch to read its content + 3. **Register** it via `POST /api/agent/pages/fetch` — this saves it as a Source and returns `source_id` and `url` + 4. **Use the registered URL** when making judgements: + - assess_criterion requires source_url + - create_commitment_event requires source_url + - update_commitment_status requires source_urls (array) + + You MUST register a page before referencing its URL in a judgement. The system will reject judgements that reference URLs not in the database. + + ## When You Find Evidence + + For every piece of evidence that tangibly affects a commitment's implementation: + 1. **Fetch and register** the page + 2. Create a CommitmentEvent with a blurb explaining WHY this evidence moves the commitment forward, backward, or is neutral — include the source_url + 3. Assess the relevant criteria against the new evidence — include the source_url + 4. Re-evaluate the overall commitment status if warranted — include all source_urls + + ## Search Strategy + + When evaluating a commitment, consider searching for: + - The commitment's keywords on canada.ca + - Related bills on parl.ca + - Budget 2025 provisions (for platform commitments only) + - Canada Gazette publications for regulatory changes + - Departmental plans and reports from the responsible department + - News releases from the responsible minister + + Use WebSearch with "site:canada.ca" or "site:gc.ca" to restrict results to official government sources. + + ## Status Change Requirements + + When updating a commitment's status, you MUST provide: + - **reasoning**: A clear 1-3 sentence explanation shown in the UI to users. Cite the specific evidence (bill number, program name, gazette reference). + - **effective_date**: The real-world date the status actually changed — NOT today's date. Use the date of the earliest evidence that justifies the new status. + - **source_urls**: All source URLs that justify the status change. + + ## Output Guidelines + + - Be factual and evidence-based + - Cite specific evidence (bill numbers, gazette references, program names, URLs) + - Note evidence gaps that could change the assessment + - Keep event descriptions concise but informative (1-3 sentences) + - If evidence is ambiguous, explain the ambiguity + PROMPT + + EVALUATE_COMMITMENT_PROMPT = <<~PROMPT + Evaluate commitment #%d. + + Use `curl -s -H "Authorization: Bearer $RAILS_API_KEY" $RAILS_API_URL/api/agent/commitments/%d` to load the full commitment details, criteria, existing evidence matches, and events. Then: + + 1. Review the existing evidence and criteria assessments + 2. Search for any new evidence on government sources that may have been missed + 3. For any new evidence found, create commitment events explaining its impact + 4. Assess each criterion that can be assessed with available evidence + 5. Determine the correct status based on the evidence hierarchy + 6. If the status should change, update it with clear reasoning + 7. Record an evaluation run for audit purposes + + Current date: %s + PROMPT + + PROCESS_ENTRY_PROMPT = <<~PROMPT + A new entry has been scraped from an RSS feed. + + Use `curl -s -H "Authorization: Bearer $RAILS_API_KEY" $RAILS_API_URL/api/agent/entries/%d` to read the entry. Then: + + 1. Read the entry content carefully + 2. Determine which commitments this entry is relevant to + 3. For each relevant commitment: + a. Link the entry via a CommitmentMatch if it provides evidence + b. Create a CommitmentEvent explaining how this evidence affects the commitment + c. Assess any criteria that this evidence helps evaluate + d. Update the commitment status if warranted + 4. Record evaluation runs for each affected commitment + + Current date: %s + PROMPT + + PROCESS_BILL_CHANGE_PROMPT = <<~PROMPT + Bill #%d has had a stage change. + + Use `curl -s -H "Authorization: Bearer $RAILS_API_KEY" $RAILS_API_URL/api/agent/bills/%d` to read the bill details and its linked commitments. Then: + + 1. Review what stage the bill has reached + 2. For each commitment linked to this bill: + a. Create a CommitmentEvent noting the bill's progress + b. Evaluate if this stage change affects the commitment's status + c. If the bill has received Royal Assent, strongly consider if the commitment is now "completed" + d. Assess relevant criteria + e. Update status if warranted + 3. If the bill is a government bill but NOT yet linked to any commitments, search for commitments it may implement and create links + 4. Record evaluation runs for each affected commitment + + Current date: %s + PROMPT + + WEEKLY_SCAN_PROMPT = <<~PROMPT + Perform a weekly proactive scan of commitment #%d. + + Use `curl -s -H "Authorization: Bearer $RAILS_API_KEY" $RAILS_API_URL/api/agent/commitments/%d` to load the full details. Then: + + 1. Review the current status and when it was last assessed + 2. Search government sources for any new evidence since the last assessment + 3. Check if any linked bills have progressed + 4. For any new evidence found, create commitment events and assess criteria + 5. Check if the target_date has passed — if so and the commitment is not completed, evaluate for "broken" status + 6. Update status if evidence warrants it + 7. Record an evaluation run + + Current date: %s + PROMPT +end diff --git a/config/initializers/cors.rb b/config/initializers/cors.rb index a5e8ac8..787e693 100644 --- a/config/initializers/cors.rb +++ b/config/initializers/cors.rb @@ -7,7 +7,7 @@ Rails.application.config.middleware.insert_before 0, Rack::Cors do allow do - origins "http://localhost:3001", "https://www.buildcanada.com" + origins /\Ahttp:\/\/localhost(:\d+)?\z/, "https://www.buildcanada.com" resource "*", headers: :any, diff --git a/config/routes.rb b/config/routes.rb index 6ff8835..1a100ad 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -36,9 +36,11 @@ get "dashboard/:government_id/at_a_glance", to: "dashboard#at_a_glance", as: :at_a_glance namespace :agent do - resources :commitments, only: [] do + resources :commitments, only: [ :index, :show ] do member do patch :status + get :sources + patch :touch_assessed end end resources :criteria, only: [ :update ] @@ -46,6 +48,10 @@ resources :commitment_events, only: [ :create ] resources :sources, only: [ :create ] resources :evaluation_runs, only: [ :create ] + resources :bills, only: [ :index, :show ] + resources :entries, only: [ :index, :show ] do + member { patch :mark_processed } + end post "pages/fetch", to: "pages#fetch" end end diff --git a/db/migrate/20260327161901_add_agent_processed_at_to_entries.rb b/db/migrate/20260327161901_add_agent_processed_at_to_entries.rb new file mode 100644 index 0000000..70ea2eb --- /dev/null +++ b/db/migrate/20260327161901_add_agent_processed_at_to_entries.rb @@ -0,0 +1,5 @@ +class AddAgentProcessedAtToEntries < ActiveRecord::Migration[8.0] + def change + add_column :entries, :agent_processed_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index ebfc8d8..1fed621 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_03_24_000001) do +ActiveRecord::Schema[8.0].define(version: 2026_03_27_161901) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -299,6 +299,7 @@ t.bigint "parent_id" t.datetime "skipped_at", precision: nil t.string "skip_reason" + t.datetime "agent_processed_at" t.index ["feed_id"], name: "index_entries_on_feed_id" t.index ["government_id"], name: "index_entries_on_government_id" t.index ["parent_id"], name: "index_entries_on_parent_id" diff --git a/lib/tasks/database_restore.rake b/lib/tasks/database_restore.rake index fbfe3dd..ce4f8ad 100644 --- a/lib/tasks/database_restore.rake +++ b/lib/tasks/database_restore.rake @@ -14,13 +14,16 @@ def db_config_for_env(target_env) enc_path = Rails.root.join("config/credentials/#{target_env}.yml.enc") key_path = Rails.root.join("config/credentials/#{target_env}.key") - unless File.exist?(enc_path) + creds = if File.exist?(enc_path) + Rails.application.encrypted(enc_path, key_path: key_path) + elsif target_env == "production" + Rails.application.credentials + else puts "No credentials file found: #{enc_path}" puts "Create with: bin/rails credentials:edit --environment #{target_env}" exit 1 end - creds = Rails.application.encrypted(enc_path, key_path: key_path) db = creds.database if db.nil? From 04d1d5366704e069fda639ad114c9eeb3df83370 Mon Sep 17 00:00:00 2001 From: xrendan Date: Fri, 27 Mar 2026 11:26:41 -0600 Subject: [PATCH 2/4] Fix RuboCop style offenses Auto-correct 226 offenses: trailing commas in hash literals and spacing inside array literal brackets. Co-Authored-By: Claude Sonnet 4.6 --- app/avo/resources/source.rb | 2 +- app/controllers/api/agent/bills_controller.rb | 4 +- .../api/agent/commitment_events_controller.rb | 2 +- .../agent/commitment_matches_controller.rb | 2 +- .../api/agent/commitments_controller.rb | 20 +- .../api/agent/criteria_controller.rb | 2 +- .../api/agent/entries_controller.rb | 4 +- app/controllers/api/agent/pages_controller.rb | 2 +- app/controllers/commitments_controller.rb | 4 +- app/controllers/feed_items_controller.rb | 4 +- app/jobs/agent_evaluate_commitment_job.rb | 6 +- app/jobs/agent_process_entry_job.rb | 6 +- app/jobs/commitment_assessment_job.rb | 8 +- app/jobs/source_document_processor_job.rb | 4 +- app/mailers/test_mailer.rb | 4 +- app/models/concerns/mcp_rack_tool.rb | 2 +- app/models/criterion_assessment.rb | 1 - app/services/ministers_fetcher.rb | 4 +- app/views/commitments/index.json.jbuilder | 2 +- .../20250805000001_create_commitments.rb | 4 +- db/migrate/20250805000003_create_criteria.rb | 6 +- ...805000004_create_commitment_departments.rb | 2 +- db/migrate/20250805000006_create_sources.rb | 2 +- ...0305000004_create_criterion_assessments.rb | 2 +- ...000005_remove_assessed_by_from_criteria.rb | 2 +- ...0260305000010_create_commitment_matches.rb | 6 +- ...000001_create_commitment_status_changes.rb | 2 +- ...20260306000002_create_commitment_events.rb | 2 +- ...60306000003_create_commitment_revisions.rb | 2 +- .../20260306000004_create_feed_items.rb | 8 +- .../20260324000001_create_evaluation_runs.rb | 2 +- lib/tasks/backfill_evidence_dates.rake | 2 +- lib/tasks/database_backup.rake | 2 +- lib/tasks/database_restore.rake | 8 +- lib/tasks/dedup.rake | 312 +++++++++--------- 35 files changed, 223 insertions(+), 224 deletions(-) diff --git a/app/avo/resources/source.rb b/app/avo/resources/source.rb index 8bbfc2e..7dd91d5 100644 --- a/app/avo/resources/source.rb +++ b/app/avo/resources/source.rb @@ -6,7 +6,7 @@ def fields field :id, as: :id field :title, as: :text field :source_type, as: :select, enum: ::Source.source_types - field :source_type_other, as: :text, hide_on: [:index] + field :source_type_other, as: :text, hide_on: [ :index ] field :url, as: :text field :date, as: :date field :government, as: :belongs_to diff --git a/app/controllers/api/agent/bills_controller.rb b/app/controllers/api/agent/bills_controller.rb index b02db30..ed98a13 100644 --- a/app/controllers/api/agent/bills_controller.rb +++ b/app/controllers/api/agent/bills_controller.rb @@ -23,7 +23,7 @@ def show relevance_score: m.relevance_score, relevance_reasoning: m.relevance_reasoning, commitment_title: m.commitment&.title, - commitment_status: m.commitment&.status, + commitment_status: m.commitment&.status } end @@ -48,7 +48,7 @@ def serialize_bill_summary(b) passed_senate_first_reading_at: b.passed_senate_first_reading_at, passed_senate_second_reading_at: b.passed_senate_second_reading_at, passed_senate_third_reading_at: b.passed_senate_third_reading_at, - received_royal_assent_at: b.received_royal_assent_at, + received_royal_assent_at: b.received_royal_assent_at } end end diff --git a/app/controllers/api/agent/commitment_events_controller.rb b/app/controllers/api/agent/commitment_events_controller.rb index b9e3647..74ddd93 100644 --- a/app/controllers/api/agent/commitment_events_controller.rb +++ b/app/controllers/api/agent/commitment_events_controller.rb @@ -26,7 +26,7 @@ def create commitment_id: event.commitment_id, event_type: event.event_type, title: event.title, - source_id: source.id, + source_id: source.id }, status: :created end end diff --git a/app/controllers/api/agent/commitment_matches_controller.rb b/app/controllers/api/agent/commitment_matches_controller.rb index b4c469f..9be9502 100644 --- a/app/controllers/api/agent/commitment_matches_controller.rb +++ b/app/controllers/api/agent/commitment_matches_controller.rb @@ -22,7 +22,7 @@ def create matchable_type: match.matchable_type, matchable_id: match.matchable_id, relevance_score: match.relevance_score, - created: match.previously_new_record?, + created: match.previously_new_record? } end end diff --git a/app/controllers/api/agent/commitments_controller.rb b/app/controllers/api/agent/commitments_controller.rb index 9b60528..1c9c9d3 100644 --- a/app/controllers/api/agent/commitments_controller.rb +++ b/app/controllers/api/agent/commitments_controller.rb @@ -29,7 +29,7 @@ def show { id: cr.id, category: cr.category, description: cr.description, verification_method: cr.verification_method, status: cr.status, - evidence_notes: cr.evidence_notes, assessed_at: cr.assessed_at, position: cr.position, + evidence_notes: cr.evidence_notes, assessed_at: cr.assessed_at, position: cr.position } end @@ -43,7 +43,7 @@ def show id: m.id, matchable_type: m.matchable_type, matchable_id: m.matchable_id, relevance_score: m.relevance_score, relevance_reasoning: m.relevance_reasoning, matched_at: m.matched_at, assessed: m.assessed, - matchable_title: matchable_title, matchable_detail: matchable_detail, + matchable_title: matchable_title, matchable_detail: matchable_detail } end @@ -51,7 +51,7 @@ def show { id: e.id, event_type: e.event_type, action_type: e.action_type, title: e.title, description: e.description, occurred_at: e.occurred_at, - metadata: e.metadata, + metadata: e.metadata } end @@ -62,7 +62,7 @@ def show id: cs.id, section: cs.section, reference: cs.reference, excerpt: cs.excerpt, relevance_note: cs.relevance_note, source_title: cs.source&.title, source_type: cs.source&.source_type, - source_url: cs.source&.url, source_date: cs.source&.date, + source_url: cs.source&.url, source_date: cs.source&.date } end @@ -74,7 +74,7 @@ def show id: cd.department_id, slug: cd.department&.slug, display_name: cd.department&.display_name, official_name: cd.department&.official_name, - is_lead: cd.is_lead, + is_lead: cd.is_lead } end @@ -83,7 +83,7 @@ def show .map do |sc| { previous_status: sc.previous_status, new_status: sc.new_status, - changed_at: sc.changed_at, reason: sc.reason, + changed_at: sc.changed_at, reason: sc.reason } end @@ -105,7 +105,7 @@ def show events: events, sources: sources, departments: departments, - status_changes: status_changes, + status_changes: status_changes } end @@ -118,7 +118,7 @@ def sources section: cs.section, reference: cs.reference, excerpt: cs.excerpt, relevance_note: cs.relevance_note, title: cs.source&.title, source_type: cs.source&.source_type, - url: cs.source&.url, date: cs.source&.date, + url: cs.source&.url, date: cs.source&.date } end render json: result @@ -159,7 +159,7 @@ def serialize_commitment_summary(c) policy_area_name: c.policy_area&.name, policy_area_slug: c.policy_area&.slug, criteria_count: c.criteria.size, - matches_count: c.commitment_matches.size, + matches_count: c.commitment_matches.size } end @@ -195,7 +195,7 @@ def status new_status: commitment.status, reasoning: reasoning, effective_date: effective_date, - source_ids: sources.pluck(:id), + source_ids: sources.pluck(:id) } end end diff --git a/app/controllers/api/agent/criteria_controller.rb b/app/controllers/api/agent/criteria_controller.rb index 373fa2e..2f9c6a1 100644 --- a/app/controllers/api/agent/criteria_controller.rb +++ b/app/controllers/api/agent/criteria_controller.rb @@ -34,7 +34,7 @@ def update previous_status: previous_status, new_status: criterion.status, evidence_notes: evidence_notes, - source_id: source.id, + source_id: source.id } end end diff --git a/app/controllers/api/agent/entries_controller.rb b/app/controllers/api/agent/entries_controller.rb index ce8b592..28cc273 100644 --- a/app/controllers/api/agent/entries_controller.rb +++ b/app/controllers/api/agent/entries_controller.rb @@ -20,7 +20,7 @@ def index id: e.id, title: e.title, url: e.url, published_at: e.published_at, scraped_at: e.scraped_at, activities_extracted_at: e.activities_extracted_at, - feed_title: e.feed&.title, + feed_title: e.feed&.title } } end @@ -37,7 +37,7 @@ def show parsed_markdown: entry.parsed_markdown, activities_extracted_at: entry.activities_extracted_at, feed_title: entry.feed&.title, - feed_source_url: entry.feed&.source_url, + feed_source_url: entry.feed&.source_url } end diff --git a/app/controllers/api/agent/pages_controller.rb b/app/controllers/api/agent/pages_controller.rb index 583cd5b..9fbc98c 100644 --- a/app/controllers/api/agent/pages_controller.rb +++ b/app/controllers/api/agent/pages_controller.rb @@ -60,7 +60,7 @@ def fetch content_markdown: parsed_markdown, published_date: published_date, source_id: source.id, - source_existed: !source.previously_new_record?, + source_existed: !source.previously_new_record? } end diff --git a/app/controllers/commitments_controller.rb b/app/controllers/commitments_controller.rb index c5f1abf..c71469f 100644 --- a/app/controllers/commitments_controller.rb +++ b/app/controllers/commitments_controller.rb @@ -52,10 +52,10 @@ def sort_direction end def page_size - [(params[:per_page] || 50).to_i, 1000].min + [ (params[:per_page] || 50).to_i, 1000 ].min end def page_offset - [(params[:page] || 1).to_i - 1, 0].max * page_size + [ (params[:page] || 1).to_i - 1, 0 ].max * page_size end end diff --git a/app/controllers/feed_items_controller.rb b/app/controllers/feed_items_controller.rb index 4ecc920..340798c 100644 --- a/app/controllers/feed_items_controller.rb +++ b/app/controllers/feed_items_controller.rb @@ -22,11 +22,11 @@ def index private def current_page - [(params[:page] || 1).to_i, 1].max + [ (params[:page] || 1).to_i, 1 ].max end def page_size - [(params[:per_page] || 50).to_i, 100].min + [ (params[:per_page] || 50).to_i, 100 ].min end def page_offset diff --git a/app/jobs/agent_evaluate_commitment_job.rb b/app/jobs/agent_evaluate_commitment_job.rb index 44edf81..7a7745a 100644 --- a/app/jobs/agent_evaluate_commitment_job.rb +++ b/app/jobs/agent_evaluate_commitment_job.rb @@ -58,7 +58,7 @@ def build_cmd(prompt, hook_script:) "--permission-mode", "bypassPermissions", "--model", ENV.fetch("AGENT_MODEL", "claude-sonnet-4-6"), "--output-format", "text", - "--settings", hook_settings, + "--settings", hook_settings ] end @@ -68,7 +68,7 @@ def allowed_tools "WebFetch(https://*.canada.ca/*)", "WebFetch(https://*.gc.ca/*)", "WebFetch(https://www.parl.ca/*)", - "WebSearch", + "WebSearch" ] end @@ -92,7 +92,7 @@ def agent_env(commitment_id: nil, entry_id: nil) "ENTRY_ID" => entry_id&.to_s, # Explicitly unset — subprocess must not access Rails credentials "RAILS_MASTER_KEY" => nil, - "SECRET_KEY_BASE" => nil, + "SECRET_KEY_BASE" => nil }.compact end end diff --git a/app/jobs/agent_process_entry_job.rb b/app/jobs/agent_process_entry_job.rb index 080c941..f82af58 100644 --- a/app/jobs/agent_process_entry_job.rb +++ b/app/jobs/agent_process_entry_job.rb @@ -50,7 +50,7 @@ def build_cmd(prompt, hook_script:) "--permission-mode", "bypassPermissions", "--model", ENV.fetch("AGENT_MODEL", "claude-opus-4-6"), "--output-format", "text", - "--settings", hook_settings, + "--settings", hook_settings ] end @@ -60,7 +60,7 @@ def allowed_tools "WebFetch(https://*.canada.ca/*)", "WebFetch(https://*.gc.ca/*)", "WebFetch(https://www.parl.ca/*)", - "WebSearch", + "WebSearch" ] end @@ -84,7 +84,7 @@ def agent_env(commitment_id: nil, entry_id: nil) "ENTRY_ID" => entry_id&.to_s, # Explicitly unset — subprocess must not access Rails credentials "RAILS_MASTER_KEY" => nil, - "SECRET_KEY_BASE" => nil, + "SECRET_KEY_BASE" => nil }.compact end end diff --git a/app/jobs/commitment_assessment_job.rb b/app/jobs/commitment_assessment_job.rb index 63a7414..799e4ad 100644 --- a/app/jobs/commitment_assessment_job.rb +++ b/app/jobs/commitment_assessment_job.rb @@ -49,7 +49,7 @@ def perform(commitment) def evidence_date_for(matchable) case matchable when Entry then matchable.published_at - when Bill then [matchable.passed_house_first_reading_at, matchable.latest_activity_at].compact.max + when Bill then [ matchable.passed_house_first_reading_at, matchable.latest_activity_at ].compact.max when StatcanDataset then matchable.last_synced_at end end @@ -93,11 +93,11 @@ def source_for(matchable, government) def source_type_for_feed(feed) title = feed.title.to_s.downcase if title.include?("gazette") - [:gazette_notice, nil] + [ :gazette_notice, nil ] elsif title.include?("committee") - [:committee_report, nil] + [ :committee_report, nil ] else - [:other, feed.title] + [ :other, feed.title ] end end end diff --git a/app/jobs/source_document_processor_job.rb b/app/jobs/source_document_processor_job.rb index d613a00..a04469f 100644 --- a/app/jobs/source_document_processor_job.rb +++ b/app/jobs/source_document_processor_job.rb @@ -85,13 +85,13 @@ def extract_pdf_pages(attachment) end def chunk_pages(pages, per_chunk: 10) - return [build_chunk(pages, 1, pages.size)] if pages.size <= per_chunk + return [ build_chunk(pages, 1, pages.size) ] if pages.size <= per_chunk chunks = [] start_idx = 0 while start_idx < pages.size - end_idx = [start_idx + per_chunk, pages.size].min + end_idx = [ start_idx + per_chunk, pages.size ].min chunks << build_chunk(pages[start_idx...end_idx], start_idx + 1, end_idx) break if end_idx >= pages.size start_idx = end_idx - 1 # 1-page overlap diff --git a/app/mailers/test_mailer.rb b/app/mailers/test_mailer.rb index ff23184..bce8b73 100644 --- a/app/mailers/test_mailer.rb +++ b/app/mailers/test_mailer.rb @@ -3,10 +3,10 @@ def test_email(recipient_email) @recipient_email = recipient_email @test_time = Time.current @environment = Rails.env - + mail( to: recipient_email, subject: "[OutcomeTracker] Test Email - #{Rails.env.capitalize}" ) end -end \ No newline at end of file +end diff --git a/app/models/concerns/mcp_rack_tool.rb b/app/models/concerns/mcp_rack_tool.rb index 3ac6888..c693007 100644 --- a/app/models/concerns/mcp_rack_tool.rb +++ b/app/models/concerns/mcp_rack_tool.rb @@ -19,7 +19,7 @@ def call(server_context:, **params) path = path_template.gsub(/:(\w+)/) { params[$1.to_sym] } query_params = params.except(*path_params) response = rack_get(path, query_params) - MCP::Tool::Response.new([{ type: "text", text: response }]) + MCP::Tool::Response.new([ { type: "text", text: response } ]) end private diff --git a/app/models/criterion_assessment.rb b/app/models/criterion_assessment.rb index 1cf0d68..7576a70 100644 --- a/app/models/criterion_assessment.rb +++ b/app/models/criterion_assessment.rb @@ -25,5 +25,4 @@ def create_feed_item occurred_at: assessed_at ) end - end diff --git a/app/services/ministers_fetcher.rb b/app/services/ministers_fetcher.rb index b27c42e..69c15f5 100644 --- a/app/services/ministers_fetcher.rb +++ b/app/services/ministers_fetcher.rb @@ -45,7 +45,7 @@ def parse_ministries_xml(xml_body) last_name: node.at_xpath("PersonOfficialLastName")&.text, title: node.at_xpath("Title")&.text, from_date: node.at_xpath("FromDateTime")&.text, - to_date: node.at_xpath("ToDateTime")&.text, + to_date: node.at_xpath("ToDateTime")&.text } end end @@ -62,7 +62,7 @@ def fetch_profile_xml(person_id) { constituency: role.at_xpath("ConstituencyName")&.text, province: role.at_xpath("ConstituencyProvinceTerritoryName")&.text, - party: role.at_xpath("CaucusShortName")&.text, + party: role.at_xpath("CaucusShortName")&.text } rescue => e Rails.logger.warn("Failed to fetch profile XML for person #{person_id}: #{e.message}") diff --git a/app/views/commitments/index.json.jbuilder b/app/views/commitments/index.json.jbuilder index b31ba64..d57dd98 100644 --- a/app/views/commitments/index.json.jbuilder +++ b/app/views/commitments/index.json.jbuilder @@ -17,5 +17,5 @@ end json.meta do json.total_count @total_count json.page (params[:page] || 1).to_i - json.per_page [(params[:per_page] || 50).to_i, 100].min + json.per_page [ (params[:per_page] || 50).to_i, 100 ].min end diff --git a/db/migrate/20250805000001_create_commitments.rb b/db/migrate/20250805000001_create_commitments.rb index 26e93c4..5b9d3ac 100644 --- a/db/migrate/20250805000001_create_commitments.rb +++ b/db/migrate/20250805000001_create_commitments.rb @@ -24,7 +24,7 @@ def change add_index :commitments, :commitment_type add_index :commitments, :status - add_index :commitments, [:government_id, :commitment_type] - add_index :commitments, [:government_id, :status] + add_index :commitments, [ :government_id, :commitment_type ] + add_index :commitments, [ :government_id, :status ] end end diff --git a/db/migrate/20250805000003_create_criteria.rb b/db/migrate/20250805000003_create_criteria.rb index a2beb20..1392060 100644 --- a/db/migrate/20250805000003_create_criteria.rb +++ b/db/migrate/20250805000003_create_criteria.rb @@ -16,8 +16,8 @@ def change end add_index :criteria, :status - add_index :criteria, [:commitment_id, :category] - add_index :criteria, [:commitment_id, :category, :status] - add_index :criteria, [:assessed_by_type, :assessed_by_id] + add_index :criteria, [ :commitment_id, :category ] + add_index :criteria, [ :commitment_id, :category, :status ] + add_index :criteria, [ :assessed_by_type, :assessed_by_id ] end end diff --git a/db/migrate/20250805000004_create_commitment_departments.rb b/db/migrate/20250805000004_create_commitment_departments.rb index d52656e..9e39376 100644 --- a/db/migrate/20250805000004_create_commitment_departments.rb +++ b/db/migrate/20250805000004_create_commitment_departments.rb @@ -8,6 +8,6 @@ def change t.timestamps end - add_index :commitment_departments, [:commitment_id, :department_id], unique: true + add_index :commitment_departments, [ :commitment_id, :department_id ], unique: true end end diff --git a/db/migrate/20250805000006_create_sources.rb b/db/migrate/20250805000006_create_sources.rb index bab53f2..ffc1317 100644 --- a/db/migrate/20250805000006_create_sources.rb +++ b/db/migrate/20250805000006_create_sources.rb @@ -10,6 +10,6 @@ def change end add_index :sources, :source_type - add_index :sources, [:government_id, :source_type] + add_index :sources, [ :government_id, :source_type ] end end diff --git a/db/migrate/20260305000004_create_criterion_assessments.rb b/db/migrate/20260305000004_create_criterion_assessments.rb index a728087..ec4fba9 100644 --- a/db/migrate/20260305000004_create_criterion_assessments.rb +++ b/db/migrate/20260305000004_create_criterion_assessments.rb @@ -11,6 +11,6 @@ def change t.timestamps end - add_index :criterion_assessments, [:criterion_id, :assessed_at] + add_index :criterion_assessments, [ :criterion_id, :assessed_at ] end end diff --git a/db/migrate/20260305000005_remove_assessed_by_from_criteria.rb b/db/migrate/20260305000005_remove_assessed_by_from_criteria.rb index f33fa51..527ec83 100644 --- a/db/migrate/20260305000005_remove_assessed_by_from_criteria.rb +++ b/db/migrate/20260305000005_remove_assessed_by_from_criteria.rb @@ -1,6 +1,6 @@ class RemoveAssessedByFromCriteria < ActiveRecord::Migration[8.0] def change - remove_index :criteria, [:assessed_by_type, :assessed_by_id] + remove_index :criteria, [ :assessed_by_type, :assessed_by_id ] remove_column :criteria, :assessed_by_type, :string remove_column :criteria, :assessed_by_id, :bigint end diff --git a/db/migrate/20260305000010_create_commitment_matches.rb b/db/migrate/20260305000010_create_commitment_matches.rb index 37f4f3d..b4f5d95 100644 --- a/db/migrate/20260305000010_create_commitment_matches.rb +++ b/db/migrate/20260305000010_create_commitment_matches.rb @@ -13,11 +13,11 @@ def change t.timestamps end - add_index :commitment_matches, [:commitment_id, :matchable_type, :matchable_id], + add_index :commitment_matches, [ :commitment_id, :matchable_type, :matchable_id ], unique: true, name: "idx_commitment_matches_unique" - add_index :commitment_matches, [:matchable_type, :matchable_id], + add_index :commitment_matches, [ :matchable_type, :matchable_id ], name: "idx_commitment_matches_matchable" - add_index :commitment_matches, [:commitment_id, :assessed], + add_index :commitment_matches, [ :commitment_id, :assessed ], name: "idx_commitment_matches_unassessed" end end diff --git a/db/migrate/20260306000001_create_commitment_status_changes.rb b/db/migrate/20260306000001_create_commitment_status_changes.rb index 41748bd..d51a4e6 100644 --- a/db/migrate/20260306000001_create_commitment_status_changes.rb +++ b/db/migrate/20260306000001_create_commitment_status_changes.rb @@ -9,6 +9,6 @@ def change t.timestamps end - add_index :commitment_status_changes, [:commitment_id, :changed_at] + add_index :commitment_status_changes, [ :commitment_id, :changed_at ] end end diff --git a/db/migrate/20260306000002_create_commitment_events.rb b/db/migrate/20260306000002_create_commitment_events.rb index ee05f5d..2c5368b 100644 --- a/db/migrate/20260306000002_create_commitment_events.rb +++ b/db/migrate/20260306000002_create_commitment_events.rb @@ -12,7 +12,7 @@ def change t.timestamps end - add_index :commitment_events, [:commitment_id, :occurred_at] + add_index :commitment_events, [ :commitment_id, :occurred_at ] add_index :commitment_events, :event_type add_index :commitment_events, :action_type end diff --git a/db/migrate/20260306000003_create_commitment_revisions.rb b/db/migrate/20260306000003_create_commitment_revisions.rb index 685f763..de6349e 100644 --- a/db/migrate/20260306000003_create_commitment_revisions.rb +++ b/db/migrate/20260306000003_create_commitment_revisions.rb @@ -12,6 +12,6 @@ def change t.timestamps end - add_index :commitment_revisions, [:commitment_id, :revision_date] + add_index :commitment_revisions, [ :commitment_id, :revision_date ] end end diff --git a/db/migrate/20260306000004_create_feed_items.rb b/db/migrate/20260306000004_create_feed_items.rb index fd42ec3..54e18fd 100644 --- a/db/migrate/20260306000004_create_feed_items.rb +++ b/db/migrate/20260306000004_create_feed_items.rb @@ -12,10 +12,10 @@ def change t.timestamps end - add_index :feed_items, [:feedable_type, :feedable_id] - add_index :feed_items, [:commitment_id, :occurred_at] - add_index :feed_items, [:policy_area_id, :occurred_at] - add_index :feed_items, [:event_type, :occurred_at] + add_index :feed_items, [ :feedable_type, :feedable_id ] + add_index :feed_items, [ :commitment_id, :occurred_at ] + add_index :feed_items, [ :policy_area_id, :occurred_at ] + add_index :feed_items, [ :event_type, :occurred_at ] add_index :feed_items, :occurred_at end end diff --git a/db/migrate/20260324000001_create_evaluation_runs.rb b/db/migrate/20260324000001_create_evaluation_runs.rb index 31a2067..9afd8f1 100644 --- a/db/migrate/20260324000001_create_evaluation_runs.rb +++ b/db/migrate/20260324000001_create_evaluation_runs.rb @@ -15,7 +15,7 @@ def change end add_index :evaluation_runs, :agent_run_id - add_index :evaluation_runs, [:commitment_id, :created_at] + add_index :evaluation_runs, [ :commitment_id, :created_at ] add_index :evaluation_runs, :trigger_type end end diff --git a/lib/tasks/backfill_evidence_dates.rake b/lib/tasks/backfill_evidence_dates.rake index 0881755..cf6d95b 100644 --- a/lib/tasks/backfill_evidence_dates.rake +++ b/lib/tasks/backfill_evidence_dates.rake @@ -23,7 +23,7 @@ end def evidence_date_for(matchable) case matchable when Entry then matchable.published_at - when Bill then [matchable.passed_house_first_reading_at, matchable.latest_activity_at].compact.max + when Bill then [ matchable.passed_house_first_reading_at, matchable.latest_activity_at ].compact.max when StatcanDataset then matchable.last_synced_at end end diff --git a/lib/tasks/database_backup.rake b/lib/tasks/database_backup.rake index 90fcee8..d5ee172 100644 --- a/lib/tasks/database_backup.rake +++ b/lib/tasks/database_backup.rake @@ -13,7 +13,7 @@ namespace :db do filename = "#{database}_backup_#{timestamp}.dump" filepath = backup_dir.join(filename) - cmd_parts = ["pg_dump"] + cmd_parts = [ "pg_dump" ] cmd_parts << "--host=#{db_config['host']}" if db_config["host"] cmd_parts << "--port=#{db_config['port']}" if db_config["port"] cmd_parts << "--username=#{db_config['username']}" if db_config["username"] diff --git a/lib/tasks/database_restore.rake b/lib/tasks/database_restore.rake index ce4f8ad..dcc2540 100644 --- a/lib/tasks/database_restore.rake +++ b/lib/tasks/database_restore.rake @@ -8,7 +8,7 @@ def db_config_for_env(target_env) port: config["port"], username: config["username"], password: config["password"], - database: config["database"], + database: config["database"] } else enc_path = Rails.root.join("config/credentials/#{target_env}.yml.enc") @@ -43,7 +43,7 @@ def db_config_for_env(target_env) port: db[:port], username: db[:username], password: db[:password], - database: db[:database], + database: db[:database] } end end @@ -62,7 +62,7 @@ end def restore_dump(dump_file, db_config) require "open3" - pg_restore_cmd = ["pg_restore"] + pg_restore_cmd = [ "pg_restore" ] pg_restore_cmd << "--host=#{db_config[:host]}" if db_config[:host] pg_restore_cmd << "--port=#{db_config[:port]}" if db_config[:port] pg_restore_cmd << "--username=#{db_config[:username]}" if db_config[:username] @@ -88,7 +88,7 @@ end namespace :db do desc "Restore database to a target environment. Usage: rake db:restore[production] or DUMP_FILE=/path rake db:restore[staging]" - task :restore, [:env] => :environment do |_t, args| + task :restore, [ :env ] => :environment do |_t, args| target_env = args[:env] || Rails.env dump_file = ENV["DUMP_FILE"] || find_latest_backup diff --git a/lib/tasks/dedup.rake b/lib/tasks/dedup.rake index 17ca3d1..d47c670 100644 --- a/lib/tasks/dedup.rake +++ b/lib/tasks/dedup.rake @@ -4,185 +4,185 @@ namespace :dedup do # extraction results. CROSS_GROUP_MERGES = [ # === Previously merged (kept for idempotency — skipped if IDs no longer active) === - [2851, 3234, 2901, 3298], - [2655, 2746, 2977, 2868, 3251], - [2869, 3252, 2918, 3313], - [2399, 2936], - [2495, 3017, 2932, 2940, 3274], - [2859, 2909, 3241, 3305], + [ 2851, 3234, 2901, 3298 ], + [ 2655, 2746, 2977, 2868, 3251 ], + [ 2869, 3252, 2918, 3313 ], + [ 2399, 2936 ], + [ 2495, 3017, 2932, 2940, 3274 ], + [ 2859, 2909, 3241, 3305 ], # === Finance === - [2509, 2555], # SR&ED claimable amount $6M - [2514, 2560], # 20% AI Deployment Tax Credit for SMEs - [2567, 2587], # Labour Mobility Tax Deduction skilled trades - [2569, 2606], # $150B private sector investment - [2469, 2574], # GST cut first-time homebuyers - [2528, 2592], # Special Import Measures Act modernization - [2530, 2594], # Debt-to-GDP declining - [2536, 2599], # Separation of capital and operating spending - [2486, 2544], # Critical Mineral Exploration Tax Credit expansion - [2548, 2657, 2979], # Sustainable Investment Guidelines - [2549, 2980], # Sustainable Bond Framework - [2793, 3189], # Immediate expensing manufacturing buildings - [2646, 2969], # Carbon contracts for difference via CGF - [2652, 2974], # Crown corp Clean Electricity ITC eligibility - [2654, 2976], # Domestic content requirements clean economy - [2666, 2988, 3265], # Prohibit investment account transfer fees - [2667, 2989], # Timely transfer of registered accounts - [2668, 2990], # Cross-border transfer fee transparency - [2670, 2991, 3267], # Credit union scaling and federal entry - [2720, 2805, 3197, 3281], # Underused Housing Tax elimination - [2839, 2892, 3222, 3292], # Finance officials civil liability protection - [2843, 3226], # Third-party cash deposits restriction - [2844, 3227], # Public-private info sharing AML - [2846, 3229], # PCMLTFA regulatory authority - [2848, 3231], # PCMLTFA all financial donations - [2849, 2897, 3296], # AML supervision and penalties - [2852, 2903, 3299], # Insured mortgage protected limit $500B - [2853, 2904, 3300], # Borrowing Authority Act limit increase - [2855, 2906, 3237, 3302], # Duty drawback charitable donations - [2867, 3250, 3312], # Canada Development Investment Corporation - [2886, 2993, 3270], # Consumer-driven banking framework - [2831, 3214, 3287], # Electronic notice-and-access financial governance - [2832, 3215, 3288], # Bearer form instruments prohibition - [2809, 2928, 2929, 3184, 3322], # Qualified investment rules registered plans - [2799, 2815, 3070, 3173, 3193], # Tiered corporate structures tax deferral - [2808, 2817, 2931, 3196, 3324], # GST/HST osteopathic services - [2819, 2820, 2821, 3199], # Luxury tax aircraft/vessels - [2822, 3200], # 2026-27 borrowing plan update - [2823, 3079, 3202], # Canada Mortgage Bonds $30B purchases - [2895, 3295], # AML technical amendments - [2790, 3164], # Medical Expense/Home Accessibility dual claim - [2792, 3166], # Carbon Rebate post-October 2026 - [2804, 3071], # FAPI foreign affiliate investment income - [2800, 3174], # Exploration expense economic viability studies - [2798, 3172], # CGF financing Clean Electricity ITC cost base - [3219, 3290], # OSFI integrity and security authorities - [3221, 3291], # Retail Payment Act confidentiality - [3035, 3275], # Economic abuse code of conduct - [2724, 3080], # Early learning childcare transfers 3% - [2998, 2999, 3271], # Stablecoin regulation - [2593, 2946], # Declining deficit trajectory - [2814, 3192], # Clean Electricity ITC legislation - [2833, 2889, 3216], # Bank branch closure public notice - [2835, 2890, 3218], # Digital ID verification bank account opening - [2841, 2894, 3294], # Minister of Finance consultation sanctions - [2810, 3163, 3185], # CRA/ESDC info sharing worker misclassification + [ 2509, 2555 ], # SR&ED claimable amount $6M + [ 2514, 2560 ], # 20% AI Deployment Tax Credit for SMEs + [ 2567, 2587 ], # Labour Mobility Tax Deduction skilled trades + [ 2569, 2606 ], # $150B private sector investment + [ 2469, 2574 ], # GST cut first-time homebuyers + [ 2528, 2592 ], # Special Import Measures Act modernization + [ 2530, 2594 ], # Debt-to-GDP declining + [ 2536, 2599 ], # Separation of capital and operating spending + [ 2486, 2544 ], # Critical Mineral Exploration Tax Credit expansion + [ 2548, 2657, 2979 ], # Sustainable Investment Guidelines + [ 2549, 2980 ], # Sustainable Bond Framework + [ 2793, 3189 ], # Immediate expensing manufacturing buildings + [ 2646, 2969 ], # Carbon contracts for difference via CGF + [ 2652, 2974 ], # Crown corp Clean Electricity ITC eligibility + [ 2654, 2976 ], # Domestic content requirements clean economy + [ 2666, 2988, 3265 ], # Prohibit investment account transfer fees + [ 2667, 2989 ], # Timely transfer of registered accounts + [ 2668, 2990 ], # Cross-border transfer fee transparency + [ 2670, 2991, 3267 ], # Credit union scaling and federal entry + [ 2720, 2805, 3197, 3281 ], # Underused Housing Tax elimination + [ 2839, 2892, 3222, 3292 ], # Finance officials civil liability protection + [ 2843, 3226 ], # Third-party cash deposits restriction + [ 2844, 3227 ], # Public-private info sharing AML + [ 2846, 3229 ], # PCMLTFA regulatory authority + [ 2848, 3231 ], # PCMLTFA all financial donations + [ 2849, 2897, 3296 ], # AML supervision and penalties + [ 2852, 2903, 3299 ], # Insured mortgage protected limit $500B + [ 2853, 2904, 3300 ], # Borrowing Authority Act limit increase + [ 2855, 2906, 3237, 3302 ], # Duty drawback charitable donations + [ 2867, 3250, 3312 ], # Canada Development Investment Corporation + [ 2886, 2993, 3270 ], # Consumer-driven banking framework + [ 2831, 3214, 3287 ], # Electronic notice-and-access financial governance + [ 2832, 3215, 3288 ], # Bearer form instruments prohibition + [ 2809, 2928, 2929, 3184, 3322 ], # Qualified investment rules registered plans + [ 2799, 2815, 3070, 3173, 3193 ], # Tiered corporate structures tax deferral + [ 2808, 2817, 2931, 3196, 3324 ], # GST/HST osteopathic services + [ 2819, 2820, 2821, 3199 ], # Luxury tax aircraft/vessels + [ 2822, 3200 ], # 2026-27 borrowing plan update + [ 2823, 3079, 3202 ], # Canada Mortgage Bonds $30B purchases + [ 2895, 3295 ], # AML technical amendments + [ 2790, 3164 ], # Medical Expense/Home Accessibility dual claim + [ 2792, 3166 ], # Carbon Rebate post-October 2026 + [ 2804, 3071 ], # FAPI foreign affiliate investment income + [ 2800, 3174 ], # Exploration expense economic viability studies + [ 2798, 3172 ], # CGF financing Clean Electricity ITC cost base + [ 3219, 3290 ], # OSFI integrity and security authorities + [ 3221, 3291 ], # Retail Payment Act confidentiality + [ 3035, 3275 ], # Economic abuse code of conduct + [ 2724, 3080 ], # Early learning childcare transfers 3% + [ 2998, 2999, 3271 ], # Stablecoin regulation + [ 2593, 2946 ], # Declining deficit trajectory + [ 2814, 3192 ], # Clean Electricity ITC legislation + [ 2833, 2889, 3216 ], # Bank branch closure public notice + [ 2835, 2890, 3218 ], # Digital ID verification bank account opening + [ 2841, 2894, 3294 ], # Minister of Finance consultation sanctions + [ 2810, 3163, 3185 ], # CRA/ESDC info sharing worker misclassification # === Defence === - [2385, 2543], # NATO 2% GDP spending target - [2541, 2602], # $30.9B defence spending over four years - [2773, 3140], # Retire obsolete CAF fleets - [2774, 3141], # Divest surplus DND property - [2775, 3142], # Energy Performance Contracts defence - [2942, 3046], # Defence Industrial Strategy + [ 2385, 2543 ], # NATO 2% GDP spending target + [ 2541, 2602 ], # $30.9B defence spending over four years + [ 2773, 3140 ], # Retire obsolete CAF fleets + [ 2774, 3141 ], # Divest surplus DND property + [ 2775, 3142 ], # Energy Performance Contracts defence + [ 2942, 3046 ], # Defence Industrial Strategy # === Transport === - [2357, 2575], # High-speed rail Windsor-Quebec - [3236, 3282], # Alto HSR legislation accelerate - [2637, 2954], # Airport lease extensions - [2638, 2955], # Airport privatization options - [2639, 2956, 3263], # Airport safety infrastructure $55.2M - [2865, 3248, 3310], # Aeronautics Act aviation safety - [2866, 3249, 3311], # Temporary orders international transport standards - [3008, 3154], # $5B Trade Diversification Corridors Fund + [ 2357, 2575 ], # High-speed rail Windsor-Quebec + [ 3236, 3282 ], # Alto HSR legislation accelerate + [ 2637, 2954 ], # Airport lease extensions + [ 2638, 2955 ], # Airport privatization options + [ 2639, 2956, 3263 ], # Airport safety infrastructure $55.2M + [ 2865, 3248, 3310 ], # Aeronautics Act aviation safety + [ 2866, 3249, 3311 ], # Temporary orders international transport standards + [ 3008, 3154 ], # $5B Trade Diversification Corridors Fund # === Environment === - [2650, 2971], # EV Availability Standard 2026 target - [2676, 2972], # Clean Fuel Regulations biofuels - [2661, 2983], # Climate Competitiveness Strategy metrics - [2717, 3075], # Output-Based Pricing System maintenance - [2647, 2973], # CEPA clean electricity agreements - [3005, 3272], # Landfill methane regulations funding + [ 2650, 2971 ], # EV Availability Standard 2026 target + [ 2676, 2972 ], # Clean Fuel Regulations biofuels + [ 2661, 2983 ], # Climate Competitiveness Strategy metrics + [ 2717, 3075 ], # Output-Based Pricing System maintenance + [ 2647, 2973 ], # CEPA clean electricity agreements + [ 3005, 3272 ], # Landfill methane regulations funding # === CRA === - [2690, 3007], # Corporate tax/GST deferral liquidity - [2718, 2732], # Wind down CRA fuel charge/DST units + [ 2690, 3007 ], # Corporate tax/GST deferral liquidity + [ 2718, 2732 ], # Wind down CRA fuel charge/DST units # === Jobs and Families === - [2682, 3006], # $570M LMDA tariff-impacted workers - [2565, 3019, 3102], # Union Training and Innovation Program - [2521, 2566], # $20M college apprenticeship training - [2742, 3103], # Foreign Credential Recognition Fund - [2913, 3245, 3308], # Government Annuities audit elimination - [3069, 2884], # Student grants/loans public institutions only + [ 2682, 3006 ], # $570M LMDA tariff-impacted workers + [ 2565, 3019, 3102 ], # Union Training and Innovation Program + [ 2521, 2566 ], # $20M college apprenticeship training + [ 2742, 3103 ], # Foreign Credential Recognition Fund + [ 2913, 3245, 3308 ], # Government Annuities audit elimination + [ 3069, 2884 ], # Student grants/loans public institutions only # === Housing & Infrastructure === - [2642, 2961], # $6B Direct Delivery infrastructure stream - [2759, 2948], # Build Communities Strong Fund - [2495, 3205, 3259], # Build Canada Homes establishment + [ 2642, 2961 ], # $6B Direct Delivery infrastructure stream + [ 2759, 2948 ], # Build Communities Strong Fund + [ 2495, 3205, 3259 ], # Build Canada Homes establishment # === Industry === - [2513, 2559], # Black Entrepreneurship Program permanent - [2662, 2984], # Dig once policy fibre optic - [2664, 2986], # Spectrum Licence Transfer Framework - [2767, 3135], # Net Zero Accelerator wind down - [2769, 3137], # Statistics Canada data collection frequency - [2921, 3317], # Predatory debt advisors penalties - [2952, 3160], # Granting council 2% savings - [2660, 2880, 2982], # Competition Act greenwashing provisions + [ 2513, 2559 ], # Black Entrepreneurship Program permanent + [ 2662, 2984 ], # Dig once policy fibre optic + [ 2664, 2986 ], # Spectrum Licence Transfer Framework + [ 2767, 3135 ], # Net Zero Accelerator wind down + [ 2769, 3137 ], # Statistics Canada data collection frequency + [ 2921, 3317 ], # Predatory debt advisors penalties + [ 2952, 3160 ], # Granting council 2% savings + [ 2660, 2880, 2982 ], # Competition Act greenwashing provisions # === Global Affairs === - [2525, 2589], # $25B export credit facility - [2526, 2590], # CanExport program diversification - [2527, 2591], # MERCOSUR/ASEAN trade agreements - [2534, 2597], # OECD global tax rules leadership - [2752, 3113, 3114], # Embassy consolidation/co-location - [2856, 2905, 3238, 3301], # Export and Import Permits Act security - [2893, 3223, 3293], # Windfall profit charge sanctions - [2824, 3207], # IDRC Board reduction + [ 2525, 2589 ], # $25B export credit facility + [ 2526, 2590 ], # CanExport program diversification + [ 2527, 2591 ], # MERCOSUR/ASEAN trade agreements + [ 2534, 2597 ], # OECD global tax rules leadership + [ 2752, 3113, 3114 ], # Embassy consolidation/co-location + [ 2856, 2905, 3238, 3301 ], # Export and Import Permits Act security + [ 2893, 3223, 3293 ], # Windfall profit charge sanctions + [ 2824, 3207 ], # IDRC Board reduction # === Heritage === - [2518, 2563], # E-book accessibility 2030 - [2696, 2864], # Artist's Resale Right - [2875, 2924, 3319], # Broadcasting Act privacy rights - [3032, 3261, 3277], # CBC/Radio-Canada funding - [2377, 3089], # Canada Strong Pass 2025 - [2739, 3097], # Heritage digital platform + [ 2518, 2563 ], # E-book accessibility 2030 + [ 2696, 2864 ], # Artist's Resale Right + [ 2875, 2924, 3319 ], # Broadcasting Act privacy rights + [ 3032, 3261, 3277 ], # CBC/Radio-Canada funding + [ 2377, 3089 ], # Canada Strong Pass 2025 + [ 2739, 3097 ], # Heritage digital platform # === Health === - [2753, 3120], # CFIA lab consolidation - [2754, 3115], # CFIA pet export digital - [2755, 3116], # CFIA vehicle washing stations - [2756, 3117], # CFIA food grading dispute resolution - [2758, 3121], # PHAC grants consolidation + [ 2753, 3120 ], # CFIA lab consolidation + [ 2754, 3115 ], # CFIA pet export digital + [ 2755, 3116 ], # CFIA vehicle washing stations + [ 2756, 3117 ], # CFIA food grading dispute resolution + [ 2758, 3121 ], # PHAC grants consolidation # === Public Safety === - [2713, 3058], # $1.76B law enforcement - [2714, 3059], # $834M CBSA enhancement - [2781, 3146], # $617.7M CBSA operations - [2783, 3037, 3148], # 1000 RCMP personnel - [2778, 3144], # Resilience Centre discontinue - [2780, 3145], # CBSA fleet lifecycle 10 years - [2784, 3149], # RCMP cannabis reimbursement $6 + [ 2713, 3058 ], # $1.76B law enforcement + [ 2714, 3059 ], # $834M CBSA enhancement + [ 2781, 3146 ], # $617.7M CBSA operations + [ 2783, 3037, 3148 ], # 1000 RCMP personnel + [ 2778, 3144 ], # Resilience Centre discontinue + [ 2780, 3145 ], # CBSA fleet lifecycle 10 years + [ 2784, 3149 ], # RCMP cannabis reimbursement $6 # === Gov Transformation === - [2788, 3152], # AI chatbots PSPC - [2789, 3151], # Redundant software/fixed lines - [3053, 3279], # Industrial Security Program + [ 2788, 3152 ], # AI chatbots PSPC + [ 2789, 3151 ], # Redundant software/fixed lines + [ 3053, 3279 ], # Industrial Security Program # === Treasury Board === - [2825, 3208], # Early Retirement Incentive Program - [2827, 3210], # 2% pension benefit rate - [2828, 3211], # Legislative amendments efficiency - [2871, 2920, 3254, 3315], # Regulatory sandboxes + [ 2825, 3208 ], # Early Retirement Incentive Program + [ 2827, 3210 ], # 2% pension benefit rate + [ 2828, 3211 ], # Legislative amendments efficiency + [ 2871, 2920, 3254, 3315 ], # Regulatory sandboxes # === Other Ministries === - [2422, 2442], # National School Food Program Canadian food - [2857, 2907], # FNFA lending to Indigenous SPVs - [2585, 2622], # GBA+ analysis all measures - [2704, 3159], # WAGE department funding - [2488, 2547], # EV charging stations 2027 - [2656, 2978], # $50M Critical Minerals admin - [2744, 3105], # Greener Homes Grant wind down - [2745, 3106], # 2 Billion Trees end - [2870, 2919, 3253], # LNG export licence 50 years - [2763, 3131], # Settlement Program eligibility limits - [2434, 2937], # Bail/sentencing stricter - [2771, 3138], # Tax Court informal procedure limits - [2772, 3139], # CHRC commissioner consolidation - [2910, 3306], # Tribunal administrative support - [2750, 3110], # Self-assessment small DFO projects + [ 2422, 2442 ], # National School Food Program Canadian food + [ 2857, 2907 ], # FNFA lending to Indigenous SPVs + [ 2585, 2622 ], # GBA+ analysis all measures + [ 2704, 3159 ], # WAGE department funding + [ 2488, 2547 ], # EV charging stations 2027 + [ 2656, 2978 ], # $50M Critical Minerals admin + [ 2744, 3105 ], # Greener Homes Grant wind down + [ 2745, 3106 ], # 2 Billion Trees end + [ 2870, 2919, 3253 ], # LNG export licence 50 years + [ 2763, 3131 ], # Settlement Program eligibility limits + [ 2434, 2937 ], # Bail/sentencing stricter + [ 2771, 3138 ], # Tax Court informal procedure limits + [ 2772, 3139 ], # CHRC commissioner consolidation + [ 2910, 3306 ], # Tribunal administrative support + [ 2750, 3110 ] # Self-assessment small DFO projects ].freeze SIMILARITY_THRESHOLD = 0.80 @@ -216,7 +216,7 @@ namespace :dedup do ActiveRecord::Base.transaction do groups.each do |group| keep = select_keeper(group[:ids]) - duplicates = group[:ids] - [keep.id] + duplicates = group[:ids] - [ keep.id ] duplicates.each do |dup_id| dup_commitment = Commitment.find(dup_id) @@ -252,7 +252,7 @@ namespace :dedup do groups << { ids: active_ids, reason: "Cross-group duplicate (manually identified)", - titles: titles, + titles: titles } end @@ -272,13 +272,13 @@ namespace :dedup do end if existing_group - existing_group[:ids] |= [a.id, b.id] - existing_group[:titles] |= [a.title, b.title] + existing_group[:ids] |= [ a.id, b.id ] + existing_group[:titles] |= [ a.title, b.title ] else groups << { - ids: [a.id, b.id], + ids: [ a.id, b.id ], reason: "Title similarity #{(similarity * 100).round(0)}% in same policy area", - titles: [a.title, b.title], + titles: [ a.title, b.title ] } end @@ -309,7 +309,7 @@ namespace :dedup do .includes(:commitment_sources, :criteria) candidates.max_by do |c| - [c.commitment_sources.size, c.criteria.size, -c.id] + [ c.commitment_sources.size, c.criteria.size, -c.id ] end end From 99d8eb7d5f04dcf4961c38f5d2cbb3c74c3b3eaf Mon Sep 17 00:00:00 2001 From: xrendan Date: Fri, 27 Mar 2026 11:32:49 -0600 Subject: [PATCH 3/4] Update brakeman to 8.0.4 to fix scan_ruby CI job The bin/brakeman script uses --ensure-latest which fails CI when the installed version is behind the latest release. Co-Authored-By: Claude Sonnet 4.6 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 76aa9db..8a6e5a4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -123,7 +123,7 @@ GEM bigdecimal (3.2.2) bootsnap (1.18.6) msgpack (~> 1.2) - brakeman (7.1.0) + brakeman (8.0.4) racc builder (3.3.0) commonmarker (2.3.1) From 95ba7673efc9706483526a86d9b0b2520e06db0d Mon Sep 17 00:00:00 2001 From: xrendan Date: Fri, 27 Mar 2026 11:35:36 -0600 Subject: [PATCH 4/4] Fix streaming: use STDERR with flush for real-time agent output Co-Authored-By: Claude Sonnet 4.6 --- app/jobs/agent_evaluate_commitment_job.rb | 5 ++++- app/jobs/agent_process_entry_job.rb | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/jobs/agent_evaluate_commitment_job.rb b/app/jobs/agent_evaluate_commitment_job.rb index 7a7745a..e569b26 100644 --- a/app/jobs/agent_evaluate_commitment_job.rb +++ b/app/jobs/agent_evaluate_commitment_job.rb @@ -39,7 +39,10 @@ def perform(commitment, trigger_type: "manual", as_of_date: nil) def stream_agent(env, cmd, chdir:) Open3.popen2e(env, *cmd, chdir: chdir) do |stdin, output, thread| stdin.close - output.each_line { |line| $stderr.print(line) } + output.each_line do |line| + STDERR.print(line) + STDERR.flush + end thread.value end end diff --git a/app/jobs/agent_process_entry_job.rb b/app/jobs/agent_process_entry_job.rb index f82af58..6ede719 100644 --- a/app/jobs/agent_process_entry_job.rb +++ b/app/jobs/agent_process_entry_job.rb @@ -31,7 +31,7 @@ def perform(entry) def stream_agent(env, cmd, chdir:) Open3.popen2e(env, *cmd, chdir: chdir) do |stdin, output, thread| stdin.close - output.each_line { |line| $stderr.print(line) } + output.each_line { |line| STDERR.print(line); STDERR.flush } thread.value end end