Implement Redis Caching for Agent API Calls

backend/app/api/v1/routes.py

@@ -30,7 +30,7 @@ async def _run_agents_in_background(report_id: str, token_id: str):
             break # Exit the async for loop on failure
 
 @router.post("/report/generate", response_model=ReportResponse)
-async def generate_report_endpoint(request: ReportRequest, background_tasks: BackgroundTasks, session: AsyncSession = Depends(get_session)):
+async def generate_report_endpoint(request: ReportRequest, background_tasks: BackgroundTasks, session: AsyncSession = Depends(get_db)):
     api_logger.info(f"Received report generation request for token_id: {request.token_id}")
     report_repository = ReportRepository(session)
     report_response = await generate_report(request, report_repository)
@@ -39,7 +39,7 @@ async def generate_report_endpoint(request: ReportRequest, background_tasks: Bac
     return report_response
 
 @router.get("/reports/{report_id}/status")
-async def get_report_status_endpoint(report_id: str, session: AsyncSession = Depends(get_session)):
+async def get_report_status_endpoint(report_id: str, session: AsyncSession = Depends(get_db)):
     api_logger.info(f"Received status request for report_id: {report_id}")
     report_repository = ReportRepository(session)
     report = await get_report_status(report_id, report_repository)
@@ -49,7 +49,7 @@ async def get_report_status_endpoint(report_id: str, session: AsyncSession = Dep
     return {"report_id": report_id, "status": report["status"]}
 
 @router.get("/reports/{report_id}/data")
-async def get_report_data_endpoint(report_id: str, session: AsyncSession = Depends(get_session)):
+async def get_report_data_endpoint(report_id: str, session: AsyncSession = Depends(get_db)):
     api_logger.info(f"Received data request for report_id: {report_id}")
     report_repository = ReportRepository(session)
     report_result = await get_report_data(report_id, report_repository)

backend/app/db/database.py

@@ -1,6 +1,5 @@
 from typing import AsyncGenerator
 from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
-from backend.app.db.base import Base
 from backend.app.core.config import settings
 
 SQLALCHEMY_DATABASE_URL = settings.DATABASE_URL

backend/app/db/migrations/env.py

@@ -1,7 +1,8 @@
+import os
+import sys
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
+from sqlalchemy import engine_from_config, pool
 
 from alembic import context
 
@@ -16,22 +17,9 @@
 
 # add your model's MetaData object here
 # for 'autogenerate' support
-import os, sys
 sys.path.append(os.getcwd())
 from backend.app.db.base import Base
 
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
 target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,

backend/app/db/tests/test_connection.py

@@ -3,7 +3,6 @@
 from dotenv import load_dotenv
 from backend.app.db.connection import DatabaseConnection, initialize_db_connection, close_db_connection
 import unittest.mock
-import asyncpg
 
 # Load environment variables from .env file for testing
 load_dotenv()

backend/app/services/agents/code_audit_agent.py

@@ -1,14 +1,49 @@
 import os
 import re
 import logging
+import json
+import hashlib
 from typing import Dict, Any, List
 import httpx
 from pydantic import BaseModel, Field
 import urllib.parse
 from backend.app.security.rate_limiter import rate_limiter
+from backend.app.utils.cache_utils import cache_request
 
 logger = logging.getLogger(__name__)
 
+def serialize_httpx_response(response: httpx.Response) -> str:
+    """Serializes an httpx.Response object to a JSON string."""
+    return json.dumps({
+        "status_code": response.status_code,
+        "headers": dict(response.headers),
+        "text": response.text,
+    })
+
+def deserialize_httpx_response(data_str: str) -> httpx.Response:
+    """Deserializes a JSON string back into a mock httpx.Response object."""
+    data = json.loads(data_str)
+
+    class MockResponse:
+        def __init__(self, status_code, headers, text):
+            self.status_code = status_code
+            self.headers = headers
+            self.text = text
+
+        def json(self):
+            # Attempt to parse text as JSON, raise JSONDecodeError if not valid JSON
+            return json.loads(self.text)
+
+        def raise_for_status(self):
+            if self.status_code >= 400:
+                # Create a dummy request for the HTTPStatusError
+                request = httpx.Request("GET", "http://cached-response/error")
+                raise httpx.HTTPStatusError(
+                    f"Bad response: {self.status_code}", request=request, response=self
+                )
+
+    return MockResponse(data["status_code"], data["headers"], data["text"])
+
 class CommitActivity(BaseModel):
     total: int
     weeks: List[Dict[str, int]]
@@ -102,7 +137,13 @@ def parse_link_header(link_header_str: str, fallback_len: int) -> int:
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitHub commits).")
                 return repo_data
-            commits_resp = await self.client.get(f"{base_url}/commits?per_page=1", headers=headers)
+            commits_resp = await cache_request(
+                url=f"{base_url}/commits?per_page=1",
+                external_api_call=lambda: self.client.get(f"{base_url}/commits?per_page=1", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.github_token.encode()).hexdigest()[:8]} if self.github_token else {}
+            )
             commits_resp.raise_for_status()
             link_header = commits_resp.headers.get('link') or commits_resp.headers.get('Link')
             repo_data['commits_count'] = parse_link_header(link_header, len(commits_resp.json()))
@@ -111,7 +152,13 @@ def parse_link_header(link_header_str: str, fallback_len: int) -> int:
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitHub contributors).")
                 return repo_data
-            contributors_resp = await self.client.get(f"{base_url}/contributors?per_page=1", headers=headers)
+            contributors_resp = await cache_request(
+                url=f"{base_url}/contributors?per_page=1",
+                external_api_call=lambda: self.client.get(f"{base_url}/contributors?per_page=1", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.github_token.encode()).hexdigest()[:8]} if self.github_token else {}
+            )
             contributors_resp.raise_for_status()
             link_header = contributors_resp.headers.get('link') or contributors_resp.headers.get('Link')
             repo_data['contributors_count'] = parse_link_header(link_header, len(contributors_resp.json()))
@@ -120,7 +167,13 @@ def parse_link_header(link_header_str: str, fallback_len: int) -> int:
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitHub releases).")
                 return repo_data
-            releases_resp = await self.client.get(f"{base_url}/releases/latest", headers=headers)
+            releases_resp = await cache_request(
+                url=f"{base_url}/releases/latest",
+                external_api_call=lambda: self.client.get(f"{base_url}/releases/latest", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.github_token.encode()).hexdigest()[:8]} if self.github_token else {}
+            )
             if releases_resp.status_code == 200:
                 repo_data['latest_release'] = releases_resp.json().get('tag_name', 'N/A')
             else:
@@ -132,7 +185,13 @@ def parse_link_header(link_header_str: str, fallback_len: int) -> int:
                 return repo_data
             search_query = urllib.parse.quote_plus(f"repo:{owner}/{repo}+type:issue")
             search_issues_url = f"https://api.github.com/search/issues?q={search_query}&per_page=1"
-            issues_search_resp = await self.client.get(search_issues_url, headers=headers)
+            issues_search_resp = await cache_request(
+                url=search_issues_url,
+                external_api_call=lambda: self.client.get(search_issues_url, headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.github_token.encode()).hexdigest()[:8]} if self.github_token else {}
+            )
             issues_search_resp.raise_for_status()
             issues_search_data = issues_search_resp.json()
             repo_data['issues_count'] = issues_search_data.get('total_count', 0)
@@ -141,7 +200,13 @@ def parse_link_header(link_header_str: str, fallback_len: int) -> int:
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitHub pull requests).")
                 return repo_data
-            pulls_resp = await self.client.get(f"{base_url}/pulls?state=all&per_page=1", headers=headers)
+            pulls_resp = await cache_request(
+                url=f"{base_url}/pulls?state=all&per_page=1",
+                external_api_call=lambda: self.client.get(f"{base_url}/pulls?state=all&per_page=1", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.github_token.encode()).hexdigest()[:8]} if self.github_token else {}
+            )
             pulls_resp.raise_for_status()
             link_header = pulls_resp.headers.get('link') or pulls_resp.headers.get('Link')
             repo_data['pull_requests_count'] = parse_link_header(link_header, len(pulls_resp.json()))
@@ -174,23 +239,41 @@ async def _fetch_gitlab_repo_data(self, project_id: str) -> Dict[str, Any]:
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitLab commits).")
                 return repo_data
-            commits_resp = await self.client.get(f"{base_url}/repository/commits?per_page=1", headers=headers)
+            commits_resp = await cache_request(
+                url=f"{base_url}/repository/commits?per_page=1",
+                external_api_call=lambda: self.client.get(f"{base_url}/repository/commits?per_page=1", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.gitlab_token.encode()).hexdigest()[:8]} if self.gitlab_token else {}
+            )
             commits_resp.raise_for_status()
             repo_data['commits_count'] = int(commits_resp.headers.get('x-total', 0))
 
             # Fetch contributors count
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitLab contributors).")
                 return repo_data
-            contributors_resp = await self.client.get(f"{base_url}/repository/contributors?per_page=1", headers=headers)
+            contributors_resp = await cache_request(
+                url=f"{base_url}/repository/contributors?per_page=1",
+                external_api_call=lambda: self.client.get(f"{base_url}/repository/contributors?per_page=1", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.gitlab_token.encode()).hexdigest()[:8]} if self.gitlab_token else {}
+            )
             contributors_resp.raise_for_status()
             repo_data['contributors_count'] = int(contributors_resp.headers.get('x-total', 0))
 
             # Fetch latest release (tags in GitLab)
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitLab tags).")
                 return repo_data
-            tags_resp = await self.client.get(f"{base_url}/repository/tags?per_page=1", headers=headers)
+            tags_resp = await cache_request(
+                url=f"{base_url}/repository/tags?per_page=1",
+                external_api_call=lambda: self.client.get(f"{base_url}/repository/tags?per_page=1", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.gitlab_token.encode()).hexdigest()[:8]} if self.gitlab_token else {}
+            )
             if tags_resp.status_code == 200 and tags_resp.json():
                 repo_data['latest_release'] = tags_resp.json()[0].get('name', 'N/A')
             else:
@@ -200,15 +283,27 @@ async def _fetch_gitlab_repo_data(self, project_id: str) -> Dict[str, Any]:
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitLab issues).")
                 return repo_data
-            issues_resp = await self.client.get(f"{base_url}/issues?scope=all&per_page=1", headers=headers)
+            issues_resp = await cache_request(
+                url=f"{base_url}/issues?scope=all&per_page=1",
+                external_api_call=lambda: self.client.get(f"{base_url}/issues?scope=all&per_page=1", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.gitlab_token.encode()).hexdigest()[:8]} if self.gitlab_token else {}
+            )
             issues_resp.raise_for_status()
             repo_data['issues_count'] = int(issues_resp.headers.get('x-total', 0))
 
             # Fetch merge requests count
             if not rate_limiter.check_rate_limit("code_audit_agent"):
                 logger.warning("Rate limit exceeded for code_audit_agent (GitLab merge requests).")
                 return repo_data
-            merge_requests_resp = await self.client.get(f"{base_url}/merge_requests?scope=all&per_page=1", headers=headers)
+            merge_requests_resp = await cache_request(
+                url=f"{base_url}/merge_requests?scope=all&per_page=1",
+                external_api_call=lambda: self.client.get(f"{base_url}/merge_requests?scope=all&per_page=1", headers=headers),
+                serializer=serialize_httpx_response,
+                deserializer=deserialize_httpx_response,
+                params={"token_hash": hashlib.sha256(self.gitlab_token.encode()).hexdigest()[:8]} if self.gitlab_token else {}
+            )
             merge_requests_resp.raise_for_status()
             repo_data['pull_requests_count'] = int(merge_requests_resp.headers.get('x-total', 0))
 

backend/app/services/agents/tests/test_code_audit_agent.py

@@ -2,7 +2,7 @@
 import pytest_asyncio
 import respx
 from httpx import Response, Request, RequestError
-from backend.app.services.agents.code_audit_agent import CodeAuditAgent, CodeMetrics, AuditSummary, CodeAuditResult
+from backend.app.services.agents.code_audit_agent import CodeAuditAgent, CodeMetrics, AuditSummary
 
 
 @pytest_asyncio.fixture

backend/app/services/nlg/__init__.py

@@ -1,2 +0,0 @@
-from .nlg_engine import NLGEngine
-from .report_nlg_engine import ReportNLGEngine

backend/app/services/nlg/tests/test_nlg_engine.py

@@ -2,7 +2,6 @@
 import json
 from unittest.mock import AsyncMock, patch
 from backend.app.services.nlg.nlg_engine import NLGEngine
-from backend.app.services.nlg.llm_client import LLMClient
 from backend.app.services.nlg.prompt_templates import get_template, fill_template
 
 # Concrete implementation for testing purposes

backend/app/services/nlg/tests/test_report_nlg_engine.py

@@ -2,7 +2,6 @@
 import json
 from unittest.mock import AsyncMock, patch
 from backend.app.services.nlg.report_nlg_engine import ReportNLGEngine
-from backend.app.services.nlg.llm_client import LLMClient
 from backend.app.services.nlg.prompt_templates import get_template, fill_template
 
 # Mock the LLMClient for all tests in this module

backend/app/services/report_processor.py

@@ -4,7 +4,7 @@
 from backend.app.services.agents.price_agent import run as price_agent_run
 from backend.app.services.agents.trend_agent import run as trend_agent_run
 from backend.app.services.agents.volume_agent import run as volume_agent_run
-from backend.app.core.storage import save_report_data, set_report_status, try_set_processing
+from backend.app.core.storage import try_set_processing
 from backend.app.services.nlg.report_nlg_engine import ReportNLGEngine
 from backend.app.services.summary.report_summary_engine import ReportSummaryEngine
 from backend.app.db.repositories.report_repository import ReportRepository

backend/app/services/summary/__init__.py

@@ -1,2 +0,0 @@
-from .summary_engine import SummaryEngine
-from .report_summary_engine import ReportSummaryEngine

backend/app/services/summary/summary_engine.py

@@ -3,7 +3,7 @@
 """
 
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List
+from typing import Any, Dict
 
 class SummaryEngine(ABC):
     """

backend/app/services/validation/tests/test_validation_engine.py

@@ -1,4 +1,3 @@
-import pytest
 from backend.app.services.validation.validation_engine import perform_cross_source_checks, normalize_missing
 
 def test_normalize_missing():

backend/app/services/validation/validation_engine.py

@@ -2,7 +2,6 @@
 Validation engine for ensuring data quality and consistency before NLG and summary generation.
 """
 
-import re
 from typing import Dict, Any, Optional, List
 from copy import deepcopy