Skip to content

Changes: Create a background task monitor_generation_time #86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
felixjordandev merged 1 commit into from
Dec 3, 2025
Merged

Changes: Create a background task monitor_generation_time #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backend/app/services/report_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import asyncio
import logging
import time
from sqlalchemy.ext.asyncio import AsyncSession # Import AsyncSession
from backend.app.core.orchestrator import Orchestrator
from backend.app.services.agents.price_agent import run as price_agent_run
from backend.app.services.agents.trend_agent import run as trend_agent_run
Expand Down
134 changes: 106 additions & 28 deletions backend/tests/test_orchestrator_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,11 @@
import asyncio
from unittest.mock import AsyncMock, patch, MagicMock
from backend.app.core.orchestrator import create_orchestrator
from backend.app.services.report_service import in_memory_reports

# Sample data for testing
SAMPLE_REPORT_ID = "test_report_123"
SAMPLE_TOKEN_ID = "ethereum"

@pytest.fixture(autouse=True)
def clear_in_memory_reports():
"""Clears in_memory_reports before each test."""
in_memory_reports.clear()
yield

@pytest.fixture
def mock_settings():
"""Mocks settings required for agent registration."""
Expand Down Expand Up @@ -54,8 +47,35 @@ async def test_orchestrator_full_integration_success(mock_settings):
mock_analyze_code_activity.return_value = {"activity_analysis": "mocked"}
mock_search_and_summarize_audit_reports.return_value = [{"audit_report": "mocked"}]

in_memory_reports[SAMPLE_REPORT_ID] = {"status": "pending", "data": {}}
orchestrator = create_orchestrator()
# Setup mock for ReportRepository and its interactions
from backend.app.db.models.report_state import ReportStatusEnum

mock_session = AsyncMock()
# Mock the async context manager for the session factory
mock_session_factory = AsyncMock(return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_session)))

orchestrator = await create_orchestrator(session_factory=mock_session_factory)
mock_report_repository = orchestrator.report_repository

# Mock initial report state (PENDING)
mock_report_state_pending = AsyncMock()
mock_report_state_pending.report_id = SAMPLE_REPORT_ID
mock_report_state_pending.status = ReportStatusEnum.PENDING
mock_report_state_pending.partial_agent_output = {}

# Mock final report state (COMPLETED)
mock_report_state_completed = AsyncMock()
mock_report_state_completed.report_id = SAMPLE_REPORT_ID
mock_report_state_completed.status = ReportStatusEnum.AGENTS_COMPLETED
mock_report_state_completed.partial_agent_output = {} # Will be updated

# Configure get_report_by_id to return pending then completed after update
mock_report_repository.get_report_by_id.side_effect = [
mock_report_state_pending, # First call during initial status check
mock_report_state_completed # Second call after update_partial for AGENTS_COMPLETED
]
mock_report_repository.update_partial.return_value = mock_report_state_completed # Ensure update returns something

Comment on lines +50 to +78
Copy link

@coderabbitai coderabbitai bot Dec 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Mocking ReportRepository methods via .side_effect/.return_value on real methods will break

Here orchestrator.report_repository is a real ReportRepository instance; its get_report_by_id / update_partial are bound async methods, not mocks. Accessing .side_effect/.return_value on them will not work and will likely raise AttributeError.

Override these attributes with AsyncMock instances instead:

@@
-        orchestrator = await create_orchestrator(session_factory=mock_session_factory)
-        mock_report_repository = orchestrator.report_repository
+        orchestrator = await create_orchestrator(session_factory=mock_session_factory)
+        mock_report_repository = orchestrator.report_repository
@@
-        # Configure get_report_by_id to return pending then completed after update
-        mock_report_repository.get_report_by_id.side_effect = [
-            mock_report_state_pending, # First call during initial status check
-            mock_report_state_completed # Second call after update_partial for AGENTS_COMPLETED
-        ]
-        mock_report_repository.update_partial.return_value = mock_report_state_completed # Ensure update returns something
+        # Configure get_report_by_id to return pending then completed after update
+        mock_report_repository.get_report_by_id = AsyncMock(side_effect=[
+            mock_report_state_pending,  # First call during initial status check
+            mock_report_state_completed  # Second call after update_partial for AGENTS_COMPLETED
+        ])
+        # Ensure update_partial is a mock and returns the completed state
+        mock_report_repository.update_partial = AsyncMock(return_value=mock_report_state_completed)

This keeps the orchestrator instance but safely mocks just the repository calls used in the test.

📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
# Setup mock for ReportRepository and its interactions
from backend.app.db.models.report_state import ReportStatusEnum
mock_session = AsyncMock()
# Mock the async context manager for the session factory
mock_session_factory = AsyncMock(return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_session)))
orchestrator = await create_orchestrator(session_factory=mock_session_factory)
mock_report_repository = orchestrator.report_repository
# Mock initial report state (PENDING)
mock_report_state_pending = AsyncMock()
mock_report_state_pending.report_id = SAMPLE_REPORT_ID
mock_report_state_pending.status = ReportStatusEnum.PENDING
mock_report_state_pending.partial_agent_output = {}
# Mock final report state (COMPLETED)
mock_report_state_completed = AsyncMock()
mock_report_state_completed.report_id = SAMPLE_REPORT_ID
mock_report_state_completed.status = ReportStatusEnum.AGENTS_COMPLETED
mock_report_state_completed.partial_agent_output = {} # Will be updated
# Configure get_report_by_id to return pending then completed after update
mock_report_repository.get_report_by_id.side_effect = [
mock_report_state_pending, # First call during initial status check
mock_report_state_completed # Second call after update_partial for AGENTS_COMPLETED
]
mock_report_repository.update_partial.return_value = mock_report_state_completed # Ensure update returns something
# Setup mock for ReportRepository and its interactions
from backend.app.db.models.report_state import ReportStatusEnum
mock_session = AsyncMock()
# Mock the async context manager for the session factory
mock_session_factory = AsyncMock(return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_session)))
orchestrator = await create_orchestrator(session_factory=mock_session_factory)
mock_report_repository = orchestrator.report_repository
# Mock initial report state (PENDING)
mock_report_state_pending = AsyncMock()
mock_report_state_pending.report_id = SAMPLE_REPORT_ID
mock_report_state_pending.status = ReportStatusEnum.PENDING
mock_report_state_pending.partial_agent_output = {}
# Mock final report state (COMPLETED)
mock_report_state_completed = AsyncMock()
mock_report_state_completed.report_id = SAMPLE_REPORT_ID
mock_report_state_completed.status = ReportStatusEnum.AGENTS_COMPLETED
mock_report_state_completed.partial_agent_output = {} # Will be updated
# Configure get_report_by_id to return pending then completed after update
mock_report_repository.get_report_by_id = AsyncMock(side_effect=[
mock_report_state_pending, # First call during initial status check
mock_report_state_completed # Second call after update_partial for AGENTS_COMPLETED
])
# Ensure update_partial is a mock and returns the completed state
mock_report_repository.update_partial = AsyncMock(return_value=mock_report_state_completed)
🤖 Prompt for AI Agents 
In backend/tests/test_orchestrator_integration.py around lines 50 to 78, the
test sets .side_effect and .return_value on real bound async methods of
orchestrator.report_repository which will fail; replace those bound methods with
AsyncMock instances on the repository before assigning side_effect/return_value
(e.g., orchestrator.report_repository.get_report_by_id = AsyncMock(...) and
orchestrator.report_repository.update_partial = AsyncMock(...)), then configure
their side_effect and return_value as needed so the orchestrator instance
remains but the repository calls are properly mocked.

result = await orchestrator.execute_agents_concurrently(SAMPLE_REPORT_ID, SAMPLE_TOKEN_ID)

# Assertions for successful execution and aggregated output
Expand All @@ -69,19 +89,13 @@ async def test_orchestrator_full_integration_success(mock_settings):
assert "code_metrics" in result["code_audit"]
assert "audit_summary" in result["code_audit"]

assert in_memory_reports[SAMPLE_REPORT_ID]["status"] == "completed"
assert in_memory_reports[SAMPLE_REPORT_ID]["data"] == result

# Verify agent calls
mock_fetch_onchain_metrics.assert_called_once()
mock_fetch_tokenomics.assert_called_once()
mock_fetch_social_data.assert_called_once()
mock_analyze_sentiment.assert_called_once()
mock_scrape_team_profiles.assert_called_once()
mock_analyze_whitepaper.assert_called_once()
mock_fetch_repo_metrics.assert_called_once()
mock_analyze_code_activity.assert_called_once()
mock_search_and_summarize_audit_reports.assert_called_once()
# Verify report repository interactions
mock_report_repository.get_report_by_id.assert_called_with(SAMPLE_REPORT_ID)
# Check that update_partial was called with AGENTS_COMPLETED status
assert mock_report_repository.update_partial.call_count > 0
update_call_args = mock_report_repository.update_partial.call_args_list[-1].args
assert update_call_args[0] == SAMPLE_REPORT_ID
assert update_call_args[1].get("status") == ReportStatusEnum.AGENTS_COMPLETED

@pytest.mark.asyncio
async def test_orchestrator_agent_timeout_handling(mock_settings):
Expand Down Expand Up @@ -111,13 +125,45 @@ async def test_orchestrator_agent_timeout_handling(mock_settings):
mock_analyze_code_activity.return_value = {"activity_analysis": "mocked"}
mock_search_and_summarize_audit_reports.return_value = [{"audit_report": "mocked"}]

in_memory_reports[SAMPLE_REPORT_ID] = {"status": "pending", "data": {}}
orchestrator = create_orchestrator()
# Setup mock for ReportRepository and its interactions
from backend.app.db.models.report_state import ReportStatusEnum
mock_session = AsyncMock()
mock_session_factory = AsyncMock(return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_session)))

orchestrator = await create_orchestrator(session_factory=mock_session_factory)
mock_report_repository = orchestrator.report_repository

# Mock initial report state (PENDING)
mock_report_state_pending = AsyncMock()
mock_report_state_pending.report_id = SAMPLE_REPORT_ID
mock_report_state_pending.status = ReportStatusEnum.PENDING
mock_report_state_pending.partial_agent_output = {}

# Mock failed report state
mock_report_state_failed = AsyncMock()
mock_report_state_failed.report_id = SAMPLE_REPORT_ID
mock_report_state_failed.status = ReportStatusEnum.AGENTS_FAILED
mock_report_state_failed.partial_agent_output = {}
mock_report_state_failed.errors = {"onchain_data_agent": True}

# Configure get_report_by_id to return pending then failed after update
mock_report_repository.get_report_by_id.side_effect = [
mock_report_state_pending, # First call during initial status check
mock_report_state_failed # Second call after update_partial for AGENTS_FAILED
]
mock_report_repository.update_partial.return_value = mock_report_state_failed

Comment on lines +128 to +155
Copy link

@coderabbitai coderabbitai bot Dec 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Same mocking issue for timeout test: methods must be replaced with AsyncMock

This block repeats the same pattern: mock_report_repository.get_report_by_id.side_effect = [...] and mock_report_repository.update_partial.return_value = ... on real async methods. Use AsyncMock assignments instead, analogous to the success test:

@@
-        orchestrator = await create_orchestrator(session_factory=mock_session_factory)
-        mock_report_repository = orchestrator.report_repository
+        orchestrator = await create_orchestrator(session_factory=mock_session_factory)
+        mock_report_repository = orchestrator.report_repository
@@
-        # Configure get_report_by_id to return pending then failed after update
-        mock_report_repository.get_report_by_id.side_effect = [
-            mock_report_state_pending, # First call during initial status check
-            mock_report_state_failed # Second call after update_partial for AGENTS_FAILED
-        ]
-        mock_report_repository.update_partial.return_value = mock_report_state_failed
+        # Configure get_report_by_id to return pending then failed after update
+        mock_report_repository.get_report_by_id = AsyncMock(side_effect=[
+            mock_report_state_pending,  # First call during initial status check
+            mock_report_state_failed   # Second call after update_partial for AGENTS_FAILED
+        ])
+        mock_report_repository.update_partial = AsyncMock(return_value=mock_report_state_failed)

Without this, the test will not run reliably.

🤖 Prompt for AI Agents 
In backend/tests/test_orchestrator_integration.py around lines 128 to 155, the
test assigns side_effect and return_value directly to coroutine methods on the
real mock_report_repository which are async and must be replaced with AsyncMock
instances; change the assignments so mock_report_repository.get_report_by_id =
AsyncMock(side_effect=[mock_report_state_pending, mock_report_state_failed]) and
mock_report_repository.update_partial =
AsyncMock(return_value=mock_report_state_failed) (and apply the same AsyncMock
replacement pattern to any other async repository methods used in this timeout
test) so the async methods behave correctly in the test.

result = await orchestrator.execute_agents_concurrently(SAMPLE_REPORT_ID, SAMPLE_TOKEN_ID)

# Assertions for timeout handling
assert "onchain_metrics" not in result
assert in_memory_reports[SAMPLE_REPORT_ID]["status"] == "failed" # Overall status should be failed due to timeout
mock_report_repository.get_report_by_id.assert_called_with(SAMPLE_REPORT_ID)
# Check that update_partial was called with AGENTS_FAILED status and error details
assert mock_report_repository.update_partial.call_count > 0
update_call_args = mock_report_repository.update_partial.call_args_list[-1].args
assert update_call_args[0] == SAMPLE_REPORT_ID
assert update_call_args[1].get("status") == ReportStatusEnum.AGENTS_FAILED
assert "onchain_data_agent" in update_call_args[1].get("errors", {})
@pytest.mark.asyncio
async def test_orchestrator_agent_exception_handling(mock_settings):
"""
Expand Down Expand Up @@ -146,10 +192,42 @@ async def test_orchestrator_agent_exception_handling(mock_settings):
mock_analyze_code_activity.return_value = {"activity_analysis": "mocked"}
mock_search_and_summarize_audit_reports.return_value = [{"audit_report": "mocked"}]

in_memory_reports[SAMPLE_REPORT_ID] = {"status": "pending", "data": {}}
orchestrator = create_orchestrator()
# Setup mock for ReportRepository and its interactions
from backend.app.db.models.report_state import ReportStatusEnum
mock_session = AsyncMock()
mock_session_factory = AsyncMock(return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_session)))

orchestrator = await create_orchestrator(session_factory=mock_session_factory)
mock_report_repository = orchestrator.report_repository

# Mock initial report state (PENDING)
mock_report_state_pending = AsyncMock()
mock_report_state_pending.report_id = SAMPLE_REPORT_ID
mock_report_state_pending.status = ReportStatusEnum.PENDING
mock_report_state_pending.partial_agent_output = {}

# Mock failed report state
mock_report_state_failed = AsyncMock()
mock_report_state_failed.report_id = SAMPLE_REPORT_ID
mock_report_state_failed.status = ReportStatusEnum.AGENTS_FAILED
mock_report_state_failed.partial_agent_output = {}
mock_report_state_failed.errors = {"onchain_data_agent": True}

# Configure get_report_by_id to return pending then failed after update
mock_report_repository.get_report_by_id.side_effect = [
mock_report_state_pending, # First call during initial status check
mock_report_state_failed # Second call after update_partial for AGENTS_FAILED
]
mock_report_repository.update_partial.return_value = mock_report_state_failed
Comment on lines +195 to +221
Copy link

@coderabbitai coderabbitai bot Dec 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Repeat of mocking issue for exception-handling test

Same concern here: get_report_by_id and update_partial are real async methods, so setting .side_effect / .return_value on them is unsafe. Mirror the AsyncMock pattern:

@@
-        orchestrator = await create_orchestrator(session_factory=mock_session_factory)
-        mock_report_repository = orchestrator.report_repository
+        orchestrator = await create_orchestrator(session_factory=mock_session_factory)
+        mock_report_repository = orchestrator.report_repository
@@
-        # Configure get_report_by_id to return pending then failed after update
-        mock_report_repository.get_report_by_id.side_effect = [
-            mock_report_state_pending, # First call during initial status check
-            mock_report_state_failed # Second call after update_partial for AGENTS_FAILED
-        ]
-        mock_report_repository.update_partial.return_value = mock_report_state_failed
+        # Configure get_report_by_id to return pending then failed after update
+        mock_report_repository.get_report_by_id = AsyncMock(side_effect=[
+            mock_report_state_pending,  # First call during initial status check
+            mock_report_state_failed   # Second call after update_partial for AGENTS_FAILED
+        ])
+        mock_report_repository.update_partial = AsyncMock(return_value=mock_report_state_failed)

This keeps all three tests consistent and prevents AttributeErrors when configuring mocks.

📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
# Setup mock for ReportRepository and its interactions
from backend.app.db.models.report_state import ReportStatusEnum
mock_session = AsyncMock()
mock_session_factory = AsyncMock(return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_session)))
orchestrator = await create_orchestrator(session_factory=mock_session_factory)
mock_report_repository = orchestrator.report_repository
# Mock initial report state (PENDING)
mock_report_state_pending = AsyncMock()
mock_report_state_pending.report_id = SAMPLE_REPORT_ID
mock_report_state_pending.status = ReportStatusEnum.PENDING
mock_report_state_pending.partial_agent_output = {}
# Mock failed report state
mock_report_state_failed = AsyncMock()
mock_report_state_failed.report_id = SAMPLE_REPORT_ID
mock_report_state_failed.status = ReportStatusEnum.AGENTS_FAILED
mock_report_state_failed.partial_agent_output = {}
mock_report_state_failed.errors = {"onchain_data_agent": True}
# Configure get_report_by_id to return pending then failed after update
mock_report_repository.get_report_by_id.side_effect = [
mock_report_state_pending, # First call during initial status check
mock_report_state_failed # Second call after update_partial for AGENTS_FAILED
]
mock_report_repository.update_partial.return_value = mock_report_state_failed
# Setup mock for ReportRepository and its interactions
from backend.app.db.models.report_state import ReportStatusEnum
mock_session = AsyncMock()
mock_session_factory = AsyncMock(return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_session)))
orchestrator = await create_orchestrator(session_factory=mock_session_factory)
mock_report_repository = orchestrator.report_repository
# Mock initial report state (PENDING)
mock_report_state_pending = AsyncMock()
mock_report_state_pending.report_id = SAMPLE_REPORT_ID
mock_report_state_pending.status = ReportStatusEnum.PENDING
mock_report_state_pending.partial_agent_output = {}
# Mock failed report state
mock_report_state_failed = AsyncMock()
mock_report_state_failed.report_id = SAMPLE_REPORT_ID
mock_report_state_failed.status = ReportStatusEnum.AGENTS_FAILED
mock_report_state_failed.partial_agent_output = {}
mock_report_state_failed.errors = {"onchain_data_agent": True}
# Configure get_report_by_id to return pending then failed after update
mock_report_repository.get_report_by_id = AsyncMock(side_effect=[
mock_report_state_pending, # First call during initial status check
mock_report_state_failed # Second call after update_partial for AGENTS_FAILED
])
mock_report_repository.update_partial = AsyncMock(return_value=mock_report_state_failed)
🤖 Prompt for AI Agents 
In backend/tests/test_orchestrator_integration.py around lines 195 to 221, the
test assigns .side_effect and .return_value directly on real async methods which
can raise AttributeErrors; change the setup so
mock_report_repository.get_report_by_id and
mock_report_repository.update_partial are AsyncMock instances (use
AsyncMock(side_effect=[...]) for get_report_by_id and
AsyncMock(return_value=...) for update_partial) mirroring the pattern used
elsewhere in the tests, and ensure the mock_session_factory/mock_session usage
remains an AsyncMock context manager as currently done.


result = await orchestrator.execute_agents_concurrently(SAMPLE_REPORT_ID, SAMPLE_TOKEN_ID)

# Assertions for exception handling
assert "tokenomics" not in result
assert in_memory_reports[SAMPLE_REPORT_ID]["status"] == "failed" # Overall status should be failed due to exception
mock_report_repository.get_report_by_id.assert_called_with(SAMPLE_REPORT_ID)
# Check that update_partial was called with AGENTS_FAILED status and error details
assert mock_report_repository.update_partial.call_count > 0
update_call_args = mock_report_repository.update_partial.call_args_list[-1].args
assert update_call_args[0] == SAMPLE_REPORT_ID
assert update_call_args[1].get("status") == ReportStatusEnum.AGENTS_FAILED
assert "onchain_data_agent" in update_call_args[1].get("errors", {})
1 change: 0 additions & 1 deletion backend/tests/test_routes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import pytest
from fastapi.testclient import TestClient
from backend.main import app
from backend.app.services.report_service import in_memory_reports
import pytest_asyncio
import anyio
import functools
Expand Down