cli runner for benchmarks

.github/workflows/benchmark.yml

@@ -0,0 +1,94 @@
+name: RAG Benchmark
+
+run-name: RAG Benchmark by ${{ github.actor }}
+
+on:
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    environment: main
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        dataset: [kunji, sneha]
+        service: [assistants, responses]
+        count: [100]
+
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
+      LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
+      LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }}
+      LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY: ${{ secrets.LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY }}
+      LOCAL_CREDENTIALS_API_KEY: ${{ secrets.LOCAL_CREDENTIALS_API_KEY }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - run: |
+          cp .env.example .env
+          sed -i 's/changethis/secret123/g' .env
+
+      - name: Run server
+        run: |
+          docker compose up -d
+          sleep 10
+
+      - name: prestart logs on failure
+        if: failure()
+        run: |
+          docker compose logs -f prestart
+          exit 1
+
+      - name: Create local credentials
+        run: |
+          curl -X POST "http://localhost:8000/api/v1/credentials/" \
+            -H "Content-Type: application/json" \
+            -H "X-API-KEY: ${{ env.LOCAL_CREDENTIALS_API_KEY }}" \
+            -d '{
+              "organization_id": 1,
+              "project_id": 1,
+              "is_active": true,
+              "credential": {
+                "openai": {
+                  "api_key": "${{ env.LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY }}"
+                }
+              }
+            }'
+
+      - name: Run benchmark
+        run: |
+          docker compose exec backend uv run ai-cli bench ${{ matrix.service }} --dataset ${{ matrix.dataset }} --count ${{ matrix.count }} | tee benchmark_output.txt
+          # Extract mean duration from benchmark output
+          MEAN_DURATION=$(grep '^Mean duration:' benchmark_output.txt | awk '{print $3}')
+          echo "## Benchmark Results for ${{ matrix.service }} - ${{ matrix.dataset }} (${{ matrix.count }} queries, ${MEAN_DURATION} avg)" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          cat benchmark_output.txt >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          # Find latest benchmark file inside container first
+          CONTAINER_LATEST=$(docker compose exec backend sh -c "ls -t bench_results_*.csv | head -n1")
+          # Copy the specific file out
+          docker compose cp backend:/app/$CONTAINER_LATEST ./
+          cp $CONTAINER_LATEST bench-${{ matrix.service }}-${{ matrix.dataset }}-${{ matrix.count }}.csv
+          ls -l bench-${{ matrix.service }}-${{ matrix.dataset }}-${{ matrix.count }}.csv
+
+      - name: backend logs on failure
+        if: failure()
+        timeout-minutes: 1
+        run: |
+          docker compose logs -f backend
+          exit 1
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-${{ matrix.service }}-${{ matrix.dataset }}-${{ matrix.count }}.csv
+          path: bench-${{ matrix.service }}-${{ matrix.dataset }}-${{ matrix.count }}.csv
+
+      - name: Cleanup
+        if: always()
+        run: docker compose down

README.md

@@ -58,13 +58,13 @@ docker compose watch
 
 This should start all necessary services for the project and will also mount file system as volume for easy development.
 
-You verify backend running by doing health-check
+You verify backend running by doing a health check
 
 ```bash
 curl http://[your-domain]:8000/api/v1/utils/health/
 ```
 
-or by visiting: http://[your-domain]:8000/api/v1/utils/health-check/ in the browser
+or by visiting: http://[your-domain]:8000/api/v1/utils/health/ in the browser
 
 ## Backend Development
 

backend/app/api/main.py

@@ -8,6 +8,7 @@
     organization,
     project,
     project_user,
+    responses,
     private,
     threads,
     users,
@@ -27,6 +28,7 @@
 api_router.include_router(organization.router)
 api_router.include_router(project.router)
 api_router.include_router(project_user.router)
+api_router.include_router(responses.router)
 api_router.include_router(threads.router)
 api_router.include_router(users.router)
 api_router.include_router(utils.router)

backend/app/api/routes/responses.py

@@ -0,0 +1,132 @@
+from typing import Optional
+
+import openai
+from pydantic import BaseModel
+from fastapi import APIRouter, Depends
+from openai import OpenAI
+from sqlmodel import Session
+
+from app.api.deps import get_current_user_org, get_db
+from app.crud.credentials import get_provider_credential
+from app.models import UserOrganization
+from app.utils import APIResponse
+
+router = APIRouter(tags=["responses"])
+
+
+def handle_openai_error(e: openai.OpenAIError) -> str:
+    """Extract error message from OpenAI error."""
+    if isinstance(e.body, dict) and "message" in e.body:
+        return e.body["message"]
+    return str(e)
+
+
+class ResponsesAPIRequest(BaseModel):
+    project_id: int
+
+    model: str
+    instructions: str
+    vector_store_ids: list[str]
+    max_num_results: Optional[int] = 20
+    temperature: Optional[float] = 0.1
+    response_id: Optional[str] = None
+
+    question: str
+
+
+class Diagnostics(BaseModel):
+    input_tokens: int
+    output_tokens: int
+    total_tokens: int
+
+    model: str
+
+
+class FileResultChunk(BaseModel):
+    score: float
+    text: str
+
+
+class _APIResponse(BaseModel):
+    status: str
+
+    response_id: str
+    message: str
+    chunks: list[FileResultChunk]
+
+    diagnostics: Optional[Diagnostics] = None
+
+
+class ResponsesAPIResponse(APIResponse[_APIResponse]):
+    pass
+
+
+def get_file_search_results(response):
+    results: list[FileResultChunk] = []
+
+    for tool_call in response.output:
+        if tool_call.type == "file_search_call":
+            results.extend(
+                [FileResultChunk(score=hit.score, text=hit.text) for hit in results]
+            )
+
+    return results
+
+
+@router.post("/responses/sync", response_model=ResponsesAPIResponse)
+async def responses_sync(
+    request: ResponsesAPIRequest,
+    _session: Session = Depends(get_db),
+    _current_user: UserOrganization = Depends(get_current_user_org),
+):
+    """
+    Temp synchronous endpoint for benchmarking OpenAI responses API
+    """
+    credentials = get_provider_credential(
+        session=_session,
+        org_id=_current_user.organization_id,
+        provider="openai",
+        project_id=request.project_id,
+    )
+    if not credentials or "api_key" not in credentials:
+        return APIResponse.failure_response(
+            error="OpenAI API key not configured for this organization."
+        )
+
+    client = OpenAI(api_key=credentials["api_key"])
+
+    try:
+        response = client.responses.create(
+            model=request.model,
+            previous_response_id=request.response_id,
+            instructions=request.instructions,
+            tools=[
+                {
+                    "type": "file_search",
+                    "vector_store_ids": request.vector_store_ids,
+                    "max_num_results": request.max_num_results,
+                }
+            ],
+            temperature=request.temperature,
+            input=[{"role": "user", "content": request.question}],
+            include=["file_search_call.results"],
+        )
+
+        response_chunks = get_file_search_results(response)
+
+        return ResponsesAPIResponse.success_response(
+            data=_APIResponse(
+                status="success",
+                response_id=response.id,
+                message=response.output_text,
+                chunks=response_chunks,
+                diagnostics=Diagnostics(
+                    input_tokens=response.usage.input_tokens,
+                    output_tokens=response.usage.output_tokens,
+                    total_tokens=response.usage.total_tokens,
+                    model=response.model,
+                ),
+            ),
+        )
+    except openai.OpenAIError as e:
+        return ResponsesAPIResponse.failure_response(error=handle_openai_error(e))

backend/app/api/routes/threads.py

@@ -288,7 +288,7 @@
         session=_session,
         org_id=_current_user.organization_id,
         provider="openai",
-        project_id=_current_user.project_id,
+        project_id=request.get("project_id"),
     )
     if not credentials or "api_key" not in credentials:
         return APIResponse.failure_response(
@@ -321,6 +321,15 @@
             message = process_message_content(
                 message_content, request.get("remove_citation", False)
             )
+
+            diagnostics = {
+                "input_tokens": run.usage.prompt_tokens,
+                "output_tokens": run.usage.completion_tokens,
+                "total_tokens": run.usage.total_tokens,
+                "model": run.model,
+            }
+            request = {**request, **{"diagnostics": diagnostics}}
+
             return create_success_response(request, message)
         else:
             return APIResponse.failure_response(