Implement Unified LLM API v1 to Simplify LLM Integrations and Enable Multi-Provider Extensibility

backend/app/alembic/versions/219033c644de_add_llm_im_jobs_table.py

@@ -0,0 +1,26 @@
+"""Add LLM in jobs table
+
+Revision ID: 219033c644de
+Revises: e7c68e43ce6f
+Create Date: 2025-10-17 15:38:33.565674
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "219033c644de"
+down_revision = "e7c68e43ce6f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TYPE jobtype ADD VALUE IF NOT EXISTS 'LLM_API'")
+
+
+def downgrade():
+    # Enum value removal requires manual intervention if 'LLM_API' is in use.
+    # If rollback is necessary, run SQL manually to recreate the enum without 'LLM_API'.
+    pass

backend/app/api/main.py

@@ -7,6 +7,7 @@
     documents,
     doc_transformation_job,
     login,
+    llm,
     organization,
     openai_conversation,
     project,
@@ -31,6 +32,7 @@
 api_router.include_router(credentials.router)
 api_router.include_router(documents.router)
 api_router.include_router(doc_transformation_job.router)
+api_router.include_router(llm.router)
 api_router.include_router(login.router)
 api_router.include_router(onboarding.router)
 api_router.include_router(openai_conversation.router)

backend/app/api/routes/llm.py

@@ -0,0 +1,58 @@
+import logging
+
+from fastapi import APIRouter
+
+from app.api.deps import AuthContextDep, SessionDep
+from app.models import LLMCallRequest, LLMCallResponse, Message
+from app.services.llm.jobs import start_job
+from app.utils import APIResponse
+
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["LLM"])
+llm_callback_router = APIRouter()
+
+
+@llm_callback_router.post(
+    "{$callback_url}",
+    name="llm_callback",
+)
+def llm_callback_notification(body: APIResponse[LLMCallResponse]):
+    """
+    Callback endpoint specification for LLM call completion.
+
+    The callback will receive:
+    - On success: APIResponse with success=True and data containing LLMCallResponse
+    - On failure: APIResponse with success=False and error message
+    - metadata field will always be included if provided in the request
+    """
+    ...
+
+
+@router.post(
+    "/llm/call",
+    response_model=APIResponse[Message],
+    callbacks=llm_callback_router.routes,
+)
+async def llm_call(
+    _current_user: AuthContextDep, _session: SessionDep, request: LLMCallRequest
+):
+    """
+    Endpoint to initiate an LLM call as a background job.
+    """
+    project_id = _current_user.project.id
+    organization_id = _current_user.organization.id
+
+    start_job(
+        db=_session,
+        request=request,
+        project_id=project_id,
+        organization_id=organization_id,
+    )
+
+    return APIResponse.success_response(
+        data=Message(
+            message=f"Your response is being generated and will be delivered via callback."
+        ),
+    )

backend/app/models/__init__.py

@@ -48,6 +48,11 @@
 
 from .job import Job, JobType, JobStatus, JobUpdate
 
+from .llm import (
+    LLMCallRequest,
+    LLMCallResponse,
+)
+
 from .message import Message
 from .model_evaluation import (
     ModelEvaluation,

backend/app/models/job.py

@@ -15,6 +15,7 @@ class JobStatus(str, Enum):
 
 class JobType(str, Enum):
     RESPONSE = "RESPONSE"
+    LLM_API = "LLM_API"
 
 
 class Job(SQLModel, table=True):

backend/app/models/llm/__init__.py

@@ -0,0 +1,2 @@
+from app.models.llm.request import LLMCallRequest, CompletionConfig, QueryParams
+from app.models.llm.response import LLMCallResponse, LLMResponse, LLMOutput, Usage

backend/app/models/llm/request.py

@@ -0,0 +1,88 @@
+from typing import Any, Literal
+
+from sqlmodel import Field, SQLModel
+from pydantic import model_validator, HttpUrl
+
+
+class ConversationConfig(SQLModel):
+    id: str | None = Field(
+        default=None,
+        description=(
+            "Identifier for an existing conversation. "
+            "Used to retrieve the previous message context and continue the chat. "
+            "If not provided and `auto_create` is True, a new conversation will be created."
+        ),
+    )
+    auto_create: bool = Field(
+        default=False,
+        description=(
+            "Only if True and no `id` is provided, a new conversation will be created automatically."
+        ),
+    )
+
+    @model_validator(mode="after")
+    def validate_conversation_logic(self):
+        if self.id and self.auto_create:
+            raise ValueError(
+                "Cannot specify both 'id' and 'auto_create=True'. "
+                "Use 'id' to continue an existing conversation, or set 'auto_create=True' to create a new one."
+            )
+        return self
+
+
+# Query Parameters (dynamic per request)
+class QueryParams(SQLModel):
+    """Query-specific parameters for each LLM call."""
+
+    input: str = Field(
+        ...,
+        min_length=1,
+        description="User input question/query/prompt, used to generate a response.",
+    )
+    conversation: ConversationConfig | None = Field(
+        default=None,
+        description="Conversation control configuration for context handling.",
+    )
+
+
+class CompletionConfig(SQLModel):
+    """Completion configuration with provider and parameters."""
+
+    provider: Literal["openai"] = Field(
+        default="openai", description="LLM provider to use"
+    )
+    params: dict[str, Any] = Field(
+        ...,
+        description="Provider-specific parameters (schema varies by provider), should exactly match the provider's endpoint params structure",
+    )
+
+
+class LLMCallConfig(SQLModel):
+    """Complete configuration for LLM call including all processing stages."""
+
+    completion: CompletionConfig = Field(..., description="Completion configuration")
+    # Future additions:
+    # classifier: ClassifierConfig | None = None
+    # pre_filter: PreFilterConfig | None = None
+
+
+class LLMCallRequest(SQLModel):
+    """User-facing API request for LLM completion."""
+
+    query: QueryParams = Field(..., description="Query-specific parameters")
+    config: LLMCallConfig = Field(..., description="Configuration for the LLM call")
+    callback_url: HttpUrl | None = Field(
+        default=None, description="Webhook URL for async response delivery"
+    )
+    include_provider_raw_response: bool = Field(
+        default=False,
+        description="Whether to include the raw LLM provider response in the output",
+    )
+    request_metadata: dict[str, Any] | None = Field(
+        default=None,
+        description=(
+            "Client-provided metadata passed through unchanged in the response. "
+            "Use this to correlate responses with requests or track request state. "
+            "The exact dictionary provided here will be returned in the response metadata field."
+        ),
+    )

backend/app/models/llm/response.py

@@ -0,0 +1,52 @@
+"""
+LLM response models.
+
+This module contains structured response models for LLM API calls.
+"""
+from sqlmodel import SQLModel, Field
+
+
+class Usage(SQLModel):
+    input_tokens: int
+    output_tokens: int
+    total_tokens: int
+
+
+class LLMOutput(SQLModel):
+    """Standardized output format for LLM responses."""
+
+    text: str = Field(..., description="Primary text content of the LLM response.")
+
+
+class LLMResponse(SQLModel):
+    """Normalized response format independent of provider."""
+
+    provider_response_id: str = Field(
+        ..., description="Unique response ID provided by the LLM provider."
+    )
+    conversation_id: str | None = Field(
+        default=None, description="Conversation or thread ID for context (if any)."
+    )
+    provider: str = Field(
+        ..., description="Name of the LLM provider (e.g., openai, anthropic)."
+    )
+    model: str = Field(
+        ..., description="Model used by the provider (e.g., gpt-4-turbo)."
+    )
+    output: LLMOutput = Field(
+        ...,
+        description="Structured output containing text and optional additional data.",
+    )
+
+
+class LLMCallResponse(SQLModel):
+    """Top-level response schema for an LLM API call."""
+
+    response: LLMResponse = Field(
+        ..., description="Normalized, structured LLM response."
+    )
+    usage: Usage = Field(..., description="Token usage and cost information.")
+    provider_raw_response: dict[str, object] | None = Field(
+        default=None,
+        description="Unmodified raw response from the LLM provider.",
+    )

backend/app/services/llm/__init__.py

@@ -0,0 +1,9 @@
+# Providers
+from app.services.llm.providers import (
+    BaseProvider,
+    OpenAIProvider,
+)
+from app.services.llm.providers import (
+    LLMProvider,
+    get_llm_provider,
+)