diff --git a/apps/api-gateway/Dockerfile b/apps/api-gateway/Dockerfile index 2a290a29..f1c91e5d 100644 --- a/apps/api-gateway/Dockerfile +++ b/apps/api-gateway/Dockerfile @@ -1,6 +1,6 @@ -# local-test/api-gateway.Dockerfile # Using a consistent base for all stages ARG BASE_IMAGE=node:22-alpine +# hadolint ignore=DL3006 FROM ${BASE_IMAGE} AS builder ARG DIR=/usr/src/app @@ -12,6 +12,7 @@ RUN yarn global add turbo@^2.0.3 RUN turbo prune api-gateway --docker && rm -f .npmrc # Installing the isolated workspace +# hadolint ignore=DL3006 FROM ${BASE_IMAGE} AS installer ARG DIR=/usr/src/app WORKDIR $DIR @@ -19,23 +20,27 @@ COPY --from=builder $DIR/out/json/ . COPY --from=builder $DIR/out/yarn.lock ./yarn.lock COPY --from=builder $DIR/turbo.json ./turbo.json COPY --from=builder $DIR/packages ./packages -RUN yarn install --ignore-scripts --frozen-lockfile --network-timeout 600000 +RUN yarn install --ignore-scripts --frozen-lockfile # Running build using turbo +# hadolint ignore=DL3006 FROM ${BASE_IMAGE} AS sourcer ARG DIR=/usr/src/app WORKDIR $DIR COPY --from=installer $DIR/ . COPY --from=builder $DIR/out/full/ . COPY --from=builder /usr/local/share/.config/yarn/global /usr/local/share/.config/yarn/global -RUN yarn build --filter=api-gateway && yarn install --production --ignore-scripts --frozen-lockfile --network-timeout 600000 +RUN yarn build --filter=api-gateway && yarn install --production --ignore-scripts --frozen-lockfile # Production stage +# hadolint ignore=DL3006 FROM ${BASE_IMAGE} AS production ARG DIR=/usr/src/app ENV NODE_ENV production WORKDIR $DIR -RUN apk add --no-cache dumb-init +RUN apk add --no-cache dumb-init~=1 +COPY --chown=node:node --from=sourcer $DIR/apps/api-gateway/package.json ./package.json + COPY --chown=node:node --from=sourcer $DIR/apps/api-gateway/dist ./dist COPY --chown=node:node --from=sourcer $DIR/node_modules $DIR/node_modules ENTRYPOINT ["/usr/bin/dumb-init", "--"] @@ -46,4 +51,4 @@ EXPOSE 3000 FROM production AS patched USER root RUN apk -U upgrade -USER node \ No newline at end of file +USER node diff --git a/apps/api-gateway/src/auth/interfaces/user.session.interface.ts b/apps/api-gateway/src/auth/interfaces/user.session.interface.ts index 778c5d4b..2de64afb 100644 --- a/apps/api-gateway/src/auth/interfaces/user.session.interface.ts +++ b/apps/api-gateway/src/auth/interfaces/user.session.interface.ts @@ -3,6 +3,7 @@ import { Request } from "express"; export enum UserRole { LEARNER = "learner", AUTHOR = "author", + ADMIN = "admin", } export interface UserSession { diff --git a/apps/api-gateway/src/main.ts b/apps/api-gateway/src/main.ts index 33ac6e9d..33524328 100644 --- a/apps/api-gateway/src/main.ts +++ b/apps/api-gateway/src/main.ts @@ -1,12 +1,13 @@ +import instana from "@instana/collector"; import { VersioningType } from "@nestjs/common"; import { NestFactory } from "@nestjs/core"; import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger"; import * as cookieParser from "cookie-parser"; +import { json, urlencoded } from "express"; import helmet from "helmet"; import { WinstonModule } from "nest-winston"; import { AppModule } from "./app.module"; import { winstonOptions } from "./logger/config"; -import instana from "@instana/collector"; instana(); @@ -15,7 +16,8 @@ async function bootstrap() { cors: false, logger: WinstonModule.createLogger(winstonOptions), }); - + app.use(json({ limit: "1000mb" })); + app.use(urlencoded({ limit: "1000mb", extended: true })); app.setGlobalPrefix("api", { exclude: ["health", "health/liveness", "health/readiness"], }); diff --git a/apps/api/.eslintrc.cjs b/apps/api/.eslintrc.cjs index 88da9ec3..3bc51a96 100644 --- a/apps/api/.eslintrc.cjs +++ b/apps/api/.eslintrc.cjs @@ -16,7 +16,10 @@ module.exports = { ], rules: { "unicorn/prefer-top-level-await": "off", - 'unicorn/no-nested-ternary': 'off', + "unicorn/no-nested-ternary": "off", + "unicorn/no-null": "off", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/require-await": "off", "unicorn/no-abusive-eslint-disable": "off", "unicorn/prevent-abbreviations": [ "error", diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index e56cd03a..5cb57286 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -1,18 +1,19 @@ -# local-test/api.Dockerfile # Using a consistent base for all stages ARG BASE_IMAGE=node:22-alpine -FROM node:22-alpine AS builder +# hadolint ignore=DL3006 +FROM ${BASE_IMAGE} AS builder ARG DIR=/usr/src/app -# Pruning using turbo +# Copy package files for dependency resolution WORKDIR $DIR COPY . . RUN yarn global add turbo@^2.0.3 RUN turbo prune api --docker && rm -f .npmrc # Installing the isolated workspace -FROM node:22-alpine AS installer +# hadolint ignore=DL3006 +FROM ${BASE_IMAGE} AS installer ARG DIR=/usr/src/app WORKDIR $DIR COPY --from=builder $DIR/out/json/ . @@ -20,36 +21,38 @@ COPY --from=builder $DIR/out/yarn.lock ./yarn.lock COPY --from=builder $DIR/turbo.json ./turbo.json COPY --from=builder $DIR/packages ./packages COPY --from=builder $DIR/apps/api/prisma ./prisma -RUN yarn install --ignore-scripts --frozen-lockfile --network-timeout 600000 +# Install build dependencies (including pkgconf) and rebuild native modules RUN apk add --no-cache python3~=3 make~=4 g++~=14 pkgconf~=2 \ + && yarn install --frozen-lockfile \ && yarn prisma generate \ && npm rebuild cld --build-from-source + # Running build using turbo -FROM node:22-alpine AS sourcer +# hadolint ignore=DL3006 +FROM ${BASE_IMAGE} AS sourcer ARG DIR=/usr/src/app WORKDIR $DIR COPY --from=installer $DIR/ . COPY --from=builder $DIR/out/full/ . COPY --from=builder /usr/local/share/.config/yarn/global /usr/local/share/.config/yarn/global -WORKDIR $DIR/apps/api -RUN npx prisma generate -WORKDIR $DIR -RUN yarn build --filter=api && yarn install --production --ignore-scripts --frozen-lockfile --network-timeout 600000 +RUN yarn build --filter=api && yarn install --production --ignore-scripts --frozen-lockfile # Production stage -FROM node:22-alpine AS production -ARG DIR=/usr/src/app +# hadolint ignore=DL3006 +FROM ${BASE_IMAGE} AS production ENV NODE_ENV production +ARG DIR=/usr/src/app WORKDIR $DIR -RUN apk add --no-cache dumb-init=1.2.5-r3 +RUN apk add --no-cache dumb-init~=1 postgresql15-client~=15 COPY --chown=node:node --from=sourcer $DIR/apps/api/package.json ./package.json COPY --chown=node:node --from=sourcer $DIR/apps/api/dist ./dist COPY --chown=node:node --from=sourcer $DIR/node_modules $DIR/node_modules COPY --chown=node:node --from=sourcer $DIR/prisma ./prisma COPY --chown=node:node --from=sourcer $DIR/apps/api/migrate.sh ./migrate.sh COPY --chown=node:node --from=sourcer $DIR/apps/api/ensureDb.js ./ensureDb.js +COPY --chown=node:node --from=sourcer $DIR/apps/api/src/scripts ./scripts ENTRYPOINT ["/usr/bin/dumb-init", "--"] -CMD ["node", "dist/main.js"] +CMD ["node", "dist/src/main.js"] EXPOSE 3000 # Patched stage with updates diff --git a/apps/api/docs/LLM_ARCHITECTURE.md b/apps/api/docs/LLM_ARCHITECTURE.md new file mode 100644 index 00000000..67650ac7 --- /dev/null +++ b/apps/api/docs/LLM_ARCHITECTURE.md @@ -0,0 +1,383 @@ +# LLM Architecture Documentation + +## Overview + +The LLM (Large Language Model) architecture provides a centralized, flexible, and scalable system for managing AI model usage across different features in the application. The architecture supports dynamic model assignment, cost tracking, usage analytics, and seamless integration of new AI providers. + +## Core Principles + +- **Dynamic Model Assignment**: Features can be dynamically assigned to different LLM models without code changes +- **Cost Tracking**: Accurate historical cost tracking with proper model attribution +- **Abstraction**: Clean separation between business logic and LLM implementation details +- **Scalability**: Easy addition of new models and providers +- **Observability**: Comprehensive usage tracking and analytics + +## Architecture Components + +### 1. Database Layer + +#### Core Models + +**`LLMModel`** + +- Represents available LLM models in the system +- Stores model metadata (provider, display name, active status) +- Primary key for all model-related operations + +```sql +CREATE TABLE "LLMModel" ( + "id" SERIAL PRIMARY KEY, + "modelKey" TEXT UNIQUE NOT NULL, -- e.g., 'gpt-4o', 'gpt-4o-mini' + "displayName" TEXT NOT NULL, -- e.g., 'GPT-4 Omni' + "provider" TEXT NOT NULL, -- e.g., 'OpenAI' + "isActive" BOOLEAN DEFAULT true, + "createdAt" TIMESTAMP DEFAULT NOW(), + "updatedAt" TIMESTAMP NOT NULL +); +``` + +**`AIFeature`** + +- Defines AI-powered features in the system +- Each feature can be assigned to different models +- Contains fallback configuration + +```sql +CREATE TABLE "AIFeature" ( + "id" SERIAL PRIMARY KEY, + "featureKey" TEXT UNIQUE NOT NULL, -- e.g., 'text_grading' + "featureType" AIFeatureType NOT NULL, + "displayName" TEXT NOT NULL, + "description" TEXT, + "isActive" BOOLEAN DEFAULT true, + "requiresModel" BOOLEAN DEFAULT true, + "defaultModelKey" TEXT, -- Fallback model + "createdAt" TIMESTAMP DEFAULT NOW(), + "updatedAt" TIMESTAMP NOT NULL +); +``` + +**`LLMFeatureAssignment`** + +- Maps AI features to specific LLM models +- Enables dynamic model switching +- Tracks assignment history and metadata + +```sql +CREATE TABLE "LLMFeatureAssignment" ( + "id" SERIAL PRIMARY KEY, + "featureId" INTEGER REFERENCES "AIFeature"(id), + "modelId" INTEGER REFERENCES "LLMModel"(id), + "isActive" BOOLEAN DEFAULT true, + "priority" INTEGER DEFAULT 0, + "assignedBy" TEXT, + "assignedAt" TIMESTAMP DEFAULT NOW(), + "deactivatedAt" TIMESTAMP, + "metadata" JSONB +); +``` + +**`LLMPricing`** + +- Stores historical pricing data for accurate cost calculations +- Supports multiple pricing sources and time-based pricing + +```sql +CREATE TABLE "LLMPricing" ( + "id" SERIAL PRIMARY KEY, + "modelId" INTEGER REFERENCES "LLMModel"(id), + "inputTokenPrice" DOUBLE PRECISION NOT NULL, + "outputTokenPrice" DOUBLE PRECISION NOT NULL, + "effectiveDate" TIMESTAMP NOT NULL, + "source" PricingSource NOT NULL, + "isActive" BOOLEAN DEFAULT true, + "metadata" JSONB +); +``` + +**`AIUsage`** + +- Tracks actual AI usage with model attribution +- Enables accurate cost calculation and analytics + +```sql +CREATE TABLE "AIUsage" ( + "id" SERIAL PRIMARY KEY, + "assignmentId" INTEGER REFERENCES "Assignment"(id), + "usageType" AIUsageType NOT NULL, + "tokensIn" INTEGER DEFAULT 0, + "tokensOut" INTEGER DEFAULT 0, + "usageCount" INTEGER DEFAULT 0, + "modelKey" TEXT, -- Actual model used + "createdAt" TIMESTAMP DEFAULT NOW() +); +``` + +#### Enums + +**`AIFeatureType`** + +```sql +CREATE TYPE "AIFeatureType" AS ENUM ( + 'TEXT_GRADING', + 'FILE_GRADING', + 'IMAGE_GRADING', + 'URL_GRADING', + 'PRESENTATION_GRADING', + 'VIDEO_GRADING', + 'QUESTION_GENERATION', + 'TRANSLATION', + 'RUBRIC_GENERATION', + 'CONTENT_MODERATION', + 'ASSIGNMENT_GENERATION', + 'LIVE_RECORDING_FEEDBACK' +); +``` + +### 2. Service Layer + +#### Core Services + +**`LlmRouter`** (`src/api/llm/core/services/llm-router.service.ts`) + +- Central registry of available LLM providers +- Routes requests to appropriate model based on feature assignments +- Provides fallback mechanisms + +Key Methods: + +```typescript +get(key: string): ILlmProvider // Get provider by model key +getForFeature(featureKey: string): Promise // Get assigned model +getAvailableModelKeys(): string[] // List all available models +``` + +**`LLMAssignmentService`** (`src/api/llm/core/services/llm-assignment.service.ts`) + +- Manages feature-to-model assignments +- Handles bulk operations and history tracking +- Provides assignment statistics + +Key Methods: + +```typescript +getAssignedModel(featureKey: string): Promise +assignModelToFeature(featureKey: string, modelKey: string): Promise +bulkAssignModels(assignments: Assignment[]): Promise +resetToDefaults(): Promise +``` + +**`LLMResolverService`** (`src/api/llm/core/services/llm-resolver.service.ts`) + +- Caching layer for model resolution (5-minute TTL) +- Performance optimization for frequent model lookups +- Cache invalidation on assignment changes + +**`LLMPricingService`** (`src/api/llm/core/services/llm-pricing.service.ts`) + +- Manages pricing data and cost calculations +- Supports historical pricing with fallback logic +- Provides cost breakdowns and analytics + +Key Methods: + +```typescript +getPricingAtDate(modelKey: string, date: Date): Promise +calculateCostWithBreakdown(modelKey: string, inputTokens: number, outputTokens: number, usageDate: Date): Promise +updatePricingHistory(pricingData: ModelPricing[]): Promise +``` + +**`PromptProcessorService`** (`src/api/llm/core/services/prompt-processor.service.ts`) + +- Processes prompts using assigned models +- Handles usage tracking with model attribution +- Supports both text and image processing + +Key Methods: + +```typescript +processPromptForFeature(prompt: PromptTemplate, assignmentId: number, usageType: AIUsageType, featureKey: string, fallbackModel?: string): Promise +processPrompt(prompt: PromptTemplate, assignmentId: number, usageType: AIUsageType, llmKey?: string): Promise +``` + +**`UsageTrackerService`** (`src/api/llm/core/services/usage-tracking.service.ts`) + +- Tracks AI usage with model attribution +- Aggregates usage statistics +- Supports cost calculation queries + +### 3. Provider Layer + +#### Interface Definition + +**`ILlmProvider`** (`src/api/llm/core/interfaces/llm-provider.interface.ts`) + +```typescript +export interface ILlmProvider { + readonly key: string; + + invoke( + messages: HumanMessage[], + options?: LlmRequestOptions + ): Promise; + invokeWithImage( + textContent: string, + imageData: string, + options?: LlmRequestOptions + ): Promise; +} +``` + +#### Current Providers + +1. **OpenAiLlmService** - GPT-4o (`gpt-4o`) +2. **OpenAiLlmMiniService** - GPT-4o Mini (`gpt-4o-mini`) +3. **Gpt4VisionPreviewLlmService** - GPT-4.1 Mini Vision (`gpt-4.1-mini`) + +### 4. API Layer + +**Admin Dashboard Controller** (`src/api/admin/controllers/admin-dashboard.controller.ts`) + +- Provides dashboard statistics with accurate cost calculations +- Supports both admin and author views +- Real-time analytics and insights + +**LLM Assignment Controller** (`src/api/admin/controllers/llm-assignment.controller.ts`) + +- CRUD operations for feature assignments +- Bulk operations and statistics +- Admin-only access with proper authentication + +**LLM Pricing Controller** (`src/api/admin/controllers/llm-pricing.controller.ts`) + +- Pricing management and cost calculation APIs +- Historical pricing data and analytics +- Cost breakdown and usage reports + +### 5. Frontend Layer + +**Admin Interface** (`apps/web/app/admin/llm-assignments/page.tsx`) + +- Visual management of feature-to-model assignments +- Real-time assignment changes with validation +- Model statistics and usage insights +- Bulk operations and reset functionality + +## Data Flow + +### 1. Model Resolution Flow + +```mermaid +graph TD + A[Feature Request] --> B[LlmRouter.getForFeature] + B --> C[LLMResolverService.resolveModelForFeature] + C --> D{Cache Hit?} + D -->|Yes| E[Return Cached Model] + D -->|No| F[LLMAssignmentService.getAssignedModel] + F --> G[Database Query] + G --> H[Cache Result] + H --> I[Return Model Key] + I --> J[LlmRouter.get] + J --> K[Return ILlmProvider] +``` + +### 2. Usage Tracking Flow + +```mermaid +graph TD + A[LLM Request] --> B[PromptProcessorService] + B --> C[Get Model Provider] + C --> D[Execute LLM Call] + D --> E[Extract Token Usage] + E --> F[UsageTrackerService.trackUsage] + F --> G[Store in AIUsage Table] + G --> H[Include Model Key] +``` + +### 3. Cost Calculation Flow + +```mermaid +graph TD + A[Cost Request] --> B[Fetch AIUsage Records] + B --> C[Extract Model Key & Date] + C --> D[LLMPricingService.getPricingAtDate] + D --> E{Pricing Found?} + E -->|Yes| F[Calculate Cost] + E -->|No| G[Find Closest Pricing] + G --> F + F --> H[Aggregate Costs] + H --> I[Return Cost Breakdown] +``` + +## Key Design Patterns + +### 1. Strategy Pattern + +- Different LLM providers implement `ILlmProvider` interface +- Enables easy addition of new providers without changing business logic + +### 2. Registry Pattern + +- `LlmRouter` maintains registry of available providers +- Central point for provider discovery and instantiation + +### 3. Caching Pattern + +- `LLMResolverService` implements intelligent caching +- Reduces database load for frequent model resolution + +### 4. Observer Pattern + +- Usage tracking happens transparently during LLM calls +- Enables comprehensive analytics without coupling + +## Configuration + +### Default Model Assignments + +The system initializes with these default assignments: + +- **GPT-4o**: Most AI features (grading, generation, moderation) +- **GPT-4o-mini**: Translation and lightweight tasks +- **GPT-4.1-mini**: Image-related processing + +### Environment Configuration + +Required environment variables: + +```env +DATABASE_URL=postgresql://... +DATABASE_URL_DIRECT=postgresql://... +OPENAI_API_KEY=sk-... +``` + +## Monitoring and Analytics + +### Usage Metrics + +- Token consumption by feature and model +- Cost breakdown by usage type +- Model performance and utilization + +### Cost Tracking + +- Real-time cost calculation with historical accuracy +- Cost attribution by feature and assignment +- Budget monitoring and alerts + +### Performance Monitoring + +- Model response times and success rates +- Cache hit ratios and performance optimization +- Error rates and failure analysis + +## Security Considerations + +- Admin-only access to model assignments and pricing +- Secure API key management for LLM providers +- Input validation and sanitization +- Rate limiting and usage quotas (future enhancement) + +--- + +_This documentation provides a comprehensive overview of the LLM architecture. For implementation details and developer guides, see the accompanying developer documentation._ diff --git a/apps/api/docs/LLM_DEVELOPER_GUIDE.md b/apps/api/docs/LLM_DEVELOPER_GUIDE.md new file mode 100644 index 00000000..9a0844b5 --- /dev/null +++ b/apps/api/docs/LLM_DEVELOPER_GUIDE.md @@ -0,0 +1,552 @@ +# LLM Developer Guide: Adding New Models + +## Overview + +This guide walks you through the process of adding new LLM providers to the existing architecture. The system is designed to make this process straightforward while maintaining consistency and reliability. + +## Table of Contents + +1. [Quick Start Checklist](#quick-start-checklist) +2. [Step-by-Step Implementation](#step-by-step-implementation) +3. [Testing Your Implementation](#testing-your-implementation) +4. [Best Practices](#best-practices) +5. [Common Issues](#common-issues) + +## Quick Start Checklist + +Before you begin, ensure you have: + +- [ ] Access to the new LLM provider's API +- [ ] API credentials and configuration details +- [ ] Understanding of the provider's pricing model +- [ ] Token counting methodology for the new provider + +## Step-by-Step Implementation + +### Step 1: Create the LLM Provider Service + +Create a new service class that implements the `ILlmProvider` interface. + +**Example: Adding Claude from Anthropic** + +For a real-world example, see the [Llama-4-Maverick implementation](#llama-4-maverick-implementation-example) below. + +```typescript +// src/api/llm/core/services/claude-llm.service.ts +import { HumanMessage } from "@langchain/core/messages"; +import { Inject, Injectable } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { Logger } from "winston"; +import { TOKEN_COUNTER } from "../../llm.constants"; +import { + ILlmProvider, + LlmRequestOptions, + LlmResponse, +} from "../interfaces/llm-provider.interface"; +import { ITokenCounter } from "../interfaces/token-counter.interface"; + +@Injectable() +export class ClaudeLlmService implements ILlmProvider { + private readonly logger: Logger; + static readonly DEFAULT_MODEL = "claude-3-opus-20240229"; + readonly key = "claude-3-opus"; // This will be the model key in the database + + constructor( + @Inject(TOKEN_COUNTER) private readonly tokenCounter: ITokenCounter, + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger + ) { + this.logger = parentLogger.child({ context: ClaudeLlmService.name }); + } + + async invoke( + messages: HumanMessage[], + options?: LlmRequestOptions + ): Promise { + try { + // Initialize your LLM client (e.g., Anthropic SDK) + const anthropic = new Anthropic({ + apiKey: process.env.ANTHROPIC_API_KEY, + }); + + // Convert messages to provider format + const content = messages + .map((m) => + typeof m.content === "string" ? m.content : JSON.stringify(m.content) + ) + .join("\n"); + + // Make the API call + const response = await anthropic.messages.create({ + model: options?.modelName || ClaudeLlmService.DEFAULT_MODEL, + max_tokens: options?.maxTokens || 4000, + temperature: options?.temperature || 0.5, + messages: [{ role: "user", content }], + }); + + // Count tokens (implement based on provider's methodology) + const inputTokens = await this.tokenCounter.countTokens( + content, + this.key + ); + const outputTokens = response.usage.output_tokens; + + return { + content: response.content[0].text, + tokenUsage: { + input: inputTokens, + output: outputTokens, + }, + }; + } catch (error) { + this.logger.error(`Claude API error: ${error.message}`); + throw error; + } + } + + async invokeWithImage( + textContent: string, + imageData: string, + options?: LlmRequestOptions + ): Promise { + // Implement image processing if supported by the provider + // If not supported, throw an appropriate error + throw new Error("Image processing not supported by Claude provider"); + } +} +``` + +### Step 2: Register the Provider in the Module + +Update the LLM module to include your new provider. + +```typescript +// src/api/llm/llm.module.ts +import { ClaudeLlmService } from "./core/services/claude-llm.service"; + +@Global() +@Module({ + providers: [ + // ... existing providers + ClaudeLlmService, + { + provide: ALL_LLM_PROVIDERS, + useFactory: ( + openai: OpenAiLlmService, + openaiMini: OpenAiLlmMiniService, + gpt4Vision: Gpt4VisionPreviewLlmService, + claude: ClaudeLlmService // Add your provider here + ) => { + return [openai, openaiMini, gpt4Vision, claude]; + }, + inject: [ + OpenAiLlmService, + OpenAiLlmMiniService, + Gpt4VisionPreviewLlmService, + ClaudeLlmService, // Add to injection list + ], + }, + // ... rest of providers + ], +}) +export class LlmModule {} +``` + +### Step 3: Add Model to Database + +Create a migration to add the new model to the `LLMModel` table. + +```bash +npx prisma migrate dev --name add_claude_model --create-only +``` + +```sql +-- migrations/TIMESTAMP_add_claude_model/migration.sql +INSERT INTO "LLMModel" ("modelKey", "displayName", "provider", "isActive", "createdAt", "updatedAt") VALUES +('claude-3-opus', 'Claude 3 Opus', 'Anthropic', true, NOW(), NOW()) +ON CONFLICT ("modelKey") DO NOTHING; +``` + +### Step 4: Add Pricing Data + +Add pricing information for the new model. + +```sql +-- Add to the same migration or create a separate one +INSERT INTO "LLMPricing" ("modelId", "inputTokenPrice", "outputTokenPrice", "effectiveDate", "source", "isActive", "metadata", "createdAt", "updatedAt") +SELECT + m.id, + 0.000015 as inputTokenPrice, -- $15 per 1M input tokens + 0.000075 as outputTokenPrice, -- $75 per 1M output tokens + NOW() as effectiveDate, + 'MANUAL'::"PricingSource" as source, + true as isActive, + jsonb_build_object( + 'note', 'Initial Claude 3 Opus pricing', + 'source', 'Anthropic website', + 'lastUpdated', NOW()::text + ) as metadata, + NOW() as createdAt, + NOW() as updatedAt +FROM "LLMModel" m +WHERE m."modelKey" = 'claude-3-opus'; +``` + +### Step 5: Update Token Counter (if needed) + +If your provider uses a different tokenization method, extend the token counter service. + +```typescript +// src/api/llm/core/services/token-counter.service.ts +async countTokens(text: string, modelKey?: string): Promise { + switch (modelKey) { + case 'claude-3-opus': + // Implement Claude-specific token counting + return this.countClaudeTokens(text); + default: + // Use existing OpenAI tokenization + return this.countOpenAITokens(text); + } +} + +private countClaudeTokens(text: string): number { + // Implement Anthropic's tokenization logic + // You might need to use their SDK or estimation method + return Math.ceil(text.length / 4); // Rough estimation +} +``` + +### Step 6: Apply Migration and Test + +```bash +# Apply the migration +npx prisma migrate dev + +# Regenerate Prisma client +npx prisma generate + +# Test the new provider +npm run test -- --testNamePattern="Claude" +``` + +## Testing Your Implementation + +### Unit Tests + +Create comprehensive unit tests for your new provider. + +```typescript +// src/api/llm/core/services/claude-llm.service.spec.ts +describe("ClaudeLlmService", () => { + let service: ClaudeLlmService; + let tokenCounter: ITokenCounter; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [ + ClaudeLlmService, + { provide: TOKEN_COUNTER, useValue: mockTokenCounter }, + { provide: WINSTON_MODULE_PROVIDER, useValue: mockLogger }, + ], + }).compile(); + + service = module.get(ClaudeLlmService); + }); + + describe("invoke", () => { + it("should successfully process a simple prompt", async () => { + // Mock Anthropic API response + const mockResponse = { + content: [{ text: "Test response" }], + usage: { output_tokens: 2 }, + }; + + jest.spyOn(service, "invoke").mockResolvedValue({ + content: "Test response", + tokenUsage: { input: 5, output: 2 }, + }); + + const result = await service.invoke([new HumanMessage("Test prompt")]); + + expect(result.content).toBe("Test response"); + expect(result.tokenUsage.output).toBe(2); + }); + + it("should handle API errors gracefully", async () => { + // Test error handling + }); + }); +}); +``` + +### Integration Tests + +Test the full flow with your new provider. + +```typescript +// Integration test example +describe("LLM Integration with Claude", () => { + it("should route requests to Claude when assigned", async () => { + // 1. Assign Claude to a feature + await assignmentService.assignModelToFeature( + "text_grading", + "claude-3-opus" + ); + + // 2. Make a request using the feature + const response = await promptProcessor.processPromptForFeature( + testPrompt, + testAssignmentId, + AIUsageType.ASSIGNMENT_GRADING, + "text_grading" + ); + + // 3. Verify Claude was used and usage was tracked + expect(response).toBeDefined(); + // Verify usage was tracked with correct model key + }); +}); +``` + +## Best Practices + +### 1. Error Handling + +- Always wrap API calls in try-catch blocks +- Log errors with sufficient context +- Provide meaningful error messages +- Implement retries for transient failures + +```typescript +async invoke(messages: HumanMessage[]): Promise { + const maxRetries = 3; + let lastError: Error; + + for (let i = 0; i < maxRetries; i++) { + try { + return await this.makeApiCall(messages); + } catch (error) { + lastError = error; + if (this.isRetryableError(error) && i < maxRetries - 1) { + await this.sleep(Math.pow(2, i) * 1000); // Exponential backoff + continue; + } + break; + } + } + + this.logger.error(`Failed after ${maxRetries} retries:`, lastError); + throw lastError; +} +``` + +### 2. Configuration Management + +- Use environment variables for API keys and configuration +- Support different models from the same provider +- Make timeout and retry settings configurable + +```typescript +interface ClaudeConfig { + apiKey: string; + baseURL?: string; + timeout?: number; + maxRetries?: number; + defaultModel?: string; +} + +private getConfig(): ClaudeConfig { + return { + apiKey: process.env.ANTHROPIC_API_KEY!, + baseURL: process.env.ANTHROPIC_BASE_URL, + timeout: parseInt(process.env.ANTHROPIC_TIMEOUT || '30000'), + maxRetries: parseInt(process.env.ANTHROPIC_MAX_RETRIES || '3'), + defaultModel: process.env.ANTHROPIC_DEFAULT_MODEL || 'claude-3-opus-20240229', + }; +} +``` + +### 3. Token Counting Accuracy + +- Use the provider's official tokenization method when available +- Implement accurate token counting for cost calculation +- Consider different tokenization for different models from the same provider + +### 4. Rate Limiting + +- Respect the provider's rate limits +- Implement client-side rate limiting if necessary +- Handle rate limit errors gracefully + +```typescript +private rateLimiter = new RateLimiter({ + tokensPerInterval: 100, + interval: 'minute' +}); + +async invoke(messages: HumanMessage[]): Promise { + await this.rateLimiter.removeTokens(1); + // ... rest of implementation +} +``` + +### 5. Monitoring and Logging + +- Log all API calls with timing information +- Track success/failure rates +- Monitor token usage and costs +- Add health checks for the new provider + +```typescript +async invoke(messages: HumanMessage[]): Promise { + const startTime = Date.now(); + const requestId = crypto.randomUUID(); + + this.logger.info(`Starting Claude request ${requestId}`); + + try { + const response = await this.makeApiCall(messages); + const duration = Date.now() - startTime; + + this.logger.info(`Claude request ${requestId} completed in ${duration}ms`); + return response; + } catch (error) { + const duration = Date.now() - startTime; + this.logger.error(`Claude request ${requestId} failed after ${duration}ms:`, error); + throw error; + } +} +``` + +## Common Issues + +### Issue 1: Token Counting Mismatch + +**Problem**: Inaccurate cost calculations due to wrong token counting. + +**Solution**: + +- Use the provider's official SDK for token counting +- If unavailable, implement estimation based on provider documentation +- Test token counting against known examples + +### Issue 2: Provider-Specific Message Formats + +**Problem**: Different providers expect different message formats. + +**Solution**: + +- Convert from the standard `HumanMessage` format to provider-specific format +- Handle system messages, images, and other content types appropriately +- Test with various message types + +### Issue 3: Inconsistent Error Responses + +**Problem**: Different error formats from different providers. + +**Solution**: + +- Standardize error handling in your provider implementation +- Map provider-specific errors to common error types +- Provide consistent error messages for the application + +### Issue 4: Missing Environment Variables + +**Problem**: Application fails when new provider credentials are missing. + +**Solution**: + +- Validate required environment variables at startup +- Provide clear error messages for missing configuration +- Consider making new providers optional during development + +```typescript +@Injectable() +export class ClaudeLlmService implements ILlmProvider { + constructor(...) { + if (!process.env.ANTHROPIC_API_KEY) { + this.logger.warn('ANTHROPIC_API_KEY not provided - Claude provider will be disabled'); + } + } + + async invoke(...): Promise { + if (!process.env.ANTHROPIC_API_KEY) { + throw new Error('Claude provider not configured - missing ANTHROPIC_API_KEY'); + } + // ... implementation + } +} +``` + +## Advanced Features + +### Supporting Multiple Models from One Provider + +```typescript +@Injectable() +export class ClaudeHaikuLlmService implements ILlmProvider { + readonly key = "claude-3-haiku"; + static readonly DEFAULT_MODEL = "claude-3-haiku-20240307"; + // ... implementation differs only in model selection +} + +// Add both to the module +{ + provide: ALL_LLM_PROVIDERS, + useFactory: (..., claude: ClaudeLlmService, claudeHaiku: ClaudeHaikuLlmService) => { + return [..., claude, claudeHaiku]; + }, + inject: [..., ClaudeLlmService, ClaudeHaikuLlmService], +} +``` + +### Adding Specialized Capabilities + +```typescript +export interface IVisionLlmProvider extends ILlmProvider { + analyzeImage( + imageData: string, + prompt?: string + ): Promise; +} + +// Implement for vision-capable models +export class ClaudeVisionService implements IVisionLlmProvider { + // ... implement both ILlmProvider and vision-specific methods +} +``` + +## Deployment Checklist + +Before deploying your new LLM provider: + +- [ ] All tests pass (unit and integration) +- [ ] Environment variables configured in all environments +- [ ] Database migrations applied +- [ ] Pricing data accurate and up-to-date +- [ ] Error handling and logging implemented +- [ ] Rate limiting configured +- [ ] Documentation updated +- [ ] Admin interface shows new model options +- [ ] Cost calculations verified with actual usage +- [ ] Provider credentials secured and rotated if needed + +## Getting Help + +If you encounter issues: + +1. Check the existing provider implementations for patterns +2. Review the interface definitions for required methods +3. Ensure your provider follows the same patterns as existing ones +4. Test thoroughly with small requests before scaling up +5. Monitor logs and metrics after deployment + +For questions or issues, refer to: + +- LLM Architecture documentation +- Existing provider implementations +- Interface definitions in `/src/api/llm/core/interfaces/` +- Test examples in `/src/api/llm/core/services/*.spec.ts` + +--- + +_This guide covers the standard process for adding new LLM providers. Each provider may have unique requirements - adapt the process as needed while maintaining consistency with the existing architecture._ diff --git a/apps/api/docs/LLM_TROUBLESHOOTING.md b/apps/api/docs/LLM_TROUBLESHOOTING.md new file mode 100644 index 00000000..9862f5fc --- /dev/null +++ b/apps/api/docs/LLM_TROUBLESHOOTING.md @@ -0,0 +1,594 @@ +# LLM Architecture Troubleshooting Guide + +## Common Issues and Solutions + +### 1. Model Assignment Issues + +#### Problem: Feature shows "No model assigned" in admin interface + +**Symptoms:** + +- Admin interface shows empty or null assignments +- Features fall back to default models unexpectedly + +**Diagnosis:** + +```bash +# Check if assignments exist +psql $DATABASE_URL -c " +SELECT f.featureKey, f.displayName, m.modelKey, lfa.isActive +FROM \"AIFeature\" f +LEFT JOIN \"LLMFeatureAssignment\" lfa ON f.id = lfa.featureId AND lfa.isActive = true +LEFT JOIN \"LLMModel\" m ON lfa.modelId = m.id +ORDER BY f.featureKey; +" +``` + +**Solutions:** + +1. Populate missing assignments: + +```sql +-- Run the default assignment migration +INSERT INTO "LLMFeatureAssignment" ("featureId", "modelId", "isActive", "priority", "assignedBy") +SELECT f.id, m.id, true, 1, 'system' +FROM "AIFeature" f, "LLMModel" m +WHERE f.featureType = 'TEXT_GRADING' AND m.modelKey = 'gpt-4o' +ON CONFLICT DO NOTHING; +``` + +2. Check for inactive assignments: + +```sql +UPDATE "LLMFeatureAssignment" SET isActive = true WHERE featureId = [FEATURE_ID]; +``` + +#### Problem: Model resolution returns wrong model + +**Symptoms:** + +- Expected model not used for specific features +- Cache returning stale data + +**Diagnosis:** + +```typescript +// Check resolver service directly +const resolverService = app.get(LLM_RESOLVER_SERVICE); +const modelKey = await resolverService.resolveModelForFeature("text_grading"); +console.log("Resolved model:", modelKey); +``` + +**Solutions:** + +1. Clear cache: + +```typescript +await resolverService.clearCache(); +``` + +2. Check assignment priority: + +```sql +-- Higher priority assignments take precedence +SELECT * FROM "LLMFeatureAssignment" +WHERE featureId = [FEATURE_ID] AND isActive = true +ORDER BY priority DESC; +``` + +### 2. Cost Calculation Issues + +#### Problem: Costs show as $0.00 or wildly inaccurate + +**Symptoms:** + +- Dashboard shows zero costs despite usage +- Historical costs don't match expected values + +**Diagnosis:** + +```sql +-- Check if pricing data exists +SELECT m.modelKey, p.inputTokenPrice, p.outputTokenPrice, p.effectiveDate, p.isActive +FROM "LLMModel" m +LEFT JOIN "LLMPricing" p ON m.id = p.modelId +WHERE p.isActive = true OR p.id IS NULL +ORDER BY m.modelKey, p.effectiveDate DESC; + +-- Check AI usage data +SELECT modelKey, COUNT(*), SUM(tokensIn), SUM(tokensOut) +FROM "AIUsage" +GROUP BY modelKey; +``` + +**Solutions:** + +1. Add missing pricing data: + +```sql +INSERT INTO "LLMPricing" ("modelId", "inputTokenPrice", "outputTokenPrice", "effectiveDate", "source", "isActive") +SELECT m.id, 0.0000025, 0.00001, '2024-01-01'::timestamp, 'MANUAL', true +FROM "LLMModel" m +WHERE m.modelKey = 'gpt-4o' +ON CONFLICT ("modelId", "effectiveDate", "source") DO NOTHING; +``` + +2. Fix missing model keys in usage data: + +```sql +-- Update old records that don't have model keys +UPDATE "AIUsage" +SET modelKey = 'gpt-4o-mini' +WHERE modelKey IS NULL AND usageType IN ('TRANSLATION'); + +UPDATE "AIUsage" +SET modelKey = 'gpt-4o' +WHERE modelKey IS NULL AND usageType NOT IN ('TRANSLATION'); +``` + +#### Problem: Pricing service returns null for valid dates + +**Symptoms:** + +- Warnings: "No pricing found for [model] at [date]" +- Fallback pricing being used incorrectly + +**Diagnosis:** + +```typescript +const pricingService = app.get(LLM_PRICING_SERVICE); +const pricing = await pricingService.getPricingAtDate("gpt-4o", new Date()); +console.log("Pricing result:", pricing); +``` + +**Solutions:** + +1. Check date ranges in pricing data: + +```sql +SELECT modelKey, MIN(effectiveDate), MAX(effectiveDate), COUNT(*) +FROM "LLMPricing" p +JOIN "LLMModel" m ON p.modelId = m.id +GROUP BY modelKey; +``` + +2. Add historical baseline pricing: + +```sql +-- Add pricing that covers all historical dates +INSERT INTO "LLMPricing" ("modelId", "inputTokenPrice", "outputTokenPrice", "effectiveDate", "source", "isActive") +SELECT m.id, + CASE WHEN m.modelKey = 'gpt-4o' THEN 0.0000025 ELSE 0.00000015 END, + CASE WHEN m.modelKey = 'gpt-4o' THEN 0.00001 ELSE 0.0000006 END, + '2023-01-01'::timestamp, 'MANUAL', true +FROM "LLMModel" m +ON CONFLICT DO NOTHING; +``` + +### 3. Provider Integration Issues + +#### Problem: New LLM provider not registering + +**Symptoms:** + +- Provider not available in router +- "No LLM provider registered for key" errors + +**Diagnosis:** + +```typescript +const router = app.get(LlmRouter); +const availableModels = router.getAvailableModelKeys(); +console.log("Available models:", availableModels); +``` + +**Solutions:** + +1. Check provider is in module injection: + +```typescript +// Ensure provider is in ALL_LLM_PROVIDERS factory +{ + provide: ALL_LLM_PROVIDERS, + useFactory: (p1, p2, newProvider) => [p1, p2, newProvider], + inject: [Provider1, Provider2, NewProvider], // Must include all providers +} +``` + +2. Verify provider implements interface correctly: + +```typescript +export class NewProvider implements ILlmProvider { + readonly key = "new-model-key"; // Must be unique + + async invoke(messages: HumanMessage[]): Promise { + // Must return proper structure + } +} +``` + +#### Problem: Provider throws unexpected errors + +**Symptoms:** + +- API calls failing with unclear errors +- Inconsistent behavior across requests + +**Diagnosis:** + +```typescript +// Test provider directly +const provider = app.get(NewLlmService); +try { + const result = await provider.invoke([new HumanMessage("Test")]); + console.log("Success:", result); +} catch (error) { + console.error("Provider error:", error); +} +``` + +**Solutions:** + +1. Add comprehensive error handling: + +```typescript +async invoke(messages: HumanMessage[]): Promise { + try { + const response = await this.client.createMessage({...}); + return this.formatResponse(response); + } catch (error) { + this.logger.error(`Provider error: ${error.message}`, { + stack: error.stack, + messages: messages.length, + modelKey: this.key + }); + + if (this.isRetryableError(error)) { + throw new RetryableError(error.message); + } + throw new ProviderError(`Failed to process request: ${error.message}`); + } +} +``` + +### 4. Usage Tracking Issues + +#### Problem: Usage not being tracked + +**Symptoms:** + +- AIUsage table empty or missing recent entries +- Dashboard shows no usage statistics + +**Diagnosis:** + +```sql +-- Check recent usage tracking +SELECT usageType, modelKey, COUNT(*), MAX(createdAt) +FROM "AIUsage" +WHERE createdAt > NOW() - INTERVAL '1 day' +GROUP BY usageType, modelKey; +``` + +**Solutions:** + +1. Verify usage tracker is called: + +```typescript +// In PromptProcessorService +await this.usageTracker.trackUsage( + assignmentId, + usageType, + result.tokenUsage.input, + result.tokenUsage.output, + llm.key // Make sure this is included +); +``` + +2. Check for errors in usage tracking: + +```typescript +try { + await this.usageTracker.trackUsage(...); +} catch (error) { + // Log but don't fail the main request + this.logger.error('Failed to track usage:', error); +} +``` + +#### Problem: Model key not being stored in usage + +**Symptoms:** + +- AIUsage records have null modelKey +- Cost calculations falling back to guessing + +**Solutions:** + +1. Update usage tracking calls: + +```typescript +// Old version (missing model key) +await this.usageTracker.trackUsage( + assignmentId, + usageType, + tokensIn, + tokensOut +); + +// New version (with model key) +await this.usageTracker.trackUsage( + assignmentId, + usageType, + tokensIn, + tokensOut, + llm.key +); +``` + +2. Backfill missing model keys: + +```sql +-- Update based on usage patterns and dates +UPDATE "AIUsage" +SET modelKey = 'gpt-4o-mini' +WHERE modelKey IS NULL + AND usageType = 'TRANSLATION'; +``` + +### 5. Admin Interface Issues + +#### Problem: Admin interface not loading model data + +**Symptoms:** + +- Loading indicators that never complete +- Empty tables or error messages + +**Diagnosis:** + +```bash +# Check API endpoints directly +curl -H "x-admin-token: YOUR_TOKEN" http://localhost:3000/api/v1/admin/llm-assignments/features + +curl -H "x-admin-token: YOUR_TOKEN" http://localhost:3000/api/v1/admin/llm-assignments/models +``` + +**Solutions:** + +1. Check admin authentication: + +```typescript +// Ensure proper admin token validation +const adminService = app.get(AdminService); +const isValid = await adminService.validateAdminToken(token); +``` + +2. Verify API responses: + +```typescript +// Check controller returns proper format +{ + success: true, + data: [...], // Array of features/models + message: "Success" +} +``` + +#### Problem: Assignment changes not persisting + +**Symptoms:** + +- UI shows changes but reverts after refresh +- API calls appear successful but data unchanged + +**Solutions:** + +1. Check transaction handling: + +```typescript +await this.prisma.$transaction(async (tx) => { + // Deactivate old assignment + await tx.lLMFeatureAssignment.updateMany({ + where: { featureId, isActive: true }, + data: { isActive: false, deactivatedAt: new Date() }, + }); + + // Create new assignment + await tx.lLMFeatureAssignment.create({ + data: { featureId, modelId, isActive: true, priority: 1 }, + }); +}); +``` + +2. Clear cache after changes: + +```typescript +await this.resolverService.clearCache(); +``` + +### 6. Performance Issues + +#### Problem: Slow model resolution + +**Symptoms:** + +- API requests taking too long +- High database load from model queries + +**Solutions:** + +1. Check cache performance: + +```typescript +const stats = await resolverService.getStats(); +console.log("Cache hit ratio:", stats.hitRatio); +``` + +2. Optimize database queries: + +```sql +-- Ensure proper indexing +CREATE INDEX IF NOT EXISTS idx_llm_assignment_feature_active +ON "LLMFeatureAssignment"(featureId, isActive) WHERE isActive = true; + +CREATE INDEX IF NOT EXISTS idx_llm_pricing_model_date +ON "LLMPricing"(modelId, effectiveDate) WHERE isActive = true; +``` + +#### Problem: Memory leaks in caching layer + +**Symptoms:** + +- Increasing memory usage over time +- Cache growing without bounds + +**Solutions:** + +1. Configure cache TTL and size limits: + +```typescript +const cache = new LRUCache({ + max: 1000, // Maximum number of items + ttl: 5 * 60 * 1000, // 5 minutes +}); +``` + +2. Monitor cache metrics: + +```typescript +setInterval(() => { + this.logger.info("Cache stats:", { + size: this.cache.size, + hits: this.cacheHits, + misses: this.cacheMisses, + }); +}, 60000); +``` + +## Debugging Tools + +### 1. Database Queries for Diagnostics + +```sql +-- Complete system overview +SELECT + f.featureKey, + f.featureType, + f.defaultModelKey, + m.modelKey as assignedModel, + lfa.priority, + lfa.assignedAt, + p.inputTokenPrice, + p.outputTokenPrice +FROM "AIFeature" f +LEFT JOIN "LLMFeatureAssignment" lfa ON f.id = lfa.featureId AND lfa.isActive = true +LEFT JOIN "LLMModel" m ON lfa.modelId = m.id +LEFT JOIN "LLMPricing" p ON m.id = p.modelId AND p.isActive = true +ORDER BY f.featureKey; + +-- Usage and cost analysis +SELECT + u.modelKey, + u.usageType, + COUNT(*) as usage_count, + SUM(u.tokensIn) as total_input_tokens, + SUM(u.tokensOut) as total_output_tokens, + AVG(p.inputTokenPrice * u.tokensIn + p.outputTokenPrice * u.tokensOut) as avg_cost +FROM "AIUsage" u +LEFT JOIN "LLMModel" m ON u.modelKey = m.modelKey +LEFT JOIN "LLMPricing" p ON m.id = p.modelId AND p.isActive = true +WHERE u.createdAt > NOW() - INTERVAL '7 days' +GROUP BY u.modelKey, u.usageType +ORDER BY total_input_tokens + total_output_tokens DESC; +``` + +### 2. Service Health Checks + +```typescript +// Add to your health check controller +@Get('llm-health') +async checkLlmHealth() { + const router = this.app.get(LlmRouter); + const assignmentService = this.app.get(LLM_ASSIGNMENT_SERVICE); + const pricingService = this.app.get(LLM_PRICING_SERVICE); + + return { + availableModels: router.getAvailableModelKeys(), + assignmentStats: await assignmentService.getAssignmentStatistics(), + pricingStats: await pricingService.getPricingStatistics(), + cacheStats: await this.resolverService.getStats() + }; +} +``` + +### 3. Log Analysis Patterns + +```bash +# Find model resolution issues +grep "Failed to resolve model" logs/*.log + +# Find pricing issues +grep "No pricing found" logs/*.log + +# Find provider errors +grep "LLM provider error" logs/*.log + +# Usage tracking failures +grep "Failed to track usage" logs/*.log +``` + +## Emergency Procedures + +### 1. Disable Problematic Provider + +```sql +UPDATE "LLMModel" SET isActive = false WHERE modelKey = 'problematic-model'; +-- This will force fallback to default models +``` + +### 2. Reset to Default Assignments + +```sql +-- Clear all assignments and let system use defaults +UPDATE "LLMFeatureAssignment" SET isActive = false; +-- Or run the reset API endpoint +curl -X POST -H "x-admin-token: TOKEN" http://localhost:3000/api/v1/admin/llm-assignments/reset-to-defaults +``` + +### 3. Clear All Caches + +```typescript +// In emergency, restart the application or clear cache manually +await resolverService.clearCache(); +``` + +## Monitoring and Alerting + +Set up alerts for: + +- High error rates from LLM providers +- Cost spikes indicating pricing issues +- Cache hit ratio dropping below 80% +- Usage tracking failures +- Model assignment changes + +Example monitoring queries: + +```sql +-- Daily cost by model +SELECT + DATE(u.createdAt) as date, + u.modelKey, + SUM(p.inputTokenPrice * u.tokensIn + p.outputTokenPrice * u.tokensOut) as daily_cost +FROM "AIUsage" u +JOIN "LLMModel" m ON u.modelKey = m.modelKey +JOIN "LLMPricing" p ON m.id = p.modelId AND p.isActive = true +WHERE u.createdAt > NOW() - INTERVAL '30 days' +GROUP BY DATE(u.createdAt), u.modelKey +ORDER BY date DESC, daily_cost DESC; +``` + +--- + +_Keep this troubleshooting guide updated as new issues are discovered and resolved._ diff --git a/apps/api/package.json b/apps/api/package.json index affddd0c..01a7bced 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -22,7 +22,9 @@ "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", "test:staged": "jest --bail --findRelatedTests --passWithNoTests --collectCoverage=false", - "test:watch": "jest --watch" + "test:watch": "jest --watch", + "create-initial-versions": "dotenv -e ./dev.env -- ts-node src/scripts/create-initial-versions.ts", + "translation-audit": "node dist/src/scripts/translation-audit.js" }, "prisma": { "seed": "ts-node prisma/seed.ts" @@ -32,7 +34,7 @@ "@instana/collector": "^4.14.0", "@langchain/community": "^0.3.26", "@langchain/core": "^0.3.46", - "@langchain/openai": "^0.5.0", + "@langchain/openai": "^0.6.9", "@nestjs/axios": "^3.0.0", "@nestjs/class-validator": "^0.13.4", "@nestjs/cli": "^10.0.0", @@ -42,17 +44,21 @@ "@nestjs/jwt": "^10.1.0", "@nestjs/passport": "^10.0.0", "@nestjs/platform-express": "^10.4.9", + "@nestjs/schedule": "^6.0.0", "@nestjs/swagger": "^7.0.1", "@nestjs/terminus": "^10.0.0", "@nestjs/websockets": "^10.4.5", "@prisma/client": "^5.3.1", + "@sendgrid/mail": "^8.1.5", + "@tanstack/react-query": "^5.85.5", + "@types/nodemailer": "^6.4.17", "@types/pdf-parse": "^1.1.5", "aws-sdk": "^2.1692.0", "axios": "^1.7.4", "bottleneck": "^2.19.5", "bullmq": "^5.21.1", "chardet": "^2.1.0", - "cheerio": "^1.1.0", + "cheerio": "^1.1.2", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cld": "^2.10.0", @@ -67,10 +73,12 @@ "jsonwebtoken": "^9.0.1", "langchain": "^0.3.23", "mammoth": "^1.9.1", - "multer": "^2.0.2", + "minisearch": "^7.1.2", + "multer": "^2.0.1", "natural": "^8.0.1", "nest-winston": "^1.9.2", "node-xlsx": "^0.24.0", + "nodemailer": "^7.0.5", "octonode": "^0.10.2", "passport": "^0.7.0", "passport-jwt": "^4.0.1", @@ -87,7 +95,8 @@ "xml2js": "^0.6.2" }, "resolutions": { - "multer": ">=2.0.2" + "multer": "2.0.2", + "form-data": "4.0.4" }, "devDependencies": { "@ianvs/prettier-plugin-sort-imports": "^4.0.2", @@ -106,7 +115,8 @@ "@typescript-eslint/parser": "^5.59.11", "dotenv-cli": "^8.0.0", "eslint": "^8.42.0", - "eslint-config-prettier": "^10.1.8", + "eslint-config-prettier": "^8.8.0", + "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-unicorn": "^47.0.0", "husky": "^8.0.3", "jest": "^29.5.0", diff --git a/apps/api/prisma/migrations/20231024040946_/migration.sql b/apps/api/prisma/migrations/20231024040946_/migration.sql new file mode 100644 index 00000000..c78b1508 --- /dev/null +++ b/apps/api/prisma/migrations/20231024040946_/migration.sql @@ -0,0 +1,102 @@ +-- CreateEnum +CREATE TYPE "AssignmentType" AS ENUM ('AI_GRADED', 'MANUAL'); + +-- CreateEnum +CREATE TYPE "AssignmentQuestionDisplayOrder" AS ENUM ('DEFINED', 'RANDOM'); + +-- CreateEnum +CREATE TYPE "QuestionType" AS ENUM ('TEXT', 'SINGLE_CORRECT', 'MULTIPLE_CORRECT', 'TRUE_FALSE', 'URL', 'UPLOAD'); + +-- CreateEnum +CREATE TYPE "ScoringType" AS ENUM ('CRITERIA_BASED', 'LOSS_PER_MISTAKE', 'AI_GRADED'); + +-- CreateTable +CREATE TABLE "Group" ( + "id" TEXT NOT NULL +); + +-- CreateTable +CREATE TABLE "AssignmentGroup" ( + "assignmentId" INTEGER NOT NULL, + "groupId" TEXT NOT NULL, + + CONSTRAINT "AssignmentGroup_pkey" PRIMARY KEY ("assignmentId","groupId") +); + +-- CreateTable +CREATE TABLE "Assignment" ( + "id" SERIAL NOT NULL, + "name" TEXT NOT NULL, + "introduction" TEXT, + "instructions" TEXT, + "gradingCriteriaOverview" TEXT, + "type" "AssignmentType" NOT NULL, + "graded" BOOLEAN, + "numAttempts" INTEGER, + "allotedTimeMinutes" INTEGER, + "attemptsPerTimeRange" INTEGER, + "attemptsTimeRangeHours" INTEGER, + "passingGrade" INTEGER, + "displayOrder" "AssignmentQuestionDisplayOrder", + "questionOrder" INTEGER[], + "published" BOOLEAN NOT NULL, + + CONSTRAINT "Assignment_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Question" ( + "id" SERIAL NOT NULL, + "totalPoints" INTEGER NOT NULL, + "numRetries" INTEGER, + "type" "QuestionType" NOT NULL, + "question" TEXT NOT NULL, + "maxWords" INTEGER, + "scoring" JSONB, + "choices" JSONB, + "answer" BOOLEAN, + "assignmentId" INTEGER NOT NULL, + "gradingContextQuestionIds" INTEGER[], + + CONSTRAINT "Question_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "AssignmentAttempt" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "userId" TEXT NOT NULL, + "submitted" BOOLEAN NOT NULL, + "grade" DOUBLE PRECISION, + "expiresAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "AssignmentAttempt_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "QuestionResponse" ( + "id" SERIAL NOT NULL, + "assignmentAttemptId" INTEGER NOT NULL, + "questionId" INTEGER NOT NULL, + "learnerResponse" TEXT NOT NULL, + "points" INTEGER NOT NULL, + "feedback" JSONB NOT NULL, + + CONSTRAINT "QuestionResponse_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Group_id_key" ON "Group"("id"); + +-- AddForeignKey +ALTER TABLE "AssignmentGroup" ADD CONSTRAINT "AssignmentGroup_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "AssignmentGroup" ADD CONSTRAINT "AssignmentGroup_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "Group"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Question" ADD CONSTRAINT "Question_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "QuestionResponse" ADD CONSTRAINT "QuestionResponse_assignmentAttemptId_fkey" FOREIGN KEY ("assignmentAttemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20231026161100_/migration.sql b/apps/api/prisma/migrations/20231026161100_/migration.sql new file mode 100644 index 00000000..c9f53477 --- /dev/null +++ b/apps/api/prisma/migrations/20231026161100_/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "QuestionResponse" ALTER COLUMN "points" SET DATA TYPE DOUBLE PRECISION; diff --git a/apps/api/prisma/migrations/20231027170418_/migration.sql b/apps/api/prisma/migrations/20231027170418_/migration.sql new file mode 100644 index 00000000..6619c256 --- /dev/null +++ b/apps/api/prisma/migrations/20231027170418_/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "timeEstimateMinutes" INTEGER; diff --git a/apps/api/prisma/migrations/20240729162940_add_empty_question_type/migration.sql b/apps/api/prisma/migrations/20240729162940_add_empty_question_type/migration.sql new file mode 100644 index 00000000..f7fd34ed --- /dev/null +++ b/apps/api/prisma/migrations/20240729162940_add_empty_question_type/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "QuestionType" ADD VALUE 'EMPTY'; diff --git a/apps/api/prisma/migrations/20240729213014_add_character_limit/migration.sql b/apps/api/prisma/migrations/20240729213014_add_character_limit/migration.sql new file mode 100644 index 00000000..ce2bcf67 --- /dev/null +++ b/apps/api/prisma/migrations/20240729213014_add_character_limit/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Question" ADD COLUMN "maxCharacters" INTEGER; diff --git a/apps/api/prisma/migrations/20240810081638_verbosity/migration.sql b/apps/api/prisma/migrations/20240810081638_verbosity/migration.sql new file mode 100644 index 00000000..4a2a08be --- /dev/null +++ b/apps/api/prisma/migrations/20240810081638_verbosity/migration.sql @@ -0,0 +1,14 @@ +/* + Warnings: + + - You are about to drop the column `numRetries` on the `Question` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "showSubmissionFeedback" BOOLEAN NOT NULL DEFAULT false, +ADD COLUMN "showAssignmentScore" BOOLEAN NOT NULL DEFAULT false, +ADD COLUMN "showQuestionScore" BOOLEAN NOT NULL DEFAULT false, +ADD COLUMN "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP; + +-- AlterTable +ALTER TABLE "Question" DROP COLUMN "numRetries"; \ No newline at end of file diff --git a/apps/api/prisma/migrations/20240813142133_remove_empty_typed_questions/migration.sql b/apps/api/prisma/migrations/20240813142133_remove_empty_typed_questions/migration.sql new file mode 100644 index 00000000..b1c9511d --- /dev/null +++ b/apps/api/prisma/migrations/20240813142133_remove_empty_typed_questions/migration.sql @@ -0,0 +1,14 @@ +/* + Warnings: + + - The values [EMPTY] on the enum `QuestionType` will be removed. If these variants are still used in the database, this will fail. + +*/ +-- AlterEnum +BEGIN; +CREATE TYPE "QuestionType_new" AS ENUM ('TEXT', 'SINGLE_CORRECT', 'MULTIPLE_CORRECT', 'TRUE_FALSE', 'URL', 'UPLOAD'); +ALTER TABLE "Question" ALTER COLUMN "type" TYPE "QuestionType_new" USING ("type"::text::"QuestionType_new"); +ALTER TYPE "QuestionType" RENAME TO "QuestionType_old"; +ALTER TYPE "QuestionType_new" RENAME TO "QuestionType"; +DROP TYPE "QuestionType_old"; +COMMIT; diff --git a/apps/api/prisma/migrations/20240923165301_add_question_display/migration.sql b/apps/api/prisma/migrations/20240923165301_add_question_display/migration.sql new file mode 100644 index 00000000..8ea945e7 --- /dev/null +++ b/apps/api/prisma/migrations/20240923165301_add_question_display/migration.sql @@ -0,0 +1,5 @@ +-- CreateEnum +CREATE TYPE "QuestionDisplay" AS ENUM ('ONE_PER_PAGE', 'ALL_PER_PAGE'); + +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "questionDisplay" "QuestionDisplay" DEFAULT 'ONE_PER_PAGE'; diff --git a/apps/api/prisma/migrations/20241008194504_adding_default_values_for_required_fields/migration.sql b/apps/api/prisma/migrations/20241008194504_adding_default_values_for_required_fields/migration.sql new file mode 100644 index 00000000..24e68926 --- /dev/null +++ b/apps/api/prisma/migrations/20241008194504_adding_default_values_for_required_fields/migration.sql @@ -0,0 +1,7 @@ +-- AlterTable +ALTER TABLE "Assignment" ALTER COLUMN "graded" SET DEFAULT false, +ALTER COLUMN "numAttempts" SET DEFAULT -1, +ALTER COLUMN "passingGrade" SET DEFAULT 50, +ALTER COLUMN "showSubmissionFeedback" SET DEFAULT true, +ALTER COLUMN "showAssignmentScore" SET DEFAULT true, +ALTER COLUMN "showQuestionScore" SET DEFAULT true; diff --git a/apps/api/prisma/migrations/20241019035436_add_job/migration.sql b/apps/api/prisma/migrations/20241019035436_add_job/migration.sql new file mode 100644 index 00000000..ce63bbf9 --- /dev/null +++ b/apps/api/prisma/migrations/20241019035436_add_job/migration.sql @@ -0,0 +1,13 @@ +-- CreateTable +CREATE TABLE "Job" ( + "id" SERIAL NOT NULL, + "userId" TEXT NOT NULL, + "assignmentId" INTEGER NOT NULL, + "status" TEXT NOT NULL, + "progress" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "result" JSONB, + + CONSTRAINT "Job_pkey" PRIMARY KEY ("id") +); diff --git a/apps/api/prisma/migrations/20241022162316_added_ai_usage/migration.sql b/apps/api/prisma/migrations/20241022162316_added_ai_usage/migration.sql new file mode 100644 index 00000000..c3e151f2 --- /dev/null +++ b/apps/api/prisma/migrations/20241022162316_added_ai_usage/migration.sql @@ -0,0 +1,23 @@ +-- CreateEnum +CREATE TYPE "AIUsageType" AS ENUM ('QUESTION_GENERATION', 'ASSIGNMENT_GENERATION', 'ASSIGNMENT_GRADING'); + +-- CreateTable +CREATE TABLE "AIUsage" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "usageType" "AIUsageType" NOT NULL, + "tokensIn" INTEGER NOT NULL DEFAULT 0, + "tokensOut" INTEGER NOT NULL DEFAULT 0, + "usageCount" INTEGER NOT NULL DEFAULT 0, + "usageDetails" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "AIUsage_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "AIUsage_assignmentId_usageType_key" ON "AIUsage"("assignmentId", "usageType"); + +-- AddForeignKey +ALTER TABLE "AIUsage" ADD CONSTRAINT "AIUsage_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20241029151320_add_question_variants/migration.sql b/apps/api/prisma/migrations/20241029151320_add_question_variants/migration.sql new file mode 100644 index 00000000..c10ca5cf --- /dev/null +++ b/apps/api/prisma/migrations/20241029151320_add_question_variants/migration.sql @@ -0,0 +1,46 @@ +-- CreateEnum +CREATE TYPE "VariantType" AS ENUM ('REWORDED', 'RANDOMIZED', 'DIFFICULTY_ADJUSTED'); + +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "questionVariationNumber" INTEGER; + +-- CreateTable +CREATE TABLE "AssignmentAttemptQuestionVariant" ( + "assignmentAttemptId" INTEGER NOT NULL, + "questionId" INTEGER NOT NULL, + "questionVariantId" INTEGER NOT NULL, + + CONSTRAINT "AssignmentAttemptQuestionVariant_pkey" PRIMARY KEY ("assignmentAttemptId","questionId") +); + +-- CreateTable +CREATE TABLE "QuestionVariant" ( + "id" SERIAL NOT NULL, + "questionId" INTEGER NOT NULL, + "variantContent" TEXT NOT NULL, + "choices" JSONB, + "maxWords" INTEGER, + "scoring" JSONB, + "answer" BOOLEAN, + "maxCharacters" INTEGER, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "difficultyLevel" INTEGER, + "variantType" "VariantType" NOT NULL, + + CONSTRAINT "QuestionVariant_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "QuestionVariant_questionId_variantContent_key" ON "QuestionVariant"("questionId", "variantContent"); + +-- AddForeignKey +ALTER TABLE "AssignmentAttemptQuestionVariant" ADD CONSTRAINT "AssignmentAttemptQuestionVariant_assignmentAttemptId_fkey" FOREIGN KEY ("assignmentAttemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "AssignmentAttemptQuestionVariant" ADD CONSTRAINT "AssignmentAttemptQuestionVariant_questionVariantId_fkey" FOREIGN KEY ("questionVariantId") REFERENCES "QuestionVariant"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "AssignmentAttemptQuestionVariant" ADD CONSTRAINT "AssignmentAttemptQuestionVariant_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "QuestionVariant" ADD CONSTRAINT "QuestionVariant_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20241104161813_new_question_types_code_images/migration.sql b/apps/api/prisma/migrations/20241104161813_new_question_types_code_images/migration.sql new file mode 100644 index 00000000..09c0028d --- /dev/null +++ b/apps/api/prisma/migrations/20241104161813_new_question_types_code_images/migration.sql @@ -0,0 +1,10 @@ +-- AlterEnum +-- This migration adds more than one value to an enum. +-- With PostgreSQL versions 11 and earlier, this is not possible +-- in a single migration. This can be worked around by creating +-- multiple migrations, each migration adding only one value to +-- the enum. + + +ALTER TYPE "QuestionType" ADD VALUE 'CODE'; +ALTER TYPE "QuestionType" ADD VALUE 'IMAGES'; diff --git a/apps/api/prisma/migrations/20241114191903_remove_question_variation_from_assignment/migration.sql b/apps/api/prisma/migrations/20241114191903_remove_question_variation_from_assignment/migration.sql new file mode 100644 index 00000000..d5514772 --- /dev/null +++ b/apps/api/prisma/migrations/20241114191903_remove_question_variation_from_assignment/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - You are about to drop the column `questionVariationNumber` on the `Assignment` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "Assignment" DROP COLUMN "questionVariationNumber"; diff --git a/apps/api/prisma/migrations/20241120205158_modify_question_type_new_response_type/migration.sql b/apps/api/prisma/migrations/20241120205158_modify_question_type_new_response_type/migration.sql new file mode 100644 index 00000000..f8366300 --- /dev/null +++ b/apps/api/prisma/migrations/20241120205158_modify_question_type_new_response_type/migration.sql @@ -0,0 +1,20 @@ +/* + Warnings: + + - The values [CODE,IMAGES] on the enum `QuestionType` will be removed. If these variants are still used in the database, this will fail. + +*/ +-- CreateEnum +CREATE TYPE "ResponseType" AS ENUM ('CODE', 'ESSAY', 'REPORT', 'PRESENTATION', 'VIDEO', 'AUDIO', 'SPREADSHEET', 'OTHER'); + +-- AlterEnum +BEGIN; +CREATE TYPE "QuestionType_new" AS ENUM ('TEXT', 'SINGLE_CORRECT', 'MULTIPLE_CORRECT', 'TRUE_FALSE', 'URL', 'UPLOAD', 'LINK_FILE'); +ALTER TABLE "Question" ALTER COLUMN "type" TYPE "QuestionType_new" USING ("type"::text::"QuestionType_new"); +ALTER TYPE "QuestionType" RENAME TO "QuestionType_old"; +ALTER TYPE "QuestionType_new" RENAME TO "QuestionType"; +DROP TYPE "QuestionType_old"; +COMMIT; + +-- AlterTable +ALTER TABLE "Question" ADD COLUMN "responseType" "ResponseType"; diff --git a/apps/api/prisma/migrations/20241205164806_adding_assignment_feedback_regrading_request_models/migration.sql b/apps/api/prisma/migrations/20241205164806_adding_assignment_feedback_regrading_request_models/migration.sql new file mode 100644 index 00000000..3c461396 --- /dev/null +++ b/apps/api/prisma/migrations/20241205164806_adding_assignment_feedback_regrading_request_models/migration.sql @@ -0,0 +1,44 @@ +-- CreateEnum +CREATE TYPE "RegradingStatus" AS ENUM ('PENDING', 'APPROVED', 'REJECTED', 'COMPLETED'); + +-- CreateTable +CREATE TABLE "AssignmentFeedback" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "attemptId" INTEGER NOT NULL, + "userId" TEXT NOT NULL, + "comments" TEXT, + "aiGradingRating" INTEGER, + "assignmentRating" INTEGER, + "aiFeedbackRating" INTEGER, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "AssignmentFeedback_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "RegradingRequest" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "userId" TEXT NOT NULL, + "attemptId" INTEGER NOT NULL, + "regradingReason" TEXT, + "regradingStatus" "RegradingStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "RegradingRequest_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "AssignmentFeedback" ADD CONSTRAINT "AssignmentFeedback_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "AssignmentFeedback" ADD CONSTRAINT "AssignmentFeedback_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "RegradingRequest" ADD CONSTRAINT "RegradingRequest_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "RegradingRequest" ADD CONSTRAINT "RegradingRequest_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20241211184012_adding_report_models/migration.sql b/apps/api/prisma/migrations/20241211184012_adding_report_models/migration.sql new file mode 100644 index 00000000..e07a03f5 --- /dev/null +++ b/apps/api/prisma/migrations/20241211184012_adding_report_models/migration.sql @@ -0,0 +1,22 @@ +-- CreateEnum +CREATE TYPE "ReportType" AS ENUM ('BUG', 'FEEDBACK', 'SUGGESTION', 'PERFORMANCE', 'FALSE_MARKING', 'OTHER'); + +-- CreateEnum +CREATE TYPE "ReportStatus" AS ENUM ('OPEN', 'IN_PROGRESS', 'RESOLVED', 'CLOSED'); + +-- CreateTable +CREATE TABLE "Report" ( + "id" SERIAL NOT NULL, + "reporterId" TEXT NOT NULL, + "assignmentId" INTEGER NOT NULL, + "attemptId" INTEGER NOT NULL, + "issueType" "ReportType" NOT NULL, + "description" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "Report_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "Report" ADD CONSTRAINT "Report_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20241212204209_modifying_reports_model_for_authors/migration.sql b/apps/api/prisma/migrations/20241212204209_modifying_reports_model_for_authors/migration.sql new file mode 100644 index 00000000..1e2a03a3 --- /dev/null +++ b/apps/api/prisma/migrations/20241212204209_modifying_reports_model_for_authors/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "Report" ADD COLUMN "author" BOOLEAN NOT NULL DEFAULT false, +ALTER COLUMN "attemptId" DROP NOT NULL; diff --git a/apps/api/prisma/migrations/20241215001452_adding_repo_type/migration.sql b/apps/api/prisma/migrations/20241215001452_adding_repo_type/migration.sql new file mode 100644 index 00000000..12d0c6ab --- /dev/null +++ b/apps/api/prisma/migrations/20241215001452_adding_repo_type/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "ResponseType" ADD VALUE 'REPO'; diff --git a/apps/api/prisma/migrations/20241216175122_adding_user_cred_model/migration.sql b/apps/api/prisma/migrations/20241216175122_adding_user_cred_model/migration.sql new file mode 100644 index 00000000..e1b04a04 --- /dev/null +++ b/apps/api/prisma/migrations/20241216175122_adding_user_cred_model/migration.sql @@ -0,0 +1,9 @@ +-- CreateTable +CREATE TABLE "UserCredential" ( + "userId" TEXT NOT NULL, + "githubToken" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "UserCredential_pkey" PRIMARY KEY ("userId") +); diff --git a/apps/api/prisma/migrations/20241217030003_adding_soft_delete_option/migration.sql b/apps/api/prisma/migrations/20241217030003_adding_soft_delete_option/migration.sql new file mode 100644 index 00000000..d1d8cec2 --- /dev/null +++ b/apps/api/prisma/migrations/20241217030003_adding_soft_delete_option/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Question" ADD COLUMN "isDeleted" BOOLEAN NOT NULL DEFAULT false; diff --git a/apps/api/prisma/migrations/20250108164245_adding_soft_delete_variant/migration.sql b/apps/api/prisma/migrations/20250108164245_adding_soft_delete_variant/migration.sql new file mode 100644 index 00000000..214f2a9e --- /dev/null +++ b/apps/api/prisma/migrations/20250108164245_adding_soft_delete_variant/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "QuestionVariant" ADD COLUMN "isDeleted" BOOLEAN NOT NULL DEFAULT false; diff --git a/apps/api/prisma/migrations/20250110222254_create_translation_models/migration.sql b/apps/api/prisma/migrations/20250110222254_create_translation_models/migration.sql new file mode 100644 index 00000000..2ceaece9 --- /dev/null +++ b/apps/api/prisma/migrations/20250110222254_create_translation_models/migration.sql @@ -0,0 +1,18 @@ +-- CreateTable +CREATE TABLE "Translation" ( + "id" SERIAL NOT NULL, + "questionId" INTEGER NOT NULL, + "languageCode" TEXT NOT NULL, + "translatedText" TEXT NOT NULL, + "untranslatedText" TEXT NOT NULL, + "translatedChoices" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "Translation_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Translation_questionId_languageCode_key" ON "Translation"("questionId", "languageCode"); + +-- AddForeignKey +ALTER TABLE "Translation" ADD CONSTRAINT "Translation_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250113150913_adding_translations_to_ai_usage/migration.sql b/apps/api/prisma/migrations/20250113150913_adding_translations_to_ai_usage/migration.sql new file mode 100644 index 00000000..73b5e280 --- /dev/null +++ b/apps/api/prisma/migrations/20250113150913_adding_translations_to_ai_usage/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "AIUsageType" ADD VALUE 'TRANSLATION'; diff --git a/apps/api/prisma/migrations/20250115180431_adding_question_variation_translations/migration.sql b/apps/api/prisma/migrations/20250115180431_adding_question_variation_translations/migration.sql new file mode 100644 index 00000000..f9e2d38e --- /dev/null +++ b/apps/api/prisma/migrations/20250115180431_adding_question_variation_translations/migration.sql @@ -0,0 +1,19 @@ +/* + Warnings: + + - A unique constraint covering the columns `[variantId,languageCode]` on the table `Translation` will be added. If there are existing duplicate values, this will fail. + +*/ +-- AlterTable +ALTER TABLE "Translation" ADD COLUMN "variantId" INTEGER, +ALTER COLUMN "questionId" DROP NOT NULL, +ALTER COLUMN "untranslatedText" DROP NOT NULL; + +-- CreateIndex +CREATE UNIQUE INDEX "unique_variant_lang" ON "Translation"("variantId", "languageCode"); + +-- AddForeignKey +ALTER TABLE "Translation" ADD CONSTRAINT "Translation_variantId_fkey" FOREIGN KEY ("variantId") REFERENCES "QuestionVariant"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- RenameIndex +ALTER INDEX "Translation_questionId_languageCode_key" RENAME TO "unique_question_lang"; diff --git a/apps/api/prisma/migrations/20250128192418_randomized_choices_flag_questionlvl/migration.sql b/apps/api/prisma/migrations/20250128192418_randomized_choices_flag_questionlvl/migration.sql new file mode 100644 index 00000000..95c44197 --- /dev/null +++ b/apps/api/prisma/migrations/20250128192418_randomized_choices_flag_questionlvl/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE "AssignmentAttempt" ADD COLUMN "questionOrder" INTEGER[]; + +-- AlterTable +ALTER TABLE "AssignmentAttemptQuestionVariant" ADD COLUMN "randomizedChoices" JSONB; + +-- AlterTable +ALTER TABLE "Question" ADD COLUMN "randomizedChoices" BOOLEAN; + +-- AlterTable +ALTER TABLE "QuestionVariant" ADD COLUMN "randomizedChoices" BOOLEAN; diff --git a/apps/api/prisma/migrations/20250128215303_making_question_variant_id_optional/migration.sql b/apps/api/prisma/migrations/20250128215303_making_question_variant_id_optional/migration.sql new file mode 100644 index 00000000..1014f4e5 --- /dev/null +++ b/apps/api/prisma/migrations/20250128215303_making_question_variant_id_optional/migration.sql @@ -0,0 +1,8 @@ +-- DropForeignKey +ALTER TABLE "AssignmentAttemptQuestionVariant" DROP CONSTRAINT "AssignmentAttemptQuestionVariant_questionVariantId_fkey"; + +-- AlterTable +ALTER TABLE "AssignmentAttemptQuestionVariant" ALTER COLUMN "questionVariantId" DROP NOT NULL; + +-- AddForeignKey +ALTER TABLE "AssignmentAttemptQuestionVariant" ADD CONSTRAINT "AssignmentAttemptQuestionVariant_questionVariantId_fkey" FOREIGN KEY ("questionVariantId") REFERENCES "QuestionVariant"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250131160706_comments_assignment_attempt_for_passsing_deadlines/migration.sql b/apps/api/prisma/migrations/20250131160706_comments_assignment_attempt_for_passsing_deadlines/migration.sql new file mode 100644 index 00000000..eb27f6c1 --- /dev/null +++ b/apps/api/prisma/migrations/20250131160706_comments_assignment_attempt_for_passsing_deadlines/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "AssignmentAttempt" ADD COLUMN "comments" TEXT; diff --git a/apps/api/prisma/migrations/20250201034438_changing_translation_constraint/migration.sql b/apps/api/prisma/migrations/20250201034438_changing_translation_constraint/migration.sql new file mode 100644 index 00000000..144c2d4a --- /dev/null +++ b/apps/api/prisma/migrations/20250201034438_changing_translation_constraint/migration.sql @@ -0,0 +1,8 @@ +-- DropIndex +DROP INDEX "unique_question_lang"; + +-- DropIndex +DROP INDEX "unique_variant_lang"; + +-- CreateIndex +CREATE INDEX "Translation_questionId_variantId_languageCode_idx" ON "Translation"("questionId", "variantId", "languageCode"); diff --git a/apps/api/prisma/migrations/20250204154022_adding_assignment_translation_feedback_translation/migration.sql b/apps/api/prisma/migrations/20250204154022_adding_assignment_translation_feedback_translation/migration.sql new file mode 100644 index 00000000..072aa4a8 --- /dev/null +++ b/apps/api/prisma/migrations/20250204154022_adding_assignment_translation_feedback_translation/migration.sql @@ -0,0 +1,49 @@ +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "languageCode" TEXT; + +-- AlterTable +ALTER TABLE "AssignmentAttempt" ADD COLUMN "preferredLanguage" TEXT; + +-- CreateTable +CREATE TABLE "AssignmentTranslation" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "languageCode" TEXT NOT NULL, + "name" TEXT NOT NULL, + "introduction" TEXT NOT NULL, + "instructions" TEXT, + "gradingCriteriaOverview" TEXT, + "translatedName" TEXT, + "translatedIntroduction" TEXT, + "translatedInstructions" TEXT, + "translatedGradingCriteriaOverview" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "AssignmentTranslation_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "FeedbackTranslation" ( + "id" SERIAL NOT NULL, + "questionId" INTEGER NOT NULL, + "languageCode" TEXT NOT NULL, + "untranslatedFeedback" JSONB NOT NULL, + "translatedFeedback" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "FeedbackTranslation_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "AssignmentTranslation_assignmentId_languageCode_key" ON "AssignmentTranslation"("assignmentId", "languageCode"); + +-- CreateIndex +CREATE UNIQUE INDEX "FeedbackTranslation_questionId_languageCode_key" ON "FeedbackTranslation"("questionId", "languageCode"); + +-- AddForeignKey +ALTER TABLE "AssignmentTranslation" ADD CONSTRAINT "AssignmentTranslation_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "FeedbackTranslation" ADD CONSTRAINT "FeedbackTranslation_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250204172050_adding_publish_job/migration.sql b/apps/api/prisma/migrations/20250204172050_adding_publish_job/migration.sql new file mode 100644 index 00000000..8b8be9b1 --- /dev/null +++ b/apps/api/prisma/migrations/20250204172050_adding_publish_job/migration.sql @@ -0,0 +1,13 @@ +-- CreateTable +CREATE TABLE "publishJob" ( + "id" SERIAL NOT NULL, + "userId" TEXT NOT NULL, + "assignmentId" INTEGER NOT NULL, + "status" TEXT NOT NULL, + "progress" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "result" JSONB, + + CONSTRAINT "publishJob_pkey" PRIMARY KEY ("id") +); diff --git a/apps/api/prisma/migrations/20250215044331_adding_percentage_publish_job/migration.sql b/apps/api/prisma/migrations/20250215044331_adding_percentage_publish_job/migration.sql new file mode 100644 index 00000000..3c9cc3fb --- /dev/null +++ b/apps/api/prisma/migrations/20250215044331_adding_percentage_publish_job/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "publishJob" ADD COLUMN "percentage" INTEGER; diff --git a/apps/api/prisma/migrations/20250313134152_adding_untranslated_choices/migration.sql b/apps/api/prisma/migrations/20250313134152_adding_untranslated_choices/migration.sql new file mode 100644 index 00000000..454913fe --- /dev/null +++ b/apps/api/prisma/migrations/20250313134152_adding_untranslated_choices/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Translation" ADD COLUMN "untranslatedChoices" JSONB; diff --git a/apps/api/prisma/migrations/20250318184857_adding_grading_presentations/migration.sql b/apps/api/prisma/migrations/20250318184857_adding_grading_presentations/migration.sql new file mode 100644 index 00000000..587af57f --- /dev/null +++ b/apps/api/prisma/migrations/20250318184857_adding_grading_presentations/migration.sql @@ -0,0 +1,9 @@ +-- AlterEnum +ALTER TYPE "AIUsageType" ADD VALUE 'LIVE_RECORDING_FEEDBACK'; + +-- AlterEnum +ALTER TYPE "ResponseType" ADD VALUE 'LIVE_RECORDING'; + +-- AlterTable +ALTER TABLE "Question" ADD COLUMN "liveRecordingConfig" JSONB, +ADD COLUMN "videoPresentationConfig" JSONB; diff --git a/apps/api/prisma/migrations/20250410201808_adding_file_saving_tables_cos/migration.sql b/apps/api/prisma/migrations/20250410201808_adding_file_saving_tables_cos/migration.sql new file mode 100644 index 00000000..4c7122c9 --- /dev/null +++ b/apps/api/prisma/migrations/20250410201808_adding_file_saving_tables_cos/migration.sql @@ -0,0 +1,65 @@ +-- CreateTable +CREATE TABLE "AuthorUpload" ( + "id" SERIAL NOT NULL, + "userId" TEXT NOT NULL, + "fileName" TEXT NOT NULL, + "fileType" TEXT NOT NULL, + "cosKey" TEXT NOT NULL, + "cosBucket" TEXT NOT NULL, + "fileSize" INTEGER, + "contentType" TEXT, + "assignmentId" INTEGER, + "questionId" INTEGER, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "AuthorUpload_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "LearnerFileUpload" ( + "id" SERIAL NOT NULL, + "userId" TEXT NOT NULL, + "assignmentId" INTEGER NOT NULL, + "attemptId" INTEGER NOT NULL, + "questionId" INTEGER NOT NULL, + "fileName" TEXT NOT NULL, + "fileType" TEXT NOT NULL, + "cosKey" TEXT NOT NULL, + "cosBucket" TEXT NOT NULL, + "fileSize" INTEGER, + "contentType" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LearnerFileUpload_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "ReportFile" ( + "id" SERIAL NOT NULL, + "reportId" INTEGER NOT NULL, + "userId" TEXT NOT NULL, + "fileName" TEXT NOT NULL, + "fileType" TEXT NOT NULL, + "cosKey" TEXT NOT NULL, + "cosBucket" TEXT NOT NULL, + "fileSize" INTEGER, + "contentType" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "ReportFile_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "LearnerFileUpload" ADD CONSTRAINT "LearnerFileUpload_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "LearnerFileUpload" ADD CONSTRAINT "LearnerFileUpload_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "LearnerFileUpload" ADD CONSTRAINT "LearnerFileUpload_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "ReportFile" ADD CONSTRAINT "ReportFile_reportId_fkey" FOREIGN KEY ("reportId") REFERENCES "Report"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250422160403_adding_assignment_types/migration.sql b/apps/api/prisma/migrations/20250422160403_adding_assignment_types/migration.sql new file mode 100644 index 00000000..1d4e5805 --- /dev/null +++ b/apps/api/prisma/migrations/20250422160403_adding_assignment_types/migration.sql @@ -0,0 +1,2 @@ +-- CreateEnum +CREATE TYPE "AssignmentTypeEnum" AS ENUM ('Quiz', 'Assignment', 'Project', 'Midterm', 'Final', 'Exam', 'Test', 'Lab', 'Homework', 'Practice', 'Assessment', 'Survey', 'Evaluation', 'Review', 'Reflection'); diff --git a/apps/api/prisma/migrations/20250427231006_adding_grading_audit/migration.sql b/apps/api/prisma/migrations/20250427231006_adding_grading_audit/migration.sql new file mode 100644 index 00000000..2f34675b --- /dev/null +++ b/apps/api/prisma/migrations/20250427231006_adding_grading_audit/migration.sql @@ -0,0 +1,28 @@ +-- CreateTable +CREATE TABLE "GradingAudit" ( + "id" SERIAL NOT NULL, + "questionId" INTEGER NOT NULL, + "assignmentId" INTEGER, + "requestPayload" TEXT NOT NULL, + "responsePayload" TEXT NOT NULL, + "gradingStrategy" TEXT NOT NULL, + "metadata" TEXT, + "timestamp" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "GradingAudit_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "GradingAudit_questionId_idx" ON "GradingAudit"("questionId"); + +-- CreateIndex +CREATE INDEX "GradingAudit_assignmentId_idx" ON "GradingAudit"("assignmentId"); + +-- CreateIndex +CREATE INDEX "GradingAudit_timestamp_idx" ON "GradingAudit"("timestamp"); + +-- AddForeignKey +ALTER TABLE "GradingAudit" ADD CONSTRAINT "GradingAudit_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "GradingAudit" ADD CONSTRAINT "GradingAudit_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250427233833_adding_metadata_to_quesiton_response/migration.sql b/apps/api/prisma/migrations/20250427233833_adding_metadata_to_quesiton_response/migration.sql new file mode 100644 index 00000000..e2e59ca7 --- /dev/null +++ b/apps/api/prisma/migrations/20250427233833_adding_metadata_to_quesiton_response/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "QuestionResponse" ADD COLUMN "gradedAt" TIMESTAMP(3), +ADD COLUMN "metadata" JSONB; diff --git a/apps/api/prisma/migrations/20250512201648_adding_show_question_option/migration.sql b/apps/api/prisma/migrations/20250512201648_adding_show_question_option/migration.sql new file mode 100644 index 00000000..4d790dea --- /dev/null +++ b/apps/api/prisma/migrations/20250512201648_adding_show_question_option/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "showQuestions" BOOLEAN NOT NULL DEFAULT true; diff --git a/apps/api/prisma/migrations/20250515203923_adding_chat_data/migration.sql b/apps/api/prisma/migrations/20250515203923_adding_chat_data/migration.sql new file mode 100644 index 00000000..498c2d00 --- /dev/null +++ b/apps/api/prisma/migrations/20250515203923_adding_chat_data/migration.sql @@ -0,0 +1,66 @@ +-- CreateEnum +CREATE TYPE "ChatRole" AS ENUM ('USER', 'ASSISTANT', 'SYSTEM'); + +-- AlterTable +ALTER TABLE "AIUsage" ADD COLUMN "userId" TEXT; + +-- CreateTable +CREATE TABLE "Chat" ( + "id" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "lastActiveAt" TIMESTAMP(3) NOT NULL, + "title" TEXT, + "assignmentId" INTEGER, + "isActive" BOOLEAN NOT NULL DEFAULT true, + + CONSTRAINT "Chat_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "ChatMessage" ( + "id" SERIAL NOT NULL, + "chatId" TEXT NOT NULL, + "role" "ChatRole" NOT NULL, + "content" TEXT NOT NULL, + "timestamp" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "toolCalls" JSONB, + + CONSTRAINT "ChatMessage_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "_ChatToUserCredential" ( + "A" TEXT NOT NULL, + "B" TEXT NOT NULL +); + +-- CreateIndex +CREATE INDEX "Chat_userId_startedAt_idx" ON "Chat"("userId", "startedAt"); + +-- CreateIndex +CREATE INDEX "Chat_assignmentId_idx" ON "Chat"("assignmentId"); + +-- CreateIndex +CREATE INDEX "ChatMessage_chatId_timestamp_idx" ON "ChatMessage"("chatId", "timestamp"); + +-- CreateIndex +CREATE UNIQUE INDEX "_ChatToUserCredential_AB_unique" ON "_ChatToUserCredential"("A", "B"); + +-- CreateIndex +CREATE INDEX "_ChatToUserCredential_B_index" ON "_ChatToUserCredential"("B"); + +-- AddForeignKey +ALTER TABLE "AIUsage" ADD CONSTRAINT "AIUsage_userId_fkey" FOREIGN KEY ("userId") REFERENCES "UserCredential"("userId") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Chat" ADD CONSTRAINT "Chat_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "ChatMessage" ADD CONSTRAINT "ChatMessage_chatId_fkey" FOREIGN KEY ("chatId") REFERENCES "Chat"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "_ChatToUserCredential" ADD CONSTRAINT "_ChatToUserCredential_A_fkey" FOREIGN KEY ("A") REFERENCES "Chat"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "_ChatToUserCredential" ADD CONSTRAINT "_ChatToUserCredential_B_fkey" FOREIGN KEY ("B") REFERENCES "UserCredential"("userId") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250520145827_adding_user_notifications_reports_system/migration.sql b/apps/api/prisma/migrations/20250520145827_adding_user_notifications_reports_system/migration.sql new file mode 100644 index 00000000..c0aa6c2f --- /dev/null +++ b/apps/api/prisma/migrations/20250520145827_adding_user_notifications_reports_system/migration.sql @@ -0,0 +1,38 @@ +-- DropForeignKey +ALTER TABLE "Report" DROP CONSTRAINT "Report_assignmentId_fkey"; + +-- AlterTable +ALTER TABLE "Report" ADD COLUMN "closureReason" TEXT, +ADD COLUMN "comments" TEXT, +ADD COLUMN "duplicateOfReportId" INTEGER, +ADD COLUMN "issueNumber" INTEGER, +ADD COLUMN "relatedToReportId" INTEGER, +ADD COLUMN "resolution" TEXT, +ADD COLUMN "similarityScore" DOUBLE PRECISION, +ADD COLUMN "status" "ReportStatus" NOT NULL DEFAULT 'OPEN', +ADD COLUMN "statusMessage" TEXT, +ALTER COLUMN "assignmentId" DROP NOT NULL; + +-- CreateTable +CREATE TABLE "UserNotification" ( + "id" SERIAL NOT NULL, + "userId" TEXT NOT NULL, + "type" TEXT NOT NULL, + "title" TEXT NOT NULL, + "message" TEXT NOT NULL, + "metadata" TEXT, + "read" BOOLEAN NOT NULL DEFAULT false, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "UserNotification_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "Report" ADD CONSTRAINT "Report_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Report" ADD CONSTRAINT "Report_duplicateOfReportId_fkey" FOREIGN KEY ("duplicateOfReportId") REFERENCES "Report"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Report" ADD CONSTRAINT "Report_relatedToReportId_fkey" FOREIGN KEY ("relatedToReportId") REFERENCES "Report"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250530185450_adding_number_of_questions_per_attempt/migration.sql b/apps/api/prisma/migrations/20250530185450_adding_number_of_questions_per_attempt/migration.sql new file mode 100644 index 00000000..0f32c2f3 --- /dev/null +++ b/apps/api/prisma/migrations/20250530185450_adding_number_of_questions_per_attempt/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "numberOfQuestionsPerAttempt" INTEGER; diff --git a/apps/api/prisma/migrations/20250611210911_cos_file_uploads/migration.sql b/apps/api/prisma/migrations/20250611210911_cos_file_uploads/migration.sql new file mode 100644 index 00000000..89ce530a --- /dev/null +++ b/apps/api/prisma/migrations/20250611210911_cos_file_uploads/migration.sql @@ -0,0 +1,28 @@ +/* + Warnings: + + - You are about to drop the `AuthorUpload` table. If the table is not empty, all the data it contains will be lost. + - You are about to drop the `LearnerFileUpload` table. If the table is not empty, all the data it contains will be lost. + - You are about to drop the `ReportFile` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropForeignKey +ALTER TABLE "LearnerFileUpload" DROP CONSTRAINT "LearnerFileUpload_assignmentId_fkey"; + +-- DropForeignKey +ALTER TABLE "LearnerFileUpload" DROP CONSTRAINT "LearnerFileUpload_attemptId_fkey"; + +-- DropForeignKey +ALTER TABLE "LearnerFileUpload" DROP CONSTRAINT "LearnerFileUpload_questionId_fkey"; + +-- DropForeignKey +ALTER TABLE "ReportFile" DROP CONSTRAINT "ReportFile_reportId_fkey"; + +-- DropTable +DROP TABLE "AuthorUpload"; + +-- DropTable +DROP TABLE "LearnerFileUpload"; + +-- DropTable +DROP TABLE "ReportFile"; diff --git a/apps/api/prisma/migrations/20250618045552_adding_image_response_type/migration.sql b/apps/api/prisma/migrations/20250618045552_adding_image_response_type/migration.sql new file mode 100644 index 00000000..44bef893 --- /dev/null +++ b/apps/api/prisma/migrations/20250618045552_adding_image_response_type/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "ResponseType" ADD VALUE 'IMAGES'; diff --git a/apps/api/prisma/migrations/20250702182506_adding_grading_job/migration.sql b/apps/api/prisma/migrations/20250702182506_adding_grading_job/migration.sql new file mode 100644 index 00000000..1f79d6e8 --- /dev/null +++ b/apps/api/prisma/migrations/20250702182506_adding_grading_job/migration.sql @@ -0,0 +1,33 @@ +-- CreateTable +CREATE TABLE "GradingJob" ( + "id" SERIAL NOT NULL, + "attemptId" INTEGER, + "assignmentId" INTEGER NOT NULL, + "userId" TEXT NOT NULL, + "status" TEXT NOT NULL, + "progress" TEXT NOT NULL, + "percentage" INTEGER, + "result" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "GradingJob_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "GradingJob_attemptId_idx" ON "GradingJob"("attemptId"); + +-- CreateIndex +CREATE INDEX "GradingJob_assignmentId_idx" ON "GradingJob"("assignmentId"); + +-- CreateIndex +CREATE INDEX "GradingJob_userId_idx" ON "GradingJob"("userId"); + +-- CreateIndex +CREATE INDEX "GradingJob_status_idx" ON "GradingJob"("status"); + +-- AddForeignKey +ALTER TABLE "GradingJob" ADD CONSTRAINT "GradingJob_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "GradingJob" ADD CONSTRAINT "GradingJob_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250708152918_adding_grading_validation_type/migration.sql b/apps/api/prisma/migrations/20250708152918_adding_grading_validation_type/migration.sql new file mode 100644 index 00000000..0a051702 --- /dev/null +++ b/apps/api/prisma/migrations/20250708152918_adding_grading_validation_type/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "AIUsageType" ADD VALUE 'GRADING_VALIDATION'; diff --git a/apps/api/prisma/migrations/20250716085011_add_contact_fields_to_feedback/migration.sql b/apps/api/prisma/migrations/20250716085011_add_contact_fields_to_feedback/migration.sql new file mode 100644 index 00000000..b31907de --- /dev/null +++ b/apps/api/prisma/migrations/20250716085011_add_contact_fields_to_feedback/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "AssignmentFeedback" ADD COLUMN "allowContact" BOOLEAN NOT NULL DEFAULT false, +ADD COLUMN "email" TEXT, +ADD COLUMN "firstName" TEXT, +ADD COLUMN "lastName" TEXT; diff --git a/apps/api/prisma/migrations/20250728142646_init/migration.sql b/apps/api/prisma/migrations/20250728142646_init/migration.sql deleted file mode 100644 index 5f4dff04..00000000 --- a/apps/api/prisma/migrations/20250728142646_init/migration.sql +++ /dev/null @@ -1,539 +0,0 @@ --- CreateEnum -CREATE TYPE "AssignmentType" AS ENUM ('AI_GRADED', 'MANUAL'); - --- CreateEnum -CREATE TYPE "AssignmentQuestionDisplayOrder" AS ENUM ('DEFINED', 'RANDOM'); - --- CreateEnum -CREATE TYPE "QuestionType" AS ENUM ('TEXT', 'SINGLE_CORRECT', 'MULTIPLE_CORRECT', 'TRUE_FALSE', 'URL', 'UPLOAD', 'LINK_FILE'); - --- CreateEnum -CREATE TYPE "ResponseType" AS ENUM ('REPO', 'CODE', 'ESSAY', 'REPORT', 'PRESENTATION', 'VIDEO', 'AUDIO', 'IMAGES', 'SPREADSHEET', 'LIVE_RECORDING', 'OTHER'); - --- CreateEnum -CREATE TYPE "QuestionDisplay" AS ENUM ('ONE_PER_PAGE', 'ALL_PER_PAGE'); - --- CreateEnum -CREATE TYPE "ScoringType" AS ENUM ('CRITERIA_BASED', 'LOSS_PER_MISTAKE', 'AI_GRADED'); - --- CreateEnum -CREATE TYPE "AIUsageType" AS ENUM ('QUESTION_GENERATION', 'ASSIGNMENT_GENERATION', 'ASSIGNMENT_GRADING', 'TRANSLATION', 'LIVE_RECORDING_FEEDBACK'); - --- CreateEnum -CREATE TYPE "ChatRole" AS ENUM ('USER', 'ASSISTANT', 'SYSTEM'); - --- CreateEnum -CREATE TYPE "VariantType" AS ENUM ('REWORDED', 'RANDOMIZED', 'DIFFICULTY_ADJUSTED'); - --- CreateEnum -CREATE TYPE "RegradingStatus" AS ENUM ('PENDING', 'APPROVED', 'REJECTED', 'COMPLETED'); - --- CreateEnum -CREATE TYPE "ReportType" AS ENUM ('BUG', 'FEEDBACK', 'SUGGESTION', 'PERFORMANCE', 'FALSE_MARKING', 'OTHER'); - --- CreateEnum -CREATE TYPE "ReportStatus" AS ENUM ('OPEN', 'IN_PROGRESS', 'RESOLVED', 'CLOSED'); - --- CreateEnum -CREATE TYPE "AssignmentTypeEnum" AS ENUM ('Quiz', 'Assignment', 'Project', 'Midterm', 'Final', 'Exam', 'Test', 'Lab', 'Homework', 'Practice', 'Assessment', 'Survey', 'Evaluation', 'Review', 'Reflection'); - --- CreateTable -CREATE TABLE "GradingAudit" ( - "id" SERIAL NOT NULL, - "questionId" INTEGER NOT NULL, - "assignmentId" INTEGER, - "requestPayload" TEXT NOT NULL, - "responsePayload" TEXT NOT NULL, - "gradingStrategy" TEXT NOT NULL, - "metadata" TEXT, - "timestamp" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "GradingAudit_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Job" ( - "id" SERIAL NOT NULL, - "userId" TEXT NOT NULL, - "assignmentId" INTEGER NOT NULL, - "status" TEXT NOT NULL, - "progress" TEXT, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - "result" JSONB, - - CONSTRAINT "Job_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "GradingJob" ( - "id" SERIAL NOT NULL, - "attemptId" INTEGER, - "assignmentId" INTEGER NOT NULL, - "userId" TEXT NOT NULL, - "status" TEXT NOT NULL, - "progress" TEXT NOT NULL, - "percentage" INTEGER, - "result" JSONB, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "GradingJob_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "publishJob" ( - "id" SERIAL NOT NULL, - "userId" TEXT NOT NULL, - "assignmentId" INTEGER NOT NULL, - "status" TEXT NOT NULL, - "progress" TEXT, - "percentage" INTEGER, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - "result" JSONB, - - CONSTRAINT "publishJob_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "AIUsage" ( - "id" SERIAL NOT NULL, - "assignmentId" INTEGER NOT NULL, - "usageType" "AIUsageType" NOT NULL, - "tokensIn" INTEGER NOT NULL DEFAULT 0, - "tokensOut" INTEGER NOT NULL DEFAULT 0, - "usageCount" INTEGER NOT NULL DEFAULT 0, - "usageDetails" JSONB, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - "userId" TEXT, - - CONSTRAINT "AIUsage_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "UserCredential" ( - "userId" TEXT NOT NULL, - "githubToken" TEXT, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "UserCredential_pkey" PRIMARY KEY ("userId") -); - --- CreateTable -CREATE TABLE "Chat" ( - "id" TEXT NOT NULL, - "userId" TEXT NOT NULL, - "startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "lastActiveAt" TIMESTAMP(3) NOT NULL, - "title" TEXT, - "assignmentId" INTEGER, - "isActive" BOOLEAN NOT NULL DEFAULT true, - - CONSTRAINT "Chat_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "ChatMessage" ( - "id" SERIAL NOT NULL, - "chatId" TEXT NOT NULL, - "role" "ChatRole" NOT NULL, - "content" TEXT NOT NULL, - "timestamp" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "toolCalls" JSONB, - - CONSTRAINT "ChatMessage_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Group" ( - "id" TEXT NOT NULL -); - --- CreateTable -CREATE TABLE "AssignmentGroup" ( - "assignmentId" INTEGER NOT NULL, - "groupId" TEXT NOT NULL, - - CONSTRAINT "AssignmentGroup_pkey" PRIMARY KEY ("assignmentId","groupId") -); - --- CreateTable -CREATE TABLE "Assignment" ( - "id" SERIAL NOT NULL, - "name" TEXT NOT NULL, - "introduction" TEXT, - "instructions" TEXT, - "gradingCriteriaOverview" TEXT, - "timeEstimateMinutes" INTEGER, - "type" "AssignmentType" NOT NULL, - "graded" BOOLEAN DEFAULT false, - "numAttempts" INTEGER DEFAULT -1, - "allotedTimeMinutes" INTEGER, - "attemptsPerTimeRange" INTEGER, - "attemptsTimeRangeHours" INTEGER, - "passingGrade" INTEGER DEFAULT 50, - "displayOrder" "AssignmentQuestionDisplayOrder", - "questionDisplay" "QuestionDisplay" DEFAULT 'ONE_PER_PAGE', - "numberOfQuestionsPerAttempt" INTEGER, - "questionOrder" INTEGER[], - "published" BOOLEAN NOT NULL, - "showAssignmentScore" BOOLEAN NOT NULL DEFAULT true, - "showQuestionScore" BOOLEAN NOT NULL DEFAULT true, - "showSubmissionFeedback" BOOLEAN NOT NULL DEFAULT true, - "showQuestions" BOOLEAN NOT NULL DEFAULT true, - "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "languageCode" TEXT, - - CONSTRAINT "Assignment_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Question" ( - "id" SERIAL NOT NULL, - "totalPoints" INTEGER NOT NULL, - "type" "QuestionType" NOT NULL, - "responseType" "ResponseType", - "question" TEXT NOT NULL, - "maxWords" INTEGER, - "scoring" JSONB, - "choices" JSONB, - "randomizedChoices" BOOLEAN, - "answer" BOOLEAN, - "assignmentId" INTEGER NOT NULL, - "gradingContextQuestionIds" INTEGER[], - "maxCharacters" INTEGER, - "isDeleted" BOOLEAN NOT NULL DEFAULT false, - "videoPresentationConfig" JSONB, - "liveRecordingConfig" JSONB, - - CONSTRAINT "Question_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "AssignmentAttempt" ( - "id" SERIAL NOT NULL, - "assignmentId" INTEGER NOT NULL, - "userId" TEXT NOT NULL, - "submitted" BOOLEAN NOT NULL, - "grade" DOUBLE PRECISION, - "expiresAt" TIMESTAMP(3), - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "questionOrder" INTEGER[], - "comments" TEXT, - "preferredLanguage" TEXT, - - CONSTRAINT "AssignmentAttempt_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "AssignmentAttemptQuestionVariant" ( - "assignmentAttemptId" INTEGER NOT NULL, - "questionId" INTEGER NOT NULL, - "questionVariantId" INTEGER, - "randomizedChoices" JSONB, - - CONSTRAINT "AssignmentAttemptQuestionVariant_pkey" PRIMARY KEY ("assignmentAttemptId","questionId") -); - --- CreateTable -CREATE TABLE "QuestionVariant" ( - "id" SERIAL NOT NULL, - "questionId" INTEGER NOT NULL, - "variantContent" TEXT NOT NULL, - "choices" JSONB, - "maxWords" INTEGER, - "scoring" JSONB, - "answer" BOOLEAN, - "maxCharacters" INTEGER, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "difficultyLevel" INTEGER, - "variantType" "VariantType" NOT NULL, - "randomizedChoices" BOOLEAN, - "isDeleted" BOOLEAN NOT NULL DEFAULT false, - - CONSTRAINT "QuestionVariant_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "QuestionResponse" ( - "id" SERIAL NOT NULL, - "assignmentAttemptId" INTEGER NOT NULL, - "questionId" INTEGER NOT NULL, - "learnerResponse" TEXT NOT NULL, - "points" DOUBLE PRECISION NOT NULL, - "feedback" JSONB NOT NULL, - "metadata" JSONB, - "gradedAt" TIMESTAMP(3), - - CONSTRAINT "QuestionResponse_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "AssignmentFeedback" ( - "id" SERIAL NOT NULL, - "assignmentId" INTEGER NOT NULL, - "attemptId" INTEGER NOT NULL, - "userId" TEXT NOT NULL, - "comments" TEXT, - "aiGradingRating" INTEGER, - "assignmentRating" INTEGER, - "aiFeedbackRating" INTEGER, - "allowContact" BOOLEAN NOT NULL DEFAULT false, - "firstName" TEXT, - "lastName" TEXT, - "email" TEXT, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "AssignmentFeedback_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "RegradingRequest" ( - "id" SERIAL NOT NULL, - "assignmentId" INTEGER NOT NULL, - "userId" TEXT NOT NULL, - "attemptId" INTEGER NOT NULL, - "regradingReason" TEXT, - "regradingStatus" "RegradingStatus" NOT NULL DEFAULT 'PENDING', - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "RegradingRequest_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Report" ( - "id" SERIAL NOT NULL, - "reporterId" TEXT NOT NULL, - "assignmentId" INTEGER, - "attemptId" INTEGER, - "issueType" "ReportType" NOT NULL, - "description" TEXT NOT NULL, - "author" BOOLEAN NOT NULL DEFAULT false, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - "status" "ReportStatus" NOT NULL DEFAULT 'OPEN', - "issueNumber" INTEGER, - "statusMessage" TEXT, - "resolution" TEXT, - "comments" TEXT, - "closureReason" TEXT, - "duplicateOfReportId" INTEGER, - "relatedToReportId" INTEGER, - "similarityScore" DOUBLE PRECISION, - - CONSTRAINT "Report_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "UserNotification" ( - "id" SERIAL NOT NULL, - "userId" TEXT NOT NULL, - "type" TEXT NOT NULL, - "title" TEXT NOT NULL, - "message" TEXT NOT NULL, - "metadata" TEXT, - "read" BOOLEAN NOT NULL DEFAULT false, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "UserNotification_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Translation" ( - "id" SERIAL NOT NULL, - "questionId" INTEGER, - "variantId" INTEGER, - "languageCode" TEXT NOT NULL, - "translatedText" TEXT NOT NULL, - "untranslatedText" TEXT, - "translatedChoices" JSONB, - "untranslatedChoices" JSONB, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "Translation_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "AssignmentTranslation" ( - "id" SERIAL NOT NULL, - "assignmentId" INTEGER NOT NULL, - "languageCode" TEXT NOT NULL, - "name" TEXT NOT NULL, - "introduction" TEXT NOT NULL, - "instructions" TEXT, - "gradingCriteriaOverview" TEXT, - "translatedName" TEXT, - "translatedIntroduction" TEXT, - "translatedInstructions" TEXT, - "translatedGradingCriteriaOverview" TEXT, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "AssignmentTranslation_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "FeedbackTranslation" ( - "id" SERIAL NOT NULL, - "questionId" INTEGER NOT NULL, - "languageCode" TEXT NOT NULL, - "untranslatedFeedback" JSONB NOT NULL, - "translatedFeedback" JSONB NOT NULL, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "FeedbackTranslation_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "_ChatToUserCredential" ( - "A" TEXT NOT NULL, - "B" TEXT NOT NULL -); - --- CreateIndex -CREATE INDEX "GradingAudit_questionId_idx" ON "GradingAudit"("questionId"); - --- CreateIndex -CREATE INDEX "GradingAudit_assignmentId_idx" ON "GradingAudit"("assignmentId"); - --- CreateIndex -CREATE INDEX "GradingAudit_timestamp_idx" ON "GradingAudit"("timestamp"); - --- CreateIndex -CREATE INDEX "GradingJob_attemptId_idx" ON "GradingJob"("attemptId"); - --- CreateIndex -CREATE INDEX "GradingJob_assignmentId_idx" ON "GradingJob"("assignmentId"); - --- CreateIndex -CREATE INDEX "GradingJob_userId_idx" ON "GradingJob"("userId"); - --- CreateIndex -CREATE INDEX "GradingJob_status_idx" ON "GradingJob"("status"); - --- CreateIndex -CREATE UNIQUE INDEX "AIUsage_assignmentId_usageType_key" ON "AIUsage"("assignmentId", "usageType"); - --- CreateIndex -CREATE INDEX "Chat_userId_startedAt_idx" ON "Chat"("userId", "startedAt"); - --- CreateIndex -CREATE INDEX "Chat_assignmentId_idx" ON "Chat"("assignmentId"); - --- CreateIndex -CREATE INDEX "ChatMessage_chatId_timestamp_idx" ON "ChatMessage"("chatId", "timestamp"); - --- CreateIndex -CREATE UNIQUE INDEX "Group_id_key" ON "Group"("id"); - --- CreateIndex -CREATE UNIQUE INDEX "QuestionVariant_questionId_variantContent_key" ON "QuestionVariant"("questionId", "variantContent"); - --- CreateIndex -CREATE INDEX "Translation_questionId_variantId_languageCode_idx" ON "Translation"("questionId", "variantId", "languageCode"); - --- CreateIndex -CREATE UNIQUE INDEX "AssignmentTranslation_assignmentId_languageCode_key" ON "AssignmentTranslation"("assignmentId", "languageCode"); - --- CreateIndex -CREATE UNIQUE INDEX "FeedbackTranslation_questionId_languageCode_key" ON "FeedbackTranslation"("questionId", "languageCode"); - --- CreateIndex -CREATE UNIQUE INDEX "_ChatToUserCredential_AB_unique" ON "_ChatToUserCredential"("A", "B"); - --- CreateIndex -CREATE INDEX "_ChatToUserCredential_B_index" ON "_ChatToUserCredential"("B"); - --- AddForeignKey -ALTER TABLE "GradingAudit" ADD CONSTRAINT "GradingAudit_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "GradingAudit" ADD CONSTRAINT "GradingAudit_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "GradingJob" ADD CONSTRAINT "GradingJob_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "GradingJob" ADD CONSTRAINT "GradingJob_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AIUsage" ADD CONSTRAINT "AIUsage_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AIUsage" ADD CONSTRAINT "AIUsage_userId_fkey" FOREIGN KEY ("userId") REFERENCES "UserCredential"("userId") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Chat" ADD CONSTRAINT "Chat_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "ChatMessage" ADD CONSTRAINT "ChatMessage_chatId_fkey" FOREIGN KEY ("chatId") REFERENCES "Chat"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AssignmentGroup" ADD CONSTRAINT "AssignmentGroup_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AssignmentGroup" ADD CONSTRAINT "AssignmentGroup_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "Group"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Question" ADD CONSTRAINT "Question_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AssignmentAttemptQuestionVariant" ADD CONSTRAINT "AssignmentAttemptQuestionVariant_assignmentAttemptId_fkey" FOREIGN KEY ("assignmentAttemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AssignmentAttemptQuestionVariant" ADD CONSTRAINT "AssignmentAttemptQuestionVariant_questionVariantId_fkey" FOREIGN KEY ("questionVariantId") REFERENCES "QuestionVariant"("id") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AssignmentAttemptQuestionVariant" ADD CONSTRAINT "AssignmentAttemptQuestionVariant_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "QuestionVariant" ADD CONSTRAINT "QuestionVariant_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "QuestionResponse" ADD CONSTRAINT "QuestionResponse_assignmentAttemptId_fkey" FOREIGN KEY ("assignmentAttemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AssignmentFeedback" ADD CONSTRAINT "AssignmentFeedback_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AssignmentFeedback" ADD CONSTRAINT "AssignmentFeedback_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "RegradingRequest" ADD CONSTRAINT "RegradingRequest_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "RegradingRequest" ADD CONSTRAINT "RegradingRequest_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "AssignmentAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Report" ADD CONSTRAINT "Report_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Report" ADD CONSTRAINT "Report_duplicateOfReportId_fkey" FOREIGN KEY ("duplicateOfReportId") REFERENCES "Report"("id") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Report" ADD CONSTRAINT "Report_relatedToReportId_fkey" FOREIGN KEY ("relatedToReportId") REFERENCES "Report"("id") ON DELETE SET NULL ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Translation" ADD CONSTRAINT "Translation_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Translation" ADD CONSTRAINT "Translation_variantId_fkey" FOREIGN KEY ("variantId") REFERENCES "QuestionVariant"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "AssignmentTranslation" ADD CONSTRAINT "AssignmentTranslation_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "FeedbackTranslation" ADD CONSTRAINT "FeedbackTranslation_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "_ChatToUserCredential" ADD CONSTRAINT "_ChatToUserCredential_A_fkey" FOREIGN KEY ("A") REFERENCES "Chat"("id") ON DELETE CASCADE ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "_ChatToUserCredential" ADD CONSTRAINT "_ChatToUserCredential_B_fkey" FOREIGN KEY ("B") REFERENCES "UserCredential"("userId") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250728232416_admin_verification_system/migration.sql b/apps/api/prisma/migrations/20250728232416_admin_verification_system/migration.sql new file mode 100644 index 00000000..c2666548 --- /dev/null +++ b/apps/api/prisma/migrations/20250728232416_admin_verification_system/migration.sql @@ -0,0 +1,43 @@ +-- CreateTable +CREATE TABLE "AdminVerificationCode" ( + "id" SERIAL NOT NULL, + "email" TEXT NOT NULL, + "code" TEXT NOT NULL, + "expiresAt" TIMESTAMP(3) NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "used" BOOLEAN NOT NULL DEFAULT false, + + CONSTRAINT "AdminVerificationCode_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "AdminSession" ( + "id" SERIAL NOT NULL, + "email" TEXT NOT NULL, + "sessionToken" TEXT NOT NULL, + "expiresAt" TIMESTAMP(3) NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "AdminSession_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "AdminVerificationCode_email_idx" ON "AdminVerificationCode"("email"); + +-- CreateIndex +CREATE INDEX "AdminVerificationCode_code_idx" ON "AdminVerificationCode"("code"); + +-- CreateIndex +CREATE INDEX "AdminVerificationCode_expiresAt_idx" ON "AdminVerificationCode"("expiresAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "AdminSession_sessionToken_key" ON "AdminSession"("sessionToken"); + +-- CreateIndex +CREATE INDEX "AdminSession_email_idx" ON "AdminSession"("email"); + +-- CreateIndex +CREATE INDEX "AdminSession_sessionToken_idx" ON "AdminSession"("sessionToken"); + +-- CreateIndex +CREATE INDEX "AdminSession_expiresAt_idx" ON "AdminSession"("expiresAt"); diff --git a/apps/api/prisma/migrations/20250731205205_add_author_assignment_table/migration.sql b/apps/api/prisma/migrations/20250731205205_add_author_assignment_table/migration.sql new file mode 100644 index 00000000..f4441623 --- /dev/null +++ b/apps/api/prisma/migrations/20250731205205_add_author_assignment_table/migration.sql @@ -0,0 +1,21 @@ +-- CreateTable +CREATE TABLE "AssignmentAuthor" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "userId" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "AssignmentAuthor_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "AssignmentAuthor_assignmentId_idx" ON "AssignmentAuthor"("assignmentId"); + +-- CreateIndex +CREATE INDEX "AssignmentAuthor_userId_idx" ON "AssignmentAuthor"("userId"); + +-- CreateIndex +CREATE UNIQUE INDEX "AssignmentAuthor_assignmentId_userId_key" ON "AssignmentAuthor"("assignmentId", "userId"); + +-- AddForeignKey +ALTER TABLE "AssignmentAuthor" ADD CONSTRAINT "AssignmentAuthor_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250806171750_add_llm_to_feature_and_pricing/migration.sql b/apps/api/prisma/migrations/20250806171750_add_llm_to_feature_and_pricing/migration.sql new file mode 100644 index 00000000..8448fbe0 --- /dev/null +++ b/apps/api/prisma/migrations/20250806171750_add_llm_to_feature_and_pricing/migration.sql @@ -0,0 +1,195 @@ +-- CreateEnum +CREATE TYPE "PricingSource" AS ENUM ('OPENAI_API', 'MANUAL', 'WEB_SCRAPING'); + +-- CreateEnum +CREATE TYPE "AIFeatureType" AS ENUM ('TEXT_GRADING', 'FILE_GRADING', 'IMAGE_GRADING', 'URL_GRADING', 'PRESENTATION_GRADING', 'VIDEO_GRADING', 'QUESTION_GENERATION', 'TRANSLATION', 'RUBRIC_GENERATION', 'CONTENT_MODERATION', 'ASSIGNMENT_GENERATION', 'LIVE_RECORDING_FEEDBACK'); + +-- CreateTable +CREATE TABLE "LLMModel" ( + "id" SERIAL NOT NULL, + "modelKey" TEXT NOT NULL, + "displayName" TEXT NOT NULL, + "provider" TEXT NOT NULL, + "isActive" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LLMModel_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "LLMPricing" ( + "id" SERIAL NOT NULL, + "modelId" INTEGER NOT NULL, + "inputTokenPrice" DOUBLE PRECISION NOT NULL, + "outputTokenPrice" DOUBLE PRECISION NOT NULL, + "effectiveDate" TIMESTAMP(3) NOT NULL, + "source" "PricingSource" NOT NULL, + "isActive" BOOLEAN NOT NULL DEFAULT true, + "metadata" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LLMPricing_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "AIFeature" ( + "id" SERIAL NOT NULL, + "featureKey" TEXT NOT NULL, + "featureType" "AIFeatureType" NOT NULL, + "displayName" TEXT NOT NULL, + "description" TEXT, + "isActive" BOOLEAN NOT NULL DEFAULT true, + "requiresModel" BOOLEAN NOT NULL DEFAULT true, + "defaultModelKey" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "AIFeature_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "LLMFeatureAssignment" ( + "id" SERIAL NOT NULL, + "featureId" INTEGER NOT NULL, + "modelId" INTEGER NOT NULL, + "isActive" BOOLEAN NOT NULL DEFAULT true, + "priority" INTEGER NOT NULL DEFAULT 0, + "assignedBy" TEXT, + "assignedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deactivatedAt" TIMESTAMP(3), + "metadata" JSONB, + + CONSTRAINT "LLMFeatureAssignment_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LLMModel_modelKey_key" ON "LLMModel"("modelKey"); + +-- CreateIndex +CREATE INDEX "LLMModel_modelKey_idx" ON "LLMModel"("modelKey"); + +-- CreateIndex +CREATE INDEX "LLMModel_provider_idx" ON "LLMModel"("provider"); + +-- CreateIndex +CREATE INDEX "LLMPricing_modelId_effectiveDate_idx" ON "LLMPricing"("modelId", "effectiveDate"); + +-- CreateIndex +CREATE INDEX "LLMPricing_effectiveDate_idx" ON "LLMPricing"("effectiveDate"); + +-- CreateIndex +CREATE INDEX "LLMPricing_isActive_idx" ON "LLMPricing"("isActive"); + +-- CreateIndex +CREATE UNIQUE INDEX "LLMPricing_modelId_effectiveDate_source_key" ON "LLMPricing"("modelId", "effectiveDate", "source"); + +-- CreateIndex +CREATE UNIQUE INDEX "AIFeature_featureKey_key" ON "AIFeature"("featureKey"); + +-- CreateIndex +CREATE INDEX "AIFeature_featureKey_idx" ON "AIFeature"("featureKey"); + +-- CreateIndex +CREATE INDEX "AIFeature_featureType_idx" ON "AIFeature"("featureType"); + +-- CreateIndex +CREATE INDEX "AIFeature_isActive_idx" ON "AIFeature"("isActive"); + +-- CreateIndex +CREATE INDEX "LLMFeatureAssignment_featureId_isActive_idx" ON "LLMFeatureAssignment"("featureId", "isActive"); + +-- CreateIndex +CREATE INDEX "LLMFeatureAssignment_modelId_idx" ON "LLMFeatureAssignment"("modelId"); + +-- CreateIndex +CREATE INDEX "LLMFeatureAssignment_isActive_idx" ON "LLMFeatureAssignment"("isActive"); + +-- CreateIndex +CREATE UNIQUE INDEX "LLMFeatureAssignment_featureId_modelId_isActive_key" ON "LLMFeatureAssignment"("featureId", "modelId", "isActive"); + +-- CreateIndex +CREATE INDEX "AssignmentAttempt_assignmentId_idx" ON "AssignmentAttempt"("assignmentId"); + +-- CreateIndex +CREATE INDEX "AssignmentAttempt_userId_idx" ON "AssignmentAttempt"("userId"); + +-- CreateIndex +CREATE INDEX "AssignmentAttempt_assignmentId_submitted_idx" ON "AssignmentAttempt"("assignmentId", "submitted"); + +-- CreateIndex +CREATE INDEX "AssignmentAttempt_createdAt_idx" ON "AssignmentAttempt"("createdAt"); + +-- CreateIndex +CREATE INDEX "AssignmentAttempt_assignmentId_userId_idx" ON "AssignmentAttempt"("assignmentId", "userId"); + +-- CreateIndex +CREATE INDEX "AssignmentFeedback_assignmentId_idx" ON "AssignmentFeedback"("assignmentId"); + +-- CreateIndex +CREATE INDEX "AssignmentFeedback_assignmentId_assignmentRating_idx" ON "AssignmentFeedback"("assignmentId", "assignmentRating"); + +-- AddForeignKey +ALTER TABLE "LLMPricing" ADD CONSTRAINT "LLMPricing_modelId_fkey" FOREIGN KEY ("modelId") REFERENCES "LLMModel"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "LLMFeatureAssignment" ADD CONSTRAINT "LLMFeatureAssignment_featureId_fkey" FOREIGN KEY ("featureId") REFERENCES "AIFeature"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "LLMFeatureAssignment" ADD CONSTRAINT "LLMFeatureAssignment_modelId_fkey" FOREIGN KEY ("modelId") REFERENCES "LLMModel"("id") ON DELETE CASCADE ON UPDATE CASCADE; + + +-- Initialize LLMModels table with default models from the system +-- This adds the three OpenAI models that are currently configured in the LLM router + +INSERT INTO "LLMModel" ("modelKey", "displayName", "provider", "isActive", "createdAt", "updatedAt") VALUES +('gpt-4o', 'GPT-4 Omni', 'OpenAI', true, NOW(), NOW()), +('gpt-4o-mini', 'GPT-4 Omni Mini', 'OpenAI', true, NOW(), NOW()), +('gpt-4.1-mini', 'GPT-4.1 Mini (Vision)', 'OpenAI', true, NOW(), NOW()) +ON CONFLICT ("modelKey") DO NOTHING; + +-- Initialize default AIFeature records for the system's AI features +INSERT INTO "AIFeature" ("featureKey", "featureType", "displayName", "description", "isActive", "requiresModel", "defaultModelKey", "createdAt", "updatedAt") VALUES +('text_grading', 'TEXT_GRADING', 'Text Response Grading', 'Automated grading of text-based responses using AI', true, true, 'gpt-4o-mini', NOW(), NOW()), +('file_grading', 'FILE_GRADING', 'File Content Grading', 'Automated grading of file uploads and documents', true, true, 'gpt-4o', NOW(), NOW()), +('image_grading', 'IMAGE_GRADING', 'Image Content Grading', 'Automated grading of image submissions', true, true, 'gpt-4.1-mini', NOW(), NOW()), +('url_grading', 'URL_GRADING', 'URL Content Grading', 'Automated grading of URL submissions', true, true, 'gpt-4o', NOW(), NOW()), +('presentation_grading', 'PRESENTATION_GRADING', 'Presentation Grading', 'Automated grading of presentation files', true, true, 'gpt-4.1-mini', NOW(), NOW()), +('video_grading', 'VIDEO_GRADING', 'Video Grading', 'Automated grading of video presentations', true, true, 'gpt-4.1-mini', NOW(), NOW()), +('question_generation', 'QUESTION_GENERATION', 'Question Generation', 'AI-powered question generation for assignments', true, true, 'gpt-4o', NOW(), NOW()), +('translation', 'TRANSLATION', 'Content Translation', 'AI-powered translation of assignment content', true, true, 'gpt-4o-mini', NOW(), NOW()), +('rubric_generation', 'RUBRIC_GENERATION', 'Rubric Generation', 'AI-powered rubric creation for assignments', true, true, 'gpt-4o', NOW(), NOW()), +('content_moderation', 'CONTENT_MODERATION', 'Content Moderation', 'AI-powered content moderation and safety checks', true, true, 'gpt-4o-mini', NOW(), NOW()), +('assignment_generation', 'ASSIGNMENT_GENERATION', 'Assignment Generation', 'AI-powered assignment creation', true, true, 'gpt-4o', NOW(), NOW()), +('live_recording_feedback', 'LIVE_RECORDING_FEEDBACK', 'Live Recording Feedback', 'AI feedback for live recordings', true, true, 'gpt-4o-mini', NOW(), NOW()) +ON CONFLICT ("featureKey") DO NOTHING; + +-- Initialize pricing data for all models with historical data going back 1 year +-- This ensures there's always pricing available for any historical date +WITH model_ids AS ( + SELECT id, "modelKey" FROM "LLMModel" WHERE "modelKey" IN ('gpt-4o', 'gpt-4o-mini', 'gpt-4.1-mini') +) +INSERT INTO "LLMPricing" ("modelId", "inputTokenPrice", "outputTokenPrice", "effectiveDate", "source", "isActive", "metadata", "createdAt", "updatedAt") +SELECT + m.id, + CASE + WHEN m."modelKey" = 'gpt-4o' THEN 0.0000025 + WHEN m."modelKey" = 'gpt-4o-mini' THEN 0.00000015 + WHEN m."modelKey" = 'gpt-4.1-mini' THEN 0.0000025 + END as inputTokenPrice, + CASE + WHEN m."modelKey" = 'gpt-4o' THEN 0.00001 + WHEN m."modelKey" = 'gpt-4o-mini' THEN 0.0000006 + WHEN m."modelKey" = 'gpt-4.1-mini' THEN 0.00001 + END as outputTokenPrice, + '2024-01-01 00:00:00'::timestamp as effectiveDate, + 'MANUAL'::"PricingSource" as source, + true as isActive, + jsonb_build_object( + 'note', 'Historical baseline pricing', + 'lastUpdated', NOW()::text + ) as metadata, + NOW() as createdAt, + NOW() as updatedAt +FROM model_ids m; \ No newline at end of file diff --git a/apps/api/prisma/migrations/20250806173928_add_model_key_to_ai_usage/migration.sql b/apps/api/prisma/migrations/20250806173928_add_model_key_to_ai_usage/migration.sql new file mode 100644 index 00000000..ead75e4a --- /dev/null +++ b/apps/api/prisma/migrations/20250806173928_add_model_key_to_ai_usage/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "AIUsage" ADD COLUMN "modelKey" TEXT; diff --git a/apps/api/prisma/migrations/20250806174307_populate_default_llm_assignments/migration.sql b/apps/api/prisma/migrations/20250806174307_populate_default_llm_assignments/migration.sql new file mode 100644 index 00000000..7b291f0a --- /dev/null +++ b/apps/api/prisma/migrations/20250806174307_populate_default_llm_assignments/migration.sql @@ -0,0 +1,23 @@ +-- Populate LLMFeatureAssignment table with default model assignments +-- Default assignments: gpt-4o for everything except translation (gpt-4o-mini) and images (gpt-4.1-mini) + +INSERT INTO "LLMFeatureAssignment" ("featureId", "modelId", "isActive", "priority", "assignedBy", "assignedAt", "metadata") +SELECT + f.id as featureId, + m.id as modelId, + true as isActive, + 1 as priority, + 'system' as assignedBy, + NOW() as assignedAt, + jsonb_build_object( + 'assignmentType', 'default', + 'reason', 'Initial system assignment based on feature requirements' + ) as metadata +FROM "AIFeature" f +CROSS JOIN "LLMModel" m +WHERE + -- Default assignments based on feature type + (f."featureType" = 'TRANSLATION' AND m."modelKey" = 'gpt-4o-mini') OR + (f."featureType" = 'IMAGE_GRADING' AND m."modelKey" = 'gpt-4.1-mini') OR + (f."featureType" NOT IN ('TRANSLATION', 'IMAGE_GRADING') AND m."modelKey" = 'gpt-4o') +ON CONFLICT ("featureId", "modelId", "isActive") DO NOTHING; \ No newline at end of file diff --git a/apps/api/prisma/migrations/20250806174359_fix_ai_feature_default_models/migration.sql b/apps/api/prisma/migrations/20250806174359_fix_ai_feature_default_models/migration.sql new file mode 100644 index 00000000..77d69634 --- /dev/null +++ b/apps/api/prisma/migrations/20250806174359_fix_ai_feature_default_models/migration.sql @@ -0,0 +1,20 @@ +-- Fix default model assignments in AIFeature table +-- Update to match the requirement: gpt-4o for everything except translation (gpt-4o-mini) and images (gpt-4.1-mini) + +UPDATE "AIFeature" SET + "defaultModelKey" = 'gpt-4o', + "updatedAt" = NOW() +WHERE "featureType" NOT IN ('TRANSLATION', 'IMAGE_GRADING') + AND "defaultModelKey" != 'gpt-4o'; + +UPDATE "AIFeature" SET + "defaultModelKey" = 'gpt-4o-mini', + "updatedAt" = NOW() +WHERE "featureType" = 'TRANSLATION' + AND "defaultModelKey" != 'gpt-4o-mini'; + +UPDATE "AIFeature" SET + "defaultModelKey" = 'gpt-4.1-mini', + "updatedAt" = NOW() +WHERE "featureType" = 'IMAGE_GRADING' + AND "defaultModelKey" != 'gpt-4.1-mini'; \ No newline at end of file diff --git a/apps/api/prisma/migrations/20250806183000_fix_missing_model_keys/migration.sql b/apps/api/prisma/migrations/20250806183000_fix_missing_model_keys/migration.sql new file mode 100644 index 00000000..15facdae --- /dev/null +++ b/apps/api/prisma/migrations/20250806183000_fix_missing_model_keys/migration.sql @@ -0,0 +1,32 @@ +-- Fix missing model keys in AIUsage table based on usage type patterns +-- This addresses the warnings about missing model keys in the cost calculation + +-- Update records with translation usage type to use gpt-4o-mini +UPDATE "AIUsage" +SET "modelKey" = 'gpt-4o-mini', "updatedAt" = NOW() +WHERE "modelKey" IS NULL + AND "usageType" = 'TRANSLATION'; + +-- Update records with live recording feedback to use gpt-4o-mini +UPDATE "AIUsage" +SET "modelKey" = 'gpt-4o-mini', "updatedAt" = NOW() +WHERE "modelKey" IS NULL + AND "usageType" = 'LIVE_RECORDING_FEEDBACK'; + +-- Update records with grading usage type to use gpt-4o +UPDATE "AIUsage" +SET "modelKey" = 'gpt-4o', "updatedAt" = NOW() +WHERE "modelKey" IS NULL + AND "usageType" = 'ASSIGNMENT_GRADING'; + +-- Update records with generation usage types to use gpt-4o +UPDATE "AIUsage" +SET "modelKey" = 'gpt-4o', "updatedAt" = NOW() +WHERE "modelKey" IS NULL + AND ("usageType" = 'QUESTION_GENERATION' + OR "usageType" = 'ASSIGNMENT_GENERATION'); + +-- Update any remaining records without model keys to use gpt-4o-mini as default +UPDATE "AIUsage" +SET "modelKey" = 'gpt-4o-mini', "updatedAt" = NOW() +WHERE "modelKey" IS NULL; \ No newline at end of file diff --git a/apps/api/prisma/migrations/20250808182841_new_version_control/migration.sql b/apps/api/prisma/migrations/20250808182841_new_version_control/migration.sql new file mode 100644 index 00000000..7211effd --- /dev/null +++ b/apps/api/prisma/migrations/20250808182841_new_version_control/migration.sql @@ -0,0 +1,122 @@ +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "currentVersionId" INTEGER; + +-- CreateTable +CREATE TABLE "AssignmentVersion" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "versionNumber" INTEGER NOT NULL, + "name" TEXT NOT NULL, + "introduction" TEXT, + "instructions" TEXT, + "gradingCriteriaOverview" TEXT, + "timeEstimateMinutes" INTEGER, + "type" "AssignmentType" NOT NULL, + "graded" BOOLEAN DEFAULT false, + "numAttempts" INTEGER DEFAULT -1, + "allotedTimeMinutes" INTEGER, + "attemptsPerTimeRange" INTEGER, + "attemptsTimeRangeHours" INTEGER, + "passingGrade" INTEGER DEFAULT 50, + "displayOrder" "AssignmentQuestionDisplayOrder", + "questionDisplay" "QuestionDisplay" DEFAULT 'ONE_PER_PAGE', + "numberOfQuestionsPerAttempt" INTEGER, + "questionOrder" INTEGER[], + "published" BOOLEAN NOT NULL DEFAULT false, + "showAssignmentScore" BOOLEAN NOT NULL DEFAULT true, + "showQuestionScore" BOOLEAN NOT NULL DEFAULT true, + "showSubmissionFeedback" BOOLEAN NOT NULL DEFAULT true, + "showQuestions" BOOLEAN NOT NULL DEFAULT true, + "languageCode" TEXT, + "createdBy" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "isDraft" BOOLEAN NOT NULL DEFAULT true, + "versionDescription" TEXT, + "isActive" BOOLEAN NOT NULL DEFAULT false, + + CONSTRAINT "AssignmentVersion_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "QuestionVersion" ( + "id" SERIAL NOT NULL, + "assignmentVersionId" INTEGER NOT NULL, + "questionId" INTEGER, + "totalPoints" INTEGER NOT NULL, + "type" "QuestionType" NOT NULL, + "responseType" "ResponseType", + "question" TEXT NOT NULL, + "maxWords" INTEGER, + "scoring" JSONB, + "choices" JSONB, + "randomizedChoices" BOOLEAN, + "answer" BOOLEAN, + "gradingContextQuestionIds" INTEGER[], + "maxCharacters" INTEGER, + "videoPresentationConfig" JSONB, + "liveRecordingConfig" JSONB, + "displayOrder" INTEGER, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "QuestionVersion_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "VersionHistory" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "fromVersionId" INTEGER, + "toVersionId" INTEGER NOT NULL, + "action" TEXT NOT NULL, + "description" TEXT, + "userId" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "metadata" JSONB, + + CONSTRAINT "VersionHistory_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "AssignmentVersion_assignmentId_idx" ON "AssignmentVersion"("assignmentId"); + +-- CreateIndex +CREATE INDEX "AssignmentVersion_isActive_idx" ON "AssignmentVersion"("isActive"); + +-- CreateIndex +CREATE INDEX "AssignmentVersion_isDraft_idx" ON "AssignmentVersion"("isDraft"); + +-- CreateIndex +CREATE UNIQUE INDEX "AssignmentVersion_assignmentId_versionNumber_key" ON "AssignmentVersion"("assignmentId", "versionNumber"); + +-- CreateIndex +CREATE INDEX "QuestionVersion_assignmentVersionId_idx" ON "QuestionVersion"("assignmentVersionId"); + +-- CreateIndex +CREATE INDEX "QuestionVersion_questionId_idx" ON "QuestionVersion"("questionId"); + +-- CreateIndex +CREATE INDEX "VersionHistory_assignmentId_idx" ON "VersionHistory"("assignmentId"); + +-- CreateIndex +CREATE INDEX "VersionHistory_createdAt_idx" ON "VersionHistory"("createdAt"); + +-- CreateIndex +CREATE INDEX "VersionHistory_userId_idx" ON "VersionHistory"("userId"); + +-- AddForeignKey +ALTER TABLE "Assignment" ADD CONSTRAINT "Assignment_currentVersionId_fkey" FOREIGN KEY ("currentVersionId") REFERENCES "AssignmentVersion"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "AssignmentVersion" ADD CONSTRAINT "AssignmentVersion_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "QuestionVersion" ADD CONSTRAINT "QuestionVersion_assignmentVersionId_fkey" FOREIGN KEY ("assignmentVersionId") REFERENCES "AssignmentVersion"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "VersionHistory" ADD CONSTRAINT "VersionHistory_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "VersionHistory" ADD CONSTRAINT "VersionHistory_fromVersionId_fkey" FOREIGN KEY ("fromVersionId") REFERENCES "AssignmentVersion"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "VersionHistory" ADD CONSTRAINT "VersionHistory_toVersionId_fkey" FOREIGN KEY ("toVersionId") REFERENCES "AssignmentVersion"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250811142045_add_assignment_draft_table/migration.sql b/apps/api/prisma/migrations/20250811142045_add_assignment_draft_table/migration.sql new file mode 100644 index 00000000..a27915e6 --- /dev/null +++ b/apps/api/prisma/migrations/20250811142045_add_assignment_draft_table/migration.sql @@ -0,0 +1,46 @@ +-- CreateTable +CREATE TABLE "AssignmentDraft" ( + "id" SERIAL NOT NULL, + "assignmentId" INTEGER NOT NULL, + "userId" TEXT NOT NULL, + "draftName" TEXT NOT NULL, + "name" TEXT NOT NULL, + "introduction" TEXT, + "instructions" TEXT, + "gradingCriteriaOverview" TEXT, + "timeEstimateMinutes" INTEGER, + "type" "AssignmentType" NOT NULL, + "graded" BOOLEAN DEFAULT false, + "numAttempts" INTEGER DEFAULT -1, + "allotedTimeMinutes" INTEGER, + "attemptsPerTimeRange" INTEGER, + "attemptsTimeRangeHours" INTEGER, + "passingGrade" INTEGER DEFAULT 50, + "displayOrder" "AssignmentQuestionDisplayOrder", + "questionDisplay" "QuestionDisplay" DEFAULT 'ONE_PER_PAGE', + "numberOfQuestionsPerAttempt" INTEGER, + "questionOrder" INTEGER[], + "published" BOOLEAN NOT NULL DEFAULT false, + "showAssignmentScore" BOOLEAN NOT NULL DEFAULT true, + "showQuestionScore" BOOLEAN NOT NULL DEFAULT true, + "showSubmissionFeedback" BOOLEAN NOT NULL DEFAULT true, + "showQuestions" BOOLEAN NOT NULL DEFAULT true, + "languageCode" TEXT, + "questionsData" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "AssignmentDraft_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "AssignmentDraft_assignmentId_idx" ON "AssignmentDraft"("assignmentId"); + +-- CreateIndex +CREATE INDEX "AssignmentDraft_userId_idx" ON "AssignmentDraft"("userId"); + +-- CreateIndex +CREATE INDEX "AssignmentDraft_createdAt_idx" ON "AssignmentDraft"("createdAt"); + +-- AddForeignKey +ALTER TABLE "AssignmentDraft" ADD CONSTRAINT "AssignmentDraft_assignmentId_fkey" FOREIGN KEY ("assignmentId") REFERENCES "Assignment"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250812020045_adding_versioning_migration/migration.sql b/apps/api/prisma/migrations/20250812020045_adding_versioning_migration/migration.sql new file mode 100644 index 00000000..331473f9 --- /dev/null +++ b/apps/api/prisma/migrations/20250812020045_adding_versioning_migration/migration.sql @@ -0,0 +1,202 @@ +-- 2025-08-12: Backfill initial versions (v1) for assignments without versions +-- Mirrors createInitialVersions() behavior with per-assignment error isolation. + +DO $$ +DECLARE + r RECORD; + v_assignment_version_id "AssignmentVersion".id%TYPE; +BEGIN + RAISE NOTICE 'Starting to create initial versions for existing assignments...'; + + -- Loop over assignments that have NO versions + FOR r IN + SELECT + a.id AS "assignmentId", + a.name, + a.introduction, + a.instructions, + a."gradingCriteriaOverview", + a."timeEstimateMinutes", + a.type, + a.graded, + a."numAttempts", + a."allotedTimeMinutes", + a."attemptsPerTimeRange", + a."attemptsTimeRangeHours", + a."passingGrade", + a."displayOrder", + a."questionDisplay", + a."numberOfQuestionsPerAttempt", + a."questionOrder", + a.published, + a."showAssignmentScore", + a."showQuestionScore", + a."showSubmissionFeedback", + a."showQuestions", + a."languageCode", + COALESCE(la."userId", 'system') AS "createdBy" + FROM "Assignment" a + LEFT JOIN "AssignmentVersion" av + ON av."assignmentId" = a.id + -- choose "first" author, mimicking array [0]; deterministic by createdAt, then id + LEFT JOIN LATERAL ( + SELECT aa."userId" + FROM "AssignmentAuthor" aa + WHERE aa."assignmentId" = a.id + ORDER BY aa."createdAt" NULLS LAST, aa.id + LIMIT 1 + ) la ON TRUE + WHERE av.id IS NULL + LOOP + BEGIN + RAISE NOTICE 'Creating version 1 for assignment "%"(ID: %)', r.name, r."assignmentId"; + + -- Create AssignmentVersion v1 (non-draft, active) + INSERT INTO "AssignmentVersion" ( + "assignmentId", + "versionNumber", + name, + introduction, + instructions, + "gradingCriteriaOverview", + "timeEstimateMinutes", + type, + graded, + "numAttempts", + "allotedTimeMinutes", + "attemptsPerTimeRange", + "attemptsTimeRangeHours", + "passingGrade", + "displayOrder", + "questionDisplay", + "numberOfQuestionsPerAttempt", + "questionOrder", + published, + "showAssignmentScore", + "showQuestionScore", + "showSubmissionFeedback", + "showQuestions", + "languageCode", + "createdBy", + "isDraft", + "versionDescription", + "isActive" + ) + VALUES ( + r."assignmentId", + 1, + r.name, + r.introduction, + r.instructions, + r."gradingCriteriaOverview", + r."timeEstimateMinutes", + r.type, + r.graded, + r."numAttempts", + r."allotedTimeMinutes", + r."attemptsPerTimeRange", + r."attemptsTimeRangeHours", + r."passingGrade", + r."displayOrder", + r."questionDisplay", + r."numberOfQuestionsPerAttempt", + r."questionOrder", + r.published, + r."showAssignmentScore", + r."showQuestionScore", + r."showSubmissionFeedback", + r."showQuestions", + r."languageCode", + r."createdBy", + FALSE, + 'Initial version created from existing assignment', + TRUE + ) + RETURNING id INTO v_assignment_version_id; + + -- Create QuestionVersion rows for non-deleted questions + INSERT INTO "QuestionVersion" ( + "assignmentVersionId", + "questionId", + "totalPoints", + type, + "responseType", + question, + "maxWords", + scoring, + choices, + "randomizedChoices", + answer, + "gradingContextQuestionIds", + "maxCharacters", + "videoPresentationConfig", + "liveRecordingConfig", + "displayOrder" + ) + SELECT + v_assignment_version_id, + q.id, + q."totalPoints", + q.type, + q."responseType", + q.question, + q."maxWords", + q.scoring, + q.choices, + q."randomizedChoices", + q.answer, + q."gradingContextQuestionIds", + q."maxCharacters", + q."videoPresentationConfig", + q."liveRecordingConfig", + ROW_NUMBER() OVER ( + ORDER BY q."createdAt", q.id + )::int AS "displayOrder" -- index + 1 equivalent + FROM "Question" q + WHERE q."assignmentId" = r."assignmentId" + AND q."isDeleted" = FALSE; + + -- Update Assignment.currentVersionId + UPDATE "Assignment" + SET "currentVersionId" = v_assignment_version_id + WHERE id = r."assignmentId"; + + -- VersionHistory entry (userId same selection as createdBy) + INSERT INTO "VersionHistory" ( + "assignmentId", + "toVersionId", + action, + description, + "userId" + ) + VALUES ( + r."assignmentId", + v_assignment_version_id, + 'initial_version_created', + 'Initial version created during migration', + r."createdBy" + ); + + RAISE NOTICE '✅ Created version 1 for assignment "%" (ID: %)', r.name, r."assignmentId"; + + EXCEPTION WHEN OTHERS THEN + -- Match script behavior: log error and continue with next assignment + RAISE WARNING '❌ Failed to create version for assignment "%" (ID: %): %', + r.name, r."assignmentId", SQLERRM; + -- Continue to next record + END; + END LOOP; + + -- Optional verification like the script's summary + RAISE NOTICE '📊 Summary:'; + PERFORM 1; -- no-op to keep block structure tidy + RAISE NOTICE '- Total assignment versions in database: %', + (SELECT COUNT(*) FROM "AssignmentVersion"); + RAISE NOTICE '- Assignments with versions: %', + (SELECT COUNT(*) FROM "Assignment" a WHERE EXISTS ( + SELECT 1 FROM "AssignmentVersion" av WHERE av."assignmentId" = a.id + )); + + RAISE NOTICE '✅ Migration block completed'; +END +$$ LANGUAGE plpgsql; diff --git a/apps/api/prisma/migrations/20250812063203_add_assignment_version_to_attempts/migration.sql b/apps/api/prisma/migrations/20250812063203_add_assignment_version_to_attempts/migration.sql new file mode 100644 index 00000000..6912d32f --- /dev/null +++ b/apps/api/prisma/migrations/20250812063203_add_assignment_version_to_attempts/migration.sql @@ -0,0 +1,8 @@ +-- AlterTable +ALTER TABLE "AssignmentAttempt" ADD COLUMN "assignmentVersionId" INTEGER; + +-- CreateIndex +CREATE INDEX "AssignmentAttempt_assignmentVersionId_idx" ON "AssignmentAttempt"("assignmentVersionId"); + +-- AddForeignKey +ALTER TABLE "AssignmentAttempt" ADD CONSTRAINT "AssignmentAttempt_assignmentVersionId_fkey" FOREIGN KEY ("assignmentVersionId") REFERENCES "AssignmentVersion"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20250813103102_adding_show_correct_answer/migration.sql b/apps/api/prisma/migrations/20250813103102_adding_show_correct_answer/migration.sql new file mode 100644 index 00000000..e0a6f65f --- /dev/null +++ b/apps/api/prisma/migrations/20250813103102_adding_show_correct_answer/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Assignment" ADD COLUMN "showCorrectAnswer" BOOLEAN NOT NULL DEFAULT true; diff --git a/apps/api/prisma/migrations/20250814160847_changing_version_to_string/migration.sql b/apps/api/prisma/migrations/20250814160847_changing_version_to_string/migration.sql new file mode 100644 index 00000000..f8044c4a --- /dev/null +++ b/apps/api/prisma/migrations/20250814160847_changing_version_to_string/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "AssignmentVersion" ALTER COLUMN "versionNumber" SET DATA TYPE TEXT; diff --git a/apps/api/prisma/migrations/20250819045429_upscaling_prices/migration.sql b/apps/api/prisma/migrations/20250819045429_upscaling_prices/migration.sql new file mode 100644 index 00000000..2720e753 --- /dev/null +++ b/apps/api/prisma/migrations/20250819045429_upscaling_prices/migration.sql @@ -0,0 +1,21 @@ +-- CreateTable +CREATE TABLE "LLMPriceUpscaling" ( + "id" SERIAL NOT NULL, + "globalFactor" DOUBLE PRECISION, + "usageTypeFactors" JSONB, + "reason" TEXT, + "appliedBy" TEXT, + "isActive" BOOLEAN NOT NULL DEFAULT true, + "effectiveDate" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deactivatedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "LLMPriceUpscaling_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "LLMPriceUpscaling_isActive_effectiveDate_idx" ON "LLMPriceUpscaling"("isActive", "effectiveDate"); + +-- CreateIndex +CREATE INDEX "LLMPriceUpscaling_effectiveDate_idx" ON "LLMPriceUpscaling"("effectiveDate"); diff --git a/apps/api/prisma/migrations/20250825160012_add_gpt5_models/migration.sql b/apps/api/prisma/migrations/20250825160012_add_gpt5_models/migration.sql new file mode 100644 index 00000000..df4f8fb3 --- /dev/null +++ b/apps/api/prisma/migrations/20250825160012_add_gpt5_models/migration.sql @@ -0,0 +1,32 @@ +-- Add GPT-5 model variants to the LLMModel table +-- This migration adds GPT-5, GPT-5-mini, and GPT-5-nano models + +INSERT INTO "LLMModel" ("modelKey", "displayName", "provider", "isActive", "createdAt", "updatedAt") VALUES +('gpt-5', 'GPT-5', 'OpenAI', true, NOW(), NOW()), +('gpt-5-mini', 'GPT-5 Mini', 'OpenAI', true, NOW(), NOW()), +('gpt-5-nano', 'GPT-5 Nano', 'OpenAI', true, NOW(), NOW()); + +-- Add initial pricing data for the new GPT-5 models +-- Note: These are estimated prices, adjust based on actual OpenAI pricing when available +WITH new_models AS ( + SELECT id, "modelKey" FROM "LLMModel" WHERE "modelKey" IN ('gpt-5', 'gpt-5-mini', 'gpt-5-nano') +) +INSERT INTO "LLMPricing" ("modelId", "inputTokenPrice", "outputTokenPrice", "effectiveDate", "source", "isActive", "createdAt", "updatedAt") +SELECT + m.id, + CASE + WHEN m."modelKey" = 'gpt-5' THEN 0.000005 -- Estimated: Higher than GPT-4o + WHEN m."modelKey" = 'gpt-5-mini' THEN 0.0000003 -- Estimated: Lower than GPT-4o-mini + WHEN m."modelKey" = 'gpt-5-nano' THEN 0.0000001 -- Estimated: Very low for nano version + END, + CASE + WHEN m."modelKey" = 'gpt-5' THEN 0.000015 -- Estimated: Higher than GPT-4o + WHEN m."modelKey" = 'gpt-5-mini' THEN 0.0000012 -- Estimated: Lower than GPT-4o-mini + WHEN m."modelKey" = 'gpt-5-nano' THEN 0.0000004 -- Estimated: Very low for nano version + END, + NOW(), + 'MANUAL', + true, + NOW(), + NOW() +FROM new_models m; \ No newline at end of file diff --git a/apps/api/prisma/migrations/20250903045959_fix_llm_assignment_uniqueness/migration.sql b/apps/api/prisma/migrations/20250903045959_fix_llm_assignment_uniqueness/migration.sql new file mode 100644 index 00000000..832fa75c --- /dev/null +++ b/apps/api/prisma/migrations/20250903045959_fix_llm_assignment_uniqueness/migration.sql @@ -0,0 +1,58 @@ +-- Fix LLMFeatureAssignment uniqueness constraint and update default assignments + +-- First, remove any duplicate feature-model pairs (keep only the active one) +DELETE FROM "LLMFeatureAssignment" a +WHERE EXISTS ( + SELECT 1 + FROM "LLMFeatureAssignment" b + WHERE b."featureId" = a."featureId" + AND b."modelId" = a."modelId" + AND b.id < a.id +); + +-- Drop the old constraint if it exists +DROP INDEX IF EXISTS "LLMFeatureAssignment_featureId_modelId_isActive_key"; + +-- Drop the new constraint if it already exists (in case of re-run) +DROP INDEX IF EXISTS "LLMFeatureAssignment_featureId_modelId_key"; + +-- Create the new unique constraint +CREATE UNIQUE INDEX "LLMFeatureAssignment_featureId_modelId_key" ON "LLMFeatureAssignment"("featureId", "modelId"); + +-- Now delete all assignments to start fresh +DELETE FROM "LLMFeatureAssignment"; + +-- All text-based grading features will use gpt-5-mini +-- Note: Grading validation/judge is handled at the service level, not as a separate AIFeature +INSERT INTO "LLMFeatureAssignment" ("featureId", "modelId", "isActive", "priority", "assignedBy", "assignedAt", "metadata") +SELECT + f.id as featureId, + m.id as modelId, + true as isActive, + 100 as priority, + 'system' as assignedBy, + NOW() as assignedAt, + jsonb_build_object( + 'assignmentType', 'default', + 'reason', 'Updated default assignments: gpt-5-mini for text grading' + ) as metadata +FROM "AIFeature" f +CROSS JOIN "LLMModel" m +WHERE + -- Text-based grading features use gpt-5-mini + (f."featureType" IN ('TEXT_GRADING', 'FILE_GRADING', 'URL_GRADING', 'LIVE_RECORDING_FEEDBACK') + AND m."modelKey" = 'gpt-5-mini') OR + + -- Vision-capable grading features still use gpt-4.1-mini (vision model) + (f."featureType" IN ('IMAGE_GRADING', 'PRESENTATION_GRADING', 'VIDEO_GRADING') + AND m."modelKey" = 'gpt-4.1-mini') OR + + -- Translation uses gpt-4o-mini (keep existing default) + (f."featureType" = 'TRANSLATION' AND m."modelKey" = 'gpt-4o-mini') OR + + -- Generation features use gpt-4o (keep existing default) + (f."featureType" IN ('QUESTION_GENERATION', 'RUBRIC_GENERATION', 'ASSIGNMENT_GENERATION') + AND m."modelKey" = 'gpt-4o') OR + + -- Content moderation uses gpt-4o-mini (keep existing default) + (f."featureType" = 'CONTENT_MODERATION' AND m."modelKey" = 'gpt-4o-mini'); \ No newline at end of file diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 84f4f41f..accfcf3c 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -130,6 +130,7 @@ model AIUsage { updatedAt DateTime @updatedAt /// Timestamp for when the usage record was last updated userId String? /// The ID of the user who used the AI feature user UserCredential? @relation(fields: [userId], references: [userId]) /// Relation to the UserCredential model + modelKey String? /// The key of the LLM model actually used for this AI operation @@unique([assignmentId, usageType]) /// Ensure each assignment has a unique usage record for each AI feature } @@ -140,6 +141,7 @@ enum AIUsageType { ASSIGNMENT_GRADING /// AI was used for grading the assignment TRANSLATION /// AI was used for translation LIVE_RECORDING_FEEDBACK /// AI was used for live recording feedback + GRADING_VALIDATION } model UserCredential { @@ -228,8 +230,11 @@ model Assignment { showQuestionScore Boolean @default(true) /// Should the question score be shown to the learner after its submission showSubmissionFeedback Boolean @default(true) /// Should the AI provide feedback when the learner submits a question showQuestions Boolean @default(true) /// Should the questions be shown to the learner + showCorrectAnswer Boolean @default(true) /// Should the correct answer be shown to the learner after its submission updatedAt DateTime @default(now()) @updatedAt /// The DateTime at which the assignment was last updated languageCode String? /// The language code for the assignment + currentVersionId Int? /// The ID of the current active version + currentVersion AssignmentVersion? @relation("CurrentVersion", fields: [currentVersionId], references: [id]) AIUsage AIUsage[] AssignmentFeedback AssignmentFeedback[] RegradingRequest RegradingRequest[] @@ -240,7 +245,103 @@ model Assignment { Chat Chat[] GradingJob GradingJob[] + AssignmentAuthor AssignmentAuthor[] + versions AssignmentVersion[] @relation("AssignmentVersions") + versionHistory VersionHistory[] + drafts AssignmentDraft[] +} + +/// The AssignmentVersion model represents a specific version of an assignment +model AssignmentVersion { + id Int @id @default(autoincrement()) /// Unique identifier for the version + assignmentId Int /// The ID of the parent assignment + assignment Assignment @relation("AssignmentVersions", fields: [assignmentId], references: [id], onDelete: Cascade) + versionNumber String /// Semantic version number (e.g., "1.0.0", "1.0.0-rc1") + name String /// Name of the assignment in this version + introduction String? /// Introduction for this version + instructions String? /// Instructions for this version + gradingCriteriaOverview String? /// Grading criteria for this version + timeEstimateMinutes Int? /// Time estimate for this version + type AssignmentType /// Type of assignment + graded Boolean? @default(false) /// Is the assignment graded + numAttempts Int? @default(-1) /// Max attempts allowed + allotedTimeMinutes Int? /// Time allotted to complete + attemptsPerTimeRange Int? /// Attempts per time range + attemptsTimeRangeHours Int? /// Time range in hours + passingGrade Int? @default(50) /// Minimum passing grade + displayOrder AssignmentQuestionDisplayOrder? /// Question display order + questionDisplay QuestionDisplay? @default(ONE_PER_PAGE) /// Question display mode + numberOfQuestionsPerAttempt Int? /// Questions per attempt + questionOrder Int[] /// Order of questions + published Boolean @default(false) /// Is this version published + showAssignmentScore Boolean @default(true) /// Show assignment score + showQuestionScore Boolean @default(true) /// Show question score + showSubmissionFeedback Boolean @default(true) /// Show submission feedback + showQuestions Boolean @default(true) /// Show questions + languageCode String? /// Language code + createdBy String /// User who created this version + createdAt DateTime @default(now()) /// When version was created + isDraft Boolean @default(true) /// Is this a draft version + versionDescription String? /// Description of changes in this version + isActive Boolean @default(false) /// Is this the active version + questionVersions QuestionVersion[] /// Questions in this version + assignmentUsingAsCurrentVersion Assignment[] @relation("CurrentVersion") + versionHistoryFrom VersionHistory[] @relation("VersionHistoryFrom") + versionHistoryTo VersionHistory[] @relation("VersionHistoryTo") + assignmentAttempts AssignmentAttempt[] /// Attempts that used this version + + @@unique([assignmentId, versionNumber]) + @@index([assignmentId]) + @@index([isActive]) + @@index([isDraft]) +} + +/// The QuestionVersion model represents a specific version of a question within an assignment version +model QuestionVersion { + id Int @id @default(autoincrement()) /// Unique identifier for the question version + assignmentVersionId Int /// The ID of the assignment version this belongs to + assignmentVersion AssignmentVersion @relation(fields: [assignmentVersionId], references: [id], onDelete: Cascade) + questionId Int? /// Reference to original question (null for new questions in version) + totalPoints Int /// Points for this question + type QuestionType /// Type of question + responseType ResponseType? /// Expected response type + question String /// The question text + maxWords Int? /// Maximum words allowed + scoring Json? /// Scoring configuration + choices Json? /// Question choices + randomizedChoices Boolean? /// Are choices randomized + answer Boolean? /// Correct answer + gradingContextQuestionIds Int[] /// Context questions for grading + maxCharacters Int? /// Maximum characters allowed + videoPresentationConfig Json? /// Video presentation config + liveRecordingConfig Json? /// Live recording config + displayOrder Int? /// Display order within version + createdAt DateTime @default(now()) /// When this version was created + + @@index([assignmentVersionId]) + @@index([questionId]) +} + +/// The VersionHistory model tracks version transitions and changes +model VersionHistory { + id Int @id @default(autoincrement()) /// Unique identifier + assignmentId Int /// The assignment being versioned + assignment Assignment @relation(fields: [assignmentId], references: [id], onDelete: Cascade) + fromVersionId Int? /// The version being changed from (null for initial) + fromVersion AssignmentVersion? @relation("VersionHistoryFrom", fields: [fromVersionId], references: [id], onDelete: SetNull) + toVersionId Int /// The version being changed to + toVersion AssignmentVersion @relation("VersionHistoryTo", fields: [toVersionId], references: [id], onDelete: Cascade) + action String /// Action performed (created, published, restored, draft_saved) + description String? /// Description of the change + userId String /// User who performed the action + createdAt DateTime @default(now()) /// When the action occurred + metadata Json? /// Additional metadata about the change + + @@index([assignmentId]) + @@index([createdAt]) + @@index([userId]) } + enum VariantType { REWORDED /// The variant is reworded RANDOMIZED /// The variant is randomized @@ -274,22 +375,31 @@ model Question { GradingAudit GradingAudit[] } model AssignmentAttempt { - id Int @id @default(autoincrement()) - assignmentId Int - userId String - questionResponses QuestionResponse[] - submitted Boolean - grade Float? - expiresAt DateTime? - createdAt DateTime @default(now()) - questionVariants AssignmentAttemptQuestionVariant[] - AssignmentFeedback AssignmentFeedback[] - questionOrder Int[] - RegradingRequest RegradingRequest[] - comments String? /// Additional comments or notes for the assignment attempt from the system - preferredLanguage String? + id Int @id @default(autoincrement()) + assignmentId Int + assignmentVersionId Int? /// The ID of the assignment version used for this attempt + assignmentVersion AssignmentVersion? @relation(fields: [assignmentVersionId], references: [id]) + userId String + questionResponses QuestionResponse[] + submitted Boolean + grade Float? + expiresAt DateTime? + createdAt DateTime @default(now()) + questionVariants AssignmentAttemptQuestionVariant[] + AssignmentFeedback AssignmentFeedback[] + questionOrder Int[] + RegradingRequest RegradingRequest[] + comments String? /// Additional comments or notes for the assignment attempt from the system + preferredLanguage String? GradingJob GradingJob[] + + @@index([assignmentId]) + @@index([userId]) + @@index([assignmentId, submitted]) + @@index([createdAt]) + @@index([assignmentId, userId]) + @@index([assignmentVersionId]) } model AssignmentAttemptQuestionVariant { @@ -356,6 +466,9 @@ model AssignmentFeedback { email String? /// User's email (required if allowContact is true) createdAt DateTime @default(now()) /// Timestamp for when the feedback was created updatedAt DateTime @updatedAt /// Timestamp for the last update to the feedback + + @@index([assignmentId]) + @@index([assignmentId, assignmentRating]) } model RegradingRequest { @@ -470,6 +583,43 @@ model Translation { @@index([questionId, variantId, languageCode]) } +/// The AssignmentDraft model represents user-specific drafts (separate from versions) +model AssignmentDraft { + id Int @id @default(autoincrement()) /// Unique identifier for the draft + assignmentId Int /// The ID of the parent assignment + assignment Assignment @relation(fields: [assignmentId], references: [id], onDelete: Cascade) + userId String /// User who created this draft + draftName String /// User-defined name for this draft + name String /// Assignment name in this draft + introduction String? /// Introduction for this draft + instructions String? /// Instructions for this draft + gradingCriteriaOverview String? /// Grading criteria for this draft + timeEstimateMinutes Int? /// Time estimate for this draft + type AssignmentType /// Type of assignment + graded Boolean? @default(false) /// Is the assignment graded + numAttempts Int? @default(-1) /// Max attempts allowed + allotedTimeMinutes Int? /// Time allotted to complete + attemptsPerTimeRange Int? /// Attempts per time range + attemptsTimeRangeHours Int? /// Time range in hours + passingGrade Int? @default(50) /// Minimum passing grade + displayOrder AssignmentQuestionDisplayOrder? /// Question display order + questionDisplay QuestionDisplay? @default(ONE_PER_PAGE) /// Question display mode + numberOfQuestionsPerAttempt Int? /// Questions per attempt + questionOrder Int[] /// Order of questions + published Boolean @default(false) /// Is this draft published + showAssignmentScore Boolean @default(true) /// Show assignment score + showQuestionScore Boolean @default(true) /// Show question score + showSubmissionFeedback Boolean @default(true) /// Show submission feedback + showQuestions Boolean @default(true) /// Show questions + languageCode String? /// Language code + questionsData Json? /// Serialized questions data + createdAt DateTime @default(now()) /// When draft was created + updatedAt DateTime @updatedAt /// When draft was last updated + @@index([assignmentId]) + @@index([userId]) + @@index([createdAt]) +} + model AssignmentTranslation { id Int @id @default(autoincrement()) assignmentId Int @@ -498,3 +648,158 @@ model FeedbackTranslation { question Question @relation(fields: [questionId], references: [id], onDelete: Cascade) @@unique([questionId, languageCode]) } + +model AdminVerificationCode { + id Int @id @default(autoincrement()) + email String + code String + expiresAt DateTime + createdAt DateTime @default(now()) + used Boolean @default(false) + + @@index([email]) + @@index([code]) + @@index([expiresAt]) +} + +model AdminSession { + id Int @id @default(autoincrement()) + email String + sessionToken String @unique + expiresAt DateTime + createdAt DateTime @default(now()) + + @@index([email]) + @@index([sessionToken]) + @@index([expiresAt]) +} + +/// This model tracks the authors of assignments +model AssignmentAuthor { + id Int @id @default(autoincrement()) + assignmentId Int /// The ID of the assignment + userId String /// The ID of the user who authored the assignment + createdAt DateTime @default(now()) /// When the author was added + assignment Assignment @relation(fields: [assignmentId], references: [id], onDelete: Cascade) + + @@unique([assignmentId, userId]) + @@index([assignmentId]) + @@index([userId]) +} + +/// Enum for different pricing sources +enum PricingSource { + OPENAI_API /// Pricing fetched from OpenAI API + MANUAL /// Manually entered pricing + WEB_SCRAPING /// Pricing scraped from web sources +} + +/// This model represents different LLM models supported by the system +model LLMModel { + id Int @id @default(autoincrement()) + modelKey String @unique /// Unique identifier for the model (e.g., "gpt-4o", "gpt-4o-mini") + displayName String /// Human-readable name for the model + provider String /// Provider name (e.g., "OpenAI", "Anthropic") + isActive Boolean @default(true) /// Whether this model is currently supported + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + pricingHistory LLMPricing[] /// Historical pricing data for this model + featureAssignments LLMFeatureAssignment[] /// Features assigned to this model + + @@index([modelKey]) + @@index([provider]) +} + +/// This model tracks pricing changes for LLM models over time +model LLMPricing { + id Int @id @default(autoincrement()) + modelId Int /// The ID of the LLM model + model LLMModel @relation(fields: [modelId], references: [id], onDelete: Cascade) + inputTokenPrice Float /// Price per input token (e.g., 0.000001) + outputTokenPrice Float /// Price per output token (e.g., 0.000003) + effectiveDate DateTime /// When this pricing became effective + source PricingSource /// Source of this pricing information + isActive Boolean @default(true) /// Whether this pricing is currently active + metadata Json? /// Additional metadata about the pricing (e.g., API response, notes) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([modelId, effectiveDate]) + @@index([effectiveDate]) + @@index([isActive]) + @@unique([modelId, effectiveDate, source]) +} + +/// This model stores price upscaling factors that are applied to base pricing +model LLMPriceUpscaling { + id Int @id @default(autoincrement()) + globalFactor Float? /// Global upscaling factor applied to all usage types + usageTypeFactors Json? /// Usage-type specific upscaling factors (e.g., {"TRANSLATION": 1.2, "GRADING": 1.5}) + reason String? /// Reason for the price upscaling + appliedBy String? /// Email of admin who applied the upscaling + isActive Boolean @default(true) /// Whether this upscaling is currently active + effectiveDate DateTime @default(now()) /// When this upscaling became effective + deactivatedAt DateTime? /// When this upscaling was deactivated (if applicable) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([isActive, effectiveDate]) + @@index([effectiveDate]) +} + +/// Enum for different AI features in the system +enum AIFeatureType { + TEXT_GRADING /// AI grading for text-based questions + FILE_GRADING /// AI grading for file uploads + IMAGE_GRADING /// AI grading for image submissions + URL_GRADING /// AI grading for URL submissions + PRESENTATION_GRADING /// AI grading for presentation files + VIDEO_GRADING /// AI grading for video presentations + QUESTION_GENERATION /// AI-powered question generation + TRANSLATION /// AI translation services + RUBRIC_GENERATION /// AI rubric creation + CONTENT_MODERATION /// AI content moderation + ASSIGNMENT_GENERATION /// AI assignment creation + LIVE_RECORDING_FEEDBACK /// AI feedback for live recordings +} + +/// This model represents AI features available in the system +model AIFeature { + id Int @id @default(autoincrement()) + featureKey String @unique /// Unique identifier for the feature (e.g., "text_grading", "question_generation") + featureType AIFeatureType /// Type of AI feature + displayName String /// Human-readable name for the feature + description String? /// Description of what this feature does + isActive Boolean @default(true) /// Whether this feature is currently enabled + requiresModel Boolean @default(true) /// Whether this feature requires an LLM model + defaultModelKey String? /// Default model to use if no assignment exists + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // Relations + assignments LLMFeatureAssignment[] /// Current and historical model assignments + + @@index([featureKey]) + @@index([featureType]) + @@index([isActive]) +} + +/// This model tracks which LLM model is assigned to each AI feature +model LLMFeatureAssignment { + id Int @id @default(autoincrement()) + featureId Int /// The AI feature this assignment is for + feature AIFeature @relation(fields: [featureId], references: [id], onDelete: Cascade) + modelId Int /// The LLM model assigned to this feature + model LLMModel @relation(fields: [modelId], references: [id], onDelete: Cascade) + isActive Boolean @default(true) /// Whether this assignment is currently active + priority Int @default(0) /// Priority for model selection (higher = more preferred) + assignedBy String? /// Email of admin who made the assignment + assignedAt DateTime @default(now()) /// When this assignment was made + deactivatedAt DateTime? /// When this assignment was deactivated (if applicable) + metadata Json? /// Additional configuration for this assignment + + @@index([featureId, isActive]) + @@index([modelId]) + @@index([isActive]) + @@unique([featureId, modelId]) +} diff --git a/apps/api/prisma/seed.ts b/apps/api/prisma/seed.ts index 52f4696d..03d075aa 100644 --- a/apps/api/prisma/seed.ts +++ b/apps/api/prisma/seed.ts @@ -120,7 +120,7 @@ async function runPgRestore(sqlFilePath: string) { const port = process.env.POSTGRES_PORT; const command = `PGPASSWORD=${password} pg_restore -d ${database} -U ${user} -h ${host} -p ${port} ${sqlFilePath}`; return new Promise((resolve, reject) => { - exec(command, (error, stdout, stderr) => { + exec(command, (error, stdout) => { if (error) { reject(error); } else { @@ -133,16 +133,15 @@ async function runPgRestore(sqlFilePath: string) { async function main() { // eslint-disable-next-line unicorn/prefer-module const sqlFilePath = path.join(__dirname, "seed.sql"); - if (fs.existsSync(sqlFilePath)) { - await runPgRestore(sqlFilePath); - } else { - const assignment = await prisma.assignment.create({ - data: { - name: "Cybersecurity Job Listing", - type: "AI_GRADED", - introduction: - "In this project, you will explore a Cybersecurity job listing and relate it to the concepts learned in the course.", - instructions: `Before submitting your responses, please ensure you have completed the following tasks: + await (fs.existsSync(sqlFilePath) + ? runPgRestore(sqlFilePath) + : prisma.assignment.create({ + data: { + name: "Cybersecurity Job Listing", + type: "AI_GRADED", + introduction: + "In this project, you will explore a Cybersecurity job listing and relate it to the concepts learned in the course.", + instructions: `Before submitting your responses, please ensure you have completed the following tasks: **Task 1: Identify Cybersecurity Role and Find Job Listing** [A] - Identify a Cybersecurity job role that interests you. @@ -160,7 +159,7 @@ async function main() { **Task 5: Create a Plan** Develop a roadmap to become eligible for this job. `, - gradingCriteriaOverview: `The assignment is worth 10 points and requires 60% to pass. + gradingCriteriaOverview: `The assignment is worth 10 points and requires 60% to pass. [1] (1 point) Provide the URL for your selected job listing. [2] (2 points) Provide company name, job title, and job location. @@ -169,47 +168,46 @@ async function main() { [5] (3 points) Outline a plan to qualify for the job. Click "Begin Assignment" to submit your responses.`, - graded: true, - allotedTimeMinutes: 1, - displayOrder: "RANDOM", - showAssignmentScore: true, - showQuestionScore: true, - showSubmissionFeedback: true, - numAttempts: undefined, - timeEstimateMinutes: 1, - published: true, - questions: { - createMany: { - data: questions.map((q) => ({ - ...q, - scoring: q.scoring as unknown as Prisma.InputJsonValue, - choices: q.choices as unknown as Prisma.InputJsonValue, - })), + graded: true, + allotedTimeMinutes: 1, + displayOrder: "RANDOM", + showAssignmentScore: true, + showQuestionScore: true, + showSubmissionFeedback: true, + numAttempts: undefined, + timeEstimateMinutes: 1, + published: true, + questions: { + createMany: { + data: questions.map((q) => ({ + ...q, + scoring: q.scoring as unknown as Prisma.InputJsonValue, + choices: q.choices as unknown as Prisma.InputJsonValue, + })), + }, }, - }, - groups: { - create: [ - { - group: { - connectOrCreate: { - where: { - id: "test-group-id", - }, - create: { - id: "test-group-id", + groups: { + create: [ + { + group: { + connectOrCreate: { + where: { + id: "test-group-id", + }, + create: { + id: "test-group-id", + }, }, }, }, - }, - ], + ], + }, }, - }, - }); - } + })); } main() - .catch((error) => { + .catch(() => { // eslint-disable-next-line unicorn/no-process-exit process.exit(1); }) diff --git a/apps/api/src/api/Job/job.module.ts b/apps/api/src/api/Job/job.module.ts index cef69c9c..86eaf3d7 100644 --- a/apps/api/src/api/Job/job.module.ts +++ b/apps/api/src/api/Job/job.module.ts @@ -1,7 +1,7 @@ import { HttpModule } from "@nestjs/axios"; import { Module } from "@nestjs/common"; -import { JobStatusServiceV1 } from "./job-status.service"; import { PrismaService } from "src/prisma.service"; +import { JobStatusServiceV1 } from "./job-status.service"; @Module({ providers: [JobStatusServiceV1, PrismaService], diff --git a/apps/api/src/api/admin/admin.controller.spec.ts b/apps/api/src/api/admin/admin.controller.spec.ts index 2babb4f2..14f6aa62 100644 --- a/apps/api/src/api/admin/admin.controller.spec.ts +++ b/apps/api/src/api/admin/admin.controller.spec.ts @@ -1,15 +1,42 @@ import { Test, TestingModule } from "@nestjs/testing"; +import { AdminVerificationService } from "../../auth/services/admin-verification.service"; import { PrismaService } from "../../prisma.service"; +import { LLM_PRICING_SERVICE } from "../llm/llm.constants"; import { AdminController } from "./admin.controller"; +import { AdminRepository } from "./admin.repository"; import { AdminService } from "./admin.service"; describe("AdminController", () => { let controller: AdminController; beforeEach(async () => { + const mockLlmPricingService = { + calculateCost: jest.fn().mockReturnValue(0.01), + getTokenCount: jest.fn().mockReturnValue(100), + }; + + const mockAdminVerificationService = { + generateAndStoreCode: jest.fn().mockResolvedValue("123456"), + verifyCode: jest.fn().mockResolvedValue(true), + verifyAdminSession: jest + .fn() + .mockResolvedValue({ email: "admin@test.com", role: "admin" }), + createAdminSession: jest.fn().mockResolvedValue("mock-session-token"), + revokeSession: jest.fn().mockResolvedValue(), + }; + const module: TestingModule = await Test.createTestingModule({ controllers: [AdminController], - providers: [AdminService, PrismaService], + providers: [ + AdminService, + PrismaService, + AdminRepository, + { provide: LLM_PRICING_SERVICE, useValue: mockLlmPricingService }, + { + provide: AdminVerificationService, + useValue: mockAdminVerificationService, + }, + ], }).compile(); controller = module.get(AdminController); diff --git a/apps/api/src/api/admin/admin.controller.ts b/apps/api/src/api/admin/admin.controller.ts index fbb32c62..5812c2ec 100644 --- a/apps/api/src/api/admin/admin.controller.ts +++ b/apps/api/src/api/admin/admin.controller.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/require-await */ import { Body, Controller, @@ -19,6 +20,8 @@ import { ApiResponse, ApiTags, } from "@nestjs/swagger"; +import { Assignment } from "@prisma/client"; +import { AdminRepository } from "./admin.repository"; import { AdminService } from "./admin.service"; import { AdminAddAssignmentToGroupResponseDto } from "./dto/assignment/add.assignment.to.group.response.dto"; import { BaseAssignmentResponseDto } from "./dto/assignment/base.assignment.response.dto"; @@ -44,7 +47,10 @@ import { AdminUpdateAssignmentRequestDto } from "./dto/assignment/update.assignm version: "1", }) export class AdminController { - constructor(private adminService: AdminService) {} + constructor( + private adminService: AdminService, + private adminRepository: AdminRepository, + ) {} @Post("assignments/clone/:id") @ApiOperation({ @@ -77,7 +83,13 @@ export class AdminController { groupId, ); } - + @Get("/assignments") + @ApiOperation({ summary: "Get all assignments" }) + @ApiResponse({ status: 200, type: [BaseAssignmentResponseDto] }) + @ApiResponse({ status: 403 }) + getAssignments(): Promise { + return this.adminRepository.findAllAssignments(); + } @Post("/assignments") @ApiOperation({ summary: "Create an assignment" }) @ApiBody({ type: AdminCreateAssignmentRequestDto }) diff --git a/apps/api/src/api/admin/admin.module.ts b/apps/api/src/api/admin/admin.module.ts index 9796cae0..9692af1b 100644 --- a/apps/api/src/api/admin/admin.module.ts +++ b/apps/api/src/api/admin/admin.module.ts @@ -1,13 +1,37 @@ import { Module } from "@nestjs/common"; import { PassportModule } from "@nestjs/passport"; import { PrismaService } from "src/prisma.service"; +import { AdminAuthModule } from "../../auth/admin-auth.module"; import { AuthModule } from "../../auth/auth.module"; +import { LlmModule } from "../llm/llm.module"; +import { ScheduledTasksModule } from "../scheduled-tasks/scheduled-tasks.module"; import { AdminController } from "./admin.controller"; +import { AdminRepository } from "./admin.repository"; import { AdminService } from "./admin.service"; +import { AdminDashboardController } from "./controllers/admin-dashboard.controller"; +import { AssignmentAnalyticsController } from "./controllers/assignment-analytics.controller"; +import { FlaggedSubmissionsController } from "./controllers/flagged-submissions.controller"; +import { LLMAssignmentController } from "./controllers/llm-assignment.controller"; +import { LLMPricingController } from "./controllers/llm-pricing.controller"; +import { RegradingRequestsController } from "./controllers/regrading-requests.controller"; @Module({ - imports: [AuthModule, PassportModule], - controllers: [AdminController], - providers: [AdminService, PrismaService], + imports: [ + AuthModule, + PassportModule, + AdminAuthModule, + LlmModule, + ScheduledTasksModule, + ], + controllers: [ + AdminController, + AdminDashboardController, + LLMAssignmentController, + LLMPricingController, + RegradingRequestsController, + FlaggedSubmissionsController, + AssignmentAnalyticsController, + ], + providers: [AdminService, PrismaService, AdminRepository], }) export class AdminModule {} diff --git a/apps/api/src/api/admin/admin.repository.ts b/apps/api/src/api/admin/admin.repository.ts new file mode 100644 index 00000000..2f3cbfc7 --- /dev/null +++ b/apps/api/src/api/admin/admin.repository.ts @@ -0,0 +1,151 @@ +import { Injectable } from "@nestjs/common"; +import { Assignment, AssignmentAttempt } from "@prisma/client"; +import { PrismaService } from "src/prisma.service"; +import { RegradingRequestDto } from "../assignment/attempt/dto/assignment-attempt/feedback.request.dto"; + +@Injectable() +export class AdminRepository { + constructor(private prisma: PrismaService) {} + + async findAssignmentById(id: number) { + return this.prisma.assignment.findUnique({ + where: { id }, + include: { + questions: { + where: { isDeleted: false }, + include: { + variants: { + where: { isDeleted: false }, + }, + }, + }, + }, + }); + } + async findAssignmentByGroupId(groupId: string) { + return this.prisma.assignment.findMany({ + where: { + groups: { + some: { + groupId, + }, + }, + }, + include: { + questions: { + where: { isDeleted: false }, + include: { + variants: { + where: { isDeleted: false }, + }, + }, + }, + }, + }); + } + + async findAllAssignments() { + return this.prisma.assignment.findMany({ + orderBy: { + updatedAt: "desc", + }, + }); + } + + async createAssignment(data: Assignment) { + return this.prisma.assignment.create({ + data, + }); + } + + async updateAssignment(id: number, data: Assignment) { + return this.prisma.assignment.update({ + where: { id }, + data, + }); + } + + async deleteAssignment(id: number) { + return this.prisma.assignment.delete({ + where: { id }, + }); + } + + async findGroupById(id: string) { + return this.prisma.group.findUnique({ + where: { id }, + }); + } + + async createGroup(id: string) { + return this.prisma.group.create({ + data: { id }, + }); + } + + async createAssignmentGroup(assignmentId: number, groupId: string) { + return this.prisma.assignmentGroup.create({ + data: { + assignmentId, + groupId, + }, + }); + } + + async findAllFlaggedSubmissions() { + return this.prisma.regradingRequest.findMany({ + where: { + regradingStatus: "PENDING", + }, + orderBy: { + createdAt: "desc", + }, + }); + } + + async findRegradingRequestById(id: number) { + return this.prisma.regradingRequest.findUnique({ + where: { id }, + }); + } + + async updateRegradingRequest(id: number, data: RegradingRequestDto) { + return this.prisma.regradingRequest.update({ + where: { id }, + data, + }); + } + + async findAllRegradingRequests() { + return this.prisma.regradingRequest.findMany({ + orderBy: { + createdAt: "desc", + }, + }); + } + + async findAssignmentAttemptById(id: number) { + return this.prisma.assignmentAttempt.findUnique({ + where: { id }, + }); + } + + async updateAssignmentAttempt(id: number, data: AssignmentAttempt) { + return this.prisma.assignmentAttempt.update({ + where: { id }, + data, + }); + } + + async findAttemptsByAssignmentId(assignmentId: number) { + return this.prisma.assignmentAttempt.findMany({ + where: { + assignmentId, + submitted: true, + }, + include: { + questionResponses: true, + }, + }); + } +} diff --git a/apps/api/src/api/admin/admin.service.spec.ts b/apps/api/src/api/admin/admin.service.spec.ts index aa325319..8e3022ed 100644 --- a/apps/api/src/api/admin/admin.service.spec.ts +++ b/apps/api/src/api/admin/admin.service.spec.ts @@ -1,13 +1,23 @@ import { Test, TestingModule } from "@nestjs/testing"; import { PrismaService } from "../../prisma.service"; +import { LLM_PRICING_SERVICE } from "../llm/llm.constants"; import { AdminService } from "./admin.service"; describe("AdminService", () => { let service: AdminService; beforeEach(async () => { + const mockLlmPricingService = { + calculateCost: jest.fn().mockReturnValue(0.01), + getTokenCount: jest.fn().mockReturnValue(100), + }; + const module: TestingModule = await Test.createTestingModule({ - providers: [AdminService, PrismaService], + providers: [ + AdminService, + PrismaService, + { provide: LLM_PRICING_SERVICE, useValue: mockLlmPricingService }, + ], }).compile(); service = module.get(AdminService); diff --git a/apps/api/src/api/admin/admin.service.ts b/apps/api/src/api/admin/admin.service.ts index 09b4c8f3..98ce200f 100644 --- a/apps/api/src/api/admin/admin.service.ts +++ b/apps/api/src/api/admin/admin.service.ts @@ -1,5 +1,17 @@ -import { Injectable, NotFoundException } from "@nestjs/common"; +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +import { Inject, Injectable, Logger, NotFoundException } from "@nestjs/common"; +import { + UserRole, + UserSession, +} from "../../auth/interfaces/user.session.interface"; import { PrismaService } from "../../prisma.service"; +import { LLMPricingService } from "../llm/core/services/llm-pricing.service"; +import { LLM_PRICING_SERVICE } from "../llm/llm.constants"; import { AdminAddAssignmentToGroupResponseDto } from "./dto/assignment/add.assignment.to.group.response.dto"; import { BaseAssignmentResponseDto } from "./dto/assignment/base.assignment.response.dto"; import { @@ -9,9 +21,725 @@ import { import { AdminGetAssignmentResponseDto } from "./dto/assignment/get.assignment.response.dto"; import { AdminUpdateAssignmentRequestDto } from "./dto/assignment/update.assignment.request.dto"; +interface DashboardFilters { + startDate?: string; + endDate?: string; + assignmentId?: number; + assignmentName?: string; + userId?: string; +} + @Injectable() export class AdminService { - constructor(private readonly prisma: PrismaService) {} + private readonly logger = new Logger(AdminService.name); + private readonly insightsCache = new Map< + string, + { data: any; cachedAt: number } + >(); + private readonly INSIGHTS_CACHE_TTL = 1 * 60 * 1000; // 1 minute cache + + constructor( + private readonly prisma: PrismaService, + @Inject(LLM_PRICING_SERVICE) + private readonly llmPricingService: LLMPricingService, + ) {} + + /** + * Helper method to get cached insights data + */ + private getCachedInsights(assignmentId: number): any | null { + const cacheKey = `insights:${assignmentId}`; + const cached = this.insightsCache.get(cacheKey); + + if (cached && Date.now() - cached.cachedAt < this.INSIGHTS_CACHE_TTL) { + this.logger.debug(`Cache hit for assignment ${assignmentId} insights`); + return cached.data; + } + + if (cached) { + this.insightsCache.delete(cacheKey); + } + + return null; + } + + /** + * Helper method to cache insights data + */ + private setCachedInsights(assignmentId: number, data: any): void { + const cacheKey = `insights:${assignmentId}`; + this.insightsCache.set(cacheKey, { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + data, + cachedAt: Date.now(), + }); + this.logger.debug(`Cached insights for assignment ${assignmentId}`); + } + + /** + * Helper method to invalidate insights cache for an assignment + */ + private invalidateInsightsCache(assignmentId: number): void { + const cacheKey = `insights:${assignmentId}`; + this.insightsCache.delete(cacheKey); + this.logger.debug( + `Invalidated insights cache for assignment ${assignmentId}`, + ); + } + + /** + * Public method to invalidate insights cache when assignment data changes + */ + invalidateAssignmentInsightsCache(assignmentId: number): void { + this.invalidateInsightsCache(assignmentId); + } + + async getBasicAssignmentAnalytics(assignmentId: number) { + // Check if the assignment exists + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { + questions: { + where: { isDeleted: false }, + }, + }, + }); + + if (!assignment) { + throw new Error(`Assignment with ID ${assignmentId} not found`); + } + + // Get all attempts for this assignment + const attempts = await this.prisma.assignmentAttempt.findMany({ + where: { + assignmentId, + submitted: true, + }, + include: { + questionResponses: true, + }, + }); + + // Calculate average score + const totalGrades = attempts.reduce( + (sum, attempt) => sum + (attempt.grade || 0), + 0, + ); + const averageScore = + attempts.length > 0 ? (totalGrades / attempts.length) * 100 : 0; + + // Calculate median score + const grades = attempts + .map((attempt) => attempt.grade || 0) + .sort((a, b) => a - b); + const medianIndex = Math.floor(grades.length / 2); + const medianScore = + grades.length > 0 + ? (grades.length % 2 === 0 + ? (grades[medianIndex - 1] + grades[medianIndex]) / 2 + : grades[medianIndex]) * 100 + : 0; + + // Calculate completion rate + const totalAttempts = attempts.length; + const completedAttempts = attempts.filter( + (attempt) => attempt.submitted, + ).length; + const completionRate = + totalAttempts > 0 ? (completedAttempts / totalAttempts) * 100 : 0; + + // Calculate average completion time + const completionTimes = attempts + .map((attempt) => { + if (attempt.createdAt && attempt.expiresAt) { + return ( + new Date(attempt.expiresAt).getTime() - + new Date(attempt.createdAt).getTime() + ); + } + return 0; + }) + .filter((time) => time > 0); + + const avgTimeMs = + completionTimes.length > 0 + ? completionTimes.reduce((sum, time) => sum + time, 0) / + completionTimes.length + : 0; + const averageCompletionTime = Math.round(avgTimeMs / (1000 * 60)); // Convert to minutes + + // Calculate score distribution + const scoreRanges = [ + "0-10", + "11-20", + "21-30", + "31-40", + "41-50", + "51-60", + "61-70", + "71-80", + "81-90", + "91-100", + ]; + const scoreDistribution = scoreRanges.map((range) => { + const [min, max] = range.split("-").map(Number); + const count = grades.filter((grade) => { + const score = grade * 100; + return score >= min && score <= max; + }).length; + return { range, count }; + }); + + // Calculate question breakdown + const questionBreakdown = assignment.questions.map((question) => { + const responses = attempts.flatMap((attempt) => + attempt.questionResponses.filter( + (response) => response.questionId === question.id, + ), + ); + + const totalPoints = responses.reduce( + (sum, response) => sum + response.points, + 0, + ); + const averageScore = + responses.length > 0 + ? (totalPoints / (responses.length * question.totalPoints)) * 100 + : 0; + + const incorrectResponses = responses.filter( + (response) => response.points < question.totalPoints, + ); + const incorrectRate = + responses.length > 0 + ? (incorrectResponses.length / responses.length) * 100 + : 0; + + return { + questionId: question.id, + averageScore, + incorrectRate, + }; + }); + + // total number of unique users who attempted the assignment + const uniqueUsers = new Set(attempts.map((attempt) => attempt.userId)).size; + + return { + averageScore, + medianScore, + completionRate, + totalAttempts, + averageCompletionTime, + scoreDistribution, + questionBreakdown, + uniqueUsers, + }; + } + + /** + * Get assignment attempts with basic information + */ + private async getAssignmentAttempts(assignmentId: number) { + try { + const attempts = await this.prisma.assignmentAttempt.findMany({ + where: { assignmentId }, + select: { + id: true, + userId: true, + submitted: true, + grade: true, + createdAt: true, + }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + return attempts.map((attempt) => ({ + id: attempt.id, + userId: attempt.userId, + submitted: attempt.submitted, + grade: attempt.grade, + createdAt: attempt.createdAt.toISOString(), + })); + } catch (error) { + this.logger.error( + `Error fetching attempts for assignment ${assignmentId}:`, + error, + ); + return []; + } + } + + /** + * Precompute insights for popular assignments to improve performance + */ + async precomputePopularInsights(): Promise { + try { + this.logger.log( + "Starting precomputation of insights for popular assignments", + ); + + // Find the most accessed assignments in the last 7 days + const popularAssignments = await this.prisma.assignmentAttempt.groupBy({ + by: ["assignmentId"], + where: { + createdAt: { + gte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), // Last 7 days + }, + }, + _count: { + assignmentId: true, + }, + orderBy: { + _count: { + assignmentId: "desc", + }, + }, + take: 20, // Top 20 most active assignments + }); + + this.logger.log( + `Found ${popularAssignments.length} popular assignments to precompute`, + ); + + // Create a mock admin session for precomputation + const adminSession = { + assignmentId: 1, + role: UserRole.ADMIN, + groupId: "system-group", + userId: "system-user", + }; + + // Process assignments in smaller batches to avoid overwhelming the system + const batchSize = 5; + for ( + let index = 0; + index < popularAssignments.length; + index += batchSize + ) { + const batch = popularAssignments.slice(index, index + batchSize); + + await Promise.all( + batch.map(async (assignment) => { + try { + // This will compute and cache the insights + await this.getDetailedAssignmentInsights( + adminSession, + assignment.assignmentId, + ); + this.logger.debug( + `Precomputed insights for assignment ${assignment.assignmentId}`, + ); + } catch (error) { + this.logger.warn( + `Failed to precompute insights for assignment ${assignment.assignmentId}:`, + error, + ); + } + }), + ); + + // Add a small delay between batches + if (index + batchSize < popularAssignments.length) { + await new Promise((resolve) => setTimeout(resolve, 1000)); // 1 second delay + } + } + + this.logger.log( + `Completed precomputation of insights for ${popularAssignments.length} assignments`, + ); + } catch (error) { + this.logger.error("Error during insights precomputation:", error); + } + } + + /** + * Helper method to calculate costs using historical pricing data with detailed breakdown + */ + private async calculateHistoricalCosts( + aiUsageRecords: Array<{ + tokensIn: number; + tokensOut: number; + createdAt: Date; + usageType?: string; + modelKey?: string; + }>, + ): Promise<{ + totalCost: number; + costBreakdown: { + grading: number; + questionGeneration: number; + translation: number; + other: number; + }; + detailedBreakdown: Array<{ + tokensIn: number; + tokensOut: number; + inputCost: number; + outputCost: number; + totalCost: number; + usageDate: Date; + modelKey: string; + inputTokenPrice: number; + outputTokenPrice: number; + pricingEffectiveDate: Date; + usageType?: string; + calculationSteps: { + inputCalculation: string; + outputCalculation: string; + totalCalculation: string; + }; + }>; + }> { + let totalCost = 0; + const detailedBreakdown = []; + const costByType = { + grading: 0, + questionGeneration: 0, + translation: 0, + other: 0, + }; + + for (const usage of aiUsageRecords) { + // Use the actual model key from the database if available + let modelKey = usage.modelKey; + + if (!modelKey) { + // Log warning for missing model key - this shouldn't happen with new records + this.logger.warn( + `Missing model key for usage record from ${usage.createdAt.toISOString()}, falling back based on usage type`, + ); + + // Fallback logic for older records without model key stored + const usageType = usage.usageType?.toLowerCase() || ""; + if (usageType.includes("translation")) { + modelKey = "gpt-4o-mini"; + } else if ( + usageType.includes("image") || + usageType.includes("vision") + ) { + modelKey = "gpt-4.1-mini"; + } else if ( + usageType.includes("grading") || + usageType.includes("generation") + ) { + modelKey = "gpt-4o"; + } else { + modelKey = "gpt-4o-mini"; // Default for unknown types + } + } + + const costBreakdown = + await this.llmPricingService.calculateCostWithBreakdown( + modelKey, + usage.tokensIn, + usage.tokensOut, + usage.createdAt, + usage.usageType, // Pass usage type for upscaling-aware calculations + ); + + if (costBreakdown) { + totalCost += costBreakdown.totalCost; + + // Categorize costs by usage type + const usageType = usage.usageType?.toLowerCase() || "other"; + if (usageType.includes("grading")) { + costByType.grading += costBreakdown.totalCost; + } else if ( + usageType.includes("question") || + usageType.includes("generation") + ) { + costByType.questionGeneration += costBreakdown.totalCost; + } else if (usageType.includes("translation")) { + costByType.translation += costBreakdown.totalCost; + } else { + costByType.other += costBreakdown.totalCost; + } + + // Create detailed calculation steps for transparency (showing per million token pricing) + const inputPricePerMillion = costBreakdown.inputTokenPrice * 1_000_000; + const outputPricePerMillion = + costBreakdown.outputTokenPrice * 1_000_000; + const calculationSteps = { + inputCalculation: `${usage.tokensIn.toLocaleString()} tokens × $${inputPricePerMillion.toFixed( + 2, + )}/1M tokens = $${costBreakdown.inputCost.toFixed(8)}`, + outputCalculation: `${usage.tokensOut.toLocaleString()} tokens × $${outputPricePerMillion.toFixed( + 2, + )}/1M tokens = $${costBreakdown.outputCost.toFixed(8)}`, + totalCalculation: `$${costBreakdown.inputCost.toFixed( + 8, + )} + $${costBreakdown.outputCost.toFixed( + 8, + )} = $${costBreakdown.totalCost.toFixed(8)}`, + }; + + detailedBreakdown.push({ + tokensIn: usage.tokensIn, + tokensOut: usage.tokensOut, + inputCost: costBreakdown.inputCost, + outputCost: costBreakdown.outputCost, + totalCost: costBreakdown.totalCost, + usageDate: usage.createdAt, + modelKey: costBreakdown.modelKey, + inputTokenPrice: costBreakdown.inputTokenPrice, + outputTokenPrice: costBreakdown.outputTokenPrice, + pricingEffectiveDate: costBreakdown.pricingEffectiveDate, + usageType: usage.usageType, + calculationSteps, + }); + } else { + // Enhanced fallback with proper logging and transparency + this.logger.error( + `No pricing found for ${modelKey} at ${usage.createdAt.toISOString()}, using emergency fallback`, + ); + + const fallbackPrices: Record< + string, + { input: number; output: number } + > = { + "gpt-4o": { input: 0.000_002_5, output: 0.000_01 }, + "gpt-4o-mini": { input: 0.000_000_15, output: 0.000_000_6 }, + "gpt-4.1-mini": { input: 0.000_002_5, output: 0.000_01 }, + }; + + const prices = + fallbackPrices[modelKey] || fallbackPrices["gpt-4o-mini"]; + const inputCost = usage.tokensIn * prices.input; + const outputCost = usage.tokensOut * prices.output; + const fallbackCost = inputCost + outputCost; + + totalCost += fallbackCost; + costByType.other += fallbackCost; + + const inputPricePerMillion = prices.input * 1_000_000; + const outputPricePerMillion = prices.output * 1_000_000; + const calculationSteps = { + inputCalculation: `${usage.tokensIn.toLocaleString()} tokens × $${inputPricePerMillion.toFixed( + 2, + )}/1M tokens = $${inputCost.toFixed(8)} (fallback)`, + outputCalculation: `${usage.tokensOut.toLocaleString()} tokens × $${outputPricePerMillion.toFixed( + 2, + )}/1M tokens = $${outputCost.toFixed(8)} (fallback)`, + totalCalculation: `$${inputCost.toFixed(8)} + $${outputCost.toFixed( + 8, + )} = $${fallbackCost.toFixed(8)} (fallback)`, + }; + + detailedBreakdown.push({ + tokensIn: usage.tokensIn, + tokensOut: usage.tokensOut, + inputCost, + outputCost, + totalCost: fallbackCost, + usageDate: usage.createdAt, + modelKey: `${modelKey} (fallback)`, + inputTokenPrice: prices.input, + outputTokenPrice: prices.output, + pricingEffectiveDate: new Date(), // Use current date for fallback + usageType: usage.usageType, + calculationSteps, + }); + } + } + + return { + totalCost, + costBreakdown: costByType, + detailedBreakdown, + }; + } + + /** + * Helper method to get author activity insights + */ + private async getAuthorActivity( + assignmentAuthors: { userId: string; createdAt: Date }[], + ) { + if (!assignmentAuthors || assignmentAuthors.length === 0) { + return { + totalAuthors: 0, + authors: [], + activityInsights: [], + }; + } + + const authorIds = assignmentAuthors.map( + (author: { userId: string }) => author.userId, + ); + + // Get all assignments by these authors to understand their activity + const authorAssignments = await this.prisma.assignment.findMany({ + where: { + AssignmentAuthor: { + some: { + userId: { + in: authorIds, + }, + }, + }, + }, + include: { + AssignmentAuthor: true, + _count: { + select: { + questions: true, + AIUsage: true, + AssignmentFeedback: true, + }, + }, + }, + }); + + // Get assignment attempt counts separately since it's not a direct relation + const attemptCounts = await this.prisma.assignmentAttempt.groupBy({ + by: ["assignmentId"], + where: { + assignmentId: { + in: authorAssignments.map((a) => a.id), + }, + }, + _count: { + id: true, + }, + }); + + // Get recent activity for these authors (simplified approach) + const validAssignmentIds = authorAssignments + .map((a) => a.id) + .filter( + (id) => id !== null && id !== undefined && typeof id === "number", + ); + + const recentActivity = + validAssignmentIds.length > 0 + ? await this.prisma.assignmentAttempt.findMany({ + where: { + assignmentId: { + in: validAssignmentIds, + }, + }, + select: { + id: true, + assignmentId: true, + createdAt: true, + }, + orderBy: { createdAt: "desc" }, + take: 50, + }) + : []; + + // Analyze author contributions + const authorStats = authorIds.map((authorId) => { + const authoredAssignments = authorAssignments.filter((assignment) => + assignment.AssignmentAuthor.some( + (author) => author.userId === authorId, + ), + ); + + const totalAssignments = authoredAssignments.length; + const totalQuestions = authoredAssignments.reduce( + (sum, assignment) => sum + assignment._count.questions, + 0, + ); + const totalAIUsage = authoredAssignments.reduce( + (sum, assignment) => sum + assignment._count.AIUsage, + 0, + ); + const totalFeedback = authoredAssignments.reduce( + (sum, assignment) => sum + assignment._count.AssignmentFeedback, + 0, + ); + + // Calculate total attempts from attempt counts + const authorAssignmentIds = new Set(authoredAssignments.map((a) => a.id)); + const totalAttempts = attemptCounts + .filter((count) => authorAssignmentIds.has(count.assignmentId)) + .reduce((sum, count) => sum + count._count.id, 0); + + // Get recent activity for this author + const authorRecentActivity = recentActivity.filter((attempt) => + authoredAssignments.some( + (assignment) => assignment.id === attempt.assignmentId, + ), + ); + + return { + userId: authorId, + totalAssignments, + totalQuestions, + totalAttempts, + totalAIUsage, + totalFeedback, + averageAttemptsPerAssignment: + totalAssignments > 0 + ? Math.round(totalAttempts / totalAssignments) + : 0, + averageQuestionsPerAssignment: + totalAssignments > 0 + ? Math.round(totalQuestions / totalAssignments) + : 0, + recentActivityCount: authorRecentActivity.length, + joinedAt: + assignmentAuthors.find( + (author: { userId: string; createdAt: Date }) => + author.userId === authorId, + )?.createdAt || new Date(), + isActiveContributor: totalAssignments >= 3, + activityScore: Math.round( + totalAssignments * 2 + totalQuestions * 0.5 + totalAttempts * 0.1, + ), + }; + }); + + // Sort authors by activity score + authorStats.sort((a, b) => b.activityScore - a.activityScore); + + // Generate insights + const activityInsights = []; + const totalAuthors = authorStats.length; + const activeAuthors = authorStats.filter( + (author) => author.isActiveContributor, + ).length; + const mostActiveAuthor = authorStats[0]; + + if (totalAuthors > 1) { + activityInsights.push( + `This assignment has ${totalAuthors} contributing authors`, + ); + + if (activeAuthors > 0) { + activityInsights.push( + `${activeAuthors} of ${totalAuthors} authors are active contributors (3+ assignments)`, + ); + } + + if (mostActiveAuthor) { + activityInsights.push( + `Most active contributor: ${String(mostActiveAuthor.userId)} with ${ + mostActiveAuthor.totalAssignments + } assignments`, + ); + } + } else if (totalAuthors === 1) { + const singleAuthor = authorStats[0]; + activityInsights.push( + `Single author assignment by ${String(singleAuthor.userId)}`, + ); + if (singleAuthor.totalAssignments > 1) { + activityInsights.push( + `Author has created ${singleAuthor.totalAssignments} total assignments`, + ); + } + } + + return { + totalAuthors: authorStats.length, + authors: authorStats, + activityInsights, + }; + } async cloneAssignment( id: number, @@ -68,9 +796,92 @@ export class AdminService { return { id: newAssignment.id, success: true, + name: newAssignment.name, + type: newAssignment.type, }; } + // Method to get flagged submissions + async getFlaggedSubmissions() { + return this.prisma.regradingRequest.findMany({ + where: { + regradingStatus: "PENDING", + }, + orderBy: { + createdAt: "desc", + }, + }); + } + + // Method to dismiss a flagged submission + async dismissFlaggedSubmission(id: number) { + return this.prisma.regradingRequest.update({ + where: { id }, + data: { + regradingStatus: "REJECTED", + }, + }); + } + + // Method to get regrading requests + async getRegradingRequests() { + return this.prisma.regradingRequest.findMany({ + orderBy: { + createdAt: "desc", + }, + }); + } + + // Method to approve a regrading request + async approveRegradingRequest(id: number, newGrade: number) { + const request = await this.prisma.regradingRequest.findUnique({ + where: { id }, + }); + + if (!request) { + throw new Error(`Regrading request with ID ${id} not found`); + } + + // Update the regrading request status + await this.prisma.regradingRequest.update({ + where: { id }, + data: { + regradingStatus: "APPROVED", + }, + }); + + // Update the assignment attempt grade + await this.prisma.assignmentAttempt.update({ + where: { id: request.attemptId }, + data: { + grade: newGrade / 100, // Convert percentage to decimal + }, + }); + + return { success: true }; + } + + // Method to reject a regrading request + async rejectRegradingRequest(id: number, reason: string) { + const request = await this.prisma.regradingRequest.findUnique({ + where: { id }, + }); + + if (!request) { + throw new Error(`Regrading request with ID ${id} not found`); + } + + // Update the regrading request status + await this.prisma.regradingRequest.update({ + where: { id }, + data: { + regradingStatus: "REJECTED", + regradingReason: reason, + }, + }); + + return { success: true }; + } async addAssignmentToGroup( assignmentId: number, groupId: string, @@ -158,6 +969,8 @@ export class AdminService { return { id: assignment.id, + name: assignment.name, + type: assignment.type, success: true, }; } @@ -190,6 +1003,8 @@ export class AdminService { return { id: result.id, success: true, + name: result.name, + type: result.type, }; } @@ -205,9 +1020,1048 @@ export class AdminService { return { id: result.id, success: true, + name: result.name, + type: result.type, + }; + } + + async getAssignmentAnalytics( + adminSession: UserSession, + page: number, + limit: number, + search?: string, + ) { + const isAdmin = adminSession.role === UserRole.ADMIN; + const skip = (page - 1) * limit; + + // Build where clause based on role and search + const searchCondition = search + ? { + OR: [ + { name: { contains: search, mode: "insensitive" as const } }, + // Check if search term is a number and search by ID + ...(Number.isNaN(Number(search)) + ? [] + : [{ id: { equals: Number(search) } }]), + ], + } + : {}; + + const whereClause = { + ...searchCondition, + ...(isAdmin + ? {} + : { + AssignmentAuthor: { + some: { + userId: adminSession.userId, + }, + }, + }), + }; + + // Get total count for pagination (separate optimized query) + const totalCount = await this.prisma.assignment.count({ + where: whereClause, + }); + + // Get assignments with only essential data for the list view + const assignments = await this.prisma.assignment.findMany({ + where: whereClause, + skip, + take: limit, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + }, + orderBy: { updatedAt: "desc" }, + }); + + if (assignments.length === 0) { + return { + data: [], + pagination: { + total: totalCount, + page, + limit, + totalPages: Math.ceil(totalCount / limit), + }, + }; + } + + const assignmentIds = assignments.map((a) => a.id); + + // Batch fetch all analytics data with optimized queries + const [attemptStats, uniqueLearnersStats, feedbackStats] = + await Promise.all([ + // Get attempt statistics in separate queries for accuracy + Promise.all([ + // Get total attempts count (all attempts) + this.prisma.assignmentAttempt.groupBy({ + by: ["assignmentId"], + where: { + assignmentId: { in: assignmentIds }, + }, + _count: { + id: true, + }, + }), + // Get completed attempts count and average grade (only submitted attempts) + this.prisma.assignmentAttempt.groupBy({ + by: ["assignmentId"], + where: { + assignmentId: { in: assignmentIds }, + submitted: true, + }, + _count: { + id: true, + }, + _avg: { + grade: true, + }, + }), + ]).then(([totalStats, submittedStats]) => { + const totalStatsMap = new Map( + totalStats.map((s) => [s.assignmentId, s._count.id]), + ); + const submittedStatsMap = new Map( + submittedStats.map((s) => [s.assignmentId, s]), + ); + + return { totalStatsMap, submittedStatsMap }; + }), + + // Get unique learner counts in one query - use distinct users per assignment + Promise.all( + assignmentIds.map(async (assignmentId) => { + const uniqueUsers = await this.prisma.assignmentAttempt.findMany({ + where: { assignmentId }, + distinct: ["userId"], + select: { userId: true }, + }); + return { assignmentId, uniqueUsersCount: uniqueUsers.length }; + }), + ), + + // Get feedback statistics in one query + this.prisma.assignmentFeedback.groupBy({ + by: ["assignmentId"], + where: { + assignmentId: { in: assignmentIds }, + assignmentRating: { not: undefined }, + }, + _avg: { + assignmentRating: true, + }, + _count: { + id: true, + }, + }), + ]); + + const { totalStatsMap, submittedStatsMap } = attemptStats; + const uniqueLearnersMap = new Map( + uniqueLearnersStats.map((s) => [s.assignmentId, s.uniqueUsersCount]), + ); + const feedbackMap = new Map(feedbackStats.map((s) => [s.assignmentId, s])); + // const questionMap = new Map(questionStats.map(s => [s.assignmentId, s])); + + const analyticsData = await Promise.all( + assignments.map(async (assignment) => { + const totalAttempts = totalStatsMap.get(assignment.id) || 0; + const submittedData = submittedStatsMap.get(assignment.id); + const completedAttempts = submittedData?._count.id || 0; + const uniqueLearners = uniqueLearnersMap.get(assignment.id) || 0; + const feedback = feedbackMap.get(assignment.id); + // const questions = questionMap.get(assignment.id); // Future use + + const averageGrade = (submittedData?._avg.grade || 0) * 100; // Convert to percentage + const averageRating = feedback?._avg.assignmentRating || 0; + + // Get detailed AI usage data for accurate cost calculation + const aiUsageDetails = await this.prisma.aIUsage.findMany({ + where: { assignmentId: assignment.id }, + select: { + tokensIn: true, + tokensOut: true, + createdAt: true, + usageType: true, + modelKey: true, + }, + }); + + // Calculate accurate costs using historical pricing and actual models + const costData = await this.calculateHistoricalCosts(aiUsageDetails); + const totalCost = costData.totalCost; + + // Generate simple performance insights + const performanceInsights: string[] = []; + if (totalAttempts > 0) { + const completionRate = (completedAttempts / totalAttempts) * 100; + if (completionRate < 70) { + performanceInsights.push( + `Low completion rate (${Math.round( + completionRate, + )}%) - consider reducing difficulty`, + ); + } + if (averageGrade > 85) { + performanceInsights.push( + `High average grade (${Math.round( + averageGrade, + )}%) - learners are doing well`, + ); + } else if (averageGrade < 60) { + performanceInsights.push( + `Low average grade (${Math.round( + averageGrade, + )}%) - may need clearer instructions`, + ); + } + } + + // Accurate cost breakdown based on actual usage types + const costBreakdown = { + grading: Math.round(costData.costBreakdown.grading * 100) / 100, + questionGeneration: + Math.round(costData.costBreakdown.questionGeneration * 100) / 100, + translation: + Math.round(costData.costBreakdown.translation * 100) / 100, + other: Math.round(costData.costBreakdown.other * 100) / 100, + }; + + return { + id: assignment.id, + name: assignment.name, + totalCost, + uniqueLearners, + totalAttempts, + completedAttempts, + averageGrade, + averageRating, + published: assignment.published, + insights: { + questionInsights: [], // Simplified - can be loaded on-demand + performanceInsights, + costBreakdown, + detailedCostBreakdown: costData.detailedBreakdown.map((detail) => ({ + ...detail, + usageDate: detail.usageDate.toISOString(), + pricingEffectiveDate: detail.pricingEffectiveDate.toISOString(), + })), + }, + }; + }), + ); + + return { + data: analyticsData, + pagination: { + total: totalCount, + page, + limit, + totalPages: Math.ceil(totalCount / limit), + }, }; } + async getDashboardStats( + adminSession: UserSession & { userId?: string }, + filters?: DashboardFilters, + ) { + const isAdmin = adminSession.role === UserRole.ADMIN; + + // Build base where clauses for different queries with filters + const assignmentWhere: any = isAdmin + ? {} + : { + AssignmentAuthor: { + some: { + userId: adminSession.userId, + }, + }, + }; + + // Apply assignment filters + if (filters?.assignmentId) { + assignmentWhere.id = filters.assignmentId; + } + if (filters?.assignmentName) { + assignmentWhere.name = { + contains: filters.assignmentName, + mode: "insensitive", + }; + } + + // Build date filter for time-based queries + const dateFilter: any = {}; + if (filters?.startDate || filters?.endDate) { + if (filters.startDate) { + dateFilter.gte = new Date(filters.startDate); + } + if (filters.endDate) { + dateFilter.lte = new Date(filters.endDate); + } + } + + // For non-admins, we need to get assignment IDs first for attempt/feedback queries + let assignmentIds: number[] = []; + if (!isAdmin) { + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { id: true }, + }); + assignmentIds = assignments.map((a) => a.id); + } else if (filters?.assignmentId || filters?.assignmentName) { + // For admins with assignment filters, also get the filtered assignment IDs + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { id: true }, + }); + assignmentIds = assignments.map((a) => a.id); + } + + // Optimized parallel queries + const [ + totalAssignments, + publishedAssignments, + attemptStats, + feedbackCount, + reportCounts, + recentAttempts, + learnerCount, + aiUsageStats, + averageAssignmentRating, + ] = await Promise.all([ + // Total assignments + this.prisma.assignment.count({ where: assignmentWhere }), + + // Published assignments + this.prisma.assignment.count({ + where: { ...assignmentWhere, published: true }, + }), + + // Attempt statistics (total attempts + unique users in one query) + isAdmin || assignmentIds.length > 0 + ? this.prisma.assignmentAttempt + .aggregate({ + where: { + ...(isAdmin ? {} : { assignmentId: { in: assignmentIds } }), + ...(assignmentIds.length > 0 && isAdmin + ? { assignmentId: { in: assignmentIds } } + : {}), + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + ...(filters?.userId + ? { + userId: { contains: filters.userId, mode: "insensitive" }, + } + : {}), + }, + _count: { id: true }, + }) + .then(async (totalAttempts) => { + const uniqueUsers = await this.prisma.assignmentAttempt.groupBy({ + by: ["userId"], + where: { + ...(isAdmin ? {} : { assignmentId: { in: assignmentIds } }), + ...(assignmentIds.length > 0 && isAdmin + ? { assignmentId: { in: assignmentIds } } + : {}), + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + ...(filters?.userId + ? { + userId: { + contains: filters.userId, + mode: "insensitive", + }, + } + : {}), + }, + }); + return { + totalAttempts: totalAttempts._count.id, + totalUsers: uniqueUsers.length, + }; + }) + : Promise.resolve({ totalAttempts: 0, totalUsers: 0 }), + + // Total feedback + isAdmin || assignmentIds.length > 0 + ? this.prisma.assignmentFeedback.count({ + where: { + ...(isAdmin ? {} : { assignmentId: { in: assignmentIds } }), + ...(assignmentIds.length > 0 && isAdmin + ? { assignmentId: { in: assignmentIds } } + : {}), + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + ...(filters?.userId + ? { userId: { contains: filters.userId, mode: "insensitive" } } + : {}), + }, + }) + : 0, + + // Report counts (admin only) + isAdmin + ? this.prisma.report + .aggregate({ + _count: { id: true }, + where: { + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + ...(filters?.userId + ? { + userId: { contains: filters.userId, mode: "insensitive" }, + } + : {}), + }, + }) + .then(async (total) => { + const open = await this.prisma.report.count({ + where: { + status: "OPEN", + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + ...(filters?.userId + ? { + userId: { + contains: filters.userId, + mode: "insensitive", + }, + } + : {}), + }, + }); + return { totalReports: total._count.id, openReports: open }; + }) + : { totalReports: 0, openReports: 0 }, + + // Recent activity with assignment names in one query + isAdmin || assignmentIds.length > 0 + ? this.prisma.assignmentAttempt.findMany({ + where: { + ...(isAdmin ? {} : { assignmentId: { in: assignmentIds } }), + ...(assignmentIds.length > 0 && isAdmin + ? { assignmentId: { in: assignmentIds } } + : {}), + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + ...(filters?.userId + ? { userId: { contains: filters.userId, mode: "insensitive" } } + : {}), + }, + take: 10, + orderBy: { createdAt: "desc" }, + select: { + id: true, + userId: true, + submitted: true, + grade: true, + createdAt: true, + assignmentId: true, + }, + }) + : [], + + // total number of unique leaners + isAdmin || assignmentIds.length > 0 + ? this.prisma.assignmentAttempt + .groupBy({ + by: ["userId"], + where: { + ...(isAdmin ? {} : { assignmentId: { in: assignmentIds } }), + ...(assignmentIds.length > 0 && isAdmin + ? { assignmentId: { in: assignmentIds } } + : {}), + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + ...(filters?.userId + ? { + userId: { contains: filters.userId, mode: "insensitive" }, + } + : {}), + }, + }) + .then((users) => users.length) + : 0, + + // AI usage data for cost calculation + isAdmin || assignmentIds.length > 0 + ? this.prisma.aIUsage.findMany({ + where: { + ...(isAdmin + ? {} + : { + assignment: { + AssignmentAuthor: { + some: { userId: adminSession.userId }, + }, + }, + }), + ...(assignmentIds.length > 0 && isAdmin + ? { assignmentId: { in: assignmentIds } } + : {}), + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + }, + select: { + tokensIn: true, + tokensOut: true, + createdAt: true, + usageType: true, + modelKey: true, + }, + }) + : [], + + // Average Assignment Rating for all assignments + isAdmin || assignmentIds.length > 0 + ? this.prisma.assignmentFeedback.aggregate({ + where: { + ...(isAdmin ? {} : { assignmentId: { in: assignmentIds } }), + ...(assignmentIds.length > 0 && isAdmin + ? { assignmentId: { in: assignmentIds } } + : {}), + ...(Object.keys(dateFilter).length > 0 + ? { createdAt: dateFilter } + : {}), + ...(filters?.userId + ? { userId: { contains: filters.userId, mode: "insensitive" } } + : {}), + }, + _avg: { assignmentRating: true }, + }) + : { _avg: { assignmentRating: 0 } }, + ]); + + // Calculate total cost using historical pricing + const costData = await this.calculateHistoricalCosts(aiUsageStats); + const totalCost = costData.totalCost; + + // Get assignment names for recent attempts + const assignmentNames = new Map(); + if (recentAttempts.length > 0) { + const uniqueAssignmentIds = [ + ...new Set(recentAttempts.map((a: any) => a.assignmentId)), + ]; + const assignments = await this.prisma.assignment.findMany({ + where: { id: { in: uniqueAssignmentIds } }, + select: { id: true, name: true }, + }); + for (const assignment of assignments) { + assignmentNames.set(assignment.id, assignment.name); + } + } + + return { + totalAssignments, + publishedAssignments, + totalReports: reportCounts.totalReports, + openReports: reportCounts.openReports, + totalFeedback: feedbackCount, + totalLearners: learnerCount, + totalAttempts: attemptStats.totalAttempts, + totalUsers: attemptStats.totalUsers, + averageAssignmentRating: + averageAssignmentRating._avg.assignmentRating || 0, + totalCost: Math.round(totalCost * 100) / 100, + costBreakdown: { + grading: Math.round(costData.costBreakdown.grading * 100) / 100, + questionGeneration: + Math.round(costData.costBreakdown.questionGeneration * 100) / 100, + translation: Math.round(costData.costBreakdown.translation * 100) / 100, + other: Math.round(costData.costBreakdown.other * 100) / 100, + }, + userRole: isAdmin ? ("admin" as const) : ("author" as const), + recentActivity: recentAttempts.map((attempt: any) => ({ + id: attempt.id, + assignmentName: assignmentNames.get(attempt.assignmentId) ?? "Unknown", + userId: attempt.userId, + submitted: attempt.submitted, + grade: attempt.grade, + createdAt: attempt.createdAt, + })), + }; + } + + async getDetailedAssignmentInsights( + adminSession: UserSession, + assignmentId: number, + ) { + try { + // Check cache first + const cachedInsights = this.getCachedInsights(assignmentId); + if (cachedInsights) { + return cachedInsights; + } + // Validate assignmentId + if (!assignmentId || assignmentId <= 0) { + throw new Error(`Invalid assignment ID: ${assignmentId}`); + } + + const isAdmin = adminSession.role === UserRole.ADMIN; + + // Check if user has access to this assignment + const assignment = await this.prisma.assignment.findFirst({ + where: { + id: assignmentId, + ...(isAdmin + ? {} + : { + AssignmentAuthor: { + some: { + userId: adminSession.userId, + }, + }, + }), + }, + include: { + questions: { + where: { isDeleted: false }, + include: { + translations: true, + variants: { + where: { isDeleted: false }, + }, + }, + }, + AIUsage: true, + AssignmentFeedback: true, + Report: true, + AssignmentAuthor: true, + }, + }); + + if (!assignment) { + throw new NotFoundException( + `Assignment with ID ${assignmentId} not found or access denied`, + ); + } + + // Get assignment attempts count and basic stats with error handling + let totalAttempts = 0; + let submittedAttempts = 0; + let calculatedAverageGrade = 0; + + try { + // Get simple counts first + totalAttempts = await this.prisma.assignmentAttempt.count({ + where: { assignmentId }, + }); + + submittedAttempts = await this.prisma.assignmentAttempt.count({ + where: { assignmentId, submitted: true }, + }); + + // Get average grade with simple aggregation + const gradeAvg = await this.prisma.assignmentAttempt.aggregate({ + where: { assignmentId, submitted: true }, + _avg: { grade: true }, + }); + calculatedAverageGrade = (gradeAvg._avg.grade || 0) * 100; // Convert to percentage + } catch (error) { + this.logger.error( + `Error fetching attempt statistics for assignment ${assignmentId}:`, + error, + ); + // Use fallback values (already initialized above) + } + + // Process question insights in batches to prevent connection pool exhaustion + const questionInsights = []; + const batchSize = 3; // Process 3 questions at a time to avoid connection pool issues + + for ( + let index = 0; + index < assignment.questions.length; + index += batchSize + ) { + const batch = assignment.questions.slice(index, index + batchSize); + + try { + const batchResults = await Promise.all( + batch.map(async (question) => { + let totalResponses = 0; + let correctCount = 0; + let averagePoints = 0; + + try { + // Get total response count for this question + totalResponses = await this.prisma.questionResponse.count({ + where: { + questionId: question.id, + assignmentAttempt: { assignmentId }, + }, + }); + + if (totalResponses > 0) { + // Get count of correct responses using aggregate + correctCount = await this.prisma.questionResponse.count({ + where: { + questionId: question.id, + assignmentAttempt: { assignmentId }, + points: question.totalPoints, + }, + }); + + // Get average points using aggregate + const pointsAvg = + await this.prisma.questionResponse.aggregate({ + where: { + questionId: question.id, + assignmentAttempt: { assignmentId }, + }, + _avg: { points: true }, + }); + averagePoints = pointsAvg._avg.points || 0; + } + } catch (error) { + this.logger.error( + `Error fetching response statistics for question ${question.id}:`, + error, + ); + // Use fallback values (already initialized above) + } + + const correctPercentage = + totalResponses > 0 ? (correctCount / totalResponses) * 100 : 0; + + let insight = `${Math.round( + correctPercentage, + )}% of learners answered correctly`; + if (correctPercentage < 50) { + insight += ` - consider reviewing this question`; + } + + return { + id: question.id, + question: question.question, + type: question.type, + totalPoints: question.totalPoints, + correctPercentage, + averagePoints, + responseCount: totalResponses, + insight, + variants: question.variants.length, + translations: question.translations.map((t) => ({ + languageCode: t.languageCode, + })), + }; + }), + ); + questionInsights.push(...batchResults); + + // Add a small delay between batches to prevent connection pool exhaustion + if (index + batchSize < assignment.questions.length) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } catch (error) { + this.logger.error( + `Error processing question batch starting at index ${index}:`, + error, + ); + // Continue with empty results for this batch to prevent total failure + const fallbackResults = batch.map((question) => ({ + id: question.id, + question: question.question, + type: question.type, + totalPoints: question.totalPoints, + correctPercentage: 0, + averagePoints: 0, + responseCount: 0, + insight: "Data unavailable due to processing error", + variants: question.variants?.length || 0, + translations: + question.translations?.map((t) => ({ + languageCode: t.languageCode, + })) || [], + })); + questionInsights.push(...fallbackResults); + } + } + + // Calculate analytics + const uniqueLearners = await this.prisma.assignmentAttempt.groupBy({ + by: ["userId"], + where: { assignmentId }, + }); + + // Use the calculated stats instead of processing attempts array + const completedAttempts = submittedAttempts; + const averageGrade = calculatedAverageGrade; + + // Calculate total cost using historical pricing + const aiUsageRecords = assignment.AIUsage.map((usage) => ({ + tokensIn: usage.tokensIn, + tokensOut: usage.tokensOut, + createdAt: usage.createdAt, + usageType: usage.usageType, + modelKey: usage.modelKey, + })); + + const costData = await this.calculateHistoricalCosts(aiUsageRecords); + const totalCost = costData.totalCost; + + // Get author information and activity analysis + const authorActivity = await this.getAuthorActivity( + assignment.AssignmentAuthor, + ); + + const aiUsageWithCost = assignment.AIUsage.map((usage, index) => { + const detailedCost = costData.detailedBreakdown[index] || { + totalCost: 0, + inputCost: 0, + outputCost: 0, + modelKey: "unknown", + inputTokenPrice: 0, + outputTokenPrice: 0, + pricingEffectiveDate: new Date(), + calculationSteps: { + inputCalculation: "0 tokens × $0 = $0 (missing)", + outputCalculation: "0 tokens × $0 = $0 (missing)", + totalCalculation: "$0 + $0 = $0 (missing)", + }, + }; + + return { + usageType: usage.usageType, + tokensIn: usage.tokensIn, + tokensOut: usage.tokensOut, + usageCount: usage.usageCount, + inputCost: detailedCost.inputCost, + outputCost: detailedCost.outputCost, + totalCost: detailedCost.totalCost, + modelUsed: detailedCost.modelKey, + inputTokenPrice: detailedCost.inputTokenPrice, + outputTokenPrice: detailedCost.outputTokenPrice, + pricingEffectiveDate: detailedCost.pricingEffectiveDate.toISOString(), + calculationSteps: detailedCost.calculationSteps, + createdAt: usage.createdAt.toISOString(), + }; + }); + + // Calculate average rating + const ratings = assignment.AssignmentFeedback.map( + (f) => f.assignmentRating, + ).filter((r) => r !== null); + const averageRating = + ratings.length > 0 + ? ratings.reduce((sum, r) => sum + r, 0) / ratings.length + : 0; + + // Calculate total points for assignment + const totalPoints = assignment.questions.reduce( + (sum, q) => sum + q.totalPoints, + 0, + ); + + // Cost breakdown from historical pricing data + const costBreakdown = { + grading: Math.round(costData.costBreakdown.grading * 100) / 100, + questionGeneration: + Math.round(costData.costBreakdown.questionGeneration * 100) / 100, + translation: Math.round(costData.costBreakdown.translation * 100) / 100, + other: Math.round(costData.costBreakdown.other * 100) / 100, + }; + + // Generate performance insights + const performanceInsights: string[] = []; + if (completedAttempts > 0 && totalAttempts > 0) { + const completionRate = (completedAttempts / totalAttempts) * 100; + if (completionRate < 70) { + performanceInsights.push( + `Low completion rate (${Math.round( + completionRate, + )}%) - consider reducing difficulty`, + ); + } + if (averageGrade > 85) { + performanceInsights.push( + `High average grade (${Math.round( + averageGrade, + )}%) - learners are doing well`, + ); + } + if (averageGrade < 60) { + performanceInsights.push( + `Low average grade (${Math.round( + averageGrade, + )}%) - may need clearer instructions`, + ); + } + } + + const insights = { + assignment: { + id: assignment.id, + name: assignment.name, + type: assignment.type, + published: assignment.published, + introduction: assignment.introduction, + instructions: assignment.instructions, + timeEstimateMinutes: assignment.timeEstimateMinutes, + allotedTimeMinutes: assignment.allotedTimeMinutes, + passingGrade: assignment.passingGrade, + createdAt: assignment.updatedAt.toISOString(), + updatedAt: assignment.updatedAt.toISOString(), + totalPoints, + }, + analytics: { + totalCost, + uniqueLearners: uniqueLearners.length, + totalAttempts, + completedAttempts, + averageGrade, + averageRating, + costBreakdown, + performanceInsights, + }, + questions: questionInsights, + attempts: await this.getAssignmentAttempts(assignmentId), + feedback: assignment.AssignmentFeedback.map((feedback) => ({ + id: feedback.id, + userId: feedback.userId, + assignmentRating: feedback.assignmentRating, + aiGradingRating: feedback.aiGradingRating, + aiFeedbackRating: feedback.aiFeedbackRating, + comments: feedback.comments, + createdAt: feedback.createdAt.toISOString(), + })), + reports: assignment.Report.map((report) => ({ + id: report.id, + issueType: report.issueType, + description: report.description, + status: report.status, + createdAt: report.createdAt.toISOString(), + })), + aiUsage: aiUsageWithCost, + costCalculationDetails: { + totalCost: Math.round(totalCost * 100) / 100, + breakdown: costData.detailedBreakdown.map((detail) => ({ + usageType: detail.usageType || "Unknown", + tokensIn: detail.tokensIn, + tokensOut: detail.tokensOut, + modelUsed: detail.modelKey, + inputTokenPrice: detail.inputTokenPrice, + outputTokenPrice: detail.outputTokenPrice, + inputCost: Math.round(detail.inputCost * 100_000_000) / 100_000_000, // 8 decimal places + outputCost: + Math.round(detail.outputCost * 100_000_000) / 100_000_000, + totalCost: Math.round(detail.totalCost * 100_000_000) / 100_000_000, + pricingEffectiveDate: detail.pricingEffectiveDate.toISOString(), + usageDate: detail.usageDate.toISOString(), + calculationSteps: detail.calculationSteps, + })), + summary: { + totalInputTokens: costData.detailedBreakdown.reduce( + (sum, d) => sum + d.tokensIn, + 0, + ), + totalOutputTokens: costData.detailedBreakdown.reduce( + (sum, d) => sum + d.tokensOut, + 0, + ), + totalInputCost: + Math.round( + costData.detailedBreakdown.reduce( + (sum, d) => sum + d.inputCost, + 0, + ) * 100_000_000, + ) / 100_000_000, + totalOutputCost: + Math.round( + costData.detailedBreakdown.reduce( + (sum, d) => sum + d.outputCost, + 0, + ) * 100_000_000, + ) / 100_000_000, + averageInputPrice: + costData.detailedBreakdown.length > 0 + ? costData.detailedBreakdown.reduce( + (sum, d) => sum + d.inputTokenPrice, + 0, + ) / costData.detailedBreakdown.length + : 0, + averageOutputPrice: + costData.detailedBreakdown.length > 0 + ? costData.detailedBreakdown.reduce( + (sum, d) => sum + d.outputTokenPrice, + 0, + ) / costData.detailedBreakdown.length + : 0, + // eslint-disable-next-line unicorn/no-array-reduce + modelDistribution: costData.detailedBreakdown.reduce( + (accumulator: Record, detail) => { + accumulator[detail.modelKey] = + (accumulator[detail.modelKey] || 0) + detail.totalCost; + return accumulator; + }, + {} as Record, + ), + usageTypeDistribution: { + grading: Math.round(costData.costBreakdown.grading * 100) / 100, + questionGeneration: + Math.round(costData.costBreakdown.questionGeneration * 100) / + 100, + translation: + Math.round(costData.costBreakdown.translation * 100) / 100, + other: Math.round(costData.costBreakdown.other * 100) / 100, + }, + }, + }, + authorActivity: { + totalAuthors: authorActivity.totalAuthors, + authors: authorActivity.authors, + activityInsights: authorActivity.activityInsights, + }, + }; + + // Cache the result before returning + this.setCachedInsights(assignmentId, insights); + + return insights; + } catch (error) { + this.logger.error( + `Error getting detailed assignment insights for assignment ${assignmentId}:`, + error, + ); + + // Return a safe fallback response + return { + insights: { + questionInsights: [], + performanceInsights: [ + "Unable to load detailed insights due to a data processing error. Please try again later.", + ], + costBreakdown: { + grading: 0, + questionGeneration: 0, + translation: 0, + other: 0, + }, + }, + authorActivity: { + totalAuthors: 0, + authors: [], + activityInsights: ["Author activity data is currently unavailable."], + }, + }; + } + } + async removeAssignment(id: number): Promise { await this.prisma.questionResponse.deleteMany({ where: { assignmentAttempt: { assignmentId: id } }, @@ -251,7 +2105,7 @@ export class AdminService { const assignmentExists = await this.prisma.assignment.findUnique({ where: { id }, - select: { id: true }, + select: { id: true, name: true, type: true }, }); if (!assignmentExists) { @@ -265,6 +2119,592 @@ export class AdminService { return { id: id, success: true, + name: assignmentExists.name || "", + type: assignmentExists.type || "AI_GRADED", + }; + } + + async executeQuickAction( + adminSession: { email: string; role: UserRole; userId?: string }, + action: string, + limit = 10, + ) { + const isAdmin = adminSession.role === UserRole.ADMIN; + + // Build base where clauses for different queries + const assignmentWhere: any = isAdmin + ? {} + : { + AssignmentAuthor: { + some: { + userId: adminSession.userId, + }, + }, + }; + + switch (action) { + case "top-assignments-by-cost": { + return await this.getTopAssignmentsByCost(assignmentWhere, limit); + } + + case "top-assignments-by-attempts": { + return await this.getTopAssignmentsByAttempts(assignmentWhere, limit); + } + + case "top-assignments-by-learners": { + return await this.getTopAssignmentsByLearners(assignmentWhere, limit); + } + + case "most-expensive-assignments": { + return await this.getMostExpensiveAssignments(assignmentWhere, limit); + } + + case "assignments-with-most-reports": { + return await this.getAssignmentsWithMostReports(assignmentWhere, limit); + } + + case "highest-rated-assignments": { + return await this.getHighestRatedAssignments(assignmentWhere, limit); + } + + case "assignments-with-lowest-ratings": { + return await this.getAssignmentsWithLowestRatings( + assignmentWhere, + limit, + ); + } + + case "recent-high-activity": { + return await this.getRecentHighActivityAssignments( + assignmentWhere, + limit, + ); + } + + case "cost-per-learner-analysis": { + return await this.getCostPerLearnerAnalysis(assignmentWhere, limit); + } + + case "completion-rate-analysis": { + return await this.getCompletionRateAnalysis(assignmentWhere, limit); + } + + default: { + throw new Error(`Unknown quick action: ${action}`); + } + } + } + + private async getTopAssignmentsByCost(assignmentWhere: any, limit: number) { + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + AIUsage: { + select: { + tokensIn: true, + tokensOut: true, + createdAt: true, + usageType: true, + modelKey: true, + }, + }, + AssignmentFeedback: { + select: { id: true }, + }, + }, + take: Math.min(limit * 10, 1000), // Get more records for sorting, but cap at 1000 + }); + + const assignmentsWithCost = await Promise.all( + assignments.map(async (assignment) => { + const costData = await this.calculateHistoricalCosts( + assignment.AIUsage, + ); + + // Get attempt count separately + const attemptCount = await this.prisma.assignmentAttempt.count({ + where: { assignmentId: assignment.id }, + }); + + return { + id: assignment.id, + name: assignment.name, + totalCost: costData.totalCost, + costBreakdown: costData.costBreakdown, + attempts: attemptCount, + feedback: assignment.AssignmentFeedback.length, + published: assignment.published, + createdAt: assignment.updatedAt, + }; + }), + ); + + return { + title: `Top ${limit} Assignments by AI Cost`, + data: assignmentsWithCost + .sort((a, b) => b.totalCost - a.totalCost) + .slice(0, limit), + }; + } + + private async getTopAssignmentsByAttempts( + assignmentWhere: any, + limit: number, + ) { + // First get assignments with basic info + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + AssignmentFeedback: { select: { id: true } }, + }, + take: Math.min(limit * 10, 1000), // Get more records for sorting, but cap at 1000 + }); + + // Get attempt data for each assignment + const assignmentsWithAttempts = await Promise.all( + assignments.map(async (assignment) => { + const attempts = await this.prisma.assignmentAttempt.findMany({ + where: { assignmentId: assignment.id }, + select: { + userId: true, + submitted: true, + grade: true, + }, + }); + + const submittedAttempts = attempts.filter((a) => a.submitted).length; + const averageGrade = + attempts.length > 0 + ? attempts.reduce((sum, a) => sum + (a.grade || 0), 0) / + attempts.length + : 0; + + return { + id: assignment.id, + name: assignment.name, + totalAttempts: attempts.length, + submittedAttempts, + uniqueUsers: new Set(attempts.map((a) => a.userId)).size, + averageGrade: averageGrade, + feedback: assignment.AssignmentFeedback.length, + published: assignment.published, + createdAt: assignment.updatedAt, + }; + }), + ); + + return { + title: `Top ${limit} Assignments by Attempts`, + data: assignmentsWithAttempts + .sort((a, b) => b.totalAttempts - a.totalAttempts) + .slice(0, limit), + }; + } + + private async getTopAssignmentsByLearners( + assignmentWhere: any, + limit: number, + ) { + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + AssignmentFeedback: { select: { id: true } }, + }, + take: Math.min(limit * 10, 1000), // Get more records for sorting, but cap at 1000 + }); + + const assignmentsWithLearnerCount = await Promise.all( + assignments.map(async (assignment) => { + const attempts = await this.prisma.assignmentAttempt.findMany({ + where: { assignmentId: assignment.id }, + select: { + userId: true, + submitted: true, + }, + }); + + const uniqueLearners = new Set(attempts.map((a) => a.userId)).size; + const completedLearners = new Set( + attempts.filter((a) => a.submitted).map((a) => a.userId), + ).size; + + return { + id: assignment.id, + name: assignment.name, + uniqueLearners, + completedLearners, + totalAttempts: attempts.length, + completionRate: + uniqueLearners > 0 ? (completedLearners / uniqueLearners) * 100 : 0, + feedback: assignment.AssignmentFeedback.length, + published: assignment.published, + createdAt: assignment.updatedAt, + }; + }), + ); + + return { + title: `Top ${limit} Assignments by Unique Learners`, + data: assignmentsWithLearnerCount + .sort((a, b) => b.uniqueLearners - a.uniqueLearners) + .slice(0, limit), + }; + } + + private async getMostExpensiveAssignments( + assignmentWhere: any, + limit: number, + ) { + return await this.getTopAssignmentsByCost(assignmentWhere, limit); + } + + private async getAssignmentsWithMostReports( + assignmentWhere: any, + limit: number, + ) { + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + Report: { + select: { + status: true, + issueType: true, + createdAt: true, + }, + }, + AssignmentFeedback: { select: { id: true } }, + }, + take: Math.min(limit * 10, 1000), // Get more records for sorting, but cap at 1000 + }); + + // Get assignments with report data and sort by report count + const assignmentsWithReports = await Promise.all( + assignments.map(async (assignment) => { + const attemptCount = await this.prisma.assignmentAttempt.count({ + where: { assignmentId: assignment.id }, + }); + + const openReports = assignment.Report.filter( + (r: any) => r.status === "OPEN", + ).length; + const recentReports = assignment.Report.filter( + (r: any) => + new Date(r.createdAt) > + new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), + ).length; + + return { + id: assignment.id, + name: assignment.name, + totalReports: assignment.Report.length, + openReports, + recentReports, + attempts: attemptCount, + feedback: assignment.AssignmentFeedback.length, + published: assignment.published, + createdAt: assignment.updatedAt, + }; + }), + ); + + return { + title: `Top ${limit} Assignments with Most Reports`, + data: assignmentsWithReports + .sort((a, b) => b.totalReports - a.totalReports) + .slice(0, limit), + }; + } + + private async getHighestRatedAssignments( + assignmentWhere: any, + limit: number, + ) { + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + AssignmentFeedback: { + select: { + assignmentRating: true, + aiGradingRating: true, + createdAt: true, + }, + }, + }, + take: Math.min(limit * 10, 1000), // Get more records for sorting, but cap at 1000 + }); + + const assignmentsWithRatings = await Promise.all( + assignments.map(async (assignment) => { + const attemptCount = await this.prisma.assignmentAttempt.count({ + where: { assignmentId: assignment.id }, + }); + + const ratings = assignment.AssignmentFeedback.map( + (f: any) => f.assignmentRating, + ).filter((r: any) => r !== null && r !== undefined) as number[]; + + const averageRating = + ratings.length > 0 + ? ratings.reduce((sum, rating) => sum + rating, 0) / ratings.length + : 0; + + const aiRatings = assignment.AssignmentFeedback.map( + (f: any) => f.aiGradingRating, + ).filter((r: any) => r !== null && r !== undefined) as number[]; + + const averageAiRating = + aiRatings.length > 0 + ? aiRatings.reduce((sum, rating) => sum + rating, 0) / + aiRatings.length + : 0; + + return { + id: assignment.id, + name: assignment.name, + averageRating, + averageAiRating, + totalRatings: ratings.length, + attempts: attemptCount, + feedback: assignment.AssignmentFeedback.length, + published: assignment.published, + createdAt: assignment.updatedAt, + }; + }), + ); + + return { + title: `Top ${limit} Highest Rated Assignments`, + data: assignmentsWithRatings + .filter((a) => a.totalRatings > 0) + .sort((a, b) => b.averageRating - a.averageRating) + .slice(0, limit), + }; + } + + private async getAssignmentsWithLowestRatings( + assignmentWhere: any, + limit: number, + ) { + const result = await this.getHighestRatedAssignments( + assignmentWhere, + limit * 2, + ); + return { + title: `${limit} Assignments with Lowest Ratings`, + data: result.data + .sort((a, b) => a.averageRating - b.averageRating) + .slice(0, limit), + }; + } + + private async getRecentHighActivityAssignments( + assignmentWhere: any, + limit: number, + ) { + const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + + // First, get assignments that have recent attempts + const assignmentIds = await this.prisma.assignmentAttempt.findMany({ + where: { + createdAt: { gte: sevenDaysAgo }, + }, + select: { assignmentId: true }, + distinct: ["assignmentId"], + }); + + const assignments = await this.prisma.assignment.findMany({ + where: { + ...assignmentWhere, + id: { in: assignmentIds.map((a) => a.assignmentId) }, + }, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + AssignmentFeedback: { select: { id: true } }, + }, + take: Math.min(limit * 10, 1000), // Get more records for sorting, but cap at 1000 + }); + + const assignmentsWithActivity = await Promise.all( + assignments.map(async (assignment) => { + const recentAttempts = await this.prisma.assignmentAttempt.findMany({ + where: { + assignmentId: assignment.id, + createdAt: { gte: sevenDaysAgo }, + }, + select: { + userId: true, + submitted: true, + createdAt: true, + }, + }); + + const totalAttempts = await this.prisma.assignmentAttempt.count({ + where: { assignmentId: assignment.id }, + }); + + const uniqueRecentUsers = new Set( + recentAttempts.map((a: any) => a.userId), + ).size; + const recentCompletions = recentAttempts.filter( + (a: any) => a.submitted, + ).length; + + return { + id: assignment.id, + name: assignment.name, + recentAttempts: recentAttempts.length, + uniqueRecentUsers, + recentCompletions, + totalAttempts, + feedback: assignment.AssignmentFeedback.length, + published: assignment.published, + createdAt: assignment.updatedAt, + }; + }), + ); + + return { + title: `${limit} Assignments with Highest Recent Activity (7 days)`, + data: assignmentsWithActivity + .sort((a, b) => b.recentAttempts - a.recentAttempts) + .slice(0, limit), + }; + } + + private async getCostPerLearnerAnalysis(assignmentWhere: any, limit: number) { + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + AIUsage: { + select: { + tokensIn: true, + tokensOut: true, + createdAt: true, + usageType: true, + modelKey: true, + }, + }, + }, + take: Math.min(limit * 10, 1000), // Get more records for sorting, but cap at 1000 + }); + + const assignmentsWithCostPerLearner = await Promise.all( + assignments.map(async (assignment) => { + const costData = await this.calculateHistoricalCosts( + assignment.AIUsage, + ); + + const attempts = await this.prisma.assignmentAttempt.findMany({ + where: { assignmentId: assignment.id }, + select: { + userId: true, + submitted: true, + }, + }); + + const uniqueLearners = new Set(attempts.map((a: any) => a.userId)).size; + const costPerLearner = + uniqueLearners > 0 ? costData.totalCost / uniqueLearners : 0; + + return { + id: assignment.id, + name: assignment.name, + totalCost: costData.totalCost, + uniqueLearners, + costPerLearner, + totalAttempts: attempts.length, + published: assignment.published, + createdAt: assignment.updatedAt, + }; + }), + ); + + return { + title: `${limit} Assignments - Cost Per Learner Analysis`, + data: assignmentsWithCostPerLearner + .filter((a) => a.uniqueLearners > 0) + .sort((a, b) => b.costPerLearner - a.costPerLearner) + .slice(0, limit), + }; + } + + private async getCompletionRateAnalysis(assignmentWhere: any, limit: number) { + const assignments = await this.prisma.assignment.findMany({ + where: assignmentWhere, + select: { + id: true, + name: true, + published: true, + updatedAt: true, + AssignmentFeedback: { select: { id: true } }, + }, + take: Math.min(limit * 10, 1000), // Get more records for sorting, but cap at 1000 + }); + + const assignmentsWithCompletionRate = await Promise.all( + assignments.map(async (assignment) => { + const attempts = await this.prisma.assignmentAttempt.findMany({ + where: { assignmentId: assignment.id }, + select: { + userId: true, + submitted: true, + }, + }); + + const uniqueUsers = new Set(attempts.map((a: any) => a.userId)).size; + const completedUsers = new Set( + attempts.filter((a: any) => a.submitted).map((a: any) => a.userId), + ).size; + const completionRate = + uniqueUsers > 0 ? (completedUsers / uniqueUsers) * 100 : 0; + + return { + id: assignment.id, + name: assignment.name, + uniqueUsers, + completedUsers, + totalAttempts: attempts.length, + completionRate, + feedback: assignment.AssignmentFeedback.length, + published: assignment.published, + createdAt: assignment.updatedAt, + }; + }), + ); + + return { + title: `${limit} Assignments - Completion Rate Analysis`, + data: assignmentsWithCompletionRate + .filter((a) => a.uniqueUsers > 0) + .sort((a, b) => b.completionRate - a.completionRate) + .slice(0, limit), }; } } diff --git a/apps/api/src/api/admin/controllers/admin-dashboard.controller.ts b/apps/api/src/api/admin/controllers/admin-dashboard.controller.ts new file mode 100644 index 00000000..67c2b244 --- /dev/null +++ b/apps/api/src/api/admin/controllers/admin-dashboard.controller.ts @@ -0,0 +1,268 @@ +import { + Controller, + DefaultValuePipe, + Get, + Injectable, + Param, + ParseIntPipe, + Post, + Query, + Req, + UseGuards, +} from "@nestjs/common"; +import { + ApiBearerAuth, + ApiOperation, + ApiQuery, + ApiResponse, + ApiTags, +} from "@nestjs/swagger"; +import { AdminGuard } from "src/auth/guards/admin.guard"; +import { + UserRole, + UserSessionRequest, +} from "src/auth/interfaces/user.session.interface"; +import { Roles } from "src/auth/role/roles.global.guard"; +import { ScheduledTasksService } from "../../scheduled-tasks/services/scheduled-tasks.service"; +import { AdminService } from "../admin.service"; + +interface AdminSessionRequest extends Request { + adminSession: { + email: string; + role: UserRole; + sessionToken: string; + }; +} + +interface AssignmentAnalyticsResponse { + data: Array<{ + id: number; + name: string; + totalCost: number; + uniqueLearners: number; + totalAttempts: number; + completedAttempts: number; + averageGrade: number; + averageRating: number; + published: boolean; + insights: { + questionInsights: Array<{ + questionId: number; + questionText: string; + correctPercentage: number; + firstAttemptSuccessRate: number; + avgPointsEarned: number; + maxPoints: number; + insight: string; + }>; + performanceInsights: string[]; + costBreakdown: { + grading: number; + questionGeneration: number; + translation: number; + other: number; + }; + detailedCostBreakdown?: Array<{ + tokensIn: number; + tokensOut: number; + inputCost: number; + outputCost: number; + totalCost: number; + usageDate: string; + modelKey: string; + inputTokenPrice: number; + outputTokenPrice: number; + pricingEffectiveDate: string; + usageType?: string; + calculationSteps: { + inputCalculation: string; + outputCalculation: string; + totalCalculation: string; + }; + }>; + }; + }>; + pagination: { + total: number; + page: number; + limit: number; + totalPages: number; + }; +} +@ApiTags("Admin Dashboard") +@UseGuards(AdminGuard) +@ApiBearerAuth() +@Injectable() +@Controller({ + path: "admin-dashboard", + version: "1", +}) +export class AdminDashboardController { + constructor( + private adminService: AdminService, + private scheduledTasksService: ScheduledTasksService, + ) {} + + @Get("stats") + @Roles(UserRole.AUTHOR, UserRole.ADMIN) + @ApiOperation({ + summary: "Get admin dashboard statistics", + }) + @ApiQuery({ name: "startDate", required: false, type: String }) + @ApiQuery({ name: "endDate", required: false, type: String }) + @ApiQuery({ name: "assignmentId", required: false, type: Number }) + @ApiQuery({ name: "assignmentName", required: false, type: String }) + @ApiQuery({ name: "userId", required: false, type: String }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + async getDashboardStats( + @Req() request: UserSessionRequest, + @Query("startDate") startDate?: string, + @Query("endDate") endDate?: string, + @Query("assignmentId") assignmentId?: string, + @Query("assignmentName") assignmentName?: string, + @Query("userId") userId?: string, + ): Promise { + return this.adminService.getDashboardStats(request.userSession, { + startDate, + endDate, + assignmentId: assignmentId + ? Number.parseInt(assignmentId, 10) + : undefined, + assignmentName, + userId, + }); + } + @Get("quick-actions/:action") + @Roles(UserRole.AUTHOR, UserRole.ADMIN) + @ApiOperation({ + summary: "Execute predefined quick actions for dashboard insights", + }) + @ApiQuery({ name: "limit", required: false, type: Number }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + async executeQuickAction( + @Req() request: AdminSessionRequest, + @Param("action") action: string, + @Query("limit", new DefaultValuePipe(10), ParseIntPipe) limit: number, + ): Promise { + return this.adminService.executeQuickAction( + request.adminSession, + action, + limit, + ); + } + + /** + * Get assignment analytics with detailed insights + */ + @Get("analytics") + @Roles(UserRole.AUTHOR, UserRole.ADMIN) + @UseGuards(AdminGuard) + @ApiOperation({ + summary: + "Get detailed assignment analytics with insights (for authors and admins)", + }) + @ApiQuery({ name: "page", required: false, type: Number }) + @ApiQuery({ name: "limit", required: false, type: Number }) + @ApiQuery({ name: "search", required: false, type: String }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + async getAssignmentAnalytics( + @Req() request: UserSessionRequest, + @Query("page", new DefaultValuePipe(1), ParseIntPipe) page: number, + @Query("limit", new DefaultValuePipe(10), ParseIntPipe) limit: number, + @Query("search") search?: string, + ): Promise { + return await this.adminService.getAssignmentAnalytics( + request.userSession, + page, + limit, + search, + ); + } + + /** + * Get detailed insights for a specific assignment + */ + @Get("assignments/:id/insights") + @Roles(UserRole.AUTHOR, UserRole.ADMIN) + @UseGuards(AdminGuard) + @ApiOperation({ + summary: "Get detailed insights for a specific assignment", + }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + @ApiResponse({ status: 404 }) + async getDetailedAssignmentInsights( + @Req() request: UserSessionRequest, + @Param("id", ParseIntPipe) id: number, + ) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return await this.adminService.getDetailedAssignmentInsights( + request.userSession, + id, + ); + } + + /** + * Manual cleanup of old drafts + */ + @Post("cleanup/drafts") + @Roles(UserRole.ADMIN) + @UseGuards(AdminGuard) + @ApiOperation({ + summary: "Manually trigger cleanup of old drafts (Admin only)", + description: + "Deletes drafts older than the specified number of days (default: 60 days)", + }) + @ApiQuery({ + name: "daysOld", + required: false, + type: Number, + description: + "Number of days old drafts should be to get deleted (default: 60). Use 0 to delete ALL drafts.", + }) + @ApiResponse({ + status: 200, + description: "Cleanup completed successfully", + schema: { + type: "object", + properties: { + success: { type: "boolean" }, + message: { type: "string" }, + deletedCount: { type: "number" }, + }, + }, + }) + @ApiResponse({ + status: 403, + description: "Forbidden - Admin access required", + }) + async manualDraftCleanup( + @Req() request: AdminSessionRequest, + @Query("daysOld", new DefaultValuePipe(60), ParseIntPipe) daysOld: number, + ) { + try { + const result = + await this.scheduledTasksService.manualCleanupOldDrafts(daysOld); + const message = + daysOld === 0 + ? "All drafts have been deleted" + : `Draft cleanup completed for drafts older than ${daysOld} days`; + + return { + success: true, + message, + ...result, + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + return { + success: false, + message: `Draft cleanup failed: ${errorMessage}`, + }; + } + } +} diff --git a/apps/api/src/api/admin/controllers/assignment-analytics.controller.ts b/apps/api/src/api/admin/controllers/assignment-analytics.controller.ts new file mode 100644 index 00000000..3971c972 --- /dev/null +++ b/apps/api/src/api/admin/controllers/assignment-analytics.controller.ts @@ -0,0 +1,35 @@ +import { + Controller, + Get, + Injectable, + Param, + UsePipes, + ValidationPipe, +} from "@nestjs/common"; +import { ApiOperation, ApiParam, ApiResponse, ApiTags } from "@nestjs/swagger"; +import { AdminService } from "../admin.service"; + +@ApiTags("Admin") +@UsePipes( + new ValidationPipe({ + whitelist: true, + forbidNonWhitelisted: true, + }), +) +@Injectable() +@Controller({ + path: "admin/assignments", + version: "1", +}) +export class AssignmentAnalyticsController { + constructor(private adminService: AdminService) {} + + @Get(":id/analytics") + @ApiOperation({ summary: "Get analytics for an assignment" }) + @ApiParam({ name: "id", required: true }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + getAssignmentAnalytics(@Param("id") id: number) { + return this.adminService.getBasicAssignmentAnalytics(Number(id)); + } +} diff --git a/apps/api/src/api/admin/controllers/flagged-submissions.controller.ts b/apps/api/src/api/admin/controllers/flagged-submissions.controller.ts new file mode 100644 index 00000000..d981d67a --- /dev/null +++ b/apps/api/src/api/admin/controllers/flagged-submissions.controller.ts @@ -0,0 +1,51 @@ +import { + Controller, + Get, + Injectable, + Param, + Post, + UsePipes, + ValidationPipe, +} from "@nestjs/common"; +import { + ApiBearerAuth, + ApiOperation, + ApiParam, + ApiResponse, + ApiTags, +} from "@nestjs/swagger"; +import { AdminService } from "../admin.service"; + +@ApiTags("Admin") +@UsePipes( + new ValidationPipe({ + whitelist: true, + forbidNonWhitelisted: true, + }), +) +@ApiBearerAuth() +@Injectable() +@Controller({ + path: "admin/flagged-submissions", + version: "1", +}) +export class FlaggedSubmissionsController { + constructor(private adminService: AdminService) {} + + @Get() + @ApiOperation({ summary: "Get all flagged submissions" }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + getFlaggedSubmissions() { + return this.adminService.getFlaggedSubmissions(); + } + + @Post(":id/dismiss") + @ApiOperation({ summary: "Dismiss a flagged submission" }) + @ApiParam({ name: "id", required: true }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + dismissFlaggedSubmission(@Param("id") id: number) { + return this.adminService.dismissFlaggedSubmission(Number(id)); + } +} diff --git a/apps/api/src/api/admin/controllers/llm-assignment.controller.ts b/apps/api/src/api/admin/controllers/llm-assignment.controller.ts new file mode 100644 index 00000000..9d782e35 --- /dev/null +++ b/apps/api/src/api/admin/controllers/llm-assignment.controller.ts @@ -0,0 +1,511 @@ +import { + Body, + Controller, + Delete, + Get, + HttpException, + HttpStatus, + Inject, + Param, + Post, + Put, + Query, + Req, + UseGuards, +} from "@nestjs/common"; +import { + ApiBearerAuth, + ApiOperation, + ApiResponse, + ApiTags, +} from "@nestjs/swagger"; +import { AdminGuard } from "../../../auth/guards/admin.guard"; +import { + AssignmentRequest, + LLMAssignmentService, +} from "../../llm/core/services/llm-assignment.service"; +import { LLMResolverService } from "../../llm/core/services/llm-resolver.service"; +import { + LLM_ASSIGNMENT_SERVICE, + LLM_RESOLVER_SERVICE, +} from "../../llm/llm.constants"; + +interface AdminSessionRequest extends Request { + adminSession: { + email: string; + role: string; + sessionToken: string; + }; +} + +@ApiTags("Admin LLM Assignments") +@ApiBearerAuth() +@Controller({ + path: "llm-assignments", + version: "1", +}) +@UseGuards(AdminGuard) +export class LLMAssignmentController { + constructor( + @Inject(LLM_ASSIGNMENT_SERVICE) + private readonly assignmentService: LLMAssignmentService, + @Inject(LLM_RESOLVER_SERVICE) + private readonly resolverService: LLMResolverService, + ) {} + + /** + * Get all AI features with their current model assignments + */ + @Get("features") + @ApiOperation({ + summary: "Get all AI features with their current model assignments", + }) + @ApiResponse({ + status: 200, + description: "Successfully retrieved feature assignments", + }) + @ApiResponse({ + status: 401, + description: "Unauthorized - Admin authentication required", + }) + @ApiResponse({ + status: 500, + description: "Internal server error", + }) + async getAllFeatureAssignments() { + try { + const assignments = + await this.assignmentService.getAllFeatureAssignments(); + return { + success: true, + data: assignments, + }; + } catch { + throw new HttpException( + "Failed to fetch feature assignments", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get the assigned model for a specific feature + */ + @Get("features/:featureKey/model") + async getAssignedModel(@Param("featureKey") featureKey: string) { + try { + const modelKey = + await this.assignmentService.getAssignedModel(featureKey); + + if (!modelKey) { + throw new HttpException( + `No model assigned to feature ${featureKey}`, + HttpStatus.NOT_FOUND, + ); + } + + return { + success: true, + data: { + featureKey, + assignedModelKey: modelKey, + }, + }; + } catch (error) { + if (error instanceof HttpException) { + throw error; + } + throw new HttpException( + "Failed to fetch assigned model", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Assign a model to a feature + */ + @Post("assign") + async assignModelToFeature( + @Body() + body: { + featureKey: string; + modelKey: string; + priority?: number; + metadata?: any; + }, + @Req() request: AdminSessionRequest, + ) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const { featureKey, modelKey, priority, metadata } = body; + + if (!featureKey || !modelKey) { + throw new HttpException( + "featureKey and modelKey are required", + HttpStatus.BAD_REQUEST, + ); + } + + try { + const assignmentRequest: AssignmentRequest = { + featureKey, + modelKey, + priority, + assignedBy: request.adminSession.email, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + metadata, + }; + + const success = + await this.assignmentService.assignModelToFeature(assignmentRequest); + + return { + success, + message: `Successfully assigned model ${modelKey} to feature ${featureKey}`, + data: { + featureKey, + modelKey, + assignedBy: request.adminSession.email, + assignedAt: new Date().toISOString(), + }, + }; + } catch (error) { + if (error instanceof HttpException) { + throw error; + } + throw new HttpException( + `Failed to assign model ${modelKey} to feature ${featureKey}: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Remove model assignment from a feature (revert to default) + */ + @Delete("features/:featureKey/assignment") + async removeFeatureAssignment( + @Param("featureKey") featureKey: string, + @Req() request: AdminSessionRequest, + ) { + try { + const success = await this.assignmentService.removeFeatureAssignment( + featureKey, + request.adminSession.email, + ); + + if (!success) { + throw new HttpException( + `No active assignment found for feature ${featureKey}`, + HttpStatus.NOT_FOUND, + ); + } + + return { + success: true, + message: `Successfully removed model assignment for feature ${featureKey}`, + data: { + featureKey, + removedBy: request.adminSession.email, + removedAt: new Date().toISOString(), + }, + }; + } catch (error) { + if (error instanceof HttpException) { + throw error; + } + throw new HttpException( + "Failed to remove feature assignment", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get assignment history for a feature + */ + @Get("features/:featureKey/history") + async getFeatureAssignmentHistory( + @Param("featureKey") featureKey: string, + @Query("limit") limit?: string, + ) { + try { + const parsedLimit = limit ? Number.parseInt(limit, 10) : 10; + const history = await this.assignmentService.getFeatureAssignmentHistory( + featureKey, + parsedLimit, + ); + + return { + success: true, + data: { + featureKey, + history: history.map((assignment) => ({ + id: assignment.id, + modelKey: assignment.model.modelKey, + modelDisplayName: assignment.model.displayName, + isActive: assignment.isActive, + priority: assignment.priority, + assignedBy: assignment.assignedBy, + assignedAt: assignment.assignedAt, + deactivatedAt: assignment.deactivatedAt, + metadata: assignment.metadata, + })), + }, + }; + } catch (error) { + if (error instanceof HttpException) { + throw error; + } + throw new HttpException( + "Failed to fetch assignment history", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get all available models for assignment + */ + @Get("models") + @ApiOperation({ + summary: "Get all available models for assignment", + }) + @ApiResponse({ + status: 200, + description: "Successfully retrieved available models", + }) + @ApiResponse({ + status: 401, + description: "Unauthorized - Admin authentication required", + }) + @ApiResponse({ + status: 500, + description: "Internal server error", + }) + async getAvailableModels() { + try { + const models = await this.assignmentService.getAvailableModels(); + + return { + success: true, + data: models.map((model) => ({ + id: model.id, + modelKey: model.modelKey, + displayName: model.displayName, + provider: model.provider, + isActive: model.isActive, + currentPricing: model.pricingHistory[0] || null, + assignedFeatures: model.featureAssignments.map((assignment) => ({ + featureKey: assignment.feature.featureKey, + featureDisplayName: assignment.feature.displayName, + priority: assignment.priority, + })), + })), + }; + } catch { + throw new HttpException( + "Failed to fetch available models", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get assignment statistics + */ + @Get("statistics") + async getAssignmentStatistics() { + try { + const stats = await this.assignmentService.getAssignmentStatistics(); + return { + success: true, + data: stats, + }; + } catch { + throw new HttpException( + "Failed to fetch assignment statistics", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Bulk update feature assignments + */ + @Put("bulk-assign") + async bulkUpdateAssignments( + @Body() + body: { + assignments: Array<{ + featureKey: string; + modelKey: string; + priority?: number; + }>; + }, + @Req() request: AdminSessionRequest, + ) { + const { assignments } = body; + + if ( + !assignments || + !Array.isArray(assignments) || + assignments.length === 0 + ) { + throw new HttpException( + "assignments array is required", + HttpStatus.BAD_REQUEST, + ); + } + + try { + const assignmentRequests: AssignmentRequest[] = assignments.map( + (assignment) => ({ + featureKey: assignment.featureKey, + modelKey: assignment.modelKey, + priority: assignment.priority, + assignedBy: request.adminSession.email, + }), + ); + + const results = await this.assignmentService.bulkUpdateAssignments( + assignmentRequests, + request.adminSession.email, + ); + + return { + success: results.failed === 0, + message: `Bulk assignment completed: ${results.success} successful, ${results.failed} failed`, + data: { + successful: results.success, + failed: results.failed, + errors: results.errors, + assignedBy: request.adminSession.email, + assignedAt: new Date().toISOString(), + }, + }; + } catch { + throw new HttpException( + "Failed to perform bulk assignment", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Reset all assignments to defaults + */ + @Post("reset-to-defaults") + async resetToDefaults(@Req() request: AdminSessionRequest) { + try { + const resetCount = await this.assignmentService.resetToDefaults( + request.adminSession.email, + ); + + return { + success: true, + message: `Successfully reset ${resetCount} features to default models`, + data: { + resetCount, + resetBy: request.adminSession.email, + resetAt: new Date().toISOString(), + }, + }; + } catch { + throw new HttpException( + "Failed to reset assignments to defaults", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Test model assignment - get which model would be used for a feature + */ + @Get("test/:featureKey") + async testFeatureAssignment(@Param("featureKey") featureKey: string) { + try { + const modelKey = + await this.assignmentService.getAssignedModel(featureKey); + + return { + success: true, + data: { + featureKey, + resolvedModelKey: modelKey, + timestamp: new Date().toISOString(), + }, + }; + } catch { + throw new HttpException( + "Failed to test feature assignment", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Clear cache for a specific feature + */ + @Post("cache/clear/:featureKey") + async clearFeatureCache(@Param("featureKey") featureKey: string) { + try { + this.resolverService.clearCacheForFeature(featureKey); + + return { + success: true, + message: `Cache cleared for feature ${featureKey}`, + timestamp: new Date().toISOString(), + }; + } catch { + throw new HttpException( + "Failed to clear feature cache", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Clear all model assignment cache + */ + @Post("cache/clear-all") + async clearAllCache() { + try { + this.resolverService.clearAllCache(); + + return { + success: true, + message: "All model assignment cache cleared", + timestamp: new Date().toISOString(), + }; + } catch { + throw new HttpException( + "Failed to clear cache", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get cache statistics + */ + @Get("cache/stats") + async getCacheStats() { + try { + const stats = this.resolverService.getCacheStats(); + + return { + success: true, + data: stats, + timestamp: new Date().toISOString(), + }; + } catch { + throw new HttpException( + "Failed to get cache stats", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } +} diff --git a/apps/api/src/api/admin/controllers/llm-pricing.controller.ts b/apps/api/src/api/admin/controllers/llm-pricing.controller.ts new file mode 100644 index 00000000..5ecefb3e --- /dev/null +++ b/apps/api/src/api/admin/controllers/llm-pricing.controller.ts @@ -0,0 +1,440 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { + Body, + Controller, + Get, + HttpException, + HttpStatus, + Inject, + Post, + Query, + UseGuards, +} from "@nestjs/common"; +import { AdminGuard } from "../../../auth/guards/admin.guard"; +import { LLMPricingService } from "../../llm/core/services/llm-pricing.service"; +import { LLM_PRICING_SERVICE } from "../../llm/llm.constants"; + +@Controller({ + path: "llm-pricing", + version: "1", +}) +@UseGuards(AdminGuard) +export class LLMPricingController { + constructor( + @Inject(LLM_PRICING_SERVICE) + private readonly llmPricingService: LLMPricingService, + ) {} + + /** + * Get current pricing for all supported models + */ + @Get("current") + async getCurrentPricing() { + try { + const models = await this.llmPricingService.getSupportedModels(); + return { + success: true, + data: models.map((model) => ({ + id: model.id, + modelKey: model.modelKey, + displayName: model.displayName, + provider: model.provider, + isActive: model.isActive, + currentPricing: model.pricingHistory[0] || null, + })), + }; + } catch { + throw new HttpException( + "Failed to fetch current pricing", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get pricing history for a specific model + */ + @Get("history") + async getPricingHistory( + @Query("modelKey") modelKey: string, + @Query("limit") limit?: string, + ) { + if (!modelKey) { + throw new HttpException("modelKey is required", HttpStatus.BAD_REQUEST); + } + + try { + const parsedLimit = limit ? Number.parseInt(limit, 10) : 10; + const history = await this.llmPricingService.getPricingHistory( + modelKey, + parsedLimit, + ); + + return { + success: true, + data: { + modelKey, + history: history.map((pricing) => ({ + id: pricing.id, + inputTokenPrice: pricing.inputTokenPrice, + outputTokenPrice: pricing.outputTokenPrice, + effectiveDate: pricing.effectiveDate, + source: pricing.source, + isActive: pricing.isActive, + createdAt: pricing.createdAt, + metadata: pricing.metadata, + })), + }, + }; + } catch { + throw new HttpException( + "Failed to fetch pricing history", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get pricing statistics + */ + @Get("statistics") + async getPricingStatistics() { + try { + const stats = await this.llmPricingService.getPricingStatistics(); + return { + success: true, + data: stats, + }; + } catch { + throw new HttpException( + "Failed to fetch pricing statistics", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Manually refresh pricing data from external sources + */ + @Post("refresh") + async refreshPricing() { + try { + // Fetch current pricing + const currentPricing = await this.llmPricingService.fetchCurrentPricing(); + + if (currentPricing.length === 0) { + return { + success: false, + message: "No pricing data available from external sources", + data: { updatedModels: 0 }, + }; + } + + // Update pricing history + const updatedCount = + await this.llmPricingService.updatePricingHistory(currentPricing); + + return { + success: true, + message: `Successfully updated pricing for ${updatedCount} models`, + data: { + updatedModels: updatedCount, + totalModelsFetched: currentPricing.length, + lastRefresh: new Date().toISOString(), + }, + }; + } catch { + throw new HttpException( + "Failed to refresh pricing", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get supported models + */ + @Get("models") + async getSupportedModels() { + try { + const models = await this.llmPricingService.getSupportedModels(); + return { + success: true, + data: models, + }; + } catch { + throw new HttpException( + "Failed to fetch supported models", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Calculate cost breakdown for specific usage + */ + @Get("calculate-cost") + async calculateCost( + @Query("modelKey") modelKey: string, + @Query("inputTokens") inputTokens: string, + @Query("outputTokens") outputTokens: string, + @Query("usageDate") usageDate?: string, + ) { + if (!modelKey || !inputTokens || !outputTokens) { + throw new HttpException( + "modelKey, inputTokens, and outputTokens are required", + HttpStatus.BAD_REQUEST, + ); + } + + try { + const parsedInputTokens = Number.parseInt(inputTokens, 10); + const parsedOutputTokens = Number.parseInt(outputTokens, 10); + const parsedUsageDate = usageDate ? new Date(usageDate) : new Date(); + + if (Number.isNaN(parsedInputTokens) || Number.isNaN(parsedOutputTokens)) { + throw new HttpException( + "inputTokens and outputTokens must be valid numbers", + HttpStatus.BAD_REQUEST, + ); + } + + const costBreakdown = + await this.llmPricingService.calculateCostWithBreakdown( + modelKey, + parsedInputTokens, + parsedOutputTokens, + parsedUsageDate, + ); + + if (!costBreakdown) { + throw new HttpException( + `No pricing data found for model ${modelKey}`, + HttpStatus.NOT_FOUND, + ); + } + + return { + success: true, + data: costBreakdown, + }; + } catch (error) { + if (error instanceof HttpException) { + throw error; + } + throw new HttpException( + "Failed to calculate cost", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Apply price upscaling factors to all models + */ + @Post("upscale") + async upscalePricing( + @Body() + upscaleData: { + globalFactor?: number; + usageFactors?: { [usageType: string]: number }; + reason?: string; + }, + ) { + if ( + !upscaleData.globalFactor && + (!upscaleData.usageFactors || + Object.keys(upscaleData.usageFactors).length === 0) + ) { + throw new HttpException( + "Either globalFactor or at least one usage type factor must be provided", + HttpStatus.BAD_REQUEST, + ); + } + + // Validate factors are positive numbers + if ( + upscaleData.globalFactor && + (upscaleData.globalFactor <= 0 || Number.isNaN(upscaleData.globalFactor)) + ) { + throw new HttpException( + "Global factor must be a positive number", + HttpStatus.BAD_REQUEST, + ); + } + + if (upscaleData.usageFactors) { + for (const [usageType, factor] of Object.entries( + upscaleData.usageFactors, + )) { + if (factor <= 0 || Number.isNaN(factor)) { + throw new HttpException( + `Usage factor for ${usageType} must be a positive number`, + HttpStatus.BAD_REQUEST, + ); + } + } + } + + try { + const result = await this.llmPricingService.applyPriceUpscaling( + upscaleData.globalFactor, + upscaleData.usageFactors || {}, + upscaleData.reason || "Manual price upscaling via admin interface", + "admin", + ); + + return { + success: true, + message: `Successfully upscaled pricing for ${result.updatedModels} models`, + data: { + updatedModels: result.updatedModels, + globalFactor: upscaleData.globalFactor, + usageFactors: upscaleData.usageFactors, + oldUpscaling: result.oldUpscaling, + newUpscaling: result.newUpscaling, + effectiveDate: result.effectiveDate, + }, + }; + } catch (error) { + throw new HttpException( + `Failed to upscale pricing: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get current active price upscaling factors + */ + @Get("upscaling/current") + async getCurrentPriceUpscaling() { + try { + const upscaling = await this.llmPricingService.getCurrentPriceUpscaling(); + return { + success: true, + data: upscaling, + }; + } catch { + throw new HttpException( + "Failed to fetch current price upscaling", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Remove current price upscaling (revert to base pricing) + */ + @Post("upscaling/remove") + async removePriceUpscaling(@Body() data: { reason?: string }) { + try { + const removed = await this.llmPricingService.removePriceUpscaling( + data.reason || "Manual removal via admin interface", + "admin", // TODO: Get actual admin email from session + ); + + if (!removed) { + return { + success: false, + message: "No active price upscaling found to remove", + }; + } + + return { + success: true, + message: "Successfully removed price upscaling", + }; + } catch (error) { + throw new HttpException( + `Failed to remove price upscaling: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get comprehensive pricing status including cache and scraping health + */ + @Get("status") + async getPricingStatus() { + try { + const status = await this.llmPricingService.getPricingStatus(); + return { + success: true, + data: status, + }; + } catch { + throw new HttpException( + "Failed to fetch pricing status", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Test scraping functionality for a specific model + */ + @Get("test-scraping") + async testScraping(@Query("modelKey") modelKey: string) { + if (!modelKey) { + throw new HttpException("modelKey is required", HttpStatus.BAD_REQUEST); + } + + try { + const result = + await this.llmPricingService.testScrapingForModel(modelKey); + return { + success: result.success, + data: result, + }; + } catch { + throw new HttpException( + "Failed to test scraping functionality", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Get cache status and statistics + */ + @Get("cache-status") + async getCacheStatus() { + try { + const status = this.llmPricingService.getCacheStatus(); + return { + success: true, + data: status, + }; + } catch { + throw new HttpException( + "Failed to fetch cache status", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } + + /** + * Clear web scraping cache + */ + @Post("clear-cache") + async clearCache() { + try { + this.llmPricingService.clearWebScrapingCache(); + return { + success: true, + message: "Web scraping cache cleared successfully", + }; + } catch { + throw new HttpException( + "Failed to clear cache", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + } +} diff --git a/apps/api/src/api/admin/controllers/regrading-requests.controller.ts b/apps/api/src/api/admin/controllers/regrading-requests.controller.ts new file mode 100644 index 00000000..e46e9dd2 --- /dev/null +++ b/apps/api/src/api/admin/controllers/regrading-requests.controller.ts @@ -0,0 +1,84 @@ +import { + Body, + Controller, + Get, + Injectable, + Param, + Post, + UsePipes, + ValidationPipe, +} from "@nestjs/common"; +import { + ApiBearerAuth, + ApiBody, + ApiOperation, + ApiParam, + ApiResponse, + ApiTags, +} from "@nestjs/swagger"; +import { AdminService } from "../admin.service"; + +class ApproveRegradingRequestDto { + newGrade: number; +} + +class RejectRegradingRequestDto { + reason: string; +} + +@ApiTags("Admin") +@UsePipes( + new ValidationPipe({ + whitelist: true, + forbidNonWhitelisted: true, + }), +) +@ApiBearerAuth() +@Injectable() +@Controller({ + path: "admin/regrading-requests", + version: "1", +}) +export class RegradingRequestsController { + constructor(private adminService: AdminService) {} + + @Get() + @ApiOperation({ summary: "Get all regrading requests" }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + getRegradingRequests() { + return this.adminService.getRegradingRequests(); + } + + @Post(":id/approve") + @ApiOperation({ summary: "Approve a regrading request" }) + @ApiParam({ name: "id", required: true }) + @ApiBody({ type: ApproveRegradingRequestDto }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + approveRegradingRequest( + @Param("id") id: number, + @Body() approveDto: ApproveRegradingRequestDto, + ) { + return this.adminService.approveRegradingRequest( + Number(id), + approveDto.newGrade, + ); + } + + @Post(":id/reject") + @ApiOperation({ summary: "Reject a regrading request" }) + @ApiParam({ name: "id", required: true }) + @ApiBody({ type: RejectRegradingRequestDto }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + rejectRegradingRequest( + @Param("id") id: number, + @Body() rejectDto: RejectRegradingRequestDto, + ) { + return this.adminService.rejectRegradingRequest( + Number(id), + rejectDto.reason, + ); + } +} diff --git a/apps/api/src/api/admin/dto/assignment/base.assignment.response.dto.ts b/apps/api/src/api/admin/dto/assignment/base.assignment.response.dto.ts index 615a226e..97c9a6a3 100644 --- a/apps/api/src/api/admin/dto/assignment/base.assignment.response.dto.ts +++ b/apps/api/src/api/admin/dto/assignment/base.assignment.response.dto.ts @@ -15,6 +15,26 @@ export class BaseAssignmentResponseDto { }) success: boolean; + @ApiProperty({ + description: "The name of the assignment.", + type: String, + required: true, + }) + name: string; + + @ApiProperty({ + description: "The type of the assignment.", + type: String, + required: true, + }) + type: string; + @ApiPropertyOptional({ description: "Optional error message.", type: String }) error?: string; + @ApiProperty({ + description: "The number of unique users associated with the assignment.", + type: Number, + required: true, + }) + uniqueUsers?: number; } diff --git a/apps/api/src/api/assignment/attempt/attempt.service.ts b/apps/api/src/api/assignment/attempt/attempt.service.ts index e485e73d..6726f1e9 100644 --- a/apps/api/src/api/assignment/attempt/attempt.service.ts +++ b/apps/api/src/api/assignment/attempt/attempt.service.ts @@ -515,6 +515,7 @@ export class AttemptServiceV1 { grade: 0, showSubmissionFeedback: false, showQuestions: false, + showCorrectAnswer: false, feedbacksForQuestions: [], message: SUBMISSION_DEADLINE_EXCEPTION_MESSAGE, }; @@ -575,7 +576,12 @@ export class AttemptServiceV1 { } const assignment = await this.prisma.assignment.findUnique({ where: { id: assignmentId }, - include: { + select: { + showAssignmentScore: true, + showSubmissionFeedback: true, + showQuestionScore: true, + showQuestions: true, + showCorrectAnswer: true, questions: { where: { isDeleted: false }, }, @@ -633,6 +639,7 @@ export class AttemptServiceV1 { grade: assignment.showAssignmentScore ? grade : undefined, showSubmissionFeedback: assignment.showSubmissionFeedback, showQuestions: assignment.showQuestions, + showCorrectAnswer: assignment.showCorrectAnswer, feedbacksForQuestions: this.constructFeedbacksForQuestions( successfulQuestionResponses, assignment as unknown as LearnerGetAssignmentResponseDto, @@ -653,6 +660,7 @@ export class AttemptServiceV1 { showQuestions: assignment.showQuestions, grade: assignment.showAssignmentScore ? result.grade : undefined, showSubmissionFeedback: assignment.showSubmissionFeedback, + showCorrectAnswer: assignment.showCorrectAnswer, feedbacksForQuestions: this.constructFeedbacksForQuestions( successfulQuestionResponses, assignment as unknown as LearnerGetAssignmentResponseDto, @@ -738,6 +746,7 @@ export class AttemptServiceV1 { showSubmissionFeedback: true, showQuestionScore: true, showQuestions: true, + showCorrectAnswer: true, }, }); @@ -868,6 +877,21 @@ export class AttemptServiceV1 { } } + // Apply visibility settings for correct answers and if learner didnt pass + if ( + assignment.showCorrectAnswer === false && + assignmentAttempt.grade < assignment.passingGrade + ) { + for (const question of finalQuestions) { + if (question.choices) { + for (const choice of question.choices) { + delete choice.isCorrect; + delete choice.feedback; + } + } + } + } + return { ...assignmentAttempt, questions: finalQuestions.map((question) => ({ @@ -882,6 +906,7 @@ export class AttemptServiceV1 { showAssignmentScore: assignment.showAssignmentScore, showSubmissionFeedback: assignment.showSubmissionFeedback, showQuestionScore: assignment.showQuestionScore, + showCorrectAnswer: assignment.showCorrectAnswer, comments: assignmentAttempt.comments, }; } @@ -946,6 +971,7 @@ export class AttemptServiceV1 { showAssignmentScore: true, showSubmissionFeedback: true, showQuestionScore: true, + showCorrectAnswer: true, }, })) as LearnerGetAssignmentResponseDto; @@ -979,6 +1005,7 @@ export class AttemptServiceV1 { if (!translationMap.has(key)) { translationMap.set(key, {}); } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion translationMap.get(key)![t.languageCode] = { translatedText: t.translatedText, translatedChoices: t.translatedChoices, @@ -1126,8 +1153,14 @@ export class AttemptServiceV1 { if (question.choices) { for (const choice of question.choices) { delete choice.points; - delete choice.isCorrect; - delete choice.feedback; + // Only remove correct answer data if showCorrectAnswer is false + if ( + assignment.showCorrectAnswer === false && + assignmentAttempt.grade < assignment.passingGrade + ) { + delete choice.isCorrect; + delete choice.feedback; + } } } @@ -1137,8 +1170,14 @@ export class AttemptServiceV1 { if (translationObject?.translatedChoices) { for (const choice of translationObject.translatedChoices) { delete choice.points; - delete choice.isCorrect; - delete choice.feedback; + // Only remove correct answer data if showCorrectAnswer is false + if ( + assignment.showCorrectAnswer === false && + assignmentAttempt.grade < assignment.passingGrade + ) { + delete choice.isCorrect; + delete choice.feedback; + } } } } @@ -1153,13 +1192,25 @@ export class AttemptServiceV1 { ) as Choice[]; for (const choice of randomizedArray) { delete choice.points; - delete choice.isCorrect; - delete choice.feedback; + // Only remove correct answer data if showCorrectAnswer is false + if ( + assignment.showCorrectAnswer === false && + assignmentAttempt.grade < assignment.passingGrade + ) { + delete choice.isCorrect; + delete choice.feedback; + } } question.randomizedChoices = JSON.stringify(randomizedArray); } - delete question.answer; + // Only remove the answer field if showCorrectAnswer is false + if ( + assignment.showCorrectAnswer === false && + assignmentAttempt.grade < assignment.passingGrade + ) { + delete question.answer; + } } return { @@ -1176,6 +1227,7 @@ export class AttemptServiceV1 { showSubmissionFeedback: assignment.showSubmissionFeedback, showQuestionScore: assignment.showQuestionScore, showQuestions: assignment.showQuestions, + showCorrectAnswer: assignment.showCorrectAnswer, }; } @@ -1680,9 +1732,12 @@ export class AttemptServiceV1 { const { // eslint-disable-next-line @typescript-eslint/no-unused-vars responsesForQuestions, + // eslint-disable-next-line @typescript-eslint/no-unused-vars authorQuestions, + // eslint-disable-next-line @typescript-eslint/no-unused-vars authorAssignmentDetails, language, + // eslint-disable-next-line @typescript-eslint/no-unused-vars preTranslatedQuestions, ...cleanedUpdateAssignmentAttemptDto } = updateAssignmentAttemptDto; diff --git a/apps/api/src/api/assignment/attempt/dto/assignment-attempt/get.assignment.attempt.response.dto.ts b/apps/api/src/api/assignment/attempt/dto/assignment-attempt/get.assignment.attempt.response.dto.ts index e43d4fe3..d3f87a8d 100644 --- a/apps/api/src/api/assignment/attempt/dto/assignment-attempt/get.assignment.attempt.response.dto.ts +++ b/apps/api/src/api/assignment/attempt/dto/assignment-attempt/get.assignment.attempt.response.dto.ts @@ -85,6 +85,14 @@ export class GetAssignmentAttemptResponseDto extends AssignmentAttemptResponseDt required: false, }) showQuestionScore: boolean; + + @ApiProperty({ + description: "Show correct answer", + type: Boolean, + required: false, + }) + showCorrectAnswer: boolean; + @ApiPropertyOptional({ description: "The comments for the question.", type: String, diff --git a/apps/api/src/api/assignment/attempt/dto/assignment-attempt/update.assignment.attempt.response.dto.ts b/apps/api/src/api/assignment/attempt/dto/assignment-attempt/update.assignment.attempt.response.dto.ts index 2f3f8d62..2d899bfc 100644 --- a/apps/api/src/api/assignment/attempt/dto/assignment-attempt/update.assignment.attempt.response.dto.ts +++ b/apps/api/src/api/assignment/attempt/dto/assignment-attempt/update.assignment.attempt.response.dto.ts @@ -41,6 +41,14 @@ export class UpdateAssignmentAttemptResponseDto extends BaseAssignmentAttemptRes required: false, }) showQuestions: boolean; + + @ApiProperty({ + description: "Show correct answer", + type: Boolean, + required: false, + }) + showCorrectAnswer: boolean; + @ApiProperty({ description: "The total points earned by the learner.", type: Number, diff --git a/apps/api/src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto.ts b/apps/api/src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto.ts index fa54db2d..dab0c1e5 100644 --- a/apps/api/src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto.ts +++ b/apps/api/src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto.ts @@ -3,7 +3,7 @@ import { IsString } from "class-validator"; export class GeneralFeedbackDto { @ApiProperty({ - description: "The feedback earned by the learner.", + description: "The feedback earned by the leanrer.", type: String, required: true, }) @@ -55,7 +55,7 @@ export class CreateQuestionResponseAttemptResponseDto { type: Object, required: false, }) - metadata?: Record; + metadata?: Record; @ApiProperty({ description: "The total points earned.", @@ -64,6 +64,9 @@ export class CreateQuestionResponseAttemptResponseDto { }) totalPoints?: number; + // points + points?: number; + @ApiProperty({ description: "The feedback received after evaluating the question response of the learner.", diff --git a/apps/api/src/api/assignment/attempt/helper/attempts.helper.ts b/apps/api/src/api/assignment/attempt/helper/attempts.helper.ts index c8b16301..b703b77c 100644 --- a/apps/api/src/api/assignment/attempt/helper/attempts.helper.ts +++ b/apps/api/src/api/assignment/attempt/helper/attempts.helper.ts @@ -7,6 +7,7 @@ import { QuestionType } from "@prisma/client"; import axios from "axios"; import * as cheerio from "cheerio"; import { ChoiceBasedQuestionResponseModel } from "../../../llm/model/choice.based.question.response.model"; +import { FileBasedQuestionResponseModel } from "../../../llm/model/file.based.question.response.model"; import { TextBasedQuestionResponseModel } from "../../../llm/model/text.based.question.response.model"; import { TrueFalseBasedQuestionResponseModel } from "../../../llm/model/true.false.based.question.response.model"; import { UrlBasedQuestionResponseModel } from "../../../llm/model/url.based.question.response.model"; @@ -24,10 +25,12 @@ export const AttemptHelper = { | UrlBasedQuestionResponseModel | TextBasedQuestionResponseModel | ChoiceBasedQuestionResponseModel - | TrueFalseBasedQuestionResponseModel, + | TrueFalseBasedQuestionResponseModel + | FileBasedQuestionResponseModel, responseDto: CreateQuestionResponseAttemptResponseDto, ) { responseDto.totalPoints = model.points; + if (model instanceof ChoiceBasedQuestionResponseModel) { responseDto.feedback = model.feedback as ChoiceBasedFeedbackDto[]; } else if (model instanceof TrueFalseBasedQuestionResponseModel) { @@ -37,6 +40,109 @@ export const AttemptHelper = { feedback: model.feedback, }, ] as TrueFalseBasedFeedbackDto[]; + } else if (model instanceof FileBasedQuestionResponseModel) { + // Handle file-based responses with enhanced rubric data + const generalFeedbackDto = new GeneralFeedbackDto(); + generalFeedbackDto.feedback = model.feedback; + + // Add AEEG components if available and not already present in feedback + if ( + model.analysis || + model.evaluation || + model.explanation || + model.guidance + ) { + const feedbackLower = generalFeedbackDto.feedback.toLowerCase(); + const hasExistingStructure = + feedbackLower.includes("analysis:") || + feedbackLower.includes("evaluation:") || + feedbackLower.includes("explanation:") || + feedbackLower.includes("guidance:"); + + if (!hasExistingStructure) { + let aeegSection = "\n\n**Detailed Breakdown:**\n"; + if (model.analysis) { + aeegSection += `\n**Analysis:** ${model.analysis}\n`; + } + if (model.evaluation) { + aeegSection += `\n**Evaluation:** ${model.evaluation}\n`; + } + if (model.explanation) { + aeegSection += `\n**Explanation:** ${model.explanation}\n`; + } + if (model.guidance) { + aeegSection += `\n**Guidance:** ${model.guidance}\n`; + } + generalFeedbackDto.feedback += aeegSection; + } + } + + responseDto.feedback = [generalFeedbackDto]; + + // Add rubric metadata for judge validation + if (model.rubricScores && model.rubricScores.length > 0) { + responseDto.metadata = { + ...responseDto.metadata, + rubricScores: model.rubricScores, + hasDetailedRubrics: true, + rubricCount: model.rubricScores.length, + }; + } + } else if (model instanceof TextBasedQuestionResponseModel) { + // Handle text-based responses with enhanced rubric data (similar to file-based) + const generalFeedbackDto = new GeneralFeedbackDto(); + generalFeedbackDto.feedback = model.feedback; + + // Add AEEG components if available and not already present in feedback + if ( + model.analysis || + model.evaluation || + model.explanation || + model.guidance + ) { + const feedbackLower = generalFeedbackDto.feedback.toLowerCase(); + const hasExistingStructure = + feedbackLower.includes("analysis:") || + feedbackLower.includes("evaluation:") || + feedbackLower.includes("explanation:") || + feedbackLower.includes("guidance:"); + + if (!hasExistingStructure) { + let aeegSection = "\n\n**Detailed Breakdown:**\n"; + if (model.analysis) { + aeegSection += `\n**Analysis:** ${model.analysis}\n`; + } + if (model.evaluation) { + aeegSection += `\n**Evaluation:** ${model.evaluation}\n`; + } + if (model.explanation) { + aeegSection += `\n**Explanation:** ${model.explanation}\n`; + } + if (model.guidance) { + aeegSection += `\n**Guidance:** ${model.guidance}\n`; + } + generalFeedbackDto.feedback += aeegSection; + } + } + responseDto.feedback = [generalFeedbackDto]; + + // Add rubric metadata for judge validation + if (model.rubricScores && model.rubricScores.length > 0) { + responseDto.metadata = { + ...responseDto.metadata, + rubricScores: model.rubricScores, + hasDetailedRubrics: true, + rubricCount: model.rubricScores.length, + }; + } + + // Add judge metadata if available + if (model.metadata) { + responseDto.metadata = { + ...responseDto.metadata, + ...model.metadata, + }; + } } else { const generalFeedbackDto = new GeneralFeedbackDto(); generalFeedbackDto.feedback = model.feedback; @@ -169,7 +275,7 @@ export const AttemptHelper = { ); if (fileList.length > 0) { content += "Repository Files:\n"; - fileList.each((index, element) => { + fileList.each((_, element) => { const fileName = $(element) .find(".js-navigation-open") .text() diff --git a/apps/api/src/api/assignment/attempt/helper/languages.json b/apps/api/src/api/assignment/attempt/helper/languages.json index 621782f6..3320d322 100644 --- a/apps/api/src/api/assignment/attempt/helper/languages.json +++ b/apps/api/src/api/assignment/attempt/helper/languages.json @@ -14,7 +14,7 @@ { "code": "el", "name": "Ελληνικά" }, { "code": "kk", "name": "Қазақ тілі" }, { "code": "ru", "name": "Русский" }, - { "code": "uk", "name": "Українська" }, + { "code": "uk-UA", "name": "Українська" }, { "code": "ar", "name": "العربية" }, { "code": "hi", "name": "हिन्दी" }, { "code": "th", "name": "ไทย" }, diff --git a/apps/api/src/api/assignment/dto/post.assignment.request.dto.ts b/apps/api/src/api/assignment/dto/post.assignment.request.dto.ts index b60edbe0..14faf2c1 100644 --- a/apps/api/src/api/assignment/dto/post.assignment.request.dto.ts +++ b/apps/api/src/api/assignment/dto/post.assignment.request.dto.ts @@ -1,6 +1,6 @@ -import { QuestionType } from "@prisma/client"; +import { QuestionType, ResponseType } from "@prisma/client"; import { AssignmentTypeEnum } from "src/api/llm/features/question-generation/services/question-generation.service"; -import { ResponseType } from "@prisma/client"; + export interface QuestionsToGenerate { multipleChoice: number; multipleSelect: number; diff --git a/apps/api/src/api/assignment/dto/update.assignment.request.dto.ts b/apps/api/src/api/assignment/dto/update.assignment.request.dto.ts index a588dc69..3e98699e 100644 --- a/apps/api/src/api/assignment/dto/update.assignment.request.dto.ts +++ b/apps/api/src/api/assignment/dto/update.assignment.request.dto.ts @@ -199,4 +199,14 @@ export class UpdateAssignmentRequestDto { @IsOptional() @IsBoolean() showSubmissionFeedback: boolean; + + @ApiProperty({ + description: + "Should the correct answer be shown to the learner after its submission", + type: Boolean, + required: false, + }) + @IsOptional() + @IsBoolean() + showCorrectAnswer: boolean; } diff --git a/apps/api/src/api/assignment/dto/update.questions.request.dto.ts b/apps/api/src/api/assignment/dto/update.questions.request.dto.ts index fe314b14..f76fbaf3 100644 --- a/apps/api/src/api/assignment/dto/update.questions.request.dto.ts +++ b/apps/api/src/api/assignment/dto/update.questions.request.dto.ts @@ -540,12 +540,42 @@ export class UpdateAssignmentQuestionsDto { @IsBoolean() showQuestions: boolean; + @ApiProperty({ + description: + "Should the correct answer be shown to the learner after its submission", + type: Boolean, + required: false, + }) + @IsOptional() + @IsBoolean() + showCorrectAnswer: boolean; + @ApiProperty({ description: "updatedAt", required: false, }) @IsOptional() updatedAt: Date; + // versionDescription + @ApiProperty({ + description: "versionDescription", + type: String, + required: false, + }) + @IsOptional() + @IsString() + versionDescription: string; + + // versionNumber + @ApiProperty({ + description: + "versionNumber - the specific version number to create when publishing", + type: String, + required: false, + }) + @IsOptional() + @IsString() + versionNumber: string; } /** * If a questionVariant is present (not null), you can expand this class diff --git a/apps/api/src/api/assignment/guards/assignment.access.control.guard.ts b/apps/api/src/api/assignment/guards/assignment.access.control.guard.ts index 70abbbb4..644e264b 100644 --- a/apps/api/src/api/assignment/guards/assignment.access.control.guard.ts +++ b/apps/api/src/api/assignment/guards/assignment.access.control.guard.ts @@ -19,8 +19,9 @@ export class AssignmentAccessControlGuard implements CanActivate { async canActivate(context: ExecutionContext): Promise { const request = context.switchToHttp().getRequest(); const { userSession, params } = request; - const { id } = params; - const assignmentId = Number(id) || userSession.assignmentId; + const { id, assignmentId: parameterAssignmentId } = params; + const assignmentId = + Number(parameterAssignmentId) || Number(id) || userSession.assignmentId; if (!assignmentId || Number.isNaN(assignmentId)) { throw new ForbiddenException("Invalid assignment ID"); } diff --git a/apps/api/src/api/assignment/question/question.service.ts b/apps/api/src/api/assignment/question/question.service.ts index cb397102..9ee7657a 100644 --- a/apps/api/src/api/assignment/question/question.service.ts +++ b/apps/api/src/api/assignment/question/question.service.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { HttpException, HttpStatus, @@ -291,6 +292,18 @@ export class QuestionService { language, ); + let translatedChoices: Choice[] | undefined; + + // If the question has choices, translate them as well + if (question.choices && question.choices.length > 0) { + translatedChoices = + await this.llmFacadeService.generateChoicesTranslation( + question.choices, + assignmentId, + language, + ); + } + await this.prisma.translation.create({ data: { questionId, @@ -298,11 +311,18 @@ export class QuestionService { languageCode, translatedText: translatedQuestion, untranslatedText: question.question, + translatedChoices: translatedChoices + ? JSON.parse(JSON.stringify(translatedChoices)) + : undefined, + untranslatedChoices: question.choices + ? JSON.parse(JSON.stringify(question.choices)) + : undefined, }, }); return { translatedQuestion, + translatedChoices, }; } diff --git a/apps/api/src/api/assignment/v1/controllers/assignment.controller.ts b/apps/api/src/api/assignment/v1/controllers/assignment.controller.ts index c93598d7..a49d8c72 100644 --- a/apps/api/src/api/assignment/v1/controllers/assignment.controller.ts +++ b/apps/api/src/api/assignment/v1/controllers/assignment.controller.ts @@ -42,17 +42,17 @@ import { ReportRequestDTO } from "../../attempt/dto/assignment-attempt/post.assi import { ASSIGNMENT_SCHEMA_URL } from "../../constants"; import { BaseAssignmentResponseDto } from "../../dto/base.assignment.response.dto"; import { + AssignmentResponseDto, GetAssignmentResponseDto, LearnerGetAssignmentResponseDto, - AssignmentResponseDto, } from "../../dto/get.assignment.response.dto"; import { QuestionGenerationPayload } from "../../dto/post.assignment.request.dto"; import { ReplaceAssignmentRequestDto } from "../../dto/replace.assignment.request.dto"; import { UpdateAssignmentRequestDto } from "../../dto/update.assignment.request.dto"; import { - UpdateAssignmentQuestionsDto, GenerateQuestionVariantDto, QuestionDto, + UpdateAssignmentQuestionsDto, } from "../../dto/update.questions.request.dto"; import { AssignmentAccessControlGuard } from "../../guards/assignment.access.control.guard"; import { LLMResponseQuestion } from "../../question/dto/create.update.question.request.dto"; diff --git a/apps/api/src/api/assignment/v1/modules/assignment.module.ts b/apps/api/src/api/assignment/v1/modules/assignment.module.ts index ecd6037e..118bc413 100644 --- a/apps/api/src/api/assignment/v1/modules/assignment.module.ts +++ b/apps/api/src/api/assignment/v1/modules/assignment.module.ts @@ -1,15 +1,13 @@ -import { Module } from "@nestjs/common"; import { HttpModule } from "@nestjs/axios"; - -import { QuestionController } from "../../question/question.controller"; - +import { Module } from "@nestjs/common"; +import { JobStatusServiceV1 } from "src/api/Job/job-status.service"; import { LlmModule } from "../../../llm/llm.module"; +import { AttemptControllerV1 } from "../../attempt/attempt.controller"; +import { AttemptServiceV1 } from "../../attempt/attempt.service"; +import { QuestionController } from "../../question/question.controller"; import { QuestionService } from "../../question/question.service"; -import { JobStatusServiceV1 } from "src/api/Job/job-status.service"; import { AssignmentControllerV1 } from "../controllers/assignment.controller"; import { AssignmentServiceV1 } from "../services/assignment.service"; -import { AttemptControllerV1 } from "../../attempt/attempt.controller"; -import { AttemptServiceV1 } from "../../attempt/attempt.service"; @Module({ controllers: [ diff --git a/apps/api/src/api/assignment/v1/services/assignment.service.ts b/apps/api/src/api/assignment/v1/services/assignment.service.ts index e5abe0aa..400c588e 100644 --- a/apps/api/src/api/assignment/v1/services/assignment.service.ts +++ b/apps/api/src/api/assignment/v1/services/assignment.service.ts @@ -1,4 +1,9 @@ /* eslint-disable unicorn/no-null */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { HttpException, HttpStatus, @@ -148,14 +153,97 @@ export class AssignmentServiceV1 { ): Promise { const isLearner = userSession.role === UserRole.LEARNER; - const result = await this.prisma.assignment.findUnique({ - where: { id }, - include: { - questions: { - include: { variants: true }, + let result: any; + + if (isLearner) { + // For learners, fetch the active published version + const assignment = await this.prisma.assignment.findUnique({ + where: { id }, + include: { + currentVersion: { + include: { + questionVersions: { orderBy: { displayOrder: "asc" } }, + }, + }, }, - }, - }); + }); + + if (!assignment) { + throw new NotFoundException(`Assignment with Id ${id} not found.`); + } + + // If there's an active version, use it; otherwise fall back to main assignment + if (assignment.currentVersion) { + const version = assignment.currentVersion; + + // Convert versioned data to the expected assignment format + result = { + id: assignment.id, + name: version.name, + introduction: version.introduction, + instructions: version.instructions, + gradingCriteriaOverview: version.gradingCriteriaOverview, + timeEstimateMinutes: version.timeEstimateMinutes, + type: version.type, + graded: version.graded, + numAttempts: version.numAttempts, + allotedTimeMinutes: version.allotedTimeMinutes, + attemptsPerTimeRange: version.attemptsPerTimeRange, + attemptsTimeRangeHours: version.attemptsTimeRangeHours, + passingGrade: version.passingGrade, + displayOrder: version.displayOrder, + questionDisplay: version.questionDisplay, + numberOfQuestionsPerAttempt: version.numberOfQuestionsPerAttempt, + questionOrder: version.questionOrder, + published: version.published, + showAssignmentScore: version.showAssignmentScore, + showQuestionScore: version.showQuestionScore, + showSubmissionFeedback: version.showSubmissionFeedback, + showQuestions: version.showQuestions, + languageCode: version.languageCode, + updatedAt: assignment.updatedAt, + questions: version.questionVersions.map((qv: any) => ({ + id: qv.questionId || qv.id, + totalPoints: qv.totalPoints, + type: qv.type, + responseType: qv.responseType, + question: qv.question, + maxWords: qv.maxWords, + scoring: qv.scoring, + choices: qv.choices, + randomizedChoices: qv.randomizedChoices, + answer: qv.answer, + gradingContextQuestionIds: qv.gradingContextQuestionIds, + maxCharacters: qv.maxCharacters, + videoPresentationConfig: qv.videoPresentationConfig, + liveRecordingConfig: qv.liveRecordingConfig, + displayOrder: qv.displayOrder, + isDeleted: false, + variants: [], // Question versions don't have variants in the same structure + })), + }; + } else { + // Fallback to main assignment if no current version + result = await this.prisma.assignment.findUnique({ + where: { id }, + include: { + questions: { + include: { variants: true }, + }, + }, + }); + } + } else { + // For authors, use the main assignment table (current editing state) + result = await this.prisma.assignment.findUnique({ + where: { id }, + include: { + questions: { + include: { variants: true }, + }, + }, + }); + } if (!result) { throw new NotFoundException(`Assignment with Id ${id} not found.`); @@ -208,6 +296,22 @@ export class AssignmentServiceV1 { } async list(userSession: UserSession): Promise { + // If user is an author, only show assignments they've authored + if (userSession.role === UserRole.AUTHOR) { + const authoredAssignments = await this.prisma.assignment.findMany({ + where: { + AssignmentAuthor: { + some: { + userId: userSession.userId, + }, + }, + }, + }); + + return authoredAssignments; + } + + // For non-authors (learners, admins), show assignments from their group const results = await this.prisma.assignmentGroup.findMany({ where: { groupId: userSession.groupId }, include: { @@ -460,6 +564,7 @@ export class AssignmentServiceV1 { job.id, assignmentId, updateAssignmentQuestionsDto, + userId, ).catch((error) => { this.logger.error( `Error processing publishing job: ${(error as Error).message}`, @@ -487,6 +592,7 @@ export class AssignmentServiceV1 { jobId: number, assignmentId: number, updateAssignmentQuestionsDto: UpdateAssignmentQuestionsDto, + userId: string, ): Promise<{ jobId: number; message: string }> { const { introduction, @@ -604,6 +710,35 @@ export class AssignmentServiceV1 { showQuestions, }, }); + + // Store the user as an author of this assignment + try { + // Check if this author relationship already exists to avoid duplicates + const existingAuthor = await this.prisma.assignmentAuthor.findFirst( + { + where: { + assignmentId, + userId, + }, + }, + ); + + if (!existingAuthor) { + await this.prisma.assignmentAuthor.create({ + data: { + assignmentId, + userId, + }, + }); + } + } catch (error) { + // Log but don't fail the publishing process if author tracking fails + this.logger.warn( + `Failed to store assignment author: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + } }, }, { diff --git a/apps/api/src/api/assignment/v2/controllers/assignment.controller.ts b/apps/api/src/api/assignment/v2/controllers/assignment.controller.ts index c3b8a2c1..1b55d2eb 100644 --- a/apps/api/src/api/assignment/v2/controllers/assignment.controller.ts +++ b/apps/api/src/api/assignment/v2/controllers/assignment.controller.ts @@ -1,6 +1,7 @@ import { Body, Controller, + DefaultValuePipe, Get, Inject, Injectable, @@ -29,6 +30,8 @@ import { import { Request } from "express"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { Observable } from "rxjs"; +import { AdminService } from "src/api/admin/admin.service"; +import { AdminGuard } from "src/auth/guards/admin.guard"; import { UserRole, UserSessionRequest, @@ -58,6 +61,52 @@ import { JobStatusServiceV2 } from "../services/job-status.service"; import { QuestionService } from "../services/question.service"; import { ReportService } from "../services/report.repository"; +interface AdminSessionRequest extends Request { + adminSession: { + email: string; + role: UserRole; + sessionToken: string; + }; +} + +interface AssignmentAnalyticsResponse { + data: Array<{ + id: number; + name: string; + totalCost: number; + uniqueLearners: number; + totalAttempts: number; + completedAttempts: number; + averageGrade: number; + averageRating: number; + published: boolean; + insights: { + questionInsights: Array<{ + questionId: number; + questionText: string; + correctPercentage: number; + firstAttemptSuccessRate: number; + avgPointsEarned: number; + maxPoints: number; + insight: string; + }>; + performanceInsights: string[]; + costBreakdown: { + grading: number; + questionGeneration: number; + translation: number; + other: number; + }; + }; + }>; + pagination: { + total: number; + page: number; + limit: number; + totalPages: number; + }; +} + /** * Controller that handles assignment-related API endpoints */ @@ -77,6 +126,7 @@ export class AssignmentControllerV2 { private readonly reportService: ReportService, private readonly jobStatusService: JobStatusServiceV2, private readonly prisma: PrismaService, + private readonly adminService: AdminService, ) { this.logger = parentLogger.child({ context: AssignmentControllerV2.name }); } @@ -402,4 +452,33 @@ export class AssignmentControllerV2 { request.userSession.userId, ); } + + /** + * Get assignment analytics with detailed insights + */ + @Get("analytics") + @Roles(UserRole.AUTHOR, UserRole.ADMIN) + @UseGuards(AdminGuard) + @ApiOperation({ + summary: + "Get detailed assignment analytics with insights (for authors and admins)", + }) + @ApiQuery({ name: "page", required: false, type: Number }) + @ApiQuery({ name: "limit", required: false, type: Number }) + @ApiQuery({ name: "search", required: false, type: String }) + @ApiResponse({ status: 200 }) + @ApiResponse({ status: 403 }) + async getAssignmentAnalytics( + @Req() request: UserSessionRequest, + @Query("page", new DefaultValuePipe(1), ParseIntPipe) page: number, + @Query("limit", new DefaultValuePipe(10), ParseIntPipe) limit: number, + @Query("search") search?: string, + ): Promise { + return await this.adminService.getAssignmentAnalytics( + request.userSession, + page, + limit, + search, + ); + } } diff --git a/apps/api/src/api/assignment/v2/controllers/draft-management.controller.ts b/apps/api/src/api/assignment/v2/controllers/draft-management.controller.ts new file mode 100644 index 00000000..7f82b01a --- /dev/null +++ b/apps/api/src/api/assignment/v2/controllers/draft-management.controller.ts @@ -0,0 +1,216 @@ +import { + Body, + Controller, + Delete, + Get, + HttpCode, + HttpStatus, + Param, + ParseIntPipe, + Post, + Put, + Req, + UseGuards, +} from "@nestjs/common"; +import { + ApiBody, + ApiOperation, + ApiParam, + ApiResponse, + ApiTags, +} from "@nestjs/swagger"; +import { + UserRole, + UserSessionRequest, +} from "../../../../auth/interfaces/user.session.interface"; +import { Roles } from "../../../../auth/role/roles.global.guard"; +import { AssignmentAccessControlGuard } from "../../guards/assignment.access.control.guard"; +import { + DraftManagementService, + DraftSummary, + SaveDraftDto, +} from "../services/draft-management.service"; + +@ApiTags("Assignment Draft Management") +@Controller({ + path: "assignments/:assignmentId/drafts", + version: "2", +}) +@UseGuards(AssignmentAccessControlGuard) +export class DraftManagementController { + constructor(private readonly draftService: DraftManagementService) {} + + @Post() + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Save current assignment state as a draft" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiBody({ + schema: { + type: "object", + properties: { + draftName: { + type: "string", + description: "Name for this draft", + }, + assignmentData: { + type: "object", + description: "Assignment data to save", + }, + questionsData: { + type: "array", + description: "Questions data to save", + items: { type: "object" }, + }, + }, + required: ["assignmentData"], + }, + }) + @ApiResponse({ status: 201, description: "Draft saved successfully" }) + @ApiResponse({ status: 404, description: "Assignment not found" }) + @HttpCode(HttpStatus.CREATED) + async saveDraft( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Body() saveDraftDto: SaveDraftDto, + @Req() request: UserSessionRequest, + ): Promise { + return await this.draftService.saveDraft( + assignmentId, + saveDraftDto, + request.userSession, + ); + } + + @Put(":draftId") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Update an existing draft" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ + name: "draftId", + type: "number", + description: "Draft ID to update", + }) + @ApiBody({ + schema: { + type: "object", + properties: { + draftName: { + type: "string", + description: "Name for this draft", + }, + assignmentData: { + type: "object", + description: "Assignment data to save", + }, + questionsData: { + type: "array", + description: "Questions data to save", + items: { type: "object" }, + }, + }, + }, + }) + @ApiResponse({ status: 200, description: "Draft updated successfully" }) + @ApiResponse({ status: 404, description: "Draft not found" }) + async updateDraft( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("draftId", ParseIntPipe) draftId: number, + @Body() saveDraftDto: SaveDraftDto, + @Req() request: UserSessionRequest, + ): Promise { + return await this.draftService.updateDraft( + draftId, + saveDraftDto, + request.userSession, + ); + } + + @Get() + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "List all drafts for current user and assignment" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiResponse({ + status: 200, + description: "List of user drafts", + }) + @ApiResponse({ status: 404, description: "Assignment not found" }) + async listUserDrafts( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Req() request: UserSessionRequest, + ): Promise { + return await this.draftService.listUserDrafts( + assignmentId, + request.userSession, + ); + } + + @Get("latest") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Get user's latest draft for an assignment" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiResponse({ status: 200, description: "Latest draft data" }) + @ApiResponse({ status: 404, description: "No draft found" }) + async getLatestDraft( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Req() request: UserSessionRequest, + ): Promise { + return await this.draftService.getLatestDraft( + assignmentId, + request.userSession, + ); + } + + @Get(":draftId") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Get a specific draft" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ name: "draftId", type: "number", description: "Draft ID" }) + @ApiResponse({ status: 200, description: "Draft data" }) + @ApiResponse({ status: 404, description: "Draft not found" }) + async getDraft( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("draftId", ParseIntPipe) draftId: number, + @Req() request: UserSessionRequest, + ): Promise { + return await this.draftService.getDraft(draftId, request.userSession); + } + + @Delete(":draftId") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Delete a draft" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ name: "draftId", type: "number", description: "Draft ID" }) + @ApiResponse({ status: 204, description: "Draft deleted successfully" }) + @ApiResponse({ status: 404, description: "Draft not found" }) + @HttpCode(HttpStatus.NO_CONTENT) + async deleteDraft( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("draftId", ParseIntPipe) draftId: number, + @Req() request: UserSessionRequest, + ): Promise { + return await this.draftService.deleteDraft(draftId, request.userSession); + } +} diff --git a/apps/api/src/api/assignment/v2/controllers/version-management.controller.ts b/apps/api/src/api/assignment/v2/controllers/version-management.controller.ts new file mode 100644 index 00000000..c594e14e --- /dev/null +++ b/apps/api/src/api/assignment/v2/controllers/version-management.controller.ts @@ -0,0 +1,560 @@ +import { + Body, + Controller, + Delete, + Get, + HttpCode, + HttpStatus, + Param, + ParseIntPipe, + Post, + Put, + Req, + UseGuards, +} from "@nestjs/common"; +import { + ApiBody, + ApiOperation, + ApiParam, + ApiResponse, + ApiTags, +} from "@nestjs/swagger"; +import { + UserRole, + UserSessionRequest, +} from "../../../../auth/interfaces/user.session.interface"; +import { Roles } from "../../../../auth/role/roles.global.guard"; +import { AssignmentAccessControlGuard } from "../../guards/assignment.access.control.guard"; +import { + AutoSaveDto, + CompareVersionsDto, + CreateVersionDto, + RestoreVersionDto, + SaveDraftDto, + UpdateVersionDescriptionDto, + UpdateVersionNumberDto, + VersionComparison, + VersionSummary, +} from "../dtos/version-management.dto"; +import { VersionManagementService } from "../services/version-management.service"; + +@ApiTags("Assignment Version Management") +@Controller({ + path: "assignments/:assignmentId/versions", + version: "2", +}) +@UseGuards(AssignmentAccessControlGuard) +export class VersionManagementController { + constructor(private readonly versionService: VersionManagementService) {} + + @Post() + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Create a new version of an assignment" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiBody({ type: CreateVersionDto }) + @ApiResponse({ status: 201, description: "Version created successfully" }) + @ApiResponse({ status: 404, description: "Assignment not found" }) + @ApiResponse({ status: 403, description: "Insufficient permissions" }) + async createVersion( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Body() createVersionDto: CreateVersionDto, + @Req() request: UserSessionRequest, + ): Promise { + return await this.versionService.createVersion( + assignmentId, + createVersionDto, + request.userSession, + ); + } + + @Get() + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "List all versions of an assignment" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiResponse({ + status: 200, + description: "List of assignment versions", + type: [VersionSummary], + }) + @ApiResponse({ status: 404, description: "Assignment not found" }) + async listVersions( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Req() request: UserSessionRequest, + ): Promise { + console.log( + "Received assignmentId:", + assignmentId, + "type:", + typeof assignmentId, + ); + + if (!request?.userSession) { + throw new Error("User session is required"); + } + + return await this.versionService.listVersions(assignmentId); + } + + @Get(":versionId") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Get a specific version of an assignment" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ name: "versionId", type: "number", description: "Version ID" }) + @ApiResponse({ status: 200, description: "Assignment version details" }) + @ApiResponse({ status: 404, description: "Assignment or version not found" }) + async getVersion( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("versionId", ParseIntPipe) versionId: number, + ): Promise { + return await this.versionService.getVersion(assignmentId, versionId); + } + + @Post("draft") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Save assignment snapshot as a draft version" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiBody({ + schema: { + type: "object", + properties: { + versionNumber: { + type: "string", + description: "Version number (e.g., 1.0.0-rc1)", + }, + versionDescription: { + type: "string", + description: "Version description", + }, + assignmentData: { + type: "object", + description: "Assignment data snapshot", + }, + questionsData: { + type: "array", + description: "Questions data snapshot", + }, + }, + required: ["versionNumber", "assignmentData"], + }, + }) + @ApiResponse({ status: 201, description: "Draft saved successfully" }) + @ApiResponse({ status: 404, description: "Assignment not found" }) + async saveDraft( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Body() + draftData: { + versionNumber: string; + versionDescription?: string; + assignmentData: any; + questionsData?: any[]; + }, + @Req() request: UserSessionRequest, + ): Promise { + return await this.versionService.saveDraftSnapshot( + assignmentId, + draftData, + request.userSession, + ); + } + + @Put(":versionId/restore") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Restore assignment to a specific version" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ + name: "versionId", + type: "number", + description: "Version ID to restore", + }) + @ApiBody({ + schema: { + type: "object", + properties: { + createAsNewVersion: { + type: "boolean", + description: "Create as new version instead of activating existing", + }, + versionDescription: { + type: "string", + description: "Description for the restored version", + }, + }, + }, + }) + @ApiResponse({ status: 200, description: "Version restored successfully" }) + @ApiResponse({ status: 404, description: "Assignment or version not found" }) + async restoreVersion( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("versionId", ParseIntPipe) versionId: number, + @Body() restoreDto: Omit, + @Req() request: UserSessionRequest, + ): Promise { + const restoreVersionDto: RestoreVersionDto = { + ...restoreDto, + versionId, + }; + return await this.versionService.restoreVersion( + assignmentId, + restoreVersionDto, + request.userSession, + ); + } + + @Post("compare") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Compare two versions of an assignment" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiBody({ + schema: { + type: "object", + properties: { + fromVersionId: { + type: "number", + description: "Version ID to compare from", + }, + toVersionId: { + type: "number", + description: "Version ID to compare to", + }, + }, + required: ["fromVersionId", "toVersionId"], + }, + }) + @ApiResponse({ + status: 200, + description: "Version comparison", + type: VersionComparison, + }) + @ApiResponse({ status: 404, description: "Assignment or version not found" }) + @HttpCode(HttpStatus.OK) + async compareVersions( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Body() compareDto: CompareVersionsDto, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + @Req() request: UserSessionRequest, + ): Promise { + return await this.versionService.compareVersions(assignmentId, compareDto); + } + + @Get("history") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Get version history for an assignment" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiResponse({ status: 200, description: "Version history timeline" }) + @ApiResponse({ status: 404, description: "Assignment not found" }) + async getVersionHistory( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Req() request: UserSessionRequest, + ) { + console.log( + "getVersionHistory called with assignmentId:", + assignmentId, + "type:", + typeof assignmentId, + ); + return await this.versionService.getVersionHistory( + assignmentId, + request.userSession, + ); + } + + @Put(":versionId/activate") + @Roles(UserRole.AUTHOR) + @ApiOperation({ + summary: "Activate a specific version as the current version", + }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ + name: "versionId", + type: "number", + description: "Version ID to activate", + }) + @ApiResponse({ status: 200, description: "Version activated successfully" }) + @ApiResponse({ status: 404, description: "Assignment or version not found" }) + async activateVersion( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("versionId", ParseIntPipe) versionId: number, + @Req() request: UserSessionRequest, + ): Promise { + return await this.versionService.restoreVersion( + assignmentId, + { versionId, createAsNewVersion: false }, + request.userSession, + ); + } + + @Put(":versionId/publish") + @Roles(UserRole.AUTHOR) + @ApiOperation({ + summary: "Publish a specific version", + }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ + name: "versionId", + type: "number", + description: "Version ID to publish", + }) + @ApiResponse({ status: 200, description: "Version published successfully" }) + @ApiResponse({ status: 404, description: "Assignment or version not found" }) + @ApiResponse({ status: 400, description: "Version already published" }) + async publishVersion( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("versionId", ParseIntPipe) versionId: number, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + @Req() request: UserSessionRequest, + ): Promise { + return await this.versionService.publishVersion(assignmentId, versionId); + } + + @Post("auto-save") + @Roles(UserRole.AUTHOR) + @ApiOperation({ + summary: + "Auto-save assignment changes as a draft (for temporary server-side saving)", + }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiBody({ + schema: { + type: "object", + properties: { + assignmentData: { + type: "object", + description: "Partial assignment data to save", + }, + questionsData: { + type: "array", + description: "Questions data to save", + items: { type: "object" }, + }, + }, + required: ["assignmentData"], + }, + }) + @ApiResponse({ status: 201, description: "Changes auto-saved successfully" }) + @ApiResponse({ status: 404, description: "Assignment not found" }) + @HttpCode(HttpStatus.CREATED) + async autoSave( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Body() autoSaveData: AutoSaveDto, + @Req() request: UserSessionRequest, + ): Promise { + const saveDraftDto: SaveDraftDto = { + ...autoSaveData, + versionDescription: "Auto-saved changes", + }; + return await this.versionService.saveDraft( + assignmentId, + saveDraftDto, + request.userSession, + ); + } + + @Get("draft/latest") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Get user's latest draft version of an assignment" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiResponse({ status: 200, description: "Latest draft version data" }) + @ApiResponse({ status: 404, description: "No draft found" }) + async getLatestDraft( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Req() request: UserSessionRequest, + ): Promise { + if (!request?.userSession) { + throw new Error("User session is required"); + } + + return await this.versionService.getUserLatestDraft( + assignmentId, + request.userSession, + ); + } + + @Post(":versionId/restore-questions") + @Roles(UserRole.AUTHOR) + @ApiOperation({ + summary: "Restore deleted questions from a specific version", + }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ + name: "versionId", + type: "number", + description: "Version ID to restore questions from", + }) + @ApiBody({ + schema: { + type: "object", + properties: { + questionIds: { + type: "array", + items: { type: "number" }, + description: "Array of question IDs to restore", + }, + }, + required: ["questionIds"], + }, + }) + @ApiResponse({ status: 200, description: "Questions restored successfully" }) + @ApiResponse({ status: 404, description: "Version or questions not found" }) + restoreDeletedQuestions( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("versionId", ParseIntPipe) versionId: number, + @Body() body: { questionIds: number[] }, + @Req() request: UserSessionRequest, + ): Promise { + if (!request?.userSession) { + throw new Error("User session is required"); + } + + return this.versionService.restoreDeletedQuestions( + assignmentId, + versionId, + body.questionIds, + request.userSession, + ); + } + + @Put(":versionId/description") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Update version description" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ + name: "versionId", + type: "number", + description: "Version ID", + }) + @ApiBody({ type: UpdateVersionDescriptionDto }) + @ApiResponse({ + status: 200, + description: "Version description updated successfully", + }) + @ApiResponse({ status: 404, description: "Version not found" }) + async updateVersionDescription( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("versionId", ParseIntPipe) versionId: number, + @Body() updateVersionDescriptionDto: UpdateVersionDescriptionDto, + @Req() request: UserSessionRequest, + ): Promise { + return this.versionService.updateVersionDescription( + assignmentId, + versionId, + updateVersionDescriptionDto.versionDescription, + request.userSession, + ); + } + + @Put(":versionId/version-number") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Update version number" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ + name: "versionId", + type: "number", + description: "Version ID", + }) + @ApiBody({ type: UpdateVersionNumberDto }) + @ApiResponse({ + status: 200, + description: "Version number updated successfully", + }) + @ApiResponse({ status: 404, description: "Version not found" }) + @ApiResponse({ status: 400, description: "Version number already exists" }) + async updateVersionNumber( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("versionId", ParseIntPipe) versionId: number, + @Body() updateVersionNumberDto: UpdateVersionNumberDto, + @Req() request: UserSessionRequest, + ): Promise { + return this.versionService.updateVersionNumber( + assignmentId, + versionId, + updateVersionNumberDto.versionNumber, + request.userSession, + ); + } + + @Delete(":versionId") + @Roles(UserRole.AUTHOR) + @ApiOperation({ summary: "Delete a specific version" }) + @ApiParam({ + name: "assignmentId", + type: "number", + description: "Assignment ID", + }) + @ApiParam({ + name: "versionId", + type: "number", + description: "Version ID to delete", + }) + @ApiResponse({ status: 200, description: "Version deleted successfully" }) + @ApiResponse({ status: 404, description: "Version not found" }) + @ApiResponse({ status: 400, description: "Cannot delete active version" }) + @HttpCode(HttpStatus.OK) + async deleteVersion( + @Param("assignmentId", ParseIntPipe) assignmentId: number, + @Param("versionId", ParseIntPipe) versionId: number, + @Req() request: UserSessionRequest, + ): Promise<{ message: string }> { + await this.versionService.deleteVersion( + assignmentId, + versionId, + request.userSession, + ); + return { message: "Version deleted successfully" }; + } +} diff --git a/apps/api/src/api/assignment/v2/dtos/version-management.dto.ts b/apps/api/src/api/assignment/v2/dtos/version-management.dto.ts new file mode 100644 index 00000000..23108e7c --- /dev/null +++ b/apps/api/src/api/assignment/v2/dtos/version-management.dto.ts @@ -0,0 +1,228 @@ +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; +import { + IsArray, + IsBoolean, + IsNumber, + IsOptional, + IsString, +} from "class-validator"; + +export class CreateVersionDto { + @ApiPropertyOptional({ + description: "Semantic version number (e.g., '1.0.0' or '1.0.0-rc1')", + }) + @IsOptional() + @IsString() + versionNumber?: string; + + @ApiPropertyOptional({ + description: "Description of what changed in this version", + }) + @IsOptional() + @IsString() + versionDescription?: string; + + @ApiPropertyOptional({ + description: "Whether this version should be created as a draft", + default: true, + }) + @IsOptional() + @IsBoolean() + isDraft?: boolean; + + @ApiPropertyOptional({ + description: "Whether this version should be activated immediately", + default: false, + }) + @IsOptional() + @IsBoolean() + shouldActivate?: boolean; + + @ApiPropertyOptional({ + description: "Whether to update existing version if it already exists", + default: false, + }) + @IsOptional() + @IsBoolean() + updateExisting?: boolean; +} + +export class CompareVersionsDto { + @ApiProperty({ description: "Version ID to compare from" }) + @IsNumber() + fromVersionId: number; + + @ApiProperty({ description: "Version ID to compare to" }) + @IsNumber() + toVersionId: number; +} + +export class RestoreVersionDto { + @ApiProperty({ description: "Version ID to restore" }) + @IsNumber() + versionId: number; + + @ApiPropertyOptional({ + description: "Create as new version instead of activating existing", + default: false, + }) + @IsOptional() + @IsBoolean() + createAsNewVersion?: boolean; + + @ApiPropertyOptional({ description: "Description for the restored version" }) + @IsOptional() + @IsString() + versionDescription?: string; +} + +export class SaveDraftDto { + @ApiProperty({ description: "Partial assignment data to save" }) + assignmentData: Record; + + @ApiPropertyOptional({ description: "Questions data to save" }) + @IsOptional() + @IsArray() + questionsData?: Array; + + // version number + @ApiPropertyOptional({ + description: "Semantic version number (e.g., '1.0.0' or '1.0.0-rc1')", + }) + @IsOptional() + @IsString() + versionNumber?: string; + + @ApiPropertyOptional({ description: "Description for this draft save" }) + @IsOptional() + @IsString() + versionDescription?: string; +} + +export class VersionSummary { + @ApiProperty({ description: "Unique version ID" }) + id: number; + + @ApiProperty({ + description: "Semantic version number (e.g., '1.0.0' or '1.0.0-rc1')", + }) + versionNumber: string; + + @ApiPropertyOptional({ + description: "Description of changes in this version", + }) + versionDescription?: string; + + @ApiProperty({ description: "Whether this version is a draft" }) + isDraft: boolean; + + @ApiProperty({ description: "Whether this version is currently active" }) + isActive: boolean; + + @ApiProperty({ description: "Whether this version is published" }) + published: boolean; + + @ApiProperty({ description: "User ID who created this version" }) + createdBy: string; + + @ApiProperty({ description: "When this version was created" }) + createdAt: Date; + + @ApiProperty({ description: "Number of questions in this version" }) + questionCount: number; +} + +export class VersionChangeDto { + @ApiProperty({ description: "Field that changed" }) + field: string; + + @ApiPropertyOptional({ description: "Value in the from version" }) + fromValue: any; + + @ApiPropertyOptional({ description: "Value in the to version" }) + toValue: any; + + @ApiProperty({ + description: "Type of change", + enum: ["added", "modified", "removed"], + }) + changeType: "added" | "modified" | "removed"; +} + +export class QuestionChangeDto { + @ApiPropertyOptional({ description: "Original question ID if applicable" }) + questionId?: number; + + @ApiProperty({ description: "Display order of the question" }) + displayOrder: number; + + @ApiProperty({ + description: "Type of change", + enum: ["added", "modified", "removed"], + }) + changeType: "added" | "modified" | "removed"; + + @ApiPropertyOptional({ + description: "Field that changed within the question", + }) + field?: string; + + @ApiPropertyOptional({ description: "Value in the from version" }) + fromValue?: any; + + @ApiPropertyOptional({ description: "Value in the to version" }) + toValue?: any; +} + +export class VersionComparison { + @ApiProperty({ description: "Summary of the version being compared from" }) + fromVersion: VersionSummary; + + @ApiProperty({ description: "Summary of the version being compared to" }) + toVersion: VersionSummary; + + @ApiProperty({ + description: "Changes in assignment-level fields", + type: [VersionChangeDto], + }) + assignmentChanges: VersionChangeDto[]; + + @ApiProperty({ + description: "Changes in questions", + type: [QuestionChangeDto], + }) + questionChanges: QuestionChangeDto[]; +} + +export class AutoSaveDto { + @ApiProperty({ description: "Partial assignment data to auto-save" }) + assignmentData: Record; + + @ApiPropertyOptional({ description: "Questions data to auto-save" }) + @IsOptional() + @IsArray() + questionsData?: Array; +} + +export class UpdateVersionDescriptionDto { + @ApiProperty({ description: "Updated description for the version" }) + @IsString() + versionDescription: string; +} + +export class UpdateVersionNumberDto { + @ApiProperty({ description: "Updated version number" }) + @IsString() + versionNumber: string; +} + +export class VersionExistsResponse { + @ApiProperty({ description: "Whether the version already exists" }) + versionExists: boolean; + + @ApiProperty({ description: "The existing version details" }) + existingVersion: VersionSummary; + + @ApiProperty({ description: "Message about the conflict" }) + message: string; +} diff --git a/apps/api/src/api/assignment/v2/modules/assignment.module.ts b/apps/api/src/api/assignment/v2/modules/assignment.module.ts index da64c952..4502c116 100644 --- a/apps/api/src/api/assignment/v2/modules/assignment.module.ts +++ b/apps/api/src/api/assignment/v2/modules/assignment.module.ts @@ -1,24 +1,32 @@ -import { Module } from "@nestjs/common"; import { HttpModule } from "@nestjs/axios"; - +import { Module } from "@nestjs/common"; +import { AdminService } from "src/api/admin/admin.service"; +import { LlmModule } from "src/api/llm/llm.module"; +import { AdminVerificationService } from "src/auth/services/admin-verification.service"; +import { PrismaService } from "src/prisma.service"; import { AssignmentControllerV2 } from "../controllers/assignment.controller"; - -import { QuestionService } from "../services/question.service"; -import { AssignmentServiceV2 } from "../services/assignment.service"; -import { JobStatusServiceV2 } from "../services/job-status.service"; -import { ReportService } from "../services/report.repository"; - +import { DraftManagementController } from "../controllers/draft-management.controller"; +import { VersionManagementController } from "../controllers/version-management.controller"; import { AssignmentRepository } from "../repositories/assignment.repository"; import { QuestionRepository } from "../repositories/question.repository"; import { VariantRepository } from "../repositories/variant.repository"; - -import { LlmModule } from "src/api/llm/llm.module"; -import { PrismaService } from "src/prisma.service"; +import { AssignmentServiceV2 } from "../services/assignment.service"; +import { DraftManagementService } from "../services/draft-management.service"; +import { JobStatusServiceV2 } from "../services/job-status.service"; +import { QuestionService } from "../services/question.service"; +import { ReportService } from "../services/report.repository"; +import { VersionManagementService } from "../services/version-management.service"; @Module({ - controllers: [AssignmentControllerV2], + controllers: [ + AssignmentControllerV2, + VersionManagementController, + DraftManagementController, + ], providers: [ AssignmentServiceV2, + VersionManagementService, + DraftManagementService, QuestionService, ReportService, JobStatusServiceV2, @@ -26,10 +34,17 @@ import { PrismaService } from "src/prisma.service"; AssignmentRepository, QuestionRepository, VariantRepository, - + AdminVerificationService, PrismaService, + AdminService, ], imports: [HttpModule, LlmModule], - exports: [AssignmentServiceV2, QuestionService, JobStatusServiceV2], + exports: [ + AssignmentServiceV2, + VersionManagementService, + DraftManagementService, + QuestionService, + JobStatusServiceV2, + ], }) export class AssignmentModuleV2 {} diff --git a/apps/api/src/api/assignment/v2/repositories/assignment.repository.ts b/apps/api/src/api/assignment/v2/repositories/assignment.repository.ts index 73b6c3cb..b70ea837 100644 --- a/apps/api/src/api/assignment/v2/repositories/assignment.repository.ts +++ b/apps/api/src/api/assignment/v2/repositories/assignment.repository.ts @@ -6,19 +6,19 @@ import { QuestionVariant, } from "@prisma/client"; import { - UserSession, UserRole, + UserSession, } from "src/auth/interfaces/user.session.interface"; import { PrismaService } from "src/prisma.service"; import { + AssignmentResponseDto, GetAssignmentResponseDto, LearnerGetAssignmentResponseDto, - AssignmentResponseDto, } from "../../dto/get.assignment.response.dto"; import { + Choice, QuestionDto, ScoringDto, - Choice, VariantDto, VideoPresentationConfig, } from "../../dto/update.questions.request.dto"; @@ -90,6 +90,22 @@ export class AssignmentRepository { async findAllForUser( userSession: UserSession, ): Promise { + // If user is an author, only show assignments they've authored + if (userSession.role === UserRole.AUTHOR) { + const authoredAssignments = await this.prisma.assignment.findMany({ + where: { + AssignmentAuthor: { + some: { + userId: userSession.userId, + }, + }, + }, + }); + + return authoredAssignments; + } + + // For non-authors (learners, admins), show assignments from their group const results = await this.prisma.assignmentGroup.findMany({ where: { groupId: userSession.groupId }, include: { diff --git a/apps/api/src/api/assignment/v2/repositories/question.repository.ts b/apps/api/src/api/assignment/v2/repositories/question.repository.ts index 6ee0c8c8..73e0522a 100644 --- a/apps/api/src/api/assignment/v2/repositories/question.repository.ts +++ b/apps/api/src/api/assignment/v2/repositories/question.repository.ts @@ -3,10 +3,10 @@ import { Injectable, Logger } from "@nestjs/common"; import { Prisma, Question, QuestionVariant } from "@prisma/client"; import { PrismaService } from "src/prisma.service"; import { - QuestionDto, - VariantDto, Choice, + QuestionDto, ScoringDto, + VariantDto, VideoPresentationConfig, } from "../../dto/update.questions.request.dto"; @@ -71,7 +71,7 @@ export class QuestionRepository { */ async upsert(questionDto: QuestionDto): Promise { try { - const { id, variants, ...questionData } = questionDto; + const { id, ...questionData } = questionDto; if (id === undefined) { throw new Error("Question ID is required for upsert operation"); diff --git a/apps/api/src/api/assignment/v2/repositories/variant.repository.ts b/apps/api/src/api/assignment/v2/repositories/variant.repository.ts index 492e8635..b3373212 100644 --- a/apps/api/src/api/assignment/v2/repositories/variant.repository.ts +++ b/apps/api/src/api/assignment/v2/repositories/variant.repository.ts @@ -1,13 +1,13 @@ /* eslint-disable unicorn/no-null */ import { Injectable, Logger } from "@nestjs/common"; import { Prisma, QuestionVariant } from "@prisma/client"; +import { PrismaService } from "src/prisma.service"; import { Choice, ScoringDto, VariantDto, VariantType, } from "../../dto/update.questions.request.dto"; -import { PrismaService } from "src/prisma.service"; /** * Repository for Question Variant data access operations diff --git a/apps/api/src/api/assignment/v2/services/__tests__/version-management.service.spec.ts b/apps/api/src/api/assignment/v2/services/__tests__/version-management.service.spec.ts new file mode 100644 index 00000000..af58296d --- /dev/null +++ b/apps/api/src/api/assignment/v2/services/__tests__/version-management.service.spec.ts @@ -0,0 +1,310 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { NotFoundException } from "@nestjs/common"; +import { Test, TestingModule } from "@nestjs/testing"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { UserRole } from "../../../../../auth/interfaces/user.session.interface"; +import { PrismaService } from "../../../../../prisma.service"; +import { VersionManagementService } from "../version-management.service"; + +describe("VersionManagementService", () => { + let service: VersionManagementService; + + const mockPrismaService = { + assignment: { + findUnique: jest.fn(), + }, + assignmentVersion: { + create: jest.fn(), + findMany: jest.fn(), + findUnique: jest.fn(), + updateMany: jest.fn(), + update: jest.fn(), + }, + questionVersion: { + create: jest.fn(), + }, + versionHistory: { + create: jest.fn(), + findMany: jest.fn(), + }, + $transaction: jest.fn(), + }; + + const mockLogger = { + child: jest.fn().mockReturnThis(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + VersionManagementService, + { + provide: PrismaService, + useValue: mockPrismaService, + }, + { + provide: WINSTON_MODULE_PROVIDER, + useValue: mockLogger, + }, + ], + }).compile(); + + service = module.get(VersionManagementService); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe("listVersions", () => { + const mockUserSession = { + userId: "user123", + role: UserRole.AUTHOR, + }; + + it("should return versions for valid assignment", async () => { + const mockAssignment = { + id: 1, + AssignmentAuthor: [{ userId: "user123" }], + }; + + const mockVersions = [ + { + id: 1, + versionNumber: 1, + isActive: true, + isDraft: false, + createdBy: "user123", + createdAt: new Date(), + _count: { questionVersions: 5 }, + }, + { + id: 2, + versionNumber: 2, + isActive: false, + isDraft: true, + createdBy: "user123", + createdAt: new Date(), + _count: { questionVersions: 3 }, + }, + ]; + + mockPrismaService.assignment.findUnique.mockResolvedValue(mockAssignment); + mockPrismaService.assignmentVersion.findMany.mockResolvedValue( + mockVersions, + ); + + const result = await service.listVersions(1); + + expect(result).toHaveLength(2); + expect(result[0]).toHaveProperty("versionNumber", 1); + expect(result[0]).toHaveProperty("questionCount", 5); + expect(result[1]).toHaveProperty("versionNumber", 2); + expect(result[1]).toHaveProperty("questionCount", 3); + }); + + it("should throw NotFoundException for non-existent assignment", async () => { + mockPrismaService.assignment.findUnique.mockResolvedValue(null); + + await expect(service.listVersions(999)).rejects.toThrow( + NotFoundException, + ); + }); + }); + + describe("getVersion", () => { + const mockUserSession = { + userId: "user123", + role: UserRole.AUTHOR, + }; + + it("should return version details", async () => { + const mockVersion = { + id: 1, + assignmentId: 1, + versionNumber: 1, + name: "Test Assignment", + questionVersions: [{ id: 1, question: "What is 2+2?" }], + }; + + mockPrismaService.assignmentVersion.findUnique.mockResolvedValue( + mockVersion, + ); + + const result = await service.getVersion(1, 1); + + expect(result).toEqual(mockVersion); + expect( + mockPrismaService.assignmentVersion.findUnique, + ).toHaveBeenCalledWith({ + where: { id: 1, assignmentId: 1 }, + include: { questionVersions: { orderBy: { displayOrder: "asc" } } }, + }); + }); + + it("should throw NotFoundException for non-existent version", async () => { + mockPrismaService.assignmentVersion.findUnique.mockResolvedValue(null); + + await expect(service.getVersion(1, 999)).rejects.toThrow( + NotFoundException, + ); + }); + }); + + describe("compareAssignmentData", () => { + it("should detect assignment field changes", () => { + const fromVersion = { + name: "Old Name", + introduction: "Old Introduction", + published: false, + }; + + const toVersion = { + name: "New Name", + introduction: "Old Introduction", + published: true, + }; + + // Access private method for testing + const result = (service as any).compareAssignmentData( + fromVersion, + toVersion, + ); + + expect(result).toHaveLength(2); + expect(result[0]).toMatchObject({ + field: "name", + fromValue: "Old Name", + toValue: "New Name", + changeType: "modified", + }); + expect(result[1]).toMatchObject({ + field: "published", + fromValue: false, + toValue: true, + changeType: "modified", + }); + }); + + it("should handle null values", () => { + const fromVersion = { + name: "Test", + introduction: null, + }; + + const toVersion = { + name: "Test", + introduction: "Added introduction", + }; + + const result = (service as any).compareAssignmentData( + fromVersion, + toVersion, + ); + + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + field: "introduction", + fromValue: null, + toValue: "Added introduction", + changeType: "added", + }); + }); + }); + + describe("compareQuestionData", () => { + it("should detect added questions", () => { + const fromQuestions = [ + { id: 1, question: "Question 1", displayOrder: 1 }, + ]; + + const toQuestions = [ + { id: 1, question: "Question 1", displayOrder: 1 }, + { id: 2, question: "Question 2", displayOrder: 2 }, + ]; + + const result = (service as any).compareQuestionData( + fromQuestions, + toQuestions, + ); + + const addedChanges = result.filter((c: any) => c.changeType === "added"); + expect(addedChanges).toHaveLength(1); + expect(addedChanges[0]).toMatchObject({ + questionId: 2, + displayOrder: 2, + changeType: "added", + }); + }); + + it("should detect removed questions", () => { + const fromQuestions = [ + { id: 1, question: "Question 1", displayOrder: 1 }, + { id: 2, question: "Question 2", displayOrder: 2 }, + ]; + + const toQuestions = [{ id: 1, question: "Question 1", displayOrder: 1 }]; + + const result = (service as any).compareQuestionData( + fromQuestions, + toQuestions, + ); + + const removedChanges = result.filter( + (c: any) => c.changeType === "removed", + ); + expect(removedChanges).toHaveLength(1); + expect(removedChanges[0]).toMatchObject({ + questionId: 2, + displayOrder: 2, + changeType: "removed", + }); + }); + + it("should detect modified questions", () => { + const fromQuestions = [ + { id: 1, question: "Old Question", totalPoints: 5, displayOrder: 1 }, + ]; + + const toQuestions = [ + { id: 1, question: "New Question", totalPoints: 10, displayOrder: 1 }, + ]; + + const result = (service as any).compareQuestionData( + fromQuestions, + toQuestions, + ); + + const modifiedChanges = result.filter( + (c: any) => c.changeType === "modified", + ); + expect(modifiedChanges).toHaveLength(2); // question text and totalPoints + + expect(modifiedChanges).toContainEqual( + expect.objectContaining({ + questionId: 1, + field: "question", + fromValue: "Old Question", + toValue: "New Question", + changeType: "modified", + }), + ); + + expect(modifiedChanges).toContainEqual( + expect.objectContaining({ + questionId: 1, + field: "totalPoints", + fromValue: 5, + toValue: 10, + changeType: "modified", + }), + ); + }); + }); +}); diff --git a/apps/api/src/api/assignment/v2/services/assignment.service.ts b/apps/api/src/api/assignment/v2/services/assignment.service.ts index c2a2b1d2..9a7b1cbb 100644 --- a/apps/api/src/api/assignment/v2/services/assignment.service.ts +++ b/apps/api/src/api/assignment/v2/services/assignment.service.ts @@ -1,6 +1,7 @@ import { Inject, Injectable } from "@nestjs/common"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { UserSession } from "src/auth/interfaces/user.session.interface"; +import { PrismaService } from "src/prisma.service"; import { Logger } from "winston"; import { BaseAssignmentResponseDto } from "../../dto/base.assignment.response.dto"; import { @@ -20,6 +21,10 @@ import { AssignmentRepository } from "../repositories/assignment.repository"; import { JobStatusServiceV2 } from "./job-status.service"; import { QuestionService } from "./question.service"; import { TranslationService } from "./translation.service"; +import { + VersionManagementService, + VersionSummary, +} from "./version-management.service"; /** * Service for managing assignment operations @@ -31,7 +36,9 @@ export class AssignmentServiceV2 { private readonly assignmentRepository: AssignmentRepository, private readonly questionService: QuestionService, private readonly translationService: TranslationService, + private readonly versionManagementService: VersionManagementService, private readonly jobStatusService: JobStatusServiceV2, + private readonly prisma: PrismaService, @Inject(WINSTON_MODULE_PROVIDER) private parentLogger: Logger, ) { this.logger = parentLogger.child({ context: "AssignmentServiceV2" }); @@ -156,12 +163,15 @@ export class AssignmentServiceV2 { updateDto: UpdateAssignmentQuestionsDto, userId: string, ): Promise<{ jobId: number; message: string }> { + this.logger.info( + `📦 PUBLISH REQUEST: Received updateDto with versionNumber: ${updateDto.versionNumber}, versionDescription: ${updateDto.versionDescription}`, + ); const job = await this.jobStatusService.createPublishJob( assignmentId, userId, ); - this.startPublishingProcess(job.id, assignmentId, updateDto).catch( + this.startPublishingProcess(job.id, assignmentId, updateDto, userId).catch( (error: unknown) => { const errorMessage = error instanceof Error ? error.message : "Unknown error"; @@ -183,6 +193,7 @@ export class AssignmentServiceV2 { jobId: number, assignmentId: number, updateDto: UpdateAssignmentQuestionsDto, + userId: string, ): Promise { try { // Progress allocation: @@ -221,6 +232,7 @@ export class AssignmentServiceV2 { showAssignmentScore: updateDto.showAssignmentScore, showQuestionScore: updateDto.showQuestionScore, showSubmissionFeedback: updateDto.showSubmissionFeedback, + showCorrectAnswer: updateDto.showCorrectAnswer, timeEstimateMinutes: updateDto.timeEstimateMinutes, showQuestions: updateDto.showQuestions, numberOfQuestionsPerAttempt: updateDto.numberOfQuestionsPerAttempt, @@ -232,6 +244,29 @@ export class AssignmentServiceV2 { percentage: 10, }); + try { + await this.prisma.assignmentAuthor.upsert({ + where: { + assignmentId_userId: { + assignmentId, + userId, + }, + }, + update: {}, + create: { + assignmentId, + userId, + }, + }); + } catch (error) { + // Log but don't fail the publishing process if author tracking fails + this.logger.warn( + `Failed to store assignment author: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + } + let questionContentChanged = false; if (updateDto.questions && updateDto.questions.length > 0) { @@ -284,11 +319,123 @@ export class AssignmentServiceV2 { end: 90, }); } else { + // Only check language consistency if no content changes and we have existing translations await this.jobStatusService.updateJobStatus(jobId, { status: "In Progress", - progress: "No content changes detected, skipping translation", - percentage: 85, + progress: "Checking for language consistency issues", + percentage: 78, }); + + // Quick check to see if we have existing translations first + const hasExistingTranslations = + await this.prisma.assignmentTranslation.count({ + where: { assignmentId }, + }); + + if (hasExistingTranslations > 0) { + // Use quick validation instead of expensive language detection + // const isValid = await this.translationService.quickValidateAssignmentTranslations( + // assignmentId, + // ); + const isValid = true; + if (isValid) { + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: + "Translation validation passed, skipping consistency check", + percentage: 85, + }); + } else { + // Only do expensive validation if quick check fails + this.logger.warn( + `Quick validation failed for assignment ${assignmentId}, running full validation`, + ); + + const languageValidation = + await this.translationService.validateAssignmentLanguageConsistency( + assignmentId, + ); + + if (languageValidation.isConsistent) { + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: "Language consistency validated, no issues found", + percentage: 85, + }); + } else { + this.logger.warn( + `Language consistency issues detected for assignment ${assignmentId}: ${languageValidation.mismatchedLanguages.join( + ", ", + )}`, + ); + + // Language mismatch detected - force retranslation for affected languages + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: `Language mismatch detected for ${languageValidation.mismatchedLanguages.length} languages, refreshing translations`, + percentage: 80, + }); + + await this.translationService.retranslateAssignmentForLanguages( + assignmentId, + languageValidation.mismatchedLanguages, + jobId, + ); + + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: "Translation refresh completed", + percentage: 90, + }); + } + } + } else { + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: + "No existing translations to validate, skipping consistency check", + percentage: 85, + }); + } + } + + // Final translation validation + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: "Validating translation completeness", + percentage: 88, + }); + + const translationCompleteness = + await this.translationService.ensureTranslationCompleteness( + assignmentId, + ); + + if (!translationCompleteness.isComplete) { + this.logger.warn( + `Missing translations detected for assignment ${assignmentId}. Attempting to fix...`, + { missingTranslations: translationCompleteness.missingTranslations }, + ); + + // Attempt to fix missing translations + for (const missing of translationCompleteness.missingTranslations) { + try { + // Note: We're not fixing missing translations here + // This is just logging for monitoring + this.logger.warn( + `Missing translations for ${ + missing.variantId + ? `variant ${missing.variantId}` + : `question ${missing.questionId}` + }: ${missing.missingLanguages.join(", ")}`, + ); + } catch (error) { + this.logger.error( + `Failed to fix missing translation for question ${missing.questionId}`, + error, + ); + } + } } await this.jobStatusService.updateJobStatus(jobId, { @@ -305,7 +452,7 @@ export class AssignmentServiceV2 { await this.assignmentRepository.update(assignmentId, { questionOrder, - published: true, + published: updateDto.published, }); const updatedQuestions = @@ -317,6 +464,173 @@ export class AssignmentServiceV2 { return indexA - indexB; }); + // Log the questions that were found after processing + this.logger.info( + `Found ${updatedQuestions.length} questions after processing for assignment ${assignmentId}`, + { + questionIds: updatedQuestions.map((q) => q.id), + }, + ); + + // Create a new version when publishing - AFTER questions are processed and committed + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: "Creating version snapshot", + percentage: 95, + }); + + try { + this.logger.info( + `Managing version after question processing - found ${updatedQuestions.length} questions`, + ); + + const userSession = { + userId, + role: "AUTHOR", + } as unknown as UserSession; + + // Check if there's an existing draft version to update/publish + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const existingDraft = + await this.versionManagementService.getUserLatestDraft( + assignmentId, + userSession, + ); + + // Check for recently created unpublished versions (to prevent duplicates from frontend) + const latestVersion = + await this.versionManagementService.getLatestVersion(assignmentId); + + let versionResult: VersionSummary; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + if ( + existingDraft && + updateDto.published && + existingDraft?._draftVersionId + ) { + // If we have a draft and we're publishing, publish the existing draft + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const draftVersionId = existingDraft._draftVersionId; + this.logger.info( + `Found existing draft version, publishing it instead of creating new version`, + { draftVersionId }, + ); + + // Update the existing draft with current content first + await this.versionManagementService.saveDraft( + assignmentId, + { + assignmentData: { + name: updateDto.name, + introduction: updateDto.introduction, + instructions: updateDto.instructions, + gradingCriteriaOverview: updateDto.gradingCriteriaOverview, + timeEstimateMinutes: updateDto.timeEstimateMinutes, + }, + questionsData: updatedQuestions, + versionDescription: + updateDto.versionDescription ?? + `Published version - ${new Date().toLocaleDateString()}`, + versionNumber: updateDto.versionNumber, + }, + userSession, + ); + + // Then publish the updated draft + versionResult = await this.versionManagementService.publishVersion( + assignmentId, + draftVersionId, + ); + } else if ( + latestVersion && + !latestVersion.published && + updateDto.published + ) { + // There's a recently created unpublished version - publish it instead of creating new one + this.logger.info( + `Found recently created unpublished version ${latestVersion.versionNumber}, publishing it instead of creating duplicate`, + { + versionId: latestVersion.id, + versionNumber: latestVersion.versionNumber, + }, + ); + + versionResult = await this.versionManagementService.publishVersion( + assignmentId, + latestVersion.id, + ); + } else if (!existingDraft && updateDto.published) { + // No existing draft and no unpublished version - create new version directly + this.logger.info( + `No existing draft or unpublished version found, creating new version directly`, + ); + this.logger.info( + `UpdateDto contains versionNumber: ${updateDto.versionNumber}, versionDescription: ${updateDto.versionDescription}`, + ); + + versionResult = await this.versionManagementService.createVersion( + assignmentId, + { + versionNumber: updateDto.versionNumber, + versionDescription: + updateDto.versionDescription ?? + `Version - ${new Date().toLocaleDateString()}`, + isDraft: false, // Create as published directly + shouldActivate: true, + }, + userSession, + ); + } else { + // Create or update draft version (not publishing) + this.logger.info(`Saving as draft version`); + + versionResult = await this.versionManagementService.saveDraft( + assignmentId, + { + assignmentData: { + name: updateDto.name, + introduction: updateDto.introduction, + instructions: updateDto.instructions, + gradingCriteriaOverview: updateDto.gradingCriteriaOverview, + timeEstimateMinutes: updateDto.timeEstimateMinutes, + }, + questionsData: updatedQuestions, + versionDescription: + updateDto.versionDescription ?? + `Draft - ${new Date().toLocaleDateString()}`, + versionNumber: updateDto.versionNumber, + }, + userSession, + ); + } + + this.logger.info( + `Successfully managed version ${versionResult.id} for assignment ${assignmentId} during publishing with ${versionResult.questionCount} questions`, + { + versionNumber: versionResult.versionNumber, + isDraft: versionResult.isDraft, + isActive: versionResult.isActive, + published: versionResult.published, + }, + ); + } catch (versionError) { + // Log the full error details + this.logger.error( + `Failed to create version during publishing for assignment ${assignmentId}:`, + { + error: + versionError instanceof Error + ? versionError.message + : "Unknown error", + stack: + versionError instanceof Error ? versionError.stack : undefined, + assignmentId, + userId, + questionsFound: updatedQuestions.length, + }, + ); + } + await this.jobStatusService.updateJobStatus(jobId, { status: "Completed", progress: diff --git a/apps/api/src/api/assignment/v2/services/draft-management.service.ts b/apps/api/src/api/assignment/v2/services/draft-management.service.ts new file mode 100644 index 00000000..a2d9f858 --- /dev/null +++ b/apps/api/src/api/assignment/v2/services/draft-management.service.ts @@ -0,0 +1,519 @@ +import { + BadRequestException, + Inject, + Injectable, + NotFoundException, +} from "@nestjs/common"; +import { + AssignmentQuestionDisplayOrder, + QuestionDisplay, +} from "@prisma/client"; +import { JsonValue } from "aws-sdk/clients/glue"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { + UserRole, + UserSession, +} from "src/auth/interfaces/user.session.interface"; +import { Logger } from "winston"; +import { PrismaService } from "../../../../prisma.service"; + +export interface SaveDraftDto { + draftName?: string; + assignmentData: Partial<{ + name: string; + introduction: string; + instructions: string; + gradingCriteriaOverview: string; + timeEstimateMinutes: number; + type: string; + graded: boolean; + numAttempts: number; + allotedTimeMinutes: number; + attemptsPerTimeRange: number; + attemptsTimeRangeHours: number; + passingGrade: number; + displayOrder: AssignmentQuestionDisplayOrder; + questionDisplay: QuestionDisplay; + numberOfQuestionsPerAttempt: number; + questionOrder: number[]; + showAssignmentScore: boolean; + showQuestionScore: boolean; + showSubmissionFeedback: boolean; + showQuestions: boolean; + languageCode: string; + }>; + questionsData?: Array; +} + +export interface DraftSummary { + id: number; + draftName: string; + assignmentName: string; + userId: string; + createdAt: Date; + updatedAt: Date; + questionCount: number; +} + +@Injectable() +export class DraftManagementService { + private readonly logger: Logger; + + constructor( + private readonly prisma: PrismaService, + @Inject(WINSTON_MODULE_PROVIDER) private parentLogger: Logger, + ) { + this.logger = parentLogger.child({ context: "DraftManagementService" }); + } + + private parseDisplayOrder( + value: any, + ): AssignmentQuestionDisplayOrder | undefined { + if (value === "DEFINED" || value === "RANDOM") { + return value as AssignmentQuestionDisplayOrder; + } + return undefined; + } + + private parseQuestionDisplay(value: any): QuestionDisplay | undefined { + if (value === "ONE_PER_PAGE" || value === "ALL_PER_PAGE") { + return value as QuestionDisplay; + } + return undefined; + } + + async saveDraft( + assignmentId: number, + saveDraftDto: SaveDraftDto, + userSession: UserSession, + ): Promise { + this.logger.info(`Saving draft for assignment ${assignmentId}`, { + userId: userSession.userId, + draftName: saveDraftDto.draftName, + }); + + // Verify assignment exists and user has access + // await this.verifyAssignmentAccess(assignmentId, userSession); + + // Get the base assignment for reference + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { questions: { where: { isDeleted: false } } }, + }); + + if (!assignment) { + throw new NotFoundException("Assignment not found"); + } + + const draftName = + saveDraftDto.draftName || `Draft - ${new Date().toLocaleString()}`; + + return await this.prisma.$transaction(async (tx) => { + // Create assignment draft + const assignmentDraft = await tx.assignmentDraft.create({ + data: { + assignmentId, + userId: userSession.userId, + draftName, + name: saveDraftDto.assignmentData?.name || assignment.name, + introduction: + saveDraftDto.assignmentData?.introduction ?? + assignment.introduction, + instructions: + saveDraftDto.assignmentData?.instructions ?? + assignment.instructions, + gradingCriteriaOverview: + saveDraftDto.assignmentData?.gradingCriteriaOverview ?? + assignment.gradingCriteriaOverview, + timeEstimateMinutes: + saveDraftDto.assignmentData?.timeEstimateMinutes || + assignment.timeEstimateMinutes, + type: assignment.type, + graded: saveDraftDto.assignmentData?.graded ?? assignment.graded, + numAttempts: + saveDraftDto.assignmentData?.numAttempts ?? assignment.numAttempts, + allotedTimeMinutes: + saveDraftDto.assignmentData?.allotedTimeMinutes ?? + assignment.allotedTimeMinutes, + attemptsPerTimeRange: + saveDraftDto.assignmentData?.attemptsPerTimeRange ?? + assignment.attemptsPerTimeRange, + attemptsTimeRangeHours: + saveDraftDto.assignmentData?.attemptsTimeRangeHours ?? + assignment.attemptsTimeRangeHours, + passingGrade: + saveDraftDto.assignmentData?.passingGrade ?? + assignment.passingGrade, + displayOrder: + this.parseDisplayOrder(saveDraftDto.assignmentData?.displayOrder) ?? + assignment.displayOrder, + questionDisplay: + this.parseQuestionDisplay( + saveDraftDto.assignmentData?.questionDisplay, + ) ?? assignment.questionDisplay, + numberOfQuestionsPerAttempt: + saveDraftDto.assignmentData?.numberOfQuestionsPerAttempt ?? + assignment.numberOfQuestionsPerAttempt, + questionOrder: + saveDraftDto.assignmentData?.questionOrder ?? + assignment.questionOrder, + published: false, + showAssignmentScore: + saveDraftDto.assignmentData?.showAssignmentScore ?? + assignment.showAssignmentScore, + showQuestionScore: + saveDraftDto.assignmentData?.showQuestionScore ?? + assignment.showQuestionScore, + showSubmissionFeedback: + saveDraftDto.assignmentData?.showSubmissionFeedback ?? + assignment.showSubmissionFeedback, + showQuestions: + saveDraftDto.assignmentData?.showQuestions ?? + assignment.showQuestions, + languageCode: + saveDraftDto.assignmentData?.languageCode ?? + assignment.languageCode, + questionsData: saveDraftDto.questionsData + ? JSON.stringify(saveDraftDto.questionsData) + : null, + }, + }); + + this.logger.info( + `Created draft "${draftName}" for assignment ${assignmentId}`, + { + draftId: assignmentDraft.id, + userId: userSession.userId, + }, + ); + + return { + id: assignmentDraft.id, + draftName: assignmentDraft.draftName, + assignmentName: assignmentDraft.name, + userId: assignmentDraft.userId, + createdAt: assignmentDraft.createdAt, + updatedAt: assignmentDraft.updatedAt, + questionCount: saveDraftDto.questionsData?.length || 0, + }; + }); + } + + async updateDraft( + draftId: number, + saveDraftDto: SaveDraftDto, + userSession: UserSession, + ): Promise { + this.logger.info(`Updating draft ${draftId}`, { + userId: userSession.userId, + }); + + // Verify draft exists and user owns it + const existingDraft = await this.prisma.assignmentDraft.findUnique({ + where: { id: draftId }, + }); + + if (!existingDraft) { + throw new NotFoundException("Draft not found"); + } + + if (existingDraft.userId !== userSession.userId) { + throw new BadRequestException("You can only update your own drafts"); + } + + const updatedDraft = await this.prisma.assignmentDraft.update({ + where: { id: draftId }, + data: { + ...(saveDraftDto.draftName && { draftName: saveDraftDto.draftName }), + ...(saveDraftDto.assignmentData?.name && { + name: saveDraftDto.assignmentData.name, + }), + ...(saveDraftDto.assignmentData?.introduction !== undefined && { + introduction: saveDraftDto.assignmentData.introduction, + }), + ...(saveDraftDto.assignmentData?.instructions !== undefined && { + instructions: saveDraftDto.assignmentData.instructions, + }), + ...(saveDraftDto.assignmentData?.gradingCriteriaOverview !== + undefined && { + gradingCriteriaOverview: + saveDraftDto.assignmentData.gradingCriteriaOverview, + }), + ...(saveDraftDto.assignmentData?.timeEstimateMinutes && { + timeEstimateMinutes: saveDraftDto.assignmentData.timeEstimateMinutes, + }), + ...(saveDraftDto.assignmentData?.graded !== undefined && { + graded: saveDraftDto.assignmentData.graded, + }), + ...(saveDraftDto.assignmentData?.numAttempts !== undefined && { + numAttempts: saveDraftDto.assignmentData.numAttempts, + }), + ...(saveDraftDto.assignmentData?.allotedTimeMinutes !== undefined && { + allotedTimeMinutes: saveDraftDto.assignmentData.allotedTimeMinutes, + }), + ...(saveDraftDto.assignmentData?.attemptsPerTimeRange !== undefined && { + attemptsPerTimeRange: + saveDraftDto.assignmentData.attemptsPerTimeRange, + }), + ...(saveDraftDto.assignmentData?.attemptsTimeRangeHours !== + undefined && { + attemptsTimeRangeHours: + saveDraftDto.assignmentData.attemptsTimeRangeHours, + }), + ...(saveDraftDto.assignmentData?.passingGrade !== undefined && { + passingGrade: saveDraftDto.assignmentData.passingGrade, + }), + ...(saveDraftDto.assignmentData?.displayOrder !== undefined && { + displayOrder: this.parseDisplayOrder( + saveDraftDto.assignmentData.displayOrder, + ), + }), + ...(saveDraftDto.assignmentData?.questionDisplay !== undefined && { + questionDisplay: this.parseQuestionDisplay( + saveDraftDto.assignmentData.questionDisplay, + ), + }), + ...(saveDraftDto.assignmentData?.numberOfQuestionsPerAttempt !== + undefined && { + numberOfQuestionsPerAttempt: + saveDraftDto.assignmentData.numberOfQuestionsPerAttempt, + }), + ...(saveDraftDto.assignmentData?.questionOrder && { + questionOrder: saveDraftDto.assignmentData.questionOrder, + }), + ...(saveDraftDto.assignmentData?.showAssignmentScore !== undefined && { + showAssignmentScore: saveDraftDto.assignmentData.showAssignmentScore, + }), + ...(saveDraftDto.assignmentData?.showQuestionScore !== undefined && { + showQuestionScore: saveDraftDto.assignmentData.showQuestionScore, + }), + ...(saveDraftDto.assignmentData?.showSubmissionFeedback !== + undefined && { + showSubmissionFeedback: + saveDraftDto.assignmentData.showSubmissionFeedback, + }), + ...(saveDraftDto.assignmentData?.showQuestions !== undefined && { + showQuestions: saveDraftDto.assignmentData.showQuestions, + }), + ...(saveDraftDto.assignmentData?.languageCode && { + languageCode: saveDraftDto.assignmentData.languageCode, + }), + ...(saveDraftDto.questionsData && { + questionsData: JSON.stringify(saveDraftDto.questionsData), + }), + }, + }); + + return { + id: updatedDraft.id, + draftName: updatedDraft.draftName, + assignmentName: updatedDraft.name, + userId: updatedDraft.userId, + createdAt: updatedDraft.createdAt, + updatedAt: updatedDraft.updatedAt, + questionCount: saveDraftDto.questionsData?.length || 0, + }; + } + + async listUserDrafts( + assignmentId: number, + userSession: UserSession, + ): Promise { + // Verify assignment access + // await this.verifyAssignmentAccess(assignmentId, userSession); + + const drafts = await this.prisma.assignmentDraft.findMany({ + where: { + assignmentId, + userId: userSession.userId, + }, + orderBy: { updatedAt: "desc" }, + }); + + return drafts.map((draft) => ({ + id: draft.id, + draftName: draft.draftName, + assignmentName: draft.name, + userId: draft.userId, + createdAt: draft.createdAt, + updatedAt: draft.updatedAt, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + questionCount: JSON.parse(draft.questionsData as string).length ?? 0, + })); + } + + async getDraft( + draftId: number, + userSession: UserSession, + ): Promise<{ + id: number; + name: string; + introduction: string; + instructions: string; + gradingCriteriaOverview: string; + timeEstimateMinutes: number; + type: string; + graded: boolean; + numAttempts: number; + allotedTimeMinutes: number; + attemptsPerTimeRange: number; + attemptsTimeRangeHours: number; + passingGrade: number; + displayOrder: AssignmentQuestionDisplayOrder; + questionDisplay: QuestionDisplay; + numberOfQuestionsPerAttempt: number; + questionOrder: number[]; + published: boolean; + showAssignmentScore: boolean; + showQuestionScore: boolean; + showSubmissionFeedback: boolean; + showQuestions: boolean; + languageCode: string; + questions: JsonValue[]; + _isDraft?: boolean; + _draftId?: number; + _draftName?: string; + _draftUpdatedAt?: Date; + }> { + const draft = await this.prisma.assignmentDraft.findUnique({ + where: { id: draftId }, + }); + + if (!draft) { + throw new NotFoundException("Draft not found"); + } + + if (draft.userId !== userSession.userId) { + throw new BadRequestException("You can only access your own drafts"); + } + + return { + id: draft.assignmentId, + name: draft.name, + introduction: draft.introduction, + instructions: draft.instructions, + gradingCriteriaOverview: draft.gradingCriteriaOverview, + timeEstimateMinutes: draft.timeEstimateMinutes, + type: draft.type, + graded: draft.graded, + numAttempts: draft.numAttempts, + allotedTimeMinutes: draft.allotedTimeMinutes, + attemptsPerTimeRange: draft.attemptsPerTimeRange, + attemptsTimeRangeHours: draft.attemptsTimeRangeHours, + passingGrade: draft.passingGrade, + displayOrder: draft.displayOrder, + questionDisplay: draft.questionDisplay, + numberOfQuestionsPerAttempt: draft.numberOfQuestionsPerAttempt, + questionOrder: draft.questionOrder, + published: draft.published, + showAssignmentScore: draft.showAssignmentScore, + showQuestionScore: draft.showQuestionScore, + showSubmissionFeedback: draft.showSubmissionFeedback, + showQuestions: draft.showQuestions, + languageCode: draft.languageCode, + questions: + (JSON.parse(draft.questionsData as string) as unknown as JsonValue[]) ?? + [], + _isDraft: true, + _draftId: draft.id, + _draftName: draft.draftName, + _draftUpdatedAt: draft.updatedAt, + }; + } + + async deleteDraft(draftId: number, userSession: UserSession): Promise { + const draft = await this.prisma.assignmentDraft.findUnique({ + where: { id: draftId }, + }); + + if (!draft) { + throw new NotFoundException("Draft not found"); + } + + if (draft.userId !== userSession.userId) { + throw new BadRequestException("You can only delete your own drafts"); + } + + await this.prisma.assignmentDraft.delete({ + where: { id: draftId }, + }); + + this.logger.info(`Deleted draft ${draftId}`, { + userId: userSession.userId, + }); + } + + async getLatestDraft( + assignmentId: number, + userSession: UserSession, + ): Promise<{ + id: number; + name: string; + introduction: string; + instructions: string; + gradingCriteriaOverview: string; + timeEstimateMinutes: number; + type: string; + graded: boolean; + numAttempts: number; + allotedTimeMinutes: number; + attemptsPerTimeRange: number; + attemptsTimeRangeHours: number; + passingGrade: number; + displayOrder: AssignmentQuestionDisplayOrder; + questionDisplay: QuestionDisplay; + numberOfQuestionsPerAttempt: number; + questionOrder: number[]; + published: boolean; + showAssignmentScore: boolean; + showQuestionScore: boolean; + showSubmissionFeedback: boolean; + showQuestions: boolean; + languageCode: string; + questions: JsonValue[]; + _isDraft?: boolean; + _draftId?: number; + _draftName?: string; + _draftUpdatedAt?: Date; + }> { + // Verify assignment access + // await this.verifyAssignmentAccess(assignmentId, userSession); + + const latestDraft = await this.prisma.assignmentDraft.findFirst({ + where: { + assignmentId, + userId: userSession.userId, + }, + orderBy: { updatedAt: "desc" }, + }); + + if (!latestDraft) { + return null; + } + + return this.getDraft(latestDraft.id, userSession); + } + + private async verifyAssignmentAccess( + assignmentId: number, + userSession: UserSession, + ) { + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { AssignmentAuthor: true }, + }); + + if (!assignment) { + throw new NotFoundException("Assignment not found"); + } + + if (userSession.role === UserRole.AUTHOR) { + const hasAccess = assignment.AssignmentAuthor.some( + (author) => author.userId === userSession.userId, + ); + if (!hasAccess) { + throw new NotFoundException("Assignment not found"); + } + } + } +} diff --git a/apps/api/src/api/assignment/v2/services/grading-consistency.service.ts b/apps/api/src/api/assignment/v2/services/grading-consistency.service.ts new file mode 100644 index 00000000..77d66cd1 --- /dev/null +++ b/apps/api/src/api/assignment/v2/services/grading-consistency.service.ts @@ -0,0 +1,705 @@ +/* eslint-disable unicorn/no-null */ +/* eslint-disable @typescript-eslint/require-await */ +// src/api/assignment/v2/services/grading-consistency.service.ts +import * as crypto from "node:crypto"; +import { Inject, Injectable, OnModuleDestroy } from "@nestjs/common"; +import { QuestionType } from "@prisma/client"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { RubricScore } from "src/api/llm/model/file.based.question.response.model"; +import { Logger } from "winston"; +import { PrismaService } from "../../../../prisma.service"; +import { + CriteriaDto, + ScoringDto, +} from "../../dto/update.questions.request.dto"; + +interface GradingRecord { + questionId: number; + responseHash: string; + points: number; + maxPoints: number; + feedback: string; + rubricScores?: RubricScore[]; + timestamp: Date; +} + +interface ConsistencyCheck { + similar: boolean; + previousGrade?: number; + previousFeedback?: string; + deviationPercentage?: number; + shouldAdjust: boolean; +} + +interface NormalizedScore { + percentage: number; + points: number; + maxPoints: number; +} + +interface ParsedRequestPayload { + learnerTextResponse?: string; + learnerResponse?: string; + [key: string]: unknown; +} + +interface ParsedResponsePayload { + totalPoints?: number; + maxPoints?: number; + feedback?: unknown; + [key: string]: unknown; +} + +interface RubricValidationResult { + valid: boolean; + issues: string[]; + corrections: RubricScore[]; +} + +@Injectable() +export class GradingConsistencyService implements OnModuleDestroy { + private readonly logger: Logger; + private readonly gradingCache = new Map(); + private readonly cacheLocks = new Map>(); + private readonly maxCacheSize = 1000; // Maximum cache entries + private readonly cacheCleanupInterval = 3_600_000; // 1 hour + private cleanupTimer?: NodeJS.Timeout; + + constructor( + private readonly prisma: PrismaService, + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, + ) { + this.logger = parentLogger.child({ + context: GradingConsistencyService.name, + }); + + // Start periodic cache cleanup + this.cleanupTimer = setInterval(() => { + this.cleanupCache(); + }, this.cacheCleanupInterval); + } + + onModuleDestroy() { + if (this.cleanupTimer) { + clearInterval(this.cleanupTimer); + } + } + + /** + * Generate a hash for response similarity checking + */ + generateResponseHash( + response: string, + questionId: number, + questionType: QuestionType, + ): string { + try { + // Normalize the response for comparison + const normalized = this.normalizeResponse(response, questionType); + + // Create a hash combining question and normalized response + const hash = crypto + .createHash("sha256") + .update(`${questionId}:${normalized}`) + .digest("hex") + .slice(0, 32); + + return hash; + } catch (error) { + this.logger.error("Error generating response hash:", error); + // Return a fallback hash + return crypto.randomBytes(16).toString("hex"); + } + } + + /** + * Check for similar previous responses + */ + async checkConsistency( + questionId: number, + responseHash: string, + currentResponse: string, + questionType: QuestionType, + ): Promise { + try { + // Check cache first + const cacheKey = `q_${questionId}`; + const cachedRecords = this.gradingCache.get(cacheKey) || []; + + // Check cached records first (faster) + for (const record of cachedRecords) { + if (this.isSimilarHash(responseHash, record.responseHash)) { + return { + similar: true, + previousGrade: record.points, + previousFeedback: record.feedback, + deviationPercentage: 0, // Exact match + shouldAdjust: false, + }; + } + } + + // Check database for recent similar responses + const recentGradings = await this.prisma.gradingAudit.findMany({ + where: { + questionId, + timestamp: { + gte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), // Last 7 days + }, + }, + orderBy: { timestamp: "desc" }, + take: 50, // Limit for performance + select: { + id: true, + requestPayload: true, + responsePayload: true, + timestamp: true, + }, + }); + + // Analyze for similar responses + for (const grading of recentGradings) { + try { + const requestData = this.safeJsonParse( + grading.requestPayload, + ); + const responseData = this.safeJsonParse( + grading.responsePayload, + ); + + if (!requestData || !responseData) continue; + + const previousResponse = + requestData.learnerTextResponse || + requestData.learnerResponse || + ""; + + if ( + this.isSimilarResponse( + currentResponse, + previousResponse, + questionType, + ) + ) { + const deviationPercentage = 0; // Will be calculated when current grade is known + + return { + similar: true, + previousGrade: responseData.totalPoints || 0, + previousFeedback: JSON.stringify(responseData.feedback || ""), + deviationPercentage, + shouldAdjust: false, // Will be determined after grading + }; + } + } catch (error) { + this.logger.debug( + `Error parsing grading record ${grading.id}:`, + error, + ); + } + } + + return { + similar: false, + shouldAdjust: false, + }; + } catch (error) { + this.logger.error("Error checking consistency:", error); + return { + similar: false, + shouldAdjust: false, + }; + } + } + + /** + * Record a grading for future consistency checks + */ + async recordGrading( + questionId: number, + responseHash: string, + points: number, + maxPoints: number, + feedback: string, + rubricScores?: RubricScore[], + ): Promise { + try { + const record: GradingRecord = { + questionId, + responseHash, + points, + maxPoints, + feedback, + rubricScores, + timestamp: new Date(), + }; + + // Update cache atomically + await this.atomicCacheUpdate(`q_${questionId}`, record); + } catch (error) { + this.logger.error("Error recording grading:", error); + } + } + + /** + * Validate rubric score consistency + */ + validateRubricScores( + rubricScores: RubricScore[], + scoringCriteria: ScoringDto, + ): RubricValidationResult { + const issues: string[] = []; + const corrections: RubricScore[] = []; + + if (!scoringCriteria?.rubrics || !Array.isArray(scoringCriteria.rubrics)) { + return { valid: true, issues: [], corrections: rubricScores || [] }; + } + + if (!Array.isArray(rubricScores)) { + issues.push("Rubric scores is not an array"); + return { valid: false, issues, corrections: [] }; + } + + // Ensure we have scores for all rubrics + if (rubricScores.length !== scoringCriteria.rubrics.length) { + issues.push( + `Rubric count mismatch: ${rubricScores.length} scores for ${scoringCriteria.rubrics.length} rubrics`, + ); + } + + // Validate each rubric score + const maxIndex = Math.min( + rubricScores.length, + scoringCriteria.rubrics.length, + ); + for (let index = 0; index < maxIndex; index++) { + const score = rubricScores[index]; + const rubric = scoringCriteria.rubrics[index]; + + if (!score || typeof score !== "object") { + issues.push(`Invalid score object at index ${index}`); + continue; + } + + if (!rubric?.criteria || !Array.isArray(rubric.criteria)) { + continue; + } + + // Check if points are valid for this rubric + const validPoints = rubric.criteria + .filter((c: CriteriaDto) => c && typeof c.points === "number") + .map((c: CriteriaDto) => c.points); + + if (validPoints.length === 0) { + continue; + } + + const currentPoints = + typeof score.pointsAwarded === "number" ? score.pointsAwarded : 0; + + if (validPoints.includes(currentPoints)) { + corrections.push(score); + } else { + issues.push( + `Invalid points ${currentPoints} for rubric "${ + score.rubricQuestion || "Unknown" + }"`, + ); + + let closestValid = validPoints[0]; + + for (const current of validPoints) { + if ( + Math.abs(current - currentPoints) < + Math.abs(closestValid - currentPoints) + ) { + closestValid = current; + } + } + + corrections.push({ + ...score, + pointsAwarded: closestValid, + }); + } + } + + return { + valid: issues.length === 0, + issues, + corrections, + }; + } + + /** + * Get grading statistics for fairness analysis + */ + async getGradingStatistics(questionId: number): Promise<{ + averageScore: number; + standardDeviation: number; + distribution: Record; + totalGradings: number; + }> { + try { + const recentGradings = await this.prisma.gradingAudit.findMany({ + where: { questionId }, + orderBy: { timestamp: "desc" }, + take: 100, + select: { + responsePayload: true, + }, + }); + + const scores: number[] = []; + const distribution: Record = {}; + + for (const grading of recentGradings) { + try { + const response = this.safeJsonParse( + grading.responsePayload, + ); + if (!response) continue; + + const percentage = Math.round( + ((response.totalPoints || 0) / (response.maxPoints || 1)) * 100, + ); + scores.push(percentage); + + const range = `${Math.floor(percentage / 10) * 10}-${ + Math.floor(percentage / 10) * 10 + 9 + }%`; + distribution[range] = (distribution[range] || 0) + 1; + } catch { + // Skip invalid records + } + } + + const averageScore = + scores.length > 0 + ? scores.reduce((a, b) => a + b, 0) / scores.length + : 0; + + const variance = + scores.length > 0 + ? scores.reduce( + (sum, score) => sum + Math.pow(score - averageScore, 2), + 0, + ) / scores.length + : 0; + + const standardDeviation = Math.sqrt(variance); + + return { + averageScore: Math.round(averageScore * 100) / 100, + standardDeviation: Math.round(standardDeviation * 100) / 100, + distribution, + totalGradings: scores.length, + }; + } catch (error) { + this.logger.error("Error getting grading statistics:", error); + return { + averageScore: 0, + standardDeviation: 0, + distribution: {}, + totalGradings: 0, + }; + } + } + + /** + * Normalize response for comparison + */ + private normalizeResponse( + response: string, + questionType: QuestionType, + ): string { + if (!response || typeof response !== "string") { + return ""; + } + + let normalized = response.toLowerCase().trim(); + + // Limit length for performance + if (normalized.length > 1000) { + normalized = normalized.slice(0, 1000); + } + + // Remove extra whitespace + normalized = normalized.replaceAll(/\s+/g, " "); + + // Remove common punctuation for comparison + normalized = normalized.replaceAll(/[!"',.:;?]/g, ""); + + // Type-specific normalization + switch (questionType) { + case QuestionType.TEXT: { + // Remove common filler words for text comparison + const fillerWords = [ + "the", + "a", + "an", + "and", + "or", + "but", + "in", + "on", + "at", + "to", + "for", + ]; + for (const word of fillerWords) { + const regex = new RegExp(`\\b${word}\\b`, "g"); + normalized = normalized.replace(regex, ""); + } + // Remove extra spaces created by word removal + normalized = normalized.replaceAll(/\s+/g, " ").trim(); + break; + } + + case QuestionType.SINGLE_CORRECT: + case QuestionType.MULTIPLE_CORRECT: { + // Normalize choice indicators + normalized = normalized.replaceAll(/\b(option|choice|answer)\s*/gi, ""); + break; + } + + case QuestionType.TRUE_FALSE: { + // Normalize boolean responses + if (/\b(true|yes|correct|right)\b/i.test(normalized)) { + normalized = "true"; + } else if (/\b(false|no|incorrect|wrong)\b/i.test(normalized)) { + normalized = "false"; + } + break; + } + } + + return normalized; + } + + /** + * Check if two responses are similar + */ + private isSimilarResponse( + response1: string, + response2: string, + questionType: QuestionType, + ): boolean { + if (!response1 || !response2) { + return false; + } + + const normalized1 = this.normalizeResponse(response1, questionType); + const normalized2 = this.normalizeResponse(response2, questionType); + + // For exact match types (choices, true/false) + if ( + questionType === QuestionType.SINGLE_CORRECT || + questionType === QuestionType.MULTIPLE_CORRECT || + questionType === QuestionType.TRUE_FALSE + ) { + return normalized1 === normalized2; + } + + // For text responses, use similarity threshold + const similarity = this.calculateSimilarity(normalized1, normalized2); + return similarity > 0.85; // 85% similarity threshold + } + + /** + * Check if two hashes are similar (for exact matches) + */ + private isSimilarHash(hash1: string, hash2: string): boolean { + return hash1 === hash2; + } + + /** + * Calculate similarity between two strings (0-1) + */ + private calculateSimilarity(string1: string, string2: string): number { + if (!string1 || !string2) return 0; + if (string1 === string2) return 1; + + const longer = string1.length > string2.length ? string1 : string2; + const shorter = string1.length > string2.length ? string2 : string1; + + if (longer.length === 0) { + return 1; + } + + // Use a more efficient algorithm for long strings + if (longer.length > 500) { + return this.calculateJaccardSimilarity(string1, string2); + } + + const editDistance = this.getEditDistance(longer, shorter); + return (longer.length - editDistance) / longer.length; + } + + /** + * Calculate Jaccard similarity for long strings (more efficient) + */ + private calculateJaccardSimilarity(string1: string, string2: string): number { + const set1 = new Set(string1.split(" ")); + const set2 = new Set(string2.split(" ")); + + const intersection = new Set([...set1].filter((x) => set2.has(x))); + const union = new Set([...set1, ...set2]); + + return union.size > 0 ? intersection.size / union.size : 0; + } + + /** + * Calculate edit distance between two strings + */ + private getEditDistance(string1: string, string2: string): number { + const m = string1.length; + const n = string2.length; + + // Create a 2D array for dynamic programming + const dp: number[][] = Array.from({ length: m + 1 }, () => + // eslint-disable-next-line unicorn/no-new-array + new Array(n + 1).fill(0), + ); + + // Initialize first row and column + for (let index = 0; index <= m; index++) { + dp[index][0] = index; + } + for (let index = 0; index <= n; index++) { + dp[0][index] = index; + } + + // Fill the matrix + for (let index = 1; index <= m; index++) { + for (let index_ = 1; index_ <= n; index_++) { + dp[index][index_] = + string1[index - 1] === string2[index_ - 1] + ? dp[index - 1][index_ - 1] + : 1 + + Math.min( + dp[index - 1][index_], + dp[index][index_ - 1], + dp[index - 1][index_ - 1], + ); + } + } + + return dp[m][n]; + } + + /** + * Normalize score to percentage + */ + private normalizeScore(points: number, maxPoints: number): NormalizedScore { + const safeMaxPoints = maxPoints > 0 ? maxPoints : 1; + const percentage = (points / safeMaxPoints) * 100; + + return { + percentage: Math.round(percentage * 100) / 100, + points, + maxPoints: safeMaxPoints, + }; + } + + /** + * Atomically update cache to prevent race conditions + */ + private async atomicCacheUpdate( + cacheKey: string, + record: GradingRecord, + ): Promise { + // Wait for any existing operation on this key + const existingLock = this.cacheLocks.get(cacheKey); + if (existingLock !== undefined) { + await existingLock; + } + + // Create a new lock for this operation + const lockPromise = this.performCacheUpdate(cacheKey, record); + this.cacheLocks.set(cacheKey, lockPromise); + + try { + await lockPromise; + } finally { + // Remove lock after operation + this.cacheLocks.delete(cacheKey); + } + } + + /** + * Perform the actual cache update + */ + private async performCacheUpdate( + cacheKey: string, + record: GradingRecord, + ): Promise { + const existing = this.gradingCache.get(cacheKey) || []; + existing.push(record); + + // Keep only recent records in cache (last 100) + if (existing.length > 100) { + existing.shift(); + } + + this.gradingCache.set(cacheKey, existing); + + // Check cache size + if (this.gradingCache.size > this.maxCacheSize) { + this.cleanupCache(); + } + } + + /** + * Safely parse JSON with type assertion + */ + private safeJsonParse(jsonString: string): T | null { + try { + if (!jsonString || typeof jsonString !== "string") { + return null; + } + return JSON.parse(jsonString) as T; + } catch { + return null; + } + } + + /** + * Clean up old cache entries + */ + private cleanupCache(): void { + try { + const now = Date.now(); + const maxAge = 24 * 60 * 60 * 1000; // 24 hours + + for (const [key, records] of this.gradingCache.entries()) { + // Remove old records + const filteredRecords = records.filter( + (record) => now - record.timestamp.getTime() < maxAge, + ); + + if (filteredRecords.length === 0) { + this.gradingCache.delete(key); + } else if (filteredRecords.length < records.length) { + this.gradingCache.set(key, filteredRecords); + } + } + + // If still too large, remove oldest entries + if (this.gradingCache.size > this.maxCacheSize) { + const sortedKeys = [...this.gradingCache.keys()].sort(); + const keysToRemove = sortedKeys.slice( + 0, + sortedKeys.length - this.maxCacheSize, + ); + for (const key of keysToRemove) this.gradingCache.delete(key); + } + + this.logger.debug( + `Cache cleanup completed. Current size: ${this.gradingCache.size}`, + ); + } catch (error) { + this.logger.error("Error during cache cleanup:", error); + } + } +} diff --git a/apps/api/src/api/assignment/v2/services/question.service.ts b/apps/api/src/api/assignment/v2/services/question.service.ts index d3339509..485e0533 100644 --- a/apps/api/src/api/assignment/v2/services/question.service.ts +++ b/apps/api/src/api/assignment/v2/services/question.service.ts @@ -180,12 +180,6 @@ export class QuestionService { (q) => q.id === backendId, ); - const questionContentChanged = - forceTranslation || - !existingQuestion || - existingQuestion.question !== questionDto.question || - !this.areChoicesEqual(existingQuestion.choices, questionDto.choices); - if ( existingQuestion && existingQuestion.question !== questionDto.question @@ -219,30 +213,26 @@ export class QuestionService { liveRecordingConfig: questionDto.liveRecordingConfig, videoPresentationConfig: questionDto.videoPresentationConfig, gradingContextQuestionIds: questionDto.gradingContextQuestionIds, + isDeleted: false, // Explicitly set to false during publishing }); if (!existingQuestion) { frontendToBackendIdMap.set(questionDto.id, upsertedQuestion.id); } - if (questionContentChanged || forceTranslation) { - await updateProgress( - questionStartProgress + progressPerQuestion * 0.5, - `Translating question ${index + 1}`, - ); - - const translationJob = await this.jobStatusService.createJob( - assignmentId, - "system", - ); + // Always translate the original question (with variantId: null) + await updateProgress( + questionStartProgress + progressPerQuestion * 0.5, + `Translating question ${index + 1}`, + ); - await this.translationService.translateQuestion( - assignmentId, - upsertedQuestion.id, - questionDto, - translationJob.id, - ); - } + await this.translationService.translateQuestion( + assignmentId, + upsertedQuestion.id, + questionDto, + jobId || 0, // Use main job or fallback + true, // Force retranslation on every publish + ); const variantCount = questionDto.variants?.length || 0; if (variantCount > 0) { @@ -251,18 +241,14 @@ export class QuestionService { `Processing ${variantCount} variants for question ${index + 1}`, ); - const checkVariantsChanged = this.checkVariantsForChanges( - existingQuestion?.variants || [], - questionDto.variants || [], - ); - await this.processVariantsForQuestion( assignmentId, upsertedQuestion.id, questionDto.variants || [], existingQuestion?.variants || [], - undefined, - checkVariantsChanged || forceTranslation, + jobId, + // checkVariantsChanged || forceTranslation + true, // Always force translation for variants ); } @@ -640,29 +626,32 @@ export class QuestionService { variantData, ); - if (jobId && contentChanged) { - await this.jobStatusService.updateJobStatus(jobId, { - status: "In Progress", - progress: `Translating variant ${ - index + 1 - }/${totalVariants} for question #${questionId}`, - }); - - await this.translationService.translateVariant( - assignmentId, - questionId, - updatedVariant.id, - updatedVariant as unknown as VariantDto, - jobId, - ); - } else if (jobId) { - await this.jobStatusService.updateJobStatus(jobId, { - status: "In Progress", - progress: `Variant ${ - index + 1 - }/${totalVariants} unchanged, skipping translation`, - }); + if (jobId) { + await (contentChanged + ? this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: `Translating variant ${ + index + 1 + }/${totalVariants} for question #${questionId} (content changed)`, + }) + : this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: `Ensuring translations for variant ${ + index + 1 + }/${totalVariants} for question #${questionId}`, + })); } + + // Always translate variants, regardless of jobId + await this.translationService.translateVariant( + assignmentId, + questionId, + updatedVariant.id, + updatedVariant as unknown as VariantDto, + jobId || 0, // Use 0 as fallback jobId for non-job contexts + // contentChanged // Force retranslation only if content changed + true, // Always force translation for existing variants + ); } else { const newVariant = await this.variantRepository.create(variantData); @@ -673,15 +662,17 @@ export class QuestionService { index + 1 }/${totalVariants} for question #${questionId}`, }); - - await this.translationService.translateVariant( - assignmentId, - questionId, - newVariant.id, - newVariant as unknown as VariantDto, - jobId, - ); } + + // Always translate new variants, regardless of jobId + await this.translationService.translateVariant( + assignmentId, + questionId, + newVariant.id, + newVariant as unknown as VariantDto, + jobId || 0, // Use 0 as fallback jobId for non-job contexts + true, // Always force translation for new variants + ); } } } diff --git a/apps/api/src/api/assignment/v2/services/translation.service.ts b/apps/api/src/api/assignment/v2/services/translation.service.ts index 3b02259d..6e479df3 100644 --- a/apps/api/src/api/assignment/v2/services/translation.service.ts +++ b/apps/api/src/api/assignment/v2/services/translation.service.ts @@ -1,6 +1,6 @@ /* eslint-disable unicorn/no-null */ import { Injectable, Logger, NotFoundException } from "@nestjs/common"; -import { Prisma, Translation } from "@prisma/client"; +import { Prisma } from "@prisma/client"; import Bottleneck from "bottleneck"; import { LlmFacadeService } from "src/api/llm/llm-facade.service"; import { PrismaService } from "src/prisma.service"; @@ -63,30 +63,50 @@ export class TranslationService { private readonly languageTranslation: boolean; private readonly limiter: Bottleneck; - private readonly MAX_BATCH_SIZE = 50; - private readonly CONCURRENCY_LIMIT = 35; + // Performance optimized settings + private readonly MAX_BATCH_SIZE = 100; // Increased for better throughput + private readonly CONCURRENCY_LIMIT = 50; // Increased concurrency private readonly MAX_RETRY_ATTEMPTS = 2; - private readonly RETRY_DELAY_BASE = 200; - private readonly STATUS_UPDATE_INTERVAL = 10; + private readonly RETRY_DELAY_BASE = 100; // Reduced delay for faster retries + private readonly STATUS_UPDATE_INTERVAL = 20; // Reduced DB calls + private readonly OPERATION_TIMEOUT = 30_000; // 30 seconds per operation + private readonly JOB_TIMEOUT = 600_000; // 10 minutes for large jobs + private readonly MAX_STUCK_OPERATIONS = 15; // Allow more stuck operations + private readonly ADAPTIVE_BATCH_SIZE = true; // Enable adaptive batching + + // Track stuck operations and performance + private stuckOperations = new Set(); + private jobStartTimes = new Map(); + private jobCancellationFlags = new Map(); + private operationStats = { + totalOperations: 0, + successfulOperations: 0, + failedOperations: 0, + averageResponseTime: 0, + consecutiveFailures: 0, + lastFailureTime: 0, + }; constructor( private readonly prisma: PrismaService, private readonly llmFacadeService: LlmFacadeService, private readonly jobStatusService: JobStatusServiceV2, ) { - this.languageTranslation = process.env.NODE_ENV !== "development"; + this.languageTranslation = + process.env.ENABLE_TRANSLATION.toString().toLowerCase() === "true" || + false; this.limiter = new Bottleneck({ - maxConcurrent: 35, - minTime: 5, - reservoir: 200, - reservoirRefreshInterval: 5000, - reservoirRefreshAmount: 200, - highWater: 3000, + maxConcurrent: this.CONCURRENCY_LIMIT, + minTime: 2, + reservoirRefreshInterval: 3000, + reservoirRefreshAmount: 500, + highWater: 5000, strategy: Bottleneck.strategy.OVERFLOW, - timeout: 45_000, + timeout: this.OPERATION_TIMEOUT, }); setInterval(() => this.checkLimiterHealth(), 30_000); + setInterval(() => this.checkJobTimeouts(), 60_000); // Check every minute } /** @@ -100,7 +120,7 @@ export class TranslationService { items: T[], batchProcessor: (item: T) => Promise, batchSize = this.MAX_BATCH_SIZE, - concurrencyLimit = this.CONCURRENCY_LIMIT, + _concurrencyLimit = this.CONCURRENCY_LIMIT, ): Promise { const results: BatchProcessResult = { success: 0, failure: 0, dropped: 0 }; const chunks: T[][] = []; @@ -148,6 +168,7 @@ export class TranslationService { /** * Get languages available for an assignment + * A language is only available if BOTH assignment metadata AND all questions are translated * * @param assignmentId - The assignment ID * @returns Array of language codes @@ -169,6 +190,391 @@ export class TranslationService { return [...availableLanguages]; } + /** + * Helper method to detect language of text + */ + async detectLanguage(text: string, assignmentId = 1): Promise { + try { + const detectedLang = await this.llmFacadeService.getLanguageCode( + text, + assignmentId, + ); + return detectedLang && detectedLang !== "unknown" ? detectedLang : "en"; + } catch { + return "en"; + } + } + + /** + * Ensure all questions and variants have complete translations + * This is a safety check to run after publishing + * + * @param assignmentId - The assignment ID + * @returns Object with completeness status + */ + async ensureTranslationCompleteness(assignmentId: number): Promise<{ + isComplete: boolean; + missingTranslations: Array<{ + questionId: number; + variantId: number | null; + missingLanguages: string[]; + }>; + }> { + const missingTranslations: Array<{ + questionId: number; + variantId: number | null; + missingLanguages: string[]; + }> = []; + + const supportedLanguages = getAllLanguageCodes() ?? ["en"]; + + // Get all questions and variants + const questions = await this.prisma.question.findMany({ + where: { + assignmentId, + isDeleted: false, + }, + include: { + variants: { + where: { isDeleted: false }, + }, + translations: { + select: { languageCode: true, variantId: true }, + }, + }, + }); + + for (const question of questions) { + // Check question translations + const questionTranslations = question.translations.filter( + (t) => t.variantId === null, + ); + const questionLanguages = new Set( + questionTranslations.map((t) => t.languageCode), + ); + + const missingQuestionLangs = supportedLanguages.filter( + (lang) => !questionLanguages.has(lang), + ); + + if (missingQuestionLangs.length > 0) { + missingTranslations.push({ + questionId: question.id, + variantId: null, + missingLanguages: missingQuestionLangs, + }); + } + + // Check variant translations + for (const variant of question.variants) { + const variantTranslations = question.translations.filter( + (t) => t.variantId === variant.id, + ); + const variantLanguages = new Set( + variantTranslations.map((t) => t.languageCode), + ); + + const missingVariantLangs = supportedLanguages.filter( + (lang) => !variantLanguages.has(lang), + ); + + if (missingVariantLangs.length > 0) { + missingTranslations.push({ + questionId: question.id, + variantId: variant.id, + missingLanguages: missingVariantLangs, + }); + } + } + } + + return { + isComplete: missingTranslations.length === 0, + missingTranslations, + }; + } + + /** + * Quick validation that only checks if translations exist without language detection + * Much faster than full language consistency validation + * + * @param assignmentId - The assignment ID + * @returns True if basic validation passes + */ + async quickValidateAssignmentTranslations( + assignmentId: number, + ): Promise { + try { + // Just check if we have recent translations (created in last 24 hours) + const recentTranslationsCount = + await this.prisma.assignmentTranslation.count({ + where: { + assignmentId, + updatedAt: { + gte: new Date(Date.now() - 24 * 60 * 60 * 1000), // 24 hours ago + }, + }, + }); + + // If we have recent translations, assume they're good + if (recentTranslationsCount > 0) { + return true; + } + + // Otherwise, do a quick count check + const totalTranslations = await this.prisma.assignmentTranslation.count({ + where: { assignmentId }, + }); + + return totalTranslations > 0; + } catch (error: unknown) { + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.error( + `Error in quick translation validation: ${errorMessage}`, + ); + return true; // Default to assuming it's okay + } + } + + /** + * Check if the assignment content language matches the expected language codes + * This validates that translations are correctly aligned with their language codes + * WARNING: This is an expensive operation that makes API calls for language detection + * + * @param assignmentId - The assignment ID + * @returns Object with validation results and mismatched languages + */ + async validateAssignmentLanguageConsistency(assignmentId: number): Promise<{ + isConsistent: boolean; + mismatchedLanguages: string[]; + details: Array<{ + languageCode: string; + detectedLanguage: string; + needsRetranslation: boolean; + }>; + }> { + const mismatchedLanguages: string[] = []; + const details: Array<{ + languageCode: string; + detectedLanguage: string; + needsRetranslation: boolean; + }> = []; + try { + // Get all assignment translations + const assignmentTranslations = + await this.prisma.assignmentTranslation.findMany({ + where: { assignmentId }, + select: { + languageCode: true, + translatedName: true, + translatedIntroduction: true, + translatedInstructions: true, + }, + }); + + // Check each translation for language consistency + for (const translation of assignmentTranslations) { + // Skip if no translated content exists + if ( + !translation.translatedName && + !translation.translatedIntroduction && + !translation.translatedInstructions + ) { + continue; + } + + // Use the most substantial text for language detection + const textToCheck = + translation.translatedIntroduction || + translation.translatedInstructions || + translation.translatedName || + ""; + + if (textToCheck) { + const detectedLanguage = await this.llmFacadeService.getLanguageCode( + textToCheck, + assignmentId, + ); + + if (detectedLanguage && detectedLanguage !== "unknown") { + // Normalize language codes for comparison + const normalizedDetected = detectedLanguage + .toLowerCase() + .split("-")[0]; + const normalizedExpected = translation.languageCode + .toLowerCase() + .split("-")[0]; + + const isMatching = normalizedDetected === normalizedExpected; + + details.push({ + languageCode: translation.languageCode, + detectedLanguage, + needsRetranslation: !isMatching, + }); + + if (!isMatching) { + mismatchedLanguages.push(translation.languageCode); + this.logger.warn( + `Language mismatch detected for assignment ${assignmentId}: ` + + `Expected ${translation.languageCode}, but detected ${detectedLanguage}`, + ); + } + } + } + } + + // Check question and variant translations using batch processing + const translations = await this.prisma.translation.findMany({ + where: { + question: { + assignmentId, + isDeleted: false, + }, + }, + select: { + languageCode: true, + translatedText: true, + questionId: true, + variantId: true, + }, + take: 20, // Reduced sample size for faster validation + }); + + if (translations.length > 0) { + // Batch language detection for all translation texts + const textsToCheck = translations + .filter((t): t is typeof t & { translatedText: string } => + Boolean(t.translatedText), + ) + .map((t) => t.translatedText); + + if (textsToCheck.length > 0) { + const detectedLanguages = + await this.llmFacadeService.batchGetLanguageCodes( + textsToCheck, + assignmentId, + ); + + let textIndex = 0; + for (const translation of translations) { + if (translation.translatedText) { + const detectedLanguage = detectedLanguages[textIndex++]; + + if (detectedLanguage && detectedLanguage !== "unknown") { + const normalizedDetected = detectedLanguage + .toLowerCase() + .split("-")[0]; + const normalizedExpected = translation.languageCode + .toLowerCase() + .split("-")[0]; + + if (normalizedDetected !== normalizedExpected) { + if (!mismatchedLanguages.includes(translation.languageCode)) { + mismatchedLanguages.push(translation.languageCode); + } + this.logger.warn( + `Language mismatch in question/variant translation: ` + + `Expected ${translation.languageCode}, detected ${detectedLanguage} ` + + `(Question: ${translation.questionId}, Variant: ${translation.variantId})`, + ); + } + } + } + } + } + } + + return { + isConsistent: mismatchedLanguages.length === 0, + mismatchedLanguages, + details, + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.error( + `Error validating language consistency: ${errorMessage}`, + ); + return { + isConsistent: true, // Default to consistent on error + mismatchedLanguages: [], + details: [], + }; + } + } + + /** + * Check if a specific language is fully available for an assignment + * More efficient than getting all available languages when checking just one + * + * @param assignmentId - The assignment ID + * @param languageCode - The language code to check + * @returns True if language is fully available + */ + async isLanguageAvailable( + assignmentId: number, + languageCode: string, + ): Promise { + // English is always available + if (languageCode.toLowerCase() === "en") { + return true; + } + + // Check if assignment translation exists + const assignmentTranslation = + await this.prisma.assignmentTranslation.findFirst({ + where: { assignmentId, languageCode }, + }); + + if (!assignmentTranslation) { + return false; + } + + // Get all questions and variants for this assignment + const questions = await this.prisma.question.findMany({ + where: { + assignmentId, + isDeleted: false, + }, + select: { + id: true, + variants: { + where: { isDeleted: false }, + select: { id: true }, + }, + }, + }); + + if (questions.length === 0) { + // No questions, so assignment translation is sufficient + return true; + } + + // Get all question and variant IDs + const questionIds = questions.map((q) => q.id); + const variantIds = questions.flatMap((q) => q.variants.map((v) => v.id)); + const requiredCount = questionIds.length + variantIds.length; + + if (requiredCount === 0) { + // No content to translate + return true; + } + + // Count actual translations for this language + const translationCount = await this.prisma.translation.count({ + where: { + languageCode, + OR: [ + { questionId: { in: questionIds }, variantId: null }, + { variantId: { in: variantIds } }, + ], + }, + }); + + return translationCount >= requiredCount; + } /** * Detect if the limiter appears to be stalled and reset it if necessary @@ -252,6 +658,7 @@ export class TranslationService { try { const originalLanguage = await this.llmFacadeService.getLanguageCode( assignment.introduction || "en", + assignment.id, ); if (languageCode === originalLanguage) return; @@ -320,11 +727,14 @@ export class TranslationService { * Update the job status with current progress information - optimized to reduce DB calls */ private async updateJobProgress( - tracker: ProgressTracker, + tracker: ProgressTracker | undefined, currentLanguage: string, currentItem?: string | number, additionalInfo?: string, ): Promise { + if (!tracker) { + return; + } if ( tracker.completedItems % this.STATUS_UPDATE_INTERVAL !== 0 && tracker.completedItems !== tracker.totalItems @@ -359,34 +769,70 @@ export class TranslationService { } /** - * Simplified retry function for translation operations + * Enhanced retry function with timeout and circuit breaker */ private async executeWithOptimizedRetry( operationName: string, translationFunction: () => Promise, maxAttempts = this.MAX_RETRY_ATTEMPTS, - jobId?: number, + _jobId?: number, ): Promise { let attempts = 0; + const operationId = `${operationName}-${Date.now()}`; while (attempts < maxAttempts) { try { - return await translationFunction(); + this.stuckOperations.add(operationId); + + // Create timeout promise + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject( + new Error( + `Operation ${operationName} timed out after ${this.OPERATION_TIMEOUT}ms`, + ), + ); + }, this.OPERATION_TIMEOUT); + }); + + // Race between operation and timeout + const result = await Promise.race([ + translationFunction(), + timeoutPromise, + ]); + + this.stuckOperations.delete(operationId); + return result; } catch (error) { attempts++; + this.stuckOperations.delete(operationId); + const errorMessage = error instanceof Error ? error.message : String(error); + // Check if this is a timeout error + const isTimeout = errorMessage.includes("timed out"); + if (attempts >= maxAttempts) { this.logger.error( `Failed ${operationName} after ${maxAttempts} attempts: ${errorMessage}`, ); + + // Track persistently stuck operations + if (isTimeout) { + this.handleStuckOperation(operationName); + } + throw error; } + // Longer backoff for timeout errors + const baseDelay = isTimeout + ? this.RETRY_DELAY_BASE * 2 + : this.RETRY_DELAY_BASE; const jitter = Math.random() * 200; await new Promise((resolve) => - setTimeout(resolve, this.RETRY_DELAY_BASE * attempts + jitter), + setTimeout(resolve, baseDelay * attempts + jitter), ); } } @@ -394,25 +840,309 @@ export class TranslationService { throw new Error(`Max retries exceeded for ${operationName}`); } + /** + * Handle stuck operations by resetting limiter if needed + */ + private handleStuckOperation(_operationName: string): void { + this.operationStats.consecutiveFailures++; + this.operationStats.lastFailureTime = Date.now(); + + if (this.stuckOperations.size >= this.MAX_STUCK_OPERATIONS) { + this.logger.warn( + `Too many stuck operations (${this.stuckOperations.size}), resetting limiter`, + ); + this.resetLimiter(); + this.stuckOperations.clear(); + this.operationStats.consecutiveFailures = 0; + } + } + + /** + * Cancel a job and mark it for termination + */ + async cancelJob(jobId: number): Promise { + this.logger.warn(`Cancelling job ${jobId}`); + this.jobCancellationFlags.set(jobId, true); + + try { + await this.jobStatusService.updateJobStatus(jobId, { + status: "Failed", + progress: "Job cancelled due to timeout or user request", + }); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.error(`Error updating cancelled job status: ${errorMessage}`); + } + } + + /** + * Check if a job should be cancelled + */ + private isJobCancelled(jobId?: number): boolean { + if (!jobId) return false; + return this.jobCancellationFlags.get(jobId) === true; + } + + /** + * Clean up cancelled job resources + */ + private cleanupCancelledJob(jobId: number): void { + this.jobCancellationFlags.delete(jobId); + this.jobStartTimes.delete(jobId); + } + + /** + * Check for jobs that have exceeded the timeout and cancel them + */ + private checkJobTimeouts(): void { + const now = Date.now(); + const expiredJobs: number[] = []; + + for (const [jobId, startTime] of this.jobStartTimes.entries()) { + if (now - startTime > this.JOB_TIMEOUT) { + expiredJobs.push(jobId); + } + } + + for (const jobId of expiredJobs) { + this.logger.warn( + `Job ${jobId} exceeded timeout (${this.JOB_TIMEOUT}ms), cancelling`, + ); + void this.cancelJob(jobId).then(() => this.cleanupCancelledJob(jobId)); + } + } + /** * Mark a language as completed in the progress tracker */ - private incrementLanguageCompleted(tracker: ProgressTracker): void { + private incrementLanguageCompleted( + tracker: ProgressTracker | undefined, + ): void { + if (!tracker) { + return; + } tracker.languageCompleted++; } - /** - * Mark an item as completed in the progress tracker - */ - private incrementCompletedItems(tracker: ProgressTracker): void { - tracker.completedItems++; - } + /** + * Mark an item as completed in the progress tracker + */ + private incrementCompletedItems(tracker: ProgressTracker | undefined): void { + if (!tracker) { + return; + } + tracker.completedItems++; + } + + /** + * Set the current item index in the progress tracker + */ + private setCurrentItemIndex( + tracker: ProgressTracker | undefined, + index: number, + ): void { + if (!tracker) { + return; + } + tracker.currentItemIndex = index; + } + + /** + * Force retranslation of an assignment for specific languages + * Used when language mismatches are detected + * + * @param assignmentId - The assignment ID + * @param languageCodes - Array of language codes to retranslate + * @param jobId - Optional job ID for progress tracking + */ + async retranslateAssignmentForLanguages( + assignmentId: number, + languageCodes: string[], + jobId?: number, + ): Promise { + if (languageCodes.length === 0) { + return; + } + + this.logger.log( + `Force retranslating assignment ${assignmentId} for languages: ${languageCodes.join( + ", ", + )}`, + ); + + // Delete existing translations for the specified languages + await this.prisma.assignmentTranslation.deleteMany({ + where: { + assignmentId, + languageCode: { in: languageCodes }, + }, + }); + + await this.prisma.translation.deleteMany({ + where: { + question: { + assignmentId, + }, + languageCode: { in: languageCodes }, + }, + }); + + // Get assignment data + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + select: { + id: true, + name: true, + introduction: true, + instructions: true, + gradingCriteriaOverview: true, + }, + }); + + if (!assignment) { + throw new NotFoundException( + `Assignment with id ${assignmentId} not found`, + ); + } + + // Retranslate assignment for specific languages + const progressTracker = jobId + ? this.initializeProgressTracker( + jobId, + languageCodes.length, + 10, + 30, + "Retranslating assignment metadata", + languageCodes.length, + ) + : undefined; + + await this.processBatchesInParallel( + languageCodes, + async (lang: string) => { + try { + await this.translateAssignmentToLanguage( + assignment as unknown as GetAssignmentResponseDto, + lang, + ); + if (progressTracker) { + this.incrementLanguageCompleted(progressTracker); + } + return true; + } catch (error: unknown) { + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.error( + `Failed to retranslate assignment to ${lang}: ${errorMessage}`, + ); + return false; + } + }, + 10, // Smaller batch size for targeted retranslation + 25, // Lower concurrency + ); + + // Get all questions and variants + const questions = await this.prisma.question.findMany({ + where: { + assignmentId, + isDeleted: false, + }, + include: { + variants: { + where: { isDeleted: false }, + }, + }, + }); + + // Retranslate questions and variants + if (jobId) { + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: `Retranslating ${questions.length} questions for ${languageCodes.length} languages`, + percentage: 40, + }); + } + + let processedItems = 0; + const totalItems = + questions.reduce( + (accumulator, q) => accumulator + 1 + q.variants.length, + 0, + ) * languageCodes.length; + + for (const question of questions) { + // Retranslate question + for (const lang of languageCodes) { + await this.generateAndStoreTranslation( + assignmentId, + question.id, + null, + question.question, + question.choices as unknown as Choice[], + await this.llmFacadeService.getLanguageCode( + question.question, + assignmentId, + ), + lang, + ); + processedItems++; + + if (jobId && processedItems % 10 === 0) { + const percentage = + 40 + Math.floor((processedItems / totalItems) * 50); + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: `Retranslating content: ${processedItems}/${totalItems} items`, + percentage, + }); + } + } + + // Retranslate variants + for (const variant of question.variants) { + for (const lang of languageCodes) { + await this.generateAndStoreTranslation( + assignmentId, + question.id, + variant.id, + variant.variantContent, + variant.choices as unknown as Choice[], + await this.llmFacadeService.getLanguageCode( + variant.variantContent, + assignmentId, + ), + lang, + ); + processedItems++; + + if (jobId && processedItems % 10 === 0) { + const percentage = + 40 + Math.floor((processedItems / totalItems) * 50); + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: `Retranslating content: ${processedItems}/${totalItems} items`, + percentage, + }); + } + } + } + } + + if (jobId) { + await this.jobStatusService.updateJobStatus(jobId, { + status: "In Progress", + progress: `Retranslation completed for ${languageCodes.length} languages`, + percentage: 95, + }); + } - /** - * Set the current item index in the progress tracker - */ - private setCurrentItemIndex(tracker: ProgressTracker, index: number): void { - tracker.currentItemIndex = index; + this.logger.log( + `Completed retranslation for assignment ${assignmentId}, languages: ${languageCodes.join( + ", ", + )}`, + ); } /** @@ -439,6 +1169,11 @@ export class TranslationService { return; } + // Track job start time for timeout monitoring + if (jobId) { + this.jobStartTimes.set(jobId, Date.now()); + } + this.checkLimiterHealth(); const assignment = (await this.prisma.assignment.findUnique({ @@ -496,6 +1231,14 @@ export class TranslationService { supportedLanguages, async (lang: string) => { try { + // Check for job cancellation + if (jobId && this.isJobCancelled(jobId)) { + this.logger.warn( + `Job ${jobId} cancelled, stopping translation for ${lang}`, + ); + return false; + } + if (progressTracker && jobId) { await this.updateJobProgress( progressTracker, @@ -547,6 +1290,11 @@ export class TranslationService { this.logger.log( `Assignment #${assignmentId} translation results: ${results.success} successful, ${results.failure} failed, ${results.dropped} dropped/retried`, ); + + // Clean up job tracking + if (jobId) { + this.cleanupCancelledJob(jobId); + } } /** @@ -563,29 +1311,37 @@ export class TranslationService { questionId: number, question: QuestionDto, jobId: number, + forceRetranslation = false, ): Promise { + const hasValidJobId = jobId && jobId > 0; + if (!this.languageTranslation) { - await this.jobStatusService.updateJobStatus(jobId, { - status: "Completed", - progress: `Translation skipped for question #${questionId} (disabled in development mode)`, - percentage: 100, - }); + if (hasValidJobId) { + await this.jobStatusService.updateJobStatus(jobId, { + status: "Completed", + progress: `Translation skipped for question #${questionId} (disabled in development mode)`, + percentage: 100, + }); + } return; } - await this.jobStatusService.updateJobStatus(jobId, { - status: "In Progress", - progress: `Preparing question #${questionId} for translation`, - percentage: 5, - }); + // Track job start time only for valid job IDs + if (hasValidJobId) { + this.jobStartTimes.set(jobId, Date.now()); + } + + // Don't interfere with main job progress - let parent job handle progress updates const normalizedText = question.question.trim(); const normalizedChoices = question.choices ?? null; let questionLang = "en"; try { - const detectedLang = - await this.llmFacadeService.getLanguageCode(normalizedText); + const detectedLang = await this.llmFacadeService.getLanguageCode( + normalizedText, + assignmentId, + ); if (detectedLang && detectedLang !== "unknown") { questionLang = detectedLang; } @@ -614,26 +1370,54 @@ export class TranslationService { supportedLanguages.length, ); - // DELETE ALL existing translations for this question first - await this.prisma.translation.deleteMany({ - where: { - questionId: questionId, - variantId: null, - }, - }); + // Only delete existing translations if content has changed or forced + if (forceRetranslation) { + await this.prisma.translation.deleteMany({ + where: { + questionId: questionId, + variantId: null, + }, + }); + } else { + // Check if translations already exist for this question + const existingTranslations = await this.prisma.translation.findMany({ + where: { + questionId: questionId, + variantId: null, + }, + select: { languageCode: true }, + }); + + const existingLanguages = new Set( + existingTranslations.map((t) => t.languageCode), + ); + const missingLanguages = supportedLanguages.filter( + (lang) => !existingLanguages.has(lang), + ); + + if (missingLanguages.length === 0) { + // All translations already exist - no need to retranslate + if (hasValidJobId) { + this.cleanupCancelledJob(jobId); + } + return; + } + } const results = await this.processBatchesInParallel( supportedLanguages, async (lang: string) => { try { - await this.updateJobProgress( - progressTracker, - getLanguageNameFromCode(lang), - undefined, - lang === questionLang - ? "Storing original content" - : "Checking for existing translation", - ); + if (progressTracker) { + await this.updateJobProgress( + progressTracker, + getLanguageNameFromCode(lang), + undefined, + lang === questionLang + ? "Storing original content" + : "Checking for existing translation", + ); + } await this.generateAndStoreTranslation( assignmentId, @@ -645,15 +1429,17 @@ export class TranslationService { lang, ); - this.incrementLanguageCompleted(progressTracker); - await this.updateJobProgress( - progressTracker, - getLanguageNameFromCode(lang), - undefined, - lang === questionLang - ? "Original stored ✓" - : "Translation completed ✓", - ); + if (progressTracker) { + this.incrementLanguageCompleted(progressTracker); + await this.updateJobProgress( + progressTracker, + getLanguageNameFromCode(lang), + undefined, + lang === questionLang + ? "Original stored ✓" + : "Translation completed ✓", + ); + } return true; } catch (error) { @@ -669,11 +1455,11 @@ export class TranslationService { this.CONCURRENCY_LIMIT, ); - await this.jobStatusService.updateJobStatus(jobId, { - status: "Completed", - progress: `Question #${questionId} translated to ${results.success} languages (${results.failure} failed, ${results.dropped} retried)`, - percentage: 100, - }); + // Question translation complete - parent job will handle final status updates + if (hasValidJobId) { + // Clean up job tracking + this.cleanupCancelledJob(jobId); + } this.logger.log( `Question #${questionId} translation results: ${results.success} successful, ${results.failure} failed, ${results.dropped} dropped/retried`, @@ -696,29 +1482,60 @@ export class TranslationService { variantId: number, variant: VariantDto, jobId: number, + forceRetranslation = false, ): Promise { + const hasValidJobId = jobId && jobId > 0; + if (!this.languageTranslation) { - await this.jobStatusService.updateJobStatus(jobId, { - status: "Completed", - progress: `Translation skipped for variant #${variantId} (disabled in development mode)`, - percentage: 100, - }); + if (hasValidJobId) { + await this.jobStatusService.updateJobStatus(jobId, { + status: "Completed", + progress: `Translation skipped for variant #${variantId} (disabled in development mode)`, + percentage: 100, + }); + } return; } - await this.jobStatusService.updateJobStatus(jobId, { - status: "In Progress", - progress: `Preparing variant #${variantId} for translation`, - percentage: 10, - }); + // Track job start time only for valid job IDs + if (hasValidJobId) { + this.jobStartTimes.set(jobId, Date.now()); + } + // Don't interfere with main job progress - let parent job handle progress updates const normalizedText = variant.variantContent.trim(); const normalizedChoices = variant.choices ?? null; + // Check if translations already exist for this variant (unless forced) + if (!forceRetranslation) { + const existingTranslations = await this.prisma.translation.findMany({ + where: { + questionId: questionId, + variantId: variantId, + }, + select: { languageCode: true }, + }); + + const supportedLanguages = getAllLanguageCodes() ?? ["en"]; + const existingLanguages = new Set( + existingTranslations.map((t) => t.languageCode), + ); + const missingLanguages = supportedLanguages.filter( + (lang) => !existingLanguages.has(lang), + ); + + if (missingLanguages.length === 0) { + // Variant already translated - let parent job handle progress + return; + } + } + let variantLang = "en"; try { - const detectedLang = - await this.llmFacadeService.getLanguageCode(normalizedText); + const detectedLang = await this.llmFacadeService.getLanguageCode( + normalizedText, + assignmentId, + ); if (detectedLang && detectedLang !== "unknown") { variantLang = detectedLang; } @@ -739,26 +1556,30 @@ export class TranslationService { supportedLanguages.length, ); - // DELETE ALL existing translations for this variant first - await this.prisma.translation.deleteMany({ - where: { - questionId: questionId, - variantId: variantId, - }, - }); + // Only delete existing translations if content has changed (forceRetranslation = true) + if (forceRetranslation) { + await this.prisma.translation.deleteMany({ + where: { + questionId: questionId, + variantId: variantId, + }, + }); + } const results = await this.processBatchesInParallel( supportedLanguages, async (lang: string) => { try { - await this.updateJobProgress( - progressTracker, - getLanguageNameFromCode(lang), - undefined, - lang === variantLang - ? "Storing original content" - : "Checking for existing translation", - ); + if (progressTracker) { + await this.updateJobProgress( + progressTracker, + getLanguageNameFromCode(lang), + undefined, + lang === variantLang + ? "Storing original content" + : "Checking for existing translation", + ); + } await this.generateAndStoreTranslation( assignmentId, @@ -770,15 +1591,17 @@ export class TranslationService { lang, ); - this.incrementLanguageCompleted(progressTracker); - await this.updateJobProgress( - progressTracker, - getLanguageNameFromCode(lang), - undefined, - lang === variantLang - ? "Original stored ✓" - : "Translation completed ✓", - ); + if (progressTracker) { + this.incrementLanguageCompleted(progressTracker); + await this.updateJobProgress( + progressTracker, + getLanguageNameFromCode(lang), + undefined, + lang === variantLang + ? "Original stored ✓" + : "Translation completed ✓", + ); + } return true; } catch (error) { @@ -794,11 +1617,11 @@ export class TranslationService { this.CONCURRENCY_LIMIT, ); - await this.jobStatusService.updateJobStatus(jobId, { - status: "Completed", - progress: `Variant #${variantId} translated to ${results.success} languages (${results.failure} failed, ${results.dropped} retried)`, - percentage: 100, - }); + // Variant translation complete - parent job will handle final status updates + if (hasValidJobId) { + // Clean up job tracking + this.cleanupCancelledJob(jobId); + } this.logger.log( `Variant #${variantId} translation results: ${results.success} successful, ${results.failure} failed, ${results.dropped} dropped/retried`, @@ -998,11 +1821,29 @@ export class TranslationService { } } - await Promise.all(translationPromises); - try { - await this.prisma.assignmentTranslation.create({ - data: { + await Promise.all(translationPromises); + + // Only create assignment translation if all fields were translated successfully + await this.prisma.assignmentTranslation.upsert({ + where: { + assignmentId_languageCode: { + assignmentId: assignment.id, + languageCode: lang, + }, + }, + update: { + name: assignment.name || "", + translatedName: translatedData.name, + instructions: assignment.instructions || "", + translatedInstructions: translatedData.instructions, + gradingCriteriaOverview: assignment.gradingCriteriaOverview || "", + translatedGradingCriteriaOverview: + translatedData.gradingCriteriaOverview, + introduction: assignment.introduction || "", + translatedIntroduction: translatedData.introduction, + }, + create: { assignment: { connect: { id: assignment.id } }, languageCode: lang, name: assignment.name || "", @@ -1016,102 +1857,16 @@ export class TranslationService { translatedIntroduction: translatedData.introduction, }, }); - } catch (error: unknown) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger.error(`Error creating translation record: ${errorMessage}`); - throw error; - } - } - - /** - * Find an existing translation for reuse - * Optimized query performance - * - * @param text - The text to translate - * @param choices - The choices to translate - * @param languageCode - The target language code - * @returns Existing translation if found - */ - private async findExistingTranslation( - text: string, - choices: Choice[] | null, - languageCode: string, - ): Promise { - try { - return await this.prisma.translation.findFirst({ - where: { - languageCode, - untranslatedText: text, - untranslatedChoices: { equals: this.prepareJsonValue(choices) }, - }, - select: { - id: true, - questionId: true, - variantId: true, - languageCode: true, - translatedText: true, - translatedChoices: true, - untranslatedText: true, - untranslatedChoices: true, - createdAt: true, - }, - }); - } catch (error: unknown) { - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger.error(`Error finding existing translation: ${errorMessage}`); - return null; - } - } - - /** - * Link an existing translation to a question or variant - * Optimized for performance - * - * @param questionId - The question ID - * @param variantId - The variant ID (or null) - * @param existingTranslation - The existing translation to reuse - * @param normalizedText - The original text - * @param normalizedChoices - The original choices - * @param languageCode - The target language code - */ - private async linkExistingTranslation( - questionId: number, - variantId: number | null, - existingTranslation: Translation, - normalizedText: string, - normalizedChoices: Choice[] | null, - languageCode: string, - ): Promise { - try { - const existingCount = await this.prisma.translation.count({ - where: { - questionId, - variantId, - languageCode, - }, - }); - - if (existingCount === 0) { - await this.prisma.translation.create({ - data: { - questionId, - variantId, - languageCode, - untranslatedText: normalizedText, - untranslatedChoices: this.prepareJsonValue(normalizedChoices), - translatedText: existingTranslation.translatedText, - translatedChoices: - existingTranslation.translatedChoices ?? Prisma.JsonNull, - }, - }); - } - } catch (error: unknown) { + } catch (translationError: unknown) { const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger.error(`Error linking existing translation: ${errorMessage}`); - throw error; + translationError instanceof Error + ? translationError.message + : String(translationError); + this.logger.warn( + `Skipping assignment translation creation for ${lang} due to translation failure for assignment ${assignment.id}: ${errorMessage}`, + ); + // Don't create assignment translation for this language + throw translationError; } } @@ -1129,6 +1884,7 @@ export class TranslationService { */ /** * Generate and store translation (creates new record each time) + * Enhanced with better source language handling and context awareness */ private async generateAndStoreTranslation( assignmentId: number, @@ -1139,46 +1895,84 @@ export class TranslationService { sourceLanguage: string, targetLanguage: string, ): Promise { - // OPTIMIZATION: Check if we already have a translation for this EXACT text - const existingTranslation = await this.prisma.translation.findFirst({ - where: { - languageCode: targetLanguage, - untranslatedText: originalText, // Exact match on the text content - untranslatedChoices: { equals: this.prepareJsonValue(originalChoices) }, - }, - orderBy: { createdAt: "desc" }, // Get the most recent one - }); - - if (existingTranslation) { - // Reuse existing translation for this exact content - await this.prisma.translation.create({ - data: { - questionId, - variantId, - languageCode: targetLanguage, - untranslatedText: originalText, - untranslatedChoices: this.prepareJsonValue(originalChoices), - translatedText: existingTranslation.translatedText, // Reuse existing translation - translatedChoices: existingTranslation.translatedChoices, - }, - }); - return; - } + // Check if we need to reuse existing translation + // Only reuse if it's from the same assignment (context-aware) + // const existingTranslation = await this.prisma.translation.findFirst({ + // where: { + // question: { + // assignmentId: assignmentId, // Same assignment context + // }, + // languageCode: targetLanguage, + // untranslatedText: originalText, + // untranslatedChoices: { equals: this.prepareJsonValue(originalChoices) }, + // }, + // orderBy: { createdAt: "desc" }, + // }); + + // if (existingTranslation) { + // // Reuse existing translation for this exact content + // try { + // await this.prisma.translation.create({ + // data: { + // questionId, + // variantId, + // languageCode: targetLanguage, + // untranslatedText: originalText, + // untranslatedChoices: this.prepareJsonValue(originalChoices), + // translatedText: existingTranslation.translatedText, // Reuse existing translation + // translatedChoices: existingTranslation.translatedChoices, + // }, + // }); + // } catch (createError) { + // // Check if record now exists due to race condition + // const existingRecord = await this.prisma.translation.findFirst({ + // where: { + // questionId, + // variantId, + // languageCode: targetLanguage, + // }, + // }); + + // if (!existingRecord) { + // // If still no existing record, re-throw the original error + // throw createError; + // } + // // If record exists now, silently continue (race condition resolved) + // } + // return; + // } // No existing translation found - generate new one if (sourceLanguage.toLowerCase() === targetLanguage.toLowerCase()) { // Same language - store original content - await this.prisma.translation.create({ - data: { - questionId, - variantId, - languageCode: targetLanguage, - untranslatedText: originalText, - untranslatedChoices: this.prepareJsonValue(originalChoices), - translatedText: originalText, - translatedChoices: this.prepareJsonValue(originalChoices), - }, - }); + try { + await this.prisma.translation.create({ + data: { + questionId, + variantId, + languageCode: targetLanguage, + untranslatedText: originalText, + untranslatedChoices: this.prepareJsonValue(originalChoices), + translatedText: originalText, + translatedChoices: this.prepareJsonValue(originalChoices), + }, + }); + } catch (createError) { + // Check if record now exists due to race condition + const existingRecord = await this.prisma.translation.findFirst({ + where: { + questionId, + variantId, + languageCode: targetLanguage, + }, + }); + + if (!existingRecord) { + // If still no existing record, re-throw the original error + throw createError; + } + // If record exists now, silently continue (race condition resolved) + } return; } @@ -1237,20 +2031,51 @@ export class TranslationService { ); } - await Promise.all(translationPromises); + try { + await Promise.all(translationPromises); - // Create new translation record - await this.prisma.translation.create({ - data: { - questionId, - variantId, - languageCode: targetLanguage, - untranslatedText: originalText, - untranslatedChoices: this.prepareJsonValue(originalChoices), - translatedText, - translatedChoices: this.prepareJsonValue(translatedChoices), - }, - }); + // Create new translation record only if all translations succeeded + try { + await this.prisma.translation.create({ + data: { + questionId, + variantId, + languageCode: targetLanguage, + untranslatedText: originalText, + untranslatedChoices: this.prepareJsonValue(originalChoices), + translatedText, + translatedChoices: this.prepareJsonValue(translatedChoices), + }, + }); + } catch (createError) { + // Check if record now exists due to race condition + const existingRecord = await this.prisma.translation.findFirst({ + where: { + questionId, + variantId, + languageCode: targetLanguage, + }, + }); + + if (!existingRecord) { + // If still no existing record, re-throw the original error + throw createError; + } + // If record exists now, silently continue (race condition resolved) + this.logger.debug( + `Translation record already exists for question ${questionId} in ${targetLanguage} (race condition resolved)`, + ); + } + } catch (translationError) { + // If any translation failed, don't create the translation record + // This language will not be available for this question/variant + this.logger.warn( + `Skipping translation record creation for ${targetLanguage} due to translation failure for question ${questionId}${ + variantId ? ` variant ${variantId}` : "" + }`, + ); + throw translationError; + } } /** * Prepare a value for storage as Prisma.JsonValue diff --git a/apps/api/src/api/assignment/v2/services/version-management.service.ts b/apps/api/src/api/assignment/v2/services/version-management.service.ts new file mode 100644 index 00000000..66c88ecc --- /dev/null +++ b/apps/api/src/api/assignment/v2/services/version-management.service.ts @@ -0,0 +1,2148 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { + BadRequestException, + ConflictException, + Inject, + Injectable, + NotFoundException, +} from "@nestjs/common"; +import { + Assignment, + AssignmentVersion, + Prisma, + Question, +} from "@prisma/client"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { + UserRole, + UserSession, +} from "src/auth/interfaces/user.session.interface"; +import { Logger } from "winston"; +import { PrismaService } from "../../../../prisma.service"; +import { QuestionDto } from "../../dto/update.questions.request.dto"; + +export interface CreateVersionDto { + versionNumber?: string; + versionDescription?: string; + isDraft?: boolean; + shouldActivate?: boolean; + updateExisting?: boolean; + versionId?: number; // ID of the version to update when updateExisting is true +} + +export interface CompareVersionsDto { + fromVersionId: number; + toVersionId: number; +} + +export interface RestoreVersionDto { + versionId: number; + createAsNewVersion?: boolean; + versionDescription?: string; +} + +export interface SaveDraftDto { + assignmentData: Partial; + questionsData?: Array; + versionDescription?: string; + versionNumber?: string; +} + +export interface VersionSummary { + id: number; + versionNumber: string; + versionDescription?: string; + isDraft: boolean; + isActive: boolean; + published: boolean; + createdBy: string; + createdAt: Date; + questionCount: number; + wasAutoIncremented?: boolean; + originalVersionNumber?: string; +} + +export interface VersionComparison { + fromVersion: VersionSummary; + toVersion: VersionSummary; + assignmentChanges: Array<{ + field: string; + fromValue: any; + toValue: any; + changeType: "added" | "modified" | "removed"; + }>; + questionChanges: Array<{ + questionId?: number; + displayOrder: number; + changeType: "added" | "modified" | "removed"; + field?: string; + fromValue?: any; + toValue?: any; + }>; +} + +@Injectable() +export class VersionManagementService { + private readonly logger: Logger; + + constructor( + private readonly prisma: PrismaService, + @Inject(WINSTON_MODULE_PROVIDER) private parentLogger: Logger, + ) { + this.logger = parentLogger.child({ context: "VersionManagementService" }); + } + + /** + * Get the most recently created version for an assignment + */ + async getLatestVersion(assignmentId: number): Promise { + // Verify assignment access + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { AssignmentAuthor: true }, + }); + + if (!assignment) { + throw new NotFoundException("Assignment not found"); + } + + const latestVersion = await this.prisma.assignmentVersion.findFirst({ + where: { assignmentId }, + include: { _count: { select: { questionVersions: true } } }, + orderBy: { createdAt: "desc" }, + }); + + if (!latestVersion) { + return null; + } + + return { + id: latestVersion.id, + versionNumber: latestVersion.versionNumber, + versionDescription: latestVersion.versionDescription, + isDraft: latestVersion.isDraft, + isActive: latestVersion.isActive, + published: latestVersion.published, + createdBy: latestVersion.createdBy, + createdAt: latestVersion.createdAt, + questionCount: latestVersion._count.questionVersions, + }; + } + + async createVersion( + assignmentId: number, + createVersionDto: CreateVersionDto, + userSession: UserSession, + ): Promise { + this.logger.info( + `🚀 CREATE VERSION: Starting for assignment ${assignmentId}`, + { + createVersionDto, + userId: userSession.userId, + }, + ); + + // Log the key decision factors + this.logger.info(`🔍 VERSION CREATE PARAMS:`, { + updateExisting: createVersionDto.updateExisting, + versionId: createVersionDto.versionId, + versionNumber: createVersionDto.versionNumber, + isDraft: createVersionDto.isDraft, + }); + + // Verify assignment exists and user has access + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { + questions: { where: { isDeleted: false } }, + versions: { orderBy: { createdAt: "desc" }, take: 1 }, + }, + }); + + if (!assignment) { + throw new NotFoundException("Assignment not found"); + } + + this.logger.info(`Creating version for assignment ${assignmentId}`, { + assignmentName: assignment.name, + questionsFound: assignment.questions.length, + questionIds: assignment.questions.map((q) => q.id), + createVersionDto, + }); + + // If no legacy questions found, this might be a new assignment created through version control + if (assignment.questions.length === 0) { + this.logger.warn( + `No legacy questions found for assignment ${assignmentId}. Creating version with empty questions.`, + ); + } + + // Handle version number - if not provided, generate a default one + let versionNumber: string; + + if (createVersionDto.versionNumber) { + // Validate provided semantic version format + const semanticVersionRegex = /^\d+\.\d+\.\d+(?:-rc\d+)?$/; + if (!semanticVersionRegex.test(createVersionDto.versionNumber)) { + throw new BadRequestException( + "Version number must follow semantic versioning format (e.g., '1.0.0' or '1.0.0-rc1')", + ); + } + versionNumber = createVersionDto.versionNumber; + } else { + // Generate default version number (legacy behavior) + const latestVersion = await this.prisma.assignmentVersion.findFirst({ + where: { assignmentId }, + orderBy: { createdAt: "desc" }, + select: { versionNumber: true }, + }); + + if (latestVersion && /^\d+\.\d+\.\d+/.test(latestVersion.versionNumber)) { + // If latest version is semantic, increment patch version + const match = latestVersion.versionNumber.match(/^(\d+)\.(\d+)\.(\d+)/); + if (match) { + const [, major, minor, patch] = match; + versionNumber = `${major}.${minor}.${Number.parseInt(patch) + 1}`; + } else { + versionNumber = "1.0.0"; + } + } else { + // Default to 1.0.0 for first version + versionNumber = "1.0.0"; + } + + // Add -rc suffix if it's a draft + if (createVersionDto.isDraft) { + versionNumber += "-rc1"; + } + } + + // Handle updateExisting with versionId first (highest priority) + if (createVersionDto.updateExisting && createVersionDto.versionId) { + this.logger.info( + `🔄 UPDATE PATH: Updating existing version ${createVersionDto.versionId} directly for assignment ${assignmentId}`, + { + versionId: createVersionDto.versionId, + versionNumber: createVersionDto.versionNumber, + versionDescription: createVersionDto.versionDescription, + }, + ); + return await this.updateExistingVersion( + assignmentId, + createVersionDto.versionId, + createVersionDto, + userSession, + ); + } + + // Check for duplicate version numbers and handle conflicts + const originalVersionNumber = versionNumber; + const finalVersionNumber = versionNumber; + const wasAutoIncremented = false; + + const existingVersion = await this.prisma.assignmentVersion.findFirst({ + where: { + assignmentId, + versionNumber: finalVersionNumber, + }, + include: { _count: { select: { questionVersions: true } } }, + }); + + if (existingVersion) { + if (createVersionDto.updateExisting) { + // Update existing version instead of creating new one (fallback to version number lookup) + this.logger.info( + `Updating existing version ${existingVersion.id} found by version number ${finalVersionNumber}`, + ); + return await this.updateExistingVersion( + assignmentId, + existingVersion.id, + createVersionDto, + userSession, + ); + } else { + // Return a special response indicating the version exists + const versionExistsError = new ConflictException({ + message: `Version ${finalVersionNumber} already exists for this assignment`, + versionExists: true, + existingVersion: { + id: existingVersion.id, + versionNumber: existingVersion.versionNumber, + versionDescription: existingVersion.versionDescription, + isDraft: existingVersion.isDraft, + isActive: existingVersion.isActive, + published: existingVersion.published, + createdBy: existingVersion.createdBy, + createdAt: existingVersion.createdAt, + questionCount: existingVersion._count.questionVersions, + }, + }); + throw versionExistsError; + } + } + + // Update versionNumber to use the final (possibly incremented) version + versionNumber = finalVersionNumber; + + return await this.prisma.$transaction(async (tx) => { + // Create assignment version + const assignmentVersion = await tx.assignmentVersion.create({ + data: { + assignmentId, + versionNumber, + name: assignment.name, + introduction: assignment.introduction, + instructions: assignment.instructions, + gradingCriteriaOverview: assignment.gradingCriteriaOverview, + timeEstimateMinutes: assignment.timeEstimateMinutes, + type: assignment.type, + graded: assignment.graded, + numAttempts: assignment.numAttempts, + allotedTimeMinutes: assignment.allotedTimeMinutes, + attemptsPerTimeRange: assignment.attemptsPerTimeRange, + attemptsTimeRangeHours: assignment.attemptsTimeRangeHours, + passingGrade: assignment.passingGrade, + displayOrder: assignment.displayOrder, + questionDisplay: assignment.questionDisplay, + numberOfQuestionsPerAttempt: assignment.numberOfQuestionsPerAttempt, + questionOrder: assignment.questionOrder, + published: !createVersionDto.isDraft, // Published only if not a draft + showAssignmentScore: assignment.showAssignmentScore, + showQuestionScore: assignment.showQuestionScore, + showSubmissionFeedback: assignment.showSubmissionFeedback, + showQuestions: assignment.showQuestions, + languageCode: assignment.languageCode, + createdBy: userSession.userId, + isDraft: createVersionDto.isDraft ?? true, + versionDescription: wasAutoIncremented + ? `${ + createVersionDto.versionDescription || "" + } (Auto-incremented from ${originalVersionNumber} due to version conflict)`.trim() + : createVersionDto.versionDescription, + isActive: createVersionDto.shouldActivate ?? false, + }, + }); + + // Create question versions + this.logger.info( + `Creating ${assignment.questions.length} question versions for assignment version ${assignmentVersion.id}`, + ); + + for (const [index, question] of assignment.questions.entries()) { + const questionVersion = await tx.questionVersion.create({ + data: { + assignmentVersionId: assignmentVersion.id, + questionId: question.id, + totalPoints: question.totalPoints, + type: question.type, + responseType: question.responseType, + question: question.question, + maxWords: question.maxWords, + scoring: question.scoring, + choices: question.choices, + randomizedChoices: question.randomizedChoices, + answer: question.answer, + gradingContextQuestionIds: question.gradingContextQuestionIds, + maxCharacters: question.maxCharacters, + videoPresentationConfig: question.videoPresentationConfig, + liveRecordingConfig: question.liveRecordingConfig, + displayOrder: index + 1, + }, + }); + + this.logger.debug( + `Created question version ${questionVersion.id} for question ${ + question.id + } (${question.question.slice(0, 50)}...)`, + ); + } + + this.logger.info( + `Successfully created all ${assignment.questions.length} question versions`, + ); + + // Update current version if should activate + if (createVersionDto.shouldActivate) { + // Deactivate other versions + await tx.assignmentVersion.updateMany({ + where: { assignmentId, id: { not: assignmentVersion.id } }, + data: { isActive: false }, + }); + + // Update assignment currentVersionId + await tx.assignment.update({ + where: { id: assignmentId }, + data: { currentVersionId: assignmentVersion.id }, + }); + } + + // Create version history entry + await tx.versionHistory.create({ + data: { + assignmentId, + fromVersionId: assignment.currentVersionId, + toVersionId: assignmentVersion.id, + action: createVersionDto.isDraft + ? "draft_created" + : "version_created", + description: createVersionDto.versionDescription, + userId: userSession.userId, + }, + }); + + this.logger.info( + `Created version ${versionNumber} for assignment ${assignmentId}${ + wasAutoIncremented ? " (auto-incremented)" : "" + }`, + { + versionId: assignmentVersion.id, + originalVersionNumber: wasAutoIncremented + ? originalVersionNumber + : undefined, + }, + ); + + return { + id: assignmentVersion.id, + versionNumber: assignmentVersion.versionNumber, + versionDescription: assignmentVersion.versionDescription, + isDraft: assignmentVersion.isDraft, + isActive: assignmentVersion.isActive, + published: assignmentVersion.published, + createdBy: assignmentVersion.createdBy, + createdAt: assignmentVersion.createdAt, + questionCount: assignment.questions.length, + wasAutoIncremented, + originalVersionNumber: wasAutoIncremented + ? originalVersionNumber + : undefined, + }; + }); + } + + async listVersions(assignmentId: number): Promise { + // Verify assignment access + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { AssignmentAuthor: true, currentVersion: true }, + }); + + if (!assignment) { + this.logger.error(`❌ Assignment ${assignmentId} not found`); + throw new NotFoundException("Assignment not found"); + } + + const versions = await this.prisma.assignmentVersion.findMany({ + where: { assignmentId }, + include: { _count: { select: { questionVersions: true } } }, + orderBy: { createdAt: "desc" }, + }); + + const versionSummaries = versions.map((version) => ({ + id: version.id, + versionNumber: version.versionNumber, + versionDescription: version.versionDescription, + isDraft: version.isDraft, + isActive: version.isActive, + published: version.published, + createdBy: version.createdBy, + createdAt: version.createdAt, + questionCount: version._count.questionVersions, + })); + + return versionSummaries; + } + + async getVersion( + assignmentId: number, + versionId: number, + ): Promise< + AssignmentVersion & { + questionVersions: any[]; + } + > { + const version = await this.prisma.assignmentVersion.findUnique({ + where: { id: versionId, assignmentId }, + include: { questionVersions: { orderBy: { displayOrder: "asc" } } }, + }); + + if (!version) { + throw new NotFoundException("Version not found"); + } + + this.logger.info( + `Found version ${versionId} for assignment ${assignmentId}`, + { + versionId: version.id, + assignmentId: version.assignmentId, + questionVersionsCount: version.questionVersions.length, + questionVersions: version.questionVersions.map((qv) => ({ + id: qv.id, + questionId: qv.questionId, + question: qv.question?.slice(0, 50) + "...", + })), + }, + ); + + // Transform the response to match the expected format + return { + id: version.id, + versionNumber: version.versionNumber, + versionDescription: version.versionDescription, + isDraft: version.isDraft, + isActive: version.isActive, + createdBy: version.createdBy, + createdAt: version.createdAt, + assignmentId: version.assignmentId, + name: version.name, + introduction: version.introduction, + instructions: version.instructions, + gradingCriteriaOverview: version.gradingCriteriaOverview, + timeEstimateMinutes: version.timeEstimateMinutes, + type: version.type, + graded: version.graded, + numAttempts: version.numAttempts, + allotedTimeMinutes: version.allotedTimeMinutes, + attemptsPerTimeRange: version.attemptsPerTimeRange, + attemptsTimeRangeHours: version.attemptsTimeRangeHours, + passingGrade: version.passingGrade, + displayOrder: version.displayOrder, + questionDisplay: version.questionDisplay, + numberOfQuestionsPerAttempt: version.numberOfQuestionsPerAttempt, + questionOrder: version.questionOrder, + published: version.published, + showAssignmentScore: version.showAssignmentScore, + showQuestionScore: version.showQuestionScore, + showSubmissionFeedback: version.showSubmissionFeedback, + showQuestions: version.showQuestions, + languageCode: version.languageCode, + // Transform questionVersions to the expected questions format + questionVersions: version.questionVersions.map((qv) => ({ + id: qv.id, + questionId: qv.questionId, + totalPoints: qv.totalPoints, + type: qv.type, + responseType: qv.responseType, + question: qv.question, + maxWords: qv.maxWords, + scoring: qv.scoring, + choices: qv.choices, + randomizedChoices: qv.randomizedChoices, + answer: qv.answer, + gradingContextQuestionIds: qv.gradingContextQuestionIds, + maxCharacters: qv.maxCharacters, + videoPresentationConfig: qv.videoPresentationConfig, + liveRecordingConfig: qv.liveRecordingConfig, + displayOrder: qv.displayOrder, + })), + }; + } + + async saveDraft( + assignmentId: number, + saveDraftDto: SaveDraftDto, + userSession: UserSession, + ): Promise { + this.logger.info(`Saving draft for assignment ${assignmentId}`, { + userId: userSession.userId, + }); + + // Check if there's an existing draft version + const existingDraft = await this.prisma.assignmentVersion.findFirst({ + where: { + assignmentId, + isDraft: true, + createdBy: userSession.userId, + }, + orderBy: { createdAt: "desc" }, + }); + + return await (existingDraft + ? this.updateExistingDraft(existingDraft.id, saveDraftDto) + : this.createDraftVersion(assignmentId, saveDraftDto, userSession)); + } + + async restoreVersion( + assignmentId: number, + restoreVersionDto: RestoreVersionDto, + userSession: UserSession, + ): Promise { + this.logger.info( + `Restoring version ${restoreVersionDto.versionId} for assignment ${assignmentId}`, + { + userId: userSession.userId, + }, + ); + + const versionToRestore = await this.prisma.assignmentVersion.findUnique({ + where: { id: restoreVersionDto.versionId, assignmentId }, + include: { questionVersions: { orderBy: { displayOrder: "asc" } } }, + }); + + if (!versionToRestore) { + throw new NotFoundException("Version to restore not found"); + } + + return await this.prisma.$transaction(async (tx) => { + if (restoreVersionDto.createAsNewVersion) { + // Create new version from restored data + const nextVersionNumber = await this.getNextVersionNumber( + assignmentId, + tx, + ); + + const restoredVersion = await tx.assignmentVersion.create({ + data: { + assignmentId, + versionNumber: nextVersionNumber, + name: versionToRestore.name, + introduction: versionToRestore.introduction, + instructions: versionToRestore.instructions, + gradingCriteriaOverview: versionToRestore.gradingCriteriaOverview, + timeEstimateMinutes: versionToRestore.timeEstimateMinutes, + type: versionToRestore.type, + graded: versionToRestore.graded, + numAttempts: versionToRestore.numAttempts, + allotedTimeMinutes: versionToRestore.allotedTimeMinutes, + attemptsPerTimeRange: versionToRestore.attemptsPerTimeRange, + attemptsTimeRangeHours: versionToRestore.attemptsTimeRangeHours, + passingGrade: versionToRestore.passingGrade, + displayOrder: versionToRestore.displayOrder, + questionDisplay: versionToRestore.questionDisplay, + numberOfQuestionsPerAttempt: + versionToRestore.numberOfQuestionsPerAttempt, + questionOrder: versionToRestore.questionOrder, + published: false, // New restored versions start unpublished + showAssignmentScore: versionToRestore.showAssignmentScore, + showQuestionScore: versionToRestore.showQuestionScore, + showSubmissionFeedback: versionToRestore.showSubmissionFeedback, + showQuestions: versionToRestore.showQuestions, + languageCode: versionToRestore.languageCode, + createdBy: userSession.userId, + isDraft: true, // Restored versions start as drafts + versionDescription: + restoreVersionDto.versionDescription || + `Restored from version ${versionToRestore.versionNumber}`, + isActive: false, + }, + }); + + // Restore question versions + for (const questionVersion of versionToRestore.questionVersions) { + await tx.questionVersion.create({ + data: { + assignmentVersionId: restoredVersion.id, + questionId: questionVersion.questionId, + totalPoints: questionVersion.totalPoints, + type: questionVersion.type, + responseType: questionVersion.responseType, + question: questionVersion.question, + maxWords: questionVersion.maxWords, + scoring: questionVersion.scoring, + choices: questionVersion.choices, + randomizedChoices: questionVersion.randomizedChoices, + answer: questionVersion.answer, + gradingContextQuestionIds: + questionVersion.gradingContextQuestionIds, + maxCharacters: questionVersion.maxCharacters, + videoPresentationConfig: questionVersion.videoPresentationConfig, + liveRecordingConfig: questionVersion.liveRecordingConfig, + displayOrder: questionVersion.displayOrder, + }, + }); + } + + // Create version history + await tx.versionHistory.create({ + data: { + assignmentId, + fromVersionId: versionToRestore.id, + toVersionId: restoredVersion.id, + action: "version_restored", + description: `Restored from version ${versionToRestore.versionNumber}`, + userId: userSession.userId, + }, + }); + + return { + id: restoredVersion.id, + versionNumber: restoredVersion.versionNumber, + versionDescription: restoredVersion.versionDescription, + isDraft: restoredVersion.isDraft, + isActive: restoredVersion.isActive, + published: restoredVersion.published, + createdBy: restoredVersion.createdBy, + createdAt: restoredVersion.createdAt, + questionCount: versionToRestore.questionVersions.length, + }; + } else { + // Activate existing version + const isRcVersion = /-rc\d+$/.test(versionToRestore.versionNumber); + + if (isRcVersion) { + // For RC versions, publish as final version and activate + return await this.activateRcVersion( + assignmentId, + restoreVersionDto.versionId, + userSession, + tx, + ); + } else { + // For regular versions, check if published before activation + if (!versionToRestore.published) { + throw new BadRequestException( + `Version ${versionToRestore.versionNumber} cannot be activated because it has not been published yet. Please publish the version first before activating it.`, + ); + } + + await tx.assignmentVersion.updateMany({ + where: { assignmentId }, + data: { isActive: false }, + }); + + await tx.assignmentVersion.update({ + where: { id: restoreVersionDto.versionId }, + data: { isActive: true }, + }); + + await tx.assignment.update({ + where: { id: assignmentId }, + data: { currentVersionId: restoreVersionDto.versionId }, + }); + + // Create version history + await tx.versionHistory.create({ + data: { + assignmentId, + fromVersionId: null, + toVersionId: restoreVersionDto.versionId, + action: "version_activated", + description: `Activated version ${versionToRestore.versionNumber}`, + userId: userSession.userId, + }, + }); + + return { + id: versionToRestore.id, + versionNumber: versionToRestore.versionNumber, + versionDescription: versionToRestore.versionDescription, + isDraft: versionToRestore.isDraft, + isActive: true, + published: versionToRestore.published, + createdBy: versionToRestore.createdBy, + createdAt: versionToRestore.createdAt, + questionCount: versionToRestore.questionVersions.length, + }; + } + } + }); + } + + async publishVersion( + assignmentId: number, + versionId: number, + ): Promise { + const version = await this.prisma.assignmentVersion.findUnique({ + where: { id: versionId, assignmentId }, + include: { _count: { select: { questionVersions: true } } }, + }); + + if (!version) { + throw new NotFoundException( + `Version with ID ${versionId} not found for assignment ${assignmentId}`, + ); + } + + if (version.published) { + throw new BadRequestException( + `Version ${version.versionNumber} is already published`, + ); + } + + // Remove -rc suffix when publishing and handle conflicts + let publishedVersionNumber = version.versionNumber; + if (publishedVersionNumber.includes("-rc")) { + publishedVersionNumber = publishedVersionNumber.replace(/-rc\d+$/, ""); + + // Check if the published version number already exists + const existingPublishedVersion = + await this.prisma.assignmentVersion.findFirst({ + where: { + assignmentId, + versionNumber: publishedVersionNumber, + id: { not: versionId }, + }, + }); + + if (existingPublishedVersion) { + // Auto-increment patch version to resolve conflict + const versionMatch = publishedVersionNumber.match( + /^(\d+)\.(\d+)\.(\d+)$/, + ); + if (versionMatch) { + const [, major, minor, patch] = versionMatch; + let newPatch = Number.parseInt(patch) + 1; + let newVersionNumber = `${major}.${minor}.${newPatch}`; + + // Keep incrementing until we find an available version number + while ( + await this.prisma.assignmentVersion.findFirst({ + where: { + assignmentId, + versionNumber: newVersionNumber, + id: { not: versionId }, + }, + }) + ) { + newPatch++; + newVersionNumber = `${major}.${minor}.${newPatch}`; + } + + publishedVersionNumber = newVersionNumber; + this.logger.info( + `Resolved version conflict by incrementing patch: ${version.versionNumber} → ${publishedVersionNumber}`, + ); + } else { + throw new ConflictException( + `Published version ${publishedVersionNumber} already exists and version format is not recognizable.`, + ); + } + } + } + + const originalVersionNumber = version.versionNumber; + const wasAutoIncremented = + publishedVersionNumber !== originalVersionNumber.replace(/-rc\d+$/, ""); + + // Update the version to mark it as published and no longer a draft + const updatedVersion = await this.prisma.assignmentVersion.update({ + where: { id: versionId }, + data: { + published: true, + isDraft: false, + versionNumber: publishedVersionNumber, + // Update description to note auto-increment if it happened + versionDescription: wasAutoIncremented + ? `${ + version.versionDescription || "" + } (Auto-incremented from ${originalVersionNumber} due to version conflict)`.trim() + : version.versionDescription, + }, + include: { _count: { select: { questionVersions: true } } }, + }); + + this.logger.info( + `Successfully published version: ${originalVersionNumber} → ${publishedVersionNumber}${ + wasAutoIncremented ? " (auto-incremented)" : "" + }`, + ); + + return { + id: updatedVersion.id, + versionNumber: updatedVersion.versionNumber, + versionDescription: updatedVersion.versionDescription, + isDraft: updatedVersion.isDraft, + isActive: updatedVersion.isActive, + published: updatedVersion.published, + createdBy: updatedVersion.createdBy, + createdAt: updatedVersion.createdAt, + questionCount: updatedVersion._count.questionVersions, + // Include additional info about auto-increment for frontend + wasAutoIncremented, + originalVersionNumber: wasAutoIncremented + ? originalVersionNumber + : undefined, + }; + } + + async compareVersions( + assignmentId: number, + compareDto: CompareVersionsDto, + ): Promise { + const [fromVersion, toVersion] = await Promise.all([ + this.prisma.assignmentVersion.findUnique({ + where: { id: compareDto.fromVersionId, assignmentId }, + include: { questionVersions: { orderBy: { displayOrder: "asc" } } }, + }), + this.prisma.assignmentVersion.findUnique({ + where: { id: compareDto.toVersionId, assignmentId }, + include: { questionVersions: { orderBy: { displayOrder: "asc" } } }, + }), + ]); + + if (!fromVersion || !toVersion) { + throw new NotFoundException("One or both versions not found"); + } + + const assignmentChanges = this.compareAssignmentData( + fromVersion, + toVersion, + ); + const questionChanges = this.compareQuestionData( + fromVersion.questionVersions, + toVersion.questionVersions, + ); + + return { + fromVersion: { + id: fromVersion.id, + versionNumber: fromVersion.versionNumber, + versionDescription: fromVersion.versionDescription, + isDraft: fromVersion.isDraft, + isActive: fromVersion.isActive, + published: fromVersion.published, + createdBy: fromVersion.createdBy, + createdAt: fromVersion.createdAt, + questionCount: fromVersion.questionVersions.length, + }, + toVersion: { + id: toVersion.id, + versionNumber: toVersion.versionNumber, + versionDescription: toVersion.versionDescription, + isDraft: toVersion.isDraft, + isActive: toVersion.isActive, + published: toVersion.published, + createdBy: toVersion.createdBy, + createdAt: toVersion.createdAt, + questionCount: toVersion.questionVersions.length, + }, + assignmentChanges, + questionChanges, + }; + } + + async getVersionHistory(assignmentId: number, _userSession: UserSession) { + // Verify access + // await this.verifyAssignmentAccess(assignmentId, userSession); + + return await this.prisma.versionHistory.findMany({ + where: { assignmentId }, + include: { + fromVersion: { select: { versionNumber: true } }, + toVersion: { select: { versionNumber: true } }, + }, + orderBy: { createdAt: "desc" }, + }); + } + + // Private helper methods + private async updateExistingDraft( + draftId: number, + saveDraftDto: SaveDraftDto, + ): Promise { + return await this.prisma.$transaction(async (tx) => { + const updatedDraft = await tx.assignmentVersion.update({ + where: { id: draftId }, + data: { + versionDescription: saveDraftDto.versionDescription, + ...(saveDraftDto.assignmentData?.name && { + name: saveDraftDto.assignmentData.name, + }), + ...(saveDraftDto.assignmentData?.introduction !== undefined && { + introduction: saveDraftDto.assignmentData.introduction, + }), + ...(saveDraftDto.assignmentData?.instructions !== undefined && { + instructions: saveDraftDto.assignmentData.instructions, + }), + ...(saveDraftDto.assignmentData?.gradingCriteriaOverview !== + undefined && { + gradingCriteriaOverview: + saveDraftDto.assignmentData.gradingCriteriaOverview, + }), + ...(saveDraftDto.assignmentData?.timeEstimateMinutes && { + timeEstimateMinutes: + saveDraftDto.assignmentData.timeEstimateMinutes, + }), + }, + include: { _count: { select: { questionVersions: true } } }, + }); + + // Delete existing question versions for this draft + await tx.questionVersion.deleteMany({ + where: { assignmentVersionId: draftId }, + }); + + // Create new question versions if provided + if (saveDraftDto.questionsData && saveDraftDto.questionsData.length > 0) { + for (const [ + index, + questionData, + ] of saveDraftDto.questionsData.entries()) { + await tx.questionVersion.create({ + data: { + assignmentVersionId: draftId, + questionId: questionData.id || null, + totalPoints: questionData.totalPoints || 0, + type: questionData.type, + responseType: questionData.responseType, + question: questionData.question, + maxWords: questionData.maxWords, + scoring: questionData.scoring as any, + choices: questionData.choices as any, + randomizedChoices: questionData.randomizedChoices, + answer: questionData.answer, + gradingContextQuestionIds: + questionData.gradingContextQuestionIds || [], + maxCharacters: questionData.maxCharacters, + videoPresentationConfig: + questionData.videoPresentationConfig as any, + liveRecordingConfig: questionData.liveRecordingConfig, + displayOrder: index + 1, + }, + }); + } + } + + return { + id: updatedDraft.id, + versionNumber: updatedDraft.versionNumber, + versionDescription: updatedDraft.versionDescription, + isDraft: updatedDraft.isDraft, + isActive: updatedDraft.isActive, + published: updatedDraft.published, + createdBy: updatedDraft.createdBy, + createdAt: updatedDraft.createdAt, + questionCount: saveDraftDto.questionsData?.length || 0, + }; + }); + } + + private async updateExistingVersion( + assignmentId: number, + versionId: number, + updateData: CreateVersionDto, + userSession: UserSession, + ): Promise { + this.logger.info( + `🔄 UPDATE EXISTING VERSION: Starting update for version ${versionId} on assignment ${assignmentId}`, + { + versionId, + assignmentId, + updateData, + userId: userSession.userId, + }, + ); + + // Get current assignment data + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { + questions: { where: { isDeleted: false } }, + }, + }); + + if (!assignment) { + throw new NotFoundException("Assignment not found"); + } + + return await this.prisma.$transaction(async (tx) => { + // Update the assignment version + const updatedVersion = await tx.assignmentVersion.update({ + where: { id: versionId }, + data: { + versionDescription: updateData.versionDescription, + isDraft: updateData.isDraft ?? true, + isActive: updateData.shouldActivate ?? false, + published: !(updateData.isDraft ?? true), // Published only if not a draft + name: assignment.name, + introduction: assignment.introduction, + instructions: assignment.instructions, + gradingCriteriaOverview: assignment.gradingCriteriaOverview, + timeEstimateMinutes: assignment.timeEstimateMinutes, + type: assignment.type, + graded: assignment.graded, + numAttempts: assignment.numAttempts, + allotedTimeMinutes: assignment.allotedTimeMinutes, + attemptsPerTimeRange: assignment.attemptsPerTimeRange, + attemptsTimeRangeHours: assignment.attemptsTimeRangeHours, + passingGrade: assignment.passingGrade, + displayOrder: assignment.displayOrder, + questionDisplay: assignment.questionDisplay, + numberOfQuestionsPerAttempt: assignment.numberOfQuestionsPerAttempt, + questionOrder: assignment.questionOrder, + showAssignmentScore: assignment.showAssignmentScore, + showQuestionScore: assignment.showQuestionScore, + showSubmissionFeedback: assignment.showSubmissionFeedback, + showQuestions: assignment.showQuestions, + languageCode: assignment.languageCode, + }, + include: { _count: { select: { questionVersions: true } } }, + }); + + // Delete existing question versions + await tx.questionVersion.deleteMany({ + where: { assignmentVersionId: versionId }, + }); + + // Create new question versions with current data + for (const [index, question] of assignment.questions.entries()) { + await tx.questionVersion.create({ + data: { + assignmentVersionId: versionId, + questionId: question.id, + totalPoints: question.totalPoints, + type: question.type, + responseType: question.responseType, + question: question.question, + maxWords: question.maxWords, + scoring: question.scoring, + choices: question.choices, + randomizedChoices: question.randomizedChoices, + answer: question.answer, + gradingContextQuestionIds: question.gradingContextQuestionIds, + maxCharacters: question.maxCharacters, + videoPresentationConfig: question.videoPresentationConfig, + liveRecordingConfig: question.liveRecordingConfig, + displayOrder: index + 1, + }, + }); + } + + // Update current version if should activate + if (updateData.shouldActivate) { + // Deactivate other versions + await tx.assignmentVersion.updateMany({ + where: { assignmentId, id: { not: versionId } }, + data: { isActive: false }, + }); + + // Update assignment currentVersionId + await tx.assignment.update({ + where: { id: assignmentId }, + data: { currentVersionId: versionId }, + }); + } + + // Create version history entry + await tx.versionHistory.create({ + data: { + assignmentId, + toVersionId: versionId, + action: "version_updated", + description: `Version ${updatedVersion.versionNumber} updated: ${updateData.versionDescription}`, + userId: userSession.userId, + }, + }); + + this.logger.info( + `✅ UPDATE EXISTING VERSION: Successfully updated version ${updatedVersion.versionNumber} (ID: ${updatedVersion.id}) for assignment ${assignmentId}`, + ); + + return { + id: updatedVersion.id, + versionNumber: updatedVersion.versionNumber, + versionDescription: updatedVersion.versionDescription, + isDraft: updatedVersion.isDraft, + isActive: updatedVersion.isActive, + published: updatedVersion.published, + createdBy: updatedVersion.createdBy, + createdAt: updatedVersion.createdAt, + questionCount: assignment.questions.length, + }; + }); + } + + private async getNextVersionNumber( + assignmentId: number, + tx: any, + ): Promise { + const lastVersion = await tx.assignmentVersion.findFirst({ + where: { assignmentId }, + orderBy: { createdAt: "desc" }, + }); + + if (!lastVersion || !lastVersion.versionNumber) { + return "1.0.0"; + } + + // If it's already a semantic version, increment patch + if (/^\d+\.\d+\.\d+(-rc\d+)?$/.test(lastVersion.versionNumber)) { + const match = lastVersion.versionNumber.match(/^(\d+)\.(\d+)\.(\d+)/); + if (match) { + const major = Number.parseInt(match[1], 10); + const minor = Number.parseInt(match[2], 10); + const patch = Number.parseInt(match[3], 10) + 1; + return `${major}.${minor}.${patch}`; + } + } + + // Legacy numeric version or unrecognized format, start fresh + return "1.0.0"; + } + + private compareAssignmentData(from: any, to: any) { + const changes = []; + const fields = [ + "name", + "introduction", + "instructions", + "gradingCriteriaOverview", + "published", + ]; + + for (const field of fields) { + if (from[field] !== to[field]) { + changes.push({ + field, + fromValue: from[field], + toValue: to[field], + changeType: + from[field] === null + ? "added" + : to[field] === null + ? "removed" + : "modified", + }); + } + } + return changes; + } + + private compareQuestionData(fromQuestions: any[], toQuestions: any[]) { + const changes = []; + + // Create maps for easier comparison + const fromMap = new Map( + fromQuestions.map((q) => [q.questionId || q.id, q]), + ); + const toMap = new Map(toQuestions.map((q) => [q.questionId || q.id, q])); + + // Find added questions + for (const [questionId, question] of toMap) { + if (!fromMap.has(questionId)) { + changes.push({ + questionId, + displayOrder: question.displayOrder, + changeType: "added" as const, + }); + } + } + + // Find removed questions + for (const [questionId, question] of fromMap) { + if (!toMap.has(questionId)) { + changes.push({ + questionId, + displayOrder: question.displayOrder, + changeType: "removed" as const, + }); + } + } + + // Find modified questions + for (const [questionId, fromQuestion] of fromMap) { + const toQuestion = toMap.get(questionId); + if (toQuestion) { + const fieldsToCompare = [ + "question", + "totalPoints", + "type", + "responseType", + "scoring", + "choices", + ]; + + for (const field of fieldsToCompare) { + const fromValue = fromQuestion[field]; + const toValue = toQuestion[field]; + + // Deep comparison for objects + const fromString = + typeof fromValue === "object" + ? JSON.stringify(fromValue) + : fromValue; + const toString_ = + typeof toValue === "object" ? JSON.stringify(toValue) : toValue; + + if (fromString !== toString_) { + changes.push({ + questionId, + displayOrder: toQuestion.displayOrder, + changeType: "modified" as const, + field, + fromValue, + toValue, + }); + } + } + } + } + + return changes; + } + + private async verifyAssignmentAccess( + assignmentId: number, + userSession: UserSession, + ) { + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { AssignmentAuthor: true }, + }); + + if (!assignment) { + throw new NotFoundException("Assignment not found"); + } + + if (userSession.role === UserRole.AUTHOR) { + const hasAccess = assignment.AssignmentAuthor.some( + (author) => author.userId === userSession.userId, + ); + if (!hasAccess) { + throw new NotFoundException("Assignment not found"); + } + } + } + + private async createDraftVersion( + assignmentId: number, + saveDraftDto: SaveDraftDto, + userSession: UserSession, + ): Promise { + // Get the base assignment for reference + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { AssignmentAuthor: true }, + }); + + if (!assignment) { + throw new NotFoundException("Assignment not found"); + } + + return await this.prisma.$transaction(async (tx) => { + // Generate semantic version number for draft + const lastVersion = await tx.assignmentVersion.findFirst({ + where: { assignmentId }, + orderBy: { createdAt: "desc" }, + }); + + let nextVersionNumber = "1.0.0-rc1"; // Default first draft version + if (lastVersion && lastVersion.versionNumber) { + // If latest is semantic version, increment RC number + if (/^\d+\.\d+\.\d+(-rc\d+)?$/.test(lastVersion.versionNumber)) { + const rcMatch = lastVersion.versionNumber.match(/-rc(\d+)$/); + if (rcMatch) { + const baseVersion = lastVersion.versionNumber.replace( + /-rc\d+$/, + "", + ); + const rcNumber = Number.parseInt(rcMatch[1], 10) + 1; + nextVersionNumber = `${baseVersion}-rc${rcNumber}`; + } else { + // No RC suffix, add RC1 to same version + nextVersionNumber = `${lastVersion.versionNumber}-rc1`; + } + } else { + // Legacy numeric version, start semantic versioning + nextVersionNumber = "1.0.0-rc1"; + } + } + + // Create assignment version with draft data + const assignmentVersion = await tx.assignmentVersion.create({ + data: { + assignmentId, + versionNumber: nextVersionNumber, + name: saveDraftDto.assignmentData?.name || assignment.name, + introduction: + saveDraftDto.assignmentData?.introduction ?? + assignment.introduction, + instructions: + saveDraftDto.assignmentData?.instructions ?? + assignment.instructions, + gradingCriteriaOverview: + saveDraftDto.assignmentData?.gradingCriteriaOverview ?? + assignment.gradingCriteriaOverview, + timeEstimateMinutes: + saveDraftDto.assignmentData?.timeEstimateMinutes || + assignment.timeEstimateMinutes, + type: assignment.type, + graded: assignment.graded, + numAttempts: assignment.numAttempts, + allotedTimeMinutes: assignment.allotedTimeMinutes, + attemptsPerTimeRange: assignment.attemptsPerTimeRange, + attemptsTimeRangeHours: assignment.attemptsTimeRangeHours, + passingGrade: assignment.passingGrade, + displayOrder: assignment.displayOrder, + questionDisplay: assignment.questionDisplay, + numberOfQuestionsPerAttempt: assignment.numberOfQuestionsPerAttempt, + questionOrder: assignment.questionOrder, + published: false, + showAssignmentScore: assignment.showAssignmentScore, + showQuestionScore: assignment.showQuestionScore, + showSubmissionFeedback: assignment.showSubmissionFeedback, + showQuestions: assignment.showQuestions, + languageCode: assignment.languageCode, + createdBy: userSession.userId, + isDraft: true, + versionDescription: + saveDraftDto.versionDescription || "Draft version", + isActive: false, + }, + }); + + // Create question versions if provided + if (saveDraftDto.questionsData && saveDraftDto.questionsData.length > 0) { + for (const [ + index, + questionData, + ] of saveDraftDto.questionsData.entries()) { + await tx.questionVersion.create({ + data: { + assignmentVersionId: assignmentVersion.id, + questionId: questionData.id || null, + totalPoints: questionData.totalPoints || 0, + type: questionData.type, + responseType: questionData.responseType, + question: questionData.question, + maxWords: questionData.maxWords, + scoring: questionData.scoring as any, + choices: questionData.choices as any, + randomizedChoices: questionData.randomizedChoices, + answer: questionData.answer, + gradingContextQuestionIds: + questionData.gradingContextQuestionIds || [], + maxCharacters: questionData.maxCharacters, + videoPresentationConfig: + questionData.videoPresentationConfig as any, + liveRecordingConfig: questionData.liveRecordingConfig, + displayOrder: index + 1, + }, + }); + } + } + + // Record in version history + await tx.versionHistory.create({ + data: { + assignmentId, + toVersionId: assignmentVersion.id, + action: "draft_created", + description: saveDraftDto.versionDescription, + userId: userSession.userId, + }, + }); + + return { + id: assignmentVersion.id, + versionNumber: assignmentVersion.versionNumber, + versionDescription: assignmentVersion.versionDescription, + isDraft: assignmentVersion.isDraft, + isActive: assignmentVersion.isActive, + published: assignmentVersion.published, + createdBy: assignmentVersion.createdBy, + createdAt: assignmentVersion.createdAt, + questionCount: saveDraftDto.questionsData?.length || 0, + }; + }); + } + + async getUserLatestDraft( + assignmentId: number, + userSession: UserSession, + ): Promise<{ + questions: any[]; + _isDraftVersion: boolean; + _draftVersionId: number | null; + id: number; + name: string; + introduction: string | null; + instructions: string | null; + gradingCriteriaOverview: string | null; + timeEstimateMinutes: number | null; + type: string; + graded: boolean; + numAttempts: number | null; + allotedTimeMinutes: number | null; + passingGrade: number | null; + displayOrder: string | null; + questionDisplay: string | null; + numberOfQuestionsPerAttempt: number | null; + questionOrder: number[] | null; + published: boolean; + showAssignmentScore: boolean; + showQuestionScore: boolean; + showSubmissionFeedback: boolean; + showQuestions: boolean; + languageCode: string | null; + }> { + // await this.verifyAssignmentAccess(assignmentId, userSession); + + const latestDraft = await this.prisma.assignmentVersion.findFirst({ + where: { + assignmentId, + isDraft: true, + createdBy: userSession.userId, + }, + include: { + questionVersions: { orderBy: { displayOrder: "asc" } }, + }, + orderBy: { createdAt: "desc" }, + }); + + if (!latestDraft) { + return null; + } + + // Convert to assignment format + return { + id: assignmentId, + name: latestDraft.name, + introduction: latestDraft.introduction, + instructions: latestDraft.instructions, + gradingCriteriaOverview: latestDraft.gradingCriteriaOverview, + timeEstimateMinutes: latestDraft.timeEstimateMinutes, + type: latestDraft.type, + graded: latestDraft.graded, + numAttempts: latestDraft.numAttempts, + allotedTimeMinutes: latestDraft.allotedTimeMinutes, + passingGrade: latestDraft.passingGrade, + displayOrder: latestDraft.displayOrder, + questionDisplay: latestDraft.questionDisplay, + numberOfQuestionsPerAttempt: latestDraft.numberOfQuestionsPerAttempt, + questionOrder: latestDraft.questionOrder, + published: latestDraft.published, + showAssignmentScore: latestDraft.showAssignmentScore, + showQuestionScore: latestDraft.showQuestionScore, + showSubmissionFeedback: latestDraft.showSubmissionFeedback, + showQuestions: latestDraft.showQuestions, + languageCode: latestDraft.languageCode, + questions: latestDraft.questionVersions.map((qv) => ({ + id: qv.questionId, + totalPoints: qv.totalPoints, + type: qv.type, + responseType: qv.responseType, + question: qv.question, + maxWords: qv.maxWords, + scoring: qv.scoring, + choices: qv.choices, + randomizedChoices: qv.randomizedChoices, + answer: qv.answer, + gradingContextQuestionIds: qv.gradingContextQuestionIds, + maxCharacters: qv.maxCharacters, + videoPresentationConfig: qv.videoPresentationConfig, + liveRecordingConfig: qv.liveRecordingConfig, + displayOrder: qv.displayOrder, + isDeleted: false, + })), + _isDraftVersion: true, + _draftVersionId: latestDraft.id, + }; + } + + async restoreDeletedQuestions( + assignmentId: number, + versionId: number, + questionIds: number[], + userSession: UserSession, + ): Promise { + this.logger.info( + `Restoring deleted questions ${questionIds.join( + ", ", + )} from version ${versionId} for assignment ${assignmentId}`, + { userId: userSession.userId }, + ); + + // await this.verifyAssignmentAccess(assignmentId, userSession); + + const sourceVersion = await this.prisma.assignmentVersion.findUnique({ + where: { id: versionId, assignmentId }, + include: { + questionVersions: { + where: { questionId: { in: questionIds } }, + }, + }, + }); + + if (!sourceVersion) { + throw new NotFoundException("Source version not found"); + } + + if (sourceVersion.questionVersions.length === 0) { + throw new NotFoundException("No questions found in source version"); + } + + return await this.prisma.$transaction(async (tx) => { + // Get or create a draft version to restore questions to + let targetDraft = await tx.assignmentVersion.findFirst({ + where: { + assignmentId, + isDraft: true, + createdBy: userSession.userId, + }, + orderBy: { createdAt: "desc" }, + }); + + if (!targetDraft) { + // Create a new draft version + const nextVersionNumber = await this.getNextVersionNumber( + assignmentId, + tx, + ); + targetDraft = await tx.assignmentVersion.create({ + data: { + assignmentId, + versionNumber: nextVersionNumber, + name: sourceVersion.name, + introduction: sourceVersion.introduction, + instructions: sourceVersion.instructions, + gradingCriteriaOverview: sourceVersion.gradingCriteriaOverview, + timeEstimateMinutes: sourceVersion.timeEstimateMinutes, + type: sourceVersion.type, + graded: sourceVersion.graded, + numAttempts: sourceVersion.numAttempts, + allotedTimeMinutes: sourceVersion.allotedTimeMinutes, + attemptsPerTimeRange: sourceVersion.attemptsPerTimeRange, + attemptsTimeRangeHours: sourceVersion.attemptsTimeRangeHours, + passingGrade: sourceVersion.passingGrade, + displayOrder: sourceVersion.displayOrder, + questionDisplay: sourceVersion.questionDisplay, + numberOfQuestionsPerAttempt: + sourceVersion.numberOfQuestionsPerAttempt, + questionOrder: sourceVersion.questionOrder, + published: false, + showAssignmentScore: sourceVersion.showAssignmentScore, + showQuestionScore: sourceVersion.showQuestionScore, + showSubmissionFeedback: sourceVersion.showSubmissionFeedback, + showQuestions: sourceVersion.showQuestions, + languageCode: sourceVersion.languageCode, + createdBy: userSession.userId, + isDraft: true, + versionDescription: `Draft with restored questions from version ${sourceVersion.versionNumber}`, + isActive: false, + }, + }); + } + + // Restore the questions to the draft version + for (const questionVersion of sourceVersion.questionVersions) { + await tx.questionVersion.create({ + data: { + assignmentVersionId: targetDraft.id, + questionId: questionVersion.questionId, + totalPoints: questionVersion.totalPoints, + type: questionVersion.type, + responseType: questionVersion.responseType, + question: questionVersion.question, + maxWords: questionVersion.maxWords, + scoring: questionVersion.scoring, + choices: questionVersion.choices, + randomizedChoices: questionVersion.randomizedChoices, + answer: questionVersion.answer, + gradingContextQuestionIds: + questionVersion.gradingContextQuestionIds, + maxCharacters: questionVersion.maxCharacters, + videoPresentationConfig: questionVersion.videoPresentationConfig, + liveRecordingConfig: questionVersion.liveRecordingConfig, + displayOrder: questionVersion.displayOrder, + }, + }); + + // Unmark the question as deleted in the main table + await tx.question.update({ + where: { id: questionVersion.questionId }, + data: { isDeleted: false }, + }); + } + + // Record in version history + await tx.versionHistory.create({ + data: { + assignmentId, + fromVersionId: versionId, + toVersionId: targetDraft.id, + action: "questions_restored", + description: `Restored ${questionIds.length} deleted questions from version ${sourceVersion.versionNumber}`, + userId: userSession.userId, + }, + }); + + this.logger.info( + `Successfully restored ${questionIds.length} questions from version ${versionId} to draft ${targetDraft.id}`, + ); + + return { + id: targetDraft.id, + versionNumber: targetDraft.versionNumber, + versionDescription: targetDraft.versionDescription, + isDraft: targetDraft.isDraft, + isActive: targetDraft.isActive, + published: targetDraft.published, + createdBy: targetDraft.createdBy, + createdAt: targetDraft.createdAt, + questionCount: sourceVersion.questionVersions.length, + }; + }); + } + + async updateVersionDescription( + assignmentId: number, + versionId: number, + versionDescription: string, + userSession: UserSession, + ): Promise { + this.logger.info(`Updating version description for version ${versionId}`, { + assignmentId, + versionId, + userId: userSession.userId, + }); + + // Verify the version exists and belongs to the assignment + const version = await this.prisma.assignmentVersion.findFirst({ + where: { + id: versionId, + assignmentId: assignmentId, + }, + }); + + if (!version) { + throw new NotFoundException("Version not found"); + } + + // Update the version description + const updatedVersion = await this.prisma.assignmentVersion.update({ + where: { id: versionId }, + data: { versionDescription }, + include: { + questionVersions: true, + }, + }); + + this.logger.info( + `Successfully updated version description for version ${versionId}`, + ); + + return { + id: updatedVersion.id, + versionNumber: updatedVersion.versionNumber, + versionDescription: updatedVersion.versionDescription, + isDraft: updatedVersion.isDraft, + isActive: updatedVersion.isActive, + published: updatedVersion.published, + createdBy: updatedVersion.createdBy, + createdAt: updatedVersion.createdAt, + questionCount: updatedVersion.questionVersions.length, + }; + } + + async updateVersionNumber( + assignmentId: number, + versionId: number, + versionNumber: string, + userSession: UserSession, + ): Promise { + this.logger.info(`Updating version number for version ${versionId}`, { + assignmentId, + versionId, + newVersionNumber: versionNumber, + userId: userSession.userId, + }); + + // Verify the version exists and belongs to the assignment + const version = await this.prisma.assignmentVersion.findFirst({ + where: { + id: versionId, + assignmentId: assignmentId, + }, + }); + + if (!version) { + throw new NotFoundException("Version not found"); + } + + // Check if the new version number already exists for this assignment + const existingVersion = await this.prisma.assignmentVersion.findFirst({ + where: { + assignmentId: assignmentId, + versionNumber: versionNumber, + id: { not: versionId }, // Exclude the current version being updated + }, + }); + + if (existingVersion) { + throw new BadRequestException( + `Version number "${versionNumber}" already exists for this assignment`, + ); + } + + // Update the version number + const updatedVersion = await this.prisma.assignmentVersion.update({ + where: { id: versionId }, + data: { versionNumber }, + include: { + questionVersions: true, + }, + }); + + this.logger.info( + `Successfully updated version number for version ${versionId} to ${versionNumber}`, + ); + + return { + id: updatedVersion.id, + versionNumber: updatedVersion.versionNumber, + versionDescription: updatedVersion.versionDescription, + isDraft: updatedVersion.isDraft, + isActive: updatedVersion.isActive, + published: updatedVersion.published, + createdBy: updatedVersion.createdBy, + createdAt: updatedVersion.createdAt, + questionCount: updatedVersion.questionVersions.length, + }; + } + + async deleteVersion( + assignmentId: number, + versionId: number, + userSession: UserSession, + ): Promise { + this.logger.info( + `Deleting version ${versionId} for assignment ${assignmentId}`, + { + assignmentId, + versionId, + userId: userSession.userId, + }, + ); + + // First, verify the version exists and belongs to the assignment + const version = await this.prisma.assignmentVersion.findFirst({ + where: { + id: versionId, + assignmentId: assignmentId, + }, + }); + + if (!version) { + throw new NotFoundException("Version not found"); + } + + // Prevent deletion of active versions + if (version.isActive) { + throw new BadRequestException("Cannot delete the active version"); + } + + // Use a transaction to ensure data integrity + await this.prisma.$transaction(async (prisma) => { + // Delete associated question versions first (due to foreign key constraints) + await prisma.questionVersion.deleteMany({ + where: { assignmentVersionId: versionId }, + }); + + // Delete the assignment version + await prisma.assignmentVersion.delete({ + where: { id: versionId }, + }); + + this.logger.info(`Successfully deleted version ${versionId}`, { + assignmentId, + versionId, + userId: userSession.userId, + }); + }); + } + + /** + * Save assignment snapshot as a draft version + */ + /** + * Activate an RC version by publishing it as a final version and then activating it + */ + private async activateRcVersion( + assignmentId: number, + rcVersionId: number, + userSession: UserSession, + tx?: Prisma.TransactionClient, + ): Promise { + const prisma = tx || this.prisma; + + this.logger.info( + `🚀 ACTIVATE RC VERSION: Starting for RC version ${rcVersionId}`, + { + assignmentId, + rcVersionId, + userId: userSession.userId, + }, + ); + + // Get the RC version + const rcVersion = await prisma.assignmentVersion.findUnique({ + where: { id: rcVersionId, assignmentId }, + include: { questionVersions: { orderBy: { displayOrder: "asc" } } }, + }); + + if (!rcVersion) { + throw new NotFoundException("RC version not found"); + } + + if (!/-rc\d+$/.test(rcVersion.versionNumber)) { + throw new BadRequestException("Version is not an RC version"); + } + + // Calculate final version number by removing RC suffix + let finalVersionNumber = rcVersion.versionNumber.replace(/-rc\d+$/, ""); + + // Check if final version already exists and auto-increment if needed + const existingFinalVersion = await prisma.assignmentVersion.findFirst({ + where: { + assignmentId, + versionNumber: finalVersionNumber, + id: { not: rcVersionId }, + }, + }); + + if (existingFinalVersion) { + // Auto-increment patch version to resolve conflict + const versionMatch = finalVersionNumber.match(/^(\d+)\.(\d+)\.(\d+)$/); + if (versionMatch) { + const [, major, minor, patch] = versionMatch; + let newPatch = Number.parseInt(patch) + 1; + let newVersionNumber = `${major}.${minor}.${newPatch}`; + + // Keep incrementing until we find an available version number + while ( + await prisma.assignmentVersion.findFirst({ + where: { + assignmentId, + versionNumber: newVersionNumber, + id: { not: rcVersionId }, + }, + }) + ) { + newPatch++; + newVersionNumber = `${major}.${minor}.${newPatch}`; + } + + finalVersionNumber = newVersionNumber; + this.logger.info( + `Resolved version conflict by incrementing patch: ${rcVersion.versionNumber} → ${finalVersionNumber}`, + ); + } + } + + const operation = tx ? "within transaction" : "standalone"; + this.logger.info(`🔄 Publishing RC as final version (${operation})`, { + originalVersion: rcVersion.versionNumber, + finalVersion: finalVersionNumber, + }); + + // Update the RC version to become the final published version + const publishedVersion = await prisma.assignmentVersion.update({ + where: { id: rcVersionId }, + data: { + versionNumber: finalVersionNumber, + published: true, + isDraft: false, + versionDescription: rcVersion.versionDescription + ? `${rcVersion.versionDescription} (Published from RC ${rcVersion.versionNumber})` + : `Published from RC ${rcVersion.versionNumber}`, + }, + include: { questionVersions: true }, + }); + + // Deactivate all other versions + await prisma.assignmentVersion.updateMany({ + where: { assignmentId, id: { not: rcVersionId } }, + data: { isActive: false }, + }); + + // Activate the newly published version + await prisma.assignmentVersion.update({ + where: { id: rcVersionId }, + data: { isActive: true }, + }); + + // Update assignment current version + await prisma.assignment.update({ + where: { id: assignmentId }, + data: { currentVersionId: rcVersionId }, + }); + + // Create version history + await prisma.versionHistory.create({ + data: { + assignmentId, + fromVersionId: null, + toVersionId: rcVersionId, + action: "rc_version_activated", + description: `RC ${rcVersion.versionNumber} published as ${finalVersionNumber} and activated`, + userId: userSession.userId, + }, + }); + + this.logger.info(`✅ RC version activated successfully`, { + originalVersion: rcVersion.versionNumber, + finalVersion: finalVersionNumber, + versionId: rcVersionId, + }); + + return { + id: publishedVersion.id, + versionNumber: publishedVersion.versionNumber, + versionDescription: publishedVersion.versionDescription, + isDraft: publishedVersion.isDraft, + isActive: true, + published: publishedVersion.published, + createdBy: publishedVersion.createdBy, + createdAt: publishedVersion.createdAt, + questionCount: publishedVersion.questionVersions.length, + }; + } + + async saveDraftSnapshot( + assignmentId: number, + draftData: { + versionNumber: string; + versionDescription?: string; + assignmentData: Assignment; + questionsData?: Question[]; + }, + userSession: UserSession, + ): Promise { + // Verify assignment access + const assignment = await this.prisma.assignment.findUnique({ + where: { id: assignmentId }, + include: { AssignmentAuthor: true, questions: true }, + }); + + if (!assignment) { + throw new NotFoundException(`Assignment ${assignmentId} not found`); + } + + // Check for version conflicts + const existingVersion = await this.prisma.assignmentVersion.findFirst({ + where: { + assignmentId, + versionNumber: draftData.versionNumber, + }, + }); + + if (existingVersion) { + throw new ConflictException( + `Version ${draftData.versionNumber} already exists for this assignment`, + ); + } + + try { + return await this.prisma.$transaction(async (tx) => { + // Create assignment version with snapshot data + const assignmentVersion = await tx.assignmentVersion.create({ + data: { + assignmentId, + versionNumber: draftData.versionNumber, + versionDescription: + draftData.versionDescription || "Draft snapshot", + isDraft: true, + isActive: false, + published: false, + createdBy: userSession.userId, + name: draftData.assignmentData.name || assignment.name, + introduction: + draftData.assignmentData.introduction ?? assignment.introduction, + instructions: + draftData.assignmentData.instructions ?? assignment.instructions, + gradingCriteriaOverview: + draftData.assignmentData.gradingCriteriaOverview ?? + assignment.gradingCriteriaOverview, + timeEstimateMinutes: + draftData.assignmentData.timeEstimateMinutes || + assignment.timeEstimateMinutes, + type: draftData.assignmentData.type || assignment.type, + graded: draftData.assignmentData.graded ?? assignment.graded, + numAttempts: + draftData.assignmentData.numAttempts ?? assignment.numAttempts, + allotedTimeMinutes: + draftData.assignmentData.allotedTimeMinutes ?? + assignment.allotedTimeMinutes, + attemptsPerTimeRange: + draftData.assignmentData.attemptsPerTimeRange ?? + assignment.attemptsPerTimeRange, + attemptsTimeRangeHours: + draftData.assignmentData.attemptsTimeRangeHours ?? + assignment.attemptsTimeRangeHours, + passingGrade: + draftData.assignmentData.passingGrade ?? assignment.passingGrade, + displayOrder: + draftData.assignmentData.displayOrder ?? assignment.displayOrder, + questionDisplay: + draftData.assignmentData.questionDisplay ?? + assignment.questionDisplay, + numberOfQuestionsPerAttempt: + draftData.assignmentData.numberOfQuestionsPerAttempt ?? + assignment.numberOfQuestionsPerAttempt, + questionOrder: + draftData.assignmentData.questionOrder ?? + assignment.questionOrder ?? + [], + showAssignmentScore: + draftData.assignmentData.showAssignmentScore ?? + assignment.showAssignmentScore ?? + true, + showQuestionScore: + draftData.assignmentData.showQuestionScore ?? + assignment.showQuestionScore ?? + true, + showSubmissionFeedback: + draftData.assignmentData.showSubmissionFeedback ?? + assignment.showSubmissionFeedback ?? + true, + showQuestions: + draftData.assignmentData.showQuestions ?? + assignment.showQuestions ?? + true, + languageCode: + draftData.assignmentData.languageCode ?? assignment.languageCode, + }, + }); + + // Create question versions from snapshot + const questionsData = draftData.questionsData || []; + + for (const [index, questionData] of questionsData.entries()) { + await tx.questionVersion.create({ + data: { + assignmentVersionId: assignmentVersion.id, + questionId: questionData.id || undefined, + totalPoints: questionData.totalPoints || 0, + type: questionData.type, + responseType: questionData.responseType, + question: questionData.question, + maxWords: questionData.maxWords || undefined, + scoring: questionData.scoring || undefined, + choices: questionData.choices || undefined, + randomizedChoices: questionData.randomizedChoices || undefined, + answer: questionData.answer || undefined, + gradingContextQuestionIds: + questionData.gradingContextQuestionIds || [], + maxCharacters: questionData.maxCharacters || undefined, + videoPresentationConfig: + questionData.videoPresentationConfig || undefined, + liveRecordingConfig: + questionData.liveRecordingConfig || undefined, + displayOrder: index + 1, + }, + }); + } + + return { + id: assignmentVersion.id, + versionNumber: assignmentVersion.versionNumber, + versionDescription: assignmentVersion.versionDescription, + isDraft: assignmentVersion.isDraft, + isActive: assignmentVersion.isActive, + published: assignmentVersion.published, + createdBy: assignmentVersion.createdBy, + createdAt: assignmentVersion.createdAt, + questionCount: questionsData.length, + }; + }); + } catch (error) { + this.logger.error("Failed to save draft snapshot:", error); + throw error; + } + } +} diff --git a/apps/api/src/api/assignment/v2/tests/unit/__mocks__/ common-mocks.ts b/apps/api/src/api/assignment/v2/tests/unit/__mocks__/ common-mocks.ts index 1dd9267b..7698adb1 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/__mocks__/ common-mocks.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/__mocks__/ common-mocks.ts @@ -545,7 +545,7 @@ export const sampleReactVariants = { }, ], createdAt: "2025-04-30T22:47:52.581Z", - variantType: "REWORDED", + variantType: VariantType.REWORDED, randomizedChoices: true, }, ], @@ -591,7 +591,7 @@ export const sampleReactVariants = { }, ], createdAt: "2025-04-30T22:47:53.159Z", - variantType: "REWORDED", + variantType: VariantType.REWORDED, randomizedChoices: true, }, ], @@ -764,7 +764,7 @@ export const createMockQuestionVariant = ( */ export const createMockVariantDto = ( overrides: Partial = {}, - questionId = 1, + _questionId = 1, variantType: VariantType = VariantType.REWORDED, ): VariantDto => { const baseVariantDto: VariantDto = { @@ -961,12 +961,13 @@ export const createMockAssignment = ( questionOrder: [1, 2], published: false, showAssignmentScore: true, + showCorrectAnswer: true, showQuestionScore: true, showSubmissionFeedback: true, showQuestions: true, updatedAt: new Date(), languageCode: "en", - + currentVersionId: 1, ...overrides, }); @@ -1061,6 +1062,7 @@ export const createMockUpdateAssignmentDto = ( showQuestionScore: true, showSubmissionFeedback: true, showQuestions: true, + showCorrectAnswer: true, ...overrides, }); @@ -1114,7 +1116,9 @@ export const createMockUpdateAssignmentQuestionsDto = ( showQuestionScore: true, showSubmissionFeedback: true, showQuestions: true, - + showCorrectAnswer: true, + versionNumber: "0.0.1", + versionDescription: "Updated questions version", updatedAt: new Date(), questions: includeQuestions ? [ @@ -1148,6 +1152,7 @@ export const createMockAssignmentAttempt = ( questionOrder: [1, 2], comments: null, preferredLanguage: "en", + assignmentVersionId: 1, }; return { @@ -1574,6 +1579,7 @@ export const createMockPrismaService = () => ({ findMany: jest.fn().mockResolvedValue([createMockAssignmentTranslation()]), create: jest.fn().mockResolvedValue(createMockAssignmentTranslation()), update: jest.fn().mockResolvedValue(createMockAssignmentTranslation()), + count: jest.fn().mockResolvedValue(1), }, assignmentFeedback: { findUnique: jest.fn().mockResolvedValue(createMockAssignmentFeedback()), @@ -1796,6 +1802,10 @@ export const createMockTranslationService = () => ({ translateAssignment: jest.fn().mockResolvedValue(undefined), translateQuestion: jest.fn().mockResolvedValue(undefined), translateVariant: jest.fn().mockResolvedValue(undefined), + ensureTranslationCompleteness: jest.fn().mockResolvedValue({ + missingTranslations: [], + allComplete: true, + }), applyTranslationsToAttempt: jest .fn() .mockImplementation((attempt: unknown): unknown => attempt), @@ -1855,7 +1865,7 @@ export const createMockLlmFacadeService = () => ({ }), generateQuestionTranslation: jest .fn() - .mockImplementation((assignmentId: number, text: string) => + .mockImplementation((_assignmentId: number, text: string) => Promise.resolve(`Translated: ${text}`), ), generateChoicesTranslation: jest.fn().mockResolvedValue([ @@ -1942,3 +1952,69 @@ export const createMockAttemptService = () => ({ getRegradingStatus: jest.fn().mockResolvedValue(createMockRegradingRequest()), createReport: jest.fn().mockResolvedValue(undefined), }); + +/** + * Create a mock VersionManagementService with pre-defined implementations + */ +export const createMockVersionManagementService = () => ({ + createVersion: jest.fn().mockResolvedValue({ + id: 1, + versionNumber: 1, + versionDescription: "Test version", + isDraft: false, + isActive: true, + createdBy: "test-user", + createdAt: new Date(), + questionCount: 2, + }), + listVersions: jest.fn().mockResolvedValue([]), + getVersion: jest.fn().mockResolvedValue(null), + saveDraft: jest.fn().mockResolvedValue({ + id: 1, + versionNumber: 1, + versionDescription: "Draft version", + isDraft: true, + isActive: false, + createdBy: "test-user", + createdAt: new Date(), + questionCount: 2, + }), + restoreVersion: jest.fn().mockResolvedValue({ + id: 2, + versionNumber: 2, + versionDescription: "Restored version", + isDraft: false, + isActive: true, + createdBy: "test-user", + createdAt: new Date(), + questionCount: 2, + }), + compareVersions: jest.fn().mockResolvedValue({ + fromVersion: { + id: 1, + versionNumber: 1, + versionDescription: "From version", + isDraft: false, + isActive: false, + createdBy: "test-user", + createdAt: new Date(), + questionCount: 2, + }, + toVersion: { + id: 2, + versionNumber: 2, + versionDescription: "To version", + isDraft: false, + isActive: true, + createdBy: "test-user", + createdAt: new Date(), + questionCount: 2, + }, + assignmentChanges: [], + questionChanges: [], + }), + getVersionHistory: jest.fn().mockResolvedValue([]), + getUserLatestDraft: jest.fn().mockResolvedValue(null), + restoreDeletedQuestions: jest.fn().mockResolvedValue(undefined), + updateVersionDescription: jest.fn().mockResolvedValue(undefined), +}); diff --git a/apps/api/src/api/assignment/v2/tests/unit/repositories/assignment.repository.spec.ts b/apps/api/src/api/assignment/v2/tests/unit/repositories/assignment.repository.spec.ts index b9960b14..d9da036b 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/repositories/assignment.repository.spec.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/repositories/assignment.repository.spec.ts @@ -1,19 +1,18 @@ /* eslint-disable @typescript-eslint/no-unsafe-argument */ /* eslint-disable unicorn/no-null */ /* eslint-disable @typescript-eslint/unbound-method */ -import { Test, TestingModule } from "@nestjs/testing"; import { NotFoundException } from "@nestjs/common"; -import { PrismaService } from "src/prisma.service"; -import { Assignment, QuestionType } from "@prisma/client"; +import { Test, TestingModule } from "@nestjs/testing"; +import { QuestionType } from "@prisma/client"; import { GetAssignmentResponseDto } from "src/api/assignment/dto/get.assignment.response.dto"; import { ScoringDto } from "src/api/assignment/dto/update.questions.request.dto"; -import { AssignmentRepository } from "../../../repositories/assignment.repository"; +import { PrismaService } from "src/prisma.service"; import { createMockAssignment, sampleAuthorSession, sampleLearnerSession, } from "../__mocks__/ common-mocks"; -import { BaseAssignmentResponseDto } from "src/api/admin/dto/assignment/base.assignment.response.dto"; +import { AssignmentRepository } from "../../../repositories/assignment.repository"; describe("AssignmentRepository", () => { let repository: AssignmentRepository; diff --git a/apps/api/src/api/assignment/v2/tests/unit/repositories/question.repository.spec.ts b/apps/api/src/api/assignment/v2/tests/unit/repositories/question.repository.spec.ts index 577fac65..d5d3592b 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/repositories/question.repository.spec.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/repositories/question.repository.spec.ts @@ -11,12 +11,7 @@ /* eslint-disable @typescript-eslint/unbound-method */ import { Logger } from "@nestjs/common"; import { Test, TestingModule } from "@nestjs/testing"; -import { - Question, - QuestionType, - QuestionVariant, - ResponseType, -} from "@prisma/client"; +import { Question, QuestionType, QuestionVariant } from "@prisma/client"; import { QuestionDto, ScoringDto, diff --git a/apps/api/src/api/assignment/v2/tests/unit/repositories/report.repository.spec.ts b/apps/api/src/api/assignment/v2/tests/unit/repositories/report.repository.spec.ts index af595df2..4470ef9a 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/repositories/report.repository.spec.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/repositories/report.repository.spec.ts @@ -8,20 +8,20 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/unbound-method */ -import { Test, TestingModule } from "@nestjs/testing"; import { BadRequestException, Logger, NotFoundException, UnprocessableEntityException, } from "@nestjs/common"; -import { PrismaService } from "src/prisma.service"; +import { Test, TestingModule } from "@nestjs/testing"; import { ReportType } from "@prisma/client"; -import { ReportService } from "../../../services/report.repository"; +import { PrismaService } from "src/prisma.service"; import { createMockAssignment, createMockReport, } from "../__mocks__/ common-mocks"; +import { ReportService } from "../../../services/report.repository"; describe("ReportService", () => { let service: ReportService; diff --git a/apps/api/src/api/assignment/v2/tests/unit/repositories/variant.repository.spec.ts b/apps/api/src/api/assignment/v2/tests/unit/repositories/variant.repository.spec.ts index 090b8160..f9304dd7 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/repositories/variant.repository.spec.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/repositories/variant.repository.spec.ts @@ -5,9 +5,9 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable unicorn/no-null */ import { Test, TestingModule } from "@nestjs/testing"; -import { PrismaService } from "src/prisma.service"; import { Prisma, VariantType } from "@prisma/client"; -import { VariantRepository } from "../../../repositories/variant.repository"; +import { VariantDto } from "src/api/assignment/dto/update.questions.request.dto"; +import { PrismaService } from "src/prisma.service"; import { createMockPrismaService, createMockQuestionVariant, @@ -16,7 +16,7 @@ import { sampleChoiceB, sampleChoiceC, } from "../__mocks__/ common-mocks"; -import { VariantDto } from "src/api/assignment/dto/update.questions.request.dto"; +import { VariantRepository } from "../../../repositories/variant.repository"; describe("VariantRepository", () => { let variantRepository: VariantRepository; diff --git a/apps/api/src/api/assignment/v2/tests/unit/services/assignment.service.spec.ts b/apps/api/src/api/assignment/v2/tests/unit/services/assignment.service.spec.ts index acc50028..5d0a367c 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/services/assignment.service.spec.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/services/assignment.service.spec.ts @@ -4,10 +4,13 @@ /* eslint-disable @typescript-eslint/unbound-method */ import { Test, TestingModule } from "@nestjs/testing"; -import { QuestionType, VariantType } from "@prisma/client"; +import { QuestionType } from "@prisma/client"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { UpdateAssignmentRequestDto } from "src/api/assignment/dto/update.assignment.request.dto"; -import { UpdateAssignmentQuestionsDto } from "src/api/assignment/dto/update.questions.request.dto"; +import { + UpdateAssignmentQuestionsDto, + VariantType as VariantTypeDto, +} from "src/api/assignment/dto/update.questions.request.dto"; import { QuestionService } from "src/api/assignment/v2/services/question.service"; import { LlmFacadeService } from "src/api/llm/llm-facade.service"; import { PrismaService } from "src/prisma.service"; @@ -26,6 +29,7 @@ import { createMockTranslationService, createMockUpdateAssignmentDto, createMockUpdateAssignmentQuestionsDto, + createMockVersionManagementService, sampleAuthorSession, sampleLearnerSession, } from "../__mocks__/ common-mocks"; @@ -33,12 +37,16 @@ import { AssignmentRepository } from "../../../repositories/assignment.repositor import { AssignmentServiceV2 } from "../../../services/assignment.service"; import { JobStatusServiceV2 } from "../../../services/job-status.service"; import { TranslationService } from "../../../services/translation.service"; +import { VersionManagementService } from "../../../services/version-management.service"; describe("AssignmentServiceV2 – full unit-suite", () => { let service: AssignmentServiceV2; let assignmentRepository: ReturnType; let questionService: ReturnType; let translationService: ReturnType; + let versionManagementService: ReturnType< + typeof createMockVersionManagementService + >; let jobStatusService: ReturnType; let logger: ReturnType; @@ -46,6 +54,7 @@ describe("AssignmentServiceV2 – full unit-suite", () => { assignmentRepository = createMockAssignmentRepository(); questionService = createMockQuestionService(); translationService = createMockTranslationService(); + versionManagementService = createMockVersionManagementService(); jobStatusService = createMockJobStatusService(); const llmService = createMockLlmFacadeService(); logger = createMockLogger(); @@ -56,6 +65,10 @@ describe("AssignmentServiceV2 – full unit-suite", () => { { provide: AssignmentRepository, useValue: assignmentRepository }, { provide: QuestionService, useValue: questionService }, { provide: TranslationService, useValue: translationService }, + { + provide: VersionManagementService, + useValue: versionManagementService, + }, { provide: JobStatusServiceV2, useValue: jobStatusService }, { provide: LlmFacadeService, useValue: llmService }, { provide: PrismaService, useValue: createMockPrismaService() }, @@ -212,7 +225,7 @@ describe("AssignmentServiceV2 – full unit-suite", () => { 1, "author-123", ); - expect(spy).toHaveBeenCalledWith(1, 1, dto); + expect(spy).toHaveBeenCalledWith(1, 1, dto, "author-123"); expect(response).toEqual({ jobId: 1, message: "Publishing started" }); }); @@ -245,7 +258,12 @@ describe("AssignmentServiceV2 – full unit-suite", () => { .spyOn(service as any, "haveQuestionContentsChanged") .mockReturnValue(true); - await service["startPublishingProcess"](jobId, assignmentId, dto); + await service["startPublishingProcess"]( + jobId, + assignmentId, + dto, + "author-123", + ); expect( questionService.processQuestionsForPublishing, @@ -297,7 +315,12 @@ describe("AssignmentServiceV2 – full unit-suite", () => { }); assignmentRepository.findById.mockResolvedValue(existingAssignment); - await service["startPublishingProcess"](jobId, assignmentId, dto); + await service["startPublishingProcess"]( + jobId, + assignmentId, + dto, + "author-123", + ); expect(translationService.translateAssignment).not.toHaveBeenCalled(); expect(questionService.updateQuestionGradingContext).not.toHaveBeenCalled(); @@ -340,6 +363,7 @@ describe("AssignmentServiceV2 – full unit-suite", () => { showQuestionScore: false, showSubmissionFeedback: false, showQuestions: false, + showCorrectAnswer: false, }; expect( @@ -401,14 +425,22 @@ describe("AssignmentServiceV2 – full unit-suite", () => { const existing = [ createMockQuestionDto({ variants: [ - { id: 101, variantContent: "A", variantType: VariantType.REWORDED }, + { + id: 101, + variantContent: "A", + variantType: VariantTypeDto.REWORDED, + }, ], }), ]; const updated = [ createMockQuestionDto({ variants: [ - { id: 101, variantContent: "B", variantType: VariantType.REWORDED }, + { + id: 101, + variantContent: "B", + variantType: VariantTypeDto.REWORDED, + }, ], }), ]; diff --git a/apps/api/src/api/assignment/v2/tests/unit/services/job-status.service.spec.ts b/apps/api/src/api/assignment/v2/tests/unit/services/job-status.service.spec.ts index e596ef67..3b083a6e 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/services/job-status.service.spec.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/services/job-status.service.spec.ts @@ -7,7 +7,6 @@ /* eslint-disable unicorn/no-useless-undefined */ /* eslint-disable unicorn/no-null */ import { Test, TestingModule } from "@nestjs/testing"; -import { Job } from "@prisma/client"; import { firstValueFrom } from "rxjs"; import { PrismaService } from "src/prisma.service"; import { @@ -182,7 +181,7 @@ describe("JobStatusServiceV2", () => { it("should complete and remove a job stream", async () => { const jobId = 1; - const stream = jobStatusService.getPublishJobStatusStream(jobId); + jobStatusService.getPublishJobStatusStream(jobId); const subject = (jobStatusService as any).jobStatusStreams.get(jobId); jest.spyOn(subject, "complete"); @@ -441,7 +440,7 @@ describe("JobStatusServiceV2", () => { percentage: 50, }; - const stream = jobStatusService.getPublishJobStatusStream(jobId); + jobStatusService.getPublishJobStatusStream(jobId); const subject = (jobStatusService as any).jobStatusStreams.get(jobId); jest.spyOn(subject, "next"); @@ -469,7 +468,7 @@ describe("JobStatusServiceV2", () => { result: { data: "some result" }, }; - const stream = jobStatusService.getPublishJobStatusStream(jobId); + jobStatusService.getPublishJobStatusStream(jobId); const subject = (jobStatusService as any).jobStatusStreams.get(jobId); jest.spyOn(subject, "next"); @@ -525,7 +524,7 @@ describe("JobStatusServiceV2", () => { percentage: 50, }; - const stream = jobStatusService.getPublishJobStatusStream(jobId); + jobStatusService.getPublishJobStatusStream(jobId); const subject = (jobStatusService as any).jobStatusStreams.get(jobId); jest.spyOn(subject, "next"); @@ -555,7 +554,7 @@ describe("JobStatusServiceV2", () => { percentage: 50, }; - const stream = jobStatusService.getPublishJobStatusStream(jobId); + jobStatusService.getPublishJobStatusStream(jobId); const subject = (jobStatusService as any).jobStatusStreams.get(jobId); diff --git a/apps/api/src/api/assignment/v2/tests/unit/services/question.service.spec.ts b/apps/api/src/api/assignment/v2/tests/unit/services/question.service.spec.ts index 62fb7ef8..54dbdc7f 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/services/question.service.spec.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/services/question.service.spec.ts @@ -4,36 +4,33 @@ /* eslint-disable unicorn/no-null */ /* eslint-disable unicorn/no-useless-undefined */ /* eslint-disable @typescript-eslint/no-unsafe-argument */ -import { Test, TestingModule } from "@nestjs/testing"; import { BadRequestException, NotFoundException } from "@nestjs/common"; -import { PrismaService } from "src/prisma.service"; -import { QuestionService } from "../../../services/question.service"; -import { QuestionRepository } from "../../../repositories/question.repository"; -import { VariantRepository } from "../../../repositories/variant.repository"; -import { TranslationService } from "../../../services/translation.service"; -import { LlmFacadeService } from "src/api/llm/llm-facade.service"; -import { JobStatusServiceV2 } from "../../../services/job-status.service"; - +import { Test, TestingModule } from "@nestjs/testing"; +import { QuestionType, ResponseType } from "@prisma/client"; import { - QuestionDto, - VariantDto, - GenerateQuestionVariantDto, - VariantType, Choice, + GenerateQuestionVariantDto, + VariantDto, } from "src/api/assignment/dto/update.questions.request.dto"; -import { QuestionType, ResponseType } from "@prisma/client"; +import { LlmFacadeService } from "src/api/llm/llm-facade.service"; +import { PrismaService } from "src/prisma.service"; import { + createMockJob, + createMockJobStatusService, + createMockLlmFacadeService, createMockPrismaService, + createMockQuestionDto, + createMockQuestionGenerationPayload, createMockQuestionRepository, - createMockVariantRepository, createMockTranslationService, - createMockLlmFacadeService, - createMockJobStatusService, - createMockQuestionDto, createMockVariantDto, - createMockJob, - createMockQuestionGenerationPayload, + createMockVariantRepository, } from "../__mocks__/ common-mocks"; +import { QuestionRepository } from "../../../repositories/question.repository"; +import { VariantRepository } from "../../../repositories/variant.repository"; +import { JobStatusServiceV2 } from "../../../services/job-status.service"; +import { QuestionService } from "../../../services/question.service"; +import { TranslationService } from "../../../services/translation.service"; describe("QuestionService", () => { let questionService: QuestionService; @@ -247,7 +244,7 @@ describe("QuestionService", () => { expect(translationService.translateQuestion).toHaveBeenCalled(); }); - it("should skip translation for unchanged content", async () => { + it("should only translate questions when content changes", async () => { const assignmentId = 1; const jobId = 1; @@ -262,7 +259,46 @@ describe("QuestionService", () => { jobId, ); - expect(translationService.translateQuestion).not.toHaveBeenCalled(); + expect(translationService.translateQuestion).toHaveBeenCalledWith( + assignmentId, + question.id, + question, + jobId, + true, // questionContentChanged should be true for unchanged content - will retranslate + ); + }); + + it("should force translation when question content changes", async () => { + const assignmentId = 1; + const jobId = 1; + + const existingQuestion = createMockQuestionDto({ + id: 1, + question: "Original question text", + }); + const updatedQuestion = createMockQuestionDto({ + id: 1, + question: "Updated question text", + }); + + questionRepository.findByAssignmentId.mockResolvedValue([ + existingQuestion, + ]); + questionRepository.upsert.mockResolvedValue(updatedQuestion); + + await questionService.processQuestionsForPublishing( + assignmentId, + [updatedQuestion], + jobId, + ); + + expect(translationService.translateQuestion).toHaveBeenCalledWith( + assignmentId, + updatedQuestion.id, + updatedQuestion, + jobId, + true, // questionContentChanged should be true - will force retranslation + ); }); }); diff --git a/apps/api/src/api/assignment/v2/tests/unit/services/report.service.spec.ts b/apps/api/src/api/assignment/v2/tests/unit/services/report.service.spec.ts index 3085867b..e8c45915 100644 --- a/apps/api/src/api/assignment/v2/tests/unit/services/report.service.spec.ts +++ b/apps/api/src/api/assignment/v2/tests/unit/services/report.service.spec.ts @@ -3,20 +3,20 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/unbound-method */ /* eslint-disable @typescript-eslint/no-unsafe-call */ -import { Test, TestingModule } from "@nestjs/testing"; -import { PrismaService } from "src/prisma.service"; -import { ReportType } from "@prisma/client"; import { BadRequestException, NotFoundException, UnprocessableEntityException, } from "@nestjs/common"; -import { ReportService } from "../../../services/report.repository"; +import { Test, TestingModule } from "@nestjs/testing"; +import { ReportType } from "@prisma/client"; +import { PrismaService } from "src/prisma.service"; import { - createMockPrismaService, createMockAssignment, + createMockPrismaService, createMockReport, } from "../__mocks__/ common-mocks"; +import { ReportService } from "../../../services/report.repository"; describe("ReportService", () => { let service: ReportService; diff --git a/apps/api/src/api/assignment/v2/tests/utils/test-utils.ts b/apps/api/src/api/assignment/v2/tests/utils/test-utils.ts index 3fa295f0..351f66f1 100644 --- a/apps/api/src/api/assignment/v2/tests/utils/test-utils.ts +++ b/apps/api/src/api/assignment/v2/tests/utils/test-utils.ts @@ -5,8 +5,8 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ /* eslint-disable @typescript-eslint/no-unsafe-return */ -import { Test, TestingModule } from "@nestjs/testing"; import { INestApplication } from "@nestjs/common"; +import { Test, TestingModule } from "@nestjs/testing"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { UserRole, diff --git a/apps/api/src/api/attempt/attempt.constants.ts b/apps/api/src/api/attempt/attempt.constants.ts new file mode 100644 index 00000000..63a95ea4 --- /dev/null +++ b/apps/api/src/api/attempt/attempt.constants.ts @@ -0,0 +1,6 @@ +// Dependency injection tokens for AttemptModule +export const GRADING_AUDIT_SERVICE = "GRADING_AUDIT_SERVICE"; +export const GRADING_CONSISTENCY_SERVICE = "GRADING_CONSISTENCY_SERVICE"; +export const ATTEMPT_VALIDATION_SERVICE = "ATTEMPT_VALIDATION_SERVICE"; +export const FILE_CONTENT_EXTRACTION_SERVICE = + "FILE_CONTENT_EXTRACTION_SERVICE"; diff --git a/apps/api/src/api/attempt/attempt.controller.ts b/apps/api/src/api/attempt/attempt.controller.ts index 310d9dda..3a06dcf8 100644 --- a/apps/api/src/api/attempt/attempt.controller.ts +++ b/apps/api/src/api/attempt/attempt.controller.ts @@ -11,7 +11,6 @@ import { Post, Query, Req, - Res, Sse, UseGuards, } from "@nestjs/common"; @@ -23,7 +22,6 @@ import { ApiTags, } from "@nestjs/swagger"; import { ReportType } from "@prisma/client"; -import { Response as ExpressResponse } from "express"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { Observable } from "rxjs"; import { @@ -52,7 +50,9 @@ import { } from "../assignment/attempt/dto/assignment-attempt/get.assignment.attempt.response.dto"; import { ReportRequestDTO } from "../assignment/attempt/dto/assignment-attempt/post.assignment.report.dto"; import { AssignmentAttemptAccessControlGuard } from "../assignment/attempt/guards/assignment.attempt.access.control.guard"; +import { GRADING_AUDIT_SERVICE } from "./attempt.constants"; import { AttemptServiceV2 } from "./services/attempt.service"; +import { GradingAuditService } from "./services/question-response/grading-audit.service"; @ApiTags("Attempts") @Injectable() @@ -65,6 +65,8 @@ export class AttemptControllerV2 { constructor( @Inject(WINSTON_MODULE_PROVIDER) private parentLogger: Logger, private readonly attemptService: AttemptServiceV2, + @Inject(GRADING_AUDIT_SERVICE) + private readonly gradingAuditService: GradingAuditService, ) { this.logger = parentLogger.child({ context: AttemptControllerV2.name }); } @@ -443,4 +445,66 @@ export class AttemptControllerV2 { return { message: "Report submitted successfully" }; } + + @Get("grading/monitoring") + @Roles(UserRole.AUTHOR) + @ApiOperation({ + summary: "Get grading architecture monitoring data for debugging", + description: + "Returns statistics about grading audit records and logs usage summary", + }) + @ApiResponse({ + status: 200, + description: "Grading monitoring data", + schema: { + type: "object", + properties: { + message: { type: "string" }, + statistics: { + type: "object", + properties: { + totalGradings: { type: "number" }, + strategiesByCount: { + type: "array", + items: { + type: "object", + properties: { + strategy: { type: "string" }, + count: { type: "number" }, + }, + }, + }, + mostActiveQuestions: { + type: "array", + items: { + type: "object", + properties: { + questionId: { type: "number" }, + count: { type: "number" }, + }, + }, + }, + }, + }, + }, + }, + }) + @ApiResponse({ status: 403, description: "Forbidden - Author role required" }) + async getGradingMonitoring(): Promise<{ + message: string; + statistics: any; + }> { + // Log the usage summary to Winston logs + await this.gradingAuditService.logArchitectureUsageSummary(); + + // Also return statistics for API response + const statistics = + await this.gradingAuditService.getGradingUsageStatistics(); + + return { + message: + "Grading architecture usage summary logged. Check application logs for detailed output.", + statistics, + }; + } } diff --git a/apps/api/src/api/attempt/attempt.module.ts b/apps/api/src/api/attempt/attempt.module.ts index 1e55e27e..07922408 100644 --- a/apps/api/src/api/attempt/attempt.module.ts +++ b/apps/api/src/api/attempt/attempt.module.ts @@ -1,11 +1,17 @@ import { Module } from "@nestjs/common"; +import { PrismaService } from "../../prisma.service"; import { AssignmentAttemptAccessControlGuard } from "../assignment/attempt/guards/assignment.attempt.access.control.guard"; import { QuestionService } from "../assignment/question/question.service"; import { AssignmentModuleV2 } from "../assignment/v2/modules/assignment.module"; import { AssignmentRepository } from "../assignment/v2/repositories/assignment.repository"; +import { GradingConsistencyService } from "../assignment/v2/services/grading-consistency.service"; import { S3Service } from "../files/services/s3.service"; import { ImageGradingService } from "../llm/features/grading/services/image-grading.service"; import { LlmModule } from "../llm/llm.module"; +import { + FILE_CONTENT_EXTRACTION_SERVICE, + GRADING_AUDIT_SERVICE, +} from "./attempt.constants"; import { AttemptControllerV2 } from "./attempt.controller"; import { ChoiceGradingStrategy } from "./common/strategies/choice-grading.strategy"; import { FileGradingStrategy } from "./common/strategies/file-grading.strategy"; @@ -49,13 +55,20 @@ import { TranslationService } from "./services/translation/translation.service"; PresentationGradingStrategy, ChoiceGradingStrategy, TrueFalseGradingStrategy, - GradingAuditService, - FileContentExtractionService, + GradingConsistencyService, + { + provide: GRADING_AUDIT_SERVICE, + useClass: GradingAuditService, + }, + PrismaService, + { + provide: FILE_CONTENT_EXTRACTION_SERVICE, + useClass: FileContentExtractionService, + }, ImageGradingStrategy, ImageGradingService, S3Service, AssignmentRepository, - QuestionResponseService, QuestionVariantService, diff --git a/apps/api/src/api/attempt/common/interfaces/grading-strategy.interface.ts b/apps/api/src/api/attempt/common/interfaces/grading-strategy.interface.ts index 64b68dd0..48b45eba 100644 --- a/apps/api/src/api/attempt/common/interfaces/grading-strategy.interface.ts +++ b/apps/api/src/api/attempt/common/interfaces/grading-strategy.interface.ts @@ -1,6 +1,6 @@ -import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; +import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; import { GradingContext } from "./grading-context.interface"; /** diff --git a/apps/api/src/api/attempt/common/interfaces/question-response-handler.interfaces.ts b/apps/api/src/api/attempt/common/interfaces/question-response-handler.interfaces.ts index 11f8c8a8..64527277 100644 --- a/apps/api/src/api/attempt/common/interfaces/question-response-handler.interfaces.ts +++ b/apps/api/src/api/attempt/common/interfaces/question-response-handler.interfaces.ts @@ -1,7 +1,7 @@ import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; -import { QuestionAnswerContext } from "../../../llm/model/base.question.evaluate.model"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; +import { QuestionAnswerContext } from "../../../llm/model/base.question.evaluate.model"; /** * Interface for question response handlers diff --git a/apps/api/src/api/attempt/common/strategies/abstract-grading.strategy.ts b/apps/api/src/api/attempt/common/strategies/abstract-grading.strategy.ts index c19c5857..96b88b75 100644 --- a/apps/api/src/api/attempt/common/strategies/abstract-grading.strategy.ts +++ b/apps/api/src/api/attempt/common/strategies/abstract-grading.strategy.ts @@ -1,66 +1,70 @@ -/* eslint-disable unicorn/no-null */ -import { Injectable } from "@nestjs/common"; +// src/api/assignment/v2/common/strategies/abstract-grading.strategy.ts +import { Inject, Injectable, Optional } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { CreateQuestionResponseAttemptResponseDto, GeneralFeedbackDto, } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; -import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; -import { LocalizationService } from "../../common/utils/localization.service"; +import { + QuestionDto, + RubricDto, + ScoringDto, +} from "src/api/assignment/dto/update.questions.request.dto"; +import { GradingConsistencyService } from "src/api/assignment/v2/services/grading-consistency.service"; +import { IGradingJudgeService } from "src/api/llm/features/grading/interfaces/grading-judge.interface"; +import { GRADING_JUDGE_SERVICE } from "src/api/llm/llm.constants"; +import { RubricScore } from "src/api/llm/model/file.based.question.response.model"; +import { Logger } from "winston"; +import { GRADING_AUDIT_SERVICE } from "../../attempt.constants"; import { GradingAuditService } from "../../services/question-response/grading-audit.service"; import { GradingContext } from "../interfaces/grading-context.interface"; import { IGradingStrategy } from "../interfaces/grading-strategy.interface"; +import { LocalizationService } from "../utils/localization.service"; + +export interface GradingValidationResult { + isValid: boolean; + issues: string[]; + corrections?: { + points?: number; + feedback?: string; + rubricScores?: any[]; + }; +} + +export interface FeedbackTone { + tone: "positive" | "negative" | "neutral"; + confidence: number; +} @Injectable() -export abstract class AbstractGradingStrategy - implements IGradingStrategy -{ +export abstract class AbstractGradingStrategy implements IGradingStrategy { + protected readonly logger?: Logger; + constructor( protected readonly localizationService: LocalizationService, - protected readonly gradingAuditService?: GradingAuditService, - ) {} - - /** - * Process a question response from start to finish - * @param question The question to grade - * @param requestDto The student's response - * @param context Additional context for grading - * @returns The graded response - */ - async handleResponse( - question: QuestionDto, - requestDto: CreateQuestionResponseAttemptRequestDto, - context: GradingContext, - ): Promise<{ - responseDto: CreateQuestionResponseAttemptResponseDto; - learnerResponse: T; - }> { - await this.validateResponse(question, requestDto); - - const learnerResponse = await this.extractLearnerResponse(requestDto); - - const responseDto = await this.gradeResponse( - question, - learnerResponse, - context, - ); + @Inject(GRADING_AUDIT_SERVICE) + protected readonly gradingAuditService: GradingAuditService, + @Optional() + protected readonly consistencyService?: GradingConsistencyService, + @Optional() + @Inject(GRADING_JUDGE_SERVICE) + protected readonly gradingJudgeService?: IGradingJudgeService, + @Optional() @Inject(WINSTON_MODULE_PROVIDER) parentLogger?: Logger, + ) { + if (this.consistencyService) { + this.consistencyService = undefined; + } - if (this.gradingAuditService) { - await this.gradingAuditService.recordGrading({ - questionId: question.id, - assignmentId: context.assignmentId, - requestDto, - responseDto, - gradingStrategy: this.constructor.name, - metadata: context.metadata, + if (parentLogger) { + this.logger = parentLogger.child({ + context: this.constructor.name, }); } - - return { responseDto, learnerResponse }; } /** - * Default implementation of validateResponse - should be overridden by subclasses + * Validate the response format */ abstract validateResponse( question: QuestionDto, @@ -68,14 +72,14 @@ export abstract class AbstractGradingStrategy ): Promise; /** - * Default implementation of extractLearnerResponse - should be overridden by subclasses + * Extract the learner response in the appropriate format */ abstract extractLearnerResponse( requestDto: CreateQuestionResponseAttemptRequestDto, ): Promise; /** - * Default implementation of gradeResponse - should be overridden by subclasses + * Grade the response and return the result */ abstract gradeResponse( question: QuestionDto, @@ -84,52 +88,632 @@ export abstract class AbstractGradingStrategy ): Promise; /** - * Create a response DTO with an error message + * Create a response DTO with feedback */ - protected createErrorResponse( - errorMessage: string, - language?: string, + protected createResponseDto( + points: number, + feedback: any[], ): CreateQuestionResponseAttemptResponseDto { const responseDto = new CreateQuestionResponseAttemptResponseDto(); - responseDto.totalPoints = 0; + responseDto.totalPoints = this.sanitizePoints(points); + responseDto.feedback = Array.isArray(feedback) ? feedback : []; + responseDto.metadata = {}; + return responseDto; + } + /** + * Create a general feedback DTO + */ + protected createGeneralFeedback(message: string): GeneralFeedbackDto { const feedback = new GeneralFeedbackDto(); - feedback.feedback = this.localizationService.getLocalizedString( - errorMessage, - language, - ); + feedback.feedback = message || ""; + return feedback; + } - responseDto.feedback = [feedback]; - return responseDto; + /** + * Validate grading consistency and fairness + */ + protected async validateGradingConsistency( + response: CreateQuestionResponseAttemptResponseDto, + question: QuestionDto, + context: GradingContext, + learnerResponseText: string, + ): Promise { + const issues: string[] = []; + const corrections: { + points?: number; + feedback?: string; + rubricScores?: RubricDto[]; + } = {}; + + try { + // Validate points are within valid range + if ( + !this.isValidNumber(response.totalPoints) || + response.totalPoints < 0 + ) { + issues.push("Points cannot be negative or invalid"); + corrections.points = 0; + } + + if (response.totalPoints > question.totalPoints) { + issues.push(`Points exceed maximum (${question.totalPoints})`); + corrections.points = question.totalPoints; + } + + // Check consistency with similar responses + if ( + this.consistencyService && + typeof this.consistencyService.generateResponseHash === "function" + ) { + try { + const responseHash = this.consistencyService.generateResponseHash( + learnerResponseText, + question.id, + question.type, + ); + + const consistencyCheck = + await this.consistencyService.checkConsistency( + question.id, + responseHash, + learnerResponseText, + question.type, + ); + + if (consistencyCheck.similar && consistencyCheck.shouldAdjust) { + const deviationPercentage = + consistencyCheck.deviationPercentage || 0; + + if (deviationPercentage > 15) { + // More than 15% deviation + issues.push( + `Similar response previously graded differently (${deviationPercentage.toFixed( + 1, + )}% deviation)`, + ); + + // Suggest adjustment to maintain consistency + if (consistencyCheck.previousGrade !== undefined) { + corrections.points = consistencyCheck.previousGrade; + corrections.feedback = this.adjustFeedbackForConsistency( + response.feedback, + consistencyCheck.previousFeedback, + ); + } + } + } + } catch (error) { + this.logger?.warn("Consistency check failed:", error); + } + } + + // Validate feedback alignment with score + const scorePercentage = + (response.totalPoints / question.totalPoints) * 100; + const feedbackTone = this.analyzeFeedbackTone(response.feedback); + + if ( + scorePercentage >= 90 && + feedbackTone.tone === "negative" && + feedbackTone.confidence > 0.7 + ) { + issues.push("Negative feedback tone doesn't match high score"); + } else if ( + scorePercentage < 50 && + feedbackTone.tone === "positive" && + feedbackTone.confidence > 0.7 + ) { + issues.push("Positive feedback tone doesn't match low score"); + } + + // Validate rubric math if present + if (response.metadata?.rubricScores) { + const mathValidation = this.validateRubricMath( + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + response.metadata.rubricScores, + response.totalPoints, + question.scoring, + ); + + if ( + !mathValidation.valid && + mathValidation.correctedTotal !== undefined + ) { + issues.push("Rubric scores don't match total points"); + corrections.points = mathValidation.correctedTotal; + } + } + } catch (error) { + this.logger?.error("Error validating grading consistency:", error); + } + + return { + isValid: issues.length === 0, + issues, + corrections: + Object.keys(corrections).length > 0 ? corrections : undefined, + }; } /** - * Helper function to create a correctly formatted response DTO + * Ensure rubric scores are mathematically consistent */ - protected createResponseDto( - points: number, - feedbackItems: Array>, - ): CreateQuestionResponseAttemptResponseDto { - const responseDto = new CreateQuestionResponseAttemptResponseDto(); - responseDto.totalPoints = points; - responseDto.feedback = feedbackItems as GeneralFeedbackDto[]; - return responseDto; + protected validateRubricMath( + rubricScores: RubricScore[], + totalPoints: number, + scoringCriteria: ScoringDto, + ): { valid: boolean; correctedTotal?: number } { + if (!Array.isArray(rubricScores) || rubricScores.length === 0) { + return { valid: true }; + } + + try { + let calculatedTotal = 0; + for (const score of rubricScores) { + calculatedTotal += this.extractPointsFromRubricScore(score); + } + if (Math.abs(calculatedTotal - totalPoints) > 0.01) { + return { + valid: false, + correctedTotal: calculatedTotal, + }; + } + if ( + this.consistencyService && + scoringCriteria && + typeof this.consistencyService.validateRubricScores === "function" + ) { + const validation = this.consistencyService.validateRubricScores( + rubricScores, + scoringCriteria, + ); + + if (!validation.valid) { + const corrections: RubricScore[] = validation.corrections ?? []; + let correctedTotal = 0; + + for (const score of corrections) { + correctedTotal += this.extractPointsFromRubricScore(score); + } + + return { + valid: false, + correctedTotal, + }; + } + } + + return { valid: true }; + } catch (error) { + this.logger?.error("Error validating rubric math:", error); + return { valid: true }; // Don't fail grading due to validation error + } } /** - * Parse a JSON field from a database record safely + * Analyze feedback tone */ - protected parseJsonField(field: any): any { - if (!field) return null; + private analyzeFeedbackTone(feedback: any): FeedbackTone { + try { + const feedbackText = this.extractTextFromFeedback(feedback).toLowerCase(); + + const positiveWords = [ + "excellent", + "great", + "good", + "well done", + "impressive", + "strong", + "effective", + "successful", + "outstanding", + "perfect", + "exceptional", + "superb", + "brilliant", + "fantastic", + "wonderful", + ]; - if (typeof field === "string") { - try { - return JSON.parse(field); - } catch { - return null; + const negativeWords = [ + "poor", + "weak", + "insufficient", + "lacking", + "needs improvement", + "failed", + "incorrect", + "inadequate", + "wrong", + "missing", + "incomplete", + "unsatisfactory", + "below", + "deficient", + "problematic", + ]; + + let positiveCount = 0; + let negativeCount = 0; + + for (const word of positiveWords) { + if (feedbackText.includes(word)) positiveCount++; + } + + for (const word of negativeWords) { + if (feedbackText.includes(word)) negativeCount++; + } + + const totalWords = positiveCount + negativeCount; + if (totalWords === 0) { + return { tone: "neutral", confidence: 1 }; } + + const confidence = Math.min(totalWords / 5, 1); // Max confidence at 5 words + + if (positiveCount > negativeCount * 2) { + return { tone: "positive", confidence }; + } else if (negativeCount > positiveCount * 2) { + return { tone: "negative", confidence }; + } else { + return { tone: "neutral", confidence }; + } + } catch { + return { tone: "neutral", confidence: 0.5 }; } + } + + /** + * Extract text from various feedback formats + */ + private extractTextFromFeedback(feedback: any): string { + try { + if (typeof feedback === "string") { + return feedback; + } + + if (Array.isArray(feedback)) { + return feedback + .map((f) => { + if (typeof f === "string") return f; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-return + if (f?.feedback) return f.feedback; + return JSON.stringify(f); + }) + .join(" "); + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + if (feedback?.feedback) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access + return feedback.feedback; + } + + return JSON.stringify(feedback); + } catch { + return ""; + } + } + + /** + * Adjust feedback to maintain consistency + */ + private adjustFeedbackForConsistency( + currentFeedback: any, + previousFeedback?: string, + ): string { + try { + const currentText = this.extractTextFromFeedback(currentFeedback); + + if (!previousFeedback) return currentText; + + // Add consistency note to feedback + const adjustedFeedback = `${currentText}\n\n**Note**: This response is similar to previous submissions and has been graded consistently.`; + + return adjustedFeedback; + } catch { + return this.extractTextFromFeedback(currentFeedback); + } + } + + /** + * Record grading for audit and consistency (non-blocking) + * This method will log all activities and continue even if recording fails + */ + protected async recordGrading( + question: QuestionDto, + requestDto: CreateQuestionResponseAttemptRequestDto, + responseDto: CreateQuestionResponseAttemptResponseDto, + context: GradingContext, + gradingStrategy: string, + ): Promise { + const startTime = Date.now(); + + this.logger?.info("Starting grading record process", { + questionId: question.id, + questionType: question.type, + assignmentId: context.assignmentId, + gradingStrategy, + responseType: question.responseType, + totalPoints: responseDto.totalPoints, + maxPoints: question.totalPoints, + }); + + try { + // Record in audit service (non-blocking) + if (!this.gradingAuditService) { + this.logger?.error("GradingAuditService is not available", { + questionId: question.id, + gradingStrategy, + hasService: !!this.gradingAuditService, + }); + return; // Exit early if service is not available + } + + // Type guard: Check if this is actually a GradingAuditService instance + if (typeof this.gradingAuditService.recordGrading !== "function") { + this.logger?.error( + "GradingAuditService.recordGrading is not a function - wrong service type injected", + { + questionId: question.id, + gradingStrategy, + serviceType: typeof this.gradingAuditService, + serviceConstructor: this.gradingAuditService.constructor.name, + availableMethods: Object.getOwnPropertyNames( + Object.getPrototypeOf(this.gradingAuditService), + ), + expectedService: "GradingAuditService", + actualService: this.gradingAuditService.constructor.name, + }, + ); + + // Gracefully skip audit recording if wrong service type is injected + this.logger?.warn( + "Skipping grading audit due to dependency injection issue", + { + questionId: question.id, + gradingStrategy, + }, + ); + return; + } + + const auditPromise = this.gradingAuditService.recordGrading({ + questionId: question.id, + assignmentId: context.assignmentId, + requestDto, + responseDto, + gradingStrategy, + metadata: { + language: context.language, + userRole: context.userRole, + timestamp: new Date().toISOString(), + version: "2.0", + processingTime: Date.now() - startTime, + }, + }); + + // Record in consistency service (non-blocking) + const consistencyPromise = this.recordConsistencyData( + question, + requestDto, + responseDto, + context, + ); + + // Wait for both operations but don't throw if they fail + const results = await Promise.allSettled([ + auditPromise, + consistencyPromise, + ]); + + // Log results but don't fail + for (const [index, result] of results.entries()) { + const operation = index === 0 ? "audit" : "consistency"; + if (result.status === "fulfilled") { + this.logger?.debug(`Successfully recorded ${operation} data`, { + questionId: question.id, + operation, + }); + } else { + this.logger?.warn(`Failed to record ${operation} data - continuing`, { + questionId: question.id, + operation, + error: + result.reason instanceof Error + ? result.reason.message + : String(result.reason), + }); + } + } + + const duration = Date.now() - startTime; + this.logger?.info("Completed grading record process", { + questionId: question.id, + gradingStrategy, + duration, + auditSuccess: results[0].status === "fulfilled", + consistencySuccess: results[1].status === "fulfilled", + }); + } catch (error) { + // This shouldn't happen with Promise.allSettled, but just in case + this.logger?.error("Unexpected error in recordGrading", { + questionId: question.id, + gradingStrategy, + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }); + } + } + + /** + * Record consistency data separately for better error handling + */ + private async recordConsistencyData( + question: QuestionDto, + requestDto: CreateQuestionResponseAttemptRequestDto, + responseDto: CreateQuestionResponseAttemptResponseDto, + _context: GradingContext, + ): Promise { + if (!this.consistencyService) { + this.logger?.debug( + "Consistency service not available - skipping consistency recording", + ); + return; + } + + // Additional safety check and type guard + if (typeof this.consistencyService.generateResponseHash !== "function") { + this.logger?.error( + "GradingConsistencyService.generateResponseHash is not a function - wrong service type injected", + { + questionId: question.id, + serviceType: typeof this.consistencyService, + serviceConstructor: this.consistencyService.constructor.name, + availableMethods: Object.getOwnPropertyNames( + Object.getPrototypeOf(this.consistencyService), + ), + expectedService: "GradingConsistencyService", + actualService: this.consistencyService.constructor.name, + }, + ); + return; + } + + try { + const responseText = this.extractResponseText(requestDto); + const responseHash = this.consistencyService.generateResponseHash( + responseText, + question.id, + question.type, + ); + + await this.consistencyService.recordGrading( + question.id, + responseHash, + responseDto.totalPoints, + question.totalPoints, + JSON.stringify(responseDto.feedback), + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + responseDto.metadata?.rubricScores, + ); + } catch (error) { + this.logger?.error("Error in consistency recording", { + questionId: question.id, + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }); + } + } + + /** + * Extract response text for consistency checking + */ + private extractResponseText( + requestDto: CreateQuestionResponseAttemptRequestDto, + ): string { + if (requestDto.learnerTextResponse) return requestDto.learnerTextResponse; + if (requestDto.learnerChoices) return requestDto.learnerChoices.join(","); + if (requestDto.learnerAnswerChoice !== undefined) + return String(requestDto.learnerAnswerChoice); + if (requestDto.learnerUrlResponse) return requestDto.learnerUrlResponse; + if (requestDto.learnerFileResponse) + return requestDto.learnerFileResponse + .map((f) => f.filename || "file") + .join(","); + if (requestDto.learnerPresentationResponse) + return JSON.stringify(requestDto.learnerPresentationResponse); + return ""; + } + + /** + * Generate feedback with proper score alignment + */ + protected generateAlignedFeedback( + points: number, + maxPoints: number, + analysis: string, + evaluation: string, + explanation: string, + guidance: string, + rubricDetails?: string, + ): string { + const percentage = + maxPoints > 0 ? Math.round((points / maxPoints) * 100) : 0; + const gradeContext = this.getGradeContext(percentage); + + return ` +**Overall Performance**: ${gradeContext} + +**Analysis:** +${analysis} + +**Evaluation:** +${evaluation}${rubricDetails || ""} + +**Score Explanation:** +You earned ${points} out of ${maxPoints} points (${percentage}%). +${explanation} + +**Guidance for Improvement:** +${guidance} + +--- +*This grade has been validated for consistency and fairness.* +`.trim(); + } + + /** + * Get grade context message based on percentage + */ + private getGradeContext(percentage: number): string { + if (percentage >= 95) return "Outstanding work!"; + if (percentage >= 90) return "Excellent performance!"; + if (percentage >= 85) return "Very good work!"; + if (percentage >= 80) return "Good performance."; + if (percentage >= 75) return "Above average work."; + if (percentage >= 70) return "Satisfactory performance."; + if (percentage >= 65) return "Adequate work with room for improvement."; + if (percentage >= 60) return "Below average performance."; + if (percentage >= 50) return "Needs significant improvement."; + return "Substantial improvement required."; + } + + /** + * Sanitize points to ensure valid number + */ + private sanitizePoints(points: any): number { + if ( + typeof points === "number" && + !Number.isNaN(points) && + Number.isFinite(points) + ) { + return Math.max(0, points); + } + return 0; + } + + /** + * Check if value is a valid number + */ + private isValidNumber(value: any): boolean { + return ( + typeof value === "number" && + !Number.isNaN(value) && + Number.isFinite(value) + ); + } + + /** + * Extract points from rubric score object safely + */ + private extractPointsFromRubricScore(score: RubricScore): number { + if (!score || typeof score !== "object") return 0; - return field; + // Use awarded points (fallback to 0). Max points is not the achieved score. + const points = + typeof score.pointsAwarded === "number" ? score.pointsAwarded : 0; + return this.sanitizePoints(points); } } diff --git a/apps/api/src/api/attempt/common/strategies/choice-grading.strategy.ts b/apps/api/src/api/attempt/common/strategies/choice-grading.strategy.ts index 4ed6fcfa..ec76935d 100644 --- a/apps/api/src/api/attempt/common/strategies/choice-grading.strategy.ts +++ b/apps/api/src/api/attempt/common/strategies/choice-grading.strategy.ts @@ -1,6 +1,12 @@ /* eslint-disable @typescript-eslint/require-await */ -import { BadRequestException, Injectable } from "@nestjs/common"; +import { + BadRequestException, + Inject, + Injectable, + Optional, +} from "@nestjs/common"; import { QuestionType } from "@prisma/client"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { ChoiceBasedFeedbackDto, @@ -11,6 +17,8 @@ import { QuestionDto, } from "src/api/assignment/dto/update.questions.request.dto"; import { ScoringType } from "src/api/assignment/question/dto/create.update.question.request.dto"; +import { Logger } from "winston"; +import { GRADING_AUDIT_SERVICE } from "../../attempt.constants"; import { GradingAuditService } from "../../services/question-response/grading-audit.service"; import { GradingContext } from "../interfaces/grading-context.interface"; import { LocalizationService } from "../utils/localization.service"; @@ -20,9 +28,17 @@ import { AbstractGradingStrategy } from "./abstract-grading.strategy"; export class ChoiceGradingStrategy extends AbstractGradingStrategy { constructor( protected readonly localizationService: LocalizationService, + @Inject(GRADING_AUDIT_SERVICE) protected readonly gradingAuditService: GradingAuditService, + @Optional() @Inject(WINSTON_MODULE_PROVIDER) parentLogger?: Logger, ) { - super(localizationService, gradingAuditService); + super( + localizationService, + gradingAuditService, + undefined, + undefined, + parentLogger, + ); } /** @@ -92,25 +108,29 @@ export class ChoiceGradingStrategy extends AbstractGradingStrategy { learnerResponse: string[], context: GradingContext, ): Promise { + let responseDto: CreateQuestionResponseAttemptResponseDto; + if (question.type === QuestionType.SINGLE_CORRECT) { const result = await this.gradeSingleChoice( question, learnerResponse, context, ); - return result.responseDto; + responseDto = result.responseDto; } else if (question.type === QuestionType.MULTIPLE_CORRECT) { const result = await this.gradeMultipleChoice( question, learnerResponse, context, ); - return result.responseDto; + responseDto = result.responseDto; } else { throw new BadRequestException( `Unsupported choice question type: ${question.type}`, ); } + + return responseDto; } /** diff --git a/apps/api/src/api/attempt/common/strategies/file-grading.strategy.ts b/apps/api/src/api/attempt/common/strategies/file-grading.strategy.ts index a7b9a177..5a805481 100644 --- a/apps/api/src/api/attempt/common/strategies/file-grading.strategy.ts +++ b/apps/api/src/api/attempt/common/strategies/file-grading.strategy.ts @@ -1,11 +1,31 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/require-await */ -import { BadRequestException, Injectable } from "@nestjs/common"; +import { + BadRequestException, + Inject, + Injectable, + Optional, +} from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; -import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; +import { + ChoiceBasedFeedbackDto, + CreateQuestionResponseAttemptResponseDto, + GeneralFeedbackDto, + TrueFalseBasedFeedbackDto, +} from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; import { AttemptHelper } from "src/api/assignment/attempt/helper/attempts.helper"; import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; import { LlmFacadeService } from "src/api/llm/llm-facade.service"; import { FileUploadQuestionEvaluateModel } from "src/api/llm/model/file.based.question.evaluate.model"; +import { Logger } from "winston"; +import { IGradingJudgeService } from "../../../llm/features/grading/interfaces/grading-judge.interface"; +import { GRADING_JUDGE_SERVICE } from "../../../llm/llm.constants"; +import { + FILE_CONTENT_EXTRACTION_SERVICE, + GRADING_AUDIT_SERVICE, +} from "../../attempt.constants"; import { ExtractedFileContent, FileContentExtractionService, @@ -22,11 +42,23 @@ export class FileGradingStrategy extends AbstractGradingStrategy< > { constructor( private readonly llmFacadeService: LlmFacadeService, + @Inject(FILE_CONTENT_EXTRACTION_SERVICE) private readonly fileContentExtractionService: FileContentExtractionService, protected readonly localizationService: LocalizationService, + @Inject(GRADING_AUDIT_SERVICE) protected readonly gradingAuditService: GradingAuditService, + @Optional() + @Inject(GRADING_JUDGE_SERVICE) + protected readonly gradingJudgeService?: IGradingJudgeService, + @Optional() @Inject(WINSTON_MODULE_PROVIDER) parentLogger?: Logger, ) { - super(localizationService, gradingAuditService); + super( + localizationService, + gradingAuditService, + undefined, + gradingJudgeService, + parentLogger, + ); } /** @@ -111,9 +143,59 @@ export class FileGradingStrategy extends AbstractGradingStrategy< context.language, ); - const responseDto = new CreateQuestionResponseAttemptResponseDto(); + let responseDto = new CreateQuestionResponseAttemptResponseDto(); AttemptHelper.assignFeedbackToResponse(gradingModel, responseDto); + // Fix mathematical inconsistency: ensure total points matches rubric scores sum + let rubricSum = 0; + + if (Array.isArray(responseDto.metadata?.rubricScores)) { + for (const score of responseDto.metadata.rubricScores) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment + const points = score.pointsAwarded ?? score.points ?? score.score ?? 0; + if (typeof points === "number") { + rubricSum += points; + } + } + + if (rubricSum === responseDto.totalPoints) { + this.logger?.info("Math is already consistent in FileGradingStrategy", { + questionId: question.id, + totalPoints: responseDto.totalPoints, + rubricSum, + }); + } else { + this.logger?.warn( + "Mathematical inconsistency detected - correcting total points", + { + questionId: question.id, + originalTotal: responseDto.totalPoints, + rubricSum, + rubricScores: responseDto.metadata.rubricScores, + }, + ); + + const originalTotal = responseDto.totalPoints; + responseDto.totalPoints = rubricSum; + responseDto.metadata.mathCorrected = true; + responseDto.metadata.originalTotal = originalTotal; + + this.logger?.info("Applied math correction in FileGradingStrategy", { + questionId: question.id, + correctedFrom: originalTotal, + correctedTo: rubricSum, + }); + } + } + + // Iterative grading improvement with judge validation + responseDto = await this.iterativeGradingWithJudge( + question, + learnerResponse, + responseDto, + context, + ); + responseDto.metadata = { ...responseDto.metadata, fileCount: learnerResponse.length, @@ -132,6 +214,38 @@ export class FileGradingStrategy extends AbstractGradingStrategy< extractionStatus: this.getExtractionStatus(extractedFiles), }; + // Judge validation is handled in iterativeGradingWithJudge method above + // This eliminates duplicate validation calls and reduces token usage + + // Record grading for audit and consistency (non-blocking) + try { + await this.recordGrading( + question, + { + learnerFileResponse: learnerResponse, + } as CreateQuestionResponseAttemptRequestDto, + responseDto, + context, + "FileGradingStrategy", + ); + } catch (error) { + // Log error but don't fail grading - audit is supplementary + this.logger?.error("Grading audit failed but continuing with grading", { + questionId: question.id, + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }); + + // Add audit failure to metadata for troubleshooting + responseDto.metadata = { + ...responseDto.metadata, + auditFailure: { + error: error instanceof Error ? error.message : String(error), + timestamp: new Date().toISOString(), + }, + }; + } + return responseDto; } @@ -174,10 +288,7 @@ export class FileGradingStrategy extends AbstractGradingStrategy< summary += ` - ${contentLength} characters`; if (hasCode) { - const language = this.detectProgrammingLanguage( - extracted.filename, - extracted.content, - ); + const language = this.detectProgrammingLanguage(extracted.filename); summary += `, ${language} code detected`; } @@ -217,7 +328,7 @@ export class FileGradingStrategy extends AbstractGradingStrategy< /** * Detect programming language from filename and content */ - private detectProgrammingLanguage(filename: string, content: string): string { + private detectProgrammingLanguage(filename: string): string { const extension = filename.split(".").pop()?.toLowerCase() || ""; const languageMap: Record = { @@ -298,4 +409,284 @@ export class FileGradingStrategy extends AbstractGradingStrategy< return { successful, failed, partial }; } + + /** + * Create a summary of learner response for judge validation + */ + private createLearnerResponseSummary( + learnerResponse: LearnerFileUpload[], + ): string { + const fileNames = learnerResponse.map((file) => file.filename).join(", "); + const contentSummary = learnerResponse + .map( + (file) => `${file.filename} - ${file.content?.length || 0} characters`, + ) + .join("; "); + + return `Files uploaded: ${fileNames}. Content: ${contentSummary}`; + } + + /** + * Iteratively improve grading with judge validation + * Preserves initial rubric scores and only adjusts feedback to prevent math inconsistencies + */ + private async iterativeGradingWithJudge( + question: QuestionDto, + learnerResponse: LearnerFileUpload[], + initialResponseDto: CreateQuestionResponseAttemptResponseDto, + context: GradingContext, + ): Promise { + const maxAttempts = 3; // Limit iterations to prevent infinite loops + let currentResponseDto = initialResponseDto; + let attempt = 1; + let previousJudgeFeedback = ""; + + // Preserve original rubric scores to prevent mathematical inconsistencies + const originalRubricScores = currentResponseDto.metadata?.rubricScores + ? JSON.parse(JSON.stringify(currentResponseDto.metadata.rubricScores)) + : []; + + // If no judge service available, return initial grading + if (!this.gradingJudgeService) { + this.logger?.debug("Judge service not available for iterative grading", { + questionId: question.id, + }); + return currentResponseDto; + } + + while (attempt <= maxAttempts) { + this.logger?.info(`Judge validation attempt ${attempt}/${maxAttempts}`, { + questionId: question.id, + currentPoints: currentResponseDto.totalPoints, + maxPoints: question.totalPoints, + }); + + try { + // Debug rubric scores before sending to judge + this.logger?.debug("Debug: Rubric scores for judge validation", { + questionId: question.id, + attempt, + hasMetadata: !!currentResponseDto.metadata, + metadataKeys: currentResponseDto.metadata + ? Object.keys(currentResponseDto.metadata) + : [], + rubricScoresLength: + currentResponseDto.metadata?.rubricScores?.length || 0, + rubricScores: currentResponseDto.metadata?.rubricScores || [], + }); + + // Validate current grading with judge + const judgeResult = await this.gradingJudgeService.validateGrading({ + question: question.question, + learnerResponse: this.createLearnerResponseSummary(learnerResponse), + scoringCriteria: question.scoring, + proposedGrading: { + points: currentResponseDto.totalPoints, + maxPoints: question.totalPoints, + feedback: JSON.stringify(currentResponseDto.feedback), + rubricScores: currentResponseDto.metadata?.rubricScores || [], + }, + assignmentId: context.assignmentId, + }); + + if (judgeResult.approved) { + this.logger?.info(`Judge approved grading on attempt ${attempt}`, { + questionId: question.id, + finalPoints: currentResponseDto.totalPoints, + attempts: attempt, + }); + + // Add metadata about judge validation + currentResponseDto.metadata = { + ...currentResponseDto.metadata, + judgeValidated: true, + judgeApproved: true, + validationAttempts: attempt, + judgeFeedback: judgeResult.feedback, + }; + + return currentResponseDto; + } + + // Judge rejected - prepare feedback for re-grading + const judgeFeedback = this.formatJudgeFeedback( + judgeResult, + previousJudgeFeedback, + ); + previousJudgeFeedback = judgeResult.feedback; + + this.logger?.warn(`Judge rejected grading on attempt ${attempt}`, { + questionId: question.id, + issues: judgeResult.issues, + suggestedPoints: judgeResult.corrections?.points, + judgeFeedback: judgeResult.feedback, + }); + + // If this is the last attempt, apply judge corrections directly + if (attempt === maxAttempts) { + this.logger?.warn( + "Max attempts reached, applying judge corrections", + { + questionId: question.id, + originalPoints: currentResponseDto.totalPoints, + judgePoints: judgeResult.corrections?.points, + }, + ); + + if (judgeResult.corrections?.points !== undefined) { + currentResponseDto.totalPoints = judgeResult.corrections.points; + } + if (judgeResult.corrections?.feedback) { + currentResponseDto.feedback = [ + { + feedback: judgeResult.corrections.feedback, + }, + ]; + } + + currentResponseDto.metadata = { + ...currentResponseDto.metadata, + judgeValidated: true, + judgeApproved: false, + validationAttempts: attempt, + judgeFeedback: judgeResult.feedback, + judgeOverride: true, + judgeIssues: judgeResult.issues, + }; + + return currentResponseDto; + } + + // CRITICAL FIX: Only adjust feedback, preserve rubric scores to prevent math inconsistencies + this.logger?.info("Adjusting feedback only, preserving rubric scores", { + questionId: question.id, + attempt: attempt + 1, + judgeFeedback: judgeFeedback, + preservedRubricCount: originalRubricScores.length, + }); + + // Create improved response with preserved rubric scores + const improvedResponseDto = + new CreateQuestionResponseAttemptResponseDto(); + + let correctTotal = 0; + + for (const score of originalRubricScores) { + const points = + score.pointsAwarded ?? score.points ?? score.score ?? 0; + if (typeof points === "number") { + correctTotal += points; + } + } + + improvedResponseDto.totalPoints = correctTotal; + improvedResponseDto.metadata = { + ...currentResponseDto.metadata, + rubricScores: originalRubricScores, + judgeIterationAttempt: attempt, + preservedRubricScores: true, + mathCorrectedInJudge: true, + }; + + // Only update feedback based on judge suggestions + const enhancedFeedback = this.createEnhancedFeedback( + currentResponseDto.feedback, + judgeFeedback, + ); + improvedResponseDto.feedback = enhancedFeedback; + + currentResponseDto = improvedResponseDto; + attempt++; + } catch (error) { + this.logger?.error(`Judge validation failed on attempt ${attempt}`, { + questionId: question.id, + error: error instanceof Error ? error.message : String(error), + }); + + // On error, return current grading + currentResponseDto.metadata = { + ...currentResponseDto.metadata, + judgeValidated: false, + validationError: + error instanceof Error ? error.message : String(error), + validationAttempts: attempt, + }; + + return currentResponseDto; + } + } + + return currentResponseDto; + } + + /** + * Create enhanced feedback without changing rubric scores + */ + private createEnhancedFeedback( + originalFeedback: + | ChoiceBasedFeedbackDto[] + | GeneralFeedbackDto[] + | TrueFalseBasedFeedbackDto[], + judgeFeedback: string, + ): any[] { + try { + const enhancedFeedback = [...(originalFeedback || [])]; + + enhancedFeedback.push({ + feedback: `**Additional Feedback Based on Quality Review:**\n${judgeFeedback}`, + }); + + return enhancedFeedback; + } catch (error) { + this.logger?.warn("Failed to enhance feedback, using original", { + error: error instanceof Error ? error.message : String(error), + }); + return originalFeedback || []; + } + } + + /** + * Format judge feedback for re-grading + */ + private formatJudgeFeedback( + judgeResult: { + issues?: string[]; + feedback?: string; + corrections?: { + points?: number; + feedback?: string; + }; + approved?: boolean; + validationAttempts?: number; + judgeFeedback?: string; + judgeIssues?: string[]; + judgeOverride?: boolean; + }, + previousFeedback: string, + ): string { + let feedback = `Previous grading was rejected by the judge. Issues identified:\n`; + + if (Array.isArray(judgeResult.issues)) { + feedback += + judgeResult.issues + .map((issue: string, index: number) => `${index + 1}. ${issue}`) + .join("\n") + "\n"; + } + + if (judgeResult.feedback) { + feedback += `\nJudge feedback: ${judgeResult.feedback}\n`; + } + + if (judgeResult.corrections?.points !== undefined) { + feedback += `\nSuggested points: ${judgeResult.corrections.points}\n`; + } + + if (previousFeedback) { + feedback += `\nPrevious feedback: ${previousFeedback}\n`; + } + + feedback += `\nPlease revise the grading to address these issues and ensure it aligns with the rubric.`; + + return feedback; + } } diff --git a/apps/api/src/api/attempt/common/strategies/image-grading.strategy.ts b/apps/api/src/api/attempt/common/strategies/image-grading.strategy.ts index 2be7ab75..6a740c28 100644 --- a/apps/api/src/api/attempt/common/strategies/image-grading.strategy.ts +++ b/apps/api/src/api/attempt/common/strategies/image-grading.strategy.ts @@ -1,13 +1,16 @@ /* eslint-disable @typescript-eslint/require-await */ -import { BadRequestException, Injectable } from "@nestjs/common"; +import { + BadRequestException, + Inject, + Injectable, + Optional, +} from "@nestjs/common"; import { QuestionType, ResponseType } from "@prisma/client"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; import { AttemptHelper } from "src/api/assignment/attempt/helper/attempts.helper"; -import { - QuestionDto, - ScoringDto, -} from "src/api/assignment/dto/update.questions.request.dto"; +import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; import { ScoringType } from "src/api/assignment/question/dto/create.update.question.request.dto"; import { ImageGradingService } from "src/api/llm/features/grading/services/image-grading.service"; import { @@ -16,6 +19,8 @@ import { LearnerImageUpload, } from "src/api/llm/model/image.based.evalutate.model"; import { ImageBasedQuestionResponseModel } from "src/api/llm/model/image.based.response.model"; +import { Logger } from "winston"; +import { GRADING_AUDIT_SERVICE } from "../../attempt.constants"; import { GradingAuditService } from "../../services/question-response/grading-audit.service"; import { GradingContext } from "../interfaces/grading-context.interface"; import { LocalizationService } from "../utils/localization.service"; @@ -43,9 +48,17 @@ export class ImageGradingStrategy extends AbstractGradingStrategy< constructor( private readonly imageGradingService: ImageGradingService, protected readonly localizationService: LocalizationService, + @Inject(GRADING_AUDIT_SERVICE) protected readonly gradingAuditService: GradingAuditService, + @Optional() @Inject(WINSTON_MODULE_PROVIDER) parentLogger?: Logger, ) { - super(localizationService, gradingAuditService); + super( + localizationService, + gradingAuditService, + undefined, + undefined, + parentLogger, + ); } async validateResponse( @@ -160,10 +173,9 @@ export class ImageGradingStrategy extends AbstractGradingStrategy< ); // Validate the grading result - const validatedResult = this.validateGradingConsistency( + const validatedResult = this.validateGradingConsistencyImage( gradingResult, - question.totalPoints, - question.scoring, + question, ); // Create response DTO @@ -190,15 +202,25 @@ export class ImageGradingStrategy extends AbstractGradingStrategy< `Successfully graded question ${question.id} - awarded ${responseDto.totalPoints}/${question.totalPoints} points`, ); + await this.recordGrading( + question, + { + learnerFileResponse: learnerResponse, + } as CreateQuestionResponseAttemptRequestDto, + responseDto, + context, + "ImageGradingStrategy", + ); + return responseDto; } - private validateGradingConsistency( - gradingResult: { points?: number; feedback?: string }, - maxPoints: number, - scoring: ScoringDto | undefined, + private validateGradingConsistencyImage( + response: ImageBasedQuestionResponseModel, + question: QuestionDto, ): ImageBasedQuestionResponseModel { - const points = gradingResult.points || 0; - const feedback = gradingResult.feedback || ""; + const points = response.points || 0; + const feedback = response.feedback || ""; + const maxPoints = question.totalPoints || 0; // Ensure points are within valid range const validatedPoints = Math.min(Math.max(points, 0), maxPoints); @@ -247,23 +269,27 @@ export class ImageGradingStrategy extends AbstractGradingStrategy< private extractPointsFromFeedback(feedback: string): number[] { const points: number[] = []; - // Look for patterns like "X points", "awarded X", "score: X", etc. + // Look for patterns specifically for final scores, avoiding intermediate scores const patterns = [ - /(\d+)\s*points?\s*(?:awarded|given|earned)/gi, - /(?:awarded|score|total):\s*(\d+)/gi, - /(\d+)\s*\/\s*\d+\s*points?/gi, + /(?:total\s*score|final\s*score|overall\s*score):\s*(\d+)/gi, + /(?:awarded|final\s*grade):\s*(\d+)\s*(?:points?|pts?)?$/gi, + /^(?:score|total):\s*(\d+)\s*(?:\/\s*\d+)?/gm, // Line starting with score + /(\d+)\s*(?:points?|pts?)\s*(?:out\s*of|\/)\s*\d+\s*(?:total|maximum)?$/gi, // Final score format ]; for (const pattern of patterns) { const matches = feedback.matchAll(pattern); for (const match of matches) { const point = Number.parseInt(match[1], 10); - if (!Number.isNaN(point)) { + if (!Number.isNaN(point) && point >= 0) { points.push(point); } } } + // If no specific final score patterns found, avoid parsing intermediate scores + // to prevent the false positive warnings + return points; } diff --git a/apps/api/src/api/attempt/common/strategies/presentation-grading.strategy.ts b/apps/api/src/api/attempt/common/strategies/presentation-grading.strategy.ts index 24979d88..33f882fa 100644 --- a/apps/api/src/api/attempt/common/strategies/presentation-grading.strategy.ts +++ b/apps/api/src/api/attempt/common/strategies/presentation-grading.strategy.ts @@ -1,12 +1,20 @@ /* eslint-disable @typescript-eslint/require-await */ -import { Injectable, BadRequestException } from "@nestjs/common"; -import { LlmFacadeService } from "src/api/llm/llm-facade.service"; -import { PresentationQuestionEvaluateModel } from "src/api/llm/model/presentation.question.evaluate.model"; -import { VideoPresentationQuestionEvaluateModel } from "src/api/llm/model/video-presentation.question.evaluate.model"; +import { + BadRequestException, + Inject, + Injectable, + Optional, +} from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; -import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; import { AttemptHelper } from "src/api/assignment/attempt/helper/attempts.helper"; +import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; +import { LlmFacadeService } from "src/api/llm/llm-facade.service"; +import { PresentationQuestionEvaluateModel } from "src/api/llm/model/presentation.question.evaluate.model"; +import { VideoPresentationQuestionEvaluateModel } from "src/api/llm/model/video-presentation.question.evaluate.model"; +import { Logger } from "winston"; +import { GRADING_AUDIT_SERVICE } from "../../attempt.constants"; import { GradingAuditService } from "../../services/question-response/grading-audit.service"; import { LearnerPresentationResponse, @@ -21,9 +29,17 @@ export class PresentationGradingStrategy extends AbstractGradingStrategy { + let responseDto: CreateQuestionResponseAttemptResponseDto; + if (question.responseType === "LIVE_RECORDING") { - return this.gradeLiveRecording(question, learnerResponse, context); + responseDto = await this.gradeLiveRecording( + question, + learnerResponse, + context, + ); } else if (question.responseType === "PRESENTATION") { - return this.gradePresentation(question, learnerResponse, context); + responseDto = await this.gradePresentation( + question, + learnerResponse, + context, + ); } else { throw new BadRequestException( `Unsupported presentation response type: ${question.responseType}`, ); } + + await this.recordGrading( + question, + { + learnerPresentationResponse: learnerResponse, + } as CreateQuestionResponseAttemptRequestDto, + responseDto, + context, + `PresentationGradingStrategy-${question.responseType}`, + ); + + return responseDto; } /** diff --git a/apps/api/src/api/attempt/common/strategies/text-grading.strategy.ts b/apps/api/src/api/attempt/common/strategies/text-grading.strategy.ts index 723ab8d8..f4281289 100644 --- a/apps/api/src/api/attempt/common/strategies/text-grading.strategy.ts +++ b/apps/api/src/api/attempt/common/strategies/text-grading.strategy.ts @@ -1,11 +1,23 @@ +/* eslint-disable unicorn/no-null */ /* eslint-disable @typescript-eslint/require-await */ -import { BadRequestException, Injectable } from "@nestjs/common"; +// src/api/assignment/v2/common/strategies/text-grading.strategy.ts +import { + BadRequestException, + Inject, + Injectable, + Optional, +} from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; import { AttemptHelper } from "src/api/assignment/attempt/helper/attempts.helper"; import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; +import { IGradingJudgeService } from "src/api/llm/features/grading/interfaces/grading-judge.interface"; import { LlmFacadeService } from "src/api/llm/llm-facade.service"; +import { GRADING_JUDGE_SERVICE } from "src/api/llm/llm.constants"; import { TextBasedQuestionEvaluateModel } from "src/api/llm/model/text.based.question.evaluate.model"; +import { Logger } from "winston"; +import { GRADING_AUDIT_SERVICE } from "../../attempt.constants"; import { GradingAuditService } from "../../services/question-response/grading-audit.service"; import { GradingContext } from "../interfaces/grading-context.interface"; import { LocalizationService } from "../utils/localization.service"; @@ -13,12 +25,25 @@ import { AbstractGradingStrategy } from "./abstract-grading.strategy"; @Injectable() export class TextGradingStrategy extends AbstractGradingStrategy { + protected readonly logger: Logger; + constructor( - private readonly llmFacadeService: LlmFacadeService, + protected readonly llmFacadeService: LlmFacadeService, protected readonly localizationService: LocalizationService, + @Inject(GRADING_AUDIT_SERVICE) protected readonly gradingAuditService: GradingAuditService, + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, + @Optional() + @Inject(GRADING_JUDGE_SERVICE) + protected readonly gradingJudgeService?: IGradingJudgeService, ) { - super(localizationService, gradingAuditService); + super( + localizationService, + gradingAuditService, + undefined, // Don't inject consistency service to avoid DI conflicts + gradingJudgeService, + parentLogger, + ); } /** @@ -52,33 +77,61 @@ export class TextGradingStrategy extends AbstractGradingStrategy { } /** - * Grade the text response using LLM + * Grade the text response using LLM (judge validation is handled within the LLM service) */ async gradeResponse( question: QuestionDto, learnerResponse: string, context: GradingContext, ): Promise { - const textBasedQuestionEvaluateModel = new TextBasedQuestionEvaluateModel( - question.question, - context.questionAnswerContext, - context.assignmentInstructions, - learnerResponse, - question.totalPoints, - question.scoring?.type ?? "", - question.scoring, - question.responseType ?? "OTHER", - ); + try { + // Create evaluation model + const textBasedQuestionEvaluateModel = new TextBasedQuestionEvaluateModel( + question.question, + context.questionAnswerContext, + context.assignmentInstructions, + learnerResponse, + question.totalPoints, + question.scoring?.type ?? "", + question.scoring, + question.responseType ?? "OTHER", + ); - const gradingModel = await this.llmFacadeService.gradeTextBasedQuestion( - textBasedQuestionEvaluateModel, - context.assignmentId, - context.language, - ); + // Get grading from LLM (includes internal judge validation with retry logic) + const gradingModel = await this.llmFacadeService.gradeTextBasedQuestion( + textBasedQuestionEvaluateModel, + context.assignmentId, + context.language, + ); - const responseDto = new CreateQuestionResponseAttemptResponseDto(); - AttemptHelper.assignFeedbackToResponse(gradingModel, responseDto); + const responseDto = new CreateQuestionResponseAttemptResponseDto(); + AttemptHelper.assignFeedbackToResponse(gradingModel, responseDto); - return responseDto; + // Record grading for audit + await this.recordGrading( + question, + { + learnerTextResponse: learnerResponse, + } as CreateQuestionResponseAttemptRequestDto, + responseDto, + context, + "TextGradingStrategy", + ); + + // Add strategy metadata + responseDto.metadata = { + ...responseDto.metadata, + strategyUsed: "TextGradingStrategy", + }; + + return responseDto; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.error(`Error in text grading: ${errorMessage}`); + throw new BadRequestException( + `Failed to grade text response: ${errorMessage}`, + ); + } } } diff --git a/apps/api/src/api/attempt/common/strategies/true-false-grading.strategy.ts b/apps/api/src/api/attempt/common/strategies/true-false-grading.strategy.ts index 91635129..ee969e40 100644 --- a/apps/api/src/api/attempt/common/strategies/true-false-grading.strategy.ts +++ b/apps/api/src/api/attempt/common/strategies/true-false-grading.strategy.ts @@ -1,11 +1,19 @@ /* eslint-disable @typescript-eslint/require-await */ -import { BadRequestException, Injectable } from "@nestjs/common"; +import { + BadRequestException, + Inject, + Injectable, + Optional, +} from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; import { Choice, QuestionDto, } from "src/api/assignment/dto/update.questions.request.dto"; +import { Logger } from "winston"; +import { GRADING_AUDIT_SERVICE } from "../../attempt.constants"; import { GradingAuditService } from "../../services/question-response/grading-audit.service"; import { GradingContext } from "../interfaces/grading-context.interface"; import { LocalizationService } from "../utils/localization.service"; @@ -15,9 +23,17 @@ import { AbstractGradingStrategy } from "./abstract-grading.strategy"; export class TrueFalseGradingStrategy extends AbstractGradingStrategy { constructor( protected readonly localizationService: LocalizationService, + @Inject(GRADING_AUDIT_SERVICE) protected readonly gradingAuditService: GradingAuditService, + @Optional() @Inject(WINSTON_MODULE_PROVIDER) parentLogger?: Logger, ) { - super(localizationService, gradingAuditService); + super( + localizationService, + gradingAuditService, + undefined, + undefined, + parentLogger, + ); } /** @@ -28,7 +44,7 @@ export class TrueFalseGradingStrategy extends AbstractGradingStrategy { requestDto: CreateQuestionResponseAttemptRequestDto, ): Promise { if ( - requestDto.learnerAnswerChoice === null && + requestDto.learnerAnswerChoice === null || requestDto.learnerAnswerChoice === undefined ) { throw new BadRequestException( @@ -98,14 +114,6 @@ export class TrueFalseGradingStrategy extends AbstractGradingStrategy { ), ); } - if (correctAnswer === undefined) { - throw new BadRequestException( - this.localizationService.getLocalizedString( - "missingCorrectAnswer", - context.language, - ), - ); - } const isCorrect = learnerResponse === correctAnswer; const feedback = isCorrect @@ -149,6 +157,17 @@ export class TrueFalseGradingStrategy extends AbstractGradingStrategy { awardedPoints: pointsAwarded, }; + // Record grading for audit and consistency + await this.recordGrading( + question, + { + learnerAnswerChoice: learnerResponse, + } as CreateQuestionResponseAttemptRequestDto, + responseDto, + context, + "TrueFalseGradingStrategy", + ); + return responseDto; } diff --git a/apps/api/src/api/attempt/common/strategies/url-grading.strategy.ts b/apps/api/src/api/attempt/common/strategies/url-grading.strategy.ts index 848717c3..fb81b42f 100644 --- a/apps/api/src/api/attempt/common/strategies/url-grading.strategy.ts +++ b/apps/api/src/api/attempt/common/strategies/url-grading.strategy.ts @@ -2,15 +2,23 @@ /* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/require-await */ -import { BadRequestException, Injectable } from "@nestjs/common"; +import { + BadRequestException, + Inject, + Injectable, + Optional, +} from "@nestjs/common"; import axios from "axios"; import * as cheerio from "cheerio"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; import { AttemptHelper } from "src/api/assignment/attempt/helper/attempts.helper"; import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; import { LlmFacadeService } from "src/api/llm/llm-facade.service"; import { UrlBasedQuestionEvaluateModel } from "src/api/llm/model/url.based.question.evaluate.model"; +import { Logger } from "winston"; +import { GRADING_AUDIT_SERVICE } from "../../attempt.constants"; import { GradingAuditService } from "../../services/question-response/grading-audit.service"; import { GradingContext } from "../interfaces/grading-context.interface"; import { LocalizationService } from "../utils/localization.service"; @@ -21,9 +29,17 @@ export class UrlGradingStrategy extends AbstractGradingStrategy { constructor( private readonly llmFacadeService: LlmFacadeService, protected readonly localizationService: LocalizationService, + @Inject(GRADING_AUDIT_SERVICE) protected readonly gradingAuditService: GradingAuditService, + @Optional() @Inject(WINSTON_MODULE_PROVIDER) parentLogger?: Logger, ) { - super(localizationService, gradingAuditService); + super( + localizationService, + gradingAuditService, + undefined, + undefined, + parentLogger, + ); } /** @@ -103,6 +119,16 @@ export class UrlGradingStrategy extends AbstractGradingStrategy { status: "error", }; + await this.recordGrading( + question, + { + learnerUrlResponse: learnerResponse, + } as CreateQuestionResponseAttemptRequestDto, + responseDto, + context, + "UrlGradingStrategy-Failed", + ); + return responseDto; } @@ -138,6 +164,17 @@ export class UrlGradingStrategy extends AbstractGradingStrategy { gradingModel.gradingRationale || "URL content evaluated", }; + // Record grading for audit and consistency (successful case) + await this.recordGrading( + question, + { + learnerUrlResponse: learnerResponse, + } as CreateQuestionResponseAttemptRequestDto, + responseDto, + context, + "UrlGradingStrategy-Success", + ); + return responseDto; } diff --git a/apps/api/src/api/attempt/services/attempt-feedback.service.ts b/apps/api/src/api/attempt/services/attempt-feedback.service.ts index b9b6dad6..67a9305c 100644 --- a/apps/api/src/api/attempt/services/attempt-feedback.service.ts +++ b/apps/api/src/api/attempt/services/attempt-feedback.service.ts @@ -1,15 +1,15 @@ import { + BadRequestException, + ForbiddenException, Injectable, NotFoundException, - ForbiddenException, - BadRequestException, } from "@nestjs/common"; -import { PrismaService } from "../../../prisma.service"; -import { UserSession } from "../../../auth/interfaces/user.session.interface"; import { AssignmentFeedbackDto, AssignmentFeedbackResponseDto, } from "src/api/assignment/attempt/dto/assignment-attempt/feedback.request.dto"; +import { UserSession } from "../../../auth/interfaces/user.session.interface"; +import { PrismaService } from "../../../prisma.service"; @Injectable() export class AttemptFeedbackService { diff --git a/apps/api/src/api/attempt/services/attempt-grading.service.ts b/apps/api/src/api/attempt/services/attempt-grading.service.ts index 95adecb8..9dedfe88 100644 --- a/apps/api/src/api/attempt/services/attempt-grading.service.ts +++ b/apps/api/src/api/attempt/services/attempt-grading.service.ts @@ -1,7 +1,6 @@ import { Injectable } from "@nestjs/common"; import { Assignment } from "@prisma/client"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; -import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; @Injectable() export class AttemptGradingService { diff --git a/apps/api/src/api/attempt/services/attempt-regrading.service.ts b/apps/api/src/api/attempt/services/attempt-regrading.service.ts index c98aa8fd..2117d192 100644 --- a/apps/api/src/api/attempt/services/attempt-regrading.service.ts +++ b/apps/api/src/api/attempt/services/attempt-regrading.service.ts @@ -1,17 +1,17 @@ import { + BadRequestException, + ForbiddenException, Injectable, NotFoundException, - ForbiddenException, - BadRequestException, } from "@nestjs/common"; -import { PrismaService } from "../../../prisma.service"; import { RegradingStatus } from "@prisma/client"; -import { UserSession } from "../../../auth/interfaces/user.session.interface"; import { RegradingRequestDto, - RequestRegradingResponseDto, RegradingStatusResponseDto, + RequestRegradingResponseDto, } from "src/api/assignment/attempt/dto/assignment-attempt/feedback.request.dto"; +import { UserSession } from "../../../auth/interfaces/user.session.interface"; +import { PrismaService } from "../../../prisma.service"; @Injectable() export class AttemptRegradingService { diff --git a/apps/api/src/api/attempt/services/attempt-reporting.service.ts b/apps/api/src/api/attempt/services/attempt-reporting.service.ts index 93e0c886..02e87a3e 100644 --- a/apps/api/src/api/attempt/services/attempt-reporting.service.ts +++ b/apps/api/src/api/attempt/services/attempt-reporting.service.ts @@ -3,8 +3,8 @@ import { NotFoundException, UnprocessableEntityException, } from "@nestjs/common"; -import { PrismaService } from "../../../prisma.service"; import { ReportType } from "@prisma/client"; +import { PrismaService } from "../../../prisma.service"; @Injectable() export class AttemptReportingService { diff --git a/apps/api/src/api/attempt/services/attempt-submission.service.ts b/apps/api/src/api/attempt/services/attempt-submission.service.ts index 8b7683f7..f1f53d54 100644 --- a/apps/api/src/api/attempt/services/attempt-submission.service.ts +++ b/apps/api/src/api/attempt/services/attempt-submission.service.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable unicorn/no-null */ import { HttpService } from "@nestjs/axios"; import { @@ -5,7 +6,14 @@ import { InternalServerErrorException, NotFoundException, } from "@nestjs/common"; -import { AssignmentAttempt, Question } from "@prisma/client"; +import { + AssignmentAttempt, + AssignmentQuestionDisplayOrder, + Question, + QuestionType, + QuestionVariant, + ResponseType, +} from "@prisma/client"; import { JsonValue } from "@prisma/client/runtime/library"; import { GRADE_SUBMISSION_EXCEPTION } from "src/api/assignment/attempt/api-exceptions/exceptions"; import { BaseAssignmentAttemptResponseDto } from "src/api/assignment/attempt/dto/assignment-attempt/base.assignment.attempt.response.dto"; @@ -74,28 +82,87 @@ export class AttemptSubmissionService { const attemptExpiresAt = this.calculateAttemptExpiresAt(assignment); + // Get the assignment with its active version for version-aware attempt creation + const assignmentWithActiveVersion = await this.prisma.assignment.findUnique( + { + where: { id: assignmentId }, + include: { + currentVersion: { + include: { + questionVersions: true, + }, + }, + // Fallback to legacy questions if no active version exists + questions: { + where: { isDeleted: false }, + include: { + variants: { + where: { isDeleted: false }, + }, + }, + }, + }, + }, + ); + + if (!assignmentWithActiveVersion) { + throw new NotFoundException( + `Assignment with Id ${assignmentId} not found.`, + ); + } + + const activeVersionId = assignmentWithActiveVersion.currentVersionId; + const assignmentAttempt = await this.prisma.assignmentAttempt.create({ data: { expiresAt: attemptExpiresAt, submitted: false, assignmentId, + assignmentVersionId: activeVersionId, // Store the active version ID grade: undefined, userId: userSession.userId, questionOrder: [], }, }); - const questions = (await this.prisma.question.findMany({ - where: { - assignmentId, - isDeleted: false, - }, - include: { - variants: { - where: { isDeleted: false }, - }, - }, - })) as unknown as QuestionDto[]; + // Use questions from active version if available, otherwise fallback to legacy questions + const questions: QuestionDto[] = + assignmentWithActiveVersion?.currentVersion?.questionVersions?.length > 0 + ? assignmentWithActiveVersion.currentVersion.questionVersions.map( + (qv) => ({ + id: qv.questionId || qv.id, // Use questionId if available, fallback to qv.id + question: qv.question, + type: qv.type, + assignmentId: assignmentId, // Set from parameter since QuestionVersion doesn't have this + totalPoints: qv.totalPoints, + maxWords: qv.maxWords, + maxCharacters: qv.maxCharacters, + choices: qv.choices as unknown as Choice[], + scoring: qv.scoring as unknown as ScoringDto, + answer: qv.answer, + variants: [], // QuestionVersions don't have variants, use empty array + gradingContextQuestionIds: qv.gradingContextQuestionIds, + responseType: qv.responseType, + isDeleted: false, // QuestionVersions are not deleted by definition + randomizedChoices: qv.randomizedChoices, + videoPresentationConfig: + qv.videoPresentationConfig as unknown as VideoPresentationConfig, + liveRecordingConfig: qv.liveRecordingConfig as object, + }), + ) + : ((assignmentWithActiveVersion?.questions || []).map((q) => ({ + ...q, + scoring: q.scoring as unknown as ScoringDto, + choices: q.choices as unknown as Choice[], + videoPresentationConfig: + q.videoPresentationConfig as unknown as VideoPresentationConfig, + liveRecordingConfig: q.liveRecordingConfig as object, + variants: (q.variants || []).map((v: QuestionVariant) => ({ + ...v, + choices: v.choices as unknown as Choice[], + scoring: v.scoring as unknown as ScoringDto, + })), + })) as QuestionDto[]); // match number of questions to the assignment settings numberOfQuestionsPerAttempt if ( @@ -195,7 +262,7 @@ export class AttemptSubmissionService { request: UserSessionRequest, progressCallback?: (progress: string, percentage?: number) => Promise, ): Promise { - const { role, userId } = request.userSession; + const { role } = request.userSession; if (role === UserRole.LEARNER) { return this.updateLearnerAttempt( attemptId, @@ -231,6 +298,11 @@ export class AttemptSubmissionService { questionVariants: { include: { questionVariant: { include: { variantOf: true } } }, }, + assignmentVersion: { + include: { + questionVersions: true, + }, + }, }, }); @@ -240,9 +312,31 @@ export class AttemptSubmissionService { ); } - const questions = await this.prisma.question.findMany({ - where: { assignmentId: assignmentAttempt.assignmentId }, - }); + // Use version-specific questions if available, otherwise fallback to legacy questions + const questions: unknown[] = + assignmentAttempt.assignmentVersionId && + assignmentAttempt.assignmentVersion?.questionVersions?.length > 0 + ? assignmentAttempt.assignmentVersion.questionVersions.map((qv) => ({ + id: qv.questionId || qv.id, + question: qv.question, + type: qv.type, + assignmentId: assignmentAttempt.assignmentId, + totalPoints: qv.totalPoints, + maxWords: qv.maxWords, + maxCharacters: qv.maxCharacters, + choices: qv.choices, + scoring: qv.scoring, + answer: qv.answer, + gradingContextQuestionIds: qv.gradingContextQuestionIds, + responseType: qv.responseType, + isDeleted: false, + randomizedChoices: qv.randomizedChoices, + videoPresentationConfig: qv.videoPresentationConfig, + liveRecordingConfig: qv.liveRecordingConfig, + })) + : await this.prisma.question.findMany({ + where: { assignmentId: assignmentAttempt.assignmentId }, + }); const assignment = await this.prisma.assignment.findUnique({ where: { id: assignmentAttempt.assignmentId }, @@ -255,6 +349,7 @@ export class AttemptSubmissionService { showSubmissionFeedback: true, showQuestionScore: true, showQuestions: true, + showCorrectAnswer: true, }, }); @@ -264,48 +359,75 @@ export class AttemptSubmissionService { ); } - const questionDtos: EnhancedAttemptQuestionDto[] = questions.map((q) => ({ - id: q.id, - question: q.question, - type: q.type, - assignmentId: q.assignmentId, - totalPoints: q.totalPoints, - maxWords: q.maxWords || undefined, - maxCharacters: q.maxCharacters || undefined, - choices: this.parseJsonValue(q.choices, []), - scoring: this.parseJsonValue(q.scoring, { - type: ScoringType.CRITERIA_BASED, - showRubricsToLearner: false, - rubrics: [], - }), - answer: - typeof q.answer === "boolean" - ? String(q.answer) - : q.answer !== null && q.answer !== undefined - ? String(q.answer) - : undefined, - gradingContextQuestionIds: q.gradingContextQuestionIds || [], - responseType: q.responseType || undefined, - isDeleted: q.isDeleted, - randomizedChoices: - typeof q.randomizedChoices === "string" - ? q.randomizedChoices - : JSON.stringify(q.randomizedChoices ?? false), - videoPresentationConfig: - this.parseJsonValue( - q.videoPresentationConfig, - null, - ), - liveRecordingConfig: this.parseJsonValue | null>( - q.liveRecordingConfig, - null, - ), - })); + const questionDtos: EnhancedAttemptQuestionDto[] = questions.map((q) => { + const question = q as Record; + + const answerValue = + typeof question.answer === "boolean" + ? String(question.answer) + : question.answer !== null && question.answer !== undefined + ? String(question.answer) + : undefined; + + const randomizedChoicesValue: string = + typeof question.randomizedChoices === "string" + ? question.randomizedChoices + : JSON.stringify(question.randomizedChoices ?? false); + + return { + id: question.id as number, + question: question.question as string, + type: question.type as QuestionType, + assignmentId: question.assignmentId as number, + totalPoints: question.totalPoints as number, + maxWords: (question.maxWords as number) || undefined, + maxCharacters: (question.maxCharacters as number) || undefined, + choices: this.parseJsonValue(question.choices, []), + scoring: this.parseJsonValue(question.scoring, { + type: ScoringType.CRITERIA_BASED, + showRubricsToLearner: false, + rubrics: [], + }), + answer: answerValue, + gradingContextQuestionIds: + (question.gradingContextQuestionIds as number[]) || [], + responseType: (question.responseType as ResponseType) || undefined, + isDeleted: question.isDeleted as boolean, + randomizedChoices: randomizedChoicesValue, + videoPresentationConfig: + this.parseJsonValue( + question.videoPresentationConfig, + null, + ), + liveRecordingConfig: this.parseJsonValue | null>(question.liveRecordingConfig, null), + }; + }); const formattedAttempt: AssignmentAttemptWithRelations = { ...assignmentAttempt, questionVariants: assignmentAttempt.questionVariants.map((qv) => ({ questionId: qv.questionId, + questionVariant: qv.questionVariant + ? { + ...qv.questionVariant, + answer: + typeof qv.questionVariant.answer === "boolean" + ? String(qv.questionVariant.answer) + : qv.questionVariant.answer, + variantOf: qv.questionVariant.variantOf + ? { + ...qv.questionVariant.variantOf, + answer: + typeof qv.questionVariant.variantOf.answer === "boolean" + ? String(qv.questionVariant.variantOf.answer) + : qv.questionVariant.variantOf.answer, + } + : undefined, + } + : null, randomizedChoices: typeof qv.randomizedChoices === "string" ? qv.randomizedChoices @@ -335,6 +457,7 @@ export class AttemptSubmissionService { showSubmissionFeedback: assignment.showSubmissionFeedback, showQuestions: assignment.showQuestions, showQuestionScore: assignment.showQuestionScore, + showCorrectAnswer: assignment.showCorrectAnswer, comments: assignmentAttempt.comments, }; } @@ -361,6 +484,11 @@ export class AttemptSubmissionService { }, }, }, + assignmentVersion: { + include: { + questionVersions: true, + }, + }, }, }); @@ -370,7 +498,7 @@ export class AttemptSubmissionService { ); } - const assignment = await this.prisma.assignment.findUnique({ + const assignment = (await this.prisma.assignment.findUnique({ where: { id: assignmentAttempt.assignmentId }, select: { questions: true, @@ -381,13 +509,50 @@ export class AttemptSubmissionService { showSubmissionFeedback: true, showQuestions: true, showQuestionScore: true, + showCorrectAnswer: true, }, - }); + })) as { + questions: Question[]; + questionOrder: number[]; + displayOrder: AssignmentQuestionDisplayOrder | null; + passingGrade: number; + showAssignmentScore: boolean; + showSubmissionFeedback: boolean; + showQuestions: boolean; + showQuestionScore: boolean; + showCorrectAnswer: boolean; + }; + + // Get version-specific questions for translation if available + const questionsForTranslation: QuestionDto[] = + assignmentAttempt.assignmentVersionId && + assignmentAttempt.assignmentVersion?.questionVersions?.length > 0 + ? (assignmentAttempt.assignmentVersion.questionVersions.map((qv) => ({ + id: qv.questionId || qv.id, + question: qv.question, + type: qv.type, + assignmentId: assignmentAttempt.assignmentId, + totalPoints: qv.totalPoints, + maxWords: qv.maxWords, + maxCharacters: qv.maxCharacters, + choices: qv.choices as unknown as Choice[], + scoring: qv.scoring as unknown as ScoringDto, + answer: qv.answer, + variants: [], + gradingContextQuestionIds: qv.gradingContextQuestionIds, + responseType: qv.responseType, + isDeleted: false, + randomizedChoices: qv.randomizedChoices, + videoPresentationConfig: + qv.videoPresentationConfig as unknown as VideoPresentationConfig, + liveRecordingConfig: qv.liveRecordingConfig as object, + })) as QuestionDto[]) + : (assignment.questions as unknown as QuestionDto[]); const translations = await this.translationService.getTranslationsForAttempt( assignmentAttempt, - assignment.questions as unknown as QuestionDto[], + questionsForTranslation, ); const formattedAttempt: AssignmentAttemptWithRelations = { @@ -425,7 +590,7 @@ export class AttemptSubmissionService { normalizedLanguage, ); - this.removeSensitiveData(finalQuestions); + this.removeSensitiveData(finalQuestions, assignment); return { ...assignmentAttempt, @@ -435,6 +600,7 @@ export class AttemptSubmissionService { showSubmissionFeedback: assignment.showSubmissionFeedback, showQuestionScore: assignment.showQuestionScore, showQuestions: assignment.showQuestions, + showCorrectAnswer: assignment.showCorrectAnswer, }; } @@ -572,6 +738,7 @@ export class AttemptSubmissionService { grade: assignment.showAssignmentScore ? result.grade : undefined, showQuestions: assignment.showQuestions, showSubmissionFeedback: assignment.showSubmissionFeedback, + showCorrectAnswer: assignment.showCorrectAnswer, feedbacksForQuestions: this.gradingService.constructFeedbacksForQuestions( successfulQuestionResponses, @@ -658,6 +825,7 @@ export class AttemptSubmissionService { grade: assignment.showAssignmentScore ? grade : undefined, showQuestions: assignment.showQuestions, showSubmissionFeedback: assignment.showSubmissionFeedback, + showCorrectAnswer: assignment.showCorrectAnswer, feedbacksForQuestions: this.gradingService.constructFeedbacksForQuestions( successfulQuestionResponses, @@ -701,6 +869,7 @@ export class AttemptSubmissionService { feedbacksForQuestions: [], message: "The attempt deadline has passed.", showQuestions: false, + showCorrectAnswer: false, }; } @@ -909,6 +1078,7 @@ export class AttemptSubmissionService { showSubmissionFeedback?: boolean; showQuestionScore?: boolean; showQuestions?: boolean; + showCorrectAnswer?: boolean; }, ): void { if (assignment.showAssignmentScore === false) { @@ -942,7 +1112,12 @@ export class AttemptSubmissionService { /** * Remove sensitive data from questions */ - private removeSensitiveData(questions: AttemptQuestionDto[]): void { + private removeSensitiveData( + questions: AttemptQuestionDto[], + assignment: { + showCorrectAnswer?: boolean; + }, + ): void { for (const question of questions) { if (!question.scoring?.showRubricsToLearner) { delete question.scoring?.rubrics; @@ -951,8 +1126,10 @@ export class AttemptSubmissionService { if (question.choices) { for (const choice of question.choices) { delete choice.points; - delete choice.isCorrect; - delete choice.feedback; + if (assignment.showCorrectAnswer === false) { + delete choice.isCorrect; + delete choice.feedback; + } } } @@ -962,8 +1139,10 @@ export class AttemptSubmissionService { if (translationObject?.translatedChoices) { for (const choice of translationObject.translatedChoices) { delete choice.points; - delete choice.isCorrect; - delete choice.feedback; + if (assignment.showCorrectAnswer === false) { + delete choice.isCorrect; + delete choice.feedback; + } } } } @@ -984,8 +1163,10 @@ export class AttemptSubmissionService { if (Array.isArray(randomizedArray)) { for (const choice of randomizedArray) { delete choice.points; - delete choice.isCorrect; - delete choice.feedback; + if (assignment.showCorrectAnswer === false) { + delete choice.isCorrect; + delete choice.feedback; + } } question.randomizedChoices = JSON.stringify(randomizedArray); } else { diff --git a/apps/api/src/api/attempt/services/attempt-validation.service.ts b/apps/api/src/api/attempt/services/attempt-validation.service.ts index fda249cf..81178e96 100644 --- a/apps/api/src/api/attempt/services/attempt-validation.service.ts +++ b/apps/api/src/api/attempt/services/attempt-validation.service.ts @@ -1,16 +1,16 @@ import { Injectable, UnprocessableEntityException } from "@nestjs/common"; -import { PrismaService } from "../../../prisma.service"; -import { UserSession } from "../../../auth/interfaces/user.session.interface"; import { IN_PROGRESS_SUBMISSION_EXCEPTION, - TIME_RANGE_ATTEMPTS_SUBMISSION_EXCEPTION_MESSAGE, MAX_ATTEMPTS_SUBMISSION_EXCEPTION_MESSAGE, SUBMISSION_DEADLINE_EXCEPTION_MESSAGE, + TIME_RANGE_ATTEMPTS_SUBMISSION_EXCEPTION_MESSAGE, } from "src/api/assignment/attempt/api-exceptions/exceptions"; import { GetAssignmentResponseDto, LearnerGetAssignmentResponseDto, } from "src/api/assignment/dto/get.assignment.response.dto"; +import { UserSession } from "../../../auth/interfaces/user.session.interface"; +import { PrismaService } from "../../../prisma.service"; @Injectable() export class AttemptValidationService { diff --git a/apps/api/src/api/attempt/services/attempt.service.ts b/apps/api/src/api/attempt/services/attempt.service.ts index 5765ae74..eca427ec 100644 --- a/apps/api/src/api/attempt/services/attempt.service.ts +++ b/apps/api/src/api/attempt/services/attempt.service.ts @@ -452,17 +452,6 @@ export class AttemptServiceV2 { }, 30_000); // Every 30 seconds try { - // You can send progress updates during the grading process - const sendProgress = (message: string) => { - response.write( - `data: ${JSON.stringify({ - status: "processing", - message, - })}\n\n`, - ); - }; - - // Call your existing submission service with progress callback const result = await this.submissionService.updateAssignmentAttempt( attemptId, assignmentId, diff --git a/apps/api/src/api/attempt/services/file-content-extraction.ts b/apps/api/src/api/attempt/services/file-content-extraction.ts index d1b220ed..2f8254c7 100644 --- a/apps/api/src/api/attempt/services/file-content-extraction.ts +++ b/apps/api/src/api/attempt/services/file-content-extraction.ts @@ -1,16 +1,16 @@ /* eslint-disable */ +import * as crypto from "crypto"; +import { Readable } from "node:stream"; +import * as path from "path"; import { BadRequestException, Injectable, Logger } from "@nestjs/common"; -import { S3Service } from "src/api/files/services/s3.service"; -import { LearnerFileUpload } from "../common/interfaces/attempt.interface"; -import pdfParse from "pdf-parse"; -import * as mammoth from "mammoth"; -import * as XLSX from "xlsx"; import csv from "csv-parser"; -import { Readable } from "node:stream"; +import * as mammoth from "mammoth"; +import pdfParse from "pdf-parse"; +import { S3Service } from "src/api/files/services/s3.service"; import * as unzipper from "unzipper"; +import * as XLSX from "xlsx"; import { parseStringPromise } from "xml2js"; -import * as path from "path"; -import * as crypto from "crypto"; +import { LearnerFileUpload } from "../common/interfaces/attempt.interface"; export interface ExtractedFileContent { filename: string; @@ -164,7 +164,9 @@ export class FileContentExtractionService { return { filename: file.filename, content: - `[ERROR extracting ${file.filename}: ${error instanceof Error ? error.message : "Unknown error"}]\n` + + `[ERROR extracting ${file.filename}: ${ + error instanceof Error ? error.message : "Unknown error" + }]\n` + `File type: ${file.fileType}\n` + `This file could not be processed, but it exists in the submission.`, fileType: file.fileType, @@ -615,7 +617,9 @@ export class FileContentExtractionService { const strings = this.extractStringsFromBinary(buffer); if (strings.length > 0) { return { - text: `[BINARY FILE: ${filename}]\nExtracted strings:\n${strings.join("\n")}`, + text: `[BINARY FILE: ${filename}]\nExtracted strings:\n${strings.join( + "\n", + )}`, encoding: "binary", extractedText: `Found ${strings.length} text strings`, }; @@ -845,8 +849,12 @@ export class FileContentExtractionService { extractedText += `Total Cells: ${notebook.cells?.length || 0}\n`; if (notebook.metadata) { - extractedText += `Language: ${notebook.metadata.language_info?.name || "Unknown"}\n`; - extractedText += `Kernel: ${notebook.metadata.kernelspec?.display_name || "Unknown"}\n`; + extractedText += `Language: ${ + notebook.metadata.language_info?.name || "Unknown" + }\n`; + extractedText += `Kernel: ${ + notebook.metadata.kernelspec?.display_name || "Unknown" + }\n`; } extractedText += "\n"; @@ -857,7 +865,9 @@ export class FileContentExtractionService { `outputs=${cell.outputs?.length || 0}`, ); - extractedText += `\n=== CELL ${index + 1} [${cell.cell_type.toUpperCase()}]`; + extractedText += `\n=== CELL ${ + index + 1 + } [${cell.cell_type.toUpperCase()}]`; if (cell.execution_count) { extractedText += ` [${cell.execution_count}]`; } @@ -1079,7 +1089,9 @@ export class FileContentExtractionService { const slideCount = Math.max(1, Math.floor(strings.length / 10)); return { - text: `[LEGACY PPT FILE]\nExtracted text fragments:\n${strings.join("\n")}`, + text: `[LEGACY PPT FILE]\nExtracted text fragments:\n${strings.join( + "\n", + )}`, extractedText: strings.join("\n"), additionalMetadata: { slideCount }, }; @@ -1206,7 +1218,9 @@ export class FileContentExtractionService { for (const entry of zip.files) { const size = entry.uncompressedSize || 0; - extractedContent += `${entry.path} (${this.formatFileSize(size)})\n`; + extractedContent += `${entry.path} (${this.formatFileSize( + size, + )})\n`; fileList.push(entry.path); if (this.isTextFile(entry.path) && size < 100000) { @@ -1228,7 +1242,9 @@ export class FileContentExtractionService { } } catch (error) { extractedContent += `Error reading archive: ${error}\n`; - extractedContent += `Archive size: ${this.formatFileSize(buffer.length)}\n`; + extractedContent += `Archive size: ${this.formatFileSize( + buffer.length, + )}\n`; } } else { extractedContent += `Archive type: ${type}\n`; @@ -1310,7 +1326,9 @@ export class FileContentExtractionService { summary += `Length: ${parsed.length}\n`; } else { summary += `Keys: ${Object.keys(parsed).length}\n`; - summary += `Top-level keys: ${Object.keys(parsed).slice(0, 10).join(", ")}`; + summary += `Top-level keys: ${Object.keys(parsed) + .slice(0, 10) + .join(", ")}`; if (Object.keys(parsed).length > 10) { summary += "..."; } @@ -1333,7 +1351,9 @@ export class FileContentExtractionService { const objects = lines.map((line) => JSON.parse(line)); return { - text: `=== JSONL DATA ===\nLines: ${objects.length}\n\n${JSON.stringify(objects, null, 2)}`, + text: `=== JSONL DATA ===\nLines: ${ + objects.length + }\n\n${JSON.stringify(objects, null, 2)}`, extractedText: JSON.stringify(objects, null, 2), encoding: "utf8", }; @@ -1782,9 +1802,21 @@ export class FileContentExtractionService { const id3v1 = buffer.slice(-128); if (id3v1.slice(0, 3).toString() === "TAG") { result += "\nID3v1 Tags found:\n"; - result += `Title: ${id3v1.slice(3, 33).toString().replace(/\0/g, "").trim()}\n`; - result += `Artist: ${id3v1.slice(33, 63).toString().replace(/\0/g, "").trim()}\n`; - result += `Album: ${id3v1.slice(63, 93).toString().replace(/\0/g, "").trim()}\n`; + result += `Title: ${id3v1 + .slice(3, 33) + .toString() + .replace(/\0/g, "") + .trim()}\n`; + result += `Artist: ${id3v1 + .slice(33, 63) + .toString() + .replace(/\0/g, "") + .trim()}\n`; + result += `Album: ${id3v1 + .slice(63, 93) + .toString() + .replace(/\0/g, "") + .trim()}\n`; result += `Year: ${id3v1.slice(93, 97).toString()}\n`; } } @@ -2481,7 +2513,9 @@ export class FileContentExtractionService { ? XLSX.utils.decode_range(worksheet["!ref"]) : null; if (range) { - allText += `Range: ${worksheet["!ref"]} (${range.e.r - range.s.r + 1} rows × ${range.e.c - range.s.c + 1} cols)\n\n`; + allText += `Range: ${worksheet["!ref"]} (${ + range.e.r - range.s.r + 1 + } rows × ${range.e.c - range.s.c + 1} cols)\n\n`; } const csvText = XLSX.utils.sheet_to_csv(worksheet, { diff --git a/apps/api/src/api/attempt/services/grading-factory.service.ts b/apps/api/src/api/attempt/services/grading-factory.service.ts index 4b5de0fc..5618acd2 100644 --- a/apps/api/src/api/attempt/services/grading-factory.service.ts +++ b/apps/api/src/api/attempt/services/grading-factory.service.ts @@ -1,7 +1,6 @@ /* eslint-disable unicorn/no-null */ import { Injectable } from "@nestjs/common"; import { QuestionType } from "@prisma/client"; -import { ImageGradingService } from "src/api/llm/features/grading/services/image-grading.service"; import { IGradingStrategy } from "../common/interfaces/grading-strategy.interface"; import { ChoiceGradingStrategy } from "../common/strategies/choice-grading.strategy"; import { FileGradingStrategy } from "../common/strategies/file-grading.strategy"; diff --git a/apps/api/src/api/attempt/services/question-response/grading-audit.service.ts b/apps/api/src/api/attempt/services/question-response/grading-audit.service.ts index d973c192..3bfe4eb0 100644 --- a/apps/api/src/api/attempt/services/question-response/grading-audit.service.ts +++ b/apps/api/src/api/attempt/services/question-response/grading-audit.service.ts @@ -1,7 +1,9 @@ /* eslint-disable unicorn/no-null */ -import { Injectable } from "@nestjs/common"; +import { Inject, Injectable } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { CreateQuestionResponseAttemptResponseDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.response.dto"; +import { Logger } from "winston"; import { PrismaService } from "../../../../prisma.service"; /** @@ -26,14 +28,34 @@ export interface GradingIssue { */ @Injectable() export class GradingAuditService { - constructor(private readonly prisma: PrismaService) {} + private readonly logger: Logger; + + constructor( + private readonly prisma: PrismaService, + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, + ) { + this.logger = parentLogger.child({ + context: GradingAuditService.name, + }); + } /** * Record a grading action for audit purposes * @param record The grading record to store */ + /** + * Record grading for audit purposes (non-blocking) + * This method will not throw errors to prevent grading failures + */ async recordGrading(record: GradingAuditRecord): Promise { try { + this.logger.info("Recording grading audit", { + questionId: record.questionId, + assignmentId: record.assignmentId, + gradingStrategy: record.gradingStrategy, + metadata: record.metadata, + }); + await this.prisma.gradingAudit.create({ data: { questionId: record.questionId, @@ -45,8 +67,33 @@ export class GradingAuditService { timestamp: new Date(), }, }); - } catch { - console.error("Error recording grading audit:", record); + + this.logger.info("Successfully recorded grading audit", { + questionId: record.questionId, + assignmentId: record.assignmentId, + gradingStrategy: record.gradingStrategy, + }); + } catch (error) { + // Log error but don't throw to prevent grading failures + this.logger.error( + "Failed to record grading audit - continuing grading process", + { + error: error instanceof Error ? error.message : String(error), + questionId: record.questionId, + assignmentId: record.assignmentId, + gradingStrategy: record.gradingStrategy, + stack: error instanceof Error ? error.stack : undefined, + record: { + questionId: record.questionId, + assignmentId: record.assignmentId, + gradingStrategy: record.gradingStrategy, + // Don't log full payloads in error to avoid log spam + }, + }, + ); + + // TODO: Consider adding alerting/monitoring for audit failures + // For now, we continue without throwing to not break grading } } @@ -175,4 +222,163 @@ export class GradingAuditService { return issues; } + + /** + * Get grading architecture usage statistics + */ + async getGradingUsageStatistics(timeRange?: { + from: Date; + to: Date; + }): Promise<{ + totalGradings: number; + strategiesByCount: { strategy: string; count: number }[]; + gradingsByDay: { date: string; count: number }[]; + averagePointsAwarded: number; + mostActiveQuestions: { questionId: number; count: number }[]; + errorRate: number; + }> { + const whereClause = timeRange + ? { + timestamp: { + gte: timeRange.from, + lte: timeRange.to, + }, + } + : {}; + + try { + // Get total gradings + const totalGradings = await this.prisma.gradingAudit.count({ + where: whereClause, + }); + + // Get strategies by count + const strategyCounts = await this.prisma.gradingAudit.groupBy({ + by: ["gradingStrategy"], + where: whereClause, + _count: { + id: true, + }, + orderBy: { + _count: { + id: "desc", + }, + }, + }); + + const strategiesByCount = strategyCounts.map((item) => ({ + strategy: item.gradingStrategy, + count: item._count.id, + })); + + // Get gradings by day (last 7 days) + const sevenDaysAgo = new Date(); + sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7); + + const dailyCounts = await this.prisma.gradingAudit.groupBy({ + by: ["timestamp"], + where: { + ...whereClause, + timestamp: { + gte: sevenDaysAgo, + }, + }, + _count: { + id: true, + }, + }); + + const gradingsByDay: { [key: string]: number } = {}; + for (const item of dailyCounts) { + const date = item.timestamp.toISOString().split("T")[0]; + gradingsByDay[date] = (gradingsByDay[date] || 0) + item._count.id; + } + + const gradingsByDayArray = Object.entries(gradingsByDay).map( + ([date, count]) => ({ + date, + count: count, + }), + ); + + // Calculate average points (this would need to parse response payloads) + const averagePointsAwarded = 0; // TODO: Implement by parsing responsePayload + + // Get most active questions + const questionCounts = await this.prisma.gradingAudit.groupBy({ + by: ["questionId"], + where: whereClause, + _count: { + id: true, + }, + orderBy: { + _count: { + id: "desc", + }, + }, + take: 10, + }); + + const mostActiveQuestions = questionCounts.map((item) => ({ + questionId: item.questionId, + count: item._count.id, + })); + + // Error rate (would need to track failures separately) + const errorRate = 0; // TODO: Implement error tracking + + this.logger.info("Generated grading usage statistics", { + totalGradings, + strategiesCount: strategiesByCount.length, + timeRange, + }); + + return { + totalGradings, + strategiesByCount, + gradingsByDay: gradingsByDayArray, + averagePointsAwarded, + mostActiveQuestions, + errorRate, + }; + } catch (error) { + this.logger.error("Failed to generate grading usage statistics", { + error: error instanceof Error ? error.message : String(error), + timeRange, + }); + throw error; + } + } + + /** + * Log architecture usage summary (call this periodically) + */ + async logArchitectureUsageSummary(): Promise { + try { + const stats = await this.getGradingUsageStatistics(); + + this.logger.info("=== GRADING ARCHITECTURE USAGE SUMMARY ===", { + totalGradingsRecorded: stats.totalGradings, + activeStrategies: stats.strategiesByCount.length, + topStrategies: stats.strategiesByCount.slice(0, 3), + mostActiveQuestions: stats.mostActiveQuestions.slice(0, 3), + recentActivity: + stats.gradingsByDay.length > 0 ? "Active" : "No recent activity", + }); + + if (stats.totalGradings === 0) { + this.logger.warn( + "⚠️ NO GRADING AUDIT RECORDS FOUND - Grading may not be working or strategies are not calling recordGrading", + ); + } else { + this.logger.info( + "✅ Grading architecture is actively being used and recorded", + ); + } + } catch (error) { + this.logger.error("Failed to log architecture usage summary", { + error: error instanceof Error ? error.message : String(error), + }); + } + } } diff --git a/apps/api/src/api/attempt/services/question-response/question-response.service.ts b/apps/api/src/api/attempt/services/question-response/question-response.service.ts index 07de0884..e504b1f1 100644 --- a/apps/api/src/api/attempt/services/question-response/question-response.service.ts +++ b/apps/api/src/api/attempt/services/question-response/question-response.service.ts @@ -5,6 +5,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { BadRequestException, + Inject, Injectable, InternalServerErrorException, NotFoundException, @@ -12,6 +13,7 @@ import { import { QuestionType } from "@prisma/client"; import axios from "axios"; import * as cheerio from "cheerio"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { authorAssignmentDetailsDTO } from "src/api/assignment/attempt/dto/assignment-attempt/create.update.assignment.attempt.request.dto"; import { CreateQuestionResponseAttemptRequestDto } from "src/api/assignment/attempt/dto/question-response/create.question.response.attempt.request.dto"; import { @@ -21,6 +23,7 @@ import { import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; import { QuestionService } from "src/api/assignment/question/question.service"; import { QuestionAnswerContext } from "src/api/llm/model/base.question.evaluate.model"; +import { Logger } from "winston"; import { UserRole } from "../../../../auth/interfaces/user.session.interface"; import { PrismaService } from "../../../../prisma.service"; import { GradingContext } from "../../common/interfaces/grading-context.interface"; @@ -29,12 +32,19 @@ import { GradingFactoryService } from "../grading-factory.service"; @Injectable() export class QuestionResponseService { + private readonly logger: Logger; + constructor( private readonly prisma: PrismaService, private readonly questionService: QuestionService, private readonly localizationService: LocalizationService, private readonly gradingFactoryService: GradingFactoryService, - ) {} + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, + ) { + this.logger = parentLogger.child({ + context: QuestionResponseService.name, + }); + } /** * Submit all questions for an assignment attempt @@ -183,44 +193,112 @@ export class QuestionResponseService { gradingContext, )); } else { + const startTime = Date.now(); + + this.logger.info("Starting question grading process", { + questionId, + questionType: question.type, + responseType: question.responseType, + assignmentId, + assignmentAttemptId, + userRole: role, + language, + }); + const gradingStrategy = this.gradingFactoryService.getStrategy( question.type, question.responseType, ); if (!gradingStrategy) { + this.logger.error("No grading strategy found", { + questionId, + questionType: question.type, + responseType: question.responseType, + }); throw new BadRequestException( `No grading strategy found for question type: ${question.type}`, ); } + this.logger.info("Using grading strategy", { + questionId, + strategyName: gradingStrategy.constructor.name, + questionType: question.type, + responseType: question.responseType, + }); + + // Validate response + this.logger.debug("Validating response", { questionId }); const isValid = await gradingStrategy.validateResponse( question, createQuestionResponseAttemptRequestDto, ); + if (!isValid) { + this.logger.warn("Response validation failed", { + questionId, + language: createQuestionResponseAttemptRequestDto.language, + strategyName: gradingStrategy.constructor.name, + }); throw new BadRequestException( `Invalid response for question ID ${questionId}: ${createQuestionResponseAttemptRequestDto.language}`, ); } + + // Extract learner response + this.logger.debug("Extracting learner response", { questionId }); learnerResponse = await gradingStrategy.extractLearnerResponse( createQuestionResponseAttemptRequestDto, ); + + // Grade the response + this.logger.info("Grading response with strategy", { + questionId, + strategyName: gradingStrategy.constructor.name, + }); + responseDto = await gradingStrategy.gradeResponse( question, learnerResponse, gradingContext, ); + if (!responseDto) { + this.logger.error("Strategy returned null/undefined response", { + questionId, + strategyName: gradingStrategy.constructor.name, + }); throw new BadRequestException( `Failed to grade response for question ID ${questionId}`, ); } + + const duration = Date.now() - startTime; + this.logger.info("Successfully completed question grading", { + questionId, + strategyName: gradingStrategy.constructor.name, + totalPoints: responseDto.totalPoints, + maxPoints: question.totalPoints, + duration, + feedback: responseDto.feedback ? "present" : "missing", + }); } } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error("Failed to process question response", { + questionId, + questionType: question.type, + responseType: question.responseType, + assignmentId, + assignmentAttemptId, + userRole: role, + error: errorMessage, + stack: error instanceof Error ? error.stack : undefined, + }); + throw new BadRequestException( `Failed to process question response: ${errorMessage}`, ); @@ -560,6 +638,9 @@ export class QuestionResponseService { } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); + console.error( + `Error fetching URL content for question ${contextQuestion.id}: ${errorMessage}`, + ); } } @@ -644,7 +725,7 @@ export class QuestionResponseService { } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); - + console.error(`Error parsing JSON field: ${errorMessage}`); return null; } } diff --git a/apps/api/src/api/attempt/services/question-variant/question-variant.service.ts b/apps/api/src/api/attempt/services/question-variant/question-variant.service.ts index 502d95ac..5a2d8807 100644 --- a/apps/api/src/api/attempt/services/question-variant/question-variant.service.ts +++ b/apps/api/src/api/attempt/services/question-variant/question-variant.service.ts @@ -1,8 +1,8 @@ /* eslint-disable unicorn/no-null */ import { Injectable } from "@nestjs/common"; -import { PrismaService } from "../../../../prisma.service"; -import { Choice } from "src/api/assignment/question/dto/create.update.question.request.dto"; import { QuestionDto } from "src/api/assignment/dto/update.questions.request.dto"; +import { Choice } from "src/api/assignment/question/dto/create.update.question.request.dto"; +import { PrismaService } from "../../../../prisma.service"; @Injectable() export class QuestionVariantService { diff --git a/apps/api/src/api/attempt/services/translation/translation.service.ts b/apps/api/src/api/attempt/services/translation/translation.service.ts index 2745c636..add6fdfd 100644 --- a/apps/api/src/api/attempt/services/translation/translation.service.ts +++ b/apps/api/src/api/attempt/services/translation/translation.service.ts @@ -1,19 +1,20 @@ /* eslint-disable unicorn/no-null */ import { Injectable } from "@nestjs/common"; -import { PrismaService } from "../../../../prisma.service"; -import { - QuestionDto, - Choice, - ScoringDto, -} from "src/api/assignment/dto/update.questions.request.dto"; -import { QuestionService } from "src/api/assignment/question/question.service"; import { AssignmentAttempt, QuestionVariant, Translation, } from "@prisma/client"; import { QuestionResponse } from "src/api/assignment/attempt/dto/assignment-attempt/create.update.assignment.attempt.request.dto"; +import { + Choice, + QuestionDto, + ScoringDto, +} from "src/api/assignment/dto/update.questions.request.dto"; +import { QuestionService } from "src/api/assignment/question/question.service"; +import { PrismaService } from "../../../../prisma.service"; import { TranslatedContent } from "../../common/utils/attempt-questions-mapper.util"; + export type VariantMapping = { questionId: number; questionVariant: QuestionVariant | null; @@ -244,10 +245,7 @@ export class TranslationService { if (typeof field === "string") { try { return JSON.parse(field); - } catch (error: unknown) { - const errorMessage = - error instanceof Error ? error.message : "Unknown error"; - + } catch { return null; } } diff --git a/apps/api/src/api/files/files.controller.ts b/apps/api/src/api/files/files.controller.ts index 5b0bf8e0..1e395e95 100644 --- a/apps/api/src/api/files/files.controller.ts +++ b/apps/api/src/api/files/files.controller.ts @@ -1,3 +1,4 @@ +/* eslint-disable */ import { BadRequestException, Body, @@ -77,6 +78,168 @@ export class FilesController { ); } + @Post("direct-upload") + @UseGuards(AuthGuard) + @UseInterceptors( + FileInterceptor("file", { + storage: memoryStorage(), + limits: { + fileSize: 10 * 1024 * 1024, // 10MB limit + }, + }), + ) + @ApiOperation({ + summary: "Direct upload file through backend (bypasses CORS)", + }) + async directUpload( + @UploadedFile() file: Express.Multer.File, + @Body() body: any, + @Req() request: UserSessionRequest, + ) { + if (!file) { + throw new BadRequestException("No file provided"); + } + + // Parse multipart form data manually + console.log("[DIRECT UPLOAD] Received body:", body); + const uploadType = body.uploadType; + let context: any = {}; + + // Parse context JSON if provided + if (body.context) { + console.log( + "[DIRECT UPLOAD] Raw context value:", + body.context, + "type:", + typeof body.context, + ); + try { + context = + typeof body.context === "string" + ? JSON.parse(body.context) + : body.context; + console.log("[DIRECT UPLOAD] Parsed context:", context); + console.log( + "[DIRECT UPLOAD] reportId from context:", + context.reportId, + "type:", + typeof context.reportId, + ); + } catch (error) { + console.error( + "[DIRECT UPLOAD] Failed to parse context:", + body.context, + error, + ); + throw new BadRequestException("Invalid context JSON"); + } + } + + const userId = request.userSession.userId; + + // Generate the appropriate bucket and key using the same logic as generateUploadUrl + const bucket = this.s3Service.getBucketName(uploadType); + console.log("[DIRECT UPLOAD] Upload type:", uploadType); + console.log("[DIRECT UPLOAD] Resolved bucket:", bucket); + console.log("[DIRECT UPLOAD] All bucket env vars:"); + console.log(" - IBM_COS_DEBUG_BUCKET:", process.env.IBM_COS_DEBUG_BUCKET); + console.log( + " - IBM_COS_AUTHOR_BUCKET:", + process.env.IBM_COS_AUTHOR_BUCKET, + ); + console.log( + " - IBM_COS_LEARNER_BUCKET:", + process.env.IBM_COS_LEARNER_BUCKET, + ); + + // Also test bucket resolution for all types + try { + console.log("[DIRECT UPLOAD] Testing bucket resolution:"); + console.log(" - debug bucket:", this.s3Service.getBucketName("debug")); + console.log(" - author bucket:", this.s3Service.getBucketName("author")); + console.log( + " - learner bucket:", + this.s3Service.getBucketName("learner"), + ); + } catch (resolutionError) { + console.error( + "[DIRECT UPLOAD] Bucket resolution error:", + resolutionError, + ); + } + + if (!bucket) { + throw new BadRequestException("Invalid upload type"); + } + + // Skip bucket access test - let the actual upload handle any bucket issues + console.log( + `[DIRECT UPLOAD] Proceeding with upload to bucket "${bucket}"...`, + ); + + // Use the same prefix generation logic from FilesService + let prefix = ""; + const normalizedPath = context.path?.startsWith("/") + ? context.path.slice(1) + : (context.path ?? ""); + + switch (uploadType) { + case "author": { + prefix = normalizedPath ? `${normalizedPath}/` : `authors/${userId}/`; + break; + } + case "learner": { + if (typeof context.assignmentId !== "number") { + throw new BadRequestException( + "Missing assignmentId in context for learner upload", + ); + } + if (typeof context.questionId !== "number") { + throw new BadRequestException( + "Missing questionId in context for learner upload", + ); + } + prefix = normalizedPath + ? `${normalizedPath}/` + : `${context.assignmentId}/${userId}/${context.questionId}/`; + break; + } + case "debug": { + if (typeof context.reportId !== "number") { + throw new BadRequestException( + "Missing reportId in context for debug upload", + ); + } + prefix = normalizedPath + ? `${normalizedPath}/` + : `debug/${context.reportId}/`; + break; + } + default: { + throw new BadRequestException("Invalid upload type"); + } + } + + // Generate unique key + const uniqueId = + Date.now().toString(36) + Math.random().toString(36).slice(2); + const key = `${prefix}${uniqueId}-${file.originalname}`; + + // Direct upload through backend + const result = await this.filesService.directUpload(file, bucket, key); + + return { + success: true, + key, + bucket, + fileType: file.mimetype, + fileName: file.originalname, + uploadType, + size: file.size, + etag: result.etag, + }; + } + @Get("access") @ApiOperation({ summary: "Get direct file access URLs using presigned URLs" }) @ApiQuery({ name: "key", required: true, description: "File key in storage" }) diff --git a/apps/api/src/api/files/files.module.ts b/apps/api/src/api/files/files.module.ts index 28747d1f..23a7bb64 100644 --- a/apps/api/src/api/files/files.module.ts +++ b/apps/api/src/api/files/files.module.ts @@ -2,10 +2,11 @@ import { Module } from "@nestjs/common"; import { MulterModule } from "@nestjs/platform-express"; import { memoryStorage } from "multer"; // ← import this + +import { PrismaService } from "src/prisma.service"; import { FilesController } from "./files.controller"; import { FilesService } from "./services/files.service"; import { S3Service } from "./services/s3.service"; -import { PrismaService } from "src/prisma.service"; @Module({ imports: [ diff --git a/apps/api/src/api/files/services/files.service.ts b/apps/api/src/api/files/services/files.service.ts index e2ce7c45..2f9cbf0e 100644 --- a/apps/api/src/api/files/services/files.service.ts +++ b/apps/api/src/api/files/services/files.service.ts @@ -126,14 +126,8 @@ export class FilesService { bucket: string, key: string, ): Promise { - try { - await this.s3Service.headBucket({ Bucket: bucket }); - } catch { - throw new NotFoundException( - "Bucket does not exist or you do not have permission to access it", - ); - } - + // Skip bucket validation - let the actual S3 operation handle any bucket issues + // This matches the behavior of presigned URL uploads which work fine const result = await this.s3Service.putObject({ Bucket: bucket, Key: key, @@ -496,7 +490,7 @@ export class FilesService { if (MIME_TYPES[extension]) return MIME_TYPES[extension]; } - const lastExtension = parts.pop()!; + const lastExtension = parts.pop(); return MIME_TYPES[lastExtension] ?? "application/octet-stream"; } } diff --git a/apps/api/src/api/github/github.controller.ts b/apps/api/src/api/github/github.controller.ts index 90a9a6df..f44f2077 100644 --- a/apps/api/src/api/github/github.controller.ts +++ b/apps/api/src/api/github/github.controller.ts @@ -4,9 +4,7 @@ import { Get, HttpException, HttpStatus, - Param, Post, - Query, Req, } from "@nestjs/common"; import { ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; diff --git a/apps/api/src/api/llm/core/interfaces/llm-provider.interface.ts b/apps/api/src/api/llm/core/interfaces/llm-provider.interface.ts index 9da368a3..0b5b88df 100644 --- a/apps/api/src/api/llm/core/interfaces/llm-provider.interface.ts +++ b/apps/api/src/api/llm/core/interfaces/llm-provider.interface.ts @@ -1,4 +1,4 @@ -import { AIMessage, HumanMessage } from "@langchain/core/messages"; +import { HumanMessage } from "@langchain/core/messages"; export interface LlmRequestOptions { temperature?: number; diff --git a/apps/api/src/api/llm/core/interfaces/prompt-processor.interface.ts b/apps/api/src/api/llm/core/interfaces/prompt-processor.interface.ts index 9f14d186..65b204a2 100644 --- a/apps/api/src/api/llm/core/interfaces/prompt-processor.interface.ts +++ b/apps/api/src/api/llm/core/interfaces/prompt-processor.interface.ts @@ -2,6 +2,17 @@ import { PromptTemplate } from "@langchain/core/prompts"; import { AIUsageType } from "@prisma/client"; export interface IPromptProcessor { + /** + * Process a prompt using assigned model for a specific feature + */ + processPromptForFeature( + prompt: PromptTemplate, + assignmentId: number, + usageType: AIUsageType, + featureKey: string, + fallbackModel?: string, + ): Promise; + /** * Process a text prompt and return the LLM response */ diff --git a/apps/api/src/api/llm/core/interfaces/token-counter.interface.ts b/apps/api/src/api/llm/core/interfaces/token-counter.interface.ts index 210bea94..edee3ccc 100644 --- a/apps/api/src/api/llm/core/interfaces/token-counter.interface.ts +++ b/apps/api/src/api/llm/core/interfaces/token-counter.interface.ts @@ -1,6 +1,8 @@ export interface ITokenCounter { /** * Count the number of tokens in a text + * @param text The text to count tokens for + * @param modelKey Optional model key for model-specific tokenization */ - countTokens(text: string): number; + countTokens(text: string, modelKey?: string): number; } diff --git a/apps/api/src/api/llm/core/interfaces/user-tracking.interface.ts b/apps/api/src/api/llm/core/interfaces/user-tracking.interface.ts index a1f5f7ac..b11c4429 100644 --- a/apps/api/src/api/llm/core/interfaces/user-tracking.interface.ts +++ b/apps/api/src/api/llm/core/interfaces/user-tracking.interface.ts @@ -9,5 +9,6 @@ export interface IUsageTracker { usageType: AIUsageType, tokensIn: number, tokensOut: number, + modelKey?: string, ): Promise; } diff --git a/apps/api/src/api/llm/core/services/gpt5-llm.service.ts b/apps/api/src/api/llm/core/services/gpt5-llm.service.ts new file mode 100644 index 00000000..2242e497 --- /dev/null +++ b/apps/api/src/api/llm/core/services/gpt5-llm.service.ts @@ -0,0 +1,170 @@ +import { HumanMessage } from "@langchain/core/messages"; +import { ChatOpenAI } from "@langchain/openai"; +import { Inject, Injectable } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { Logger } from "winston"; +import { TOKEN_COUNTER } from "../../llm.constants"; +import { + ILlmProvider, + LlmRequestOptions, + LlmResponse, +} from "../interfaces/llm-provider.interface"; +import { ITokenCounter } from "../interfaces/token-counter.interface"; + +/** + * GPT-5 provider service targeting the next-generation GPT-5 model. + * This service implements the ILlmProvider interface and provides + * enhanced capabilities and performance compared to GPT-4 models. + */ +@Injectable() +export class Gpt5LlmService implements ILlmProvider { + private readonly logger: Logger; + static readonly DEFAULT_MODEL = "gpt-5"; + readonly key = "gpt-5"; + + constructor( + @Inject(TOKEN_COUNTER) private readonly tokenCounter: ITokenCounter, + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, + ) { + this.logger = parentLogger.child({ context: Gpt5LlmService.name }); + } + + /** + * Create a ChatOpenAI instance with the given options + */ + private createChatModel(options?: LlmRequestOptions): ChatOpenAI { + return new ChatOpenAI({ + temperature: options?.temperature ?? 0.5, + modelName: options?.modelName ?? Gpt5LlmService.DEFAULT_MODEL, + maxTokens: options?.maxTokens, + }); + } + + /** + * Send a request to GPT-5 and get a response + */ + async invoke( + messages: HumanMessage[], + options?: LlmRequestOptions, + ): Promise { + const model = this.createChatModel(options); + + const inputText = messages + .map((m) => + typeof m.content === "string" ? m.content : JSON.stringify(m.content), + ) + .join("\n"); + const inputTokens = this.tokenCounter.countTokens(inputText); + + this.logger.debug(`Invoking GPT-5 with ${inputTokens} input tokens`); + + try { + const result = await model.invoke(messages); + const responseContent = result.content.toString(); + const outputTokens = this.tokenCounter.countTokens(responseContent); + + this.logger.debug(`GPT-5 responded with ${outputTokens} output tokens`); + + return { + content: responseContent, + tokenUsage: { + input: inputTokens, + output: outputTokens, + }, + }; + } catch (error) { + this.logger.error( + `GPT-5 API error: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + throw error; + } + } + + /** + * Send a request with image content to GPT-5 + */ + async invokeWithImage( + textContent: string, + imageData: string, + options?: LlmRequestOptions, + ): Promise { + const model = this.createChatModel(options); + + const processedImageData = this.normalizeImageData(imageData); + const inputTokens = this.tokenCounter.countTokens(textContent); + + // GPT-5 has improved image token estimation + const estimatedImageTokens = 200; + + this.logger.debug( + `Invoking GPT-5 with image (${inputTokens} text tokens + ~${estimatedImageTokens} image tokens)`, + ); + + try { + const result = await model.invoke([ + new HumanMessage({ + content: [ + { type: "text", text: textContent }, + { + type: "image_url", + image_url: { + url: processedImageData, + detail: options?.imageDetail || "auto", + }, + }, + ], + }), + ]); + + const responseContent = result.content.toString(); + const outputTokens = this.tokenCounter.countTokens(responseContent); + + this.logger.debug( + `GPT-5 with image responded with ${outputTokens} output tokens`, + ); + + return { + content: responseContent, + tokenUsage: { + input: inputTokens + estimatedImageTokens, + output: outputTokens, + }, + }; + } catch (error) { + this.logger.error( + `Error processing image with GPT-5: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + throw error; + } + } + + /** + * Normalize image data to ensure it has the correct format + */ + private normalizeImageData(imageData: string): string { + if (!imageData) { + throw new Error("Image data is empty or null"); + } + + if (imageData.startsWith("data:")) { + return imageData; + } + + let mimeType = "image/jpeg"; + if (imageData.startsWith("/9j/")) { + mimeType = "image/jpeg"; + } else if (imageData.startsWith("iVBORw0KGgo")) { + mimeType = "image/png"; + } else if (imageData.startsWith("R0lGOD")) { + mimeType = "image/gif"; + } else if (imageData.startsWith("UklGR")) { + mimeType = "image/webp"; + } + + return `data:${mimeType};base64,${imageData}`; + } +} diff --git a/apps/api/src/api/llm/core/services/gpt5-mini-llm.service.ts b/apps/api/src/api/llm/core/services/gpt5-mini-llm.service.ts new file mode 100644 index 00000000..b2e799bf --- /dev/null +++ b/apps/api/src/api/llm/core/services/gpt5-mini-llm.service.ts @@ -0,0 +1,171 @@ +import { HumanMessage } from "@langchain/core/messages"; +import { ChatOpenAI } from "@langchain/openai"; +import { Inject, Injectable } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { Logger } from "winston"; +import { TOKEN_COUNTER } from "../../llm.constants"; +import { + ILlmProvider, + LlmRequestOptions, + LlmResponse, +} from "../interfaces/llm-provider.interface"; +import { ITokenCounter } from "../interfaces/token-counter.interface"; + +/** + * GPT-5-mini provider service targeting the lightweight/faster GPT-5-mini model. + * This service offers enhanced performance over GPT-4o-mini with better efficiency + * and cost-effectiveness for simpler tasks. + */ +@Injectable() +export class Gpt5MiniLlmService implements ILlmProvider { + private readonly logger: Logger; + static readonly DEFAULT_MODEL = "gpt-5-mini"; + readonly key = "gpt-5-mini"; + + constructor( + @Inject(TOKEN_COUNTER) private readonly tokenCounter: ITokenCounter, + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, + ) { + this.logger = parentLogger.child({ context: Gpt5MiniLlmService.name }); + } + + /** + * Create a ChatOpenAI instance with the given options + */ + private createChatModel(options?: LlmRequestOptions): ChatOpenAI { + return new ChatOpenAI({ + modelName: options?.modelName ?? Gpt5MiniLlmService.DEFAULT_MODEL, + maxCompletionTokens: options?.maxTokens ?? 4096, + }); + } + + /** + * Send a request to GPT-5-mini and get a response + */ + async invoke( + messages: HumanMessage[], + options?: LlmRequestOptions, + ): Promise { + const model = this.createChatModel(options); + + const inputText = messages + .map((m) => + typeof m.content === "string" ? m.content : JSON.stringify(m.content), + ) + .join("\n"); + const inputTokens = this.tokenCounter.countTokens(inputText); + + this.logger.debug(`Invoking GPT-5-mini with ${inputTokens} input tokens`); + + try { + const result = await model.invoke(messages); + const responseContent = result.content.toString(); + const outputTokens = this.tokenCounter.countTokens(responseContent); + + this.logger.debug( + `GPT-5-mini responded with ${outputTokens} output tokens`, + ); + + return { + content: responseContent, + tokenUsage: { + input: inputTokens, + output: outputTokens, + }, + }; + } catch (error) { + this.logger.error( + `GPT-5-mini API error: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + throw error; + } + } + + /** + * Send a request with image content to GPT-5-mini + */ + async invokeWithImage( + textContent: string, + imageData: string, + options?: LlmRequestOptions, + ): Promise { + const model = this.createChatModel(options); + + const processedImageData = this.normalizeImageData(imageData); + const inputTokens = this.tokenCounter.countTokens(textContent); + + // GPT-5-mini has optimized image token usage + const estimatedImageTokens = 150; + + this.logger.debug( + `Invoking GPT-5-mini with image (${inputTokens} text tokens + ~${estimatedImageTokens} image tokens)`, + ); + + try { + const result = await model.invoke([ + new HumanMessage({ + content: [ + { type: "text", text: textContent }, + { + type: "image_url", + image_url: { + url: processedImageData, + detail: options?.imageDetail || "auto", + }, + }, + ], + }), + ]); + + const responseContent = result.content.toString(); + const outputTokens = this.tokenCounter.countTokens(responseContent); + + this.logger.debug( + `GPT-5-mini with image responded with ${outputTokens} output tokens`, + ); + + return { + content: responseContent, + tokenUsage: { + input: inputTokens + estimatedImageTokens, + output: outputTokens, + }, + }; + } catch (error) { + this.logger.error( + `Error processing image with GPT-5-mini: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + throw error; + } + } + + /** + * Normalize image data to ensure it has the correct format + */ + private normalizeImageData(imageData: string): string { + if (!imageData) { + throw new Error("Image data is empty or null"); + } + + if (imageData.startsWith("data:")) { + return imageData; + } + + let mimeType = "image/jpeg"; + if (imageData.startsWith("/9j/")) { + mimeType = "image/jpeg"; + } else if (imageData.startsWith("iVBORw0KGgo")) { + mimeType = "image/png"; + } else if (imageData.startsWith("R0lGOD")) { + mimeType = "image/gif"; + } else if (imageData.startsWith("UklGR")) { + mimeType = "image/webp"; + } + + return `data:${mimeType};base64,${imageData}`; + } +} diff --git a/apps/api/src/api/llm/core/services/gpt5-nano-llm.service.ts b/apps/api/src/api/llm/core/services/gpt5-nano-llm.service.ts new file mode 100644 index 00000000..c17bb2ee --- /dev/null +++ b/apps/api/src/api/llm/core/services/gpt5-nano-llm.service.ts @@ -0,0 +1,174 @@ +import { HumanMessage } from "@langchain/core/messages"; +import { ChatOpenAI } from "@langchain/openai"; +import { Inject, Injectable } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { Logger } from "winston"; +import { TOKEN_COUNTER } from "../../llm.constants"; +import { + ILlmProvider, + LlmRequestOptions, + LlmResponse, +} from "../interfaces/llm-provider.interface"; +import { ITokenCounter } from "../interfaces/token-counter.interface"; + +/** + * GPT-5-nano provider service targeting the ultra-lightweight GPT-5-nano model. + * This service is optimized for simple tasks requiring maximum speed and + * cost-effectiveness, ideal for basic text processing and quick responses. + */ +@Injectable() +export class Gpt5NanoLlmService implements ILlmProvider { + private readonly logger: Logger; + static readonly DEFAULT_MODEL = "gpt-5-nano"; + readonly key = "gpt-5-nano"; + + constructor( + @Inject(TOKEN_COUNTER) private readonly tokenCounter: ITokenCounter, + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, + ) { + this.logger = parentLogger.child({ context: Gpt5NanoLlmService.name }); + } + + /** + * Create a ChatOpenAI instance with the given options + */ + private createChatModel(options?: LlmRequestOptions): ChatOpenAI { + const config: any = { + modelName: options?.modelName ?? Gpt5NanoLlmService.DEFAULT_MODEL, + }; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + return new ChatOpenAI(config); + } + + /** + * Send a request to GPT-5-nano and get a response + */ + async invoke( + messages: HumanMessage[], + options?: LlmRequestOptions, + ): Promise { + const model = this.createChatModel(options); + + const inputText = messages + .map((m) => + typeof m.content === "string" ? m.content : JSON.stringify(m.content), + ) + .join("\n"); + const inputTokens = this.tokenCounter.countTokens(inputText); + + this.logger.debug(`Invoking GPT-5-nano with ${inputTokens} input tokens`); + + try { + const result = await model.invoke(messages); + const responseContent = result.content.toString(); + const outputTokens = this.tokenCounter.countTokens(responseContent); + + this.logger.debug( + `GPT-5-nano responded with ${outputTokens} output tokens`, + ); + + return { + content: responseContent, + tokenUsage: { + input: inputTokens, + output: outputTokens, + }, + }; + } catch (error) { + this.logger.error( + `GPT-5-nano API error: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + throw error; + } + } + + /** + * Send a request with image content to GPT-5-nano + * Note: GPT-5-nano has limited image processing capabilities + */ + async invokeWithImage( + textContent: string, + imageData: string, + options?: LlmRequestOptions, + ): Promise { + const model = this.createChatModel(options); + + const processedImageData = this.normalizeImageData(imageData); + const inputTokens = this.tokenCounter.countTokens(textContent); + + // GPT-5-nano has minimal image token usage for cost optimization + const estimatedImageTokens = 100; + + this.logger.debug( + `Invoking GPT-5-nano with image (${inputTokens} text tokens + ~${estimatedImageTokens} image tokens)`, + ); + + try { + const result = await model.invoke([ + new HumanMessage({ + content: [ + { type: "text", text: textContent }, + { + type: "image_url", + image_url: { + url: processedImageData, + detail: options?.imageDetail || "low", // Default to low detail for nano + }, + }, + ], + }), + ]); + + const responseContent = result.content.toString(); + const outputTokens = this.tokenCounter.countTokens(responseContent); + + this.logger.debug( + `GPT-5-nano with image responded with ${outputTokens} output tokens`, + ); + + return { + content: responseContent, + tokenUsage: { + input: inputTokens + estimatedImageTokens, + output: outputTokens, + }, + }; + } catch (error) { + this.logger.error( + `Error processing image with GPT-5-nano: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + throw error; + } + } + + /** + * Normalize image data to ensure it has the correct format + */ + private normalizeImageData(imageData: string): string { + if (!imageData) { + throw new Error("Image data is empty or null"); + } + + if (imageData.startsWith("data:")) { + return imageData; + } + + let mimeType = "image/jpeg"; + if (imageData.startsWith("/9j/")) { + mimeType = "image/jpeg"; + } else if (imageData.startsWith("iVBORw0KGgo")) { + mimeType = "image/png"; + } else if (imageData.startsWith("R0lGOD")) { + mimeType = "image/gif"; + } else if (imageData.startsWith("UklGR")) { + mimeType = "image/webp"; + } + + return `data:${mimeType};base64,${imageData}`; + } +} diff --git a/apps/api/src/api/llm/core/services/llm-assignment.service.ts b/apps/api/src/api/llm/core/services/llm-assignment.service.ts new file mode 100644 index 00000000..19251214 --- /dev/null +++ b/apps/api/src/api/llm/core/services/llm-assignment.service.ts @@ -0,0 +1,437 @@ +import { + forwardRef, + Inject, + Injectable, + Logger, + NotFoundException, +} from "@nestjs/common"; +import { AIFeatureType } from "@prisma/client"; +import { PrismaService } from "../../../../prisma.service"; +import { LLM_RESOLVER_SERVICE } from "../../llm.constants"; +import { LLMResolverService } from "./llm-resolver.service"; + +export interface FeatureAssignment { + id: number; + featureKey: string; + featureType: AIFeatureType; + displayName: string; + description?: string; + isActive: boolean; + requiresModel: boolean; + defaultModelKey?: string; + assignedModel?: { + id: number; + modelKey: string; + displayName: string; + provider: string; + priority: number; + assignedBy?: string; + assignedAt: Date; + }; +} + +export interface AssignmentRequest { + featureKey: string; + modelKey: string; + priority?: number; + assignedBy?: string; + metadata?: any; +} + +@Injectable() +export class LLMAssignmentService { + private readonly logger = new Logger(LLMAssignmentService.name); + + constructor( + private readonly prisma: PrismaService, + @Inject(forwardRef(() => LLM_RESOLVER_SERVICE)) + private readonly resolverService: LLMResolverService, + ) {} + + /** + * Get all AI features with their current model assignments + */ + async getAllFeatureAssignments(): Promise { + const features = await this.prisma.aIFeature.findMany({ + include: { + assignments: { + where: { isActive: true }, + orderBy: { priority: "desc" }, + take: 1, + include: { + model: true, + }, + }, + }, + orderBy: { displayName: "asc" }, + }); + + return features.map((feature) => ({ + id: feature.id, + featureKey: feature.featureKey, + featureType: feature.featureType, + displayName: feature.displayName, + description: feature.description, + isActive: feature.isActive, + requiresModel: feature.requiresModel, + defaultModelKey: feature.defaultModelKey, + assignedModel: feature.assignments[0] + ? { + id: feature.assignments[0].model.id, + modelKey: feature.assignments[0].model.modelKey, + displayName: feature.assignments[0].model.displayName, + provider: feature.assignments[0].model.provider, + priority: feature.assignments[0].priority, + assignedBy: feature.assignments[0].assignedBy, + assignedAt: feature.assignments[0].assignedAt, + } + : undefined, + })); + } + + /** + * Get the assigned model for a specific feature + */ + async getAssignedModel(featureKey: string): Promise { + const feature = await this.prisma.aIFeature.findUnique({ + where: { featureKey }, + include: { + assignments: { + where: { isActive: true }, + orderBy: { priority: "desc" }, + take: 1, + include: { model: true }, + }, + }, + }); + + if (!feature) { + this.logger.warn(`Feature ${featureKey} not found`); + return null; + } + + if (!feature.isActive) { + this.logger.warn(`Feature ${featureKey} is not active`); + return null; + } + + // Return assigned model or default model + if (feature.assignments.length > 0) { + return feature.assignments[0].model.modelKey; + } + + if (feature.defaultModelKey) { + this.logger.debug( + `Using default model ${feature.defaultModelKey} for feature ${featureKey}`, + ); + return feature.defaultModelKey; + } + + this.logger.warn( + `No model assigned to feature ${featureKey} and no default model`, + ); + return null; + } + + /** + * Assign a model to a feature + */ + async assignModelToFeature(request: AssignmentRequest): Promise { + const { + featureKey, + modelKey, + priority = 100, + assignedBy, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + metadata, + } = request; + + try { + // Find the feature + const feature = await this.prisma.aIFeature.findUnique({ + where: { featureKey }, + }); + + if (!feature) { + throw new NotFoundException(`Feature ${featureKey} not found`); + } + + // Find the model + const model = await this.prisma.lLMModel.findUnique({ + where: { modelKey }, + }); + + if (!model) { + throw new NotFoundException(`Model ${modelKey} not found`); + } + + if (!model.isActive) { + throw new Error(`Model ${modelKey} is not active`); + } + + // Check if an assignment already exists for this feature-model pair + const existingAssignment = + await this.prisma.lLMFeatureAssignment.findUnique({ + where: { + featureId_modelId: { + featureId: feature.id, + modelId: model.id, + }, + }, + }); + + await (existingAssignment + ? this.prisma.lLMFeatureAssignment.update({ + where: { id: existingAssignment.id }, + data: { + isActive: true, + priority, + assignedBy, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + metadata, + assignedAt: new Date(), + deactivatedAt: null, + }, + }) + : this.prisma.lLMFeatureAssignment.create({ + data: { + featureId: feature.id, + modelId: model.id, + isActive: true, + priority, + assignedBy, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + metadata, + }, + })); + + // Deactivate all other assignments for this feature + await this.prisma.lLMFeatureAssignment.updateMany({ + where: { + featureId: feature.id, + modelId: { not: model.id }, + isActive: true, + }, + data: { + isActive: false, + deactivatedAt: new Date(), + }, + }); + + // Clear cache for this feature since assignment changed + this.resolverService.clearCacheForFeature(featureKey); + + this.logger.log( + `Assigned model ${modelKey} to feature ${featureKey} by ${ + assignedBy || "system" + }`, + ); + return true; + } catch (error) { + this.logger.error( + `Failed to assign model ${modelKey} to feature ${featureKey}:`, + error, + ); + throw error; + } + } + + /** + * Remove model assignment from a feature (revert to default) + */ + async removeFeatureAssignment( + featureKey: string, + assignedBy?: string, + ): Promise { + try { + const feature = await this.prisma.aIFeature.findUnique({ + where: { featureKey }, + }); + + if (!feature) { + throw new NotFoundException(`Feature ${featureKey} not found`); + } + + // Deactivate current assignments + const result = await this.prisma.lLMFeatureAssignment.updateMany({ + where: { + featureId: feature.id, + isActive: true, + }, + data: { + isActive: false, + deactivatedAt: new Date(), + }, + }); + + // Clear cache for this feature since assignment was removed + this.resolverService.clearCacheForFeature(featureKey); + + this.logger.log( + `Removed model assignment for feature ${featureKey} by ${ + assignedBy || "system" + }`, + ); + return result.count > 0; + } catch (error) { + this.logger.error( + `Failed to remove assignment for feature ${featureKey}:`, + error, + ); + throw error; + } + } + + /** + * Get assignment history for a feature + */ + async getFeatureAssignmentHistory(featureKey: string, limit = 10) { + const feature = await this.prisma.aIFeature.findUnique({ + where: { featureKey }, + }); + + if (!feature) { + throw new NotFoundException(`Feature ${featureKey} not found`); + } + + return await this.prisma.lLMFeatureAssignment.findMany({ + where: { featureId: feature.id }, + include: { model: true }, + orderBy: { assignedAt: "desc" }, + take: limit, + }); + } + + /** + * Get all available models for assignment + */ + async getAvailableModels() { + return await this.prisma.lLMModel.findMany({ + where: { isActive: true }, + include: { + pricingHistory: { + where: { isActive: true }, + orderBy: { effectiveDate: "desc" }, + take: 1, + }, + featureAssignments: { + where: { isActive: true }, + include: { feature: true }, + }, + }, + orderBy: { displayName: "asc" }, + }); + } + + /** + * Get assignment statistics + */ + async getAssignmentStatistics() { + const totalFeatures = await this.prisma.aIFeature.count(); + const activeFeatures = await this.prisma.aIFeature.count({ + where: { isActive: true }, + }); + const featuresWithAssignments = await this.prisma.aIFeature.count({ + where: { + assignments: { + some: { isActive: true }, + }, + }, + }); + const featuresUsingDefaults = activeFeatures - featuresWithAssignments; + + const modelUsage = await this.prisma.lLMFeatureAssignment.groupBy({ + by: ["modelId"], + where: { isActive: true }, + _count: { featureId: true }, + orderBy: { _count: { featureId: "desc" } }, + }); + + const modelUsageWithNames = await Promise.all( + modelUsage.map(async (usage) => { + const model = await this.prisma.lLMModel.findUnique({ + where: { id: usage.modelId }, + select: { modelKey: true, displayName: true }, + }); + return { + modelKey: model?.modelKey || "unknown", + displayName: model?.displayName || "Unknown", + featureCount: usage._count.featureId, + }; + }), + ); + + return { + totalFeatures, + activeFeatures, + featuresWithAssignments, + featuresUsingDefaults, + modelUsage: modelUsageWithNames, + }; + } + + /** + * Bulk update feature assignments + */ + async bulkUpdateAssignments( + assignments: AssignmentRequest[], + assignedBy?: string, + ): Promise<{ success: number; failed: number; errors: string[] }> { + const results = { success: 0, failed: 0, errors: [] as string[] }; + + for (const assignment of assignments) { + try { + await this.assignModelToFeature({ + ...assignment, + assignedBy: assignedBy || assignment.assignedBy, + }); + results.success++; + } catch (error) { + results.failed++; + results.errors.push( + `${assignment.featureKey}: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + } + } + + this.logger.log( + `Bulk update completed: ${results.success} success, ${results.failed} failed`, + ); + return results; + } + + /** + * Reset all assignments to defaults + */ + async resetToDefaults(assignedBy?: string): Promise { + // Get all features with default models + const featuresWithDefaults = await this.prisma.aIFeature.findMany({ + where: { + defaultModelKey: { not: null }, + isActive: true, + }, + }); + + let resetCount = 0; + for (const feature of featuresWithDefaults) { + try { + await this.assignModelToFeature({ + featureKey: feature.featureKey, + modelKey: feature.defaultModelKey, + assignedBy: assignedBy || "SYSTEM_RESET", + }); + resetCount++; + } catch (error) { + this.logger.error( + `Failed to reset feature ${feature.featureKey}:`, + error, + ); + } + } + + this.logger.log(`Reset ${resetCount} features to default models`); + return resetCount; + } +} diff --git a/apps/api/src/api/llm/core/services/llm-pricing.service.ts b/apps/api/src/api/llm/core/services/llm-pricing.service.ts new file mode 100644 index 00000000..30564c5e --- /dev/null +++ b/apps/api/src/api/llm/core/services/llm-pricing.service.ts @@ -0,0 +1,1521 @@ +/* eslint-disable */ +import { Inject, Injectable, Logger } from "@nestjs/common"; +import { PricingSource } from "@prisma/client"; +import * as cheerio from "cheerio"; +import { PrismaService } from "../../../../prisma.service"; +import { LLM_RESOLVER_SERVICE } from "../../llm.constants"; +import { LLMResolverService } from "./llm-resolver.service"; + +export interface ModelPricing { + modelKey: string; + inputTokenPrice: number; + outputTokenPrice: number; + effectiveDate: Date; + source: PricingSource; + metadata?: any; +} + +export interface CostBreakdown { + inputTokens: number; + outputTokens: number; + inputCost: number; + outputCost: number; + totalCost: number; + modelKey: string; + pricingEffectiveDate: Date; + inputTokenPrice: number; + outputTokenPrice: number; +} + +// Web scraping types and constants +const OPENAI_PRICING_URL = "https://openai.com/api/pricing"; + +type ExtractResult = { + modelKey: string; + inputPerToken: number; // USD per token + outputPerToken: number; // USD per token + sourceUrl: string; + fetchedAt: string; + lastModified?: string | null; +}; + +// Helper functions for web scraping + +/** + * Helper functions for parsing pricing data + */ +function dollarsPerTokenFromPerMillion(perMillionUSD: number): number { + return perMillionUSD / 1_000_000; +} + +/** + * Strategy 1: Extract from structured data (JSON-LD, data attributes, etc.) + */ +async function extractFromStructuredData( + $: cheerio.CheerioAPI, + logger: Logger, +): Promise { + const results: ExtractResult[] = []; + + try { + // Look for JSON-LD structured data + const jsonLdScripts = $('script[type="application/ld+json"]'); + jsonLdScripts.each((i, elem) => { + try { + const jsonData = JSON.parse($(elem).text()); + if (jsonData && jsonData.offers && Array.isArray(jsonData.offers)) { + // Parse structured pricing data if available + jsonData.offers.forEach((offer: any) => { + if (offer.name && offer.price) { + const modelKey = normalizeModelName(offer.name); + if (modelKey && offer.inputPrice && offer.outputPrice) { + results.push({ + modelKey, + inputPerToken: offer.inputPrice, + outputPerToken: offer.outputPrice, + sourceUrl: OPENAI_PRICING_URL, + fetchedAt: new Date().toISOString(), + }); + } + } + }); + } + } catch { + // Skip invalid JSON + } + }); + + // Look for data attributes on pricing elements + const pricingElements = $( + "[data-model-name], [data-pricing], .pricing-card, .model-card", + ); + pricingElements.each((i, elem) => { + const $elem = $(elem); + const modelName = + $elem.attr("data-model-name") || + $elem.find("[data-model-name]").attr("data-model-name"); + const inputPrice = + $elem.attr("data-input-price") || + $elem.find("[data-input-price]").attr("data-input-price"); + const outputPrice = + $elem.attr("data-output-price") || + $elem.find("[data-output-price]").attr("data-output-price"); + + if (modelName && inputPrice && outputPrice) { + const modelKey = normalizeModelName(modelName); + if (modelKey) { + results.push({ + modelKey, + inputPerToken: Number.parseFloat(inputPrice), + outputPerToken: Number.parseFloat(outputPrice), + sourceUrl: OPENAI_PRICING_URL, + fetchedAt: new Date().toISOString(), + }); + } + } + }); + + logger.log(`Structured data extraction found ${results.length} models`); + return results; + } catch (error) { + logger.warn("Structured data extraction failed:", error); + return []; + } +} + +/** + * Strategy 2: Extract from pricing tables or card layouts + */ +async function extractFromPricingTables( + $: cheerio.CheerioAPI, + logger: Logger, +): Promise { + const results: ExtractResult[] = []; + + try { + // Look for table-based pricing + const tables = $("table, .pricing-table, .model-grid, .pricing-grid"); + tables.each((i, table) => { + const $table = $(table); + + // Look for rows or cards that contain model information + const rows = $table.find("tr, .pricing-card, .model-card, .pricing-row"); + rows.each((j, row) => { + const $row = $(row); + const text = $row.text().toLowerCase(); + + // Check if this row contains a known model + const modelKey = detectModelFromText(text); + if (modelKey) { + // Look for pricing patterns in this row + const prices = extractPricesFromElement($row); + if (prices.input && prices.output) { + results.push({ + modelKey, + inputPerToken: prices.input, + outputPerToken: prices.output, + sourceUrl: OPENAI_PRICING_URL, + fetchedAt: new Date().toISOString(), + }); + } + } + }); + }); + + // Look for card-based layouts + const cards = $( + '.card, .pricing-card, .model-card, [class*="pricing"], [class*="model"]', + ); + cards.each((i, card) => { + const $card = $(card); + const text = $card.text().toLowerCase(); + + const modelKey = detectModelFromText(text); + if (modelKey) { + const prices = extractPricesFromElement($card); + if (prices.input && prices.output) { + results.push({ + modelKey, + inputPerToken: prices.input, + outputPerToken: prices.output, + sourceUrl: OPENAI_PRICING_URL, + fetchedAt: new Date().toISOString(), + }); + } + } + }); + + logger.log(`Table/card extraction found ${results.length} models`); + return results; + } catch (error) { + logger.warn("Table/card extraction failed:", error); + return []; + } +} + +/** + * Strategy 3: Enhanced text pattern matching with multiple patterns + */ +async function extractFromTextPatterns( + $: cheerio.CheerioAPI, + logger: Logger, +): Promise { + const results: ExtractResult[] = []; + + try { + const bodyText = $("body").text(); + + // Enhanced patterns for different model types + const patterns = [ + // GPT-5 models + { + regex: + /gpt-?5(?:\s+|-)?(mini|nano)?\s*[^$]*?\$?([\d.]+)[^$]*?\$?([\d.]+)/gi, + baseModel: "gpt-5", + }, + // GPT-4o models - more flexible pattern matching + { + regex: + /gpt-?4o(?:\s+|-)?(mini)?\s*[^$]*?\$?([\d.]+)[^$]*?\$?([\d.]+)/gi, + baseModel: "gpt-4o", + }, + // Additional specific pattern for standalone gpt-4o + { + regex: + /(?:^|[^a-z])gpt.?4o(?:\s|\$|[^a-z-])[^$]*?\$?([\d.]+)[^$]*?\$?([\d.]+)/gi, + baseModel: "gpt-4o", + }, + // GPT-4.1 models + { + regex: + /gpt-?4\.1(?:\s+|-)?(mini|nano)?\s*[^$]*?\$?([\d.]+)[^$]*?\$?([\d.]+)/gi, + baseModel: "gpt-4.1", + }, + // o1/o3/o4 models + { + regex: + /o([1-4])(?:\s+|-)?(pro|mini|deep-research)?\s*[^$]*?\$?([\d.]+)[^$]*?\$?([\d.]+)/gi, + baseModel: "o", + }, + // More specific text patterns for common OpenAI page formats + { + regex: /textgpt-4o[^$]*?\$?([\d.]+)[^$]*?\$?([\d.]+)/gi, + baseModel: "gpt-4o", + }, + { + regex: /gpt-4o\s*text[^$]*?\$?([\d.]+)[^$]*?\$?([\d.]+)/gi, + baseModel: "gpt-4o", + }, + ]; + + for (const pattern of patterns) { + let match; + while ((match = pattern.regex.exec(bodyText)) !== null) { + const variant = match[1] || match[2]; // Handle different capture group positions + const inputPrice = Number.parseFloat(match.at(-2)); + const outputPrice = Number.parseFloat(match.at(-1)); + + if (!isNaN(inputPrice) && !isNaN(outputPrice)) { + let modelKey = pattern.baseModel; + if (pattern.baseModel === "o") { + modelKey = `o${match[1]}`; + if (variant && variant !== match[1]) { + modelKey += `-${variant}`; + } + } else if (variant) { + modelKey += `-${variant}`; + } + + // Convert from per-million to per-token pricing + results.push({ + modelKey: modelKey.toLowerCase(), + inputPerToken: dollarsPerTokenFromPerMillion(inputPrice), + outputPerToken: dollarsPerTokenFromPerMillion(outputPrice), + sourceUrl: OPENAI_PRICING_URL, + fetchedAt: new Date().toISOString(), + }); + } + } + } + + logger.log(`Text pattern extraction found ${results.length} models`); + return results; + } catch (error) { + logger.warn("Text pattern extraction failed:", error); + return []; + } +} + +/** + * Normalize model names to match our internal naming convention + */ +function normalizeModelName(name: string): string | null { + const normalized = name.toLowerCase().replaceAll(/[^\d.a-z-]/g, ""); + + const modelMap: Record = { + gpt5: "gpt-5", + gpt5mini: "gpt-5-mini", + gpt5nano: "gpt-5-nano", + gpt4o: "gpt-4o", + gpt4omini: "gpt-4o-mini", + gpt41mini: "gpt-4.1-mini", + gpt41nano: "gpt-4.1-nano", + o1pro: "o1-pro", + o1mini: "o1-mini", + o3pro: "o3-pro", + o3mini: "o3-mini", + o3deepresearch: "o3-deep-research", + o4mini: "o4-mini", + o4minideepresearch: "o4-mini-deep-research", + }; + + return modelMap[normalized] || normalized; +} + +/** + * Detect model name from text content + */ +function detectModelFromText(text: string): string | null { + const lowerText = text.toLowerCase(); + + // Check for specific model patterns - order matters for specificity + if (lowerText.includes("gpt-5") && lowerText.includes("nano")) + return "gpt-5-nano"; + if (lowerText.includes("gpt-5") && lowerText.includes("mini")) + return "gpt-5-mini"; + if (lowerText.includes("gpt-5")) return "gpt-5"; + + // GPT-4o - be more specific to catch standalone gpt-4o + if (lowerText.includes("gpt-4o") && lowerText.includes("mini")) + return "gpt-4o-mini"; + if (lowerText.match(/gpt-?4o(?!\w)/)) return "gpt-4o"; // Ensure not part of another word + if (lowerText.includes("gpt4o")) return "gpt-4o"; // Handle without hyphen + + if (lowerText.includes("gpt-4.1") && lowerText.includes("mini")) + return "gpt-4.1-mini"; + if (lowerText.includes("gpt-4.1") && lowerText.includes("nano")) + return "gpt-4.1-nano"; + if (lowerText.includes("gpt-4.1")) return "gpt-4.1"; + + if ( + lowerText.includes("o1-pro") || + (lowerText.includes("o1") && lowerText.includes("pro")) + ) + return "o1-pro"; + if ( + lowerText.includes("o1-mini") || + (lowerText.includes("o1") && lowerText.includes("mini")) + ) + return "o1-mini"; + if (lowerText.includes("o1")) return "o1"; + + if ( + lowerText.includes("o3-pro") || + (lowerText.includes("o3") && lowerText.includes("pro")) + ) + return "o3-pro"; + if ( + lowerText.includes("o3-mini") || + (lowerText.includes("o3") && lowerText.includes("mini")) + ) + return "o3-mini"; + if ( + lowerText.includes("o3-deep-research") || + (lowerText.includes("o3") && lowerText.includes("deep")) + ) + return "o3-deep-research"; + if (lowerText.includes("o3")) return "o3"; + + if (lowerText.includes("o4-mini-deep-research")) + return "o4-mini-deep-research"; + if ( + lowerText.includes("o4-mini") || + (lowerText.includes("o4") && lowerText.includes("mini")) + ) + return "o4-mini"; + + return null; +} + +/** + * Extract pricing numbers from a DOM element + */ +function extractPricesFromElement($elem: any): { + input?: number; + output?: number; +} { + const text = $elem.text(); + + // Look for common pricing patterns + const patterns = [ + /input[\s:]*\$?([\d.]+)[^$]*output[\s:]*\$?([\d.]+)/gi, + /\$?([\d.]+)[^$]*input[^$]*\$?([\d.]+)[^$]*output/gi, + /\$?([\d.]+)[^$]*\/[^$]*\$?([\d.]+)/gi, // $X.XX / $Y.YY format + /per\s+million.*?\$?([\d.]+)[^$]*\$?([\d.]+)/gi, + // Additional patterns for OpenAI's specific formatting + /\$?([\d.]+)[^$\d]{0,20}\$?([\d.]+)(?:\s|$)/gi, // Two prices close together + /(?:^|\s)\$?([\d.]+)[^$]*?(?:per|\/)[^$]*?\$?([\d.]+)/gi, // Price per something + ]; + + for (const pattern of patterns) { + const match = pattern.exec(text); + if (match) { + const price1 = Number.parseFloat(match[1]); + const price2 = Number.parseFloat(match[2]); + + if (!isNaN(price1) && !isNaN(price2)) { + // Assume first price is input, second is output (common convention) + return { + input: dollarsPerTokenFromPerMillion(price1), + output: dollarsPerTokenFromPerMillion(price2), + }; + } + } + } + + return {}; +} + +/** + * Strategy 4: Aggressive pattern matching for commonly missed models like gpt-4o + */ +async function extractWithAggressivePatterns( + $: cheerio.CheerioAPI, + logger: Logger, +): Promise { + const results: ExtractResult[] = []; + + try { + const bodyText = $("body").text(); + const cleanedText = bodyText.replace(/\s+/g, " ").toLowerCase(); + + // Very broad patterns to catch gpt-4o specifically + const aggressivePatterns = [ + // Look for "gpt-4o" followed by prices within reasonable distance + /gpt[-\s]?4o[^a-z][^$]*?\$?([\d.]+)[^$\d]{1,50}\$?([\d.]+)/gi, + // Look for specific page section patterns + /text[^$]*?gpt[-\s]?4o[^$]*?\$?([\d.]+)[^$\d]{1,50}\$?([\d.]+)/gi, + // Look for pricing table patterns + /gpt[-\s]?4o[^$]{0,100}([\d.]+)[^$\d]{1,50}([\d.]+)/gi, + // Very loose pattern for any two prices near "4o" + /4o[^$\d]{0,80}([\d.]+)[^$\d]{1,50}([\d.]+)/gi, + ]; + + for (const pattern of aggressivePatterns) { + let match; + while ((match = pattern.exec(cleanedText)) !== null) { + const price1 = parseFloat(match[1]); + const price2 = parseFloat(match[2]); + + if (!isNaN(price1) && !isNaN(price2) && price1 > 0 && price2 > 0) { + // Basic sanity check - input price should be less than output price typically + const inputPrice = price1 < price2 ? price1 : price2; + const outputPrice = price1 < price2 ? price2 : price1; + + // Only accept reasonable pricing ranges (OpenAI prices are usually in these ranges) + if ( + inputPrice >= 0.1 && + inputPrice <= 50 && + outputPrice >= 0.1 && + outputPrice <= 200 + ) { + results.push({ + modelKey: "gpt-4o", + inputPerToken: dollarsPerTokenFromPerMillion(inputPrice), + outputPerToken: dollarsPerTokenFromPerMillion(outputPrice), + sourceUrl: OPENAI_PRICING_URL, + fetchedAt: new Date().toISOString(), + }); + + logger.log( + `Aggressive pattern found gpt-4o: $${inputPrice}/$${outputPrice} per million`, + ); + break; // Only take the first reasonable match + } + } + } + + // If we found gpt-4o, also try to find gpt-4o-mini with similar patterns + if (results.length > 0) { + const miniPatterns = [ + /gpt[-\s]?4o[-\s]?mini[^$]*?\$?([\d.]+)[^$\d]{1,50}\$?([\d.]+)/gi, + /4o[-\s]?mini[^$]{0,80}([\d.]+)[^$\d]{1,50}([\d.]+)/gi, + ]; + + for (const miniPattern of miniPatterns) { + const miniMatch = miniPattern.exec(cleanedText); + if (miniMatch) { + const miniPrice1 = parseFloat(miniMatch[1]); + const miniPrice2 = parseFloat(miniMatch[2]); + + if ( + !isNaN(miniPrice1) && + !isNaN(miniPrice2) && + miniPrice1 > 0 && + miniPrice2 > 0 + ) { + const inputPrice = + miniPrice1 < miniPrice2 ? miniPrice1 : miniPrice2; + const outputPrice = + miniPrice1 < miniPrice2 ? miniPrice2 : miniPrice1; + + if ( + inputPrice >= 0.01 && + inputPrice <= 5 && + outputPrice >= 0.01 && + outputPrice <= 20 + ) { + results.push({ + modelKey: "gpt-4o-mini", + inputPerToken: dollarsPerTokenFromPerMillion(inputPrice), + outputPerToken: dollarsPerTokenFromPerMillion(outputPrice), + sourceUrl: OPENAI_PRICING_URL, + fetchedAt: new Date().toISOString(), + }); + + logger.log( + `Aggressive pattern found gpt-4o-mini: $${inputPrice}/$${outputPrice} per million`, + ); + break; + } + } + } + } + break; // If we found the main model, stop looking + } + } + + logger.log(`Aggressive patterns found ${results.length} models`); + return results; + } catch (error) { + logger.warn("Aggressive pattern extraction failed:", error); + return []; + } +} + +/** + * Parse pricing from OpenAI pricing page using multiple strategies for better reliability. + * Attempts structured data extraction first, falls back to text parsing if needed. + */ +async function scrapeAllPricingFromOpenAI(): Promise { + const logger = new Logger("LLMPricingScraper"); + + // Try multiple user agents to avoid detection + const userAgents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + ]; + + const randomUserAgent = + userAgents[Math.floor(Math.random() * userAgents.length)]; + + try { + // Add timeout and better headers + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 15_000); // 15 second timeout + + const res = await fetch(OPENAI_PRICING_URL, { + signal: controller.signal, + headers: { + "User-Agent": randomUserAgent, + Accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + DNT: "1", + Connection: "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Cache-Control": "max-age=0", + }, + }); + + clearTimeout(timeoutId); + + if (!res.ok) { + logger.warn( + `OpenAI pricing page returned status ${res.status}: ${res.statusText}`, + ); + return []; + } + + const html = await res.text(); + const $ = cheerio.load(html); + + // Try multiple extraction strategies + let results: ExtractResult[] = []; + + // Strategy 1: Look for JSON-LD or structured data + results = await extractFromStructuredData($, logger); + if (results.length > 0) { + logger.log( + `Successfully extracted ${results.length} models from structured data`, + ); + return results; + } + + // Strategy 2: Look for pricing tables or cards + results = await extractFromPricingTables($, logger); + if (results.length > 0) { + logger.log( + `Successfully extracted ${results.length} models from pricing tables`, + ); + return results; + } + + // Strategy 3: Enhanced text parsing with multiple patterns + results = await extractFromTextPatterns($, logger); + if (results.length > 0) { + logger.log( + `Successfully extracted ${results.length} models from text patterns`, + ); + return results; + } + + // Strategy 4: Aggressive fallback specifically for missing models + results = await extractWithAggressivePatterns($, logger); + if (results.length > 0) { + logger.log( + `Successfully extracted ${results.length} models using aggressive patterns`, + ); + return results; + } + + logger.warn("All extraction strategies failed, no pricing data found"); + return []; + } catch (error) { + logger.error("Error scraping OpenAI pricing:", error); + return []; + } +} + +/** + * Single entry point for web scraping a model's pricing with retry logic. + */ +async function resolveOneModelFromWeb( + modelKey: string, +): Promise { + const logger = new Logger("LLMPricingScraper"); + const maxRetries = 3; + const baseDelay = 2000; // 2 seconds + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + logger.log( + `Attempting to scrape pricing for ${modelKey} (attempt ${attempt}/${maxRetries})`, + ); + const allPricing = await scrapeAllPricingFromOpenAI(); + const result = allPricing.find((p) => p.modelKey === modelKey) || null; + + if (result) { + logger.log(`Successfully found pricing for ${modelKey}`); + return result; + } + + if (attempt < maxRetries) { + const delay = + baseDelay * Math.pow(2, attempt - 1) + Math.random() * 1000; // Exponential backoff with jitter + logger.warn( + `Model ${modelKey} not found in scraped data, retrying in ${delay}ms...`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } catch (error) { + logger.error( + `Error scraping pricing for ${modelKey} (attempt ${attempt}):`, + error, + ); + + if (attempt < maxRetries) { + const delay = + baseDelay * Math.pow(2, attempt - 1) + Math.random() * 1000; + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + } + + logger.warn( + `Failed to scrape pricing for ${modelKey} after ${maxRetries} attempts`, + ); + return null; +} + +@Injectable() +export class LLMPricingService { + private readonly logger = new Logger(LLMPricingService.name); + private pricingCache: Map< + string, + { data: ExtractResult[]; timestamp: number } + > = new Map(); + private readonly CACHE_TTL = 10 * 60 * 1000; // 10 minutes cache + + constructor( + private readonly prisma: PrismaService, + @Inject(LLM_RESOLVER_SERVICE) + private readonly llmResolverService: LLMResolverService, + ) {} + + /** + * Get cached pricing data or fetch fresh data if cache is expired + */ + private async getCachedPricingData(): Promise { + const cacheKey = "openai_pricing"; + const cached = this.pricingCache.get(cacheKey); + + if (cached && Date.now() - cached.timestamp < this.CACHE_TTL) { + this.logger.log("Using cached pricing data"); + return cached.data; + } + + this.logger.log("Fetching fresh pricing data from OpenAI"); + const freshData = await scrapeAllPricingFromOpenAI(); + + // Cache the results + this.pricingCache.set(cacheKey, { + data: freshData, + timestamp: Date.now(), + }); + + return freshData; + } + + /** + * Fetches current pricing from OpenAI website using web scraping + * Gets all OpenAI models from database and attempts to fetch their current pricing + */ + async fetchCurrentPricing(): Promise { + this.logger.log( + "Fetching current pricing from OpenAI website via web scraping", + ); + + try { + // Get all OpenAI models from the database + const openaiModels = await this.prisma.lLMModel.findMany({ + where: { + provider: "OpenAI", + isActive: true, + }, + }); + + this.logger.log( + `Found ${openaiModels.length} OpenAI models to fetch pricing for`, + ); + + // Use cached pricing data to reduce API calls + const scrapedPricing = await this.getCachedPricingData(); + + if (scrapedPricing.length === 0) { + this.logger.warn( + "No pricing data scraped, falling back to manual pricing", + ); + return this.getFallbackPricingForAllModels(); + } + + const currentPricing: ModelPricing[] = []; + const modelsWithPricing: Set = new Set(); + + // Match scraped data with database models + for (const model of openaiModels) { + const scrapedModel = scrapedPricing.find( + (p) => p.modelKey === model.modelKey, + ); + + if (scrapedModel) { + currentPricing.push({ + modelKey: scrapedModel.modelKey, + inputTokenPrice: scrapedModel.inputPerToken, + outputTokenPrice: scrapedModel.outputPerToken, + effectiveDate: new Date(), + source: PricingSource.WEB_SCRAPING, + metadata: { + sourceUrl: scrapedModel.sourceUrl, + fetchedAt: scrapedModel.fetchedAt, + lastModified: scrapedModel.lastModified, + }, + }); + + modelsWithPricing.add(model.modelKey); + this.logger.log( + `Successfully fetched pricing for ${model.modelKey}: input=$${scrapedModel.inputPerToken}, output=$${scrapedModel.outputPerToken}`, + ); + } else { + // Try fallback pricing for this specific model + const fallbackPricing = this.getFallbackPricing(model.modelKey); + if (fallbackPricing) { + currentPricing.push(fallbackPricing); + this.logger.warn( + `Using fallback pricing for ${model.modelKey} (not available on OpenAI website yet)`, + ); + } else { + this.logger.error(`No pricing found for model ${model.modelKey}`); + } + } + } + + this.logger.log( + `Successfully processed pricing for ${currentPricing.length} models (${ + modelsWithPricing.size + } from scraping, ${ + currentPricing.length - modelsWithPricing.size + } from fallback)`, + ); + return currentPricing; + } catch (error) { + this.logger.error("Failed to fetch current pricing:", error); + + // Return fallback pricing for known models to prevent complete failure + const fallbackPricing = this.getFallbackPricingForAllModels(); + this.logger.warn( + `Returning fallback pricing for ${fallbackPricing.length} models due to scraping failure`, + ); + return fallbackPricing; + } + } + + /** + * Get fallback pricing for a specific model when web scraping fails + * Pricing based on OpenAI Standard tier as of Jan 2025 + */ + private getFallbackPricing(modelKey: string): ModelPricing | null { + const fallbackPrices: Record = { + // GPT-4o models (Standard tier) + "gpt-4o": { input: 0.000_002_5, output: 0.000_01 }, // $2.50 / $10.00 + "gpt-4o-mini": { input: 0.000_000_15, output: 0.000_000_6 }, // $0.15 / $0.60 + + // GPT-4.1 models (Standard tier) + "gpt-4.1": { input: 0.000_002, output: 0.000_008 }, // $2.00 / $8.00 + "gpt-4.1-mini": { input: 0.000_000_4, output: 0.000_001_6 }, // $0.40 / $1.60 + "gpt-4.1-nano": { input: 0.000_000_1, output: 0.000_000_4 }, // $0.10 / $0.40 + + // GPT-5 models (Standard tier) + "gpt-5": { input: 0.000_001_25, output: 0.000_01 }, // $1.25 / $10.00 + "gpt-5-mini": { input: 0.000_000_25, output: 0.000_002 }, // $0.25 / $2.00 + "gpt-5-nano": { input: 0.000_000_05, output: 0.000_000_4 }, // $0.05 / $0.40 + + // o1 models (Standard tier) + o1: { input: 0.000_015, output: 0.000_06 }, // $15.00 / $60.00 + "o1-pro": { input: 0.000_15, output: 0.0006 }, // $150.00 / $600.00 + "o1-mini": { input: 0.000_001_1, output: 0.000_004_4 }, // $1.10 / $4.40 + + // o3 models (Standard tier) + o3: { input: 0.000_002, output: 0.000_008 }, // $2.00 / $8.00 + "o3-pro": { input: 0.000_02, output: 0.000_08 }, // $20.00 / $80.00 + "o3-mini": { input: 0.000_001_1, output: 0.000_004_4 }, // $1.10 / $4.40 + "o3-deep-research": { input: 0.000_01, output: 0.000_04 }, // $10.00 / $40.00 + + // o4 models (Standard tier) + "o4-mini": { input: 0.000_001_1, output: 0.000_004_4 }, // $1.10 / $4.40 + "o4-mini-deep-research": { input: 0.000_002, output: 0.000_008 }, // $2.00 / $8.00 + }; + + const pricing = fallbackPrices[modelKey]; + if (!pricing) return null; + + return { + modelKey, + inputTokenPrice: pricing.input, + outputTokenPrice: pricing.output, + effectiveDate: new Date(), + source: PricingSource.MANUAL, + metadata: { + lastUpdated: new Date().toISOString(), + source: "Fallback pricing", + notes: modelKey.startsWith("gpt-5") + ? "Estimated pricing for unreleased model" + : "Known pricing when web scraping failed", + }, + }; + } + + /** + * Get fallback pricing for all known models when web scraping completely fails + */ + private getFallbackPricingForAllModels(): ModelPricing[] { + const modelKeys = [ + "gpt-4o", + "gpt-4o-mini", + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4.1-nano", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "o1", + "o1-pro", + "o1-mini", + "o3", + "o3-pro", + "o3-mini", + "o3-deep-research", + "o4-mini", + "o4-mini-deep-research", + ]; + return modelKeys + .map((modelKey) => this.getFallbackPricing(modelKey)) + .filter((pricing): pricing is ModelPricing => pricing !== null); + } + + /** + * Updates pricing history in the database + */ + async updatePricingHistory(pricingData: ModelPricing[]): Promise { + let updatedCount = 0; + + for (const pricing of pricingData) { + try { + // Get the model by modelKey + const model = await this.prisma.lLMModel.findUnique({ + where: { modelKey: pricing.modelKey }, + }); + + if (!model) { + this.logger.warn(`Model ${pricing.modelKey} not found in database`); + continue; + } + + // Check if this exact pricing already exists + const existingPricing = await this.prisma.lLMPricing.findFirst({ + where: { + modelId: model.id, + inputTokenPrice: pricing.inputTokenPrice, + outputTokenPrice: pricing.outputTokenPrice, + source: pricing.source, + effectiveDate: { + gte: new Date(Date.now() - 24 * 60 * 60 * 1000), // Within last 24 hours + }, + }, + }); + + if (existingPricing) { + this.logger.debug( + `Pricing for ${pricing.modelKey} unchanged within last 24h`, + ); + continue; + } + + // Deactivate previous pricing for this model + await this.prisma.lLMPricing.updateMany({ + where: { + modelId: model.id, + isActive: true, + }, + data: { + isActive: false, + }, + }); + + // Insert new pricing + await this.prisma.lLMPricing.create({ + data: { + modelId: model.id, + inputTokenPrice: pricing.inputTokenPrice, + outputTokenPrice: pricing.outputTokenPrice, + effectiveDate: pricing.effectiveDate, + source: pricing.source, + isActive: true, + metadata: pricing.metadata, + }, + }); + + this.logger.log( + `Updated pricing for ${pricing.modelKey}: input=$${pricing.inputTokenPrice}, output=$${pricing.outputTokenPrice}`, + ); + updatedCount++; + } catch (error) { + this.logger.error( + `Failed to update pricing for ${pricing.modelKey}:`, + error, + ); + } + } + + return updatedCount; + } + + /** + * Gets pricing for a specific model at a specific date + * Falls back to closest available pricing if no exact match found + */ + async getPricingAtDate( + modelKey: string, + date: Date, + ): Promise { + const model = await this.prisma.lLMModel.findUnique({ + where: { modelKey }, + }); + + if (!model) { + this.logger.warn(`Model ${modelKey} not found`); + return null; + } + + // First try to find pricing effective on or before the requested date + let pricing = await this.prisma.lLMPricing.findFirst({ + where: { + modelId: model.id, + effectiveDate: { + lte: date, + }, + }, + orderBy: { + effectiveDate: "desc", + }, + include: { + model: true, + }, + }); + + // If no pricing found before the date, find the closest pricing after the date + if (!pricing) { + pricing = await this.prisma.lLMPricing.findFirst({ + where: { + modelId: model.id, + effectiveDate: { + gt: date, + }, + }, + orderBy: { + effectiveDate: "asc", + }, + include: { + model: true, + }, + }); + + if (pricing) { + this.logger.debug( + `Using future pricing for ${modelKey} at ${date.toISOString()}: effective ${pricing.effectiveDate.toISOString()}`, + ); + } + } + + // If still no pricing found, try to find any pricing for this model + if (!pricing) { + pricing = await this.prisma.lLMPricing.findFirst({ + where: { + modelId: model.id, + }, + orderBy: { + effectiveDate: "desc", + }, + include: { + model: true, + }, + }); + + if (pricing) { + this.logger.debug( + `Using latest available pricing for ${modelKey} at ${date.toISOString()}: effective ${pricing.effectiveDate.toISOString()}`, + ); + } + } + + if (!pricing) { + this.logger.warn(`No pricing found for ${modelKey} at any date`); + return null; + } + + return { + modelKey: pricing.model.modelKey, + inputTokenPrice: pricing.inputTokenPrice, + outputTokenPrice: pricing.outputTokenPrice, + effectiveDate: pricing.effectiveDate, + source: pricing.source, + metadata: pricing.metadata, + }; + } + + /** + * Gets current active pricing for a model + */ + async getCurrentPricing(modelKey: string): Promise { + const model = await this.prisma.lLMModel.findUnique({ + where: { modelKey }, + }); + + if (!model) { + return null; + } + + const pricing = await this.prisma.lLMPricing.findFirst({ + where: { + modelId: model.id, + isActive: true, + }, + include: { + model: true, + }, + }); + + if (!pricing) { + return null; + } + + return { + modelKey: pricing.model.modelKey, + inputTokenPrice: pricing.inputTokenPrice, + outputTokenPrice: pricing.outputTokenPrice, + effectiveDate: pricing.effectiveDate, + source: pricing.source, + metadata: pricing.metadata, + }; + } + + /** + * Calculate cost with detailed breakdown using historical pricing and current upscaling + */ + async calculateCostWithBreakdown( + modelKey: string, + inputTokens: number, + outputTokens: number, + usageDate: Date, + usageType?: string, + ): Promise { + // Use the new upscaling-aware calculation method + return await this.calculateCostWithUpscaling( + modelKey, + inputTokens, + outputTokens, + usageDate, + usageType, + ); + } + + /** + * Get all supported models + */ + async getSupportedModels() { + return await this.prisma.lLMModel.findMany({ + where: { isActive: true }, + include: { + pricingHistory: { + where: { isActive: true }, + orderBy: { effectiveDate: "desc" }, + take: 1, + }, + }, + }); + } + + /** + * Get pricing history for a model + */ + async getPricingHistory(modelKey: string, limit = 10) { + const model = await this.prisma.lLMModel.findUnique({ + where: { modelKey }, + }); + + if (!model) { + return []; + } + + return await this.prisma.lLMPricing.findMany({ + where: { modelId: model.id }, + orderBy: { effectiveDate: "desc" }, + take: limit, + include: { model: true }, + }); + } + + /** + * Get pricing statistics + */ + async getPricingStatistics() { + const models = await this.prisma.lLMModel.count(); + const activePricing = await this.prisma.lLMPricing.count({ + where: { isActive: true }, + }); + const totalPricingRecords = await this.prisma.lLMPricing.count(); + + const lastUpdate = await this.prisma.lLMPricing.findFirst({ + orderBy: { createdAt: "desc" }, + select: { createdAt: true }, + }); + + return { + totalModels: models, + activePricingRecords: activePricing, + totalPricingRecords, + lastUpdated: lastUpdate?.createdAt, + }; + } + + /** + * Apply price upscaling factors - stores scaling factors in dedicated table + */ + async applyPriceUpscaling( + globalFactor?: number, + usageFactors?: { [usageType: string]: number }, + reason?: string, + appliedBy?: string, + ): Promise<{ + updatedModels: number; + oldUpscaling: any; + newUpscaling: any; + effectiveDate: Date; + }> { + const effectiveDate = new Date(); + + this.logger.log( + `Applying price upscaling: globalFactor=${globalFactor}, usageFactors=${JSON.stringify( + usageFactors, + )}, reason=${reason}`, + ); + + try { + // Deactivate any existing active upscaling + const oldUpscaling = await this.prisma.lLMPriceUpscaling.findFirst({ + where: { isActive: true }, + }); + + if (oldUpscaling) { + await this.prisma.lLMPriceUpscaling.update({ + where: { id: oldUpscaling.id }, + data: { + isActive: false, + deactivatedAt: effectiveDate, + }, + }); + } + + // Create new upscaling record + const newUpscaling = await this.prisma.lLMPriceUpscaling.create({ + data: { + globalFactor: globalFactor || null, + usageTypeFactors: usageFactors ? JSON.stringify(usageFactors) : null, + reason: reason || "Manual price upscaling via admin interface", + appliedBy: appliedBy || "admin", + isActive: true, + effectiveDate: effectiveDate, + }, + }); + + // Clear the pricing cache to ensure new factors are applied + await this.clearPricingCache(); + + // Get count of models that will be affected + const modelsCount = await this.prisma.lLMModel.count({ + where: { isActive: true }, + }); + + this.logger.log( + `Price upscaling applied successfully. Will affect ${modelsCount} models.`, + ); + + return { + updatedModels: modelsCount, + oldUpscaling: oldUpscaling || null, + newUpscaling: { + id: newUpscaling.id, + globalFactor: newUpscaling.globalFactor, + usageTypeFactors: newUpscaling.usageTypeFactors, + reason: newUpscaling.reason, + appliedBy: newUpscaling.appliedBy, + effectiveDate: newUpscaling.effectiveDate, + }, + effectiveDate, + }; + } catch (error) { + this.logger.error("Failed to apply price upscaling:", error); + throw error; + } + } + + /** + * Get current active price upscaling factors + */ + async getCurrentPriceUpscaling() { + return await this.prisma.lLMPriceUpscaling.findFirst({ + where: { isActive: true }, + orderBy: { effectiveDate: "desc" }, + }); + } + + /** + * Remove current price upscaling (revert to base pricing) + */ + async removePriceUpscaling( + reason?: string, + removedBy?: string, + ): Promise { + try { + const activeUpscaling = await this.prisma.lLMPriceUpscaling.findFirst({ + where: { isActive: true }, + }); + + if (!activeUpscaling) { + this.logger.log("No active price upscaling to remove"); + return false; + } + + await this.prisma.lLMPriceUpscaling.update({ + where: { id: activeUpscaling.id }, + data: { + isActive: false, + deactivatedAt: new Date(), + reason: reason + ? `${activeUpscaling.reason} | Removed: ${reason}` + : activeUpscaling.reason, + }, + }); + + // Clear the pricing cache + await this.clearPricingCache(); + + this.logger.log(`Price upscaling removed by ${removedBy || "admin"}`); + return true; + } catch (error) { + this.logger.error("Failed to remove price upscaling:", error); + throw error; + } + } + + /** + * Calculate cost with upscaling factors applied + */ + async calculateCostWithUpscaling( + modelKey: string, + inputTokens: number, + outputTokens: number, + usageDate: Date, + usageType?: string, + ): Promise { + // Get base pricing + const basePricing = await this.getPricingAtDate(modelKey, usageDate); + if (!basePricing) { + return null; + } + + // Get current upscaling factors + const upscaling = await this.getCurrentPriceUpscaling(); + + let finalInputPrice = basePricing.inputTokenPrice; + let finalOutputPrice = basePricing.outputTokenPrice; + + if (upscaling) { + // Apply global factor first + if (upscaling.globalFactor && upscaling.globalFactor > 0) { + finalInputPrice *= upscaling.globalFactor; + finalOutputPrice *= upscaling.globalFactor; + } + + // Apply usage-type specific factor + if (usageType && upscaling.usageTypeFactors) { + try { + const usageFactors = JSON.parse(upscaling.usageTypeFactors as string); + const usageFactor = usageFactors[usageType]; + if (usageFactor && usageFactor > 0) { + finalInputPrice *= usageFactor; + finalOutputPrice *= usageFactor; + } + } catch (error) { + this.logger.warn("Failed to parse usage type factors:", error); + } + } + } + + const inputCost = inputTokens * finalInputPrice; + const outputCost = outputTokens * finalOutputPrice; + const totalCost = inputCost + outputCost; + + return { + inputTokens, + outputTokens, + inputCost, + outputCost, + totalCost, + modelKey, + pricingEffectiveDate: basePricing.effectiveDate, + inputTokenPrice: finalInputPrice, + outputTokenPrice: finalOutputPrice, + }; + } + + /** + * Clear pricing cache and related model assignment cache + */ + private clearPricingCache(): void { + try { + // Clear the model assignment cache in the resolver service + // This is critical because the resolver service caches model assignments for 5 minutes + this.llmResolverService.clearAllCache(); + + this.logger.log("Pricing cache and model assignment cache cleared"); + } catch (error) { + this.logger.warn("Failed to clear some caches:", error); + } + } + + /** + * Clear only the web scraping cache (useful for testing or forced refresh) + */ + clearWebScrapingCache(): void { + this.pricingCache.clear(); + this.logger.log("Web scraping cache cleared"); + } + + /** + * Get cache status and statistics + */ + getCacheStatus(): { + hasCachedData: boolean; + cacheAge: number | null; + cacheCount: number; + cacheTTL: number; + } { + const cached = this.pricingCache.get("openai_pricing"); + return { + hasCachedData: !!cached, + cacheAge: cached ? Date.now() - cached.timestamp : null, + cacheCount: this.pricingCache.size, + cacheTTL: this.CACHE_TTL, + }; + } + + /** + * Test scraping functionality for a specific model + */ + async testScrapingForModel(modelKey: string): Promise<{ + success: boolean; + pricing?: ExtractResult; + error?: string; + fallbackUsed?: boolean; + }> { + try { + this.logger.log(`Testing scraping functionality for ${modelKey}`); + + // Try to scrape fresh data (bypass cache) + this.pricingCache.delete("openai_pricing"); + const scrapedData = await this.getCachedPricingData(); + + const result = scrapedData.find((p) => p.modelKey === modelKey); + + if (result) { + return { success: true, pricing: result }; + } else { + // Check if we have fallback pricing + const fallback = this.getFallbackPricing(modelKey); + return fallback + ? { + success: true, + fallbackUsed: true, + pricing: { + modelKey: fallback.modelKey, + inputPerToken: fallback.inputTokenPrice, + outputPerToken: fallback.outputTokenPrice, + sourceUrl: "fallback", + fetchedAt: new Date().toISOString(), + }, + } + : { + success: false, + error: `Model ${modelKey} not found in scraped data or fallback`, + }; + } + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + return { success: false, error: errorMessage }; + } + } + + /** + * Get comprehensive pricing status for all known models + */ + async getPricingStatus(): Promise<{ + totalModels: number; + scrapedSuccessfully: number; + usingFallback: number; + notFound: number; + cacheStatus: any; + lastUpdate?: Date; + }> { + try { + const models = await this.prisma.lLMModel.findMany({ + where: { provider: "OpenAI", isActive: true }, + }); + + const scrapedData = await this.getCachedPricingData(); + const cacheStatus = this.getCacheStatus(); + + let scrapedSuccessfully = 0; + let usingFallback = 0; + let notFound = 0; + + for (const model of models) { + const scraped = scrapedData.find((p) => p.modelKey === model.modelKey); + if (scraped) { + scrapedSuccessfully++; + } else { + const fallback = this.getFallbackPricing(model.modelKey); + if (fallback) { + usingFallback++; + } else { + notFound++; + } + } + } + + // Get last update from database + const lastUpdate = await this.prisma.lLMPricing.findFirst({ + orderBy: { effectiveDate: "desc" }, + select: { effectiveDate: true }, + }); + + return { + totalModels: models.length, + scrapedSuccessfully, + usingFallback, + notFound, + cacheStatus, + lastUpdate: lastUpdate?.effectiveDate, + }; + } catch (error) { + this.logger.error("Error getting pricing status:", error); + throw error; + } + } +} diff --git a/apps/api/src/api/llm/core/services/llm-resolver.service.ts b/apps/api/src/api/llm/core/services/llm-resolver.service.ts new file mode 100644 index 00000000..ed2891e8 --- /dev/null +++ b/apps/api/src/api/llm/core/services/llm-resolver.service.ts @@ -0,0 +1,344 @@ +import { Inject, Injectable, Logger } from "@nestjs/common"; +import { LLM_ASSIGNMENT_SERVICE } from "../../llm.constants"; +import { LLMAssignmentService } from "./llm-assignment.service"; + +export interface TaskComplexityContext { + featureKey: string; + inputLength?: number; + responseType?: string; + requiresReasoning?: boolean; + hasMultipleCriteria?: boolean; + isValidationOnly?: boolean; + customComplexity?: "simple" | "complex"; +} + +export type TaskComplexity = "simple" | "complex"; + +/** + * Service to resolve which LLM model should be used for different features + */ +@Injectable() +export class LLMResolverService { + private readonly logger = new Logger(LLMResolverService.name); + private readonly modelCache = new Map< + string, + { modelKey: string; cachedAt: number } + >(); + private readonly CACHE_TTL = 5 * 60 * 1000; // 5 minutes cache + + constructor( + @Inject(LLM_ASSIGNMENT_SERVICE) + private readonly assignmentService: LLMAssignmentService, + ) {} + + /** + * Get the model key that should be used for a specific feature + * This method includes caching for performance + */ + async resolveModelForFeature(featureKey: string): Promise { + // Check cache first + const cached = this.modelCache.get(featureKey); + if (cached && Date.now() - cached.cachedAt < this.CACHE_TTL) { + this.logger.debug( + `Using cached model ${cached.modelKey} for feature ${featureKey}`, + ); + return cached.modelKey; + } + + try { + // Get assigned model from service + const modelKey = + await this.assignmentService.getAssignedModel(featureKey); + + if (modelKey) { + // Cache the result + this.modelCache.set(featureKey, { + modelKey, + cachedAt: Date.now(), + }); + + this.logger.debug( + `Resolved model ${modelKey} for feature ${featureKey}`, + ); + return modelKey; + } + + this.logger.warn(`No model resolved for feature ${featureKey}`); + return null; + } catch (error) { + this.logger.error( + `Failed to resolve model for feature ${featureKey}:`, + error, + ); + return null; + } + } + + /** + * Get models for multiple features at once + */ + async resolveModelsForFeatures( + featureKeys: string[], + ): Promise> { + const results = new Map(); + + // Process all features in parallel + const promises = featureKeys.map(async (featureKey) => { + const modelKey = await this.resolveModelForFeature(featureKey); + return { featureKey, modelKey }; + }); + + const resolved = await Promise.all(promises); + for (const { featureKey, modelKey } of resolved) { + results.set(featureKey, modelKey); + } + + return results; + } + + /** + * Clear cache for a specific feature (useful when assignments change) + */ + clearCacheForFeature(featureKey: string): void { + this.modelCache.delete(featureKey); + this.logger.debug(`Cleared cache for feature ${featureKey}`); + } + + /** + * Clear all cached model assignments + */ + clearAllCache(): void { + this.modelCache.clear(); + this.logger.debug("Cleared all model assignment cache"); + } + + /** + * Get cache statistics + */ + getCacheStats() { + const now = Date.now(); + let validEntries = 0; + let expiredEntries = 0; + + for (const [, entry] of this.modelCache) { + if (now - entry.cachedAt < this.CACHE_TTL) { + validEntries++; + } else { + expiredEntries++; + } + } + + return { + totalEntries: this.modelCache.size, + validEntries, + expiredEntries, + cacheTtlMs: this.CACHE_TTL, + }; + } + + /** + * Convenience method to get model key with fallback + */ + async getModelKeyWithFallback( + featureKey: string, + fallbackModel = "gpt-4o-mini", + ): Promise { + const resolvedModel = await this.resolveModelForFeature(featureKey); + + if (resolvedModel) { + return resolvedModel; + } + + this.logger.warn( + `Using fallback model ${fallbackModel} for feature ${featureKey}`, + ); + return fallbackModel; + } + + /** + * Get model based on task complexity analysis + */ + async getModelForComplexity(context: TaskComplexityContext): Promise { + // First try to get specifically assigned model + const assignedModel = await this.resolveModelForFeature(context.featureKey); + if (assignedModel) { + this.logger.debug( + `Using assigned model ${assignedModel} for feature ${context.featureKey}`, + ); + return assignedModel; + } + + // Analyze task complexity and select appropriate model + const complexity = this.analyzeTaskComplexity(context); + const selectedModel = this.selectModelForComplexity(complexity); + + this.logger.debug( + `Selected ${selectedModel} model based on ${complexity} complexity for feature ${context.featureKey}`, + ); + return selectedModel; + } + + /** + * Analyze task complexity based on context + */ + private analyzeTaskComplexity( + context: TaskComplexityContext, + ): TaskComplexity { + // Use custom complexity if provided + if (context.customComplexity) { + return context.customComplexity; + } + + // Define simple task patterns + const simpleTaskPatterns = [ + "validation", + "judge", + "math_check", + "format_check", + "sanitization", + "basic_feedback", + ]; + + const complexTaskPatterns = [ + "grading", + "generation", + "analysis", + "evaluation", + "reasoning", + "feedback", + "translation", + ]; + + // Check for validation-only tasks (always simple) + if (context.isValidationOnly) { + return "simple"; + } + + // Check feature key patterns + const featureKey = context.featureKey.toLowerCase(); + + if (simpleTaskPatterns.some((pattern) => featureKey.includes(pattern))) { + return "simple"; + } + + if (complexTaskPatterns.some((pattern) => featureKey.includes(pattern))) { + return "complex"; + } + + // Analyze based on context characteristics + let complexityScore = 0; + + // Input length factor + if (context.inputLength) { + if (context.inputLength > 10_000) + complexityScore += 2; // Large input = complex + else if (context.inputLength > 2000) complexityScore += 1; + } + + // Reasoning requirement + if (context.requiresReasoning) complexityScore += 2; + + // Multiple criteria evaluation + if (context.hasMultipleCriteria) complexityScore += 1; + + // Response type complexity + if (context.responseType) { + const complexResponseTypes = [ + "CODE", + "ESSAY", + "REPORT", + "PRESENTATION", + "VIDEO", + ]; + if (complexResponseTypes.includes(context.responseType.toUpperCase())) { + complexityScore += 1; + } + } + + // Determine complexity based on score + return complexityScore >= 2 ? "complex" : "simple"; + } + + /** + * Select model based on complexity level + */ + private selectModelForComplexity(complexity: TaskComplexity): string { + switch (complexity) { + case "simple": { + return "gpt-4o-mini"; + } // Fast, cost-effective for simple tasks + case "complex": { + return "gpt-4o"; + } // More capable for complex reasoning + default: { + return "gpt-4o-mini"; + } // Default to mini for efficiency + } + } + + /** + * Get optimal model for specific grading task + */ + async getModelForGradingTask( + featureKey: string, + responseType: string, + inputLength: number, + criteriaCount = 1, + ): Promise { + const context: TaskComplexityContext = { + featureKey, + inputLength, + responseType, + requiresReasoning: true, + hasMultipleCriteria: criteriaCount > 1, + isValidationOnly: featureKey.toLowerCase().includes("validation"), + }; + + return this.getModelForComplexity(context); + } + + /** + * Get optimal model for validation task + */ + async getModelForValidationTask( + featureKey: string, + inputLength = 0, + ): Promise { + const context: TaskComplexityContext = { + featureKey, + inputLength, + isValidationOnly: true, + customComplexity: "simple", + }; + + return this.getModelForComplexity(context); + } + + /** + * Check if a feature has a specific model assigned + */ + async isModelAssignedToFeature( + featureKey: string, + modelKey: string, + ): Promise { + const assignedModel = await this.resolveModelForFeature(featureKey); + return assignedModel === modelKey; + } + + /** + * Warmup cache by preloading common features + */ + async warmupCache(featureKeys: string[]): Promise { + this.logger.log(`Warming up cache for ${featureKeys.length} features`); + + const promises = featureKeys.map((featureKey) => + this.resolveModelForFeature(featureKey).catch((error) => { + this.logger.warn(`Failed to warmup cache for ${featureKey}:`, error); + return null; + }), + ); + + await Promise.all(promises); + this.logger.log("Cache warmup completed"); + } +} diff --git a/apps/api/src/api/llm/core/services/llm-router.service.ts b/apps/api/src/api/llm/core/services/llm-router.service.ts index f35a136d..c814145f 100644 --- a/apps/api/src/api/llm/core/services/llm-router.service.ts +++ b/apps/api/src/api/llm/core/services/llm-router.service.ts @@ -1,24 +1,105 @@ -import { Injectable, Inject } from "@nestjs/common"; +import { Inject, Injectable, Logger } from "@nestjs/common"; +import { ALL_LLM_PROVIDERS, LLM_RESOLVER_SERVICE } from "../../llm.constants"; import { ILlmProvider } from "../interfaces/llm-provider.interface"; -import { ALL_LLM_PROVIDERS } from "../../llm.constants"; +import { LLMResolverService } from "./llm-resolver.service"; @Injectable() export class LlmRouter { + private readonly logger = new Logger(LlmRouter.name); private readonly map: Map; - constructor(@Inject(ALL_LLM_PROVIDERS) providers: ILlmProvider[]) { + constructor( + @Inject(ALL_LLM_PROVIDERS) providers: ILlmProvider[], + @Inject(LLM_RESOLVER_SERVICE) + private readonly resolverService: LLMResolverService, + ) { this.map = new Map(providers.map((p) => [p.key, p])); } - /** Return provider by key, or throw if it doesn’t exist */ + /** Return provider by key, or throw if it doesn't exist */ get(key: string): ILlmProvider { const found = this.map.get(key); if (!found) throw new Error(`No LLM provider registered for key "${key}"`); return found; } + /** Get provider for a specific AI feature (uses dynamic assignment) */ + async getForFeature(featureKey: string): Promise { + try { + const assignedModelKey = + await this.resolverService.resolveModelForFeature(featureKey); + + if (assignedModelKey) { + const provider = this.map.get(assignedModelKey); + if (provider) { + this.logger.debug( + `Using assigned model ${assignedModelKey} for feature ${featureKey}`, + ); + return provider; + } else { + this.logger.warn( + `Assigned model ${assignedModelKey} not found in providers, using default`, + ); + } + } + + // Fallback to default if no assignment or provider not found + const defaultProvider = this.getDefault(); + this.logger.debug( + `Using default model ${defaultProvider.key} for feature ${featureKey}`, + ); + return defaultProvider; + } catch (error) { + this.logger.error( + `Failed to resolve model for feature ${featureKey}, using default:`, + error, + ); + return this.getDefault(); + } + } + + /** Get provider with fallback model specification */ + async getForFeatureWithFallback( + featureKey: string, + fallbackModelKey = "gpt-4o-mini", + ): Promise { + try { + const assignedModelKey = + await this.resolverService.getModelKeyWithFallback( + featureKey, + fallbackModelKey, + ); + return this.get(assignedModelKey); + } catch (error) { + this.logger.error( + `Failed to get provider for feature ${featureKey} with fallback ${fallbackModelKey}:`, + error, + ); + return this.get(fallbackModelKey); + } + } + + /** Check if a specific model is available */ + hasModel(modelKey: string): boolean { + return this.map.has(modelKey); + } + + /** Get all available model keys */ + getAvailableModelKeys(): string[] { + return [...this.map.keys()]; + } + /** Convenience default (first registered) */ getDefault(): ILlmProvider { return this.map.values().next().value; } + + /** Get statistics about model usage */ + getProviderStats() { + return { + totalProviders: this.map.size, + availableModels: [...this.map.keys()], + defaultModel: this.getDefault()?.key || "none", + }; + } } diff --git a/apps/api/src/api/llm/core/services/moderation.service.ts b/apps/api/src/api/llm/core/services/moderation.service.ts index b123043f..20352a55 100644 --- a/apps/api/src/api/llm/core/services/moderation.service.ts +++ b/apps/api/src/api/llm/core/services/moderation.service.ts @@ -1,10 +1,9 @@ -import { Injectable, HttpException, HttpStatus } from "@nestjs/common"; -import { OpenAIModerationChain } from "langchain/chains"; +import { HttpException, HttpStatus, Inject, Injectable } from "@nestjs/common"; import { sanitize } from "isomorphic-dompurify"; -import { IModerationService } from "../interfaces/moderation.interface"; +import { OpenAIModerationChain } from "langchain/chains"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; -import { Inject } from "@nestjs/common"; import { Logger } from "winston"; +import { IModerationService } from "../interfaces/moderation.interface"; @Injectable() export class ModerationService implements IModerationService { @@ -33,7 +32,9 @@ export class ModerationService implements IModerationService { ); } catch (error) { this.logger.error( - `Error validating content: ${error instanceof Error ? error.message : "Unknown error"}`, + `Error validating content: ${ + error instanceof Error ? error.message : "Unknown error" + }`, ); return true; @@ -52,7 +53,9 @@ export class ModerationService implements IModerationService { return sanitizedContent; } catch (error) { this.logger.error( - `Error sanitizing content: ${error instanceof Error ? error.message : "Unknown error"}`, + `Error sanitizing content: ${ + error instanceof Error ? error.message : "Unknown error" + }`, ); return content; diff --git a/apps/api/src/api/llm/core/services/openai-llm-mini.service.ts b/apps/api/src/api/llm/core/services/openai-llm-mini.service.ts index da380411..bb9bbda8 100644 --- a/apps/api/src/api/llm/core/services/openai-llm-mini.service.ts +++ b/apps/api/src/api/llm/core/services/openai-llm-mini.service.ts @@ -1,15 +1,15 @@ -import { Injectable, Inject } from "@nestjs/common"; -import { ChatOpenAI } from "@langchain/openai"; import { HumanMessage } from "@langchain/core/messages"; +import { ChatOpenAI } from "@langchain/openai"; +import { Inject, Injectable } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { Logger } from "winston"; +import { TOKEN_COUNTER } from "../../llm.constants"; import { ILlmProvider, LlmRequestOptions, LlmResponse, } from "../interfaces/llm-provider.interface"; import { ITokenCounter } from "../interfaces/token-counter.interface"; -import { TOKEN_COUNTER } from "../../llm.constants"; -import { WINSTON_MODULE_PROVIDER } from "nest-winston"; -import { Logger } from "winston"; /** * Light-weight provider that targets the smaller/faster gpt-4o-mini model. diff --git a/apps/api/src/api/llm/core/services/openai-llm-vision.service.ts b/apps/api/src/api/llm/core/services/openai-llm-vision.service.ts index 6b1d36a6..cffd47de 100644 --- a/apps/api/src/api/llm/core/services/openai-llm-vision.service.ts +++ b/apps/api/src/api/llm/core/services/openai-llm-vision.service.ts @@ -1,15 +1,15 @@ -import { Injectable, Inject } from "@nestjs/common"; -import { ChatOpenAI } from "@langchain/openai"; import { HumanMessage } from "@langchain/core/messages"; +import { ChatOpenAI } from "@langchain/openai"; +import { Inject, Injectable } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { Logger } from "winston"; +import { TOKEN_COUNTER } from "../../llm.constants"; import { ILlmProvider, LlmRequestOptions, LlmResponse, } from "../interfaces/llm-provider.interface"; import { ITokenCounter } from "../interfaces/token-counter.interface"; -import { TOKEN_COUNTER } from "../../llm.constants"; -import { WINSTON_MODULE_PROVIDER } from "nest-winston"; -import { Logger } from "winston"; @Injectable() export class Gpt4VisionPreviewLlmService implements ILlmProvider { @@ -128,7 +128,9 @@ export class Gpt4VisionPreviewLlmService implements ILlmProvider { }; } catch (error) { this.logger.error( - `Error processing image with GPT-4 Vision Preview: ${error instanceof Error ? error.message : "Unknown error"}`, + `Error processing image with GPT-4 Vision Preview: ${ + error instanceof Error ? error.message : "Unknown error" + }`, ); throw error; } diff --git a/apps/api/src/api/llm/core/services/openai-llm.service.ts b/apps/api/src/api/llm/core/services/openai-llm.service.ts index 192fd69d..1ed080fc 100644 --- a/apps/api/src/api/llm/core/services/openai-llm.service.ts +++ b/apps/api/src/api/llm/core/services/openai-llm.service.ts @@ -1,15 +1,15 @@ -import { Injectable, Inject } from "@nestjs/common"; -import { ChatOpenAI } from "@langchain/openai"; import { HumanMessage } from "@langchain/core/messages"; +import { ChatOpenAI } from "@langchain/openai"; +import { Inject, Injectable } from "@nestjs/common"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { Logger } from "winston"; +import { TOKEN_COUNTER } from "../../llm.constants"; import { ILlmProvider, LlmRequestOptions, LlmResponse, } from "../interfaces/llm-provider.interface"; import { ITokenCounter } from "../interfaces/token-counter.interface"; -import { TOKEN_COUNTER } from "../../llm.constants"; -import { WINSTON_MODULE_PROVIDER } from "nest-winston"; -import { Logger } from "winston"; @Injectable() export class OpenAiLlmService implements ILlmProvider { @@ -113,7 +113,9 @@ export class OpenAiLlmService implements ILlmProvider { }; } catch (error) { this.logger.error( - `Error processing image with LLM: ${error instanceof Error ? error.message : "Unknown error"}`, + `Error processing image with LLM: ${ + error instanceof Error ? error.message : "Unknown error" + }`, ); throw error; } diff --git a/apps/api/src/api/llm/core/services/prompt-processor.service.ts b/apps/api/src/api/llm/core/services/prompt-processor.service.ts index cf12cf33..1f03cc34 100644 --- a/apps/api/src/api/llm/core/services/prompt-processor.service.ts +++ b/apps/api/src/api/llm/core/services/prompt-processor.service.ts @@ -1,10 +1,14 @@ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { HumanMessage } from "@langchain/core/messages"; import { PromptTemplate } from "@langchain/core/prompts"; import { Inject, Injectable } from "@nestjs/common"; import { AIUsageType } from "@prisma/client"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; -import { decodeFields, decodeIfBase64 } from "src/helpers/decoder"; import { Logger } from "winston"; +import { decodeFields, decodeIfBase64 } from "../../../../helpers/decoder"; import { USAGE_TRACKER } from "../../llm.constants"; import { IPromptProcessor } from "../interfaces/prompt-processor.interface"; import { IUsageTracker } from "../interfaces/user-tracking.interface"; @@ -22,6 +26,37 @@ export class PromptProcessorService implements IPromptProcessor { this.logger = parentLogger.child({ context: PromptProcessorService.name }); } + /** + * Process a prompt using assigned model for a specific feature + */ + async processPromptForFeature( + prompt: PromptTemplate, + assignmentId: number, + usageType: AIUsageType, + featureKey: string, + fallbackModel = "gpt-4o-mini", + ): Promise { + try { + const llm = await this.router.getForFeatureWithFallback( + featureKey, + fallbackModel, + ); + return await this._processPromptWithProvider( + prompt, + assignmentId, + usageType, + llm, + ); + } catch (error) { + this.logger.error( + `Error processing prompt for feature ${featureKey}: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + throw error; + } + } + /** * Process a text prompt and return the LLM response */ @@ -33,62 +68,12 @@ export class PromptProcessorService implements IPromptProcessor { ): Promise { try { const llm = this.router.get(llmKey ?? "gpt-4o"); - - if (prompt.partialVariables) { - const stringVariables: { [key: string]: string | null } = {}; - - for (const key in prompt.partialVariables) { - const value = prompt.partialVariables[key]; - if ( - (typeof value === "string" || value === null) && - typeof value !== "function" - ) { - stringVariables[key] = value; - } - } - - const decodedVariables = decodeFields(stringVariables); - - for (const key in decodedVariables) { - prompt.partialVariables[key] = decodedVariables[key]; - } - } - - let input: string; - try { - input = await prompt.format({}); - - input = decodeIfBase64(input) || input; - } catch (formatError: unknown) { - const errorMessage = - formatError instanceof Error ? formatError.message : "Unknown error"; - this.logger.error(`Error formatting prompt: ${errorMessage}`, { - stack: - formatError instanceof Error - ? formatError.stack - : "No stack trace available", - promptDetails: { - template: JSON.stringify(prompt.template).slice(0, 100) + "...", - partialVariables: - JSON.stringify(prompt.partialVariables || {}).slice(0, 200) + - "...", - }, - }); - throw formatError; - } - - const result = await llm.invoke([new HumanMessage(input)]); - - const response = this.cleanResponse(result.content); - - await this.usageTracker.trackUsage( + return await this._processPromptWithProvider( + prompt, assignmentId, usageType, - result.tokenUsage.input, - result.tokenUsage.output, + llm, ); - - return response; } catch (error) { this.logger.error( `Error processing prompt: ${ @@ -111,6 +96,70 @@ export class PromptProcessorService implements IPromptProcessor { } } + /** + * Internal method to process a prompt with a specific LLM provider + */ + private async _processPromptWithProvider( + prompt: PromptTemplate, + assignmentId: number, + usageType: AIUsageType, + llm: any, + ): Promise { + if (prompt.partialVariables) { + const stringVariables: { [key: string]: string | null } = {}; + + for (const key in prompt.partialVariables) { + const value = prompt.partialVariables[key]; + if ( + (typeof value === "string" || value === null) && + typeof value !== "function" + ) { + stringVariables[key] = value; + } + } + + const decodedVariables = decodeFields(stringVariables); + + for (const key in decodedVariables) { + prompt.partialVariables[key] = decodedVariables[key]; + } + } + + let input: string; + try { + input = await prompt.format({}); + input = decodeIfBase64(input) || input; + } catch (formatError: unknown) { + const errorMessage = + formatError instanceof Error ? formatError.message : "Unknown error"; + this.logger.error(`Error formatting prompt: ${errorMessage}`, { + stack: + formatError instanceof Error + ? formatError.stack + : "No stack trace available", + promptDetails: { + template: JSON.stringify(prompt.template).slice(0, 100) + "...", + partialVariables: + JSON.stringify(prompt.partialVariables || {}).slice(0, 200) + "...", + }, + }); + throw formatError; + } + + const result = await llm.invoke([new HumanMessage(input)]); + const response = this.cleanResponse(result.content); + + await this.usageTracker.trackUsage( + assignmentId, + usageType, + result.tokenUsage.input, + result.tokenUsage.output, + llm.key, + ); + + return response; + } + /** * Process a prompt with image data and return the LLM response */ @@ -162,6 +211,7 @@ export class PromptProcessorService implements IPromptProcessor { usageType, result.tokenUsage.input, result.tokenUsage.output, + llm.key, ); return response; diff --git a/apps/api/src/api/llm/core/services/token-counter.service.ts b/apps/api/src/api/llm/core/services/token-counter.service.ts index b209e185..fed0b5d4 100644 --- a/apps/api/src/api/llm/core/services/token-counter.service.ts +++ b/apps/api/src/api/llm/core/services/token-counter.service.ts @@ -1,9 +1,8 @@ -import { Injectable } from "@nestjs/common"; import { get_encoding, Tiktoken } from "@dqbd/tiktoken"; -import { ITokenCounter } from "../interfaces/token-counter.interface"; +import { Inject, Injectable } from "@nestjs/common"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; -import { Inject } from "@nestjs/common"; import { Logger } from "winston"; +import { ITokenCounter } from "../interfaces/token-counter.interface"; @Injectable() export class TokenCounterService implements ITokenCounter { @@ -19,16 +18,38 @@ export class TokenCounterService implements ITokenCounter { /** * Count the number of tokens in the given text */ - countTokens(text: string): number { + countTokens(text: string, modelKey?: string): number { if (!text) return 0; try { - return this.encoding.encode(text).length; + if (modelKey?.includes("llama")) { + return this.countLlamaTokens(text); + } + return this.encoding.encode(text, [modelKey]).length; } catch (error) { this.logger.error( - `Error encoding text: ${error instanceof Error ? error.message : "Unknown error"}`, + `Error encoding text for model ${modelKey}: ${ + error instanceof Error ? error.message : "Unknown error" + }`, ); + // Rough fallback + return Math.ceil(text.length / 4); + } + } + /** + * Count tokens for Llama models (rough approximation) + * Llama models typically use a similar tokenization to GPT, so we use the same encoding + * but could be refined with actual Llama tokenizer if needed + */ + private countLlamaTokens(text: string): number { + try { + // For now, use the same encoding as GPT models + // In the future, this could be replaced with actual Llama tokenizer + return this.encoding.encode(text).length; + } catch { + // Fallback to character-based estimation + // Llama models typically have similar token-to-character ratios as GPT return Math.ceil(text.length / 4); } } diff --git a/apps/api/src/api/llm/core/services/usage-tracking.service.ts b/apps/api/src/api/llm/core/services/usage-tracking.service.ts index 8c24862b..45616241 100644 --- a/apps/api/src/api/llm/core/services/usage-tracking.service.ts +++ b/apps/api/src/api/llm/core/services/usage-tracking.service.ts @@ -25,6 +25,7 @@ export class UsageTrackerService implements IUsageTracker { usageType: AIUsageType, tokensIn: number, tokensOut: number, + modelKey?: string, ): Promise { try { const assignmentIdToDatabase = Number(assignmentId); @@ -54,6 +55,7 @@ export class UsageTrackerService implements IUsageTracker { tokensOut: { increment: tokensOut }, usageCount: { increment: 1 }, updatedAt: new Date(), + ...(modelKey && { modelKey }), }, create: { assignmentId: assignmentIdToDatabase, @@ -63,6 +65,7 @@ export class UsageTrackerService implements IUsageTracker { usageCount: 1, createdAt: new Date(), updatedAt: new Date(), + modelKey, }, }); diff --git a/apps/api/src/api/llm/features/grading/interfaces/grading-judge.interface.ts b/apps/api/src/api/llm/features/grading/interfaces/grading-judge.interface.ts new file mode 100644 index 00000000..3281b5b7 --- /dev/null +++ b/apps/api/src/api/llm/features/grading/interfaces/grading-judge.interface.ts @@ -0,0 +1,37 @@ +import { + RubricDto, + ScoringDto, +} from "src/api/assignment/dto/update.questions.request.dto"; +import { RubricScore } from "src/api/llm/model/file.based.question.response.model"; + +export interface GradingJudgeInput { + question: string; + learnerResponse: string; + scoringCriteria: ScoringDto; + proposedGrading: { + points: number; + maxPoints: number; + feedback: string; + rubricScores?: RubricDto[]; + analysis?: string; + evaluation?: string; + explanation?: string; + guidance?: string; + }; + assignmentId: number; +} + +export interface GradingJudgeResult { + approved: boolean; + feedback: string; + issues?: string[]; + corrections?: { + points?: number; + feedback?: string; + rubricScores?: RubricScore[]; + }; +} + +export interface IGradingJudgeService { + validateGrading(input: GradingJudgeInput): Promise; +} diff --git a/apps/api/src/api/llm/features/grading/interfaces/presentation-grading.interface.ts b/apps/api/src/api/llm/features/grading/interfaces/presentation-grading.interface.ts index 7592dbce..3e9b860f 100644 --- a/apps/api/src/api/llm/features/grading/interfaces/presentation-grading.interface.ts +++ b/apps/api/src/api/llm/features/grading/interfaces/presentation-grading.interface.ts @@ -1,6 +1,6 @@ import { PresentationQuestionEvaluateModel } from "src/api/llm/model/presentation.question.evaluate.model"; -import { LearnerLiveRecordingFeedback } from "../../../../assignment/attempt/dto/assignment-attempt/types"; import { PresentationQuestionResponseModel } from "src/api/llm/model/presentation.question.response.model"; +import { LearnerLiveRecordingFeedback } from "../../../../assignment/attempt/dto/assignment-attempt/types"; export interface IPresentationGradingService { /** diff --git a/apps/api/src/api/llm/features/grading/services/file-grading.service.ts b/apps/api/src/api/llm/features/grading/services/file-grading.service.ts index 3dc42e87..7b655a9a 100644 --- a/apps/api/src/api/llm/features/grading/services/file-grading.service.ts +++ b/apps/api/src/api/llm/features/grading/services/file-grading.service.ts @@ -7,6 +7,15 @@ import { FileUploadQuestionEvaluateModel } from "src/api/llm/model/file.based.qu import { FileBasedQuestionResponseModel } from "src/api/llm/model/file.based.question.response.model"; import { Logger } from "winston"; import { z } from "zod"; +import { IModerationService } from "../../../core/interfaces/moderation.interface"; +import { IPromptProcessor } from "../../../core/interfaces/prompt-processor.interface"; +import { LLMResolverService } from "../../../core/services/llm-resolver.service"; +import { + LLM_RESOLVER_SERVICE, + MODERATION_SERVICE, + PROMPT_PROCESSOR, +} from "../../../llm.constants"; +import { IFileGradingService } from "../interfaces/file-grading.interface"; // Define types to avoid deep instantiation issues type RubricScore = { @@ -25,10 +34,6 @@ type GradingOutput = { guidance: string; rubricScores?: RubricScore[]; }; -import { IModerationService } from "../../../core/interfaces/moderation.interface"; -import { IPromptProcessor } from "../../../core/interfaces/prompt-processor.interface"; -import { MODERATION_SERVICE, PROMPT_PROCESSOR } from "../../../llm.constants"; -import { IFileGradingService } from "../interfaces/file-grading.interface"; @Injectable() export class FileGradingService implements IFileGradingService { @@ -39,6 +44,8 @@ export class FileGradingService implements IFileGradingService { private readonly promptProcessor: IPromptProcessor, @Inject(MODERATION_SERVICE) private readonly moderationService: IModerationService, + @Inject(LLM_RESOLVER_SERVICE) + private readonly llmResolver: LLMResolverService, @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, ) { this.logger = parentLogger.child({ context: FileGradingService.name }); @@ -146,28 +153,41 @@ export class FileGradingService implements IFileGradingService { format_instructions: () => formatInstructions, }, }); + const extractedContent = learnerResponse + .map((item) => item.content) + .join(" "); + const inputLength = + question.length + + extractedContent.length + + JSON.stringify(scoringCriteria).length; + const criteriaCount = Array.isArray(scoringCriteria) + ? scoringCriteria.length + : 1; + + const selectedModel = await this.llmResolver.getModelForGradingTask( + "file_grading", + responseType, + inputLength, + criteriaCount, + ); const response = await this.promptProcessor.processPrompt( prompt, assignmentId, AIUsageType.ASSIGNMENT_GRADING, + selectedModel, ); try { - const parsedResponse = (await parser.parse(response)) as GradingOutput; + let parsedResponse = (await parser.parse(response)) as GradingOutput; - // Validate and format rubric scores if available - let rubricDetails = ""; let calculatedTotalPoints = 0; if ( parsedResponse.rubricScores && parsedResponse.rubricScores.length > 0 ) { - rubricDetails = "\n\n**Rubric Scoring:**\n"; for (const score of parsedResponse.rubricScores) { - rubricDetails += `${score.pointsAwarded}/${score.maxPoints} points\n`; - rubricDetails += `Justification: ${score.justification}\n\n`; calculatedTotalPoints += score.pointsAwarded; } @@ -179,44 +199,56 @@ export class FileGradingService implements IFileGradingService { this.logger.warn( `LLM total points (${parsedResponse.points}) doesn't match sum of rubric scores (${calculatedTotalPoints}). Using rubric sum.`, ); - parsedResponse.points = calculatedTotalPoints; + // Create corrected response object + parsedResponse = { + ...parsedResponse, + points: calculatedTotalPoints, + }; } } - // Combine the AEEG components into comprehensive feedback - const aeegFeedback = ` -**Analysis:** -${parsedResponse.analysis} - -**Evaluation:** -${parsedResponse.evaluation}${rubricDetails} - -**Explanation:** -${parsedResponse.explanation} - -**Guidance:** -${parsedResponse.guidance} -`.trim(); - - const fileBasedQuestionResponseModel = { - points: parsedResponse.points, - feedback: aeegFeedback, - }; + const fileBasedQuestionResponseModel = new FileBasedQuestionResponseModel( + parsedResponse.points, + parsedResponse.feedback, + parsedResponse.analysis, + parsedResponse.evaluation, + parsedResponse.explanation, + parsedResponse.guidance, + parsedResponse.rubricScores, + ); const parsedPoints = fileBasedQuestionResponseModel.points; + let finalModel = fileBasedQuestionResponseModel; + if (parsedPoints > maxTotalPoints) { this.logger.warn( `LLM awarded ${parsedPoints} points, which exceeds maximum of ${maxTotalPoints}. Capping at maximum.`, ); - fileBasedQuestionResponseModel.points = maxTotalPoints; + finalModel = new FileBasedQuestionResponseModel( + maxTotalPoints, + fileBasedQuestionResponseModel.feedback, + fileBasedQuestionResponseModel.analysis, + fileBasedQuestionResponseModel.evaluation, + fileBasedQuestionResponseModel.explanation, + fileBasedQuestionResponseModel.guidance, + fileBasedQuestionResponseModel.rubricScores, + ); } else if (parsedPoints < 0) { this.logger.warn( `LLM awarded negative points (${parsedPoints}). Setting to 0.`, ); - fileBasedQuestionResponseModel.points = 0; + finalModel = new FileBasedQuestionResponseModel( + 0, + fileBasedQuestionResponseModel.feedback, + fileBasedQuestionResponseModel.analysis, + fileBasedQuestionResponseModel.evaluation, + fileBasedQuestionResponseModel.explanation, + fileBasedQuestionResponseModel.guidance, + fileBasedQuestionResponseModel.rubricScores, + ); } - return fileBasedQuestionResponseModel as FileBasedQuestionResponseModel; + return finalModel; } catch (error) { this.logger.error( `Error parsing LLM response: ${ @@ -258,85 +290,40 @@ ${parsedResponse.guidance} fileTypeDescriptions[responseType] || fileTypeDescriptions.OTHER; return ` - You are an expert educator evaluating a student's ${fileTypeContext} using the AEEG (Analyze, Evaluate, Explain, Guide) approach. - - QUESTION: - {question} - - FILES SUBMITTED: - {files} - - SCORING INFORMATION: - Total Points Available: {total_points} - Scoring Type: {scoring_type} - Scoring Criteria: {scoring_criteria} - - CRITICAL GRADING INSTRUCTIONS: - You MUST grade according to the EXACT rubric provided in the scoring criteria. If the scoring type is "CRITERIA_BASED" with rubrics: - 1. Evaluate the submission against EACH rubric question provided - 2. For EACH rubric: - - Read the rubricQuestion carefully - - Review ALL criteria options for that rubric - - Select EXACTLY ONE criterion that best matches the student's performance - - Award the EXACT points specified for that selected criterion (not an average or adjusted value) - 3. Do NOT use generic grading criteria unless specifically mentioned in the rubric - 4. Do NOT interpolate between criteria levels - select the ONE that best fits - 5. The total points awarded must equal the sum of points from the selected criterion for each rubric - 6. Include a rubricScores array in your response with one entry per rubric showing: - - rubricQuestion: the exact text of the rubric question - - pointsAwarded: the exact points from the selected criterion - - maxPoints: the maximum possible points for that rubric - - justification: why you selected that specific criterion level - 7. The total points must not exceed {total_points} - - GRADING APPROACH (AEEG): - - 1. ANALYZE: Carefully examine the submitted files and describe what you observe - - Identify the key elements and structure of the submission - - Note specific techniques, approaches, or methodologies used - - Observe the quality and completeness of the work - - Recognize strengths and unique aspects of the submission - - Focus your analysis on aspects relevant to the rubric criteria - - 2. EVALUATE: For each rubric question in the scoring criteria: - - Read the rubric question carefully - - Examine how the submission addresses this specific rubric question - - Compare the submission against ALL criterion levels for this rubric - - Select EXACTLY ONE criterion that best matches the student's performance - - Award the EXACT points specified for that selected criterion - - Do NOT average, interpolate, or adjust points - use the exact value from the selected criterion - - Record your selection in the rubricScores array - - 3. EXPLAIN: Provide clear reasons for the grade based on specific observations - - For each rubric, explain why you selected that specific criterion level - - Reference specific parts of the submitted files - - Connect your observations directly to the rubric descriptions - - Justify points awarded with concrete evidence from the submission - - Clearly articulate what was well-executed - - Transparently address any deficiencies or areas that fell short - - 4. GUIDE: Offer concrete suggestions for improvement - - Provide specific, actionable feedback based on the rubric criteria - - Suggest ways to reach higher criterion levels in each rubric - - Recommend resources, techniques, or strategies for improvement - - Focus guidance on the specific skills and competencies assessed by the rubrics - - Include practical tips relevant to the submission type - - GRADING INSTRUCTIONS: - - Be fair, consistent, and constructive in your evaluation - - Use encouraging language while maintaining high standards - - Ensure all feedback is specific to the files submitted - - Consider the context and requirements of the assignment - - For CRITERIA_BASED scoring, you MUST include rubricScores array - + Grade ${fileTypeContext} using AEEG approach. + + QUESTION: {question} + FILES: {files} + POINTS: {total_points} | TYPE: {scoring_type} + CRITERIA: {scoring_criteria} + + RUBRIC RULES: + - Select EXACTLY ONE criterion per rubric (no interpolation) + - Award EXACT points from selected criterion + - Total = sum of rubric points (max {total_points}) + - Include rubricScores array with: rubricQuestion, pointsAwarded, maxPoints, justification + + AEEG APPROACH: + 1. ANALYZE: Key elements, structure, techniques, quality + 2. EVALUATE: Match submission to each rubric criterion, select best fit + 3. EXPLAIN: Justify grade with specific evidence + 4. GUIDE: Actionable improvement suggestions + LANGUAGE: {language} + + JSON Response: + - Points (rubric sum) + - Feedback (overall assessment) + - Analysis (submission examination) + - Evaluation (rubric-based scoring) + - Explanation (grade justification) + - Guidance (improvement tips) + - rubricScores array (if CRITERIA_BASED) - Respond with a JSON object containing: - - Points awarded (sum of all rubric scores) - - Comprehensive feedback incorporating all four AEEG components - - Separate fields for each AEEG component - - If scoring type is CRITERIA_BASED, include rubricScores array with score for each rubric + AVOID REDUNDANCY: Each field should contain unique information and not repeat content from other fields. + Make sure your feedback is short and concise. + {format_instructions} `; } diff --git a/apps/api/src/api/llm/features/grading/services/grading-judge.service.ts b/apps/api/src/api/llm/features/grading/services/grading-judge.service.ts new file mode 100644 index 00000000..1d51080c --- /dev/null +++ b/apps/api/src/api/llm/features/grading/services/grading-judge.service.ts @@ -0,0 +1,372 @@ +// src/llm/features/grading/services/grading-judge.service.ts +import { PromptTemplate } from "@langchain/core/prompts"; +import { Inject, Injectable } from "@nestjs/common"; +import { AIUsageType } from "@prisma/client"; +import { StructuredOutputParser } from "langchain/output_parsers"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { RubricScore } from "src/api/llm/model/file.based.question.response.model"; +import { Logger } from "winston"; +import { z } from "zod"; +import { IPromptProcessor } from "../../../core/interfaces/prompt-processor.interface"; +import { LLMResolverService } from "../../../core/services/llm-resolver.service"; +import { LLM_RESOLVER_SERVICE, PROMPT_PROCESSOR } from "../../../llm.constants"; +import { IGradingJudgeService } from "../interfaces/grading-judge.interface"; + +export interface GradingJudgeInput { + question: string; + learnerResponse: string; + scoringCriteria: any; + proposedGrading: { + points: number; + maxPoints: number; + feedback: string; + rubricScores?: RubricScore[]; + analysis?: string; + evaluation?: string; + explanation?: string; + guidance?: string; + }; + assignmentId: number; +} + +export interface GradingJudgeResult { + approved: boolean; + feedback: string; + issues?: string[]; + corrections?: { + points?: number; + feedback?: string; + rubricScores?: RubricScore[]; + }; +} + +const ParsedJudgeResponseSchema = z.object({ + approved: z.boolean(), + feedback: z.string(), + issues: z.array(z.string()).optional(), + mathematicallyCorrect: z.boolean(), + feedbackAligned: z.boolean(), + rubricAdherence: z.boolean(), + fairnessScore: z.number().min(0).max(10), + suggestedPoints: z.number().optional(), + suggestedFeedbackChanges: z.string().optional(), + correctedRubricScores: z + .array( + z.object({ + rubricQuestion: z.string(), + pointsAwarded: z.number(), + maxPoints: z.number(), + criterionSelected: z.string(), + justification: z.string(), + }), + ) + .optional(), +}); + +type ParsedJudgeResponse = z.infer; + +const judgeParserCache = new WeakMap>(); + +@Injectable() +export class GradingJudgeService implements IGradingJudgeService { + private readonly logger: Logger; + private readonly maxJudgeTimeout = 60_000; + + constructor( + @Inject(PROMPT_PROCESSOR) + private readonly promptProcessor: IPromptProcessor, + @Inject(LLM_RESOLVER_SERVICE) + private readonly llmResolver: LLMResolverService, + @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, + ) { + this.logger = parentLogger.child({ context: GradingJudgeService.name }); + } + + async validateGrading(input: GradingJudgeInput): Promise { + const startTime = Date.now(); + + try { + this.logger.info( + `Judge validating grading for assignment ${input.assignmentId}`, + ); + + this.validateInput(input); + + const parser = this.getOrCreateParser(); + const formatInstructions = parser.getFormatInstructions(); + + let actualSum = 0; + if ( + input.proposedGrading.rubricScores && + Array.isArray(input.proposedGrading.rubricScores) + ) { + const actualSum = input.proposedGrading.rubricScores + .map( + (score: { + pointsAwarded?: number; + points?: number; + score?: number; + }) => score.pointsAwarded ?? score.points ?? score.score ?? 0, + ) + .filter((points): points is number => typeof points === "number") + .reduce((a, b) => a + b, 0); + + this.logger.info( + `Calculated actual sum: ${actualSum} from ${input.proposedGrading.rubricScores.length} rubric scores`, + ); + } else { + actualSum = input.proposedGrading.points || 0; + this.logger.info( + `No rubric scores found, using proposed points as actual sum: ${actualSum}`, + ); + } + + const template = this.loadJudgeTemplate(); + + const prompt = new PromptTemplate({ + template, + inputVariables: [], + partialVariables: { + question: () => input.question || "No question provided", + learner_response: () => + input.learnerResponse || "No response provided", + scoring_criteria: () => JSON.stringify(input.scoringCriteria || {}), + proposed_points: () => String(input.proposedGrading.points || 0), + actual_sum: () => String(actualSum), + max_points: () => String(input.proposedGrading.maxPoints || 0), + proposed_feedback: () => + input.proposedGrading.feedback || "No feedback provided", + proposed_analysis: () => + input.proposedGrading.analysis || "Not provided", + proposed_evaluation: () => + input.proposedGrading.evaluation || "Not provided", + proposed_explanation: () => + input.proposedGrading.explanation || "Not provided", + proposed_guidance: () => + input.proposedGrading.guidance || "Not provided", + proposed_rubric_scores: () => + JSON.stringify(input.proposedGrading.rubricScores || []), + format_instructions: () => formatInstructions, + }, + }); + + // Use validation-optimized model selection (gpt-4o-mini for validation tasks) + const selectedModel = await this.llmResolver.getModelForValidationTask( + "text_grading", + ( + input.question + + input.learnerResponse + + JSON.stringify(input.scoringCriteria) + ).length, + ); + + const response = await this.processWithTimeout( + this.promptProcessor.processPrompt( + prompt, + input.assignmentId, + AIUsageType.GRADING_VALIDATION, + selectedModel, + ), + this.maxJudgeTimeout, + ); + + const parsedResponse = await parser.parse(response); + const result = this.buildJudgeResult(parsedResponse, input, actualSum); + + const endTime = Date.now(); + this.logger.info( + `Judge ${parsedResponse.approved ? "approved" : "rejected"} grading. ` + + `Mathematical: ${JSON.stringify( + parsedResponse.mathematicallyCorrect, + )}, ` + + `Aligned: ${JSON.stringify(parsedResponse.feedbackAligned)}, ` + + `Rubric: ${JSON.stringify(parsedResponse.rubricAdherence)}, ` + + `Fairness: ${JSON.stringify(parsedResponse.fairnessScore)}/10, ` + + `Time: ${endTime - startTime}ms`, + ); + + return result; + } catch (error) { + this.logger.error( + `Error in judge validation: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + + return { + approved: false, + feedback: + "Judge validation temporarily unavailable. Please review grading manually.", + issues: ["Judge service error - manual review required"], + }; + } + } + + private validateInput(input: GradingJudgeInput): void { + if (!input.question) { + throw new Error("Question is required for judge validation"); + } + + if (!input.learnerResponse) { + throw new Error("Learner response is required for judge validation"); + } + + if ( + typeof input.proposedGrading.points !== "number" || + input.proposedGrading.points < 0 + ) { + throw new Error("Invalid proposed points"); + } + + if ( + typeof input.proposedGrading.maxPoints !== "number" || + input.proposedGrading.maxPoints <= 0 + ) { + throw new Error("Invalid max points"); + } + } + + private buildJudgeResult( + parsedResponse: ParsedJudgeResponse, + input: GradingJudgeInput, + actualSum: number, + ): GradingJudgeResult { + // CRITICAL: Override if math is actually wrong + const mathIsCorrect = input.proposedGrading.points === actualSum; + + // If math is correct but judge thinks it's wrong, fix the judge's response + if (mathIsCorrect && !parsedResponse.mathematicallyCorrect) { + this.logger.warn( + `Judge incorrectly flagged math as wrong. Total: ${input.proposedGrading.points}, Sum: ${actualSum}. Overriding judge.`, + ); + parsedResponse.mathematicallyCorrect = true; + // If math is the only issue, approve it + if (parsedResponse.fairnessScore >= 5 && parsedResponse.rubricAdherence) { + parsedResponse.approved = true; + } + } + + // If math is wrong, always reject + if (!mathIsCorrect) { + parsedResponse.mathematicallyCorrect = false; + parsedResponse.approved = false; + } + + const result: GradingJudgeResult = { + approved: parsedResponse.approved, + feedback: parsedResponse.feedback || "No feedback provided", + issues: parsedResponse.issues || [], + }; + + if (!parsedResponse.approved) { + result.corrections = {}; + + // Only suggest math correction if math is actually wrong + if (!mathIsCorrect) { + result.corrections.points = actualSum; + result.issues = [ + `Mathematical error: Total should be ${actualSum}, not ${input.proposedGrading.points}`, + ]; + } else if (parsedResponse.fairnessScore < 5) { + // Only reject for severe unfairness + result.issues = [ + `Grading appears unfair (fairness score: ${parsedResponse.fairnessScore}/10)`, + ]; + } else if (!parsedResponse.rubricAdherence) { + // Check if it's a real rubric error or just subjective disagreement + const rubricScores = input.proposedGrading.rubricScores || []; + const hasInvalidPoints = rubricScores.some((score) => { + // This would need actual validation against criteria + // For now, just check if points are reasonable + return ( + score.pointsAwarded < 0 || score.pointsAwarded > score.maxPoints + ); + }); + + if (hasInvalidPoints) { + result.issues = ["Invalid rubric point values used"]; + } else { + // If rubric values are technically valid, don't reject for subjective disagreement + this.logger.info( + "Judge disagrees with rubric scoring but values are valid. Approving.", + ); + return { + approved: true, + feedback: + "Grading is technically correct despite subjective concerns", + issues: [], + }; + } + } + + // Don't add corrections for subjective disagreements + if ( + parsedResponse.suggestedFeedbackChanges && + parsedResponse.fairnessScore < 5 + ) { + result.corrections.feedback = parsedResponse.suggestedFeedbackChanges; + } + + if ( + parsedResponse.correctedRubricScores && + Array.isArray(parsedResponse.correctedRubricScores) && + parsedResponse.correctedRubricScores.length > 0 && + !mathIsCorrect // Only suggest rubric corrections if math is wrong + ) { + result.corrections.rubricScores = parsedResponse.correctedRubricScores; + } + } + + return result; + } + + private getOrCreateParser(): StructuredOutputParser< + typeof ParsedJudgeResponseSchema + > { + const cacheKey = {}; + let parser = judgeParserCache.get(cacheKey); + + if (!parser) { + parser = StructuredOutputParser.fromZodSchema(ParsedJudgeResponseSchema); + judgeParserCache.set(cacheKey, parser); + } + + return parser as StructuredOutputParser; + } + + private async processWithTimeout( + promise: Promise, + timeoutMs: number, + ): Promise { + const timeoutPromise = new Promise((_, reject) => { + setTimeout( + () => reject(new Error(`Operation timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + }); + + return Promise.race([promise, timeoutPromise]); + } + + private loadJudgeTemplate(): string { + return `Validate grading for MATHEMATICAL ACCURACY & TECHNICAL COMPLIANCE only. DO NOT re-grade. + +GRADING TO VALIDATE: +Points: {proposed_points} | Sum: {actual_sum} | Max: {max_points} +Scores: {proposed_rubric_scores} + +CRITERIA: {scoring_criteria} + +VALIDATION: +1. Math: {proposed_points} = {actual_sum}? If NO → REJECT +2. Valid rubric values per criteria? If NO → REJECT +3. Extremely unfair (fairness < 5/10)? If YES → REJECT + +✅ APPROVE: Math correct + valid values + fairness ≥5 +❌ REJECT: Math wrong OR invalid values OR fairness <5 + +Context: {question} | {learner_response} + +{format_instructions}`; + } +} diff --git a/apps/api/src/api/llm/features/grading/services/image-grading.service.ts b/apps/api/src/api/llm/features/grading/services/image-grading.service.ts index ece6e6be..c2ca241f 100644 --- a/apps/api/src/api/llm/features/grading/services/image-grading.service.ts +++ b/apps/api/src/api/llm/features/grading/services/image-grading.service.ts @@ -1,4 +1,5 @@ -/* eslint-disable */ +/* eslint-disable @typescript-eslint/require-await */ +/* eslint-disable unicorn/no-null */ import { PromptTemplate } from "@langchain/core/prompts"; import { HttpException, HttpStatus, Inject, Injectable } from "@nestjs/common"; import { AIUsageType } from "@prisma/client"; @@ -79,7 +80,6 @@ export class ImageGradingService implements IImageGradingService { topBucket, topKey, learnerImages, - assignmentId, ); const maxTotalPoints = this.calculateMaxPoints( @@ -225,6 +225,8 @@ GRADING STANDARDS FOR IMAGES (STRICTLY ENFORCED): Remember: Most casual photographs should score in the "Needs Improvement" or "Satisfactory" range unless they demonstrate exceptional qualities. +Make sure your feedback is short and concise. + Respond with a JSON object containing: - Points awarded (sum of all rubric scores) - Separate fields for each AEEG component (analysis, evaluation, explanation, guidance) @@ -260,23 +262,13 @@ Respond with a JSON object containing: finalPoints = 0; } - // Format rubric scores if available - let rubricDetails = ""; - if (parsed.rubricScores && parsed.rubricScores.length > 0) { - rubricDetails = "\n\n**Rubric Scoring:**\n"; - for (const score of parsed.rubricScores) { - rubricDetails += `${score.pointsAwarded}/${score.maxPoints} points\n`; - rubricDetails += `Justification: ${score.justification}\n\n`; - } - } - // Combine the AEEG components into comprehensive feedback const aeegFeedback = ` **Analysis:** ${parsed.analysis} **Evaluation:** -${parsed.evaluation}${rubricDetails} +${parsed.evaluation} **Explanation:** ${parsed.explanation} @@ -430,7 +422,6 @@ ${parsed.guidance} topBucket: string, topKey: string, learnerImages: LearnerImageUpload[], - assignmentId: number, ): Promise { if (topImageData && topImageData !== "InCos") { return await this.processDirectImageData(topImageData); diff --git a/apps/api/src/api/llm/features/grading/services/presentation-grading.service.ts b/apps/api/src/api/llm/features/grading/services/presentation-grading.service.ts index bc58c26c..cccca3f1 100644 --- a/apps/api/src/api/llm/features/grading/services/presentation-grading.service.ts +++ b/apps/api/src/api/llm/features/grading/services/presentation-grading.service.ts @@ -163,26 +163,13 @@ export class PresentationGradingService implements IPresentationGradingService { const parsedResponse = await parser.parse(response); console.log("Parsed Response:", parsedResponse); - // Format rubric scores if available - let rubricDetails = ""; - if ( - parsedResponse.rubricScores && - parsedResponse.rubricScores.length > 0 - ) { - rubricDetails = "\n\n**Rubric Scoring:**\n"; - for (const score of parsedResponse.rubricScores) { - rubricDetails += `${score.pointsAwarded}/${score.maxPoints} points\n`; - rubricDetails += `Justification: ${score.justification}\n\n`; - } - } - // Combine the AEEG components into comprehensive feedback const aeegFeedback = ` **Analysis:** ${parsedResponse.analysis} **Evaluation:** -${parsedResponse.evaluation}${rubricDetails} +${parsedResponse.evaluation} **Explanation:** ${parsedResponse.explanation} diff --git a/apps/api/src/api/llm/features/grading/services/text-grading.service.ts b/apps/api/src/api/llm/features/grading/services/text-grading.service.ts index 0252ef9c..c9d3819a 100644 --- a/apps/api/src/api/llm/features/grading/services/text-grading.service.ts +++ b/apps/api/src/api/llm/features/grading/services/text-grading.service.ts @@ -1,66 +1,313 @@ +/* eslint-disable unicorn/no-null */ // src/llm/features/grading/services/text-grading.service.ts import { PromptTemplate } from "@langchain/core/prompts"; import { HttpException, HttpStatus, Inject, Injectable } from "@nestjs/common"; import { AIUsageType } from "@prisma/client"; import { StructuredOutputParser } from "langchain/output_parsers"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { + CriteriaDto, + RubricDto, + ScoringDto, +} from "src/api/assignment/dto/update.questions.request.dto"; +import { RubricScore } from "src/api/llm/model/file.based.question.response.model"; import { TextBasedQuestionEvaluateModel } from "src/api/llm/model/text.based.question.evaluate.model"; -import { TextBasedQuestionResponseModel } from "src/api/llm/model/text.based.question.response.model"; +import { + GradingMetadata, + TextBasedQuestionResponseModel, +} from "src/api/llm/model/text.based.question.response.model"; import { Logger } from "winston"; import { z } from "zod"; - -// Define types to avoid deep instantiation issues -type RubricScore = { - rubricQuestion: string; - pointsAwarded: number; - maxPoints: number; - justification: string; -}; - -type GradingOutput = { - points: number; - feedback: string; - analysis: string; - evaluation: string; - explanation: string; - guidance: string; - rubricScores?: RubricScore[]; -}; import { IModerationService } from "../../../core/interfaces/moderation.interface"; import { IPromptProcessor } from "../../../core/interfaces/prompt-processor.interface"; import { + GRADING_JUDGE_SERVICE, MODERATION_SERVICE, PROMPT_PROCESSOR, RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS, } from "../../../llm.constants"; +import { IGradingJudgeService } from "../interfaces/grading-judge.interface"; import { ITextGradingService } from "../interfaces/text-grading.interface"; +export interface GradingValidation { + isValid: boolean; + issues: string[]; + suggestedCorrections?: { + points?: number; + feedback?: string; + rubricScores?: RubricScore[]; + }; +} + +const GradingAttemptSchema = z.object({ + points: z + .number() + .min(0) + .describe("Total points awarded (sum of all rubric scores)"), + feedback: z + .string() + .describe("Comprehensive feedback following the AEEG approach"), + analysis: z + .string() + .describe( + "Detailed analysis of what is observed in the learner's response", + ), + evaluation: z + .string() + .describe( + "Evaluation of how well the response meets each assessment aspect", + ), + explanation: z + .string() + .describe("Clear reasons for the grade based on specific observations"), + guidance: z.string().describe("Concrete suggestions for improvement"), + rubricScores: z + .array( + z.object({ + rubricQuestion: z.string().describe("The rubric question"), + pointsAwarded: z + .number() + .min(0) + .describe("Points awarded for this rubric"), + maxPoints: z + .number() + .describe("Maximum points available for this rubric"), + criterionSelected: z + .string() + .describe("The specific criterion level selected"), + justification: z + .string() + .describe("Detailed justification for the score"), + }), + ) + .describe("Individual scores for each rubric criterion") + .optional(), + gradingRationale: z + .string() + .describe("Internal rationale for ensuring consistent grading"), +}); + +export type GradingAttempt = z.infer; + +// Singleton parser instance to avoid recreation +let singletonParser: StructuredOutputParser< + typeof GradingAttemptSchema +> | null = null; + @Injectable() export class TextGradingService implements ITextGradingService { private readonly logger: Logger; + private readonly maxRetries = 3; + private readonly retryDelay = 1000; // ms constructor( @Inject(PROMPT_PROCESSOR) private readonly promptProcessor: IPromptProcessor, @Inject(MODERATION_SERVICE) private readonly moderationService: IModerationService, + @Inject(GRADING_JUDGE_SERVICE) + private readonly gradingJudgeService: IGradingJudgeService, @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, ) { this.logger = parentLogger.child({ context: TextGradingService.name }); } /** - * Grade a text-based question response + * Grade a text-based question response with judge validation */ async gradeTextBasedQuestion( textBasedQuestionEvaluateModel: TextBasedQuestionEvaluateModel, assignmentId: number, language?: string, ): Promise { + const startTime = Date.now(); + + try { + const { question, learnerResponse, totalPoints, scoringCriteria } = + textBasedQuestionEvaluateModel; + + // Sanitize learner response to prevent injection attacks + const sanitizedLearnerResponse = this.sanitizeInput(learnerResponse); + + // Validate the learner's response + const isValidResponse = await this.moderationService.validateContent( + sanitizedLearnerResponse, + ); + if (!isValidResponse) { + throw new HttpException( + "Learner response validation failed", + HttpStatus.BAD_REQUEST, + ); + } + + // Calculate maximum possible points from rubrics + const maxPossiblePoints = this.calculateMaxPossiblePoints( + scoringCriteria as ScoringDto, + totalPoints, + ); + + // Generate content hash for consistency checking + const contentHash = this.generateContentHash(learnerResponse, question); + + // Attempt grading with judge validation loop + let gradingAttempt: GradingAttempt | null = null; + let judgeApproved = false; + let attemptCount = 0; + let previousJudgeFeedback: string | null = null; + + while (!judgeApproved && attemptCount < this.maxRetries) { + attemptCount++; + this.logger.info( + `Grading attempt ${attemptCount}/${this.maxRetries} for assignment ${assignmentId}`, + ); + + try { + // Generate grading + gradingAttempt = await this.generateGrading( + textBasedQuestionEvaluateModel, + maxPossiblePoints, + contentHash, + assignmentId, + language, + previousJudgeFeedback, + ); + + // Validate with judge + const judgeResult = await this.validateWithJudge( + question, + sanitizedLearnerResponse, + scoringCriteria as ScoringDto, + gradingAttempt, + maxPossiblePoints, + assignmentId, + ); + + if (judgeResult.approved) { + judgeApproved = true; + this.logger.info( + `Judge approved grading on attempt ${attemptCount}`, + ); + } else { + // Format judge feedback to be more actionable for the TA + previousJudgeFeedback = this.formatJudgeFeedbackForTA( + judgeResult, + attemptCount, + ); + this.logger.warn( + `Judge rejected grading attempt ${attemptCount}: ${judgeResult.feedback}`, + ); + + // Apply judge's corrections if provided + if (judgeResult.corrections && gradingAttempt) { + const originalPoints = gradingAttempt.points; + gradingAttempt = this.applyJudgeCorrections( + gradingAttempt, + judgeResult.corrections, + ); + + // If corrections were applied, check if we should approve + if ( + this.areCorrectionsMinor( + judgeResult.corrections, + originalPoints, + maxPossiblePoints, + ) + ) { + judgeApproved = true; + this.logger.info( + "Minor corrections applied, approving grading", + ); + } + } + + // Add exponential backoff delay before retry + if (!judgeApproved && attemptCount < this.maxRetries) { + const backoffDelay = + this.retryDelay * Math.pow(2, attemptCount - 1); + await this.delay(Math.min(backoffDelay, 5000)); // Cap at 5 seconds + } + } + } catch (error) { + this.logger.error( + `Error in grading attempt ${attemptCount}: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + + // On last attempt, use the grading even if judge fails + if (attemptCount === this.maxRetries && gradingAttempt) { + judgeApproved = true; + this.logger.warn( + "Using grading despite judge failure on final attempt", + ); + } + } + } + + if (!gradingAttempt) { + throw new HttpException( + "Failed to generate grading after all attempts", + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } + + // Generate final feedback + const finalFeedback = this.generateAlignedFeedback( + gradingAttempt, + maxPossiblePoints, + ); + + const endTime = Date.now(); + this.logger.info( + `Graded text question - Points: ${gradingAttempt.points}/${maxPossiblePoints}, ` + + `Content Hash: ${contentHash}, Judge Approved: ${judgeApproved.toString()}, ` + + `Time: ${endTime - startTime}ms, Attempts: ${attemptCount}`, + ); + + // Create properly typed metadata + const metadata: GradingMetadata = { + judgeApproved, + attempts: attemptCount, + gradingTimeMs: endTime - startTime, + contentHash, + }; + + // Return the enhanced validated response + return new TextBasedQuestionResponseModel( + gradingAttempt.points, + finalFeedback, + gradingAttempt.analysis, + gradingAttempt.evaluation, + gradingAttempt.explanation, + gradingAttempt.guidance, + this.ensureRequiredRubricFields(gradingAttempt.rubricScores), + gradingAttempt.gradingRationale, + metadata, + ); + } catch (error) { + this.logger.error( + `Failed to grade text question: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + throw error; + } + } + + /** + * Generate a grading attempt + */ + private async generateGrading( + textBasedQuestionEvaluateModel: TextBasedQuestionEvaluateModel, + maxPossiblePoints: number, + contentHash: string, + assignmentId: number, + language?: string, + previousJudgeFeedback?: string | null, + ): Promise { const { question, learnerResponse, - totalPoints, scoringCriteriaType, scoringCriteria, previousQuestionsAnswersContext, @@ -68,48 +315,37 @@ export class TextGradingService implements ITextGradingService { responseType, } = textBasedQuestionEvaluateModel; - // Validate the learner's response - const validateLearnerResponse = - await this.moderationService.validateContent(learnerResponse); - if (!validateLearnerResponse) { - throw new HttpException( - "Learner response validation failed", - HttpStatus.BAD_REQUEST, - ); - } - - // Use simple Zod schema to avoid deep instantiation - const parser = StructuredOutputParser.fromZodSchema( - z.object({ - points: z.number(), - feedback: z.string(), - analysis: z.string(), - evaluation: z.string(), - explanation: z.string(), - guidance: z.string(), - rubricScores: z - .array( - z.object({ - rubricQuestion: z.string(), - pointsAwarded: z.number(), - maxPoints: z.number(), - justification: z.string(), - }), - ) - .optional(), - }), - ); - + // Get or create parser + const parser = this.getOrCreateParser(); const formatInstructions = parser.getFormatInstructions(); // Add response-specific instructions based on the type const responseSpecificInstruction = - (RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS[responseType] as - | string - | undefined) ?? ""; + RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS[ + responseType as keyof typeof RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS + ] ?? ""; + + // Load the enhanced grading template with proper rubric data + const template = this.loadEnhancedTextGradingTemplate(); - // Load the grading template - const template = await this.loadTextGradingTemplate(); + // Debug rubric data being passed to LLM + this.logger.info("Rubric data being passed to LLM", { + assignmentId, + scoringCriteriaType, + hasRubrics: !!(scoringCriteria as ScoringDto)?.rubrics, + rubricCount: (scoringCriteria as ScoringDto)?.rubrics?.length || 0, + rubrics: + (scoringCriteria as ScoringDto)?.rubrics?.map((r, index) => ({ + index: index, + question: r.rubricQuestion, + criteriaCount: r.criteria?.length || 0, + criteria: + r.criteria?.map((c) => ({ + description: c.description, + points: c.points, + })) || [], + })) || [], + }); const prompt = new PromptTemplate({ template, @@ -121,148 +357,572 @@ export class TextGradingService implements ITextGradingService { previous_questions_and_answers: () => JSON.stringify(previousQuestionsAnswersContext ?? []), learner_response: () => learnerResponse, - total_points: () => totalPoints.toString(), + total_points: () => maxPossiblePoints.toString(), scoring_type: () => scoringCriteriaType, scoring_criteria: () => JSON.stringify(scoringCriteria), format_instructions: () => formatInstructions, grading_type: () => responseType, language: () => language ?? "en", + content_hash: () => contentHash, + judge_feedback: () => previousJudgeFeedback || "No previous feedback", }, }); - // Process the prompt through the LLM - const response = await this.promptProcessor.processPrompt( + // Process the prompt through the LLM using dynamic model assignment + const response = await this.promptProcessor.processPromptForFeature( prompt, assignmentId, AIUsageType.ASSIGNMENT_GRADING, + "text_grading", + "gpt-4o-mini", // fallback for text grading ); - try { - // Parse the response into the expected output format - const parsedResponse = (await parser.parse(response)) as GradingOutput; - console.log("Parsed grading response:", parsedResponse); - - // Format rubric scores if available - let rubricDetails = ""; - if ( - parsedResponse.rubricScores && - parsedResponse.rubricScores.length > 0 - ) { - rubricDetails = "\n\n**Rubric Scoring:**\n"; - for (const score of parsedResponse.rubricScores) { - rubricDetails += `${score.pointsAwarded}/${score.maxPoints} points\n`; - rubricDetails += `Justification: ${score.justification}\n\n`; - } - } + const parsedResponse = await parser.parse(response); - // Combine the AEEG components into comprehensive feedback - const aeegFeedback = ` -**Analysis:** -${parsedResponse.analysis} - -**Evaluation:** -${parsedResponse.evaluation} - -**Explanation:** -${parsedResponse.explanation} + // Log the LLM's original grading for audit purposes + this.logger.info( + `LLM grading result - Points: ${parsedResponse.points}/${maxPossiblePoints}, ` + + `Rubric scores: ${parsedResponse.rubricScores?.length || 0} items`, + ); -**Guidance:** -${parsedResponse.guidance}${rubricDetails} -`.trim(); + return parsedResponse; + } - // Return the response with combined feedback - return { - points: parsedResponse.points, - feedback: aeegFeedback, - } as TextBasedQuestionResponseModel; + /** + * Validate grading with judge service + */ + private async validateWithJudge( + question: string, + learnerResponse: string, + scoringCriteria: ScoringDto, + gradingAttempt: GradingAttempt, + maxPossiblePoints: number, + assignmentId: number, + ) { + try { + return await this.gradingJudgeService.validateGrading({ + question, + learnerResponse, + scoringCriteria, + proposedGrading: { + points: gradingAttempt.points, + maxPoints: maxPossiblePoints, + feedback: this.generateAlignedFeedback( + gradingAttempt, + maxPossiblePoints, + ), + rubricScores: gradingAttempt.rubricScores as RubricDto[], + analysis: gradingAttempt.analysis, + evaluation: gradingAttempt.evaluation, + explanation: gradingAttempt.explanation, + guidance: gradingAttempt.guidance, + }, + assignmentId, + }); } catch (error) { this.logger.error( - `Error parsing LLM response: ${ + `Judge validation failed: ${ error instanceof Error ? error.message : "Unknown error" }`, ); - throw new HttpException( - "Failed to parse grading response", - HttpStatus.INTERNAL_SERVER_ERROR, + // Return approved by default if judge fails + return { + approved: true, + feedback: "Judge validation failed, approving by default", + }; + } + } + + /** + * Apply corrections from judge + */ + private applyJudgeCorrections( + gradingAttempt: GradingAttempt, + corrections: { + points?: number; + feedback?: string; + rubricScores?: RubricScore[]; + }, + ): GradingAttempt { + const corrected = { ...gradingAttempt }; + + if (corrections.points !== undefined) { + corrected.points = corrections.points; + } + + if (corrections.feedback) { + corrected.explanation = `${corrected.explanation}\n\n**Judge Adjustment**: ${corrections.feedback}`; + } + + if (corrections.rubricScores && Array.isArray(corrections.rubricScores)) { + corrected.rubricScores = corrections.rubricScores; + // Recalculate total points from rubric scores + corrected.points = corrections.rubricScores.reduce( + (sum: number, score: RubricScore) => sum + (score.pointsAwarded || 0), + 0, + ); + } + + return corrected; + } + + /** + * Check if corrections are minor enough to auto-approve + */ + private areCorrectionsMinor( + corrections: { + points?: number; + feedback?: string; + rubricScores?: RubricScore[]; + }, + originalPoints?: number, + maxPoints?: number, + ): boolean { + // Only consider minor if it's just a small point adjustment + if (corrections.rubricScores) return false; + if (corrections.feedback) return false; + + if ( + corrections.points !== undefined && + originalPoints !== undefined && + maxPoints !== undefined + ) { + const pointDifference = Math.abs(corrections.points - originalPoints); + const percentageChange = + maxPoints > 0 ? (pointDifference / maxPoints) * 100 : 0; + + // Auto-approve if point adjustment is small (within 5% of max points) + return percentageChange <= 5; + } + return false; + } + + /** + * Sanitize user input to prevent prompt injection and other attacks + */ + private sanitizeInput(input: string): string { + if (!input || typeof input !== "string") { + return ""; + } + + // Remove or escape potentially dangerous patterns + return ( + input + // Remove null bytes and control characters except newlines and tabs + .replaceAll(/[^\t\n\r\u0020-\u007E\u00A0-\uFFFF]/gu, "") + // Limit consecutive newlines to prevent prompt breaking + .replaceAll(/\n{3,}/g, "\n\n") + // Remove potential prompt injection markers + .replaceAll(/(?:^|\n)\s*(?:system|user|assistant|human):/gi, "") + // Remove common LLM instruction patterns + .replaceAll( + /(?:^|\n)\s*(?:ignore|disregard|forget).*?(?:instruction|prompt|rule)/gi, + "", + ) + // Truncate if too long + .slice(0, 10_000) + .trim() + ); + } + + /** + * Compare numbers with zero tolerance - must be exactly equal + */ + private areNumbersEqual(a: number, b: number, tolerance = 0): boolean { + return Math.abs(a - b) <= tolerance; + } + + /** + * Get or create parser (singleton for performance) + */ + private getOrCreateParser(): StructuredOutputParser< + typeof GradingAttemptSchema + > { + if (!singletonParser) { + singletonParser = StructuredOutputParser.fromZodSchema( + z.object({ + points: z + .number() + .min(0) + .describe("Total points awarded (sum of all rubric scores)"), + feedback: z + .string() + .describe("Comprehensive feedback following the AEEG approach"), + analysis: z + .string() + .describe( + "Detailed analysis of what is observed in the learner's response", + ), + evaluation: z + .string() + .describe( + "Evaluation of how well the response meets each assessment aspect", + ), + explanation: z + .string() + .describe( + "Clear reasons for the grade based on specific observations", + ), + guidance: z.string().describe("Concrete suggestions for improvement"), + rubricScores: z + .array( + z.object({ + rubricQuestion: z.string().describe("The rubric question"), + pointsAwarded: z + .number() + .min(0) + .describe("Points awarded for this rubric"), + maxPoints: z + .number() + .describe("Maximum points available for this rubric"), + criterionSelected: z + .string() + .describe("The specific criterion level selected"), + justification: z + .string() + .describe("Detailed justification for the score"), + }), + ) + .describe("Individual scores for each rubric criterion") + .optional(), + gradingRationale: z + .string() + .describe("Internal rationale for ensuring consistent grading"), + }), ); } + + return singletonParser; + } + + /** + * Calculate maximum possible points from scoring criteria + */ + private calculateMaxPossiblePoints( + scoringCriteria: ScoringDto, + defaultTotal: number, + ): number { + if ( + !scoringCriteria || + !Array.isArray(scoringCriteria.rubrics) || + scoringCriteria.rubrics.length === 0 + ) { + return defaultTotal; + } + + let maxPoints = 0; + for (const rubric of scoringCriteria.rubrics) { + if (rubric?.criteria && Array.isArray(rubric.criteria)) { + const rubricMax = Math.max( + 0, + ...rubric.criteria + .filter((c: CriteriaDto) => typeof c?.points === "number") + .map((c: CriteriaDto) => c.points), + ); + maxPoints += rubricMax; + } + } + + return maxPoints > 0 ? maxPoints : defaultTotal; + } + + /** + * Generate a content hash for consistency checking + */ + private generateContentHash( + learnerResponse: string, + question: string, + ): string { + // Create a normalized version of the response for comparison + const normalizedResponse = learnerResponse + .toLowerCase() + .replaceAll(/[^\s\w]/g, "") + .replaceAll(/\s+/g, " ") + .trim() + .slice(0, 1000); // Limit length for performance + + const normalizedQuestion = question + .toLowerCase() + .replaceAll(/[^\s\w]/g, "") + .replaceAll(/\s+/g, " ") + .trim() + .slice(0, 500); + + // Use a more efficient hashing approach + const combined = `${normalizedQuestion}:${normalizedResponse}`; + return Buffer.from(combined).toString("base64").slice(0, 16); + } + + /** + * Validate feedback tone alignment with score percentage + */ + private validateFeedbackAlignment( + gradingAttempt: GradingAttempt, + scorePercentage: number, + ): string | null { + const feedback = gradingAttempt.feedback || ""; + const explanation = gradingAttempt.explanation || ""; + const guidance = gradingAttempt.guidance || ""; + const allText = `${feedback} ${explanation} ${guidance}`.toLowerCase(); + + // Define tone indicators + const positiveWords = [ + "excellent", + "outstanding", + "great", + "strong", + "impressive", + "well done", + "good job", + ]; + const negativeWords = [ + "poor", + "weak", + "inadequate", + "lacking", + "missing", + "incomplete", + "fails", + "incorrect", + ]; + const encouragingWords = ["keep up", "continue", "maintain", "build on"]; + const criticalWords = [ + "needs improvement", + "must improve", + "requires work", + "significant issues", + ]; + + const positiveCount = positiveWords.reduce( + (count, word) => count + (allText.includes(word) ? 1 : 0), + 0, + ); + const negativeCount = negativeWords.reduce( + (count, word) => count + (allText.includes(word) ? 1 : 0), + 0, + ); + const encouragingCount = encouragingWords.reduce( + (count, word) => count + (allText.includes(word) ? 1 : 0), + 0, + ); + const criticalCount = criticalWords.reduce( + (count, word) => count + (allText.includes(word) ? 1 : 0), + 0, + ); + + // Check alignment based on score percentage + if ( + scorePercentage >= 85 && + (negativeCount > positiveCount || criticalCount > encouragingCount) + ) { + return `High score (${Math.round( + scorePercentage, + )}%) but overly negative feedback tone`; + } + + if ( + scorePercentage <= 40 && + (positiveCount > negativeCount || encouragingCount > criticalCount) + ) { + return `Low score (${Math.round( + scorePercentage, + )}%) but overly positive feedback tone`; + } + + if ( + scorePercentage >= 70 && + scorePercentage < 85 && + negativeCount > positiveCount + 2 + ) { + return `Good score (${Math.round( + scorePercentage, + )}%) but excessively critical feedback`; + } + + return null; // Alignment is acceptable } /** - * Load the text grading template with AEEG approach + * Convert optional rubric scores to required format */ - // eslint-disable-next-line @typescript-eslint/require-await - private async loadTextGradingTemplate(): Promise { + private ensureRequiredRubricFields( + rubricScores?: RubricScore[], + ): RubricScore[] { + if (!rubricScores || !Array.isArray(rubricScores)) { + return []; + } + + return rubricScores.map((score, index) => ({ + rubricQuestion: score.rubricQuestion || `Rubric ${index + 1}`, + pointsAwarded: score.pointsAwarded || 0, + maxPoints: score.maxPoints || 0, + criterionSelected: score.criterionSelected || "Default criterion", + justification: score.justification || "Auto-generated justification", + })); + } + + /** + * Generate ultra-concise feedback - no redundancy + */ + private generateAlignedFeedback( + gradingAttempt: GradingAttempt, + maxPossiblePoints: number, + ): string { + const percentage = + maxPossiblePoints > 0 + ? Math.round((gradingAttempt.points / maxPossiblePoints) * 100) + : 0; + + // Super simple format: just the essential info + const conciseFeedback = `${gradingAttempt.explanation} + +${gradingAttempt.guidance} + +**Score: ${gradingAttempt.points}/${maxPossiblePoints} (${percentage}%)**`.trim(); + + return conciseFeedback; + } + + /** + * Get contextual introduction based on score percentage + */ + private getScoreContext(percentage: number): string { + if (percentage >= 95) { + return "You achieved an outstanding score."; + } else if (percentage >= 90) { + return "You achieved an excellent score."; + } else if (percentage >= 85) { + return "You achieved a very good score."; + } else if (percentage >= 80) { + return "You achieved a good score."; + } else if (percentage >= 75) { + return "You achieved an above average score."; + } else if (percentage >= 70) { + return "You achieved a satisfactory score."; + } else if (percentage >= 65) { + return "You achieved an adequate score with room for improvement."; + } else if (percentage >= 60) { + return "Your score indicates areas for improvement."; + } else if (percentage >= 50) { + return "Your score indicates significant room for improvement."; + } else { + return "Your score indicates substantial areas needing improvement."; + } + } + + /** + * Format judge feedback to be more actionable for the grading assistant + */ + private formatJudgeFeedbackForTA( + judgeResult: { + feedback?: string; + issues?: string[]; + corrections?: { + points?: number; + feedback?: string; + rubricScores?: unknown[]; + }; + }, + attemptNumber: number, + ): string { + let formattedFeedback = `📋 GRADING FEEDBACK - ATTEMPT ${attemptNumber}:\n\n`; + + // Extract issues from judge feedback + if (judgeResult.issues && Array.isArray(judgeResult.issues)) { + formattedFeedback += `🚨 CRITICAL ISSUES TO FIX:\n`; + for (const [index, issue] of judgeResult.issues.entries()) { + formattedFeedback += `${index + 1}. ${issue}\n`; + } + formattedFeedback += "\n"; + } + + // Add structured feedback based on common patterns + const feedback = judgeResult.feedback || ""; + formattedFeedback += `📝 DETAILED FEEDBACK:\n${feedback}\n\n`; + + // Add specific corrections if available + if (judgeResult.corrections) { + formattedFeedback += `✅ REQUIRED CORRECTIONS:\n`; + if (judgeResult.corrections.points !== undefined) { + formattedFeedback += `• Adjust total points to: ${judgeResult.corrections.points}\n`; + } + if (judgeResult.corrections.feedback) { + formattedFeedback += `• Update feedback: ${judgeResult.corrections.feedback}\n`; + } + if (judgeResult.corrections.rubricScores) { + formattedFeedback += `• Fix rubric scores as specified\n`; + } + formattedFeedback += "\n"; + } + + // Add learning guidance + formattedFeedback += `🎯 WHAT YOU MUST DO DIFFERENTLY:\n`; + if (feedback.includes("mathematical")) { + formattedFeedback += `• Double-check ALL math: Total points MUST equal sum of rubric scores\n`; + } + if (feedback.includes("feedback") && feedback.includes("align")) { + formattedFeedback += `• Ensure your explanations clearly justify the scores given\n`; + formattedFeedback += `• Use specific student quotes as evidence for each point awarded/deducted\n`; + } + if (feedback.includes("rubric")) { + formattedFeedback += `• Follow rubric criteria exactly - pick the ONE criterion that best fits\n`; + formattedFeedback += `• Use EXACT point values from the criteria, no custom points\n`; + } + if (feedback.includes("specific") || feedback.includes("evidence")) { + formattedFeedback += `• Quote specific parts of the student response to justify scores\n`; + formattedFeedback += `• Provide concrete evidence for every point awarded or deducted\n`; + } + + formattedFeedback += `\n💡 REMEMBER: This feedback helps you improve accuracy. Learn from it!`; + + return formattedFeedback; + } + + /** + * Utility function for delays + */ + private delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + /** + * Load the enhanced text grading template with robust validation + */ + private loadEnhancedTextGradingTemplate(): string { return ` - You are an expert educator evaluating a student's response to a question using the AEEG (Analyze, Evaluate, Explain, Guide) approach. - - QUESTION: - {question} + You are an educational grading assistant helping evaluate student work fairly and accurately. - ASSIGNMENT INSTRUCTIONS: - {assignment_instructions} + PREVIOUS FEEDBACK: {judge_feedback} + If feedback above exists, please address the issues mentioned. - PREVIOUS QUESTIONS AND ANSWERS: - {previous_questions_and_answers} - - STUDENT'S RESPONSE: + STUDENT RESPONSE TO GRADE: {learner_response} - RESPONSE TYPE SPECIFIC INSTRUCTIONS: - {responseSpecificInstruction} - - SCORING INFORMATION: - Total Points Available: {total_points} - Scoring Type: {scoring_type} - Scoring Criteria: {scoring_criteria} - - CRITICAL GRADING INSTRUCTIONS: - You MUST grade according to the EXACT rubric provided in the scoring criteria. If the scoring type is "CRITERIA_BASED" with rubrics: - 1. Evaluate the response against EACH rubric question provided - 2. Award points based ONLY on the criteria descriptions provided for each rubric - 3. Do NOT use generic essay criteria like "creativity" or "depth of analysis" unless specifically mentioned in the rubric - 4. For each rubric, select the criterion that best matches the student's performance and award those exact points - 5. The total points awarded must equal the sum of points from all rubrics - 6. Include specific justification for why you selected each criterion level + DETAILED RUBRIC CRITERIA AND SCORING: + {scoring_criteria} + Max Points Available: {total_points} - GRADING APPROACH (AEEG): + CRITICAL INSTRUCTIONS: - 1. ANALYZE: Carefully examine the student's response and describe what you observe - - Identify key points made by the student - - Note the structure and organization of their response - - Recognize any evidence, examples, or reasoning provided - - Observe the clarity and coherence of their communication - - Focus on aspects relevant to the rubric criteria + 1. CAREFULLY examine each rubric question and its scoring criteria + 2. For each rubric, select the ONE criterion that BEST matches the student's response + 3. Use the EXACT point value from the selected criterion - no custom points allowed + 4. Your total points MUST equal the sum of all rubric scores + 5. Quote specific parts of the student response as evidence for your scoring decisions + 6. Provide constructive feedback based on what you observe - 2. EVALUATE: For each rubric question in the scoring criteria: - - Read the rubric question carefully - - Compare the response against each criterion level - - Select the criterion that best matches the student's performance - - Award the exact points specified for that criterion - - Do NOT average or adjust points - use the exact values provided + RUBRIC SCORING REQUIREMENTS: + - You must score ALL rubric questions provided in the criteria + - Each rubric score must use a valid point value from its criteria options + - Justify each score with specific evidence from the student response + - Total points = sum of all individual rubric scores (this is mandatory) - 3. EXPLAIN: Provide clear reasons for the grade based on specific observations - - Justify why you selected each criterion level for each rubric - - Reference specific parts of the student's response that led to your decisions - - Connect your observations from the analysis to the evaluation outcomes - - Make the grading rationale transparent and understandable - - Ensure explanations align with the rubric criteria used - - 4. GUIDE: Offer concrete suggestions for improvement - - Provide specific, actionable feedback based on the rubric criteria - - Suggest ways to reach higher criterion levels in each rubric - - Recommend resources or strategies for improvement - - Focus guidance on the specific skills assessed by the rubrics - - LANGUAGE: {language} - - Respond with a JSON object containing: - - Points awarded (sum of all rubric scores) - - Comprehensive feedback incorporating all four AEEG components - - Separate fields for each AEEG component - - If scoring type is CRITERIA_BASED, include rubricScores array with score for each rubric - - Format your response according to: + Question: {question} + Assignment Instructions: {assignment_instructions} + Language: {language} + + Make sure your feedback is short and concise. + {format_instructions} `; } diff --git a/apps/api/src/api/llm/features/grading/services/url-grading.service.ts b/apps/api/src/api/llm/features/grading/services/url-grading.service.ts index 5adb5fe0..c8c724b7 100644 --- a/apps/api/src/api/llm/features/grading/services/url-grading.service.ts +++ b/apps/api/src/api/llm/features/grading/services/url-grading.service.ts @@ -74,7 +74,7 @@ export class UrlGradingService implements IUrlGradingService { const formatInstructions = parser.getFormatInstructions(); const responseSpecificInstruction: string = - (RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS[responseType] as string) ?? ""; + RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS[responseType] ?? ""; const prompt = new PromptTemplate({ template: this.loadUrlGradingTemplate(), diff --git a/apps/api/src/api/llm/features/question-generation/interfaces/question-validator.interface.ts b/apps/api/src/api/llm/features/question-generation/interfaces/question-validator.interface.ts index 89b52132..b3cee17e 100644 --- a/apps/api/src/api/llm/features/question-generation/interfaces/question-validator.interface.ts +++ b/apps/api/src/api/llm/features/question-generation/interfaces/question-validator.interface.ts @@ -1,6 +1,6 @@ import { EnhancedQuestionsToGenerate } from "src/api/assignment/dto/post.assignment.request.dto"; -import { ValidationResult } from "../services/question-validator.service"; import { DifficultyLevel } from "../services/question-generation.service"; +import { ValidationResult } from "../services/question-validator.service"; /** * Interface for the question validator service diff --git a/apps/api/src/api/llm/features/question-generation/question-generation.module.ts b/apps/api/src/api/llm/features/question-generation/question-generation.module.ts index dac517b2..c1eff938 100644 --- a/apps/api/src/api/llm/features/question-generation/question-generation.module.ts +++ b/apps/api/src/api/llm/features/question-generation/question-generation.module.ts @@ -1,10 +1,10 @@ import { Module } from "@nestjs/common"; import { WinstonModule } from "nest-winston"; import { PrismaService } from "../../../../prisma.service"; -import { QuestionGenerationService } from "./services/question-generation.service"; import { QUESTION_GENERATION_SERVICE } from "../../llm.constants"; -import { QuestionTemplateService } from "./services/question-template.service"; import { LlmModule } from "../../llm.module"; +import { QuestionGenerationService } from "./services/question-generation.service"; +import { QuestionTemplateService } from "./services/question-template.service"; @Module({ imports: [LlmModule, WinstonModule], diff --git a/apps/api/src/api/llm/features/question-generation/services/question-generation.service.ts b/apps/api/src/api/llm/features/question-generation/services/question-generation.service.ts index 4d16cf64..95d0106b 100644 --- a/apps/api/src/api/llm/features/question-generation/services/question-generation.service.ts +++ b/apps/api/src/api/llm/features/question-generation/services/question-generation.service.ts @@ -236,7 +236,7 @@ export class QuestionGenerationService implements IQuestionGenerationService { for (let attempt = 0; attempt < this.MAX_GENERATION_RETRIES; attempt++) { try { // Create prompt focused on current batch - const parser = this.createOutputParser(types, counts); + const parser = this.createOutputParser(types); const prompt = this.createBatchPrompt( types, counts, @@ -361,7 +361,6 @@ export class QuestionGenerationService implements IQuestionGenerationService { // If we failed completely, use template questions as fallback if (!success && generatedQuestions.length < totalCount) { this.logger.warn("Generation failed, using fallbacks"); - const remaining = totalCount - generatedQuestions.length; const fallbacks = this.generateFallbackQuestions( types, counts.map((count) => @@ -388,7 +387,6 @@ export class QuestionGenerationService implements IQuestionGenerationService { private createOutputParser( types: QuestionType[], - counts: number[], ): StructuredOutputParser { return StructuredOutputParser.fromZodSchema( z.object({ @@ -674,8 +672,7 @@ FORMAT INSTRUCTIONS: question: question.question?.replaceAll("```", "").trim(), totalPoints: question.totalPoints || this.getDefaultPoints(question.type), type: question.type, - responseType: - question.responseType || this.getDefaultResponseType(question.type), + responseType: question.responseType || this.getDefaultResponseType(), difficultyLevel: question.difficultyLevel, maxWords: question.maxWords || @@ -1238,7 +1235,7 @@ FORMAT INSTRUCTIONS: question: questionText, totalPoints: this.getDefaultPoints(type, difficultyLevel), type: type, - responseType: this.getDefaultResponseType(type), + responseType: this.getDefaultResponseType(), difficultyLevel: difficultyLevel, scoring: this.getDefaultScoring(type, difficultyLevel), }; @@ -1668,7 +1665,7 @@ FORMAT INSTRUCTIONS: return undefined; } - private getDefaultResponseType(questionType: QuestionType): ResponseType { + private getDefaultResponseType(): ResponseType { return ResponseType.OTHER; } private getDefaultChoices( @@ -2043,7 +2040,7 @@ FORMAT INSTRUCTIONS: .array( z.object({ choice: z.enum(["true", "false", "True", "False"]), - points: z.number().min(1), + points: z.number().min(0), feedback: z.string().optional(), isCorrect: z.boolean(), }), @@ -2137,6 +2134,7 @@ QUALITY REQUIREMENTS: - Ensure distractors remain equally plausible - Provide educational feedback for each choice - Keep original point distribution + - IMPORTANT: Points must be non-negative integers (>= 0) for all questions 4. Avoid simply: - Changing minor words or punctuation diff --git a/apps/api/src/api/llm/features/rubric/interfaces/rubric.interface.ts b/apps/api/src/api/llm/features/rubric/interfaces/rubric.interface.ts index efd22ef5..925955a2 100644 --- a/apps/api/src/api/llm/features/rubric/interfaces/rubric.interface.ts +++ b/apps/api/src/api/llm/features/rubric/interfaces/rubric.interface.ts @@ -1,8 +1,8 @@ import { - QuestionDto, - ScoringDto, Choice, + QuestionDto, RubricDto, + ScoringDto, } from "../../../../assignment/dto/update.questions.request.dto"; export interface IRubricService { diff --git a/apps/api/src/api/llm/features/rubric/services/rubric.service.ts b/apps/api/src/api/llm/features/rubric/services/rubric.service.ts index c05aa676..b43a2b4a 100644 --- a/apps/api/src/api/llm/features/rubric/services/rubric.service.ts +++ b/apps/api/src/api/llm/features/rubric/services/rubric.service.ts @@ -1,25 +1,24 @@ /* eslint-disable unicorn/no-null */ -import { Injectable, Inject } from "@nestjs/common"; -import { AIUsageType } from "@prisma/client"; import { PromptTemplate } from "@langchain/core/prompts"; +import { Inject, Injectable } from "@nestjs/common"; +import { AIUsageType } from "@prisma/client"; import { StructuredOutputParser } from "langchain/output_parsers"; -import { z } from "zod"; - -import { PROMPT_PROCESSOR } from "../../../llm.constants"; -import { IPromptProcessor } from "../../../core/interfaces/prompt-processor.interface"; -import { IRubricService } from "../interfaces/rubric.interface"; -import { - QuestionDto, - ScoringDto, - RubricDto, - Choice, -} from "../../../../assignment/dto/update.questions.request.dto"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; -import { Logger } from "winston"; import { Criteria, ScoringType, } from "src/api/assignment/question/dto/create.update.question.request.dto"; +import { Logger } from "winston"; +import { z } from "zod"; +import { + Choice, + QuestionDto, + RubricDto, + ScoringDto, +} from "../../../../assignment/dto/update.questions.request.dto"; +import { IPromptProcessor } from "../../../core/interfaces/prompt-processor.interface"; +import { PROMPT_PROCESSOR } from "../../../llm.constants"; +import { IRubricService } from "../interfaces/rubric.interface"; @Injectable() export class RubricService implements IRubricService { @@ -39,7 +38,6 @@ export class RubricService implements IRubricService { async createMarkingRubric( question: QuestionDto, assignmentId: number, - rubricIndex?: number, ): Promise { const rubricTemplate = this.selectRubricTemplate(question.type); @@ -381,12 +379,13 @@ export class RubricService implements IRubricService { }); try { - const response = await this.promptProcessor.processPrompt( + const response = await this.promptProcessor.processPromptForFeature( prompt, assignmentId, - AIUsageType.QUESTION_GENERATION, + AIUsageType.ASSIGNMENT_GRADING, + "rubric_generation", + "gpt-4o-mini", ); - let parsed: | { newRubrics?: { diff --git a/apps/api/src/api/llm/features/translation/interfaces/translation.interface.ts b/apps/api/src/api/llm/features/translation/interfaces/translation.interface.ts index b3a72cc7..681938fe 100644 --- a/apps/api/src/api/llm/features/translation/interfaces/translation.interface.ts +++ b/apps/api/src/api/llm/features/translation/interfaces/translation.interface.ts @@ -5,7 +5,15 @@ export interface ITranslationService { * Detect the language of text */ - getLanguageCode(text: string): Promise; + getLanguageCode(text: string, assignmentId?: number): Promise; + + /** + * Batch detect languages for multiple texts + */ + batchGetLanguageCodes( + texts: string[], + assignmentId?: number, + ): Promise; /** * Translate a question to a target language diff --git a/apps/api/src/api/llm/features/translation/services/translation.service.ts b/apps/api/src/api/llm/features/translation/services/translation.service.ts index c4d4ca10..859021cd 100644 --- a/apps/api/src/api/llm/features/translation/services/translation.service.ts +++ b/apps/api/src/api/llm/features/translation/services/translation.service.ts @@ -1,21 +1,30 @@ -import { Injectable, Inject, HttpException, HttpStatus } from "@nestjs/common"; -import { AIUsageType } from "@prisma/client"; +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable unicorn/prefer-module */ +import * as fs from "node:fs"; +import * as path from "node:path"; import { PromptTemplate } from "@langchain/core/prompts"; +import { HttpException, HttpStatus, Inject, Injectable } from "@nestjs/common"; +import { AIUsageType } from "@prisma/client"; +import cld from "cld"; import { StructuredOutputParser } from "langchain/output_parsers"; +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { decodeIfBase64 } from "src/helpers/decoder"; +import { Logger } from "winston"; import { z } from "zod"; -import cld from "cld"; - -import { PROMPT_PROCESSOR } from "../../../llm.constants"; +import { Choice } from "../../../../assignment/dto/update.questions.request.dto"; import { IPromptProcessor } from "../../../core/interfaces/prompt-processor.interface"; +import { PROMPT_PROCESSOR } from "../../../llm.constants"; import { ITranslationService } from "../interfaces/translation.interface"; -import { Choice } from "../../../../assignment/dto/update.questions.request.dto"; -import { WINSTON_MODULE_PROVIDER } from "nest-winston"; -import { Logger } from "winston"; -import { decodeIfBase64 } from "src/helpers/decoder"; + +interface LanguageMapping { + code: string; + name: string; +} @Injectable() export class TranslationService implements ITranslationService { private readonly logger: Logger; + private languageMap: Map = new Map(); constructor( @Inject(PROMPT_PROCESSOR) @@ -23,23 +32,365 @@ export class TranslationService implements ITranslationService { @Inject(WINSTON_MODULE_PROVIDER) parentLogger: Logger, ) { this.logger = parentLogger.child({ context: TranslationService.name }); + this.loadLanguageMap(); } /** - * Detect the language of text using cld library + * Load language mappings from languages.json file */ + private loadLanguageMap(): void { + try { + // Try multiple possible paths for the languages.json file + const possiblePaths = [ + path.join(process.cwd(), "../../apps/web/public/languages.json"), // When running from api directory + path.join(process.cwd(), "../web/public/languages.json"), // Alternative path + path.join( + __dirname, + "../../../../../../apps/web/public/languages.json", + ), // From compiled js location + path.join(__dirname, "../../../../../web/public/languages.json"), // Alternative compiled location + ]; + + let languagesPath: string | null; + for (const testPath of possiblePaths) { + if (fs.existsSync(testPath)) { + languagesPath = testPath; + break; + } + } + + if (languagesPath) { + const languagesData = fs.readFileSync(languagesPath, "utf8"); + const languages: LanguageMapping[] = JSON.parse(languagesData); + + for (const lang of languages) { + this.languageMap.set(lang.code, lang.name); + } + + this.logger.debug( + `Loaded ${this.languageMap.size} language mappings from ${languagesPath}`, + ); + } else { + this.logger.warn( + "Languages file not found in any expected location, using default mappings", + ); + this.loadDefaultLanguageMap(); + } + } catch (error) { + this.logger.error( + `Error loading language mappings: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + this.loadDefaultLanguageMap(); + } + } + + /** + * Load default language mappings as fallback + */ + private loadDefaultLanguageMap(): void { + const defaultMappings: LanguageMapping[] = [ + { code: "en", name: "English" }, + { code: "id", name: "Bahasa Indonesia" }, + { code: "de", name: "Deutsch" }, + { code: "es", name: "Español" }, + { code: "fr", name: "Français" }, + { code: "it", name: "Italiano" }, + { code: "hu", name: "Magyar" }, + { code: "nl", name: "Nederlands" }, + { code: "pl", name: "Polski" }, + { code: "pt", name: "Português" }, + { code: "sv", name: "Svenska" }, + { code: "tr", name: "Türkçe" }, + { code: "el", name: "Ελληνικά" }, + { code: "kk", name: "Қазақ тілі" }, + { code: "ru", name: "Русский" }, + { code: "uk-UA", name: "Українська" }, + { code: "ar", name: "العربية" }, + { code: "hi", name: "हिन्दी" }, + { code: "th", name: "ไทย" }, + { code: "ko", name: "한국어" }, + { code: "zh-CN", name: "简体中文" }, + { code: "zh-TW", name: "繁體中文" }, + { code: "ja", name: "日本語" }, + ]; + + for (const lang of defaultMappings) { + this.languageMap.set(lang.code, lang.name); + } + } - async getLanguageCode(text: string): Promise { + /** + * Get language name from language code + */ + private getLanguageName(languageCode: string): string { + return this.languageMap.get(languageCode) || languageCode; + } + + /** + * Batch detect languages for multiple texts using CLD first, falling back to GPT-5-nano + */ + async batchGetLanguageCodes( + texts: string[], + assignmentId = 1, + ): Promise { + if (texts.length === 0) return []; + + // Step 1: Try CLD on all texts first (fast and free) + const results: unknown[] = Array.from({ length: texts.length }).fill( + "unknown", + ); + const textsNeedingGPT: Array<{ text: string; index: number }> = []; + + for (const [index, text] of texts.entries()) { + if (!text || !text.trim()) { + continue; + } + + const decodedText = decodeIfBase64(text) || text; + + try { + const cldResponse = await cld.detect(decodedText); + const detectedLanguage = cldResponse.languages[0]; + + // Check if CLD is confident enough + if (detectedLanguage && detectedLanguage.percent >= 80) { + results[index] = detectedLanguage.code; + this.logger.debug( + `CLD batch detected language for text ${index}: ${detectedLanguage.code} (${detectedLanguage.percent}% confidence)`, + ); + } else { + // Mark for GPT-5-nano processing + textsNeedingGPT.push({ + text: decodedText.slice(0, 500), + index: index, + }); + } + } catch { + // Mark for GPT-5-nano processing + textsNeedingGPT.push({ + text: decodedText.slice(0, 500), + index: index, + }); + } + } + + // Step 2: Use GPT-5-nano for texts that CLD couldn't handle confidently + if ( + textsNeedingGPT.length === 0 && + results.every((r) => typeof r === "string") + ) { + return results; + } + + this.logger.debug( + `CLD processed ${texts.length - textsNeedingGPT.length}/${ + texts.length + } texts confidently, using GPT-5-nano for ${ + textsNeedingGPT.length + } remaining texts`, + ); + + const parser = StructuredOutputParser.fromZodSchema( + z.object({ + detections: z.array( + z.object({ + textIndex: z + .number() + .describe("The index of the text in the input array"), + languageCode: z + .string() + .describe("The detected language code (e.g., 'en', 'es', 'fr')"), + confidence: z + .number() + .min(0) + .max(1) + .describe("Confidence score between 0 and 1"), + }), + ), + }), + ); + + const formatInstructions = parser.getFormatInstructions(); + + // Create input texts for the model (only the texts that need GPT processing) + const inputTexts = textsNeedingGPT + .map((item, index) => `Text ${index}: ${item.text}`) + .join("\n\n"); + + const prompt = new PromptTemplate({ + template: `You are a language detection expert. Analyze the following texts and identify their languages. + +TEXTS TO ANALYZE: +{texts} + +INSTRUCTIONS: +1. For each text, detect its language +2. Return the standard ISO 639-1 language code (e.g., 'en' for English, 'es' for Spanish) +3. For Chinese, specify 'zh-CN' for simplified or 'zh-TW' for traditional +4. If you cannot determine the language, return 'unknown' as the language code +5. Provide a confidence score between 0 and 1 for each detection +6. Return results for all texts in the order they appear (Text 0, Text 1, etc.) + +{format_instructions}`, + inputVariables: [], + partialVariables: { + texts: inputTexts, + format_instructions: formatInstructions, + }, + }); + + try { + const response = await this.promptProcessor.processPrompt( + prompt, + assignmentId, + AIUsageType.TRANSLATION, + "gpt-5-nano", + ); + + const parsedResponse = await parser.parse(response); + + // Map GPT-5-nano results back to the original result array + for (const detection of parsedResponse.detections) { + const gptTextItem = textsNeedingGPT[detection.textIndex]; + if (gptTextItem) { + results[gptTextItem.index] = detection.languageCode; + + this.logger.debug( + `GPT-5-nano batch detected language for text ${gptTextItem.index}: ${detection.languageCode} (${detection.confidence} confidence)`, + ); + + if (detection.confidence < 0.5) { + this.logger.warn( + `Low confidence (${detection.confidence}) in GPT-5-nano batch language detection for text at index ${gptTextItem.index}`, + ); + } + } + } + + if (results.every((r) => typeof r === "string")) { + return results; + } + } catch (error) { + this.logger.error( + `Error in batch language detection with GPT-5-nano: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + // Fallback to individual detection for critical cases + return Promise.all( + texts.map((text) => this.getLanguageCode(text, assignmentId)), + ); + } + } + + /** + * Detect the language of text using CLD first, falling back to GPT-5-nano + */ + async getLanguageCode(text: string, assignmentId = 1): Promise { if (!text) return "unknown"; const decodedText = decodeIfBase64(text) || text; + // Step 1: Try CLD first (fast and free) + try { + const cldResponse = await cld.detect(decodedText); + const detectedLanguage = cldResponse.languages[0]; + + // Check if CLD is confident enough + if (detectedLanguage && detectedLanguage.percent >= 80) { + this.logger.debug( + `CLD detected language: ${detectedLanguage.code} (${detectedLanguage.percent}% confidence)`, + ); + return detectedLanguage.code; + } else if (detectedLanguage) { + this.logger.debug( + `CLD low confidence (${detectedLanguage.percent}%), falling back to GPT-5-nano`, + ); + // Fall through to GPT-5-nano + } + } catch (cldError) { + this.logger.debug( + `CLD failed: ${ + cldError instanceof Error ? cldError.message : "Unknown error" + }, falling back to GPT-5-nano`, + ); + // Fall through to GPT-5-nano + } + + // Step 2: Fall back to GPT-5-nano for difficult cases + const textSample = decodedText.slice(0, 500); + + const parser = StructuredOutputParser.fromZodSchema( + z.object({ + languageCode: z + .string() + .describe( + "The detected language code (e.g., 'en', 'es', 'fr', 'zh-CN')", + ), + confidence: z + .number() + .min(0) + .max(1) + .describe("Confidence score between 0 and 1"), + }), + ); + + const formatInstructions = parser.getFormatInstructions(); + + const prompt = new PromptTemplate({ + template: `You are a language detection expert. Analyze the following text and identify its language. + +TEXT: +{text} + +INSTRUCTIONS: +1. Detect the language of the text +2. Return the standard ISO 639-1 language code (e.g., 'en' for English, 'es' for Spanish) +3. For Chinese, specify 'zh-CN' for simplified or 'zh-TW' for traditional +4. If you cannot determine the language, return 'unknown' as the language code +5. Provide a confidence score between 0 and 1 + +{format_instructions}`, + inputVariables: [], + partialVariables: { + text: textSample, + format_instructions: formatInstructions, + }, + }); + try { - const response = await cld.detect(decodedText); - return response.languages[0].code; + const response = await this.promptProcessor.processPrompt( + prompt, + assignmentId, + AIUsageType.TRANSLATION, + "gpt-5-nano", + ); + + const parsedResponse = await parser.parse(response); + + this.logger.debug( + `GPT-5-nano detected language: ${parsedResponse.languageCode} (${parsedResponse.confidence} confidence)`, + ); + + if (parsedResponse.confidence < 0.5) { + this.logger.warn( + `Low confidence (${ + parsedResponse.confidence + }) in GPT-5-nano language detection for: "${textSample.slice( + 0, + 50, + )}..."`, + ); + } + + return parsedResponse.languageCode; } catch (error) { this.logger.error( - `Error detecting language: ${error instanceof Error ? error.message : "Unknown error"}`, + `Error detecting language with GPT-5-nano: ${ + error instanceof Error ? error.message : "Unknown error" + }`, ); return "unknown"; } @@ -57,6 +408,9 @@ export class TranslationService implements ITranslationService { const cleanedText = decodedQuestionText.replaceAll(/<[^>]*>?/gm, ""); + // Convert language code to language name for the LLM + const targetLanguageName = this.getLanguageName(targetLanguage); + const parser = StructuredOutputParser.fromZodSchema( z.object({ translatedText: z.string().nonempty("Translated text cannot be empty"), @@ -70,7 +424,7 @@ export class TranslationService implements ITranslationService { inputVariables: [], partialVariables: { question_text: cleanedText, - target_language: targetLanguage, + target_language: targetLanguageName, format_instructions: formatInstructions, }, }); @@ -87,7 +441,9 @@ export class TranslationService implements ITranslationService { return parsedResponse.translatedText; } catch (error) { this.logger.error( - `Error translating question: ${error instanceof Error ? error.message : "Unknown error"}`, + `Error translating question: ${ + error instanceof Error ? error.message : "Unknown error" + }`, ); throw new HttpException( "Failed to translate question", @@ -97,11 +453,12 @@ export class TranslationService implements ITranslationService { } /** * Generate comprehensive translations for choice objects including feedback + * Validates existing translations and retranslates if language doesn't match * * @param choices - Original choices array to translate * @param assignmentId - The assignment ID for tracking * @param targetLanguage - The target language code - * @returns Translated choices with all content translated + * @returns Translated choices with both choice text and feedback translated and validated */ async generateChoicesTranslation( choices: Choice[] | null | undefined, @@ -117,27 +474,98 @@ export class TranslationService implements ITranslationService { try { this.logger.debug( - `Translating text for ${choices.length} choices to ${targetLanguage}`, + `Batch translating text for ${choices.length} choices to ${targetLanguage}`, ); + // Collect all texts for batch language detection + const textsToCheck: string[] = []; + const textMap: Array<{ + choiceIndex: number; + type: "choice" | "feedback"; + textIndex: number; + }> = []; + + for (const [choiceIndex, choice] of choices.entries()) { + if (choice.choice) { + textMap.push({ + choiceIndex, + type: "choice", + textIndex: textsToCheck.length, + }); + textsToCheck.push(choice.choice); + } + if (choice.feedback) { + textMap.push({ + choiceIndex, + type: "feedback", + textIndex: textsToCheck.length, + }); + textsToCheck.push(choice.feedback); + } + } + + // Batch language detection + let needsTranslationFlags: boolean[] = []; + if (textsToCheck.length > 0) { + needsTranslationFlags = await this.batchShouldRetranslate( + textsToCheck, + targetLanguage, + assignmentId, + ); + } + + // Process translations based on batch results const translatedChoices = await Promise.all( - choices.map(async (choice) => { + choices.map(async (choice, choiceIndex) => { const translatedChoice = { ...choice }; - const choiceText = translatedChoice.choice; - if (choiceText) { + + // Handle choice text + const choiceMapping = textMap.find( + (m) => m.choiceIndex === choiceIndex && m.type === "choice", + ); + if (choiceMapping && needsTranslationFlags[choiceMapping.textIndex]) { try { const translatedText = await this.translateText( - choiceText, + choice.choice, targetLanguage, assignmentId, ); + translatedChoice.choice = translatedText; + this.logger.debug( + `Batch retranslated choice text to ${targetLanguage}`, + ); + } catch (error) { + this.logger.error( + `Failed to translate choice text: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + } - if (translatedChoice.choice !== undefined) { - translatedChoice.choice = translatedText; - } + // Handle feedback text + const feedbackMapping = textMap.find( + (m) => m.choiceIndex === choiceIndex && m.type === "feedback", + ); + if ( + feedbackMapping && + needsTranslationFlags[feedbackMapping.textIndex] + ) { + try { + const translatedFeedback = await this.translateText( + choice.feedback, + targetLanguage, + assignmentId, + ); + translatedChoice.feedback = translatedFeedback; + this.logger.debug( + `Batch retranslated choice feedback to ${targetLanguage}`, + ); } catch (error) { this.logger.error( - `Failed to translate choice text: ${error instanceof Error ? error.message : String(error)}`, + `Failed to translate choice feedback: ${ + error instanceof Error ? error.message : String(error) + }`, ); } } @@ -149,13 +577,187 @@ export class TranslationService implements ITranslationService { return translatedChoices; } catch (error) { this.logger.error( - `Error translating choice text: ${error instanceof Error ? error.message : String(error)}`, + `Error translating choice text: ${ + error instanceof Error ? error.message : String(error) + }`, ); return choices; } } + /** + * Batch determine if texts should be retranslated based on language detection + * + * @param texts - Array of texts to check + * @param targetLanguage - The expected target language code + * @param assignmentId - The assignment ID for tracking + * @returns Promise - array of booleans indicating which texts need retranslation + */ + private async batchShouldRetranslate( + texts: string[], + targetLanguage: string, + assignmentId: number, + ): Promise { + if (texts.length === 0) return []; + + // Filter out empty texts + const validTexts = texts.filter((text) => text && text.trim().length > 0); + if (validTexts.length === 0) { + return texts.map(() => false); + } + + try { + const detectedLanguages = await this.batchGetLanguageCodes( + texts, + assignmentId, + ); + + return texts.map((text, index) => { + if (!text || text.trim().length === 0) { + return false; + } + + const detectedLanguage = detectedLanguages[index]; + + // Skip retranslation if we can't detect the language + if (detectedLanguage === "unknown") { + this.logger.debug( + `Could not detect language for text, skipping validation: "${text.slice( + 0, + 50, + )}..."`, + ); + return false; + } + + // Normalize language codes for comparison + const normalizedDetected = this.normalizeLanguageCode(detectedLanguage); + const normalizedTarget = this.normalizeLanguageCode(targetLanguage); + + const needsRetranslation = normalizedDetected !== normalizedTarget; + + if (needsRetranslation) { + this.logger.info( + `Language mismatch detected in batch. Expected: ${normalizedTarget}, Found: ${normalizedDetected}. Text: "${text.slice( + 0, + 50, + )}..."`, + ); + } + + return needsRetranslation; + }); + } catch (error) { + this.logger.error( + `Error during batch language validation: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + // Fallback to individual validation + return Promise.all( + texts.map((text) => + this.shouldRetranslate(text, targetLanguage, assignmentId), + ), + ); + } + } + + /** + * Determine if text should be retranslated based on language detection + * + * @param text - The text to check + * @param targetLanguage - The expected target language code + * @param assignmentId - The assignment ID for tracking + * @returns Promise - true if text needs retranslation + */ + private async shouldRetranslate( + text: string, + targetLanguage: string, + assignmentId: number, + ): Promise { + if (!text || text.trim().length === 0) { + return false; + } + + try { + const detectedLanguage = await this.getLanguageCode(text, assignmentId); + + // Skip retranslation if we can't detect the language + if (detectedLanguage === "unknown") { + this.logger.debug( + `Could not detect language for text, skipping validation: "${text.slice( + 0, + 50, + )}..."`, + ); + return false; + } + + // Normalize language codes for comparison + const normalizedDetected = this.normalizeLanguageCode(detectedLanguage); + const normalizedTarget = this.normalizeLanguageCode(targetLanguage); + + const needsRetranslation = normalizedDetected !== normalizedTarget; + + if (needsRetranslation) { + this.logger.info( + `Language mismatch detected. Expected: ${normalizedTarget}, Found: ${normalizedDetected}. Text: "${text.slice( + 0, + 50, + )}..."`, + ); + } else { + this.logger.debug(`Text language matches target: ${normalizedTarget}`); + } + + return needsRetranslation; + } catch (error) { + this.logger.error( + `Error during language validation: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + // On error, assume retranslation is needed to be safe + return true; + } + } + + /** + * Normalize language codes for comparison (handles variants like zh-CN vs zh) + * + * @param languageCode - The language code to normalize + * @returns string - The normalized language code + */ + private normalizeLanguageCode(languageCode: string): string { + if (!languageCode) return "unknown"; + + const code = languageCode.toLowerCase(); + + // Handle common language variants + const languageMap: Record = { + "zh-cn": "zh", + "zh-tw": "zh", + "zh-hk": "zh", + "en-us": "en", + "en-gb": "en", + "en-ca": "en", + "es-es": "es", + "es-mx": "es", + "pt-br": "pt", + "pt-pt": "pt", + "fr-fr": "fr", + "fr-ca": "fr", + "de-de": "de", + "it-it": "it", + "ru-ru": "ru", + "ja-jp": "ja", + "ko-kr": "ko", + }; + + return languageMap[code] || code.split("-")[0]; + } + /** * Translate arbitrary text to a target language */ @@ -168,6 +770,9 @@ export class TranslationService implements ITranslationService { const decodedText = decodeIfBase64(text) || text; + // Convert language code to language name for the LLM + const targetLanguageName = this.getLanguageName(targetLanguage); + const parser = StructuredOutputParser.fromZodSchema( z.object({ translatedText: z.string().nonempty("Translated text cannot be empty"), @@ -181,7 +786,7 @@ export class TranslationService implements ITranslationService { inputVariables: [], partialVariables: { text: decodedText, - target_language: targetLanguage, + target_language: targetLanguageName, format_instructions: formatInstructions, }, }); @@ -199,7 +804,9 @@ export class TranslationService implements ITranslationService { return parsedResponse.translatedText; } catch (error) { this.logger.error( - `Error translating text: ${error instanceof Error ? error.message : "Unknown error"}`, + `Error translating text: ${ + error instanceof Error ? error.message : "Unknown error" + }`, ); throw new HttpException( "Failed to translate text", diff --git a/apps/api/src/api/llm/grading/config/policy.config.ts b/apps/api/src/api/llm/grading/config/policy.config.ts new file mode 100644 index 00000000..ae79ad7f --- /dev/null +++ b/apps/api/src/api/llm/grading/config/policy.config.ts @@ -0,0 +1,74 @@ +export interface PolicyThresholds { + TAU_TOTAL: number; + TAU_CRITERION: number; + CONFIDENCE_THRESHOLD: number; + AGREEMENT_THRESHOLD: number; + MAX_RETRIES: number; + EVIDENCE_REQUIRED: boolean; +} + +export interface EarlyExitPolicy { + ACCEPT_ON_HIGH_CONFIDENCE: boolean; + SKIP_JUDGES_ON_SIMPLE_QUESTIONS: boolean; + MCQ_AUTO_ACCEPT: boolean; + TRUE_FALSE_AUTO_ACCEPT: boolean; +} + +export interface GradingPolicyConfig { + thresholds: PolicyThresholds; + earlyExit: EarlyExitPolicy; + tiebreakStrategy: "third_judge" | "meta_decider" | "hybrid"; +} + +export const DEFAULT_POLICY_CONFIG: GradingPolicyConfig = { + thresholds: { + TAU_TOTAL: 2, + TAU_CRITERION: 1, + CONFIDENCE_THRESHOLD: 0.7, + AGREEMENT_THRESHOLD: 0.6, + MAX_RETRIES: 2, + EVIDENCE_REQUIRED: true, + }, + earlyExit: { + ACCEPT_ON_HIGH_CONFIDENCE: true, + SKIP_JUDGES_ON_SIMPLE_QUESTIONS: true, + MCQ_AUTO_ACCEPT: true, + TRUE_FALSE_AUTO_ACCEPT: true, + }, + tiebreakStrategy: "hybrid", +}; + +export const createDynamicThresholds = ( + totalMax: number, +): PolicyThresholds => ({ + ...DEFAULT_POLICY_CONFIG.thresholds, + TAU_TOTAL: Math.max(1, totalMax * 0.05), + TAU_CRITERION: Math.max(0.5, totalMax * 0.02), +}); + +export const getQuestionTypePolicy = ( + questionType: string, +): Partial => { + switch (questionType) { + case "TRUE_FALSE": + case "SINGLE_CORRECT": + case "MULTIPLE_CORRECT": { + return { + MCQ_AUTO_ACCEPT: true, + SKIP_JUDGES_ON_SIMPLE_QUESTIONS: true, + }; + } + + case "TEXT": + case "UPLOAD": { + return { + MCQ_AUTO_ACCEPT: false, + SKIP_JUDGES_ON_SIMPLE_QUESTIONS: false, + }; + } + + default: { + return {}; + } + } +}; diff --git a/apps/api/src/api/llm/grading/enhanced-automated-grading.module.ts b/apps/api/src/api/llm/grading/enhanced-automated-grading.module.ts new file mode 100644 index 00000000..09fdf58a --- /dev/null +++ b/apps/api/src/api/llm/grading/enhanced-automated-grading.module.ts @@ -0,0 +1,42 @@ +import { Module } from "@nestjs/common"; +import { CompareNode } from "./nodes/compare.node"; +import { DecisionNode } from "./nodes/decision.node"; +import { EnhancedGradeNode } from "./nodes/enhanced-grade.node"; +import { EnhancedValidateNode } from "./nodes/enhanced-validate.node"; +import { EvidenceNode } from "./nodes/evidence.node"; +import { JudgeNode } from "./nodes/judge.node"; +import { TiebreakNode } from "./nodes/tiebreak.node"; +import { EnhancedAutomatedGradingService } from "./services/enhanced-automated-grading.service"; +import { EnhancedPolicyService } from "./services/enhanced-policy.service"; +import { EvidenceService } from "./services/evidence.service"; +import { MetaDeciderService } from "./services/meta-decider.service"; +import { MonitoringService } from "./services/monitoring.service"; + +@Module({ + providers: [ + // Main service + EnhancedAutomatedGradingService, + + // Core services + EvidenceService, + EnhancedPolicyService, + MetaDeciderService, + MonitoringService, + + // Graph nodes + EnhancedGradeNode, + EnhancedValidateNode, + JudgeNode, + EvidenceNode, + CompareNode, + TiebreakNode, + DecisionNode, + ], + exports: [ + EnhancedAutomatedGradingService, + EvidenceService, + EnhancedPolicyService, + MonitoringService, + ], +}) +export class EnhancedAutomatedGradingModule {} diff --git a/apps/api/src/api/llm/grading/examples/type-safe-grading.example.ts b/apps/api/src/api/llm/grading/examples/type-safe-grading.example.ts new file mode 100644 index 00000000..a9b78d02 --- /dev/null +++ b/apps/api/src/api/llm/grading/examples/type-safe-grading.example.ts @@ -0,0 +1,254 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { EnhancedAutomatedGradingService } from "../services/enhanced-automated-grading.service"; +import { MonitoringService } from "../services/monitoring.service"; +import { + FinalGradeData, + GradingContextData, + RubricCriterion, +} from "../types/grading.types"; + +@Injectable() +export class TypeSafeGradingExample { + private readonly logger = new Logger(TypeSafeGradingExample.name); + + constructor( + private readonly gradingService: EnhancedAutomatedGradingService, + private readonly monitoringService: MonitoringService, + ) {} + + async demonstrateTypeSafeGrading(): Promise { + // Define rubric with full type safety + const rubric: RubricCriterion[] = [ + { + id: "accuracy", + description: + "Answer demonstrates factual accuracy and correct understanding", + maxPoints: 10, + keywords: ["correct", "accurate", "precise"], + }, + { + id: "completeness", + description: "Answer addresses all aspects of the question", + maxPoints: 8, + keywords: ["complete", "comprehensive", "thorough"], + }, + { + id: "clarity", + description: "Answer is well-organized and clearly expressed", + maxPoints: 7, + keywords: ["clear", "organized", "coherent"], + }, + ]; + + // Create fully typed grading context + const gradingContext: GradingContextData = { + questionId: "q_001", + learnerAnswer: + "Photosynthesis is the process by which plants convert sunlight, carbon dioxide, and water into glucose and oxygen. This occurs in the chloroplasts using chlorophyll.", + rubric, + questionType: "TEXT", + responseType: "essay", + timeout: 60_000, + maxRetries: 2, + }; + + try { + // Execute grading with full type safety + const result = + await this.gradingService.executeGradingPipeline(gradingContext); + + // Type-safe result handling + if (result.success && result.finalGrade) { + this.logSuccessfulGrading(result.finalGrade); + this.analyzeGradingMetrics(result); + } else { + this.handleGradingErrors(result.errors, result.warnings); + } + + // Get system health with proper typing + const systemHealth = this.gradingService.getSystemHealth(); + this.logSystemHealth(systemHealth); + } catch (error) { + this.logger.error("Type-safe grading example failed:", error); + } + } + + private logSuccessfulGrading(finalGrade: FinalGradeData): void { + this.logger.log("Grading completed successfully", { + selectedSource: finalGrade.selectedSource, + totalAwarded: finalGrade.grade.totalAwarded, + totalMax: finalGrade.grade.totalMax, + confidence: finalGrade.grade.confidence, + processingSteps: finalGrade.processingSteps, + }); + + // Type-safe criteria analysis + for (const award of finalGrade.grade.criteriaAwards) { + this.logger.debug( + `Criterion ${award.criterionId}: ${award.awarded}/${award.maxPoints}`, + { + justification: award.justification, + evidence: award.evidence ?? "No evidence provided", + }, + ); + } + } + + private analyzeGradingMetrics(result: { + processingTimeMs: number; + riskLevel: "low" | "medium" | "high"; + fallbackUsed: boolean; + debugInfo?: { + processingSteps?: string[]; + nodeExecutionTimes?: Record; + errorRecovery?: unknown; + circuitBreakerStatus?: Record; + }; + }): void { + const performanceMetrics = { + processingTime: result.processingTimeMs, + riskAssessment: result.riskLevel, + systemReliability: result.fallbackUsed ? "degraded" : "optimal", + }; + + this.logger.log("Performance analysis", performanceMetrics); + + // Risk-based alerting + if (result.riskLevel === "high") { + this.logger.warn( + "High-risk grading detected - manual review recommended", + ); + } + + if (result.fallbackUsed) { + this.logger.warn( + "Fallback mechanisms were used - system performance degraded", + ); + } + } + + private handleGradingErrors(errors: string[], warnings: string[]): void { + for (const error of errors) { + this.logger.error(`Grading error: ${error}`); + } + + for (const warning of warnings) { + this.logger.warn(`Grading warning: ${warning}`); + } + } + + private logSystemHealth(health: { + totalRequests: number; + successRate: number; + averageProcessingTime: number; + errorRate: number; + circuitBreakerStatus: Record; + fallbackUsageRate: number; + lastHealthCheck: number; + }): void { + this.logger.log("System health status", { + requests: health.totalRequests, + successRate: `${(health.successRate * 100).toFixed(1)}%`, + avgProcessingTime: `${health.averageProcessingTime}ms`, + errorRate: `${(health.errorRate * 100).toFixed(2)}%`, + fallbackRate: `${(health.fallbackUsageRate * 100).toFixed(2)}%`, + circuitBreakers: Object.keys(health.circuitBreakerStatus).filter( + (key) => health.circuitBreakerStatus[key], + ), + }); + } + + async demonstrateBatchProcessing(): Promise { + const batchContexts: GradingContextData[] = [ + { + questionId: "batch_001", + learnerAnswer: + "Water cycle involves evaporation, condensation, and precipitation.", + rubric: [ + { + id: "understanding", + description: "Demonstrates understanding of water cycle", + maxPoints: 10, + }, + ], + questionType: "TEXT", + timeout: 30_000, + maxRetries: 1, + }, + { + questionId: "batch_002", + learnerAnswer: "True, because the Earth revolves around the Sun.", + rubric: [ + { + id: "correctness", + description: "Answer is factually correct", + maxPoints: 5, + }, + ], + questionType: "TRUE_FALSE", + timeout: 15_000, + maxRetries: 1, + }, + ]; + + try { + const results = + await this.gradingService.processGradingBatch(batchContexts); + const stats = this.gradingService.getProcessingStats(results); + + this.logger.log("Batch processing completed", { + totalRequests: results.length, + successRate: (stats.successRate * 100).toFixed(1) + "%", + avgProcessingTime: stats.avgProcessingTimeMs.toFixed(0) + "ms", + riskDistribution: stats.riskDistribution, + fallbackRate: (stats.fallbackRate * 100).toFixed(1) + "%", + }); + } catch (error) { + this.logger.error("Batch processing failed:", error); + } + } + + demonstrateAdvancedMetrics(): void { + // Get detailed metrics with time window + const metrics = this.gradingService.getMetrics(3_600_000); // Last hour + + this.logger.log("Advanced metrics analysis", { + requestCount: metrics.requestCount, + successRate: (metrics.successRate * 100).toFixed(1) + "%", + avgProcessingTime: metrics.averageProcessingTime.toFixed(0) + "ms", + llmUsage: { + totalCalls: metrics.llmUsage.totalCalls, + totalTokens: metrics.llmUsage.totalTokens, + avgTokensPerCall: metrics.llmUsage.avgTokensPerCall.toFixed(0), + }, + nodePerformance: Object.entries(metrics.nodePerformance).map( + ([node, perf]) => ({ + node, + avgTime: perf.avgTime.toFixed(0) + "ms", + successRate: (perf.successRate * 100).toFixed(1) + "%", + }), + ), + fallbackUsage: { + count: metrics.fallbackUsage.count, + rate: (metrics.fallbackUsage.rate * 100).toFixed(1) + "%", + }, + errorAnalysis: metrics.errorAnalysis, + }); + } +} + +// Example usage with proper error handling +export async function runTypeSafeGradingExample( + gradingService: EnhancedAutomatedGradingService, + monitoringService: MonitoringService, +): Promise { + const example = new TypeSafeGradingExample(gradingService, monitoringService); + + try { + await example.demonstrateTypeSafeGrading(); + await example.demonstrateBatchProcessing(); + example.demonstrateAdvancedMetrics(); + } catch (error) { + console.error("Example execution failed:", error); + } +} diff --git a/apps/api/src/api/llm/grading/graph/grade.graph.ts b/apps/api/src/api/llm/grading/graph/grade.graph.ts new file mode 100644 index 00000000..3474c06d --- /dev/null +++ b/apps/api/src/api/llm/grading/graph/grade.graph.ts @@ -0,0 +1,113 @@ +import { SimpleGradingGraph } from "./simple-graph"; +import { + GradingGraphState, + shouldRunJudgeA, + shouldRunJudgeB, + shouldRunTiebreak, +} from "./state"; + +type NodeFunction = (state: GradingGraphState) => Promise; + +export class GradingGraph { + private graph: SimpleGradingGraph; + + constructor( + private gradeNode: NodeFunction, + private validateNode: NodeFunction, + private judgeANode: NodeFunction, + private evidenceNode: NodeFunction, + private compareNode: NodeFunction, + private decisionNode: NodeFunction, + ) { + this.graph = new SimpleGradingGraph(); + this.buildGraph(); + } + + private buildGraph() { + this.graph + .addNode("grade", this.gradeNode) + .addNode("validate", this.validateNode) + .addNode("judgeA", this.judgeANode) + .addNode("evidence", this.evidenceNode) + .addNode("compare", this.compareNode) + .addNode("decision", this.decisionNode); + + this.graph.setEntryPoint("grade"); + + this.graph.addEdge("grade", "validate"); + + this.graph.addConditionalEdges( + "validate", + this.shouldContinueFromValidate, + { + evidence: "evidence", + retry_grade: "grade", + error: "END", + }, + ); + + this.graph.addConditionalEdges("evidence", this.shouldRunJudges, { + judgeA: "judgeA", + decision: "decision", + }); + + this.graph.addConditionalEdges("judgeA", this.shouldRunJudgeBConditional, { + judgeB: "judgeB", + compare: "compare", + }); + + this.graph.addEdge("judgeB", "compare"); + + this.graph.addConditionalEdges( + "compare", + this.shouldRunTiebreakConditional, + { + tiebreak: "tiebreak", + decision: "decision", + }, + ); + + this.graph.addEdge("tiebreak", "decision"); + this.graph.addEdge("decision", "END"); + } + + private shouldContinueFromValidate = (state: GradingGraphState): string => { + if (!state.shouldContinue) { + return "error"; + } + + if (!state.graderResult?.isValid && state.retry_count < 2) { + return "retry_grade"; + } + + return "evidence"; + }; + + private shouldRunJudges = (state: GradingGraphState): string => { + if (shouldRunJudgeA(state)) { + return "judgeA"; + } + + return "decision"; + }; + + private shouldRunJudgeBConditional = (state: GradingGraphState): string => { + if (shouldRunJudgeB(state)) { + return "judgeB"; + } + + return "compare"; + }; + + private shouldRunTiebreakConditional = (state: GradingGraphState): string => { + if (shouldRunTiebreak(state)) { + return "tiebreak"; + } + + return "decision"; + }; + + compile() { + return this.graph.compile(); + } +} diff --git a/apps/api/src/api/llm/grading/graph/simple-graph.ts b/apps/api/src/api/llm/grading/graph/simple-graph.ts new file mode 100644 index 00000000..24586c16 --- /dev/null +++ b/apps/api/src/api/llm/grading/graph/simple-graph.ts @@ -0,0 +1,120 @@ +import { GradingGraphState } from "./state"; + +type NodeFunction = (state: GradingGraphState) => Promise; +type ConditionalFunction = (state: GradingGraphState) => string; + +interface GraphEdge { + from: string; + to: string | ConditionalFunction; + conditions?: Record; +} + +export class SimpleGradingGraph { + private nodes = new Map(); + private edges: GraphEdge[] = []; + private entryPoint = ""; + + addNode(name: string, nodeFunction: NodeFunction): this { + this.nodes.set(name, nodeFunction); + return this; + } + + addEdge(from: string, to: string): this { + this.edges.push({ from, to }); + return this; + } + + addConditionalEdges( + from: string, + condition: ConditionalFunction, + routes: Record, + ): this { + this.edges.push({ from, to: condition, conditions: routes }); + return this; + } + + setEntryPoint(nodeName: string): this { + this.entryPoint = nodeName; + return this; + } + + compile(): { + invoke: (state: GradingGraphState) => Promise; + } { + return { + invoke: async ( + initialState: GradingGraphState, + ): Promise => { + let currentState = initialState; + let currentNode = this.entryPoint; + const visitedNodes = new Set(); + const maxIterations = 20; // Prevent infinite loops + let iteration = 0; + + while ( + currentNode && + currentNode !== "END" && + iteration < maxIterations + ) { + iteration++; + + // Execute current node + const nodeFunction = this.nodes.get(currentNode); + if (!nodeFunction) { + throw new Error(`Node '${currentNode}' not found`); + } + + // Prevent infinite loops + const nodeKey = `${currentNode}_${iteration}`; + if (visitedNodes.has(nodeKey)) { + break; + } + visitedNodes.add(nodeKey); + + try { + currentState = await nodeFunction(currentState); + + // Check if processing should stop + if (!currentState.shouldContinue) { + break; + } + } catch (error) { + currentState = { + ...currentState, + errors: [ + ...currentState.errors, + `Node ${currentNode} failed: ${String(error)}`, + ], + shouldContinue: false, + }; + break; + } + + // Determine next node + currentNode = this.getNextNode(currentNode, currentState); + } + + return currentState; + }, + }; + } + + private getNextNode(currentNode: string, state: GradingGraphState): string { + const edge = this.edges.find((edgeItem) => edgeItem.from === currentNode); + if (!edge) { + return "END"; + } + + if (typeof edge.to === "string") { + return edge.to; + } + + // Conditional edge + if (typeof edge.to === "function" && edge.conditions) { + const condition = edge.to(state); + return edge.conditions[condition] ?? "END"; + } + + return "END"; + } +} diff --git a/apps/api/src/api/llm/grading/graph/state.ts b/apps/api/src/api/llm/grading/graph/state.ts new file mode 100644 index 00000000..c54d3569 --- /dev/null +++ b/apps/api/src/api/llm/grading/graph/state.ts @@ -0,0 +1,239 @@ +import { + CircuitBreakerData, + ErrorRecoveryData, + GradingContextData, + GraphStateData, + ProcessingMetricsData, +} from "../types/grading.types"; + +export interface GradingGraphState extends GraphStateData { + retry_count: number; + processing_start_time: number; + error_recovery: ErrorRecoveryData; + processing_metrics: ProcessingMetricsData; + node_circuit_breakers: Record; + fallback_used: boolean; + max_processing_time: number; +} + +export const initialState = ( + context: GradingContextData, +): GradingGraphState => ({ + context, + graderResult: undefined, + judgeAResult: undefined, + judgeBResult: undefined, + evidenceVerification: undefined, + comparison: undefined, + tiebreakResult: undefined, + finalGrade: undefined, + errors: [], + currentStep: "grade", + shouldContinue: true, + retry_count: 0, + processing_start_time: Date.now(), + error_recovery: { + attempts: 0, + recoveryStrategy: "retry", + fallbackUsed: false, + }, + processing_metrics: { + nodeExecutionTimes: {}, + llmTokensUsed: 0, + cacheHits: 0, + }, + node_circuit_breakers: {}, + fallback_used: false, + max_processing_time: context.timeout || 60_000, +}); + +export const updateState = ( + state: GradingGraphState, + updates: Partial, +): GradingGraphState => ({ + ...state, + ...updates, +}); + +export const addError = ( + state: GradingGraphState, + error: string, + nodeName?: string, +): GradingGraphState => { + const errorWithContext = nodeName ? `[${nodeName}] ${error}` : error; + + const updatedRecovery = { + ...state.error_recovery, + attempts: state.error_recovery.attempts + 1, + lastError: error, + }; + + return { + ...state, + errors: [...state.errors, errorWithContext], + error_recovery: updatedRecovery, + }; +}; + +export const canRetry = ( + state: GradingGraphState, + maxRetries?: number, +): boolean => { + const limit = maxRetries || state.context.maxRetries || 2; + return state.retry_count < limit && state.error_recovery.attempts < limit * 2; +}; + +export const incrementRetry = ( + state: GradingGraphState, +): GradingGraphState => ({ + ...state, + retry_count: state.retry_count + 1, + error_recovery: { + ...state.error_recovery, + attempts: state.error_recovery.attempts + 1, + }, +}); + +export const getProcessingDuration = (state: GradingGraphState): number => { + return Date.now() - state.processing_start_time; +}; + +export const isTimeoutExceeded = (state: GradingGraphState): boolean => { + return getProcessingDuration(state) > state.max_processing_time; +}; + +export const recordNodeExecution = ( + state: GradingGraphState, + nodeName: string, + executionTime: number, +): GradingGraphState => ({ + ...state, + processing_metrics: { + ...state.processing_metrics, + nodeExecutionTimes: { + ...state.processing_metrics.nodeExecutionTimes, + [nodeName]: executionTime, + }, + }, +}); + +export const updateCircuitBreaker = ( + state: GradingGraphState, + nodeName: string, + success: boolean, +): GradingGraphState => { + const currentBreaker = state.node_circuit_breakers[nodeName] || { + failures: 0, + isOpen: false, + resetTimeout: 60_000, + }; + + const updatedBreaker = success + ? { + ...currentBreaker, + failures: Math.max(0, currentBreaker.failures - 1), + } + : { + ...currentBreaker, + failures: currentBreaker.failures + 1, + lastFailure: Date.now(), + isOpen: currentBreaker.failures >= 2, + }; + + return { + ...state, + node_circuit_breakers: { + ...state.node_circuit_breakers, + [nodeName]: updatedBreaker, + }, + }; +}; + +export const isNodeCircuitBreakerOpen = ( + state: GradingGraphState, + nodeName: string, +): boolean => { + const breaker = state.node_circuit_breakers[nodeName]; + if (!breaker || !breaker.isOpen) return false; + + const now = Date.now(); + if (breaker.lastFailure && now - breaker.lastFailure > breaker.resetTimeout) { + return false; + } + + return true; +}; + +export const shouldAbortProcessing = (state: GradingGraphState): boolean => { + return ( + isTimeoutExceeded(state) || + state.errors.length > 10 || + state.error_recovery.attempts > 10 || + state.error_recovery.recoveryStrategy === "abort" + ); +}; + +export const determineFallbackStrategy = ( + state: GradingGraphState, + nodeName: string, +): "retry" | "fallback" | "skip" | "abort" => { + if (shouldAbortProcessing(state)) return "abort"; + + if (isTimeoutExceeded(state)) return "fallback"; + + if (isNodeCircuitBreakerOpen(state, nodeName)) return "skip"; + + if (canRetry(state)) return "retry"; + + return "fallback"; +}; + +export const shouldRunJudgeA = ( + state: GradingGraphState, + confidenceThreshold = 0.7, +): boolean => { + if (!state.graderResult) return false; + + const lowConfidence = state.graderResult.confidence < confidenceThreshold; + const hasEvidenceIssues = + state.evidenceVerification && + state.evidenceVerification.invalidCriteriaIds.length > 0; + + return lowConfidence || !!hasEvidenceIssues; +}; + +export const shouldRunJudgeB = ( + state: GradingGraphState, + totalThreshold = 2, +): boolean => { + if (!state.graderResult || !state.judgeAResult) return false; + + const totalDelta = Math.abs( + state.graderResult.totalAwarded - state.judgeAResult.totalAwarded, + ); + + return totalDelta > totalThreshold; +}; + +export const shouldRunTiebreak = ( + state: GradingGraphState, + totalThreshold = 2, + agreementThreshold = 0.6, +): boolean => { + if (!state.comparison) return false; + + const highTotalDelta = + state.comparison.graderVsJudgeA.totalDelta > totalThreshold; + const lowAgreement = + state.comparison.graderVsJudgeA.agreementPct < agreementThreshold; + + if (state.comparison.judgeAVsJudgeB) { + const judgeDelta = + state.comparison.judgeAVsJudgeB.totalDelta > totalThreshold; + const judgeAgreement = + state.comparison.judgeAVsJudgeB.agreementPct < agreementThreshold; + return highTotalDelta || lowAgreement || judgeDelta || judgeAgreement; + } + + return highTotalDelta || lowAgreement; +}; diff --git a/apps/api/src/api/llm/grading/nodes/compare.node.ts b/apps/api/src/api/llm/grading/nodes/compare.node.ts new file mode 100644 index 00000000..c75e824f --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/compare.node.ts @@ -0,0 +1,102 @@ +import { Injectable } from "@nestjs/common"; +import { GradingGraphState } from "../graph/state"; +import { + CriterionAwardData, + GradeData, + JudgeComparisonData, + ValidatedGradeData, +} from "../types/grading.types"; + +@Injectable() +export class CompareNode { + async execute(state: GradingGraphState): Promise { + if (!state.graderResult || !state.judgeAResult) { + return { + ...state, + errors: [ + ...state.errors, + "Missing grader or judge results for comparison", + ], + currentStep: "decision", + shouldContinue: true, + }; + } + + try { + const comparison = this.computeComparison(state); + + return { + ...state, + comparison, + currentStep: "tiebreak_check", + shouldContinue: true, + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown comparison error"; + + return { + ...state, + errors: [...state.errors, `Comparison failed: ${errorMessage}`], + currentStep: "decision", + shouldContinue: true, + }; + } + } + + private computeComparison(state: GradingGraphState): JudgeComparisonData { + const graderVsJudgeA = this.compareGrades( + state.graderResult, + state.judgeAResult, + ); + + const judgeAVsJudgeB = state.judgeBResult + ? this.compareGrades(state.judgeAResult, state.judgeBResult) + : undefined; + + return { + graderVsJudgeA, + judgeAVsJudgeB, + }; + } + + private compareGrades( + grade1: ValidatedGradeData | GradeData, + grade2: GradeData, + ): { + totalDelta: number; + criterionDeltas: Array<{ criterionId: string; delta: number }>; + agreementPct: number; + } { + const totalDelta = Math.abs( + (grade1.totalAwarded ?? 0) - (grade2.totalAwarded ?? 0), + ); + + const criterionDeltas = (grade1.criteriaAwards ?? []).map( + (award1: CriterionAwardData) => { + const award2 = (grade2.criteriaAwards ?? []).find( + (award: CriterionAwardData) => + award.criterionId === award1.criterionId, + ); + const delta = award2 + ? Math.abs((award1.awarded ?? 0) - (award2.awarded ?? 0)) + : (award1.awarded ?? 0); + + return { + criterionId: award1.criterionId ?? "", + delta, + }; + }, + ); + + const agreementCount = criterionDeltas.filter((cd) => cd.delta <= 1).length; + const agreementPct = + criterionDeltas.length > 0 ? agreementCount / criterionDeltas.length : 0; + + return { + totalDelta, + criterionDeltas, + agreementPct, + }; + } +} diff --git a/apps/api/src/api/llm/grading/nodes/decision.node.ts b/apps/api/src/api/llm/grading/nodes/decision.node.ts new file mode 100644 index 00000000..76d54371 --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/decision.node.ts @@ -0,0 +1,162 @@ +import { Injectable } from "@nestjs/common"; +import { GradingGraphState } from "../graph/state"; +import { + FinalGradeData, + GradeData, + ValidatedGradeData, +} from "../types/grading.types"; + +interface PolicyService { + decide( + graderResult: any, + judgeAResult?: any, + judgeBResult?: any, + comparison?: any, + tiebreakResult?: any, + ): { + selectedSource: "grader" | "judges" | "tiebreak"; + reasoning: string; + }; +} + +@Injectable() +export class DecisionNode { + constructor(private policyService: PolicyService) {} + + async execute(state: GradingGraphState): Promise { + if (!state.graderResult) { + return { + ...state, + errors: [...state.errors, "No grader result for final decision"], + shouldContinue: false, + currentStep: "error", + }; + } + + try { + const decision = this.policyService.decide( + state.graderResult, + state.judgeAResult, + state.judgeBResult, + state.comparison, + state.tiebreakResult, + ); + + const selectedGrade = this.selectGrade(state, decision.selectedSource); + const processingSteps = this.determineProcessingSteps(state); + const metadata = this.buildMetadata(state); + + const finalGrade: FinalGradeData = { + selectedSource: decision.selectedSource, + grade: selectedGrade, + reasoning: decision.reasoning, + processingSteps, + metadata, + }; + + return { + ...state, + finalGrade, + currentStep: "completed", + shouldContinue: false, + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown decision error"; + + return { + ...state, + errors: [...state.errors, `Final decision failed: ${errorMessage}`], + shouldContinue: false, + currentStep: "error", + }; + } + } + + private selectGrade( + state: GradingGraphState, + selectedSource: "grader" | "judges" | "tiebreak", + ): ValidatedGradeData | GradeData | undefined { + switch (selectedSource) { + case "grader": { + return state.graderResult; + } + + case "judges": { + return state.judgeAResult || state.graderResult; + } + + case "tiebreak": { + if (state.tiebreakResult?.result) { + return state.tiebreakResult.result; + } + if (state.tiebreakResult?.metaDecision === "accept_grader") { + return state.graderResult; + } + if (state.tiebreakResult?.metaDecision === "accept_judges") { + return state.judgeAResult || state.graderResult; + } + return state.graderResult; + } + + default: { + return state.graderResult; + } + } + } + + private determineProcessingSteps( + state: GradingGraphState, + ): Array< + | "grade" + | "validate" + | "judgeA" + | "judgeB" + | "evidence" + | "compare" + | "tiebreak" + | "decision" + > { + const steps: Array< + | "grade" + | "validate" + | "judgeA" + | "judgeB" + | "evidence" + | "compare" + | "tiebreak" + | "decision" + > = ["grade", "validate", "evidence"]; + + if (state.judgeAResult) steps.push("judgeA"); + if (state.judgeBResult) steps.push("judgeB"); + if (state.comparison) steps.push("compare"); + if (state.tiebreakResult) steps.push("tiebreak"); + + steps.push("decision"); + + return steps; + } + + private buildMetadata(state: GradingGraphState) { + const totalProcessingTimeMs = Date.now() - state.processing_start_time; + + let llmCalls = 1; + if (state.judgeAResult) llmCalls++; + if (state.judgeBResult) llmCalls++; + if (state.tiebreakResult?.method === "third_judge") llmCalls++; + + let earlyExitReason: string | undefined; + if (!state.judgeAResult) { + earlyExitReason = "High confidence grader result with valid evidence"; + } else if (!state.judgeBResult) { + earlyExitReason = "Judge A agreed with grader within thresholds"; + } + + return { + totalProcessingTimeMs, + llmCalls, + earlyExitReason, + }; + } +} diff --git a/apps/api/src/api/llm/grading/nodes/enhanced-grade.node.ts b/apps/api/src/api/llm/grading/nodes/enhanced-grade.node.ts new file mode 100644 index 00000000..bdbeea70 --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/enhanced-grade.node.ts @@ -0,0 +1,419 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import { Injectable, Logger } from "@nestjs/common"; +import { + addError, + determineFallbackStrategy, + GradingGraphState, + isNodeCircuitBreakerOpen, + recordNodeExecution, + shouldAbortProcessing, + updateCircuitBreaker, +} from "../graph/state"; +import { GradeSchema } from "../schemas/zod-schemas"; +import { + GradeData, + GradingContextData, + LLMGradingRequest, + RubricCriterion, +} from "../types/grading.types"; + +interface LLMGradingService { + gradeWithRubric(context: LLMGradingRequest): Promise; +} + +interface GradingConfig { + defaultTimeout: number; + maxRetries: number; + enableFallbackGrading: boolean; + fallbackConfidence: number; + enablePromptOptimization: boolean; +} + +@Injectable() +export class EnhancedGradeNode { + private readonly logger = new Logger(EnhancedGradeNode.name); + private readonly config: GradingConfig; + + constructor( + private llmGradingService: LLMGradingService, + config?: Partial, + ) { + this.config = { + defaultTimeout: 60_000, + maxRetries: 3, + enableFallbackGrading: true, + fallbackConfidence: 0.5, + enablePromptOptimization: true, + ...config, + }; + } + + async execute(state: GradingGraphState): Promise { + const startTime = Date.now(); + const nodeName = "grade"; + + try { + if (shouldAbortProcessing(state)) { + return addError( + state, + "Processing aborted due to timeout or excessive errors", + nodeName, + ); + } + + if (isNodeCircuitBreakerOpen(state, nodeName)) { + this.logger.warn("Circuit breaker open, using fallback grading"); + return this.executeWithFallback(state, nodeName, startTime); + } + + const gradingResult = await this.performGradingWithTimeout(state); + + if (gradingResult) { + const successState = { + ...state, + graderResult: { + ...gradingResult, + isValid: true, + validationErrors: [], + arithmeticFixed: false, + }, + currentStep: "validate", + shouldContinue: true, + }; + + return this.recordSuccessAndReturn(successState, nodeName, startTime); + } + + return this.handleGradingFailure( + state, + new Error("Grading returned null result"), + nodeName, + startTime, + ); + } catch (error) { + return this.handleGradingFailure(state, error, nodeName, startTime); + } + } + + private async performGradingWithTimeout( + state: GradingGraphState, + ): Promise { + const timeout = state.context.timeout || this.config.defaultTimeout; + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + reject(new Error(`Grading timeout after ${timeout}ms`)); + }, timeout); + + this.llmGradingService + .gradeWithRubric({ + ...this.optimizeGradingContext(state.context), + timeout: timeout - 5000, + }) + .then((result) => { + clearTimeout(timeoutId); + const parsed = GradeSchema.parse(result) as GradeData; + resolve(parsed); + }) + .catch((error) => { + clearTimeout(timeoutId); + reject(error); + }); + }); + } + + private optimizeGradingContext( + context: GradingContextData, + ): LLMGradingRequest { + if (!this.config.enablePromptOptimization) { + return { + questionId: context.questionId, + learnerAnswer: context.learnerAnswer, + rubric: context.rubric, + questionType: context.questionType, + responseType: context.responseType, + timeout: context.timeout, + }; + } + + const optimizedRubric: RubricCriterion[] = context.rubric.map( + (criterion) => ({ + id: criterion.id, + description: this.truncateDescription(criterion.description), + maxPoints: criterion.maxPoints, + keywords: criterion.keywords?.slice(0, 5), + }), + ); + + const optimizedAnswer = this.truncateAnswer( + context.learnerAnswer, + context.questionType, + ); + + return { + ...context, + learnerAnswer: optimizedAnswer, + rubric: optimizedRubric, + }; + } + + private truncateDescription(description: string): string { + if (description.length <= 200) return description; + + const sentences = description.split(/[!.?]+/); + let truncated = sentences[0]; + + for ( + let index = 1; + index < sentences.length && truncated.length < 200; + index++ + ) { + truncated += ". " + sentences[index]; + } + + return truncated.length > 200 ? truncated.slice(0, 197) + "..." : truncated; + } + + private truncateAnswer(answer: string, questionType: string): string { + const maxLength = this.getMaxAnswerLength(questionType); + + if (answer.length <= maxLength) return answer; + + const words = answer.split(/\s+/); + let truncated = ""; + + for (const word of words) { + if ((truncated + " " + word).length > maxLength - 20) break; + truncated += (truncated ? " " : "") + word; + } + + return truncated + "... [truncated]"; + } + + private getMaxAnswerLength(questionType: string): number { + switch (questionType) { + case "TRUE_FALSE": + case "SINGLE_CORRECT": + case "MULTIPLE_CORRECT": { + return 500; + } + case "TEXT": { + return 5000; + } + case "UPLOAD": + case "URL": { + return 10_000; + } + default: { + return 3000; + } + } + } + + private handleGradingFailure( + state: GradingGraphState, + error: any, + nodeName: string, + startTime: number, + ): GradingGraphState { + const errorMessage = + error instanceof Error ? error.message : "Unknown grading error"; + this.logger.error(`Grading failed: ${errorMessage}`, error); + + const strategy = determineFallbackStrategy(state, nodeName); + const executionTime = Date.now() - startTime; + + switch (strategy) { + case "retry": { + if (state.retry_count < this.config.maxRetries) { + const retryState = { + ...addError( + state, + `Grading attempt ${ + state.retry_count + 1 + } failed: ${errorMessage}`, + nodeName, + ), + retry_count: state.retry_count + 1, + currentStep: "grade", + shouldContinue: true, + }; + return recordNodeExecution(retryState, nodeName, executionTime); + } + break; + } + + case "fallback": { + return this.executeWithFallback(state, nodeName, startTime); + } + + case "skip": { + const skipState = addError( + state, + "Grading skipped due to circuit breaker", + nodeName, + ); + return { + ...recordNodeExecution(skipState, nodeName, executionTime), + shouldContinue: false, + currentStep: "error", + }; + } + + case "abort": { + const abortState = addError( + state, + "Grading aborted due to system constraints", + nodeName, + ); + return { + ...recordNodeExecution(abortState, nodeName, executionTime), + shouldContinue: false, + currentStep: "error", + }; + } + } + + const errorState = addError( + state, + `Grading failed: ${errorMessage}`, + nodeName, + ); + return { + ...updateCircuitBreaker( + recordNodeExecution(errorState, nodeName, executionTime), + nodeName, + false, + ), + shouldContinue: false, + currentStep: "error", + }; + } + + private executeWithFallback( + state: GradingGraphState, + nodeName: string, + startTime: number, + ): GradingGraphState { + if (!this.config.enableFallbackGrading) { + const errorState = addError(state, "Fallback grading disabled", nodeName); + return recordNodeExecution(errorState, nodeName, Date.now() - startTime); + } + + this.logger.log("Using fallback grading"); + + try { + const fallbackGrade = this.generateFallbackGrade(state.context); + + const successState = { + ...state, + graderResult: { + ...fallbackGrade, + isValid: true, + validationErrors: [ + "Generated using fallback grading due to LLM failure", + ], + arithmeticFixed: false, + }, + currentStep: "validate", + shouldContinue: true, + fallback_used: true, + }; + + return recordNodeExecution( + successState, + nodeName, + Date.now() - startTime, + ); + } catch (fallbackError) { + const errorState = addError( + state, + `Fallback grading failed: ${ + fallbackError instanceof Error + ? fallbackError.message + : String(fallbackError) + }`, + nodeName, + ); + return recordNodeExecution(errorState, nodeName, Date.now() - startTime); + } + } + + private generateFallbackGrade(context: GradingContextData): GradeData { + const totalMax = context.rubric.reduce( + (sum, criterion) => sum + criterion.maxPoints, + 0, + ); + + const criteriaAwards = context.rubric.map((criterion) => { + const baseScore = criterion.maxPoints * 0.6; + const randomVariation = + (Math.random() - 0.5) * (criterion.maxPoints * 0.2); + const awarded = Math.max( + 0, + Math.min(criterion.maxPoints, baseScore + randomVariation), + ); + + return { + criterionId: criterion.id, + awarded: Math.round(awarded * 10) / 10, + maxPoints: criterion.maxPoints, + justification: + "Score assigned using fallback evaluation due to system constraints.", + evidence: this.extractSimpleEvidence( + context.learnerAnswer, + criterion.keywords, + ), + }; + }); + + const totalAwarded = criteriaAwards.reduce( + (sum, award) => sum + award.awarded, + 0, + ); + + return { + criteriaAwards, + totalAwarded: Math.round(totalAwarded * 10) / 10, + totalMax, + overallFeedback: + "This grade was assigned using automated fallback evaluation. Manual review recommended.", + confidence: this.config.fallbackConfidence, + }; + } + + private extractSimpleEvidence( + answer: string, + keywords?: string[], + ): string | undefined { + if (!keywords || keywords.length === 0) return undefined; + + const answerLower = answer.toLowerCase(); + const matchedKeywords = keywords.filter((keyword) => + answerLower.includes(keyword.toLowerCase()), + ); + + if (matchedKeywords.length === 0) return undefined; + + const sentences = answer.split(/[!.?]+/); + for (const sentence of sentences) { + for (const keyword of matchedKeywords) { + if (sentence.toLowerCase().includes(keyword.toLowerCase())) { + return sentence.trim().slice(0, 100); + } + } + } + + return undefined; + } + + private recordSuccessAndReturn( + state: GradingGraphState, + nodeName: string, + startTime: number, + ): GradingGraphState { + const executionTime = Date.now() - startTime; + const successState = updateCircuitBreaker(state, nodeName, true); + return recordNodeExecution(successState, nodeName, executionTime); + } +} diff --git a/apps/api/src/api/llm/grading/nodes/enhanced-validate.node.ts b/apps/api/src/api/llm/grading/nodes/enhanced-validate.node.ts new file mode 100644 index 00000000..ea31751b --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/enhanced-validate.node.ts @@ -0,0 +1,411 @@ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +/* eslint-disable @typescript-eslint/restrict-plus-operands */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +import { Injectable, Logger } from "@nestjs/common"; +import { + addError, + determineFallbackStrategy, + GradingGraphState, + isNodeCircuitBreakerOpen, + recordNodeExecution, + shouldAbortProcessing, + updateCircuitBreaker, +} from "../graph/state"; +import { ValidatedGradeSchema } from "../schemas/zod-schemas"; + +interface ValidationConfig { + strictMode: boolean; + maxTolerancePct: number; + enableAutoFix: boolean; + timeoutMs: number; +} + +@Injectable() +export class EnhancedValidateNode { + private readonly logger = new Logger(EnhancedValidateNode.name); + private readonly config: ValidationConfig; + + constructor(config?: Partial) { + this.config = { + strictMode: false, + maxTolerancePct: 0.1, + enableAutoFix: true, + timeoutMs: 10_000, + ...config, + }; + } + + async execute(state: GradingGraphState): Promise { + const startTime = Date.now(); + const nodeName = "validate"; + + try { + if (shouldAbortProcessing(state)) { + return addError( + state, + "Processing aborted due to timeout or excessive errors", + nodeName, + ); + } + + if (isNodeCircuitBreakerOpen(state, nodeName)) { + this.logger.warn("Circuit breaker open, using fallback validation"); + return this.executeWithFallback(state, nodeName, startTime); + } + + if (!state.graderResult) { + const errorState = addError( + state, + "No grader result to validate", + nodeName, + ); + return updateCircuitBreaker(errorState, nodeName, false); + } + + const validationResult = await this.validateGradeWithTimeout( + state.graderResult, + ); + + if (validationResult.isValid) { + const successState = { + ...state, + graderResult: validationResult, + currentStep: "evidence", + shouldContinue: true, + }; + + return this.recordSuccessAndReturn(successState, nodeName, startTime); + } + + const strategy = determineFallbackStrategy(state, nodeName); + return this.handleValidationFailure( + state, + validationResult, + strategy, + nodeName, + startTime, + ); + } catch (error) { + return this.handleCriticalError(state, error, nodeName, startTime); + } + } + + private async validateGradeWithTimeout(grade: any): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Validation timeout")); + }, this.config.timeoutMs); + + try { + const result = this.validateGrade(grade); + clearTimeout(timeout); + resolve(result); + } catch (error) { + clearTimeout(timeout); + reject(error); + } + }); + } + + private validateGrade(grade: any) { + const validationErrors: string[] = []; + let arithmeticFixed = false; + const gradeClone = { ...grade }; + + if (!grade) { + throw new Error("No grade to validate"); + } + + try { + const calculatedTotal = gradeClone.criteriaAwards.reduce( + (sum: number, award: any) => sum + (award.awarded || 0), + 0, + ); + const calculatedMax = gradeClone.criteriaAwards.reduce( + (sum: number, award: any) => sum + (award.maxPoints || 0), + 0, + ); + + const totalTolerance = calculatedTotal * this.config.maxTolerancePct; + const maxTolerance = calculatedMax * this.config.maxTolerancePct; + + if ( + Math.abs(calculatedTotal - gradeClone.totalAwarded) > totalTolerance + ) { + if (this.config.enableAutoFix) { + gradeClone.totalAwarded = calculatedTotal; + arithmeticFixed = true; + } else { + validationErrors.push( + `Total awarded mismatch: calculated=${calculatedTotal}, provided=${gradeClone.totalAwarded}`, + ); + } + } + + if (Math.abs(calculatedMax - gradeClone.totalMax) > maxTolerance) { + if (this.config.enableAutoFix) { + gradeClone.totalMax = calculatedMax; + arithmeticFixed = true; + } else { + validationErrors.push( + `Total max mismatch: calculated=${calculatedMax}, provided=${gradeClone.totalMax}`, + ); + } + } + + for (const award of gradeClone.criteriaAwards) { + if (award.awarded < 0) { + if (this.config.enableAutoFix) { + award.awarded = 0; + arithmeticFixed = true; + } else { + validationErrors.push( + `Negative score for criterion ${award.criterionId}`, + ); + } + } + + if ( + award.awarded > + award.maxPoints + award.maxPoints * this.config.maxTolerancePct + ) { + if (this.config.enableAutoFix) { + award.awarded = award.maxPoints; + arithmeticFixed = true; + } else { + validationErrors.push( + `Score exceeds max for criterion ${award.criterionId}`, + ); + } + } + + if (!award.justification || award.justification.trim().length === 0) { + if (this.config.enableAutoFix) { + award.justification = "Score assigned based on rubric criteria"; + arithmeticFixed = true; + } else { + validationErrors.push( + `Missing justification for criterion ${award.criterionId}`, + ); + } + } + + if (award.justification && award.justification.length > 500) { + if (this.config.enableAutoFix) { + award.justification = award.justification.slice(0, 497) + "..."; + arithmeticFixed = true; + } else { + validationErrors.push( + `Justification too long for criterion ${award.criterionId}`, + ); + } + } + } + + if (gradeClone.confidence < 0 || gradeClone.confidence > 1) { + if (this.config.enableAutoFix) { + gradeClone.confidence = Math.max( + 0, + Math.min(1, gradeClone.confidence), + ); + arithmeticFixed = true; + } else { + validationErrors.push("Confidence must be between 0 and 1"); + } + } + + if ( + gradeClone.overallFeedback && + gradeClone.overallFeedback.length > 1000 + ) { + if (this.config.enableAutoFix) { + gradeClone.overallFeedback = + gradeClone.overallFeedback.slice(0, 997) + "..."; + arithmeticFixed = true; + } else { + validationErrors.push("Overall feedback too long"); + } + } + + if (this.config.strictMode && validationErrors.length > 0) { + throw new Error( + `Strict validation failed: ${validationErrors.join(", ")}`, + ); + } + + try { + ValidatedGradeSchema.parse({ + ...gradeClone, + isValid: validationErrors.length === 0, + validationErrors, + arithmeticFixed, + }); + } catch (zodError) { + validationErrors.push( + `Schema validation failed: ${ + zodError instanceof Error ? zodError.message : String(zodError) + }`, + ); + } + + return { + ...gradeClone, + isValid: validationErrors.length === 0, + validationErrors, + arithmeticFixed, + }; + } catch (error) { + this.logger.error("Validation processing error:", error); + return { + ...gradeClone, + isValid: false, + validationErrors: [ + `Validation processing failed: ${ + error instanceof Error ? error.message : String(error) + }`, + ], + arithmeticFixed, + }; + } + } + + private handleValidationFailure( + state: GradingGraphState, + validationResult: any, + strategy: string, + nodeName: string, + startTime: number, + ): GradingGraphState { + const executionTime = Date.now() - startTime; + + switch (strategy) { + case "retry": { + if (state.retry_count < (state.context.maxRetries || 2)) { + const retryState = { + ...state, + graderResult: validationResult, + retry_count: state.retry_count + 1, + currentStep: "grade", + shouldContinue: true, + }; + return recordNodeExecution(retryState, nodeName, executionTime); + } + break; + } + + case "fallback": { + return this.executeWithFallback(state, nodeName, startTime); + } + + case "skip": { + this.logger.warn("Skipping validation due to circuit breaker"); + const skipState = addError( + state, + "Validation skipped due to repeated failures", + nodeName, + ); + return recordNodeExecution(skipState, nodeName, executionTime); + } + + case "abort": { + const abortState = addError( + state, + "Validation aborted due to system constraints", + nodeName, + ); + return { + ...recordNodeExecution(abortState, nodeName, executionTime), + shouldContinue: false, + currentStep: "error", + }; + } + } + + const failedState = addError( + state, + `Validation failed after retries: ${validationResult.validationErrors.join( + ", ", + )}`, + nodeName, + ); + + return { + ...updateCircuitBreaker(failedState, nodeName, false), + graderResult: validationResult, + shouldContinue: false, + currentStep: "error", + }; + } + + private executeWithFallback( + state: GradingGraphState, + nodeName: string, + startTime: number, + ): GradingGraphState { + this.logger.log("Using fallback validation"); + + if (!state.graderResult) { + const errorState = addError( + state, + "No grader result for fallback validation", + nodeName, + ); + return recordNodeExecution(errorState, nodeName, Date.now() - startTime); + } + + const fallbackResult = { + ...state.graderResult, + isValid: true, + validationErrors: ["Fallback validation used due to system constraints"], + arithmeticFixed: false, + }; + + const successState = { + ...state, + graderResult: fallbackResult, + currentStep: "evidence", + shouldContinue: true, + fallback_used: true, + }; + + return recordNodeExecution(successState, nodeName, Date.now() - startTime); + } + + private handleCriticalError( + state: GradingGraphState, + error: any, + nodeName: string, + startTime: number, + ): GradingGraphState { + const errorMessage = + error instanceof Error ? error.message : "Unknown validation error"; + this.logger.error(`Critical validation error: ${errorMessage}`, error); + + const errorState = addError( + state, + `Critical validation error: ${errorMessage}`, + nodeName, + ); + const updatedState = updateCircuitBreaker(errorState, nodeName, false); + + return { + ...recordNodeExecution(updatedState, nodeName, Date.now() - startTime), + shouldContinue: false, + currentStep: "error", + }; + } + + private recordSuccessAndReturn( + state: GradingGraphState, + nodeName: string, + startTime: number, + ): GradingGraphState { + const executionTime = Date.now() - startTime; + const successState = updateCircuitBreaker(state, nodeName, true); + return recordNodeExecution(successState, nodeName, executionTime); + } +} diff --git a/apps/api/src/api/llm/grading/nodes/evidence.node.ts b/apps/api/src/api/llm/grading/nodes/evidence.node.ts new file mode 100644 index 00000000..7b09a6e6 --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/evidence.node.ts @@ -0,0 +1,66 @@ +import { Injectable } from "@nestjs/common"; +import { GradingGraphState } from "../graph/state"; +import { EvidenceService } from "../services/evidence.service"; + +@Injectable() +export class EvidenceNode { + constructor(private evidenceService: EvidenceService) {} + + async execute(state: GradingGraphState): Promise { + if (!state.graderResult) { + return { + ...state, + errors: [...state.errors, "No grader result for evidence verification"], + shouldContinue: false, + currentStep: "error", + }; + } + + try { + const evidenceVerification = await this.evidenceService.verifyEvidence( + state.context.learnerAnswer, + state.graderResult, + ); + + let updatedGraderResult = state.graderResult; + + if ( + !evidenceVerification.ok && + evidenceVerification.invalidCriteriaIds.length > 0 + ) { + updatedGraderResult = this.evidenceService.zeroOutInvalidCriteria( + state.graderResult, + evidenceVerification.invalidCriteriaIds, + ); + } + + return { + ...state, + graderResult: updatedGraderResult, + evidenceVerification, + currentStep: "judge_check", + shouldContinue: true, + }; + } catch (error) { + const errorMessage = + error instanceof Error + ? error.message + : "Unknown evidence verification error"; + + return { + ...state, + errors: [ + ...state.errors, + `Evidence verification failed: ${errorMessage}`, + ], + evidenceVerification: { + ok: false, + invalidCriteriaIds: [], + details: [], + }, + currentStep: "judge_check", + shouldContinue: true, + }; + } + } +} diff --git a/apps/api/src/api/llm/grading/nodes/grade.node.ts b/apps/api/src/api/llm/grading/nodes/grade.node.ts new file mode 100644 index 00000000..8d47ac11 --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/grade.node.ts @@ -0,0 +1,90 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { Injectable } from "@nestjs/common"; +import { GradingGraphState } from "../graph/state"; +import { GradeSchema } from "../schemas/zod-schemas"; +import { GradeData, ValidatedGradeData } from "../types/grading.types"; + +interface LLMGradingService { + gradeWithRubric(context: { + questionId: string; + learnerAnswer: string; + rubric: Array<{ + id: string; + description: string; + maxPoints: number; + keywords?: string[]; + }>; + questionType: string; + responseType?: string; + }): Promise; +} + +@Injectable() +export class GradeNode { + constructor(private llmGradingService: LLMGradingService) {} + + async execute(state: GradingGraphState): Promise { + try { + const result = await this.llmGradingService.gradeWithRubric({ + questionId: state.context.questionId, + learnerAnswer: state.context.learnerAnswer, + rubric: state.context.rubric, + questionType: state.context.questionType, + responseType: state.context.responseType, + }); + + const parsed = GradeSchema.parse(result); + + return { + ...state, + graderResult: { + ...(parsed as GradeData), + isValid: true, + validationErrors: [], + arithmeticFixed: false, + } as ValidatedGradeData, + currentStep: "validate", + shouldContinue: true, + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown grading error"; + + return { + ...state, + errors: [...state.errors, `Grading failed: ${errorMessage}`], + shouldContinue: false, + currentStep: "error", + }; + } + } + + private createOptimizedPrompt(context: GradingGraphState["context"]): string { + const criteriaList = context.rubric + .map((c) => `${c.id}: ${c.description} (${c.maxPoints}pts)`) + .join("\n"); + + return `Grade this answer using the provided criteria: + +ANSWER: "${context.learnerAnswer}" + +CRITERIA: +${criteriaList} + +Return JSON with exact structure: +{ + "criteriaAwards": [{"criterionId": "...", "awarded": 0-${Math.max( + ...context.rubric.map((r) => r.maxPoints), + )}, "maxPoints": N, "justification": "1-2 sentences", "evidence": "exact quote or omit"}], + "totalAwarded": N, + "totalMax": ${context.rubric.reduce((sum, c) => sum + c.maxPoints, 0)}, + "overallFeedback": "brief summary", + "confidence": 0.0-1.0 +} + +Requirements: +- Justifications: 1-2 sentences max +- Evidence: exact quotes from answer, omit if none +- Be precise with scoring`; + } +} diff --git a/apps/api/src/api/llm/grading/nodes/judge.node.ts b/apps/api/src/api/llm/grading/nodes/judge.node.ts new file mode 100644 index 00000000..ee4bdd5c --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/judge.node.ts @@ -0,0 +1,179 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { Injectable } from "@nestjs/common"; +import { GradingGraphState } from "../graph/state"; +import { GradeSchema } from "../schemas/zod-schemas"; +import { GradeData } from "../types/grading.types"; + +interface LLMJudgeService { + judgeGrading(context: { + questionId: string; + learnerAnswer: string; + rubric: Array<{ + id: string; + description: string; + maxPoints: number; + }>; + specificCriteria?: string[]; + }): Promise; +} + +@Injectable() +export class JudgeNode { + constructor(private llmJudgeService: LLMJudgeService) {} + + async executeJudgeA(state: GradingGraphState): Promise { + return this.executeJudge(state, "judgeA"); + } + + async executeJudgeB(state: GradingGraphState): Promise { + const differingCriteria = this.getDifferingCriteria(state); + return this.executeJudge(state, "judgeB", differingCriteria); + } + + private async executeJudge( + state: GradingGraphState, + judgeName: "judgeA" | "judgeB", + specificCriteria?: string[], + ): Promise { + try { + const result = await this.llmJudgeService.judgeGrading({ + questionId: state.context.questionId, + learnerAnswer: state.context.learnerAnswer, + rubric: specificCriteria + ? state.context.rubric.filter((r) => specificCriteria.includes(r.id)) + : state.context.rubric, + specificCriteria, + }); + + const parsed = GradeSchema.parse(result); + + return judgeName === "judgeA" + ? { + ...state, + judgeAResult: parsed as GradeData, + currentStep: "judgeB_check", + shouldContinue: true, + } + : { + ...state, + judgeBResult: specificCriteria + ? this.mergePartialGrade( + state, + parsed as GradeData, + specificCriteria, + ) + : (parsed as GradeData), + currentStep: "compare", + shouldContinue: true, + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : `Unknown ${judgeName} error`; + + return { + ...state, + errors: [...state.errors, `${judgeName} failed: ${errorMessage}`], + currentStep: "compare", + shouldContinue: true, + }; + } + } + + private getDifferingCriteria(state: GradingGraphState): string[] { + if (!state.graderResult || !state.judgeAResult) { + return []; + } + + const differingCriteria: string[] = []; + + for (const graderAward of state.graderResult.criteriaAwards) { + const judgeAward = state.judgeAResult.criteriaAwards.find( + (a) => a.criterionId === graderAward.criterionId, + ); + + if ( + judgeAward && + Math.abs(graderAward.awarded - judgeAward.awarded) > 1 + ) { + differingCriteria.push(graderAward.criterionId); + } + } + + return differingCriteria; + } + + private mergePartialGrade( + state: GradingGraphState, + partialGrade: { + criteriaAwards: Array<{ criterionId: string; awarded: number }>; + totalAwarded: number; + overallFeedback: string; + confidence: number; + }, + updatedCriteria: string[], + ): any { + if (!state.judgeAResult) { + return partialGrade; + } + + const mergedAwards = state.judgeAResult.criteriaAwards.map((award) => { + if (updatedCriteria.includes(award.criterionId)) { + const updatedAward = partialGrade.criteriaAwards.find( + (a: { criterionId: string; awarded: number; evidence?: string[] }) => + a.criterionId === award.criterionId, + ); + return updatedAward || award; + } + return award; + }); + + const totalAwarded = mergedAwards.reduce( + (sum, award) => sum + award.awarded, + 0, + ); + + return { + ...state.judgeAResult, + criteriaAwards: mergedAwards, + totalAwarded, + overallFeedback: `${ + state.judgeAResult.overallFeedback + } [Updated: ${updatedCriteria.join(", ")}]`, + }; + } + + private createJudgePrompt( + context: GradingGraphState["context"], + specificCriteria?: string[], + ): string { + const targetRubric = specificCriteria + ? context.rubric.filter((r) => specificCriteria.includes(r.id)) + : context.rubric; + + const criteriaList = targetRubric + .map((c) => `${c.id}: ${c.description} (${c.maxPoints}pts)`) + .join("\n"); + + const instruction = specificCriteria + ? `Re-grade ONLY these specific criteria: ${specificCriteria.join(", ")}` + : "Grade this answer independently"; + + return `${instruction} + +ANSWER: "${context.learnerAnswer}" + +CRITERIA: +${criteriaList} + +Provide independent scoring without seeing previous grades. + +Return JSON: +{ + "criteriaAwards": [...], + "totalAwarded": N, + "totalMax": ${targetRubric.reduce((sum, c) => sum + c.maxPoints, 0)}, + "overallFeedback": "brief assessment", + "confidence": 0.0-1.0 +}`; + } +} diff --git a/apps/api/src/api/llm/grading/nodes/tiebreak.node.ts b/apps/api/src/api/llm/grading/nodes/tiebreak.node.ts new file mode 100644 index 00000000..2d00e775 --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/tiebreak.node.ts @@ -0,0 +1,121 @@ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { Injectable } from "@nestjs/common"; +import { GradingGraphState } from "../graph/state"; +import { GradeSchema } from "../schemas/zod-schemas"; +import { GradeData, TiebreakResultData } from "../types/grading.types"; + +interface LLMThirdJudgeService { + judgeGrading(context: { + questionId: string; + learnerAnswer: string; + rubric: Array<{ + id: string; + description: string; + maxPoints: number; + }>; + }): Promise; +} + +interface MetaDeciderService { + decide(features: { + deltaA: number; + deltaB: number; + agreementPct: number; + evidenceDensity: number; + }): Promise<"accept_grader" | "accept_judges" | "tiebreak">; +} + +@Injectable() +export class TiebreakNode { + constructor( + private llmThirdJudgeService: LLMThirdJudgeService, + private metaDeciderService: MetaDeciderService, + ) {} + + async execute(state: GradingGraphState): Promise { + if (!state.comparison || !state.graderResult || !state.judgeAResult) { + return { + ...state, + errors: [...state.errors, "Insufficient data for tiebreak"], + currentStep: "decision", + shouldContinue: true, + }; + } + + try { + const tiebreakResult = await this.performTiebreak(state); + + return { + ...state, + tiebreakResult, + currentStep: "decision", + shouldContinue: true, + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown tiebreak error"; + + return { + ...state, + errors: [...state.errors, `Tiebreak failed: ${errorMessage}`], + tiebreakResult: { + method: "meta_decider", + metaDecision: "accept_grader", + confidence: 0.5, + }, + currentStep: "decision", + shouldContinue: true, + }; + } + } + + private async performTiebreak( + state: GradingGraphState, + ): Promise { + const features = this.extractFeatures(state); + + const useMetaDecider = Math.random() > 0.5; + + if (useMetaDecider) { + const metaDecision = await this.metaDeciderService.decide(features); + + return { + method: "meta_decider", + metaDecision, + confidence: 0.8, + }; + } + + const thirdJudgeResult = await this.llmThirdJudgeService.judgeGrading({ + questionId: state.context.questionId, + learnerAnswer: state.context.learnerAnswer, + rubric: state.context.rubric, + }); + + const parsed = GradeSchema.parse(thirdJudgeResult); + + return { + method: "third_judge", + result: parsed as GradeData, + confidence: (parsed as GradeData).confidence, + }; + } + + private extractFeatures(state: GradingGraphState) { + const comparison = state.comparison; + + const evidenceCriteriaCount = state.graderResult.criteriaAwards.filter( + (award) => award.evidence && award.evidence.length > 0, + ).length; + + const evidenceDensity = + evidenceCriteriaCount / state.graderResult.criteriaAwards.length; + + return { + deltaA: comparison.graderVsJudgeA.totalDelta, + deltaB: comparison.judgeAVsJudgeB?.totalDelta ?? 0, + agreementPct: comparison.graderVsJudgeA.agreementPct, + evidenceDensity, + }; + } +} diff --git a/apps/api/src/api/llm/grading/nodes/validate.node.ts b/apps/api/src/api/llm/grading/nodes/validate.node.ts new file mode 100644 index 00000000..491b56d5 --- /dev/null +++ b/apps/api/src/api/llm/grading/nodes/validate.node.ts @@ -0,0 +1,156 @@ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/require-await */ +import { Injectable } from "@nestjs/common"; +import { GradingGraphState } from "../graph/state"; +import { ValidatedGradeSchema } from "../schemas/zod-schemas"; + +@Injectable() +export class ValidateNode { + async execute(state: GradingGraphState): Promise { + if (!state.graderResult) { + return { + ...state, + errors: [...state.errors, "No grader result to validate"], + shouldContinue: false, + currentStep: "error", + }; + } + + try { + const validationResult = this.validateGrade(state.graderResult); + + if (validationResult.isValid) { + return { + ...state, + graderResult: validationResult, + currentStep: "evidence", + shouldContinue: true, + }; + } + + if (state.retry_count < 2) { + return { + ...state, + graderResult: validationResult, + retry_count: state.retry_count + 1, + currentStep: "grade", + shouldContinue: true, + }; + } + + return { + ...state, + graderResult: validationResult, + errors: [ + ...state.errors, + `Validation failed after retries: ${validationResult.validationErrors.join( + ", ", + )}`, + ], + shouldContinue: false, + currentStep: "error", + }; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown validation error"; + + return { + ...state, + errors: [...state.errors, `Validation error: ${errorMessage}`], + shouldContinue: false, + currentStep: "error", + }; + } + } + + private validateGrade(grade: GradingGraphState["graderResult"]) { + const validationErrors: string[] = []; + let arithmeticFixed = false; + + if (!grade) { + throw new Error("No grade to validate"); + } + + const calculatedTotal = grade.criteriaAwards.reduce( + (sum, award) => sum + award.awarded, + 0, + ); + const calculatedMax = grade.criteriaAwards.reduce( + (sum, award) => sum + award.maxPoints, + 0, + ); + + if (Math.abs(calculatedTotal - grade.totalAwarded) > 0.01) { + grade.totalAwarded = calculatedTotal; + arithmeticFixed = true; + } + + if (Math.abs(calculatedMax - grade.totalMax) > 0.01) { + grade.totalMax = calculatedMax; + arithmeticFixed = true; + } + + for (const award of grade.criteriaAwards) { + if (award.awarded < 0) { + award.awarded = 0; + arithmeticFixed = true; + validationErrors.push( + `Negative score clamped to 0 for criterion ${award.criterionId}`, + ); + } + + if (award.awarded > award.maxPoints) { + award.awarded = award.maxPoints; + arithmeticFixed = true; + validationErrors.push( + `Score clamped to max for criterion ${award.criterionId}`, + ); + } + + if (!award.justification || award.justification.trim().length === 0) { + validationErrors.push( + `Missing justification for criterion ${award.criterionId}`, + ); + } + + if (award.justification && award.justification.length > 500) { + validationErrors.push( + `Justification too long for criterion ${award.criterionId}`, + ); + } + } + + if (grade.confidence < 0 || grade.confidence > 1) { + grade.confidence = Math.max(0, Math.min(1, grade.confidence)); + arithmeticFixed = true; + } + + if (grade.overallFeedback && grade.overallFeedback.length > 1000) { + validationErrors.push("Overall feedback too long"); + } + + try { + ValidatedGradeSchema.parse({ + ...grade, + isValid: validationErrors.length === 0, + validationErrors, + arithmeticFixed, + }); + } catch (zodError) { + validationErrors.push( + `Schema validation failed: ${ + zodError instanceof Error ? zodError.message : String(zodError) + }`, + ); + } + + return { + ...grade, + isValid: validationErrors.length === 0, + validationErrors, + arithmeticFixed, + }; + } +} diff --git a/apps/api/src/api/llm/grading/schemas/zod-schemas.ts b/apps/api/src/api/llm/grading/schemas/zod-schemas.ts new file mode 100644 index 00000000..456d252f --- /dev/null +++ b/apps/api/src/api/llm/grading/schemas/zod-schemas.ts @@ -0,0 +1,242 @@ +import { z } from "zod"; + +export const CriterionAwardSchema = z.object({ + criterionId: z.string().min(1), + awarded: z.number().min(0).finite(), + maxPoints: z.number().min(0).finite(), + justification: z.string().min(1).max(500).trim(), + evidence: z.string().max(1000).trim().optional(), +}); + +export const GradeSchema = z + .object({ + criteriaAwards: z.array(CriterionAwardSchema).min(1), + totalAwarded: z.number().min(0).finite(), + totalMax: z.number().min(0.1).finite(), + overallFeedback: z.string().max(1000).trim(), + confidence: z.number().min(0).max(1).finite(), + }) + .refine( + (data) => { + const calculatedTotal = data.criteriaAwards.reduce( + (sum, award) => sum + award.awarded, + 0, + ); + return Math.abs(calculatedTotal - data.totalAwarded) <= 0.1; + }, + { + message: + "totalAwarded must equal sum of criterion awards (tolerance: 0.1)", + path: ["totalAwarded"], + }, + ) + .refine( + (data) => { + const calculatedMax = data.criteriaAwards.reduce( + (sum, award) => sum + award.maxPoints, + 0, + ); + return Math.abs(calculatedMax - data.totalMax) <= 0.1; + }, + { + message: + "totalMax must equal sum of criterion maxPoints (tolerance: 0.1)", + path: ["totalMax"], + }, + ) + .refine((data) => data.totalAwarded <= data.totalMax + 0.1, { + message: "totalAwarded cannot exceed totalMax", + path: ["totalAwarded"], + }); + +export const ValidatedGradeSchema = z + .object({ + criteriaAwards: z.array(CriterionAwardSchema).min(1), + totalAwarded: z.number().min(0).finite(), + totalMax: z.number().min(0.1).finite(), + overallFeedback: z.string().max(1000).trim(), + confidence: z.number().min(0).max(1).finite(), + isValid: z.boolean(), + validationErrors: z.array(z.string()), + arithmeticFixed: z.boolean().default(false), + }) + .refine( + (data) => { + const calculatedTotal = data.criteriaAwards.reduce( + (sum, award) => sum + award.awarded, + 0, + ); + return Math.abs(calculatedTotal - data.totalAwarded) <= 0.1; + }, + { + message: + "totalAwarded must equal sum of criterion awards (tolerance: 0.1)", + path: ["totalAwarded"], + }, + ) + .refine( + (data) => { + const calculatedMax = data.criteriaAwards.reduce( + (sum, award) => sum + award.maxPoints, + 0, + ); + return Math.abs(calculatedMax - data.totalMax) <= 0.1; + }, + { + message: + "totalMax must equal sum of criterion maxPoints (tolerance: 0.1)", + path: ["totalMax"], + }, + ) + .refine((data) => data.totalAwarded <= data.totalMax + 0.1, { + message: "totalAwarded cannot exceed totalMax", + path: ["totalAwarded"], + }); + +export const EvidenceVerificationSchema = z.object({ + ok: z.boolean(), + invalidCriteriaIds: z.array(z.string()), + details: z.array( + z.object({ + criterionId: z.string(), + issue: z.enum([ + "missing_evidence", + "evidence_not_found", + "fuzzy_match_failed", + ]), + evidence: z.string().optional(), + }), + ), +}); + +export const JudgeComparisonSchema = z.object({ + graderVsJudgeA: z.object({ + totalDelta: z.number(), + criterionDeltas: z.array( + z.object({ + criterionId: z.string(), + delta: z.number(), + }), + ), + agreementPct: z.number().min(0).max(1), + }), + judgeAVsJudgeB: z + .object({ + totalDelta: z.number(), + criterionDeltas: z.array( + z.object({ + criterionId: z.string(), + delta: z.number(), + }), + ), + agreementPct: z.number().min(0).max(1), + }) + .optional(), +}); + +export const TiebreakResultSchema = z.object({ + method: z.enum(["third_judge", "meta_decider"]), + result: GradeSchema.optional(), + metaDecision: z + .enum(["accept_grader", "accept_judges", "tiebreak"]) + .optional(), + confidence: z.number().min(0).max(1), +}); + +export const FinalGradeSchema = z.object({ + selectedSource: z.enum(["grader", "judges", "tiebreak"]), + grade: GradeSchema, + reasoning: z.string().max(500), + processingSteps: z.array( + z.enum([ + "grade", + "validate", + "judgeA", + "judgeB", + "evidence", + "compare", + "tiebreak", + "decision", + ]), + ), + metadata: z.object({ + totalProcessingTimeMs: z.number(), + llmCalls: z.number(), + earlyExitReason: z.string().optional(), + }), +}); + +export const GradingContextSchema = z.object({ + questionId: z.string().min(1), + learnerAnswer: z.string().min(1).max(50_000), + rubric: z + .array( + z.object({ + id: z.string().min(1), + description: z.string().min(10).max(2000), + maxPoints: z.number().min(0.1).max(1000).finite(), + keywords: z.array(z.string().min(1)).optional(), + }), + ) + .min(1) + .max(20), + questionType: z.enum([ + "TEXT", + "UPLOAD", + "URL", + "TRUE_FALSE", + "SINGLE_CORRECT", + "MULTIPLE_CORRECT", + ]), + responseType: z.string().max(50).optional(), + timeout: z.number().min(5000).max(300_000).default(60_000), + maxRetries: z.number().min(0).max(5).default(2), +}); + +export const GraphStateSchema = z.object({ + context: GradingContextSchema, + graderResult: ValidatedGradeSchema.optional(), + judgeAResult: GradeSchema.optional(), + judgeBResult: GradeSchema.optional(), + evidenceVerification: EvidenceVerificationSchema.optional(), + comparison: JudgeComparisonSchema.optional(), + tiebreakResult: TiebreakResultSchema.optional(), + finalGrade: FinalGradeSchema.optional(), + errors: z.array(z.string()).default([]), + currentStep: z.string(), + shouldContinue: z.boolean().default(true), +}); + +export const ErrorRecoverySchema = z.object({ + attempts: z.number().min(0).max(10), + lastError: z.string().optional(), + recoveryStrategy: z.enum(["retry", "fallback", "skip", "abort"]), + fallbackUsed: z.boolean().default(false), +}); + +export const CircuitBreakerSchema = z.object({ + failures: z.number().min(0), + lastFailure: z.number().optional(), + isOpen: z.boolean().default(false), + resetTimeout: z.number().min(1000).default(60_000), +}); + +export const ProcessingMetricsSchema = z.object({ + nodeExecutionTimes: z.record(z.string(), z.number()), + memoryUsage: z.number().optional(), + llmTokensUsed: z.number().min(0).default(0), + cacheHits: z.number().min(0).default(0), +}); + +export type CriterionAward = z.infer; +export type Grade = z.infer; +export type ValidatedGrade = z.infer; +export type EvidenceVerification = z.infer; +export type JudgeComparison = z.infer; +export type TiebreakResult = z.infer; +export type FinalGrade = z.infer; +export type GradingContext = z.infer; +export type GraphState = z.infer; +export type ErrorRecovery = z.infer; +export type CircuitBreaker = z.infer; +export type ProcessingMetrics = z.infer; diff --git a/apps/api/src/api/llm/grading/services/enhanced-automated-grading.service.ts b/apps/api/src/api/llm/grading/services/enhanced-automated-grading.service.ts new file mode 100644 index 00000000..880079be --- /dev/null +++ b/apps/api/src/api/llm/grading/services/enhanced-automated-grading.service.ts @@ -0,0 +1,622 @@ +import { Injectable, Logger, OnModuleDestroy } from "@nestjs/common"; +import { v4 as uuidv4 } from "uuid"; +import { GradingGraph } from "../graph/grade.graph"; +import { GradingGraphState, initialState } from "../graph/state"; +import { CompareNode } from "../nodes/compare.node"; +import { DecisionNode } from "../nodes/decision.node"; +import { EnhancedGradeNode } from "../nodes/enhanced-grade.node"; +import { EnhancedValidateNode } from "../nodes/enhanced-validate.node"; +import { EvidenceNode } from "../nodes/evidence.node"; +import { JudgeNode } from "../nodes/judge.node"; +import { FinalGradeData, GradingContextData } from "../types/grading.types"; +import { EnhancedPolicyService } from "./enhanced-policy.service"; +import { EvidenceService } from "./evidence.service"; +import { MetaDeciderService } from "./meta-decider.service"; +import { MonitoringService, SystemHealth } from "./monitoring.service"; + +interface GradingResult { + requestId: string; + success: boolean; + finalGrade?: FinalGradeData; + errors: string[]; + warnings: string[]; + processingTimeMs: number; + riskLevel: "low" | "medium" | "high"; + fallbackUsed: boolean; + debugInfo?: { + processingSteps?: string[]; + nodeExecutionTimes?: Record; + errorRecovery?: any; + circuitBreakerStatus?: Record; + }; +} + +interface GradingServiceConfig { + maxConcurrentRequests: number; + enableBatching: boolean; + batchSize: number; + batchTimeoutMs: number; + enableDebugMode: boolean; + enableCaching: boolean; + defaultTimeout: number; + enableFailfast: boolean; +} + +@Injectable() +export class EnhancedAutomatedGradingService implements OnModuleDestroy { + private readonly logger = new Logger(EnhancedAutomatedGradingService.name); + private graph: GradingGraph | null = null; + private readonly config: GradingServiceConfig; + private readonly activeRequests = new Map>(); + private readonly requestQueue: Array<{ + context: GradingContextData; + resolve: (result: GradingResult) => void; + reject: (error: Error) => void; + timestamp: number; + }> = []; + private batchTimer: NodeJS.Timeout | null = null; + private shutdownRequested = false; + + constructor( + private enhancedGradeNode: EnhancedGradeNode, + private enhancedValidateNode: EnhancedValidateNode, + private judgeNode: JudgeNode, + private evidenceNode: EvidenceNode, + private compareNode: CompareNode, + private decisionNode: DecisionNode, + private enhancedPolicyService: EnhancedPolicyService, + private evidenceService: EvidenceService, + private metaDeciderService: MetaDeciderService, + private monitoringService: MonitoringService, + config?: Partial, + ) { + this.config = { + maxConcurrentRequests: 50, + enableBatching: true, + batchSize: 10, + batchTimeoutMs: 5000, + enableDebugMode: false, + enableCaching: true, + defaultTimeout: 120_000, + enableFailfast: true, + ...config, + }; + + this.initializeGraph(); + this.logger.log( + "Enhanced Automated Grading Service initialized", + this.config, + ); + } + + async executeGradingPipeline( + context: GradingContextData, + ): Promise { + if (this.shutdownRequested) { + throw new Error("Service is shutting down, cannot process new requests"); + } + + const requestId = uuidv4(); + + try { + if (this.activeRequests.size >= this.config.maxConcurrentRequests) { + if (this.config.enableFailfast) { + throw new Error("Maximum concurrent requests exceeded"); + } + + if (this.config.enableBatching) { + return this.queueForBatch(context); + } + } + + const resultPromise = this.processGradingRequest(requestId, context); + this.activeRequests.set(requestId, resultPromise); + + try { + const result = await resultPromise; + return result; + } finally { + this.activeRequests.delete(requestId); + } + } catch (error) { + this.activeRequests.delete(requestId); + const errorMessage = + error instanceof Error ? error.message : "Unknown error"; + + this.logger.error( + `Failed to execute grading pipeline for ${requestId}:`, + error, + ); + + return { + requestId, + success: false, + errors: [errorMessage], + warnings: [], + processingTimeMs: 0, + riskLevel: "high", + fallbackUsed: false, + }; + } + } + + private async processGradingRequest( + requestId: string, + context: GradingContextData, + ): Promise { + const startTime = Date.now(); + + try { + this.monitoringService.startGradingRequest(requestId, context); + + const validatedContext = await this.validateAndSanitizeContext(context); + + if (this.shouldEarlyExit(validatedContext)) { + return this.executeSimpleGrading( + requestId, + validatedContext, + startTime, + ); + } + + return this.executeFullGrading(requestId, validatedContext, startTime); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown processing error"; + + const result: GradingResult = { + requestId, + success: false, + errors: [`Pipeline execution failed: ${errorMessage}`], + warnings: [], + processingTimeMs: Date.now() - startTime, + riskLevel: "high", + fallbackUsed: false, + }; + + this.monitoringService.completeGradingRequest( + requestId, + false, + undefined, + { + attempts: 1, + lastError: errorMessage, + recoveryStrategy: "abort", + fallbackUsed: false, + }, + ); + + return result; + } + } + + private async validateAndSanitizeContext( + context: GradingContextData, + ): Promise { + const sanitized = { ...context }; + + if (sanitized.learnerAnswer.length > 50_000) { + sanitized.learnerAnswer = + sanitized.learnerAnswer.slice(0, 47_000) + + "... [truncated for processing]"; + this.logger.warn("Answer truncated due to length"); + } + + if (sanitized.rubric.length > 20) { + sanitized.rubric = sanitized.rubric.slice(0, 20); + this.logger.warn("Rubric truncated to 20 criteria"); + } + + sanitized.rubric = sanitized.rubric.filter( + (criterion) => + criterion.maxPoints > 0 && criterion.description.length > 0, + ); + + if (sanitized.rubric.length === 0) { + throw new Error("No valid rubric criteria found"); + } + + return sanitized; + } + + private shouldEarlyExit(context: GradingContextData): boolean { + return ( + ["TRUE_FALSE", "SINGLE_CORRECT", "MULTIPLE_CORRECT"].includes( + context.questionType, + ) && + context.rubric.length <= 5 && + context.learnerAnswer.length < 1000 + ); + } + + private async executeSimpleGrading( + requestId: string, + context: GradingContextData, + startTime: number, + ): Promise { + const state = initialState(context); + const warnings: string[] = []; + + try { + let currentState = await this.executeNodeWithRecovery( + requestId, + "grade", + () => this.enhancedGradeNode.execute(state), + ); + + currentState = await this.executeNodeWithRecovery( + requestId, + "validate", + () => this.enhancedValidateNode.execute(currentState), + ); + + currentState = await this.executeNodeWithRecovery( + requestId, + "evidence", + () => this.evidenceNode.execute(currentState), + ); + + if ( + this.enhancedPolicyService.shouldEarlyExit( + currentState.graderResult, + currentState.evidenceVerification, + currentState.context.questionType, + { + questionType: context.questionType, + fallbackUsed: currentState.fallback_used, + errorCount: currentState.errors.length, + }, + ) + ) { + currentState = await this.executeNodeWithRecovery( + requestId, + "decision", + () => this.decisionNode.execute(currentState), + ); + + const result: GradingResult = { + requestId, + success: true, + finalGrade: currentState.finalGrade, + errors: currentState.errors, + warnings: [...warnings, "Early exit processing used"], + processingTimeMs: Date.now() - startTime, + riskLevel: currentState.fallback_used ? "medium" : "low", + fallbackUsed: currentState.fallback_used, + debugInfo: this.config.enableDebugMode + ? { + processingSteps: currentState.finalGrade?.processingSteps, + nodeExecutionTimes: + currentState.processing_metrics.nodeExecutionTimes, + } + : undefined, + }; + + this.monitoringService.completeGradingRequest( + requestId, + true, + currentState.finalGrade, + currentState.error_recovery, + ); + + return result; + } + + return this.executeFullGrading( + requestId, + context, + startTime, + currentState, + ); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Simple grading failed"; + + const result: GradingResult = { + requestId, + success: false, + errors: [errorMessage], + warnings, + processingTimeMs: Date.now() - startTime, + riskLevel: "high", + fallbackUsed: false, + }; + + this.monitoringService.completeGradingRequest(requestId, false); + return result; + } + } + + private async executeFullGrading( + requestId: string, + context: GradingContextData, + startTime: number, + initialStateOverride?: GradingGraphState, + ): Promise { + try { + if (!this.graph) { + throw new Error("Graph not initialized"); + } + + const state = initialStateOverride || initialState(context); + const compiledGraph = this.graph.compile(); + + const result = await Promise.race([ + compiledGraph.invoke(state), + new Promise((_, reject) => + setTimeout( + () => reject(new Error("Pipeline timeout")), + context.timeout || this.config.defaultTimeout, + ), + ), + ]); + + const warnings: string[] = []; + + if (result.errors.length > 0) { + warnings.push( + `${result.errors.length} errors occurred during processing`, + ); + } + + if (result.fallback_used) { + warnings.push("Fallback mechanisms were used"); + } + + const gradingResult: GradingResult = { + requestId, + success: result.finalGrade !== undefined, + finalGrade: result.finalGrade, + errors: result.errors, + warnings, + processingTimeMs: Date.now() - startTime, + riskLevel: this.assessOverallRisk(result), + fallbackUsed: result.fallback_used, + debugInfo: this.config.enableDebugMode + ? { + processingSteps: result.finalGrade?.processingSteps, + nodeExecutionTimes: result.processing_metrics.nodeExecutionTimes, + errorRecovery: result.error_recovery, + circuitBreakerStatus: result.node_circuit_breakers, + } + : undefined, + }; + + this.monitoringService.completeGradingRequest( + requestId, + gradingResult.success, + result.finalGrade, + result.error_recovery, + ); + + return gradingResult; + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Full grading pipeline failed"; + + const result: GradingResult = { + requestId, + success: false, + errors: [errorMessage], + warnings: [], + processingTimeMs: Date.now() - startTime, + riskLevel: "high", + fallbackUsed: false, + }; + + this.monitoringService.completeGradingRequest(requestId, false); + return result; + } + } + + private async executeNodeWithRecovery( + requestId: string, + nodeName: string, + nodeExecution: () => Promise, + ): Promise { + const startTime = Date.now(); + + try { + const result = await nodeExecution(); + const executionTime = Date.now() - startTime; + + this.monitoringService.recordNodeExecution( + requestId, + nodeName, + executionTime, + true, + ); + return result; + } catch (error) { + const executionTime = Date.now() - startTime; + this.monitoringService.recordNodeExecution( + requestId, + nodeName, + executionTime, + false, + ); + + this.logger.error( + `Node ${nodeName} failed for request ${requestId}:`, + error, + ); + throw error; + } + } + + private assessOverallRisk( + state: GradingGraphState, + ): "low" | "medium" | "high" { + if (state.errors.length > 5 || state.error_recovery.attempts > 5) { + return "high"; + } + + if (state.fallback_used || state.errors.length > 0) { + return "medium"; + } + + const confidence = state.finalGrade?.grade?.confidence || 0; + if (confidence < 0.7) { + return "medium"; + } + + return "low"; + } + + private async queueForBatch( + context: GradingContextData, + ): Promise { + return new Promise((resolve, reject) => { + this.requestQueue.push({ + context, + resolve, + reject, + timestamp: Date.now(), + }); + + if (this.requestQueue.length >= this.config.batchSize) { + void this.processBatch(); + } else if (!this.batchTimer) { + this.batchTimer = setTimeout( + () => void this.processBatch(), + this.config.batchTimeoutMs, + ); + } + }); + } + + private async processBatch(): Promise { + if (this.batchTimer) { + clearTimeout(this.batchTimer); + this.batchTimer = null; + } + + const batch = this.requestQueue.splice(0, this.config.batchSize); + if (batch.length === 0) return; + + this.logger.debug(`Processing batch of ${batch.length} requests`); + + const results = await Promise.allSettled( + batch.map(({ context }) => this.executeGradingPipeline(context)), + ); + + for (const [index, result] of results.entries()) { + const { resolve, reject } = batch[index]; + + if (result.status === "fulfilled") { + resolve(result.value); + } else { + reject(result.reason as Error); + } + } + } + + async processGradingBatch( + contexts: GradingContextData[], + ): Promise { + if (this.shutdownRequested) { + throw new Error("Service is shutting down"); + } + + this.logger.log(`Processing batch of ${contexts.length} grading requests`); + + const results = await Promise.allSettled( + contexts.map((context) => this.executeGradingPipeline(context)), + ); + + return results.map((result) => { + if (result.status === "fulfilled") { + return result.value; + } else { + const errorMessage = + result.reason instanceof Error + ? result.reason.message + : "Unknown batch error"; + return { + requestId: uuidv4(), + success: false, + errors: [`Batch processing failed: ${errorMessage}`], + warnings: [], + processingTimeMs: 0, + riskLevel: "high" as const, + fallbackUsed: false, + }; + } + }); + } + + getProcessingStats(results: GradingResult[]): { + successRate: number; + avgProcessingTimeMs: number; + totalErrors: number; + riskDistribution: Record; + fallbackRate: number; + } { + const successful = results.filter((r) => r.success); + const riskDistribution = { low: 0, medium: 0, high: 0 }; + + for (const r of results) riskDistribution[r.riskLevel]++; + + return { + successRate: successful.length / results.length, + avgProcessingTimeMs: + results.reduce((sum, r) => sum + r.processingTimeMs, 0) / + results.length, + totalErrors: results.reduce((sum, r) => sum + r.errors.length, 0), + riskDistribution, + fallbackRate: + results.filter((r) => r.fallbackUsed).length / results.length, + }; + } + + getSystemHealth(): SystemHealth { + return this.monitoringService.getSystemHealth(); + } + + getMetrics(timeWindowMs?: number) { + return this.monitoringService.getAggregatedMetrics(timeWindowMs); + } + + private initializeGraph(): void { + try { + this.graph = new GradingGraph( + (state) => this.enhancedGradeNode.execute(state), + (state) => this.enhancedValidateNode.execute(state), + (state) => this.judgeNode.executeJudgeA(state), + (state) => this.evidenceNode.execute(state), + (state) => this.compareNode.execute(state), + (state) => this.decisionNode.execute(state), + ); + + this.logger.log("Grading graph initialized successfully"); + } catch (error) { + this.logger.error("Failed to initialize grading graph:", error); + throw new Error("Graph initialization failed"); + } + } + + async gracefulShutdown(): Promise { + this.logger.log("Starting graceful shutdown..."); + this.shutdownRequested = true; + + if (this.batchTimer) { + clearTimeout(this.batchTimer); + await this.processBatch(); + } + + await Promise.allSettled(this.activeRequests.values()); + + for (const { reject } of this.requestQueue) { + reject(new Error("Service shutdown")); + } + this.requestQueue.length = 0; + + this.logger.log("Graceful shutdown completed"); + } + + onModuleDestroy(): void { + this.gracefulShutdown().catch((error) => { + this.logger.error("Error during shutdown:", error); + }); + } +} diff --git a/apps/api/src/api/llm/grading/services/enhanced-policy.service.ts b/apps/api/src/api/llm/grading/services/enhanced-policy.service.ts new file mode 100644 index 00000000..bf26b2ca --- /dev/null +++ b/apps/api/src/api/llm/grading/services/enhanced-policy.service.ts @@ -0,0 +1,471 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { + createDynamicThresholds, + DEFAULT_POLICY_CONFIG, + getQuestionTypePolicy, + GradingPolicyConfig, +} from "../config/policy.config"; +import { + DecisionContext, + EvidenceVerificationData, + GradeData, + JudgeComparisonData, + PolicyDecision, + TiebreakResultData, +} from "../types/grading.types"; + +@Injectable() +export class EnhancedPolicyService { + private readonly logger = new Logger(EnhancedPolicyService.name); + private config: GradingPolicyConfig; + + constructor(config?: Partial) { + this.config = { + ...DEFAULT_POLICY_CONFIG, + ...config, + }; + } + + decide( + graderResult: GradeData, + judgeAResult?: GradeData, + judgeBResult?: GradeData, + comparison?: JudgeComparisonData, + tiebreakResult?: TiebreakResultData, + context?: DecisionContext, + ): PolicyDecision { + try { + this.validateInputs(graderResult); + + if (tiebreakResult) { + return this.decideTiebreak( + tiebreakResult, + graderResult, + judgeAResult, + context, + ); + } + + if (!judgeAResult) { + return this.decideEarlyExit(graderResult, context); + } + + if (!judgeBResult) { + return this.decideSingleJudge( + graderResult, + judgeAResult, + comparison, + context, + ); + } + + return this.decideMultipleJudges( + graderResult, + judgeAResult, + judgeBResult, + comparison, + context, + ); + } catch (error) { + this.logger.error("Policy decision failed:", error); + return this.getFailsafeDecision(graderResult); + } + } + + shouldEarlyExit( + graderResult: GradeData, + evidenceVerification?: EvidenceVerificationData, + questionType?: string, + context?: DecisionContext, + ): boolean { + if (!this.config.earlyExit.ACCEPT_ON_HIGH_CONFIDENCE) { + return false; + } + + try { + if (context?.fallbackUsed) { + this.logger.warn("Fallback grading used, skipping early exit"); + return false; + } + + if (context?.errorCount && context.errorCount > 0) { + return false; + } + + const questionPolicy = questionType + ? getQuestionTypePolicy(questionType) + : {}; + + if ( + questionType === "TRUE_FALSE" && + questionPolicy.TRUE_FALSE_AUTO_ACCEPT + ) { + return ( + graderResult.confidence >= 0.5 && this.isReasonableScore(graderResult) + ); + } + + if ( + ["SINGLE_CORRECT", "MULTIPLE_CORRECT"].includes(questionType || "") && + questionPolicy.MCQ_AUTO_ACCEPT + ) { + return ( + graderResult.confidence >= 0.6 && this.isReasonableScore(graderResult) + ); + } + + const highConfidence = + graderResult.confidence >= this.config.thresholds.CONFIDENCE_THRESHOLD; + const validEvidence = + !evidenceVerification || + evidenceVerification.ok || + this.hasMinimalEvidence(graderResult); + const reasonableScore = this.isReasonableScore(graderResult); + const noAnomalies = !this.detectScoringAnomalies(graderResult); + + return highConfidence && validEvidence && reasonableScore && noAnomalies; + } catch (error) { + this.logger.error("Early exit evaluation failed:", error); + return false; + } + } + + private validateInputs(graderResult: GradeData): void { + if (!graderResult) { + throw new Error("Grader result is required"); + } + + if ( + !graderResult.criteriaAwards || + !Array.isArray(graderResult.criteriaAwards) + ) { + throw new Error("Invalid grader result structure"); + } + + if ( + typeof graderResult.totalAwarded !== "number" || + graderResult.totalAwarded < 0 + ) { + throw new Error("Invalid total awarded score"); + } + + if ( + typeof graderResult.confidence !== "number" || + graderResult.confidence < 0 || + graderResult.confidence > 1 + ) { + throw new Error("Invalid confidence value"); + } + } + + private decideTiebreak( + tiebreakResult: TiebreakResultData, + graderResult: GradeData, + judgeAResult?: GradeData, + context?: DecisionContext, + ): PolicyDecision { + if (tiebreakResult.method === "third_judge" && tiebreakResult.result) { + const confidence = Math.min(0.9, tiebreakResult.confidence || 0.7); + return { + selectedSource: "tiebreak", + reasoning: `Third judge resolution with ${confidence.toFixed( + 2, + )} confidence`, + confidence, + riskLevel: this.assessRiskLevel(confidence, context), + fallbackUsed: context?.fallbackUsed || false, + }; + } + + if (tiebreakResult.metaDecision === "accept_grader") { + return { + selectedSource: "grader", + reasoning: "Meta-decider recommends original grader result", + confidence: Math.min(0.8, graderResult.confidence), + riskLevel: this.assessRiskLevel(graderResult.confidence, context), + fallbackUsed: context?.fallbackUsed || false, + }; + } + + if (tiebreakResult.metaDecision === "accept_judges") { + const judgeConfidence = judgeAResult ? judgeAResult.confidence : 0.7; + return { + selectedSource: "judges", + reasoning: "Meta-decider recommends judge consensus", + confidence: Math.min(0.8, judgeConfidence), + riskLevel: this.assessRiskLevel(judgeConfidence, context), + fallbackUsed: context?.fallbackUsed || false, + }; + } + + return { + selectedSource: "grader", + reasoning: + "Tiebreak failed, defaulting to grader with reduced confidence", + confidence: Math.min(0.5, graderResult.confidence), + riskLevel: "high", + fallbackUsed: context?.fallbackUsed || false, + }; + } + + private decideEarlyExit( + graderResult: GradeData, + context?: DecisionContext, + ): PolicyDecision { + const confidence = context?.fallbackUsed + ? Math.min(0.6, graderResult.confidence) + : graderResult.confidence; + + return { + selectedSource: "grader", + reasoning: "Early exit: High confidence grader with valid evidence", + confidence, + riskLevel: this.assessRiskLevel(confidence, context), + fallbackUsed: context?.fallbackUsed || false, + }; + } + + private decideSingleJudge( + graderResult: GradeData, + judgeAResult: GradeData, + comparison?: JudgeComparisonData, + context?: DecisionContext, + ): PolicyDecision { + const thresholds = this.getThresholds(context?.totalMax); + + const withinThreshold = + !comparison || + comparison.graderVsJudgeA.totalDelta <= thresholds.TAU_TOTAL; + const goodAgreement = + !comparison || + comparison.graderVsJudgeA.agreementPct >= + this.config.thresholds.AGREEMENT_THRESHOLD; + + const graderReliable = !this.detectScoringAnomalies(graderResult); + const judgeReliable = !this.detectScoringAnomalies(judgeAResult); + + if (withinThreshold && goodAgreement && graderReliable) { + return { + selectedSource: "grader", + reasoning: "Grader and Judge A agree within thresholds", + confidence: Math.min( + graderResult.confidence, + judgeAResult.confidence + 0.1, + ), + riskLevel: "low", + fallbackUsed: context?.fallbackUsed || false, + }; + } + + if ( + judgeReliable && + (!graderReliable || judgeAResult.confidence > graderResult.confidence) + ) { + return { + selectedSource: "judges", + reasoning: "Judge A shows higher reliability or confidence than grader", + confidence: judgeAResult.confidence, + riskLevel: this.assessRiskLevel(judgeAResult.confidence, context), + fallbackUsed: context?.fallbackUsed || false, + }; + } + + return { + selectedSource: "grader", + reasoning: "Preferring grader despite disagreement with Judge A", + confidence: Math.min(0.7, graderResult.confidence), + riskLevel: "medium", + fallbackUsed: context?.fallbackUsed || false, + }; + } + + private decideMultipleJudges( + graderResult: GradeData, + judgeAResult: GradeData, + judgeBResult: GradeData, + comparison?: JudgeComparisonData, + context?: DecisionContext, + ): PolicyDecision { + const thresholds = this.getThresholds(context?.totalMax); + + if (!comparison) { + return { + selectedSource: "judges", + reasoning: "No comparison data, averaging judge results", + confidence: Math.min(judgeAResult.confidence, judgeBResult.confidence), + riskLevel: "medium", + fallbackUsed: context?.fallbackUsed || false, + }; + } + + const graderJudgeADelta = comparison.graderVsJudgeA.totalDelta; + const judgeABDelta = comparison.judgeAVsJudgeB?.totalDelta ?? 0; + + const allAgree = + graderJudgeADelta <= thresholds.TAU_TOTAL && + judgeABDelta <= thresholds.TAU_TOTAL; + + if (allAgree) { + const avgConfidence = + (graderResult.confidence + + judgeAResult.confidence + + judgeBResult.confidence) / + 3; + return { + selectedSource: "grader", + reasoning: + "All sources agree within thresholds, keeping original grader result", + confidence: avgConfidence, + riskLevel: "low", + fallbackUsed: context?.fallbackUsed || false, + }; + } + + const judgesAgree = judgeABDelta <= thresholds.TAU_TOTAL; + if (judgesAgree) { + const judgeConfidence = + (judgeAResult.confidence + judgeBResult.confidence) / 2; + return { + selectedSource: "judges", + reasoning: "Judges agree with each other but disagree with grader", + confidence: judgeConfidence, + riskLevel: this.assessRiskLevel(judgeConfidence, context), + fallbackUsed: context?.fallbackUsed || false, + }; + } + + const graderMostReliable = + !this.detectScoringAnomalies(graderResult) && + graderResult.confidence >= + Math.max(judgeAResult.confidence, judgeBResult.confidence); + + if (graderMostReliable) { + return { + selectedSource: "grader", + reasoning: + "Grader shows highest reliability among conflicting assessments", + confidence: Math.min(0.7, graderResult.confidence), + riskLevel: "medium", + fallbackUsed: context?.fallbackUsed || false, + }; + } + + return { + selectedSource: "judges", + reasoning: + "Complex disagreement, averaging judge results as safest option", + confidence: Math.min( + 0.6, + (judgeAResult.confidence + judgeBResult.confidence) / 2, + ), + riskLevel: "high", + fallbackUsed: context?.fallbackUsed || false, + }; + } + + private getFailsafeDecision(graderResult: GradeData): PolicyDecision { + this.logger.warn("Using failsafe decision due to policy evaluation error"); + + return { + selectedSource: "grader", + reasoning: "Failsafe decision due to policy evaluation error", + confidence: Math.min(0.4, graderResult?.confidence || 0.4), + riskLevel: "high", + fallbackUsed: true, + }; + } + + private isReasonableScore(result: GradeData): boolean { + if (result.totalAwarded > result.totalMax * 1.1) return false; + if (result.totalAwarded < 0) return false; + + for (const award of result.criteriaAwards) { + if (award.awarded > award.maxPoints * 1.1) return false; + if (award.awarded < 0) return false; + } + + return true; + } + + private detectScoringAnomalies(result: GradeData): boolean { + try { + const variance = this.calculateScoreVariance(result); + if (variance > result.totalMax * 0.5) return true; + + const extremeScores = result.criteriaAwards.filter( + (award: { awarded: number; maxPoints: number }) => + award.awarded === 0 || award.awarded === award.maxPoints, + ).length; + + const extremeRatio = extremeScores / result.criteriaAwards.length; + if (extremeRatio > 0.8) return true; + + const hasImpossibleScores = result.criteriaAwards.some( + (award: { awarded: number; maxPoints: number }) => + award.awarded > award.maxPoints || award.awarded < 0, + ); + + return hasImpossibleScores; + } catch (error) { + this.logger.warn("Anomaly detection failed:", error); + return false; + } + } + + private calculateScoreVariance(result: GradeData): number { + const ratios = result.criteriaAwards.map((award) => + award.maxPoints > 0 ? award.awarded / award.maxPoints : 0, + ); + + const mean = ratios.reduce((sum, ratio) => sum + ratio, 0) / ratios.length; + const variance = + ratios.reduce((sum, ratio) => sum + Math.pow(ratio - mean, 2), 0) / + ratios.length; + + return Math.sqrt(variance); + } + + private hasMinimalEvidence(result: GradeData): boolean { + const withEvidence = result.criteriaAwards.filter( + (award) => + award.awarded > 0 && award.evidence && award.evidence.length > 10, + ).length; + + const scoredCriteria = result.criteriaAwards.filter( + (award) => award.awarded > 0, + ).length; + + return scoredCriteria === 0 || withEvidence / scoredCriteria >= 0.3; + } + + private assessRiskLevel( + confidence: number, + context?: DecisionContext, + ): "low" | "medium" | "high" { + if ( + context?.fallbackUsed || + (context?.errorCount && context.errorCount > 2) + ) { + return "high"; + } + + if (confidence >= 0.8) return "low"; + if (confidence >= 0.6) return "medium"; + return "high"; + } + + private getThresholds(totalMax?: number) { + return totalMax + ? createDynamicThresholds(totalMax) + : this.config.thresholds; + } + + updateConfig(newConfig: Partial): void { + this.config = { ...this.config, ...newConfig }; + this.logger.log("Policy configuration updated"); + } + + getConfig(): GradingPolicyConfig { + return { ...this.config }; + } +} diff --git a/apps/api/src/api/llm/grading/services/evidence.service.ts b/apps/api/src/api/llm/grading/services/evidence.service.ts new file mode 100644 index 00000000..3646fbda --- /dev/null +++ b/apps/api/src/api/llm/grading/services/evidence.service.ts @@ -0,0 +1,439 @@ +/* eslint-disable unicorn/no-useless-undefined */ +/* eslint-disable @typescript-eslint/require-await */ +import { Injectable, Logger } from "@nestjs/common"; +import MiniSearch from "minisearch"; +import { + CircuitBreakerData, + EvidenceMatch, + EvidenceVerificationData, + GradeData, + ValidatedGradeData, +} from "../types/grading.types"; + +interface EvidenceServiceConfig { + fuzzyThreshold: number; + minQuoteLength: number; + maxAnswerLength: number; + searchTimeout: number; + enableFallbacks: boolean; +} + +interface SearchDocument { + id: number; + text: string; + original: string; + position: number; + length: number; +} + +interface SearchResult { + id: number; + score: number; + match: Record; + terms: string[]; + queryTerms: string[]; +} + +type MiniSearchInstance = { + addAll: (documents: SearchDocument[]) => void; + search: (query: string, options?: Record) => SearchResult[]; +}; + +@Injectable() +export class EvidenceService { + private readonly logger = new Logger(EvidenceService.name); + private readonly config: EvidenceServiceConfig; + private circuitBreaker: CircuitBreakerData = { + failures: 0, + isOpen: false, + resetTimeout: 60_000, + }; + + constructor(config?: Partial) { + this.config = { + fuzzyThreshold: 0.7, + minQuoteLength: 10, + maxAnswerLength: 50_000, + searchTimeout: 30_000, + enableFallbacks: true, + ...config, + }; + } + + async verifyEvidence( + answer: string, + grade: GradeData, + ): Promise { + const startTime = Date.now(); + + try { + if (this.isCircuitBreakerOpen()) { + this.logger.warn("Circuit breaker open, using fallback verification"); + return this.fallbackVerification(answer, grade); + } + + if (answer.length > this.config.maxAnswerLength) { + this.logger.warn( + `Answer too long (${answer.length} chars), truncating`, + ); + answer = answer.slice(0, Math.max(0, this.config.maxAnswerLength)); + } + + const invalidCriteriaIds: string[] = []; + const details: Array<{ + criterionId: string; + issue: "missing_evidence" | "evidence_not_found" | "fuzzy_match_failed"; + evidence?: string; + }> = []; + + const searchIndex = await this.createSearchIndexWithTimeout(answer); + if (!searchIndex) { + this.logger.error("Failed to create search index, using fallback"); + return this.fallbackVerification(answer, grade); + } + + for (const award of grade.criteriaAwards) { + if (award.awarded === 0) { + continue; + } + + if ( + !award.evidence || + award.evidence.trim().length < this.config.minQuoteLength + ) { + invalidCriteriaIds.push(award.criterionId); + details.push({ + criterionId: award.criterionId, + issue: "missing_evidence", + evidence: award.evidence, + }); + continue; + } + + try { + const match = await this.findEvidenceMatchWithFallbacks( + answer, + award.evidence, + searchIndex, + ); + if (!match) { + invalidCriteriaIds.push(award.criterionId); + details.push({ + criterionId: award.criterionId, + issue: "evidence_not_found", + evidence: award.evidence, + }); + } else if (match.similarity < this.config.fuzzyThreshold) { + invalidCriteriaIds.push(award.criterionId); + details.push({ + criterionId: award.criterionId, + issue: "fuzzy_match_failed", + evidence: award.evidence, + }); + } + } catch (error) { + this.logger.error( + `Evidence matching failed for ${award.criterionId}:`, + error, + ); + invalidCriteriaIds.push(award.criterionId); + details.push({ + criterionId: award.criterionId, + issue: "evidence_not_found", + evidence: award.evidence, + }); + } + } + + this.recordSuccess(); + const processingTime = Date.now() - startTime; + this.logger.debug( + `Evidence verification completed in ${processingTime}ms`, + ); + + return { + ok: invalidCriteriaIds.length === 0, + invalidCriteriaIds, + details, + }; + } catch (error) { + this.recordFailure(); + this.logger.error("Evidence verification failed:", error); + + if (this.config.enableFallbacks) { + return this.fallbackVerification(answer, grade); + } + + throw error; + } + } + + private async createSearchIndexWithTimeout( + text: string, + ): Promise { + return new Promise((resolve) => { + const timeout = setTimeout(() => { + this.logger.warn("Search index creation timed out"); + void resolve(undefined); + }, this.config.searchTimeout); + + try { + const sentences = this.splitIntoSentences(text); + const documents: SearchDocument[] = sentences.map( + (sentence, index) => ({ + id: index, + text: sentence.toLowerCase(), + original: sentence, + position: text.indexOf(sentence), + length: sentence.length, + }), + ); + + const miniSearch = new MiniSearch({ + fields: ["text"], + storeFields: ["original", "position", "length"], + tokenize: (string: string) => + string.split(/\s+/).filter((token) => token.length > 1), + processTerm: (term: string) => term.toLowerCase(), + }) as MiniSearchInstance; + + miniSearch.addAll(documents); + clearTimeout(timeout); + resolve(miniSearch); + } catch (error) { + clearTimeout(timeout); + this.logger.error("Error creating search index:", error); + resolve(undefined); + } + }); + } + + private async findEvidenceMatchWithFallbacks( + answer: string, + evidence: string, + searchIndex: MiniSearchInstance, + ): Promise { + const cleanEvidence = evidence.trim().toLowerCase(); + const cleanAnswer = answer.toLowerCase(); + + const exactMatch = cleanAnswer.indexOf(cleanEvidence); + if (exactMatch !== -1) { + return { + quote: evidence, + position: exactMatch, + similarity: 1, + method: "exact", + }; + } + + try { + const fuzzyMatch = this.performFuzzySearch(cleanEvidence, searchIndex); + if (fuzzyMatch && fuzzyMatch.similarity >= this.config.fuzzyThreshold) { + return fuzzyMatch; + } + } catch (error) { + this.logger.warn("Fuzzy search failed, trying keyword fallback:", error); + } + + if (this.config.enableFallbacks) { + return this.performKeywordMatch(cleanEvidence, cleanAnswer); + } + + return undefined; + } + + private performFuzzySearch( + evidence: string, + searchIndex: MiniSearchInstance, + ): EvidenceMatch | undefined { + const searchResults = searchIndex.search(evidence, { + fuzzy: 0.3, + prefix: true, + boost: { text: 2 }, + combineWith: "AND", + }); + + if (searchResults.length === 0) { + return undefined; + } + + const bestMatch = searchResults[0]; + const matchedDocument = { + original: String(bestMatch.match?.original?.[0] || ""), + position: 0, + }; + const similarity = this.calculateAdvancedSimilarity( + evidence, + matchedDocument.original.toLowerCase(), + ); + + return { + quote: matchedDocument.original, + position: matchedDocument.position, + similarity, + method: "fuzzy", + }; + } + + private performKeywordMatch( + evidence: string, + answer: string, + ): EvidenceMatch | undefined { + const evidenceWords = evidence + .split(/\s+/) + .filter((word) => word.length > 3); + const matchedWords = evidenceWords.filter((word) => answer.includes(word)); + + if (matchedWords.length === 0) return undefined; + + const similarity = matchedWords.length / evidenceWords.length; + if (similarity < 0.5) return undefined; + + const firstMatch = answer.indexOf(matchedWords[0]); + return { + quote: matchedWords.join(" "), + position: firstMatch, + similarity, + method: "keyword", + }; + } + + private calculateAdvancedSimilarity(text1: string, text2: string): number { + const words1 = text1.split(/\s+/).filter((w) => w.length > 1); + const words2 = text2.split(/\s+/).filter((w) => w.length > 1); + + if (words1.length === 0 || words2.length === 0) return 0; + + const set1 = new Set(words1); + const set2 = new Set(words2); + + const intersection = new Set([...set1].filter((x) => set2.has(x))); + const union = new Set([...set1, ...set2]); + + const jaccardSimilarity = intersection.size / union.size; + + const lengthSimilarity = + Math.min(text1.length, text2.length) / + Math.max(text1.length, text2.length); + + return jaccardSimilarity * 0.7 + lengthSimilarity * 0.3; + } + + private splitIntoSentences(text: string): string[] { + return text + .split(/[!.?]+/) + .map((s) => s.trim()) + .filter((s) => s.length > 5); + } + + private fallbackVerification( + answer: string, + grade: GradeData, + ): EvidenceVerificationData { + this.logger.log("Using fallback evidence verification"); + + const invalidCriteriaIds: string[] = []; + const details: Array<{ + criterionId: string; + issue: "missing_evidence" | "evidence_not_found" | "fuzzy_match_failed"; + evidence?: string; + }> = []; + + for (const award of grade.criteriaAwards) { + if (award.awarded === 0) continue; + + if ( + !award.evidence || + award.evidence.trim().length < this.config.minQuoteLength + ) { + invalidCriteriaIds.push(award.criterionId); + details.push({ + criterionId: award.criterionId, + issue: "missing_evidence", + evidence: award.evidence, + }); + } else { + const simpleMatch = answer + .toLowerCase() + .includes(award.evidence.toLowerCase()); + if (!simpleMatch) { + invalidCriteriaIds.push(award.criterionId); + details.push({ + criterionId: award.criterionId, + issue: "evidence_not_found", + evidence: award.evidence, + }); + } + } + } + + return { + ok: invalidCriteriaIds.length === 0, + invalidCriteriaIds, + details, + }; + } + + private isCircuitBreakerOpen(): boolean { + if (!this.circuitBreaker.isOpen) return false; + + const now = Date.now(); + if ( + this.circuitBreaker.lastFailure && + now - this.circuitBreaker.lastFailure > this.circuitBreaker.resetTimeout + ) { + this.circuitBreaker.isOpen = false; + this.circuitBreaker.failures = 0; + this.logger.log("Circuit breaker reset"); + return false; + } + + return true; + } + + private recordSuccess(): void { + if (this.circuitBreaker.failures > 0) { + this.circuitBreaker.failures = Math.max( + 0, + this.circuitBreaker.failures - 1, + ); + } + } + + private recordFailure(): void { + this.circuitBreaker.failures++; + this.circuitBreaker.lastFailure = Date.now(); + + if (this.circuitBreaker.failures >= 3) { + this.circuitBreaker.isOpen = true; + this.logger.warn("Circuit breaker opened due to failures"); + } + } + + zeroOutInvalidCriteria( + grade: ValidatedGradeData, + invalidCriteriaIds: string[], + ): ValidatedGradeData { + const updatedCriteriaAwards = grade.criteriaAwards.map((award) => { + if (invalidCriteriaIds.includes(award.criterionId)) { + return { + ...award, + awarded: 0, + justification: `${award.justification} [Evidence verification failed]`, + }; + } + return award; + }); + + const newTotalAwarded = updatedCriteriaAwards.reduce( + (sum, award) => sum + award.awarded, + 0, + ); + + return { + ...grade, + criteriaAwards: updatedCriteriaAwards, + totalAwarded: newTotalAwarded, + }; + } +} diff --git a/apps/api/src/api/llm/grading/services/meta-decider.service.ts b/apps/api/src/api/llm/grading/services/meta-decider.service.ts new file mode 100644 index 00000000..4ecb7612 --- /dev/null +++ b/apps/api/src/api/llm/grading/services/meta-decider.service.ts @@ -0,0 +1,72 @@ +import { Injectable } from "@nestjs/common"; + +export interface MetaDecisionFeatures { + deltaA: number; + deltaB: number; + agreementPct: number; + evidenceDensity: number; +} + +export type MetaDecision = "accept_grader" | "accept_judges" | "tiebreak"; + +@Injectable() +export class MetaDeciderService { + async decide(features: MetaDecisionFeatures): Promise { + const decisionScore = this.calculateDecisionScore(features); + + if (decisionScore < -0.3) { + return "accept_grader"; + } else if (decisionScore > 0.3) { + return "accept_judges"; + } else { + return "tiebreak"; + } + } + + private calculateDecisionScore(features: MetaDecisionFeatures): number { + let score = 0; + + score += this.deltaAContribution(features.deltaA); + score += this.deltaBContribution(features.deltaB); + score += this.agreementContribution(features.agreementPct); + score += this.evidenceContribution(features.evidenceDensity); + + return Math.max(-1, Math.min(1, score)); + } + + private deltaAContribution(deltaA: number): number { + if (deltaA <= 1) return -0.4; + if (deltaA <= 2) return -0.1; + if (deltaA <= 3) return 0.2; + return 0.4; + } + + private deltaBContribution(deltaB: number): number { + if (deltaB === 0) return 0; + if (deltaB <= 1) return 0.3; + if (deltaB <= 2) return 0.1; + return -0.2; + } + + private agreementContribution(agreementPct: number): number { + if (agreementPct >= 0.8) return -0.3; + if (agreementPct >= 0.6) return -0.1; + if (agreementPct >= 0.4) return 0.1; + return 0.3; + } + + private evidenceContribution(evidenceDensity: number): number { + if (evidenceDensity >= 0.8) return -0.2; + if (evidenceDensity >= 0.5) return -0.1; + if (evidenceDensity >= 0.2) return 0.1; + return 0.2; + } + + async loadONNXModel(): Promise { + throw new Error("ONNX model loading not implemented yet"); + } + + async predictWithONNX(): Promise { + throw new Error("ONNX prediction not implemented yet"); + } +} diff --git a/apps/api/src/api/llm/grading/services/monitoring.service.ts b/apps/api/src/api/llm/grading/services/monitoring.service.ts new file mode 100644 index 00000000..5a7858d8 --- /dev/null +++ b/apps/api/src/api/llm/grading/services/monitoring.service.ts @@ -0,0 +1,475 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { + ErrorRecoveryData, + FinalGradeData, + GradingContextData, +} from "../types/grading.types"; + +interface GradingMetrics { + requestId: string; + timestamp: number; + context: Partial; + processingSteps: string[]; + nodeExecutionTimes: Record; + totalProcessingTime: number; + llmCalls: number; + llmTokensUsed: number; + cacheHits: number; + memoryUsage?: number; + errorCount: number; + circuitBreakerTriggered: boolean; + fallbackUsed: boolean; + finalGrade?: Partial; + errorRecovery: ErrorRecoveryData; +} + +export interface SystemHealth { + totalRequests: number; + successRate: number; + averageProcessingTime: number; + errorRate: number; + circuitBreakerStatus: Record; + fallbackUsageRate: number; + memoryUsage?: number; + lastHealthCheck: number; +} + +interface AlertThresholds { + maxErrorRate: number; + maxProcessingTime: number; + maxMemoryUsage?: number; + minSuccessRate: number; + maxConcurrentRequests: number; +} + +@Injectable() +export class MonitoringService { + private readonly logger = new Logger(MonitoringService.name); + private metrics: Map = new Map(); + private systemHealth: SystemHealth = { + totalRequests: 0, + successRate: 1, + averageProcessingTime: 0, + errorRate: 0, + circuitBreakerStatus: {}, + fallbackUsageRate: 0, + lastHealthCheck: Date.now(), + }; + private alertThresholds: AlertThresholds = { + maxErrorRate: 0.1, + maxProcessingTime: 120_000, + minSuccessRate: 0.9, + maxConcurrentRequests: 100, + }; + private concurrentRequests = 0; + private readonly maxMetricsRetention = 10_000; + private readonly healthCheckInterval = 30_000; + + constructor() { + this.startHealthCheckTimer(); + } + + startGradingRequest(requestId: string, context: GradingContextData): void { + this.concurrentRequests++; + + const metrics: GradingMetrics = { + requestId, + timestamp: Date.now(), + context: { + questionId: context.questionId, + questionType: context.questionType, + responseType: context.responseType, + }, + processingSteps: [], + nodeExecutionTimes: {}, + totalProcessingTime: 0, + llmCalls: 0, + llmTokensUsed: 0, + cacheHits: 0, + errorCount: 0, + circuitBreakerTriggered: false, + fallbackUsed: false, + errorRecovery: { + attempts: 0, + recoveryStrategy: "retry", + fallbackUsed: false, + }, + }; + + this.metrics.set(requestId, metrics); + this.systemHealth.totalRequests++; + + this.logger.debug(`Started monitoring request ${requestId}`); + + if (this.concurrentRequests > this.alertThresholds.maxConcurrentRequests) { + this.triggerAlert( + "HIGH_CONCURRENCY", + `Concurrent requests: ${this.concurrentRequests}`, + ); + } + } + + recordNodeExecution( + requestId: string, + nodeName: string, + executionTime: number, + success: boolean, + llmTokens?: number, + ): void { + const metrics = this.metrics.get(requestId); + if (!metrics) { + this.logger.warn(`No metrics found for request ${requestId}`); + return; + } + + metrics.processingSteps.push(nodeName); + metrics.nodeExecutionTimes[nodeName] = executionTime; + + if (llmTokens) { + metrics.llmCalls++; + metrics.llmTokensUsed += llmTokens; + } + + if (!success) { + metrics.errorCount++; + } + + this.logger.debug( + `Node ${nodeName} executed for ${requestId} in ${executionTime}ms, success: ${success.toString()}`, + ); + } + + recordCircuitBreakerTriggered(requestId: string, nodeName: string): void { + const metrics = this.metrics.get(requestId); + if (metrics) { + metrics.circuitBreakerTriggered = true; + } + + this.systemHealth.circuitBreakerStatus[nodeName] = true; + + this.logger.warn( + `Circuit breaker triggered for ${nodeName} in request ${requestId}`, + ); + this.triggerAlert( + "CIRCUIT_BREAKER", + `Circuit breaker opened for ${nodeName}`, + ); + } + + recordFallbackUsed(requestId: string, fallbackType: string): void { + const metrics = this.metrics.get(requestId); + if (metrics) { + metrics.fallbackUsed = true; + metrics.errorRecovery.fallbackUsed = true; + } + + this.logger.log(`Fallback ${fallbackType} used for request ${requestId}`); + } + + recordCacheHit(requestId: string): void { + const metrics = this.metrics.get(requestId); + if (metrics) { + metrics.cacheHits++; + } + } + + recordMemoryUsage(requestId: string, memoryUsage: number): void { + const metrics = this.metrics.get(requestId); + if (metrics) { + metrics.memoryUsage = memoryUsage; + } + + if ( + this.alertThresholds.maxMemoryUsage && + memoryUsage > this.alertThresholds.maxMemoryUsage + ) { + this.triggerAlert("HIGH_MEMORY_USAGE", `Memory usage: ${memoryUsage} MB`); + } + } + + completeGradingRequest( + requestId: string, + success: boolean, + finalGrade?: FinalGradeData, + errorRecovery?: ErrorRecoveryData, + ): void { + const metrics = this.metrics.get(requestId); + if (!metrics) { + this.logger.warn(`No metrics found for completing request ${requestId}`); + return; + } + + this.concurrentRequests = Math.max(0, this.concurrentRequests - 1); + + const totalTime = Date.now() - metrics.timestamp; + metrics.totalProcessingTime = totalTime; + metrics.finalGrade = finalGrade + ? { + selectedSource: finalGrade.selectedSource, + reasoning: finalGrade.reasoning, + processingSteps: finalGrade.processingSteps, + } + : undefined; + + if (errorRecovery) { + metrics.errorRecovery = errorRecovery; + } + + this.updateSystemHealth(); + this.logCompletionMetrics(requestId, metrics, success); + + if (totalTime > this.alertThresholds.maxProcessingTime) { + this.triggerAlert( + "SLOW_PROCESSING", + `Request ${requestId} took ${totalTime}ms`, + ); + } + + if (metrics.errorCount > 5) { + this.triggerAlert( + "HIGH_ERROR_COUNT", + `Request ${requestId} had ${metrics.errorCount} errors`, + ); + } + + this.cleanupOldMetrics(); + } + + getRequestMetrics(requestId: string): GradingMetrics | undefined { + return this.metrics.get(requestId); + } + + getSystemHealth(): SystemHealth { + return { ...this.systemHealth }; + } + + getAggregatedMetrics(timeWindowMs = 3_600_000): { + requestCount: number; + averageProcessingTime: number; + successRate: number; + nodePerformance: Record; + llmUsage: { + totalCalls: number; + totalTokens: number; + avgTokensPerCall: number; + }; + fallbackUsage: { count: number; rate: number }; + errorAnalysis: Record; + } { + const cutoff = Date.now() - timeWindowMs; + const recentMetrics = [...this.metrics.values()].filter( + (m) => m.timestamp > cutoff, + ); + + if (recentMetrics.length === 0) { + return { + requestCount: 0, + averageProcessingTime: 0, + successRate: 1, + nodePerformance: {}, + llmUsage: { totalCalls: 0, totalTokens: 0, avgTokensPerCall: 0 }, + fallbackUsage: { count: 0, rate: 0 }, + errorAnalysis: {}, + }; + } + + const requestCount = recentMetrics.length; + const successfulRequests = recentMetrics.filter((m) => m.finalGrade).length; + const successRate = + requestCount > 0 ? successfulRequests / requestCount : 1; + + const totalProcessingTime = recentMetrics.reduce( + (sum, m) => sum + m.totalProcessingTime, + 0, + ); + const averageProcessingTime = totalProcessingTime / requestCount; + + const nodePerformance: Record< + string, + { avgTime: number; successRate: number } + > = {}; + const nodeStats: Record< + string, + { totalTime: number; count: number; successes: number } + > = {}; + + for (const metrics of recentMetrics) { + for (const [node, time] of Object.entries(metrics.nodeExecutionTimes)) { + if (!nodeStats[node]) { + nodeStats[node] = { totalTime: 0, count: 0, successes: 0 }; + } + nodeStats[node].totalTime += time; + nodeStats[node].count++; + if (metrics.finalGrade) nodeStats[node].successes++; + } + } + + for (const [node, stats] of Object.entries(nodeStats)) { + nodePerformance[node] = { + avgTime: stats.totalTime / stats.count, + successRate: stats.successes / stats.count, + }; + } + + const totalLLMCalls = recentMetrics.reduce((sum, m) => sum + m.llmCalls, 0); + const totalTokens = recentMetrics.reduce( + (sum, m) => sum + m.llmTokensUsed, + 0, + ); + const avgTokensPerCall = + totalLLMCalls > 0 ? totalTokens / totalLLMCalls : 0; + + const fallbackCount = recentMetrics.filter((m) => m.fallbackUsed).length; + const fallbackRate = requestCount > 0 ? fallbackCount / requestCount : 0; + + const errorAnalysis: Record = {}; + for (const metrics of recentMetrics) { + if (metrics.errorCount > 0) { + for (const step of metrics.processingSteps) { + errorAnalysis[step] = (errorAnalysis[step] || 0) + 1; + } + } + } + + return { + requestCount, + averageProcessingTime, + successRate, + nodePerformance, + llmUsage: { totalCalls: totalLLMCalls, totalTokens, avgTokensPerCall }, + fallbackUsage: { count: fallbackCount, rate: fallbackRate }, + errorAnalysis, + }; + } + + private updateSystemHealth(): void { + const recentMetrics = [...this.metrics.values()] + .filter((m) => Date.now() - m.timestamp < 300_000) + .slice(-1000); + + if (recentMetrics.length > 0) { + const successCount = recentMetrics.filter((m) => m.finalGrade).length; + this.systemHealth.successRate = successCount / recentMetrics.length; + + const totalTime = recentMetrics.reduce( + (sum, m) => sum + m.totalProcessingTime, + 0, + ); + this.systemHealth.averageProcessingTime = + totalTime / recentMetrics.length; + + const errorCount = recentMetrics.reduce( + (sum, m) => sum + (m.errorCount > 0 ? 1 : 0), + 0, + ); + this.systemHealth.errorRate = errorCount / recentMetrics.length; + + const fallbackCount = recentMetrics.filter((m) => m.fallbackUsed).length; + this.systemHealth.fallbackUsageRate = + fallbackCount / recentMetrics.length; + } + + this.systemHealth.lastHealthCheck = Date.now(); + } + + private logCompletionMetrics( + requestId: string, + metrics: GradingMetrics, + success: boolean, + ): void { + const logData = { + requestId, + success, + totalTime: metrics.totalProcessingTime, + nodeExecutionTimes: metrics.nodeExecutionTimes, + llmCalls: metrics.llmCalls, + tokensUsed: metrics.llmTokensUsed, + errorCount: metrics.errorCount, + fallbackUsed: metrics.fallbackUsed, + circuitBreakerTriggered: metrics.circuitBreakerTriggered, + finalSource: metrics.finalGrade?.selectedSource, + }; + + if (success) { + this.logger.log(`Request ${requestId} completed successfully`, logData); + } else { + this.logger.error(`Request ${requestId} failed`, logData); + } + } + + private cleanupOldMetrics(): void { + if (this.metrics.size > this.maxMetricsRetention) { + const sortedMetrics = [...this.metrics.entries()].sort( + (a, b) => b[1].timestamp - a[1].timestamp, + ); + + const toDelete = sortedMetrics.slice(this.maxMetricsRetention); + for (const [requestId] of toDelete) { + this.metrics.delete(requestId); + } + + this.logger.debug(`Cleaned up ${toDelete.length} old metrics entries`); + } + } + + private startHealthCheckTimer(): void { + setInterval(() => { + this.performHealthCheck(); + }, this.healthCheckInterval); + } + + private performHealthCheck(): void { + const health = this.getSystemHealth(); + + if (health.successRate < this.alertThresholds.minSuccessRate) { + this.triggerAlert( + "LOW_SUCCESS_RATE", + `Success rate: ${health.successRate.toFixed(3)}`, + ); + } + + if (health.errorRate > this.alertThresholds.maxErrorRate) { + this.triggerAlert( + "HIGH_ERROR_RATE", + `Error rate: ${health.errorRate.toFixed(3)}`, + ); + } + + if (health.averageProcessingTime > this.alertThresholds.maxProcessingTime) { + this.triggerAlert( + "SLOW_AVERAGE_PROCESSING", + `Avg time: ${health.averageProcessingTime}ms`, + ); + } + + this.logger.debug("Health check completed", { + successRate: health.successRate, + errorRate: health.errorRate, + avgProcessingTime: health.averageProcessingTime, + concurrentRequests: this.concurrentRequests, + }); + } + + private triggerAlert(alertType: string, message: string): void { + this.logger.warn(`ALERT [${alertType}]: ${message}`); + } + + updateAlertThresholds(newThresholds: Partial): void { + this.alertThresholds = { ...this.alertThresholds, ...newThresholds }; + this.logger.log("Alert thresholds updated", this.alertThresholds); + } + + resetMetrics(): void { + this.metrics.clear(); + this.systemHealth = { + totalRequests: 0, + successRate: 1, + averageProcessingTime: 0, + errorRate: 0, + circuitBreakerStatus: {}, + fallbackUsageRate: 0, + lastHealthCheck: Date.now(), + }; + this.logger.log("Metrics reset"); + } +} diff --git a/apps/api/src/api/llm/grading/types/grading.types.ts b/apps/api/src/api/llm/grading/types/grading.types.ts new file mode 100644 index 00000000..fdac8c1d --- /dev/null +++ b/apps/api/src/api/llm/grading/types/grading.types.ts @@ -0,0 +1,184 @@ +export interface CriterionAwardData { + criterionId: string; + awarded: number; + maxPoints: number; + justification: string; + evidence?: string; +} + +export interface GradeData { + criteriaAwards: CriterionAwardData[]; + totalAwarded: number; + totalMax: number; + overallFeedback: string; + confidence: number; +} + +export interface ValidatedGradeData extends GradeData { + isValid: boolean; + validationErrors: string[]; + arithmeticFixed: boolean; +} + +export interface EvidenceVerificationData { + ok: boolean; + invalidCriteriaIds: string[]; + details: Array<{ + criterionId: string; + issue: "missing_evidence" | "evidence_not_found" | "fuzzy_match_failed"; + evidence?: string; + }>; +} + +export interface JudgeComparisonData { + graderVsJudgeA: { + totalDelta: number; + criterionDeltas: Array<{ + criterionId: string; + delta: number; + }>; + agreementPct: number; + }; + judgeAVsJudgeB?: { + totalDelta: number; + criterionDeltas: Array<{ + criterionId: string; + delta: number; + }>; + agreementPct: number; + }; +} + +export interface TiebreakResultData { + method: "third_judge" | "meta_decider"; + result?: GradeData; + metaDecision?: "accept_grader" | "accept_judges" | "tiebreak"; + confidence: number; +} + +export interface FinalGradeData { + selectedSource: "grader" | "judges" | "tiebreak"; + grade: GradeData; + reasoning: string; + processingSteps: Array< + | "grade" + | "validate" + | "judgeA" + | "judgeB" + | "evidence" + | "compare" + | "tiebreak" + | "decision" + >; + metadata: { + totalProcessingTimeMs: number; + llmCalls: number; + earlyExitReason?: string; + }; +} + +export interface RubricCriterion { + id: string; + description: string; + maxPoints: number; + keywords?: string[]; +} + +export interface GradingContextData { + questionId: string; + learnerAnswer: string; + rubric: RubricCriterion[]; + questionType: + | "TEXT" + | "UPLOAD" + | "URL" + | "TRUE_FALSE" + | "SINGLE_CORRECT" + | "MULTIPLE_CORRECT"; + responseType?: string; + timeout: number; + maxRetries: number; +} + +export interface ErrorRecoveryData { + attempts: number; + lastError?: string; + recoveryStrategy: "retry" | "fallback" | "skip" | "abort"; + fallbackUsed: boolean; +} + +export interface CircuitBreakerData { + failures: number; + lastFailure?: number; + isOpen: boolean; + resetTimeout: number; +} + +export interface ProcessingMetricsData { + nodeExecutionTimes: Record; + memoryUsage?: number; + llmTokensUsed: number; + cacheHits: number; +} + +export interface GraphStateData { + context: GradingContextData; + graderResult?: ValidatedGradeData; + judgeAResult?: GradeData; + judgeBResult?: GradeData; + evidenceVerification?: EvidenceVerificationData; + comparison?: JudgeComparisonData; + tiebreakResult?: TiebreakResultData; + finalGrade?: FinalGradeData; + errors: string[]; + currentStep: string; + shouldContinue: boolean; +} + +export interface LLMGradingRequest { + questionId: string; + learnerAnswer: string; + rubric: RubricCriterion[]; + questionType: string; + responseType?: string; + timeout?: number; +} + +export interface LLMJudgeRequest { + questionId: string; + learnerAnswer: string; + rubric: RubricCriterion[]; + specificCriteria?: string[]; +} + +export interface MetaDecisionFeatures { + deltaA: number; + deltaB: number; + agreementPct: number; + evidenceDensity: number; +} + +export type MetaDecision = "accept_grader" | "accept_judges" | "tiebreak"; + +export interface EvidenceMatch { + quote: string; + position: number; + similarity: number; + method: "exact" | "fuzzy" | "keyword"; +} + +export interface PolicyDecision { + selectedSource: "grader" | "judges" | "tiebreak"; + reasoning: string; + confidence: number; + riskLevel: "low" | "medium" | "high"; + fallbackUsed: boolean; +} + +export interface DecisionContext { + questionType?: string; + totalMax?: number; + processingTime?: number; + errorCount?: number; + fallbackUsed?: boolean; +} diff --git a/apps/api/src/api/llm/llm-facade.service.ts b/apps/api/src/api/llm/llm-facade.service.ts index 4259b254..1cc01017 100644 --- a/apps/api/src/api/llm/llm-facade.service.ts +++ b/apps/api/src/api/llm/llm-facade.service.ts @@ -1,36 +1,41 @@ -import { Injectable, Inject } from "@nestjs/common"; +import { Inject, Injectable } from "@nestjs/common"; import { QuestionType } from "@prisma/client"; - -import { - PROMPT_PROCESSOR, - MODERATION_SERVICE, - TEXT_GRADING_SERVICE, - FILE_GRADING_SERVICE, - IMAGE_GRADING_SERVICE, - URL_GRADING_SERVICE, - PRESENTATION_GRADING_SERVICE, - VIDEO_PRESENTATION_GRADING_SERVICE, - QUESTION_GENERATION_SERVICE, - RUBRIC_SERVICE, - TRANSLATION_SERVICE, -} from "./llm.constants"; - +import { WINSTON_MODULE_PROVIDER } from "nest-winston"; +import { Logger } from "winston"; +import { LearnerLiveRecordingFeedback } from "../assignment/attempt/dto/assignment-attempt/types"; import { QuestionsToGenerate } from "../assignment/dto/post.assignment.request.dto"; import { + Choice, QuestionDto, + RubricDto, ScoringDto, - Choice, VariantDto, - RubricDto, } from "../assignment/dto/update.questions.request.dto"; -import { LearnerLiveRecordingFeedback } from "../assignment/attempt/dto/assignment-attempt/types"; -import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { IModerationService } from "./core/interfaces/moderation.interface"; import { IPromptProcessor } from "./core/interfaces/prompt-processor.interface"; +import { IFileGradingService } from "./features/grading/interfaces/file-grading.interface"; +import { IImageGradingService } from "./features/grading/interfaces/image-grading.interface"; +import { IPresentationGradingService } from "./features/grading/interfaces/presentation-grading.interface"; import { ITextGradingService } from "./features/grading/interfaces/text-grading.interface"; +import { IUrlGradingService } from "./features/grading/interfaces/url-grading.interface"; +import { IVideoPresentationGradingService } from "./features/grading/interfaces/video-grading.interface"; import { IQuestionGenerationService } from "./features/question-generation/interfaces/question-generation.interface"; +import { AssignmentTypeEnum } from "./features/question-generation/services/question-generation.service"; import { IRubricService } from "./features/rubric/interfaces/rubric.interface"; import { ITranslationService } from "./features/translation/interfaces/translation.interface"; +import { + FILE_GRADING_SERVICE, + IMAGE_GRADING_SERVICE, + MODERATION_SERVICE, + PRESENTATION_GRADING_SERVICE, + PROMPT_PROCESSOR, + QUESTION_GENERATION_SERVICE, + RUBRIC_SERVICE, + TEXT_GRADING_SERVICE, + TRANSLATION_SERVICE, + URL_GRADING_SERVICE, + VIDEO_PRESENTATION_GRADING_SERVICE, +} from "./llm.constants"; import { FileUploadQuestionEvaluateModel } from "./model/file.based.question.evaluate.model"; import { FileBasedQuestionResponseModel } from "./model/file.based.question.response.model"; import { ImageBasedQuestionEvaluateModel } from "./model/image.based.evalutate.model"; @@ -43,13 +48,6 @@ import { UrlBasedQuestionEvaluateModel } from "./model/url.based.question.evalua import { UrlBasedQuestionResponseModel } from "./model/url.based.question.response.model"; import { VideoPresentationQuestionEvaluateModel } from "./model/video-presentation.question.evaluate.model"; import { VideoPresentationQuestionResponseModel } from "./model/video-presentation.question.response.model"; -import { Logger } from "winston"; -import { IFileGradingService } from "./features/grading/interfaces/file-grading.interface"; -import { IImageGradingService } from "./features/grading/interfaces/image-grading.interface"; -import { IPresentationGradingService } from "./features/grading/interfaces/presentation-grading.interface"; -import { IUrlGradingService } from "./features/grading/interfaces/url-grading.interface"; -import { IVideoPresentationGradingService } from "./features/grading/interfaces/video-grading.interface"; -import { AssignmentTypeEnum } from "./features/question-generation/services/question-generation.service"; /** * LLM Facade Service @@ -311,8 +309,18 @@ export class LlmFacadeService { * Detect the language of text */ - async getLanguageCode(text: string): Promise { - return this.translationService.getLanguageCode(text); + async getLanguageCode(text: string, assignmentId?: number): Promise { + return this.translationService.getLanguageCode(text, assignmentId); + } + + /** + * Batch detect languages for multiple texts + */ + async batchGetLanguageCodes( + texts: string[], + assignmentId: number, + ): Promise { + return this.translationService.batchGetLanguageCodes(texts, assignmentId); } /** diff --git a/apps/api/src/api/llm/llm.constants.ts b/apps/api/src/api/llm/llm.constants.ts index d3bc5b57..b193f0c9 100644 --- a/apps/api/src/api/llm/llm.constants.ts +++ b/apps/api/src/api/llm/llm.constants.ts @@ -1,11 +1,21 @@ +// src/llm/llm.constants.ts + +// LLM Provider tokens export const OPENAI_LLM_PROVIDER_4o = "OPENAI_LLM_PROVIDER_4o"; export const OPENAI_LLM_PROVIDER_mini = "OPENAI_LLM_PROVIDER_mini"; +// export const LLAMA_LLM_PROVIDER = "LLAMA_LLM_PROVIDER"; +export const ALL_LLM_PROVIDERS = Symbol("ALL_LLM_PROVIDERS"); + +// Core service tokens export const PROMPT_PROCESSOR = "PROMPT_PROCESSOR"; export const MODERATION_SERVICE = "MODERATION_SERVICE"; export const TOKEN_COUNTER = "TOKEN_COUNTER"; export const USAGE_TRACKER = "USAGE_TRACKER"; -export const ALL_LLM_PROVIDERS = Symbol("ALL_LLM_PROVIDERS"); +export const LLM_PRICING_SERVICE = "LLM_PRICING_SERVICE"; +export const LLM_ASSIGNMENT_SERVICE = "LLM_ASSIGNMENT_SERVICE"; +export const LLM_RESOLVER_SERVICE = "LLM_RESOLVER_SERVICE"; +// Grading service tokens export const TEXT_GRADING_SERVICE = "TEXT_GRADING_SERVICE"; export const FILE_GRADING_SERVICE = "FILE_GRADING_SERVICE"; export const IMAGE_GRADING_SERVICE = "IMAGE_GRADING_SERVICE"; @@ -13,24 +23,35 @@ export const URL_GRADING_SERVICE = "URL_GRADING_SERVICE"; export const PRESENTATION_GRADING_SERVICE = "PRESENTATION_GRADING_SERVICE"; export const VIDEO_PRESENTATION_GRADING_SERVICE = "VIDEO_PRESENTATION_GRADING_SERVICE"; +export const GRADING_JUDGE_SERVICE = "GRADING_JUDGE_SERVICE"; + +// Feature service tokens export const QUESTION_GENERATION_SERVICE = "QUESTION_GENERATION_SERVICE"; export const RUBRIC_SERVICE = "RUBRIC_SERVICE"; export const TRANSLATION_SERVICE = "TRANSLATION_SERVICE"; export const VALIDATOR_SERVICE = "VALIDATOR_SERVICE"; +// Default model export const DEFAULT_LLM_MODEL = "gpt-4o"; +// Available LLM models +export const AVAILABLE_MODELS = { + OPENAI_GPT4O: "gpt-4o", + OPENAI_GPT4O_MINI: "gpt-4o-mini", + // LLAMA_4_MAVERICK: "llama-4-maverick", +} as const; + +// Response type specific instructions export const RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS = { CODE: ` **Feedback Structure:** Provide feedback in the following format: 1. **Accuracy**: Assess whether the response meets the task requirements and identify any discrepancies. - 2. **Functionality**: Evaluate whether the response works as expected and achieves the intended outcome. - 3. **Efficiency**: Discuss the approach taken and identify any areas for optimization. - 4. **Style**: Examine the clarity, readability, and presentation of the response, noting areas for improvement. - 5. **Practices**: Comment on adherence to best practices, including maintainability, modularity, and clarity. - 6. **Strengths**: Highlight notable features or aspects of the response that demonstrate understanding or innovation. + 2. **Efficiency**: Discuss the approach taken and identify any areas for optimization. + 3. **Style**: Examine the clarity, readability, and presentation of the response, noting areas for improvement. + 4. **Practices**: Comment on adherence to best practices, including maintainability, modularity, and clarity. + 5. **Strengths**: Highlight notable features or aspects of the response that demonstrate understanding or innovation. **Instructions for Feedback:** - Ensure feedback is constructive and actionable. @@ -91,6 +112,15 @@ export const RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS = { - **Engagement**: Determine how well the image captures attention and conveys its message. - **Professionalism**: Ensure adherence to professional standards in visual communication. `, + IMAGES: ` + Critique the image submission based on: + - **Content Relevance**: Ensure the image aligns with the assignment question and objectives. + - **Visual Clarity**: Assess the quality, focus, and overall presentation of the image. + - **Creativity**: Evaluate the originality and thoughtfulness of the image. + - **Technical Execution**: Comment on composition, lighting, and any editing techniques used. + - **Engagement**: Determine how well the image captures attention and conveys its message. + - **Professionalism**: Ensure adherence to professional standards in visual communication. + `, URL: ` Critique the URL submission based on: - **Content Relevance**: Ensure the linked content directly addresses the assignment question. @@ -125,4 +155,40 @@ export const RESPONSE_TYPE_SPECIFIC_INSTRUCTIONS = { - **Detail**: Determine if the rubric provides sufficient detail for effective evaluation. - **Professionalism**: Ensure adherence to professional standards in tone and presentation. `, + REPO: ` + Critique the repository submission based on: + - **Repository Structure**: Evaluate organization, file naming, and directory structure. + - **Code Quality**: Assess code readability, maintainability, and adherence to best practices. + - **Documentation**: Review README quality, code comments, and API documentation. + - **Testing**: Evaluate test coverage, quality, and CI/CD setup if applicable. + - **Version Control**: Comment on commit history, branching strategy, and collaboration practices. + - **Technical Excellence**: Highlight innovative solutions or exceptional implementation details. + `, + SPREADSHEET: ` + Critique the spreadsheet submission based on: + - **Data Organization**: Evaluate structure, naming conventions, and data relationships. + - **Formula Accuracy**: Assess correctness and efficiency of formulas and calculations. + - **Visual Presentation**: Review formatting, charts, and visual elements for clarity. + - **Documentation**: Comment on cell comments, legend, and explanatory notes. + - **Functionality**: Evaluate any macros, pivot tables, or advanced features used. + - **Professional Standards**: Ensure adherence to spreadsheet best practices. + `, + LIVE_RECORDING: ` + Critique the live recording submission based on: + - **Content Delivery**: Evaluate accuracy and completeness of presented information. + - **Presentation Skills**: Assess confidence, clarity, and engagement techniques. + - **Time Management**: Comment on pacing and adherence to time constraints. + - **Audience Interaction**: Review handling of questions or interactive elements. + - **Technical Execution**: Evaluate audio/video quality and any technical issues. + - **Overall Impact**: Determine effectiveness in conveying the intended message. + `, + OTHER: ` + Provide comprehensive feedback based on: + - **Content Quality**: Evaluate relevance, accuracy, and completeness. + - **Organization**: Assess structure and logical flow of information. + - **Technical Execution**: Comment on the quality of implementation or presentation. + - **Creativity and Innovation**: Highlight unique approaches or solutions. + - **Adherence to Requirements**: Ensure all assignment criteria are met. + - **Areas for Improvement**: Provide specific, actionable suggestions. + `, }; diff --git a/apps/api/src/api/llm/llm.module.ts b/apps/api/src/api/llm/llm.module.ts index 665ce57d..dfddef1c 100644 --- a/apps/api/src/api/llm/llm.module.ts +++ b/apps/api/src/api/llm/llm.module.ts @@ -1,14 +1,23 @@ import { Global, Module } from "@nestjs/common"; import { PrismaService } from "src/prisma.service"; import { S3Service } from "../files/services/s3.service"; +import { Gpt5LlmService } from "./core/services/gpt5-llm.service"; +import { Gpt5MiniLlmService } from "./core/services/gpt5-mini-llm.service"; +import { Gpt5NanoLlmService } from "./core/services/gpt5-nano-llm.service"; +import { LLMAssignmentService } from "./core/services/llm-assignment.service"; +import { LLMPricingService } from "./core/services/llm-pricing.service"; +import { LLMResolverService } from "./core/services/llm-resolver.service"; import { LlmRouter } from "./core/services/llm-router.service"; import { ModerationService } from "./core/services/moderation.service"; import { OpenAiLlmMiniService } from "./core/services/openai-llm-mini.service"; +import { Gpt4VisionPreviewLlmService } from "./core/services/openai-llm-vision.service"; import { OpenAiLlmService } from "./core/services/openai-llm.service"; +// import { LlamaLlmService } from "./core/services/llama-llm.service"; import { PromptProcessorService } from "./core/services/prompt-processor.service"; import { TokenCounterService } from "./core/services/token-counter.service"; import { UsageTrackerService } from "./core/services/usage-tracking.service"; import { FileGradingService } from "./features/grading/services/file-grading.service"; +import { GradingJudgeService } from "./features/grading/services/grading-judge.service"; import { ImageGradingService } from "./features/grading/services/image-grading.service"; import { PresentationGradingService } from "./features/grading/services/presentation-grading.service"; import { TextGradingService } from "./features/grading/services/text-grading.service"; @@ -22,7 +31,11 @@ import { LlmFacadeService } from "./llm-facade.service"; import { ALL_LLM_PROVIDERS, FILE_GRADING_SERVICE, + GRADING_JUDGE_SERVICE, IMAGE_GRADING_SERVICE, + LLM_ASSIGNMENT_SERVICE, + LLM_PRICING_SERVICE, + LLM_RESOLVER_SERVICE, MODERATION_SERVICE, PRESENTATION_GRADING_SERVICE, PROMPT_PROCESSOR, @@ -36,7 +49,6 @@ import { VALIDATOR_SERVICE, VIDEO_PRESENTATION_GRADING_SERVICE, } from "./llm.constants"; -import { Gpt4VisionPreviewLlmService } from "./core/services/openai-llm-vision.service"; @Global() @Module({ @@ -46,6 +58,10 @@ import { Gpt4VisionPreviewLlmService } from "./core/services/openai-llm-vision.s OpenAiLlmService, OpenAiLlmMiniService, Gpt4VisionPreviewLlmService, + Gpt5LlmService, + Gpt5MiniLlmService, + Gpt5NanoLlmService, + // LlamaLlmService, LlmRouter, { provide: ALL_LLM_PROVIDERS, @@ -53,17 +69,28 @@ import { Gpt4VisionPreviewLlmService } from "./core/services/openai-llm-vision.s p1: OpenAiLlmService, p2: OpenAiLlmMiniService, p3: Gpt4VisionPreviewLlmService, + p4: Gpt5LlmService, + p5: Gpt5MiniLlmService, + p6: Gpt5NanoLlmService, + // p7: LlamaLlmService, ) => { - return [p1, p2, p3]; + return [p1, p2, p3, p4, p5, p6]; }, inject: [ OpenAiLlmService, OpenAiLlmMiniService, Gpt4VisionPreviewLlmService, + Gpt5LlmService, + Gpt5MiniLlmService, + Gpt5NanoLlmService, + // LlamaLlmService, ], }, S3Service, - + { + provide: GRADING_JUDGE_SERVICE, + useClass: GradingJudgeService, + }, { provide: VALIDATOR_SERVICE, useClass: QuestionValidatorService, @@ -84,6 +111,18 @@ import { Gpt4VisionPreviewLlmService } from "./core/services/openai-llm-vision.s provide: USAGE_TRACKER, useClass: UsageTrackerService, }, + { + provide: LLM_PRICING_SERVICE, + useClass: LLMPricingService, + }, + { + provide: LLM_ASSIGNMENT_SERVICE, + useClass: LLMAssignmentService, + }, + { + provide: LLM_RESOLVER_SERVICE, + useClass: LLMResolverService, + }, { provide: TEXT_GRADING_SERVICE, @@ -133,6 +172,10 @@ import { Gpt4VisionPreviewLlmService } from "./core/services/openai-llm-vision.s MODERATION_SERVICE, TOKEN_COUNTER, USAGE_TRACKER, + GRADING_JUDGE_SERVICE, + LLM_PRICING_SERVICE, + LLM_ASSIGNMENT_SERVICE, + LLM_RESOLVER_SERVICE, TEXT_GRADING_SERVICE, FILE_GRADING_SERVICE, IMAGE_GRADING_SERVICE, diff --git a/apps/api/src/api/llm/model/file.based.question.response.model.ts b/apps/api/src/api/llm/model/file.based.question.response.model.ts index 03dab680..402c2f98 100644 --- a/apps/api/src/api/llm/model/file.based.question.response.model.ts +++ b/apps/api/src/api/llm/model/file.based.question.response.model.ts @@ -1,9 +1,36 @@ +// Define types to align with grading service output +export type RubricScore = { + rubricQuestion?: string; + pointsAwarded?: number; + maxPoints?: number; + justification?: string; + criterionSelected?: string; // if you want to keep it +}; + export class FileBasedQuestionResponseModel { readonly points: number; readonly feedback: string; + readonly analysis?: string; + readonly evaluation?: string; + readonly explanation?: string; + readonly guidance?: string; + readonly rubricScores?: RubricScore[]; - constructor(points: number, feedback: string) { + constructor( + points: number, + feedback: string, + analysis?: string, + evaluation?: string, + explanation?: string, + guidance?: string, + rubricScores?: RubricScore[], + ) { this.points = points; this.feedback = feedback; + this.analysis = analysis; + this.evaluation = evaluation; + this.explanation = explanation; + this.guidance = guidance; + this.rubricScores = rubricScores; } } diff --git a/apps/api/src/api/llm/model/text.based.question.response.model.ts b/apps/api/src/api/llm/model/text.based.question.response.model.ts index ad6afabc..655df4d5 100644 --- a/apps/api/src/api/llm/model/text.based.question.response.model.ts +++ b/apps/api/src/api/llm/model/text.based.question.response.model.ts @@ -1,9 +1,74 @@ +// Define types to align with grading service output + +import { RubricScore } from "./file.based.question.response.model"; + +export interface GradingMetadata { + judgeApproved: boolean; + attempts: number; + gradingTimeMs: number; + contentHash: string; +} + export class TextBasedQuestionResponseModel { - readonly points: number; - readonly feedback: string; + public readonly points: number; + public readonly feedback: string; + public readonly analysis?: string; + public readonly evaluation?: string; + public readonly explanation?: string; + public readonly guidance?: string; + public readonly rubricScores?: RubricScore[]; + public readonly gradingRationale?: string; + public readonly metadata?: GradingMetadata; - constructor(points: number, feedback: string) { + constructor( + points: number, + feedback: string, + analysis?: string, + evaluation?: string, + explanation?: string, + guidance?: string, + rubricScores?: RubricScore[], + gradingRationale?: string, + metadata?: GradingMetadata, + ) { this.points = points; this.feedback = feedback; + this.analysis = analysis; + this.evaluation = evaluation; + this.explanation = explanation; + this.guidance = guidance; + this.rubricScores = rubricScores; + this.gradingRationale = gradingRationale; + this.metadata = metadata; + } + + /** + * Create a simplified response with just the essential fields + */ + static createSimple( + points: number, + feedback: string, + explanation?: string, + guidance?: string, + metadata?: GradingMetadata, + ): TextBasedQuestionResponseModel { + return new TextBasedQuestionResponseModel( + points, + feedback, + undefined, // analysis + undefined, // evaluation + explanation, + guidance, + undefined, // rubricScores + undefined, // gradingRationale + metadata, + ); + } + + /** + * Get percentage score + */ + getPercentage(maxPoints: number): number { + return maxPoints > 0 ? Math.round((this.points / maxPoints) * 100) : 0; } } diff --git a/apps/api/src/api/report/controllers/report.controller.ts b/apps/api/src/api/report/controllers/report.controller.ts index 30b2c3c6..3eccc08a 100644 --- a/apps/api/src/api/report/controllers/report.controller.ts +++ b/apps/api/src/api/report/controllers/report.controller.ts @@ -1,24 +1,33 @@ +/* eslint-disable @typescript-eslint/require-await */ import { - Controller, - Post, + BadRequestException, Body, + Controller, + DefaultValuePipe, Get, + Injectable, Param, - Req, + ParseIntPipe, Patch, - BadRequestException, - Injectable, + Post, + Query, + Req, + UploadedFile, UseGuards, + UseInterceptors, } from "@nestjs/common"; +import { FileInterceptor } from "@nestjs/platform-express"; +import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger"; import { ReportStatus } from "@prisma/client"; -import { ReportsService } from "../services/report.service"; +import { memoryStorage } from "multer"; +import { AssignmentAccessControlGuard } from "src/api/assignment/guards/assignment.access.control.guard"; +import { AdminGuard } from "src/auth/guards/admin.guard"; import { UserRole, UserSessionRequest, } from "src/auth/interfaces/user.session.interface"; import { Roles } from "src/auth/role/roles.global.guard"; -import { ApiTags } from "@nestjs/swagger"; -import { AssignmentAccessControlGuard } from "src/api/assignment/guards/assignment.access.control.guard"; +import { ReportsService } from "../services/report.service"; @ApiTags("Reports") @Injectable() @@ -28,7 +37,94 @@ import { AssignmentAccessControlGuard } from "src/api/assignment/guards/assignme }) export class ReportsController { constructor(private readonly reportsService: ReportsService) {} + @Get("feedback") + @UseGuards(AdminGuard) + @ApiOperation({ + summary: "Get all assignment feedback with pagination and filtering", + }) + @ApiQuery({ name: "page", required: false, type: Number, example: 1 }) + @ApiQuery({ name: "limit", required: false, type: Number, example: 20 }) + @ApiQuery({ name: "search", required: false, type: String }) + @ApiQuery({ name: "assignmentId", required: false, type: Number }) + @ApiQuery({ name: "allowContact", required: false, type: Boolean }) + @ApiQuery({ name: "startDate", required: false, type: String }) + @ApiQuery({ name: "endDate", required: false, type: String }) + async getFeedback( + @Query("page", new DefaultValuePipe(1), ParseIntPipe) page: number, + @Query("limit", new DefaultValuePipe(20), ParseIntPipe) limit: number, + @Query("search") search?: string, + @Query("allowContact") allowContact?: string, + @Query("startDate") startDate?: string, + @Query("endDate") endDate?: string, + @Req() request?: UserSessionRequest, + ) { + // Convert admin session to user session format for the service + const userSession = request?.userSession + ? { + userId: request.userSession.userId, + role: request.userSession.role, + assignmentId: undefined, + groupId: undefined, + } + : request?.userSession; + + return this.reportsService.getFeedback({ + page, + limit, + search, + assignmentId: undefined, + allowContact: + allowContact === "true" + ? true + : allowContact === "false" + ? false + : undefined, + startDate, + endDate, + userSession, + }); + } + + @Get() + @UseGuards(AdminGuard) + @ApiOperation({ summary: "Get all reports with pagination and filtering" }) + @ApiQuery({ name: "page", required: false, type: Number, example: 1 }) + @ApiQuery({ name: "limit", required: false, type: Number, example: 20 }) + @ApiQuery({ name: "search", required: false, type: String }) + @ApiQuery({ name: "assignmentId", required: false, type: Number }) + @ApiQuery({ name: "status", required: false, type: String }) + @ApiQuery({ name: "issueType", required: false, type: String }) + @ApiQuery({ name: "startDate", required: false, type: String }) + @ApiQuery({ name: "endDate", required: false, type: String }) + async getReports( + @Query("page", new DefaultValuePipe(1), ParseIntPipe) page: number, + @Query("limit", new DefaultValuePipe(20), ParseIntPipe) limit: number, + @Query("search") search?: string, + @Query("status") status?: string, + @Query("issueType") issueType?: string, + @Query("startDate") startDate?: string, + @Query("endDate") endDate?: string, + ) { + return this.reportsService.getReports({ + page, + limit, + search, + assignmentId: undefined, + status: status, + issueType: issueType, + startDate, + endDate, + }); + } @Post() + @UseInterceptors( + FileInterceptor("screenshot", { + storage: memoryStorage(), + limits: { + fileSize: 10 * 1024 * 1024, // 10MB limit + }, + }), + ) async reportIssue( @Body() dto: { @@ -43,6 +139,7 @@ export class ReportsController { userRole?: string; additionalDetails?: Record; }, + @UploadedFile() screenshot: Express.Multer.File, @Req() request: UserSessionRequest, ): Promise<{ message: string; issueNumber?: number; reportId?: number }> { const reportDto = { @@ -58,8 +155,12 @@ export class ReportsController { userEmail: dto.userEmail, }, }; - - return this.reportsService.reportIssue(reportDto, request.userSession); + console.log("Reporting issue:", reportDto); + return this.reportsService.reportIssue( + reportDto, + request.userSession, + screenshot, + ); } @Get("assignment/:id") @@ -112,6 +213,29 @@ export class ReportsController { ); } + @Patch(":id/screenshot") + @UseGuards(AssignmentAccessControlGuard) + async addScreenshotToReport( + @Param("id") id: string, + @Body() + screenshotData: { + screenshotUrl: string; + bucket?: string; + }, + @Req() request: UserSessionRequest, + ) { + const userId = request.userSession?.userId; + if (!userId) { + throw new BadRequestException("User ID is required"); + } + return this.reportsService.addScreenshotToReport( + Number(id), + screenshotData.screenshotUrl, + userId, + screenshotData.bucket, + ); + } + @Post("feedback") @UseGuards(AssignmentAccessControlGuard) async sendUserFeedback( diff --git a/apps/api/src/api/report/report.module.ts b/apps/api/src/api/report/report.module.ts index 10f8849a..bdf4432d 100644 --- a/apps/api/src/api/report/report.module.ts +++ b/apps/api/src/api/report/report.module.ts @@ -1,14 +1,25 @@ +import { HttpModule } from "@nestjs/axios"; import { Module } from "@nestjs/common"; +import { ConfigModule } from "@nestjs/config"; +import { AdminAuthModule } from "src/auth/admin-auth.module"; +import { PrismaService } from "src/prisma.service"; +import { FilesService } from "../files/services/files.service"; +import { S3Service } from "../files/services/s3.service"; +import { NotificationsService } from "../user/services/notification.service"; import { ReportsController } from "./controllers/report.controller"; import { FloService } from "./services/flo.service"; import { ReportsService } from "./services/report.service"; -import { ConfigModule } from "@nestjs/config"; -import { HttpModule } from "@nestjs/axios"; -import { NotificationsService } from "../user/services/notification.service"; @Module({ - providers: [ReportsService, FloService, NotificationsService], + providers: [ + ReportsService, + FloService, + NotificationsService, + PrismaService, + FilesService, + S3Service, + ], controllers: [ReportsController], - imports: [ConfigModule, HttpModule], + imports: [ConfigModule, HttpModule, AdminAuthModule], }) export class ReportsModule {} diff --git a/apps/api/src/api/report/services/report.service.ts b/apps/api/src/api/report/services/report.service.ts index ff1dedc7..cb3e1efc 100644 --- a/apps/api/src/api/report/services/report.service.ts +++ b/apps/api/src/api/report/services/report.service.ts @@ -1,26 +1,339 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable unicorn/no-null */ import { Injectable, InternalServerErrorException, NotFoundException, } from "@nestjs/common"; -import { ReportStatus, ReportType } from "@prisma/client"; +import { Prisma, ReportStatus, ReportType } from "@prisma/client"; import axios from "axios"; +import * as jwt from "jsonwebtoken"; import * as natural from "natural"; +import { FilesService } from "src/api/files/services/files.service"; import { NotificationsService } from "src/api/user/services/notification.service"; -import { UserRole } from "src/auth/interfaces/user.session.interface"; +import { + UserRole, + UserSession, +} from "src/auth/interfaces/user.session.interface"; import { PrismaService } from "src/prisma.service"; import { ReportIssueDto } from "../types/report.types"; import { FloService } from "./flo.service"; +interface FeedbackFilterParameters { + page: number; + limit: number; + search?: string; + assignmentId?: number; + allowContact?: boolean; + startDate?: string; + endDate?: string; + userSession?: UserSession; +} + +interface ReportFilterParameters { + page: number; + limit: number; + search?: string; + assignmentId?: number; + status?: string; + issueType?: string; + startDate?: string; + endDate?: string; +} @Injectable() export class ReportsService { + private ghTokenCache: { value: string; expiresAt: number } | null = null; + private ghInstallationIdCache: number | null = null; + constructor( private readonly floService: FloService, private readonly prisma: PrismaService, + private readonly filesService: FilesService, private readonly notificationsService: NotificationsService, ) {} + private getPrivateKey(): string { + const raw = process.env.GITHUB_APP_PRIVATE_KEY || ""; + const processed = raw.includes("\\n") ? raw.replaceAll("\\n", "\n") : raw; + return processed; + } + + private buildAppJWT(): string { + const appId = process.env.GITHUB_APP_ID; + if (!appId) { + throw new InternalServerErrorException("GITHUB_APP_ID missing"); + } + const privateKey = this.getPrivateKey(); + const now = Math.floor(Date.now() / 1000); + + try { + const token = jwt.sign( + { + iat: now - 60, + exp: now + 9 * 60, + iss: appId, + }, + privateKey, + { algorithm: "RS256" }, + ); + return token; + } catch (error) { + console.error("Failed to create JWT:", error); + throw new InternalServerErrorException("Failed to create GitHub App JWT"); + } + } + + private async getInstallationId(): Promise { + if (this.ghInstallationIdCache) return this.ghInstallationIdCache; + + const explicit = process.env.GITHUB_APP_INSTALLATION_ID; + if (explicit) { + this.ghInstallationIdCache = Number(explicit); + return this.ghInstallationIdCache; + } + + const owner = process.env.GITHUB_OWNER; + const repo = process.env.GITHUB_REPO; + if (!owner || !repo) { + throw new InternalServerErrorException( + "GITHUB_OWNER or GITHUB_REPO missing", + ); + } + + try { + const appJwt = this.buildAppJWT(); + + const { data } = await axios.get( + `https://api.github.com/repos/${owner}/${repo}/installation`, + { + headers: { + Authorization: `Bearer ${appJwt}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + }, + ); + // data.id is the installation id + this.ghInstallationIdCache = Number(data.id); + return this.ghInstallationIdCache; + } catch (error) { + console.error("Failed to get installation ID:", error); + if (axios.isAxiosError(error)) { + console.error("Response status:", error.response?.status); + console.error("Response data:", error.response?.data); + } + throw new InternalServerErrorException( + "Failed to get GitHub App installation ID", + ); + } + } + private async getInstallationToken(): Promise { + const now = Math.floor(Date.now() / 1000); + + if (this.ghTokenCache && this.ghTokenCache.expiresAt - 60 > now) { + return this.ghTokenCache.value; + } + + try { + const installationId = await this.getInstallationId(); + const appJwt = this.buildAppJWT(); + + const { data } = await axios.post( + `https://api.github.com/app/installations/${installationId}/access_tokens`, + {}, + { + headers: { + Authorization: `Bearer ${appJwt}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + }, + ); + + const token = data.token as string; + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + const expiresAt = Math.floor(new Date(data.expires_at).getTime() / 1000); + this.ghTokenCache = { value: token, expiresAt }; + return token; + } catch (error) { + console.error("Failed to get installation token:", error); + if (axios.isAxiosError(error)) { + console.error("Response status:", error.response?.status); + console.error("Response data:", error.response?.data); + } + throw new InternalServerErrorException( + "Failed to get GitHub App installation token", + ); + } + } + + async getFeedback(parameters: FeedbackFilterParameters) { + const { + page, + limit, + search, + assignmentId, + allowContact, + startDate, + endDate, + userSession, + } = parameters; + const skip = (page - 1) * limit; + + const where: Prisma.AssignmentFeedbackWhereInput = {}; + + // If user is an author, only show feedback for their assignments + if (userSession && userSession.role === UserRole.AUTHOR) { + where.assignment = { + AssignmentAuthor: { + some: { + userId: userSession.userId, + }, + }, + }; + } + + if (assignmentId) { + where.assignmentId = assignmentId; + } + + if (allowContact !== undefined) { + where.allowContact = allowContact; + } + + if (search) { + where.OR = [ + { comments: { contains: search, mode: "insensitive" } }, + { firstName: { contains: search, mode: "insensitive" } }, + { lastName: { contains: search, mode: "insensitive" } }, + { email: { contains: search, mode: "insensitive" } }, + ]; + } + + if (startDate || endDate) { + where.createdAt = {}; + if (startDate) { + where.createdAt.gte = new Date(startDate); + } + if (endDate) { + where.createdAt.lte = new Date(endDate); + } + } + + const [data, total] = await Promise.all([ + this.prisma.assignmentFeedback.findMany({ + where, + skip, + take: limit, + include: { + assignment: { + select: { + id: true, + name: true, + }, + }, + assignmentAttempt: { + select: { + id: true, + grade: true, + submitted: true, + }, + }, + }, + orderBy: { + createdAt: "desc", + }, + }), + this.prisma.assignmentFeedback.count({ where }), + ]); + + return { + data, + pagination: { + total, + page, + limit, + totalPages: Math.ceil(total / limit), + }, + }; + } + + async getReports(parameters: ReportFilterParameters) { + const { + page, + limit, + search, + assignmentId, + status, + issueType, + startDate, + endDate, + } = parameters; + const skip = (page - 1) * limit; + + const where: Prisma.ReportWhereInput = {}; + + if (assignmentId) { + where.assignmentId = assignmentId; + } + + if (status) { + where.status = status as unknown; + } + + if (issueType) { + where.issueType = issueType as unknown; + } + + if (search) { + where.OR = [ + { description: { contains: search, mode: "insensitive" } }, + { statusMessage: { contains: search, mode: "insensitive" } }, + { resolution: { contains: search, mode: "insensitive" } }, + { comments: { contains: search, mode: "insensitive" } }, + ]; + } + + if (startDate || endDate) { + where.createdAt = {}; + if (startDate) { + where.createdAt.gte = new Date(startDate); + } + if (endDate) { + where.createdAt.lte = new Date(endDate); + } + } + + const [data, total] = await Promise.all([ + this.prisma.report.findMany({ + where, + skip, + take: limit, + include: { + assignment: { + select: { + id: true, + name: true, + }, + }, + }, + orderBy: { + createdAt: "desc", + }, + }), + this.prisma.report.count({ where }), + ]); + return { + data, + pagination: { + total, + page, + limit, + totalPages: Math.ceil(total / limit), + }, + }; + } private async createGithubIssue( title: string, body: string, @@ -28,41 +341,56 @@ export class ReportsService { ): Promise<{ number: number; [key: string]: any }> { const githubOwner = process.env.GITHUB_OWNER; const githubRepo = process.env.GITHUB_REPO; - const token = process.env.GITHUB_APP_TOKEN; + + const token = await this.getInstallationToken(); if (!githubOwner || !githubRepo || !token) { + const missingConfig = []; + if (!githubOwner) missingConfig.push("GITHUB_OWNER"); + if (!githubRepo) missingConfig.push("GITHUB_REPO"); + if (!token) missingConfig.push("installation token"); + console.error("Missing GitHub configuration:", missingConfig); throw new InternalServerErrorException( - "GitHub repository configuration or token missing", + `GitHub repository configuration or token missing: ${missingConfig.join( + ", ", + )}`, ); } try { - const response = await axios.post( - `https://api.github.com/repos/${githubOwner}/${githubRepo}/issues`, - { - title, - body, - labels, - }, - { - headers: { - Authorization: `token ${token}`, - Accept: "application/vnd.github.v3+json", - }, + const url = `https://api.github.com/repos/${githubOwner}/${githubRepo}/issues`; + const payload = { title, body, labels }; + const response = await axios.post(url, payload, { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", }, - ); + }); return response.data as { number: number; [key: string]: any }; } catch (error) { - const error_ = axios.isAxiosError(error) - ? new InternalServerErrorException( - `Failed to create GitHub issue: ${error.message}`, - ) - : new InternalServerErrorException( - `Failed to create GitHub issue: ${ - error instanceof Error ? error.message : "Unknown error" - }`, - ); - throw error_; + console.error("Failed to create GitHub issue:", error); + + if (axios.isAxiosError(error)) { + console.error("HTTP Status:", error.response?.status); + console.error("Response headers:", error.response?.headers); + console.error("Response data:", error.response?.data); + + const errorMessage: string = + error.response?.data?.message || error.message; + const status = error.response?.status; + + throw new InternalServerErrorException( + `Failed to create GitHub issue (${status}): ${errorMessage}`, + ); + } else { + const errorMessage = + error instanceof Error ? error.message : "Unknown error"; + console.error("Non-HTTP error:", errorMessage); + throw new InternalServerErrorException( + `Failed to create GitHub issue: ${errorMessage}`, + ); + } } } @@ -74,7 +402,7 @@ export class ReportsService { }> { const githubOwner = process.env.GITHUB_OWNER; const githubRepo = process.env.GITHUB_REPO; - const token = process.env.GITHUB_APP_TOKEN; + const token = await this.getInstallationToken(); if (!githubOwner || !githubRepo || !token) { throw new InternalServerErrorException( "GitHub repository configuration or token missing", @@ -86,8 +414,9 @@ export class ReportsService { `https://api.github.com/repos/${githubOwner}/${githubRepo}/issues/${issueNumber}`, { headers: { - Authorization: `token ${token}`, - Accept: "application/vnd.github.v3+json", + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", }, }, ); @@ -406,6 +735,7 @@ export class ReportsService { attemptId?: number; userId?: string; }, + screenshot?: Express.Multer.File, ): Promise<{ message: string; issueNumber?: number; @@ -419,7 +749,8 @@ export class ReportsService { }>; isDuplicate?: boolean; }> { - const { issueType, description, attemptId, severity } = dto; + const { issueType, description, attemptId, severity, additionalDetails } = + dto; const assignmentId = userSession?.assignmentId; if (!issueType) { @@ -456,6 +787,34 @@ export class ReportsService { }; } + let screenshotUrl: string | undefined; + if (screenshot && screenshot.buffer) { + try { + const debugBucket = process.env.IBM_COS_DEBUG_BUCKET; + if (debugBucket) { + const uniqueId = + Date.now().toString(36) + Math.random().toString(36).slice(2); + const screenshotKey = `issue-screenshots/${uniqueId}-${screenshot.originalname}`; + + await this.filesService.directUpload( + screenshot, + debugBucket, + screenshotKey, + ); + + screenshotUrl = screenshotKey; + console.log("Screenshot uploaded successfully:", screenshotUrl); + } else { + console.warn( + "IBM_COS_DEBUG_BUCKET not configured, skipping screenshot upload", + ); + } + } catch (uploadError) { + console.error("Failed to upload screenshot:", uploadError); + // Continue with report creation even if screenshot upload fails + } + } + const mappedIssueType = this.mapIssueTypeToReportType(issueType); const similarReports = await this.findSimilarReports( @@ -490,6 +849,39 @@ ${isProduction ? "PROD" : "DEV"}] [${role}] ${issueSeverity.toUpperCase()} ${ ${description} `; + // Include screenshot if provided (either from additionalDetails or uploaded file) + const finalScreenshotUrl = + screenshotUrl || additionalDetails?.screenshotUrl; + if (finalScreenshotUrl && typeof finalScreenshotUrl === "string") { + // Get environment variable for IBM COS debug bucket + const debugBucket = process.env.IBM_COS_DEBUG_BUCKET; + const cosEndpoint = process.env.IBM_COS_ENDPOINT; + + // If we uploaded the screenshot (screenshotUrl is set) or it's a full bucket path + if ( + cosEndpoint && + debugBucket && + (screenshotUrl || finalScreenshotUrl.includes(debugBucket)) + ) { + // Construct full IBM COS URL for the screenshot + const fullScreenshotUrl = screenshotUrl + ? `${cosEndpoint}/${debugBucket}/${screenshotUrl}` + : `${cosEndpoint}/${debugBucket}/${finalScreenshotUrl}`; + + issueBody += ` +### Screenshot +![Screenshot](${fullScreenshotUrl}) + +*Screenshot uploaded to IBM Cloud Object Storage: \`${finalScreenshotUrl}\`* +`; + } else { + issueBody += ` +### Screenshot +Screenshot Key: \`${finalScreenshotUrl}\` +`; + } + } + if (similarReports.length > 0) { issueBody += `\n\n### Similar Issues\n`; for (const report of similarReports.slice(0, 3)) { @@ -519,7 +911,7 @@ ${description} const githubOwner = process.env.GITHUB_OWNER; const githubRepo = process.env.GITHUB_REPO; - const token = process.env.GITHUB_APP_TOKEN; + const token = await this.getInstallationToken(); if (!githubOwner || !githubRepo || !token) { throw new InternalServerErrorException( @@ -527,7 +919,7 @@ ${description} ); } - const commentBody = ` + let commentBody = ` ## Duplicate Report Detected Another user has reported a nearly identical issue: @@ -542,13 +934,34 @@ Another user has reported a nearly identical issue: ${description} `; + // Include screenshot in duplicate report comment if provided + const screenshotUrl = additionalDetails?.screenshotUrl; + if (screenshotUrl && typeof screenshotUrl === "string") { + const debugBucket = process.env.IBM_COS_DEBUG_BUCKET; + if (debugBucket && screenshotUrl.includes(debugBucket)) { + const cosEndpoint = process.env.IBM_COS_ENDPOINT; + const fullScreenshotUrl = `${cosEndpoint}/${debugBucket}/${screenshotUrl}`; + + commentBody += ` +### Screenshot from duplicate report +![Screenshot](${fullScreenshotUrl}) +`; + } else { + commentBody += ` +### Screenshot from duplicate report +Screenshot Key: \`${screenshotUrl}\` +`; + } + } + await axios.post( `https://api.github.com/repos/${githubOwner}/${githubRepo}/issues/${parentIssueNumber}/comments`, { body: commentBody }, { headers: { - Authorization: `token ${token}`, - Accept: "application/vnd.github.v3+json", + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", }, }, ); @@ -557,8 +970,9 @@ ${description} `https://api.github.com/repos/${githubOwner}/${githubRepo}/issues/${parentIssueNumber}`, { headers: { - Authorization: `token ${token}`, - Accept: "application/vnd.github.v3+json", + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", }, }, ); @@ -570,14 +984,9 @@ ${description} closed_at?: string | null; }; - let parentClosureReason = "duplicate"; - if (issue.state === "closed") { - const { status, closureReason } = - await this.checkGitHubIssueStatus(parentIssueNumber); - if (closureReason) { - parentClosureReason = closureReason; - } + await this.checkGitHubIssueStatus(parentIssueNumber); + // Parent closure reason can be used if needed for future enhancements } } else if (highSimilarityReport?.issueNumber) { const labels = ["chat-report", "related-issue"]; @@ -595,7 +1004,7 @@ ${description} const githubOwner = process.env.GITHUB_OWNER; const githubRepo = process.env.GITHUB_REPO; - const token = process.env.GITHUB_APP_TOKEN; + const token = await this.getInstallationToken(); if (githubOwner && githubRepo && token) { const relationComment = ` @@ -612,8 +1021,9 @@ A new related issue has been created: #${issue.number} { body: relationComment }, { headers: { - Authorization: `token ${token}`, - Accept: "application/vnd.github.v3+json", + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", }, }, ); @@ -1244,7 +1654,7 @@ A new related issue has been created: #${issue.number} }> { const githubOwner = process.env.GITHUB_OWNER; const githubRepo = process.env.GITHUB_REPO; - const token = process.env.GITHUB_APP_TOKEN; + const token = await this.getInstallationToken(); if (!githubOwner || !githubRepo || !token) { throw new InternalServerErrorException( @@ -1257,8 +1667,9 @@ A new related issue has been created: #${issue.number} `https://api.github.com/repos/${githubOwner}/${githubRepo}/issues/${issueNumber}`, { headers: { - Authorization: `token ${token}`, - Accept: "application/vnd.github.v3+json", + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", }, }, ); @@ -1594,7 +2005,7 @@ A new related issue has been created: #${issue.number} try { const githubOwner = process.env.GITHUB_OWNER; const githubRepo = process.env.GITHUB_REPO; - const token = process.env.GITHUB_APP_TOKEN; + const token = await this.getInstallationToken(); if (githubOwner && githubRepo && token) { let updatedBody = report.description; @@ -1628,8 +2039,9 @@ A new related issue has been created: #${issue.number} }, { headers: { - Authorization: `token ${token}`, - Accept: "application/vnd.github.v3+json", + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", }, }, ); @@ -1811,7 +2223,7 @@ A new related issue has been created: #${issue.number} try { const githubOwner = process.env.GITHUB_OWNER; const githubRepo = process.env.GITHUB_REPO; - const token = process.env.GITHUB_APP_TOKEN; + const token = await this.getInstallationToken(); if (githubOwner && githubRepo && token) { await axios.post( @@ -1821,8 +2233,9 @@ A new related issue has been created: #${issue.number} }, { headers: { - Authorization: `token ${token}`, - Accept: "application/vnd.github.v3+json", + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", }, }, ); @@ -2074,4 +2487,81 @@ ${description} }; } } + + async addScreenshotToReport( + reportId: number, + screenshotUrl: string, + userId: string, + bucket?: string, + ) { + // Find the report and verify the user owns it + const report = await this.prisma.report.findUnique({ + where: { id: reportId }, + }); + + if (!report) { + throw new NotFoundException(`Report with ID ${reportId} not found`); + } + + if (report.reporterId !== userId) { + throw new NotFoundException(`Report with ID ${reportId} not found`); + } + + // Add screenshot URL to the GitHub issue if it exists + if (report.issueNumber) { + try { + const token = await this.getInstallationToken(); + const owner = process.env.GITHUB_OWNER; + const repo = process.env.GITHUB_REPO; + + // Construct the full COS URL for the screenshot + const debugBucket = bucket || process.env.IBM_COS_DEBUG_BUCKET; + const cosEndpoint = process.env.IBM_COS_ENDPOINT; + const fullScreenshotUrl = `${cosEndpoint}/${debugBucket}/${screenshotUrl}`; + + const commentBody = ` +### Screenshot Added + +![Screenshot](${fullScreenshotUrl}) + +*Screenshot uploaded to IBM Cloud Object Storage: \`${screenshotUrl}\`* +`; + + await axios.post( + `https://api.github.com/repos/${owner}/${repo}/issues/${report.issueNumber}/comments`, + { + body: commentBody, + }, + { + headers: { + Authorization: `token ${token}`, + Accept: "application/vnd.github.v3+json", + }, + }, + ); + + console.log(`Screenshot added to GitHub issue #${report.issueNumber}`); + } catch (error) { + console.error(`Error adding screenshot to GitHub issue:`, error); + } + } + + // Update the report's comments to include the screenshot URL + const currentComments = report.comments || ""; + const screenshotComment = `\n[Screenshot: ${screenshotUrl}]`; + + await this.prisma.report.update({ + where: { id: reportId }, + data: { + comments: currentComments + screenshotComment, + updatedAt: new Date(), + }, + }); + + return { + success: true, + message: "Screenshot added to report successfully", + screenshotUrl, + }; + } } diff --git a/apps/api/src/api/scheduled-tasks/scheduled-tasks.module.ts b/apps/api/src/api/scheduled-tasks/scheduled-tasks.module.ts new file mode 100644 index 00000000..b9b65846 --- /dev/null +++ b/apps/api/src/api/scheduled-tasks/scheduled-tasks.module.ts @@ -0,0 +1,13 @@ +import { Module } from "@nestjs/common"; +import { ScheduleModule } from "@nestjs/schedule"; +import { PrismaService } from "../../prisma.service"; +import { AdminService } from "../admin/admin.service"; +import { LlmModule } from "../llm/llm.module"; +import { ScheduledTasksService } from "./services/scheduled-tasks.service"; + +@Module({ + imports: [ScheduleModule.forRoot(), LlmModule], + providers: [ScheduledTasksService, PrismaService, AdminService], + exports: [ScheduledTasksService], +}) +export class ScheduledTasksModule {} diff --git a/apps/api/src/api/scheduled-tasks/services/scheduled-tasks.service.ts b/apps/api/src/api/scheduled-tasks/services/scheduled-tasks.service.ts new file mode 100644 index 00000000..e83cd5a1 --- /dev/null +++ b/apps/api/src/api/scheduled-tasks/services/scheduled-tasks.service.ts @@ -0,0 +1,412 @@ +import { + Inject, + Injectable, + Logger, + OnApplicationBootstrap, +} from "@nestjs/common"; +import { Cron, CronExpression } from "@nestjs/schedule"; +import { PrismaService } from "../../../prisma.service"; +import { AdminService } from "../../admin/admin.service"; +import { LLMPricingService } from "../../llm/core/services/llm-pricing.service"; +import { LLM_PRICING_SERVICE } from "../../llm/llm.constants"; + +@Injectable() +export class ScheduledTasksService implements OnApplicationBootstrap { + private readonly logger = new Logger(ScheduledTasksService.name); + + constructor( + private prismaService: PrismaService, + @Inject(LLM_PRICING_SERVICE) private llmPricingService: LLMPricingService, + private adminService: AdminService, + ) {} + + async onApplicationBootstrap() { + this.logger.log("Application started - running initial tasks"); + // Run initial tasks on startup + await Promise.all([this.migrateExistingAuthors(), this.updateLLMPricing()]); + } + + // @Cron(CronExpression.EVERY_DAY_AT_2AM) + // async republishTopAssignments() { + // this.logger.log('Starting scheduled task: Republish top 10 used assignments'); + + // try { + // // Find top 10 most attempted assignments + // const topAssignments = await this.prismaService.assignmentAttempt.groupBy({ + // by: ['assignmentId'], + // _count: { + // assignmentId: true, + // }, + // orderBy: { + // _count: { + // assignmentId: 'desc', + // }, + // }, + // take: 10, + // }); + + // this.logger.log(`Found ${topAssignments.length} top assignments to republish`); + + // // Update each assignment to trigger republishing/translation + // for (const assignment of topAssignments) { + // await this.prismaService.assignment.update({ + // where: { id: assignment.assignmentId }, + // data: { + // updatedAt: new Date(), + // published: true, // Ensure it's published + // }, + // }); + + // // Create a publish job to trigger translation + // await this.prismaService.publishJob.create({ + // data: { + // userId: 'SYSTEM_SCHEDULED_TASK', + // assignmentId: assignment.assignmentId, + // status: 'Pending', + // progress: 'Scheduled republishing of top assignment', + // percentage: 0, + // }, + // }); + + // this.logger.log(`Republished assignment ${assignment.assignmentId} with ${assignment._count.assignmentId} attempts`); + // } + + // this.logger.log('Completed scheduled task: Republish top 10 used assignments'); + // } catch (error) { + // this.logger.error('Error in republishTopAssignments:', error); + // } + // } + + @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT) + async migrateExistingAuthors() { + this.logger.log( + "Starting scheduled task: Migrate existing authors to AssignmentAuthor table", + ); + + try { + // Find authors from Report table where author=true + const reportAuthors = await this.prismaService.report.findMany({ + where: { + author: true, + assignmentId: { + not: null, + }, + }, + select: { + reporterId: true, + assignmentId: true, + }, + distinct: ["reporterId", "assignmentId"], + }); + + this.logger.log( + `Found ${reportAuthors.length} potential authors from Report table`, + ); + + // Find authors from AIUsage table (users who generated content for assignments) + const aiUsageAuthors = await this.prismaService.aIUsage.findMany({ + where: { + userId: { + not: null, + }, + usageType: { + in: ["QUESTION_GENERATION", "ASSIGNMENT_GENERATION"], + }, + }, + select: { + userId: true, + assignmentId: true, + }, + distinct: ["userId", "assignmentId"], + }); + + this.logger.log( + `Found ${aiUsageAuthors.length} potential authors from AIUsage table`, + ); + + // Find authors from Job table (users who created assignments) + const jobAuthors = await this.prismaService.job.findMany({ + select: { + userId: true, + assignmentId: true, + }, + distinct: ["userId", "assignmentId"], + }); + + this.logger.log( + `Found ${jobAuthors.length} potential authors from Job table`, + ); + + // Find authors from publishJob table (users who published assignments) + const publishJobAuthors = await this.prismaService.publishJob.findMany({ + where: { + userId: { + not: "SYSTEM_SCHEDULED_TASK", // Exclude system tasks + }, + }, + select: { + userId: true, + assignmentId: true, + }, + distinct: ["userId", "assignmentId"], + }); + // Combine all potential authors + const allPotentialAuthors = [ + ...reportAuthors.map((r) => ({ + userId: r.reporterId, + assignmentId: r?.assignmentId ?? null, + })), + ...aiUsageAuthors.map((a) => ({ + userId: a?.userId ?? null, + assignmentId: a.assignmentId, + })), + ...jobAuthors.map((index) => ({ + userId: index.userId, + assignmentId: index.assignmentId, + })), + ...publishJobAuthors.map((p) => ({ + userId: p.userId, + assignmentId: p.assignmentId, + })), + ]; + + // Remove duplicates + const uniqueAuthors = allPotentialAuthors.filter( + (author, index, self) => + index === + self.findIndex( + (a) => + a.userId === author.userId && + a.assignmentId === author.assignmentId, + ), + ); + + this.logger.log( + `Processing ${uniqueAuthors.length} unique author-assignment pairs`, + ); + + let migratedCount = 0; + let skippedCount = 0; + + // Insert authors into AssignmentAuthor table (ignore duplicates) + for (const author of uniqueAuthors) { + try { + await this.prismaService.assignmentAuthor.create({ + data: { + userId: author.userId, + assignmentId: author.assignmentId, + }, + }); + migratedCount++; + } catch { + // Skip if already exists or assignment doesn't exist + skippedCount++; + } + } + + this.logger.log( + `Completed scheduled task: Migrated ${migratedCount} authors, skipped ${skippedCount} duplicates/invalid entries`, + ); + } catch (error) { + this.logger.error("Error in migrateExistingAuthors:", error); + } + } + + // @Cron(CronExpression.EVERY_WEEK) + // async cleanupOldPublishJobs() { + // this.logger.log('Starting scheduled task: Cleanup old publish jobs'); + + // try { + // // Delete completed publish jobs older than 30 days + // const thirtyDaysAgo = new Date(); + // thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30); + + // const deletedJobs = await this.prismaService.publishJob.deleteMany({ + // where: { + // status: 'Completed', + // createdAt: { + // lt: thirtyDaysAgo, + // }, + // }, + // }); + + // this.logger.log(`Cleaned up ${deletedJobs.count} old publish jobs`); + // } catch (error) { + // this.logger.error('Error in cleanupOldPublishJobs:', error); + // } + // } + + @Cron(CronExpression.EVERY_WEEK) // Every Sunday at midnight + async cleanupOldDrafts(customDaysOld?: number) { + const daysOld = customDaysOld === undefined ? 60 : customDaysOld; // Default to 60 days + const isDeleteAll = daysOld === 0; + + this.logger.log( + `Starting ${customDaysOld === undefined ? "scheduled" : "manual"} task: ${ + isDeleteAll + ? "Delete ALL drafts" + : `Cleanup old drafts (${daysOld} days old)` + }`, + ); + + try { + let whereCondition = {}; + let logMessage = ""; + + if (isDeleteAll) { + // Delete all drafts + whereCondition = {}; + logMessage = "Looking for ALL drafts to delete"; + } else { + // Calculate date for the specified number of days ago + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - daysOld); + whereCondition = { + createdAt: { + lt: cutoffDate, + }, + }; + logMessage = `Looking for drafts older than ${cutoffDate.toISOString()} (${daysOld} days ago)`; + } + + this.logger.log(logMessage); + + // Find drafts based on condition + const oldDrafts = await this.prismaService.assignmentDraft.findMany({ + where: whereCondition, + select: { + id: true, + draftName: true, + userId: true, + createdAt: true, + assignment: { + select: { + id: true, + name: true, + }, + }, + }, + }); + + this.logger.log( + `Found ${oldDrafts.length} ${ + isDeleteAll ? "drafts" : `drafts older than ${daysOld} days` + }`, + ); + + if (oldDrafts.length === 0) { + this.logger.log( + isDeleteAll ? "No drafts to delete" : "No old drafts to cleanup", + ); + return { + deletedCount: 0, + daysOld, + cutoffDate: isDeleteAll + ? "ALL" + : new Date( + Date.now() - daysOld * 24 * 60 * 60 * 1000, + ).toISOString(), + }; + } + + // Log details of drafts to be deleted for audit purposes + for (const draft of oldDrafts) { + this.logger.log( + `Preparing to delete draft: ID=${draft.id}, Name="${draft.draftName}", ` + + `User=${draft.userId}, Assignment="${draft.assignment.name}", ` + + `Created=${draft.createdAt.toISOString()}`, + ); + } + + // Delete the drafts + const deletedDrafts = await this.prismaService.assignmentDraft.deleteMany( + { + where: whereCondition, + }, + ); + + this.logger.log( + `Completed ${ + customDaysOld === undefined ? "scheduled" : "manual" + } task: Deleted ${deletedDrafts.count} ${ + isDeleteAll ? "drafts (ALL)" : "old drafts" + }`, + ); + + return { + deletedCount: deletedDrafts.count, + daysOld, + cutoffDate: isDeleteAll + ? "ALL" + : new Date(Date.now() - daysOld * 24 * 60 * 60 * 1000).toISOString(), + }; + } catch (error) { + this.logger.error("Error in cleanupOldDrafts:", error); + throw error; + } + } + + @Cron("0 */6 * * *") // Every 6 hours + async updateLLMPricing() { + this.logger.log("Starting scheduled task: Update LLM pricing"); + + try { + // Fetch current pricing from OpenAI + const currentPricing = await this.llmPricingService.fetchCurrentPricing(); + + if (currentPricing.length === 0) { + this.logger.warn("No pricing data fetched from OpenAI"); + return; + } + + // Update pricing history + const updatedCount = + await this.llmPricingService.updatePricingHistory(currentPricing); + + this.logger.log( + `Completed scheduled task: Updated pricing for ${updatedCount} models`, + ); + + // Log pricing statistics + const stats = await this.llmPricingService.getPricingStatistics(); + this.logger.log( + `Pricing statistics: ${stats.totalModels} models, ${stats.activePricingRecords} active pricing records`, + ); + } catch (error) { + this.logger.error("Error in updateLLMPricing:", error); + } + } + + async manualUpdateLLMPricing() { + this.logger.log("Manual update of LLM pricing requested"); + await this.updateLLMPricing(); + } + + async manualCleanupOldDrafts(daysOld?: number) { + this.logger.log( + `Manual cleanup of old drafts requested${ + daysOld ? ` (${daysOld} days old)` : "" + }`, + ); + return await this.cleanupOldDrafts(daysOld); + } + + @Cron(CronExpression.EVERY_3_HOURS) // Every 3 hours + async precomputeInsights() { + this.logger.log( + "Starting scheduled task: Precompute insights for popular assignments", + ); + + try { + await this.adminService.precomputePopularInsights(); + this.logger.log("Completed scheduled task: Insights precomputation"); + } catch (error) { + this.logger.error("Error in precomputeInsights:", error); + } + } + + async manualPrecomputeInsights() { + this.logger.log("Manual precomputation of insights requested"); + await this.precomputeInsights(); + } +} diff --git a/apps/api/src/api/user/controllers/notification.controller.ts b/apps/api/src/api/user/controllers/notification.controller.ts index 223f0ca5..11a75b18 100644 --- a/apps/api/src/api/user/controllers/notification.controller.ts +++ b/apps/api/src/api/user/controllers/notification.controller.ts @@ -1,6 +1,6 @@ import { Controller, Get, Injectable, Param, Post, Req } from "@nestjs/common"; -import { NotificationsService } from "../services/notification.service"; import { UserSessionRequest } from "src/auth/interfaces/user.session.interface"; +import { NotificationsService } from "../services/notification.service"; @Injectable() @Controller({ diff --git a/apps/api/src/api/user/modules/chat.module.ts b/apps/api/src/api/user/modules/chat.module.ts index 77f5e873..84db0242 100644 --- a/apps/api/src/api/user/modules/chat.module.ts +++ b/apps/api/src/api/user/modules/chat.module.ts @@ -2,8 +2,8 @@ import { Module } from "@nestjs/common"; import { PrismaService } from "src/prisma.service"; import { ChatController } from "../controllers/chat.controller"; import { ChatAccessControlGuard } from "../guards/chat.access.control.guard"; -import { ChatService } from "../services/chat.service"; import { ChatRepository } from "../repositories/chat.repository"; +import { ChatService } from "../services/chat.service"; @Module({ controllers: [ChatController], diff --git a/apps/api/src/api/user/repositories/chat.repository.ts b/apps/api/src/api/user/repositories/chat.repository.ts index af3dc70e..370735c9 100644 --- a/apps/api/src/api/user/repositories/chat.repository.ts +++ b/apps/api/src/api/user/repositories/chat.repository.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/require-await */ import { Injectable } from "@nestjs/common"; -import { PrismaService } from "../../../prisma.service"; import { Chat, ChatMessage, ChatRole, Prisma } from "@prisma/client"; +import { PrismaService } from "../../../prisma.service"; /** * Repository for Chat-related database operations diff --git a/apps/api/src/api/user/services/chat.service.ts b/apps/api/src/api/user/services/chat.service.ts index b6e3d8b1..a8a6fd37 100644 --- a/apps/api/src/api/user/services/chat.service.ts +++ b/apps/api/src/api/user/services/chat.service.ts @@ -1,7 +1,7 @@ import { Injectable } from "@nestjs/common"; -import { Chat, ChatMessage, ChatRole, Prisma } from "@prisma/client"; -import { ChatRepository } from "../repositories/chat.repository"; +import { Chat, ChatMessage, ChatRole } from "@prisma/client"; import { JsonValue } from "@prisma/client/runtime/library"; +import { ChatRepository } from "../repositories/chat.repository"; @Injectable() export class ChatService { diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 5dc62c60..c19916f1 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -8,7 +8,9 @@ import { ConfigModule } from "@nestjs/config"; import { RouterModule } from "@nestjs/core"; import { WinstonModule } from "nest-winston"; import { ApiModule } from "./api/api.module"; +import { ScheduledTasksModule } from "./api/scheduled-tasks/scheduled-tasks.module"; import { AppService } from "./app.service"; +import { AdminAuthModule } from "./auth/admin-auth.module"; import { AuthModule } from "./auth/auth.module"; import { UserSessionMiddleware } from "./auth/middleware/user.session.middleware"; import { HealthModule } from "./health/health.module"; @@ -23,9 +25,11 @@ import { routes } from "./routes"; WinstonModule.forRoot(winstonOptions), HealthModule, ApiModule, + ScheduledTasksModule, RouterModule.register(routes), MessagingModule, AuthModule, + AdminAuthModule, ], providers: [AppService], }) @@ -37,13 +41,15 @@ export class AppModule implements NestModule { .apply(UserSessionMiddleware) .forRoutes( { path: "/v1/assignments*", method: RequestMethod.ALL }, - { path: "/v2/assignments*", method: RequestMethod.ALL }, { path: "/v1/github*", method: RequestMethod.ALL }, { path: "/v1/user-session", method: RequestMethod.GET }, { path: "/v1/reports*", method: RequestMethod.ALL }, { path: "/v1/chats*", method: RequestMethod.ALL }, { path: "/v1/notifications*", method: RequestMethod.ALL }, { path: "/v1/files*", method: RequestMethod.ALL }, + { path: "/v1/admin*", method: RequestMethod.ALL }, + { path: "/v2/assignments/*", method: RequestMethod.ALL }, + { path: "/v1/admin-dashboard/*", method: RequestMethod.GET }, ); } } diff --git a/apps/api/src/auth/admin-auth.module.ts b/apps/api/src/auth/admin-auth.module.ts new file mode 100644 index 00000000..86814aae --- /dev/null +++ b/apps/api/src/auth/admin-auth.module.ts @@ -0,0 +1,18 @@ +import { Module } from "@nestjs/common"; +import { PrismaService } from "src/prisma.service"; +import { AdminAuthController } from "./controllers/admin-auth.controller"; +import { AdminGuard } from "./guards/admin.guard"; +import { AdminEmailService } from "./services/admin-email.service"; +import { AdminVerificationService } from "./services/admin-verification.service"; + +@Module({ + controllers: [AdminAuthController], + providers: [ + PrismaService, + AdminVerificationService, + AdminEmailService, + AdminGuard, + ], + exports: [AdminVerificationService, AdminEmailService, AdminGuard], +}) +export class AdminAuthModule {} diff --git a/apps/api/src/auth/controllers/admin-auth.controller.ts b/apps/api/src/auth/controllers/admin-auth.controller.ts new file mode 100644 index 00000000..8d54efcc --- /dev/null +++ b/apps/api/src/auth/controllers/admin-auth.controller.ts @@ -0,0 +1,295 @@ +import { + BadRequestException, + Body, + Controller, + ForbiddenException, + Injectable, + Post, +} from "@nestjs/common"; +import { ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; +import { AdminEmailService } from "../services/admin-email.service"; +import { AdminVerificationService } from "../services/admin-verification.service"; + +interface SendCodeRequest { + email: string; +} + +interface VerifyCodeRequest { + email: string; + code: string; +} + +interface SendCodeResponse { + message: string; + success: boolean; +} + +interface VerifyCodeResponse { + message: string; + success: boolean; + sessionToken?: string; + expiresAt?: string; +} + +@ApiTags("Admin Authentication") +@Injectable() +@Controller({ + path: "auth/admin", + version: "1", +}) +export class AdminAuthController { + constructor( + private readonly adminVerificationService: AdminVerificationService, + private readonly adminEmailService: AdminEmailService, + ) {} + + @Post("me") + @ApiOperation({ + summary: "Get current admin user information", + description: "Returns the current admin user's role and email information", + }) + @ApiResponse({ + status: 200, + description: "Admin user information retrieved successfully", + }) + @ApiResponse({ + status: 401, + description: "Invalid or expired admin session", + }) + async getCurrentAdmin(@Body() body: { sessionToken: string }) { + const userInfo = await this.adminVerificationService.verifyAdminSession( + body.sessionToken, + ); + + if (!userInfo) { + throw new ForbiddenException("Invalid or expired admin session"); + } + + return { + email: userInfo.email, + role: userInfo.role, + isAdmin: userInfo.role === "admin", + success: true, + }; + } + + @Post("send-code") + @ApiOperation({ + summary: "Send verification code to admin email", + description: + "Sends a 6-digit verification code to the specified email if it's in the admin list", + }) + @ApiResponse({ + status: 200, + description: "Verification code sent successfully", + type: Object, + }) + @ApiResponse({ + status: 403, + description: "Email not authorized for admin access", + }) + @ApiResponse({ + status: 400, + description: "Invalid email format", + }) + async sendVerificationCode( + @Body() request: SendCodeRequest, + ): Promise { + const { email } = request; + + // Validate email format + const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (!emailRegex.test(email)) { + throw new BadRequestException("Invalid email format"); + } + + // Check if email is authorized (admin or author) + const isAuthorized = + await this.adminVerificationService.isAuthorizedEmail(email); + if (!isAuthorized) { + throw new ForbiddenException("Email not authorized for admin access"); + } + + try { + // Generate and store verification code + const code = + await this.adminVerificationService.generateAndStoreCode(email); + + // Send email with verification code + const emailSent = await this.adminEmailService.sendVerificationCode( + email, + code, + ); + + if (!emailSent) { + throw new BadRequestException("Failed to send verification code"); + } + + return { + message: "Verification code sent to your email", + success: true, + }; + } catch (error) { + if ( + error instanceof BadRequestException || + error instanceof ForbiddenException + ) { + throw error; + } + throw new BadRequestException("Failed to send verification code"); + } + } + + @Post("verify-code") + @ApiOperation({ + summary: "Verify admin access code", + description: + "Verifies the 6-digit code and returns a session token for admin access", + }) + @ApiResponse({ + status: 200, + description: "Code verified successfully, session token returned", + type: Object, + }) + @ApiResponse({ + status: 400, + description: "Invalid or expired code", + }) + @ApiResponse({ + status: 403, + description: "Email not authorized", + }) + async verifyCode( + @Body() request: VerifyCodeRequest, + ): Promise { + const { email, code } = request; + + // Validate email format + const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (!emailRegex.test(email)) { + throw new BadRequestException("Invalid email format"); + } + + // Validate code format (6 digits) + if (!/^\d{6}$/.test(code)) { + throw new BadRequestException("Invalid code format"); + } + + // Check if email is authorized (admin or author) + const isAuthorized = + await this.adminVerificationService.isAuthorizedEmail(email); + if (!isAuthorized) { + throw new ForbiddenException("Email not authorized for admin access"); + } + + try { + // Verify the code + const isValidCode = await this.adminVerificationService.verifyCode( + email, + code, + ); + + if (!isValidCode) { + throw new BadRequestException("Invalid or expired verification code"); + } + + // Generate admin session token + const sessionToken = + await this.adminVerificationService.generateAdminSession(email); + const expiresAt = new Date( + Date.now() + 24 * 60 * 60 * 1000, + ).toISOString(); // 24 hours + + return { + message: "Admin access granted", + success: true, + sessionToken, + expiresAt, + }; + } catch (error) { + if ( + error instanceof BadRequestException || + error instanceof ForbiddenException + ) { + throw error; + } + throw new BadRequestException("Failed to verify code"); + } + } + + @Post("logout") + @ApiOperation({ + summary: "Logout admin session", + description: "Revokes the admin session token", + }) + @ApiResponse({ + status: 200, + description: "Logged out successfully", + }) + async logout( + @Body() request: { sessionToken: string }, + ): Promise<{ message: string }> { + const { sessionToken } = request; + + if (!sessionToken) { + throw new BadRequestException("Session token is required"); + } + + try { + await this.adminVerificationService.revokeSession(sessionToken); + return { message: "Logged out successfully" }; + } catch { + throw new BadRequestException("Failed to logout"); + } + } + + @Post("test-email") + @ApiOperation({ + summary: "Test email configuration", + description: + "Send a test email to verify Gmail SMTP configuration is working", + }) + @ApiResponse({ + status: 200, + description: "Test email sent successfully", + }) + @ApiResponse({ + status: 400, + description: "Failed to send test email", + }) + async testEmail( + @Body() request: { email: string }, + ): Promise<{ message: string; success: boolean }> { + const { email } = request; + + // Validate email format + const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (!emailRegex.test(email)) { + throw new BadRequestException("Invalid email format"); + } + + try { + // Test connection first + const connectionOk = await this.adminEmailService.testConnection(); + if (!connectionOk) { + throw new BadRequestException("Email service not properly configured"); + } + + // Send test email + const emailSent = await this.adminEmailService.sendTestEmail(email); + if (!emailSent) { + throw new BadRequestException("Failed to send test email"); + } + + return { + message: "Test email sent successfully", + success: true, + }; + } catch (error) { + if (error instanceof BadRequestException) { + throw error; + } + throw new BadRequestException("Failed to send test email"); + } + } +} diff --git a/apps/api/src/auth/guards/admin.guard.ts b/apps/api/src/auth/guards/admin.guard.ts new file mode 100644 index 00000000..6960141d --- /dev/null +++ b/apps/api/src/auth/guards/admin.guard.ts @@ -0,0 +1,53 @@ +import { + CanActivate, + ExecutionContext, + Injectable, + UnauthorizedException, +} from "@nestjs/common"; +import { + UserRole, + UserSessionRequest, +} from "../interfaces/user.session.interface"; +import { AdminVerificationService } from "../services/admin-verification.service"; + +@Injectable() +export class AdminGuard implements CanActivate { + constructor( + private readonly adminVerificationService: AdminVerificationService, + ) {} + + async canActivate(context: ExecutionContext): Promise { + const request = context.switchToHttp().getRequest(); + + const adminTokenHeader = + request.headers["x-admin-token"] || request.headers["admin-token"]; + + const adminToken = Array.isArray(adminTokenHeader) + ? adminTokenHeader[0] + : adminTokenHeader; + + if (!adminToken) { + throw new UnauthorizedException( + "Admin authentication required. Please login with email verification.", + ); + } + + const userInfo = + await this.adminVerificationService.verifyAdminSession(adminToken); + + if (!userInfo) { + throw new UnauthorizedException( + "Invalid or expired admin session. Please login again.", + ); + } + + request.userSession = { + ...request.userSession, + userId: userInfo.email.toLowerCase(), + role: userInfo.role === "admin" ? UserRole.ADMIN : UserRole.AUTHOR, + sessionToken: adminToken, + }; + + return true; + } +} diff --git a/apps/api/src/auth/interfaces/user.session.interface.ts b/apps/api/src/auth/interfaces/user.session.interface.ts index d7daef6c..8f3b1427 100644 --- a/apps/api/src/auth/interfaces/user.session.interface.ts +++ b/apps/api/src/auth/interfaces/user.session.interface.ts @@ -12,6 +12,7 @@ export interface ClientUserSession { assignmentId: number; returnUrl?: string; launch_presentation_locale?: string; + sessionToken?: string; } export interface UserSession extends ClientUserSession { diff --git a/apps/api/src/auth/middleware/user.session.middleware.ts b/apps/api/src/auth/middleware/user.session.middleware.ts index 828f7804..048e1177 100644 --- a/apps/api/src/auth/middleware/user.session.middleware.ts +++ b/apps/api/src/auth/middleware/user.session.middleware.ts @@ -12,11 +12,17 @@ import { @Injectable() export class UserSessionMiddleware implements NestMiddleware { use(request: UserSessionRequest, _: Response, next: NextFunction) { + const userSessionHeader = request.headers["user-session"] as string; + + if (!userSessionHeader) { + console.error("Invalid user-session header format"); + throw new BadRequestException("Invalid user-session header"); + } + try { - request.userSession = JSON.parse( - request.headers["user-session"] as string, - ) as UserSession; + request.userSession = JSON.parse(userSessionHeader) as UserSession; } catch { + console.error("Invalid user-session header format"); throw new BadRequestException("Invalid user-session header"); } next(); diff --git a/apps/api/src/auth/role/roles.global.guard.ts b/apps/api/src/auth/role/roles.global.guard.ts index 317ab596..8a1c9776 100644 --- a/apps/api/src/auth/role/roles.global.guard.ts +++ b/apps/api/src/auth/role/roles.global.guard.ts @@ -5,10 +5,7 @@ import { SetMetadata, } from "@nestjs/common"; import { Reflector } from "@nestjs/core"; -import { - UserRole, - UserSessionRequest, -} from "../interfaces/user.session.interface"; +import { UserRole } from "../interfaces/user.session.interface"; export const ROLES_KEY = "roles"; export const Roles = (...roles: UserRole[]) => SetMetadata(ROLES_KEY, roles); @@ -25,8 +22,17 @@ export class RolesGlobalGuard implements CanActivate { if (!requiredRoles) { return true; } - const request: UserSessionRequest = context.switchToHttp().getRequest(); - const { userSession } = request; - return requiredRoles.includes(userSession.role); + const request: { + userSession?: { role: UserRole }; + adminSession?: { role: UserRole }; + } = context.switchToHttp().getRequest(); + + const session = request.userSession || request.adminSession; + + if (!session) { + return false; + } + + return requiredRoles.includes(session.role); } } diff --git a/apps/api/src/auth/services/admin-email.service.ts b/apps/api/src/auth/services/admin-email.service.ts new file mode 100644 index 00000000..63edbdff --- /dev/null +++ b/apps/api/src/auth/services/admin-email.service.ts @@ -0,0 +1,468 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable unicorn/prefer-module */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { Injectable, Logger } from "@nestjs/common"; +import * as nodemailer from "nodemailer"; + +/** + * AdminEmailService supports both SendGrid and Gmail SMTP for sending emails. + * + * Environment Variables: + * + * EMAIL_PROVIDER - Choose email provider ('sendgrid' | 'google'). Defaults to 'sendgrid' + * + * SendGrid Configuration: + * - SENDGRID_API_KEY: SendGrid API key (required for SendGrid) + * - SENDGRID_FROM_EMAIL: From email address (defaults to 'noreply@markapp.com') + * - SENDGRID_FROM_NAME: From name (defaults to 'Mark Admin System') + * + * Gmail Configuration: + * - GMAIL_USER: Gmail email address (required for Gmail) + * - GMAIL_APP_PASSWORD: Gmail app password (required for Gmail) + * + * Fallback Strategy: + * - If preferred provider is not available, falls back to the other provider + * - If no providers are configured, uses console logging in development + * - Fails gracefully in production when no providers are available + */ + +type EmailProvider = "sendgrid" | "google" | "none"; +const sgMail = require("@sendgrid/mail"); +sgMail.setApiKey(process.env.SENDGRID_API_KEY); +@Injectable() +export class AdminEmailService { + private readonly logger = new Logger(AdminEmailService.name); + private transporter: nodemailer.Transporter; + private emailProvider: EmailProvider; + + constructor() { + this.initializeEmailService(); + } + + private initializeEmailService() { + const providerPreference = + process.env.EMAIL_PROVIDER?.toLowerCase() || "sendgrid"; + + const sendGridApiKey = process.env.SENDGRID_API_KEY; + + const gmailUser = process.env.GMAIL_USER; + const gmailPassword = process.env.GMAIL_APP_PASSWORD; + + if (providerPreference === "sendgrid" && sendGridApiKey) { + try { + sgMail.setApiKey(sendGridApiKey); + this.emailProvider = "sendgrid"; + this.transporter = undefined; + this.logger.log("SendGrid email service initialized"); + return; + } catch (error) { + this.logger.error("Failed to initialize SendGrid:", error); + } + } + + if (providerPreference === "google" && gmailUser && gmailPassword) { + this.emailProvider = "google"; + this.transporter = nodemailer.createTransport({ + host: "smtp.gmail.com", + port: 587, + secure: false, + auth: { + user: gmailUser, + pass: gmailPassword, + }, + requireTLS: true, + }); + this.logger.log("Gmail SMTP transporter initialized"); + return; + } else if (gmailUser && gmailPassword) { + this.emailProvider = "google"; + this.transporter = nodemailer.createTransport({ + host: "smtp.gmail.com", + port: 587, + secure: false, + auth: { + user: gmailUser, + pass: gmailPassword, + }, + requireTLS: true, + }); + this.logger.log("Gmail SMTP transporter initialized (fallback)"); + return; + } else if ( + sendGridApiKey && + sgMail && + typeof sgMail.setApiKey === "function" + ) { + try { + sgMail.setApiKey(sendGridApiKey); + this.emailProvider = "sendgrid"; + this.transporter = undefined; + this.logger.log("SendGrid email service initialized (fallback)"); + } catch (error) { + this.logger.error("Failed to initialize SendGrid as fallback:", error); + this.emailProvider = "none"; + this.transporter = undefined; + } + } else { + this.emailProvider = "none"; + this.transporter = undefined; + this.logger.warn( + "No email service configured. Set SENDGRID_API_KEY or GMAIL_USER/GMAIL_APP_PASSWORD. Email service will use console logging in development.", + ); + } + } + + /** + * Send verification code email to admin using configured email provider (SendGrid or Gmail) + */ + async sendVerificationCode(email: string, code: string): Promise { + try { + if (this.emailProvider === "none") { + if (process.env.NODE_ENV === "production") { + this.logger.error("Email service not configured for production"); + return false; + } else { + this.logger.log(` +=== ADMIN VERIFICATION CODE === +Email: ${email} +Code: ${code} +Expires: 10 minutes +Provider: Development Console +===============================`); + return true; + } + } + + if (this.emailProvider === "sendgrid") { + return await this.sendVerificationCodeSendGrid(email, code); + } else if (this.emailProvider === "google") { + return await this.sendVerificationCodeGmail(email, code); + } + + return false; + } catch (error) { + this.logger.error(`Failed to send verification code to ${email}:`, error); + return false; + } + } + + /** + * Send verification code using SendGrid + */ + private async sendVerificationCodeSendGrid( + email: string, + code: string, + ): Promise { + try { + if (!sgMail || typeof sgMail.send !== "function") { + this.logger.error("SendGrid not properly initialized"); + return false; + } + + const fromEmail = + process.env.SENDGRID_FROM_EMAIL || "noreply@markapp.com"; + const fromName = process.env.SENDGRID_FROM_NAME || "Mark Admin System"; + + const mailData = { + from: { + email: fromEmail, + name: fromName, + }, + to: email, + subject: "Mark Admin Access - Verification Code", + html: this.getEmailTemplate(code), + text: this.getPlainTextTemplate(code), + }; + + await sgMail.send(mailData); + + return true; + } catch (error) { + this.logger.error( + `Failed to send verification code via SendGrid to ${email}:`, + error, + ); + return false; + } + } + + /** + * Send verification code using Gmail SMTP + */ + private async sendVerificationCodeGmail( + email: string, + code: string, + ): Promise { + try { + if (!this.transporter) { + this.logger.error("Gmail transporter not initialized"); + return false; + } + + const mailOptions = { + from: { + name: "Mark Admin System", + address: process.env.GMAIL_USER || "noreply@markapp.com", + }, + to: email, + subject: "Mark Admin Access - Verification Code", + html: this.getEmailTemplate(code), + text: this.getPlainTextTemplate(code), + }; + + await this.transporter.sendMail(mailOptions); + return true; + } catch (error) { + this.logger.error( + `Failed to send verification code via Gmail to ${email}:`, + error, + ); + return false; + } + } + + /** + * Get HTML email template + */ + private getEmailTemplate(code: string): string { + return ` + + + + + + Admin Verification Code + + + +
+
+

🛡️ Admin Access

+
+
+

+ Someone requested admin access to the Mark application with your email address. + Use the verification code below to complete your login: +

+ +
+
${code}
+
+ +
+

+ ⚠️ Security Notice: This code expires in 10 minutes. + If you did not request admin access, please ignore this email and consider changing your password. +

+
+ +

+ For security reasons, do not share this code with anyone. Mark administrators will never ask for this code. +

+
+
+

This is an automated message from Mark Admin System

+

© ${new Date().getFullYear()} Mark Application

+
+
+ + + `; + } + + /** + * Get plain text email template + */ + private getPlainTextTemplate(code: string): string { + return ` +Mark Admin Access - Verification Code + +Someone requested admin access to the Mark application with your email address. + +Your verification code is: ${code} + +This code will expire in 10 minutes. If you did not request this, please ignore this email. + +For security reasons, do not share this code with anyone. + +This is an automated message from Mark Admin System. + `; + } + + /** + * Test email service connection + */ + async testConnection(): Promise { + try { + if (this.emailProvider === "none") { + if (process.env.NODE_ENV === "production") { + this.logger.error("Email service not configured"); + return false; + } else { + this.logger.log( + "Email service ready (development mode - console logging)", + ); + return true; + } + } + + if (this.emailProvider === "sendgrid") { + this.logger.log("SendGrid email service ready"); + return true; + } + + if (this.emailProvider === "google" && this.transporter) { + await this.transporter.verify(); + this.logger.log("Gmail SMTP connection verified successfully"); + return true; + } + + return false; + } catch (error) { + this.logger.error( + `${this.emailProvider} email service connection failed:`, + error, + ); + return false; + } + } + + /** + * Send a test email to verify configuration + */ + async sendTestEmail(toEmail: string): Promise { + try { + if (this.emailProvider === "none") { + this.logger.warn( + "Cannot send test email - email service not configured", + ); + return false; + } + + // Route to appropriate email service + if (this.emailProvider === "sendgrid") { + return await this.sendTestEmailSendGrid(toEmail); + } else if (this.emailProvider === "google") { + return await this.sendTestEmailGmail(toEmail); + } + + return false; + } catch (error) { + this.logger.error(`Failed to send test email to ${toEmail}:`, error); + return false; + } + } + + /** + * Send test email using SendGrid + */ + private async sendTestEmailSendGrid(toEmail: string): Promise { + try { + if (!sgMail || typeof sgMail.send !== "function") { + this.logger.error("SendGrid not properly initialized"); + return false; + } + + const fromEmail = + process.env.SENDGRID_FROM_EMAIL || "noreply@markapp.com"; + const fromName = process.env.SENDGRID_FROM_NAME || "Mark Admin System"; + + const mailData = { + from: { + email: fromEmail, + name: fromName, + }, + to: toEmail, + subject: "Mark Admin - Email Configuration Test", + html: ` +

🎉 Email Configuration Test

+

If you received this email, your SendGrid email configuration is working correctly!

+

Provider: SendGrid

+

Timestamp: ${new Date().toISOString()}

+

This is a test message from Mark Admin System.

+ `, + text: ` +Email Configuration Test + +If you received this email, your SendGrid email configuration is working correctly! + +Provider: SendGrid +Timestamp: ${new Date().toISOString()} + +This is a test message from Mark Admin System. + `, + }; + await sgMail.send(mailData); + return true; + } catch (error) { + this.logger.error( + `Failed to send test email via SendGrid to ${toEmail}:`, + error, + ); + return false; + } + } + + /** + * Send test email using Gmail SMTP + */ + private async sendTestEmailGmail(toEmail: string): Promise { + try { + if (!this.transporter) { + this.logger.error("Gmail transporter not initialized"); + return false; + } + + const mailOptions = { + from: { + name: "Mark Admin System", + address: process.env.GMAIL_USER || "noreply@markapp.com", + }, + to: toEmail, + subject: "Mark Admin - Email Configuration Test", + html: ` +

🎉 Email Configuration Test

+

If you received this email, your Gmail SMTP configuration is working correctly!

+

Provider: Gmail SMTP

+

Timestamp: ${new Date().toISOString()}

+

This is a test message from Mark Admin System.

+ `, + text: ` +Email Configuration Test + +If you received this email, your Gmail SMTP configuration is working correctly! + +Provider: Gmail SMTP +Timestamp: ${new Date().toISOString()} + +This is a test message from Mark Admin System. + `, + }; + + await this.transporter.sendMail(mailOptions); + + return true; + } catch (error) { + this.logger.error( + `Failed to send test email via Gmail to ${toEmail}:`, + error, + ); + return false; + } + } +} diff --git a/apps/api/src/auth/services/admin-verification.service.ts b/apps/api/src/auth/services/admin-verification.service.ts new file mode 100644 index 00000000..197821e5 --- /dev/null +++ b/apps/api/src/auth/services/admin-verification.service.ts @@ -0,0 +1,173 @@ +import * as crypto from "node:crypto"; +import { Injectable } from "@nestjs/common"; +import { isAdminEmail } from "src/config/admin-emails"; +import { PrismaService } from "src/prisma.service"; + +@Injectable() +export class AdminVerificationService { + constructor(private readonly prisma: PrismaService) {} + + /** + * Generate a 6-digit verification code + */ + private generateVerificationCode(): string { + return crypto.randomInt(100_000, 999_999).toString(); + } + + /** + * Generate a verification code and store it in the database + */ + async generateAndStoreCode(email: string): Promise { + const code = this.generateVerificationCode(); + const expiresAt = new Date(Date.now() + 10 * 60 * 1000); // 10 minutes from now + + // Clean up old codes for this email + await this.prisma.adminVerificationCode.deleteMany({ + where: { email: email.toLowerCase() }, + }); + + // Store new code + await this.prisma.adminVerificationCode.create({ + data: { + email: email.toLowerCase(), + code, + expiresAt, + used: false, + }, + }); + + return code; + } + + /** + * Verify a code against stored codes + */ + async verifyCode(email: string, code: string): Promise { + const verificationRecord = + await this.prisma.adminVerificationCode.findFirst({ + where: { + email: email.toLowerCase(), + code, + used: false, + expiresAt: { + gt: new Date(), + }, + }, + }); + + if (!verificationRecord) { + return false; + } + + // Mark code as used + await this.prisma.adminVerificationCode.update({ + where: { id: verificationRecord.id }, + data: { used: true }, + }); + + return true; + } + + /** + * Generate an admin session token + */ + async generateAdminSession(email: string): Promise { + const sessionToken = crypto.randomBytes(32).toString("hex"); + const expiresAt = new Date(Date.now() + 24 * 60 * 60 * 1000); // 24 hours from now + + // Clean up old sessions for this email + await this.prisma.adminSession.deleteMany({ + where: { email: email.toLowerCase() }, + }); + + // Create new session + await this.prisma.adminSession.create({ + data: { + email: email.toLowerCase(), + sessionToken, + expiresAt, + }, + }); + + return sessionToken; + } + + /** + * Verify admin session token and return user info + */ + async verifyAdminSession( + sessionToken: string, + ): Promise<{ email: string; role: "admin" | "author" } | null> { + const session = await this.prisma.adminSession.findFirst({ + where: { + sessionToken, + expiresAt: { + gt: new Date(), + }, + }, + }); + + if (!session) { + return null; + } + + const role = isAdminEmail(session.email) ? "admin" : "author"; + + return { + email: session.email, + role, + }; + } + + /** + * Check if email is authorized (admin or has authored assignments) + */ + async isAuthorizedEmail(email: string): Promise { + // Check if it's an admin email + if (isAdminEmail(email)) { + return true; + } + + // Check if user has authored any assignments + const authorRecord = await this.prisma.assignmentAuthor.findFirst({ + where: { + userId: email.toLowerCase(), + }, + }); + + return !!authorRecord; + } + + /** + * Clean up expired codes and sessions + */ + async cleanupExpired(): Promise { + const now = new Date(); + + await Promise.all([ + this.prisma.adminVerificationCode.deleteMany({ + where: { + expiresAt: { + lt: now, + }, + }, + }), + this.prisma.adminSession.deleteMany({ + where: { + expiresAt: { + lt: now, + }, + }, + }), + ]); + } + + /** + * Revoke admin session + */ + async revokeSession(sessionToken: string): Promise { + await this.prisma.adminSession.deleteMany({ + where: { sessionToken }, + }); + } +} diff --git a/apps/api/src/config/admin-emails.ts b/apps/api/src/config/admin-emails.ts new file mode 100644 index 00000000..ab6007b8 --- /dev/null +++ b/apps/api/src/config/admin-emails.ts @@ -0,0 +1,10 @@ +export const ADMIN_EMAILS = // read from environment variable or hardcoded list + process.env.ADMIN_EMAILS + ? process.env.ADMIN_EMAILS.split(",").map((email) => + email.trim().toLowerCase(), + ) + : []; + +export function isAdminEmail(email: string): boolean { + return ADMIN_EMAILS.includes(email.toLowerCase()); +} diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index 53df22b1..082c35f2 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -17,7 +17,8 @@ async function bootstrap() { cors: false, logger: WinstonModule.createLogger(winstonOptions), }); - + app.use(json({ limit: "1000mb" })); + app.use(urlencoded({ limit: "1000mb", extended: true })); app.setGlobalPrefix("api", { exclude: ["health", "health/liveness", "health/readiness"], }); @@ -29,9 +30,6 @@ async function bootstrap() { app.use(helmet()); app.use(cookieParser()); - app.use(json({ limit: "10mb" })); - app.use(urlencoded({ limit: "10mb", extended: true })); - app.useGlobalPipes(new ValidationPipe({ whitelist: true })); app.useGlobalGuards(app.select(AuthModule).get(RolesGlobalGuard)); diff --git a/apps/api/src/scripts/create-initial-versions.ts b/apps/api/src/scripts/create-initial-versions.ts new file mode 100644 index 00000000..833eaa5e --- /dev/null +++ b/apps/api/src/scripts/create-initial-versions.ts @@ -0,0 +1,180 @@ +#!/usr/bin/env ts-node + +/** + * Script to create initial versions (version 1) for all existing assignments + * This should be run once to migrate existing assignments to the new version system + */ +import { PrismaService } from "../prisma.service"; + +async function createInitialVersions() { + const prisma = new PrismaService(); + + try { + console.log( + "Starting to create initial versions for existing assignments...", + ); + + const alreadyMigrated = await prisma.assignmentVersion.count(); + if (alreadyMigrated > 0) { + console.log("🚫 Initial versions already exist. Skipping migration."); + process.exit(0); + } + + // Find all assignments that don't have any versions yet + const assignmentsWithoutVersions = await prisma.assignment.findMany({ + where: { + versions: { + none: {}, + }, + }, + include: { + questions: { + where: { isDeleted: false }, + }, + AssignmentAuthor: true, + }, + }); + + console.log( + `Found ${assignmentsWithoutVersions.length} assignments without versions`, + ); + + let createdCount = 0; + + for (const assignment of assignmentsWithoutVersions) { + try { + await prisma.$transaction(async (tx) => { + console.log( + `Creating version 1 for assignment "${assignment.name}" (ID: ${assignment.id})`, + ); + + // Create the initial version (version 1) + const assignmentVersion = await tx.assignmentVersion.create({ + data: { + assignmentId: assignment.id, + versionNumber: "1.0.0", + name: assignment.name, + introduction: assignment.introduction, + instructions: assignment.instructions, + gradingCriteriaOverview: assignment.gradingCriteriaOverview, + timeEstimateMinutes: assignment.timeEstimateMinutes, + type: assignment.type, + graded: assignment.graded, + numAttempts: assignment.numAttempts, + allotedTimeMinutes: assignment.allotedTimeMinutes, + attemptsPerTimeRange: assignment.attemptsPerTimeRange, + attemptsTimeRangeHours: assignment.attemptsTimeRangeHours, + passingGrade: assignment.passingGrade, + displayOrder: assignment.displayOrder, + questionDisplay: assignment.questionDisplay, + numberOfQuestionsPerAttempt: + assignment.numberOfQuestionsPerAttempt, + questionOrder: assignment.questionOrder, + published: assignment.published, + showAssignmentScore: assignment.showAssignmentScore, + showQuestionScore: assignment.showQuestionScore, + showSubmissionFeedback: assignment.showSubmissionFeedback, + showQuestions: assignment.showQuestions, + languageCode: assignment.languageCode, + createdBy: assignment.AssignmentAuthor[0]?.userId || "system", + isDraft: false, // Existing published assignments become version 1 (non-draft) + versionDescription: + "Initial version created from existing assignment", + isActive: true, // Make this the active version + }, + }); + + // Create question versions for all questions + for (const [index, question] of assignment.questions.entries()) { + await tx.questionVersion.create({ + data: { + assignmentVersionId: assignmentVersion.id, + questionId: question.id, + totalPoints: question.totalPoints, + type: question.type, + responseType: question.responseType, + question: question.question, + maxWords: question.maxWords, + scoring: question.scoring, + choices: question.choices, + randomizedChoices: question.randomizedChoices, + answer: question.answer, + gradingContextQuestionIds: question.gradingContextQuestionIds, + maxCharacters: question.maxCharacters, + videoPresentationConfig: question.videoPresentationConfig, + liveRecordingConfig: question.liveRecordingConfig, + displayOrder: index + 1, + }, + }); + } + + // Set the assignment's currentVersionId to this new version + await tx.assignment.update({ + where: { id: assignment.id }, + data: { currentVersionId: assignmentVersion.id }, + }); + + // Create version history entry + await tx.versionHistory.create({ + data: { + assignmentId: assignment.id, + toVersionId: assignmentVersion.id, + action: "initial_version_created", + description: "Initial version created during migration", + userId: assignment.AssignmentAuthor[0]?.userId || "system", + }, + }); + + createdCount++; + console.log( + `✅ Created version 1 for assignment "${assignment.name}" with ${assignment.questions.length} questions`, + ); + }); + } catch (error) { + console.error( + `❌ Failed to create version for assignment "${assignment.name}" (ID: ${assignment.id}):`, + error, + ); + } + } + + console.log( + `\n🎉 Successfully created initial versions for ${createdCount} assignments`, + ); + + // Verify the results + const totalVersions = await prisma.assignmentVersion.count(); + const assignmentsWithVersions = await prisma.assignment.count({ + where: { + versions: { + some: {}, + }, + }, + }); + + console.log(`\n📊 Summary:`); + console.log(`- Total assignment versions in database: ${totalVersions}`); + console.log(`- Assignments with versions: ${assignmentsWithVersions}`); + } catch (error) { + console.error("❌ Script failed with error:", error); + process.exit(1); + } finally { + await prisma.$disconnect(); + } +} + +// Run the script if called directly +// eslint-disable-next-line unicorn/prefer-module +if (require.main === module) { + createInitialVersions() + .then(() => { + console.log("✅ Script completed successfully"); + process.exit(0); + }) + .catch((error) => { + console.error("❌ Script failed:", error); + process.exit(1); + }); +} + +export { createInitialVersions }; diff --git a/apps/api/src/scripts/translation-audit-batch.ts b/apps/api/src/scripts/translation-audit-batch.ts new file mode 100644 index 00000000..42e956c8 --- /dev/null +++ b/apps/api/src/scripts/translation-audit-batch.ts @@ -0,0 +1,502 @@ +#!/usr/bin/env ts-node +/* eslint-disable */ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { HumanMessage } from "@langchain/core/messages"; +import { PrismaClient } from "@prisma/client"; +import { OpenAiLlmMiniService } from "src/api/llm/core/services/openai-llm-mini.service"; +import { createLogger } from "winston"; +import { PromptProcessorService } from "../api/llm/core/services/prompt-processor.service"; +import { TokenCounterService } from "../api/llm/core/services/token-counter.service"; + +const prisma = new PrismaClient(); + +// Initialize the project's translation service +const logger = createLogger({ + level: "error", + silent: true, // Keep quiet for CLI usage +}); + +const promptProcessor = new PromptProcessorService( + {} as any, // We don't need the LLM router for language detection + {} as any, // We don't need usage tracker for language detection + logger, +); + +const tokenCounter = new TokenCounterService(logger); +const gpt4MiniService = new OpenAiLlmMiniService(tokenCounter, logger); + +// eslint-disable-next-line unicorn/prefer-module +const LANGUAGES_FILE_PATH = path.join( + __dirname, + "../api/assignment/attempt/helper/languages.json", +); + +const supportedLanguages: Array<{ code: string; name: string }> = JSON.parse( + fs.readFileSync(LANGUAGES_FILE_PATH, "utf8"), +); +const SUPPORTED_LANGUAGE_CODES = supportedLanguages.map((lang) => + lang.code.toLowerCase(), +); + +console.log( + `📋 Loaded ${ + SUPPORTED_LANGUAGE_CODES.length + } supported languages: ${SUPPORTED_LANGUAGE_CODES.join(", ")}`, +); + +interface TranslationTask { + id: number; + translationId: number; + languageCode: string; + originalText: string; + originalChoices: any; + assignmentId: number; +} + +interface TranslationResult { + translationId: number; + success: boolean; + translatedText?: string; + translatedChoices?: any; + error?: string; +} + +/** + * GPT-5-nano translation fallback for script usage + */ +async function translateTextWithOpenAI( + text: string, + targetLanguage: string, +): Promise { + const languageNames: Record = { + en: "English", + es: "Spanish", + fr: "French", + de: "German", + it: "Italian", + pt: "Portuguese", + ru: "Russian", + zh: "Chinese", + "zh-cn": "Chinese (Simplified)", + "zh-tw": "Chinese (Traditional)", + ja: "Japanese", + ko: "Korean", + ar: "Arabic", + hi: "Hindi", + th: "Thai", + tr: "Turkish", + pl: "Polish", + nl: "Dutch", + sv: "Swedish", + hu: "Hungarian", + el: "Greek", + "uk-ua": "Ukrainian", + kk: "Kazakh", + id: "Indonesian", + }; + + const targetLanguageName = + languageNames[targetLanguage.toLowerCase()] || targetLanguage; + + try { + const promptContent = `You are a professional translator. Translate the given text to ${targetLanguageName}. Maintain the original meaning, tone, and context. If the text contains technical terms, preserve them appropriately. Return only the translated text without any additional explanation. + +Text to translate: ${text}`; + + const response = await gpt4MiniService.invoke([ + new HumanMessage(promptContent), + ]); + + const translatedText = response.content?.trim(); + + if (!translatedText) { + throw new Error("No translation received from GPT-5-nano"); + } + + return translatedText; + } catch (error) { + throw new Error( + `Translation failed: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + } +} + +/** + * Direct OpenAI choice translation fallback for script usage + */ +async function translateChoicesWithOpenAI( + choices: any[], + targetLanguage: string, +): Promise { + // Process all choices in parallel + return Promise.all( + choices.map(async (choice) => { + const translatedChoice = { ...choice }; + + // Translate choice text and feedback in parallel + const [translatedChoiceText, translatedFeedback] = await Promise.all([ + choice.choice + ? translateTextWithOpenAI(choice.choice, targetLanguage).catch( + (error) => { + console.log( + ` ⚠️ Failed to translate choice text: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + return choice.choice; // Keep original if translation fails + }, + ) + : Promise.resolve(choice.choice), + choice.feedback + ? translateTextWithOpenAI(choice.feedback, targetLanguage).catch( + (error) => { + console.log( + ` ⚠️ Failed to translate feedback text: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + return choice.feedback; // Keep original if translation fails + }, + ) + : Promise.resolve(choice.feedback), + ]); + + translatedChoice.choice = translatedChoiceText; + translatedChoice.feedback = translatedFeedback; + + return translatedChoice; + }), + ); +} + +/** + * Process a single translation task + */ +async function processTranslation( + task: TranslationTask, + index: number, + total: number, +): Promise { + console.log( + `\n📝 [${index}/${total}] Processing translation ${task.translationId} (${task.languageCode})`, + ); + + try { + let translatedText = ""; + let translatedChoices = null; + + // Translate main text + try { + console.log(` Using assignment ID: ${task.assignmentId}`); + translatedText = await translateTextWithOpenAI( + task.originalText, + task.languageCode, + ); + + console.log(` ✅ Main text translated via service`); + } catch (serviceError) { + console.log( + ` ⚠️ Service failed: ${ + serviceError instanceof Error ? serviceError.message : "Unknown error" + }`, + ); + } + + // Translate choices if they exist + if (task.originalChoices) { + let choicesArray = []; + let originalChoicesData = task.originalChoices; + + // Handle case where choices might be stored as JSON string + if (typeof task.originalChoices === "string") { + try { + originalChoicesData = JSON.parse(task.originalChoices); + } catch (e) { + console.log(` ⚠️ Failed to parse choices JSON: ${e}`); + translatedChoices = task.originalChoices; // Keep original if can't parse + } + } + + if (Array.isArray(originalChoicesData)) { + choicesArray = originalChoicesData; + } else if ( + typeof originalChoicesData === "object" && + originalChoicesData.choices + ) { + choicesArray = originalChoicesData.choices; + } + + if (choicesArray.length > 0) { + console.log(` 🔄 Translating ${choicesArray.length} choices...`); + try { + const translatedChoicesArray = await translateChoicesWithOpenAI( + choicesArray, + task.languageCode, + ); + + // Format the result to match the original structure + if (Array.isArray(originalChoicesData)) { + translatedChoices = translatedChoicesArray; + } else { + translatedChoices = { + ...originalChoicesData, + choices: translatedChoicesArray, + }; + } + + console.log(` ✅ Choices translated successfully`); + } catch (serviceError) { + console.log( + ` ⚠️ Choice translation failed: ${ + serviceError instanceof Error + ? serviceError.message + : "Unknown error" + }`, + ); + console.log(` 📝 Keeping original choices as fallback`); + translatedChoices = task.originalChoices; // Keep original as fallback + } + } else { + console.log(` 📝 No choices found to translate`); + translatedChoices = task.originalChoices; + } + } + + return { + translationId: task.translationId, + success: true, + translatedText, + translatedChoices, + }; + } catch (error) { + console.log( + ` ❌ Translation failed: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + return { + translationId: task.translationId, + success: false, + error: error instanceof Error ? error.message : "Unknown error", + }; + } +} + +/** + * Batch process translation updates to database + */ +async function batchUpdateTranslations( + results: TranslationResult[], +): Promise<{ success: number; failed: number }> { + let successCount = 0; + let failedCount = 0; + + // Update all successful translations in parallel + await Promise.all( + results.map(async (result) => { + if (result.success && result.translatedText) { + try { + // Ensure choices are properly formatted for database storage + const choicesData = result.translatedChoices + ? typeof result.translatedChoices === "string" + ? result.translatedChoices + : JSON.stringify(result.translatedChoices) + : result.translatedChoices; + + await prisma.translation.update({ + where: { id: result.translationId }, + data: { + translatedText: result.translatedText, + translatedChoices: choicesData, + }, + }); + console.log(`✅ Updated translation ${result.translationId}`); + successCount++; + } catch (error) { + console.log( + `❌ Failed to update ${result.translationId}: ${ + error instanceof Error ? error.message : "Unknown error" + }`, + ); + failedCount++; + } + } else { + failedCount++; + } + }), + ); + + return { success: successCount, failed: failedCount }; +} + +/** + * Retranslate specific records and update them in the database with batch processing + */ +export async function markForRetranslationBatch( + translationIds: number[], + assignmentId?: number, + includeAll = false, +): Promise { + if (!process.env.OPENAI_API_KEY) { + console.log( + `❌ OPENAI_API_KEY is required for retranslation. Please set this environment variable.`, + ); + return; + } + + console.log( + `🔄 Retranslating ${translationIds.length} records with batch processing...\n`, + ); + + const BATCH_SIZE = 5; // Process 5 translations at a time + let totalSuccess = 0; + let totalError = 0; + + // First, fetch all translation records with their source data + console.log(`📥 Fetching translation records...`); + const translationRecords = await Promise.all( + translationIds.map((id) => + prisma.translation.findUnique({ + where: { id }, + include: { + question: { + include: { + assignment: { + include: { + currentVersion: true, + }, + }, + }, + }, + variant: { + include: { + variantOf: { + include: { + assignment: { + include: { + currentVersion: true, + }, + }, + }, + }, + }, + }, + }, + }), + ), + ); + + // Filter and prepare translation tasks + const tasks: TranslationTask[] = []; + for (const translation of translationRecords) { + if (!translation) continue; + + // Check if active (unless includeAll) + if (!includeAll) { + let isActive = false; + if (translation.question) { + isActive = + !translation.question.isDeleted && + translation.question.assignment?.currentVersion?.isActive === true && + translation.question.assignment?.currentVersion?.isDraft === false; + } else if (translation.variant) { + isActive = + !translation.variant.isDeleted && + !translation.variant.variantOf.isDeleted && + translation.variant.variantOf.assignment?.currentVersion?.isActive === + true && + translation.variant.variantOf.assignment?.currentVersion?.isDraft === + false; + } + + if (!isActive) { + console.log(`⚠️ Skipping inactive translation ${translation.id}`); + continue; + } + } + + // Prepare task + let originalText = ""; + let originalChoices = null; + let taskAssignmentId = 0; + + if (translation.question) { + originalText = translation.question.question; + originalChoices = translation.question.choices; + taskAssignmentId = assignmentId || translation.question.assignmentId || 0; + } else if (translation.variant) { + originalText = translation.variant.variantContent; + originalChoices = translation.variant.choices; + taskAssignmentId = + assignmentId || translation.variant.variantOf.assignmentId || 0; + } + + if (originalText) { + tasks.push({ + id: tasks.length, + translationId: translation.id, + languageCode: translation.languageCode, + originalText, + originalChoices, + assignmentId: taskAssignmentId, + }); + } + } + + console.log(`📋 Prepared ${tasks.length} translations for processing\n`); + + // Process in batches + for (let i = 0; i < tasks.length; i += BATCH_SIZE) { + const batch = tasks.slice(i, Math.min(i + BATCH_SIZE, tasks.length)); + const batchNumber = Math.floor(i / BATCH_SIZE) + 1; + const totalBatches = Math.ceil(tasks.length / BATCH_SIZE); + + console.log( + `\n📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} translations)...`, + ); + + // Process batch in parallel + const batchResults = await Promise.all( + batch.map((task, index) => + processTranslation(task, i + index + 1, tasks.length), + ), + ); + + // Update database for this batch + const { success, failed } = await batchUpdateTranslations(batchResults); + totalSuccess += success; + totalError += failed; + + // Small delay between batches to avoid rate limiting + if (i + BATCH_SIZE < tasks.length) { + console.log(`⏳ Waiting before next batch...`); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + + console.log(`\n📊 Retranslation Summary:`); + console.log(` ✅ Successful: ${totalSuccess}`); + console.log(` ❌ Errors: ${totalError}`); + console.log(` 📋 Total: ${translationIds.length}`); + + if (totalSuccess > 0) { + console.log(`\n🎉 Successfully retranslated ${totalSuccess} records!`); + } + + if (totalError > 0) { + console.log( + `\n⚠️ ${totalError} records failed. Check the logs above for details.`, + ); + } +} + +// Export for use in main script +if (require.main === module) { + // This file is not meant to be run directly + console.log( + "This is a module for batch translation processing. Import it from the main script.", + ); +} diff --git a/apps/api/src/scripts/translation-audit.ts b/apps/api/src/scripts/translation-audit.ts new file mode 100644 index 00000000..51ab7367 --- /dev/null +++ b/apps/api/src/scripts/translation-audit.ts @@ -0,0 +1,1471 @@ +#!/usr/bin/env ts-node +/* eslint-disable */ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { HumanMessage } from "@langchain/core/messages"; +import { PrismaClient } from "@prisma/client"; +import * as cld from "cld"; +import { OpenAiLlmMiniService } from "src/api/llm/core/services/openai-llm-mini.service"; +import { createLogger } from "winston"; +import { PromptProcessorService } from "../api/llm/core/services/prompt-processor.service"; +import { TokenCounterService } from "../api/llm/core/services/token-counter.service"; +import { TranslationService } from "../api/llm/features/translation/services/translation.service"; +import { markForRetranslationBatch } from "./translation-audit-batch"; + +const prisma = new PrismaClient(); + +// Initialize the project's translation service +const logger = createLogger({ + level: "error", + silent: true, // Keep quiet for CLI usage +}); + +const promptProcessor = new PromptProcessorService( + {} as any, // We don't need the LLM router for language detection + {} as any, // We don't need usage tracker for language detection + logger, +); + +const tokenCounter = new TokenCounterService(logger); +const gpt4MiniService = new OpenAiLlmMiniService(tokenCounter, logger); + +// eslint-disable-next-line unicorn/prefer-module +const LANGUAGES_FILE_PATH = path.join( + __dirname, + "../api/assignment/attempt/helper/languages.json", +); + +const supportedLanguages: Array<{ code: string; name: string }> = JSON.parse( + fs.readFileSync(LANGUAGES_FILE_PATH, "utf8"), +); +const SUPPORTED_LANGUAGE_CODES = supportedLanguages.map((lang) => + lang.code.toLowerCase(), +); + +console.log( + `📋 Loaded ${ + SUPPORTED_LANGUAGE_CODES.length + } supported languages: ${SUPPORTED_LANGUAGE_CODES.join(", ")}`, +); + +interface BadTranslation { + id: number; + questionId: number | null; + variantId: number | null; + languageCode: string; + detectedLanguage: string; + translatedText: string; + translatedChoices: any; + issue: string; +} + +/** + * Filter detected language to only return supported languages + */ +function filterToSupportedLanguage(detectedLang: string): string { + if (!detectedLang || detectedLang === "unknown") { + return "unknown"; + } + + const normalizedDetected = normalizeLanguageCode(detectedLang); + + // Always allow English detection (even if not in supported list) + // This is important for detecting untranslated content + if (normalizedDetected === "en") { + return "en"; + } + + // Check if detected language is in our supported list + const isSupported = SUPPORTED_LANGUAGE_CODES.some( + (supportedCode) => + normalizeLanguageCode(supportedCode) === normalizedDetected, + ); + + return isSupported ? normalizedDetected : "unknown"; +} + +/** + * Check if two languages are similar enough to be considered equivalent for choice text + */ +function areSimilarLanguages(detected: string, expected: string): boolean { + const similar = { + // Romance language confusion groups + fr: ["it", "es", "pt"], + it: ["fr", "es", "pt"], + es: ["pt", "fr", "it"], + pt: ["es", "fr", "it"], + + // Chinese variants + "zh-cn": ["zh-tw", "zh"], + "zh-tw": ["zh-cn", "zh"], + zh: ["zh-cn", "zh-tw"], + + // English variants + en: ["en-us", "en-gb", "en-ca"], + }; + + const normalizedDetected = normalizeLanguageCode(detected); + const normalizedExpected = normalizeLanguageCode(expected); + + if (normalizedDetected === normalizedExpected) { + return true; + } + + return similar[normalizedExpected]?.includes(normalizedDetected) || false; +} + +/** + * Use the project's translation service for language detection + * This provides the same enhanced multi-engine detection with caching + */ +async function detectLanguageRobust(text: string): Promise<{ + detected: string; + confidence: number; + engines: { cld?: string; openai?: string; patterns?: string }; + consensus: boolean; + rawDetections: { cld?: string; openai?: string; patterns?: string }; + cldConfidence?: number; +}> { + if (!text || text.trim().length === 0) { + return { + detected: "unknown", + confidence: 0, + engines: {}, + consensus: false, + rawDetections: {}, + }; + } + + const rawDetections: { cld?: string; openai?: string; patterns?: string } = + {}; + const engines: { cld?: string; openai?: string; patterns?: string } = {}; + + // Try CLD detection first + let cldLang = "unknown"; + let cldConfidence = 0; + try { + const cldResponse = await cld.detect(text); + cldLang = cldResponse.languages[0].code; + cldConfidence = cldResponse.languages[0].percent / 100; + rawDetections.cld = `${cldLang} (${Math.round(cldConfidence * 100)}%)`; + engines.cld = filterToSupportedLanguage(cldLang); + } catch { + rawDetections.cld = "error"; + engines.cld = "error"; + } + + // Use GPT-5-nano for more accurate language detection + let openaiLang = "unknown"; + let openaiConfidence = 0; + try { + const promptContent = `Detect the language of the following text and respond with ONLY the ISO 639-1 language code (e.g., 'en' for English, 'fr' for French, 'es' for Spanish, 'it' for Italian, 'zh-CN' for Simplified Chinese, 'zh-TW' for Traditional Chinese, etc.). + +If the text contains mixed languages, identify the primary language. If you cannot determine the language with confidence, respond with 'unknown'. + +Text: "${text}" + +Language code:`; + + const response = await gpt4MiniService.invoke([ + new HumanMessage(promptContent), + ]); + + const detectedCode = response.content?.trim().toLowerCase() || "unknown"; + + // Validate the response is a valid language code + if ( + detectedCode && + detectedCode !== "unknown" && + detectedCode.match(/^[a-z]{2}(-[a-z]{2,4})?$/i) + ) { + openaiLang = detectedCode; + openaiConfidence = 0.9; // High confidence for LLM detection + rawDetections.openai = `${openaiLang} (90%)`; + engines.openai = filterToSupportedLanguage(openaiLang); + } else { + rawDetections.openai = "unknown"; + engines.openai = "unknown"; + } + } catch (error) { + console.error("Error using GPT-5-nano for language detection:", error); + rawDetections.openai = "error"; + engines.openai = "error"; + } + + // Determine final detected language + let detected = "unknown"; + let confidence = 0; + let consensus = false; + + // Prefer OpenAI detection if available and confident + if (openaiLang !== "unknown" && openaiConfidence > 0) { + detected = filterToSupportedLanguage(openaiLang); + confidence = openaiConfidence; + // Check consensus between CLD and OpenAI + consensus = + normalizeLanguageCode(cldLang) === normalizeLanguageCode(openaiLang); + } else if (cldLang !== "unknown" && cldConfidence > 0.5) { + // Fall back to CLD if OpenAI failed + detected = filterToSupportedLanguage(cldLang); + confidence = cldConfidence; + consensus = false; + } + + return { + detected, + confidence, + engines, + consensus, + rawDetections, + cldConfidence, + }; +} + +/** + * Normalize language codes for comparison + */ +function normalizeLanguageCode(languageCode: string): string { + const code = languageCode.toLowerCase(); + + // Handle Chinese variants + if (code === "zh-cn" || code === "zh-hans") return "zh-cn"; + if (code === "zh-tw" || code === "zh-hant") return "zh-tw"; + if (code === "zh") return "zh-cn"; // Default Chinese to simplified + + // Handle other common variants + if (code.startsWith("en-")) return "en"; + if (code.startsWith("es-")) return "es"; + if (code.startsWith("fr-")) return "fr"; + if (code.startsWith("de-")) return "de"; + if (code.startsWith("pt-")) return "pt"; + + // Return base language code + return code.split("-")[0]; +} + +/** + * Check if text in translatedChoices has language mismatches with improved accuracy + */ +async function checkChoicesLanguage( + translatedChoices: any, + expectedLanguage: string, +): Promise<{ hasIssue: boolean; details: string; debugInfo?: string[] }> { + if (!translatedChoices) { + return { hasIssue: false, details: "No choices to check" }; + } + + let choices: any[] = []; + + // Handle different JSON structures + if (Array.isArray(translatedChoices)) { + choices = translatedChoices; + } else if (typeof translatedChoices === "object") { + // Could be a JSON object with choices array + choices = + translatedChoices.choices || translatedChoices.translatedChoices || []; + } + + if (choices.length === 0) { + return { hasIssue: false, details: "No choices found in JSON" }; + } + + const issues: string[] = []; + const debugInfo: string[] = []; + + // Process all choices in parallel + const choiceResults = await Promise.all( + choices.map(async (choice, index) => { + const results: { + choiceText?: { detection: any; issue?: string }; + feedback?: { detection: any; issue?: string }; + } = {}; + + // Check choice text and feedback in parallel + const [choiceTextDetection, feedbackDetection] = await Promise.all([ + choice.choice + ? detectLanguageRobust(choice.choice) + : Promise.resolve(null), + choice.feedback + ? detectLanguageRobust(choice.feedback) + : Promise.resolve(null), + ]); + + // Process choice text detection + if (choiceTextDetection && choice.choice) { + results.choiceText = { detection: choiceTextDetection }; + + if ( + choiceTextDetection.detected !== "unknown" && + choiceTextDetection.confidence > 0.5 && + !areSimilarLanguages(choiceTextDetection.detected, expectedLanguage) + ) { + results.choiceText.issue = `Choice ${ + index + 1 + } text: expected ${expectedLanguage}, detected ${ + choiceTextDetection.detected + } (${Math.round(choiceTextDetection.confidence * 100)}% confidence)`; + } + } + + // Process feedback detection + if (feedbackDetection && choice.feedback) { + results.feedback = { detection: feedbackDetection }; + + if ( + feedbackDetection.detected !== "unknown" && + feedbackDetection.confidence > 0.6 && + !areSimilarLanguages(feedbackDetection.detected, expectedLanguage) + ) { + results.feedback.issue = `Choice ${ + index + 1 + } feedback: expected ${expectedLanguage}, detected ${ + feedbackDetection.detected + } (${Math.round(feedbackDetection.confidence * 100)}% confidence)`; + } + } + + return { index, choice, results }; + }), + ); + + // Collect results + for (const { index, choice, results } of choiceResults) { + // Add debug info + if (results.choiceText) { + debugInfo.push( + `Choice ${index + 1} text: "${choice.choice}" -> ${JSON.stringify( + results.choiceText.detection.rawDetections, + )} (confidence: ${results.choiceText.detection.confidence.toFixed(2)})`, + ); + + if (results.choiceText.issue) { + issues.push(results.choiceText.issue); + } + } + + if (results.feedback) { + debugInfo.push( + `Choice ${index + 1} feedback: "${choice.feedback}" -> ${JSON.stringify( + results.feedback.detection.rawDetections, + )} (confidence: ${results.feedback.detection.confidence.toFixed(2)})`, + ); + + if (results.feedback.issue) { + issues.push(results.feedback.issue); + } + } + } + + return { + hasIssue: issues.length > 0, + details: + issues.length > 0 + ? issues.join("; ") + : "All choices match expected language", + debugInfo, + }; +} + +/** + * Find all translation records with language mismatches + */ +async function findBadTranslations( + isDebugMode = false, + limit?: number, + assignmentIds?: number[], + includeAll = false, +): Promise { + console.log("🔍 Scanning translation table for language mismatches...\n"); + + const queryOptions: any = { + orderBy: { id: "asc" }, + }; + + // Add assignment ID filtering if specified + if (assignmentIds && assignmentIds.length > 0) { + // First, get the question IDs and variant IDs for the specified assignments + const questionQuery: any = { + where: { + assignmentId: { + in: assignmentIds, + }, + }, + select: { + id: true, + variants: { + select: { + id: true, + }, + }, + }, + }; + + // Add active filtering unless includeAll flag is set + if (!includeAll) { + questionQuery.where.isDeleted = false; + // Need to include the assignment relation to check currentVersion + questionQuery.include = { + assignment: { + include: { + currentVersion: true, + }, + }, + variants: { + where: { + isDeleted: false, + }, + }, + }; + // Remove the select since we're using include now + delete questionQuery.select; + } + + let questions = await prisma.question.findMany(questionQuery); + + // Filter for active questions in active versions if not includeAll + if (!includeAll) { + questions = questions.filter( + (q: any) => + q.assignment?.currentVersion?.isActive === true && + q.assignment?.currentVersion?.isDraft === false, + ); + } + + const questionIds = questions.map((q: any) => q.id); + const variantIds = questions.flatMap((q: any) => + q.variants.map((v: any) => v.id), + ); + + const statusText = includeAll ? "all" : "active"; + console.log( + `📋 Found ${questionIds.length} ${statusText} questions and ${ + variantIds.length + } ${statusText} variants for assignment IDs: ${assignmentIds.join(", ")}`, + ); + + // Now filter translations based on these IDs + queryOptions.where = { + OR: [ + { + questionId: { + in: questionIds, + }, + }, + { + variantId: { + in: variantIds, + }, + }, + ], + }; + + // Use select for performance + queryOptions.select = { + id: true, + questionId: true, + variantId: true, + languageCode: true, + translatedText: true, + translatedChoices: true, + }; + } else { + // When not filtering by assignment, optionally filter globally for active questions and versions + if (!includeAll) { + queryOptions.where = { + OR: [ + { + // Translations for questions - only active questions from active versions + questionId: { + not: null, + }, + question: { + isDeleted: false, + assignment: { + currentVersion: { + isActive: true, + isDraft: false, + }, + }, + }, + }, + { + // Translations for variants - only active variants of active questions from active versions + variantId: { + not: null, + }, + variant: { + isDeleted: false, + variantOf: { + isDeleted: false, + assignment: { + currentVersion: { + isActive: true, + isDraft: false, + }, + }, + }, + }, + }, + ], + }; + + console.log( + "📋 Filtering globally for active questions and variants in published assignment versions", + ); + } else { + console.log( + "📋 Including ALL questions and variants (active, deleted, draft versions) - no filtering applied", + ); + } + + // Use select for better performance + queryOptions.select = { + id: true, + questionId: true, + variantId: true, + languageCode: true, + translatedText: true, + translatedChoices: true, + }; + } + + // Add limit if specified + if (limit && limit > 0) { + queryOptions.take = limit; + console.log(`📋 Limiting scan to first ${limit} translation records`); + } + + const translations = await prisma.translation.findMany(queryOptions); + + console.log(`📊 Found ${translations.length} translation records to check\n`); + + const badTranslations: BadTranslation[] = []; + const BATCH_SIZE = 10; // Process 10 translations at a time + + // Process translations in batches + for (let i = 0; i < translations.length; i += BATCH_SIZE) { + const batch = translations.slice( + i, + Math.min(i + BATCH_SIZE, translations.length), + ); + const batchNumber = Math.floor(i / BATCH_SIZE) + 1; + const totalBatches = Math.ceil(translations.length / BATCH_SIZE); + + console.log( + `📦 Processing batch ${batchNumber}/${totalBatches} (${batch.length} records)...`, + ); + + // Process all translations in this batch in parallel + const batchResults = await Promise.all( + batch.map(async (translation) => { + const issues: string[] = []; + const debugInfo: string[] = []; + + // Check main text and choices in parallel + const [mainTextResult, choicesResult] = await Promise.all([ + // Check main translated text + translation.translatedText + ? detectLanguageRobust(translation.translatedText) + : Promise.resolve(null), + // Check translated choices + translation.translatedChoices + ? checkChoicesLanguage( + translation.translatedChoices, + translation.languageCode, + ) + : Promise.resolve(null), + ]); + + // Process main text detection result + if (mainTextResult && translation.translatedText) { + if (isDebugMode) { + debugInfo.push( + `Main text: "${translation.translatedText.slice( + 0, + 100, + )}..." -> ${JSON.stringify( + mainTextResult.rawDetections, + )} (confidence: ${mainTextResult.confidence.toFixed(2)})`, + ); + } + + if ( + mainTextResult.detected !== "unknown" && + mainTextResult.confidence > 0.5 && + !areSimilarLanguages( + mainTextResult.detected, + translation.languageCode, + ) + ) { + issues.push( + `Main text: expected ${translation.languageCode}, detected ${ + mainTextResult.detected + } (${Math.round(mainTextResult.confidence * 100)}% confidence)`, + ); + } + } + + // Process choices detection result + if (choicesResult) { + if (choicesResult.hasIssue) { + issues.push(`Choices: ${choicesResult.details}`); + } + + if (isDebugMode && (issues.length > 0 || choicesResult.debugInfo)) { + debugInfo.push(...(choicesResult.debugInfo || [])); + } + } + + // Return result for this translation + if (issues.length > 0) { + return { + id: translation.id, + questionId: translation.questionId, + variantId: translation.variantId, + languageCode: translation.languageCode, + detectedLanguage: mainTextResult?.detected || "unknown", + translatedText: translation.translatedText, + translatedChoices: translation.translatedChoices, + issue: issues.join(" | "), + debugInfo: isDebugMode ? debugInfo : undefined, + }; + } + + // Return debug info even if no issues found (for debug mode) + if (isDebugMode && debugInfo.length > 0) { + return { + id: translation.id, + languageCode: translation.languageCode, + debugInfo, + hasNoIssues: true, + }; + } + + return null; + }), + ); + + // Process batch results + for (const result of batchResults) { + if (result && !result.hasNoIssues) { + badTranslations.push(result as BadTranslation); + } + + // Show debug info if available + if (isDebugMode && result?.debugInfo && result.debugInfo.length > 0) { + console.log(`\n🐛 DEBUG Record ${result.id}:`); + console.log(` Expected: ${result.languageCode}`); + console.log( + ` Issues: ${ + result.hasNoIssues ? "None" : (result as BadTranslation).issue + }`, + ); + for (const info of result.debugInfo) console.log(` ${info}`); + } + } + + // Small delay between batches to avoid overwhelming the API + if (i + BATCH_SIZE < translations.length) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + return badTranslations; +} + +/** + * Delete translation records only (without retranslating) + */ +async function deleteTranslationsOnly( + translationIds: number[], + includeAll = false, +): Promise { + console.log(`🗑️ Deleting ${translationIds.length} translation records...\n`); + + const deletedRecords: Array<{ + id: number; + questionId?: number; + variantId?: number; + languageCode: string; + }> = []; + + for (const translationId of translationIds) { + try { + // Get the translation record before deletion for logging, including active status checks + const translation = await prisma.translation.findUnique({ + where: { id: translationId }, + include: { + question: { + include: { + assignment: { + include: { + currentVersion: true, + }, + }, + }, + }, + variant: { + include: { + variantOf: { + include: { + assignment: { + include: { + currentVersion: true, + }, + }, + }, + }, + }, + }, + }, + }); + + if (!translation) { + console.log(`❌ Translation record ${translationId} not found`); + continue; + } + + // Check if the translation is for an active question/variant (unless includeAll flag is set) + if (!includeAll) { + let isActive = false; + if (translation.question) { + isActive = + !translation.question.isDeleted && + translation.question.assignment?.currentVersion?.isActive === + true && + translation.question.assignment?.currentVersion?.isDraft === false; + } else if (translation.variant) { + isActive = + !translation.variant.isDeleted && + !translation.variant.variantOf.isDeleted && + translation.variant.variantOf.assignment?.currentVersion + ?.isActive === true && + translation.variant.variantOf.assignment?.currentVersion + ?.isDraft === false; + } + + if (!isActive) { + console.log( + `⚠️ Skipping deletion of translation ${translationId} - associated question/variant is not active or in an inactive assignment version`, + ); + continue; + } + } + + // Delete the record + await prisma.translation.delete({ + where: { id: translationId }, + }); + + deletedRecords.push({ + id: translation.id, + questionId: translation.questionId || undefined, + variantId: translation.variantId || undefined, + languageCode: translation.languageCode, + }); + + console.log( + `✅ Deleted translation record ${translationId} (${translation.languageCode})`, + ); + } catch (error) { + console.log(`❌ Error deleting record ${translationId}:`, error); + } + } + + if (deletedRecords.length > 0) { + console.log( + `\n🔄 Deleted ${deletedRecords.length} translation records. These will be regenerated automatically when:`, + ); + console.log( + ` 1. A learner requests the assignment in the target language`, + ); + console.log(` 2. You manually trigger translation via the API`); + console.log(` 3. Assignment translation job is run`); + + console.log(`\n📋 Deleted records summary:`); + for (const record of deletedRecords) { + if (record.questionId) { + console.log( + ` Question ${record.questionId} -> ${record.languageCode} (Translation ID: ${record.id})`, + ); + } else if (record.variantId) { + console.log( + ` Variant ${record.variantId} -> ${record.languageCode} (Translation ID: ${record.id})`, + ); + } + } + } else { + console.log(`\n📋 No records were deleted.`); + } +} + +/** + * Delete all translation records for a specific assignment + */ +async function deleteAssignmentTranslations( + assignmentId: number, + includeAll = false, +): Promise { + console.log( + `🗑️ Deleting ALL translation records for assignment ${assignmentId}...\n`, + ); + + try { + // Get all questions and variants for this assignment + const questionQuery: any = { + where: { + assignmentId: assignmentId, + }, + select: { + id: true, + question: true, + variants: { + select: { + id: true, + variantContent: true, + }, + }, + }, + }; + + // Add active filtering unless includeAll flag is set + if (!includeAll) { + questionQuery.where.isDeleted = false; + questionQuery.include = { + assignment: { + include: { + currentVersion: true, + }, + }, + variants: { + where: { + isDeleted: false, + }, + select: { + id: true, + variantContent: true, + }, + }, + }; + delete questionQuery.select; + } + + const questions = await prisma.question.findMany(questionQuery); + + // Filter for active questions in active versions if not includeAll + const validQuestions = includeAll + ? questions + : questions.filter( + (q: any) => + q.assignment?.currentVersion?.isActive === true && + q.assignment?.currentVersion?.isDraft === false, + ); + + const questionIds = validQuestions.map((q: any) => q.id); + const variantIds = validQuestions.flatMap((q: any) => + q.variants.map((v: any) => v.id), + ); + + console.log( + `📋 Found ${questionIds.length} questions and ${variantIds.length} variants for assignment ${assignmentId}`, + ); + + // Step 1: Delete translation records for questions + let deletedQuestionTranslations = 0; + if (questionIds.length > 0) { + const questionTranslationResult = await prisma.translation.deleteMany({ + where: { + questionId: { + in: questionIds, + }, + }, + }); + deletedQuestionTranslations = questionTranslationResult.count; + console.log( + `✅ Deleted ${deletedQuestionTranslations} question translation records`, + ); + } + + // Step 2: Delete translation records for variants + let deletedVariantTranslations = 0; + if (variantIds.length > 0) { + const variantTranslationResult = await prisma.translation.deleteMany({ + where: { + variantId: { + in: variantIds, + }, + }, + }); + deletedVariantTranslations = variantTranslationResult.count; + console.log( + `✅ Deleted ${deletedVariantTranslations} variant translation records`, + ); + } + + // Step 3: Delete assignmentTranslation records + const assignmentTranslationQuery: any = { + where: { + assignmentId: assignmentId, + }, + }; + + // Add active filtering for assignmentTranslation if not includeAll + if (!includeAll) { + assignmentTranslationQuery.where.assignment = { + currentVersion: { + isActive: true, + isDraft: false, + }, + }; + } + + const assignmentTranslationResult = + await prisma.assignmentTranslation.deleteMany(assignmentTranslationQuery); + const deletedAssignmentTranslations = assignmentTranslationResult.count; + console.log( + `✅ Deleted ${deletedAssignmentTranslations} assignment translation records`, + ); + + // Summary + const totalDeleted = + deletedQuestionTranslations + + deletedVariantTranslations + + deletedAssignmentTranslations; + + console.log(`\n📊 Deletion Summary for Assignment ${assignmentId}:`); + console.log(` 🗂️ Question translations: ${deletedQuestionTranslations}`); + console.log(` 🔄 Variant translations: ${deletedVariantTranslations}`); + console.log( + ` 📋 Assignment translations: ${deletedAssignmentTranslations}`, + ); + console.log(` 📊 Total deleted: ${totalDeleted}`); + + if (totalDeleted > 0) { + console.log( + `\n🔄 All translation records for assignment ${assignmentId} have been deleted.`, + ); + console.log(` These will be regenerated automatically when:`); + console.log( + ` 1. A learner requests the assignment in any target language`, + ); + console.log(` 2. You manually trigger translation via the API`); + console.log(` 3. Assignment translation job is run`); + } else { + console.log( + `\n📋 No translation records found for assignment ${assignmentId}.`, + ); + } + } catch (error) { + console.error( + `❌ Error deleting translations for assignment ${assignmentId}:`, + error, + ); + throw error; + } +} + +/** + * Delete bad translation records so they can be regenerated + */ +async function deleteAndRetranslate( + translationIds: number[], + includeAll = false, +): Promise { + console.log( + `🗑️ Deleting ${translationIds.length} bad translation records...\n`, + ); + + const deletedRecords: Array<{ + id: number; + questionId?: number; + variantId?: number; + languageCode: string; + }> = []; + + for (const translationId of translationIds) { + try { + // Get the translation record before deletion for logging, including active status checks + const translation = await prisma.translation.findUnique({ + where: { id: translationId }, + include: { + question: { + include: { + assignment: { + include: { + currentVersion: true, + }, + }, + }, + }, + variant: { + include: { + variantOf: { + include: { + assignment: { + include: { + currentVersion: true, + }, + }, + }, + }, + }, + }, + }, + }); + + if (!translation) { + console.log(`❌ Translation record ${translationId} not found`); + continue; + } + + // Check if the translation is for an active question/variant (unless includeAll flag is set) + if (!includeAll) { + let isActive = false; + if (translation.question) { + isActive = + !translation.question.isDeleted && + translation.question.assignment?.currentVersion?.isActive === + true && + translation.question.assignment?.currentVersion?.isDraft === false; + } else if (translation.variant) { + isActive = + !translation.variant.isDeleted && + !translation.variant.variantOf.isDeleted && + translation.variant.variantOf.assignment?.currentVersion + ?.isActive === true && + translation.variant.variantOf.assignment?.currentVersion + ?.isDraft === false; + } + + if (!isActive) { + console.log( + `⚠️ Skipping deletion of translation ${translationId} - associated question/variant is not active or in an inactive assignment version`, + ); + continue; + } + } + + // Delete the record + await prisma.translation.delete({ + where: { id: translationId }, + }); + + deletedRecords.push({ + id: translation.id, + questionId: translation.questionId || undefined, + variantId: translation.variantId || undefined, + languageCode: translation.languageCode, + }); + + console.log( + `✅ Deleted translation record ${translationId} (${translation.languageCode})`, + ); + } catch (error) { + console.log(`❌ Error deleting record ${translationId}:`, error); + } + } + + if (deletedRecords.length > 0) { + console.log( + `\n🔄 Deleted ${deletedRecords.length} translation records. These will be regenerated automatically when:`, + ); + console.log( + ` 1. A learner requests the assignment in the target language`, + ); + console.log(` 2. You manually trigger translation via the API`); + console.log(` 3. Assignment translation job is run`); + + console.log(`\n📋 Deleted records summary:`); + for (const record of deletedRecords) { + if (record.questionId) { + console.log( + ` Question ${record.questionId} -> ${record.languageCode} (Translation ID: ${record.id})`, + ); + } else if (record.variantId) { + console.log( + ` Variant ${record.variantId} -> ${record.languageCode} (Translation ID: ${record.id})`, + ); + } + } + } +} + +/** + * Main CLI function + */ +async function main() { + const command = process.argv[2]; + const isDebugMode = process.argv.includes("--debug"); + const includeAll = process.argv.includes("--include-all"); + + // Parse limit parameter (--limit=N) + let limit: number | undefined; + const limitArg = process.argv.find((arg) => arg.startsWith("--limit=")); + if (limitArg) { + const limitValue = parseInt(limitArg.split("=")[1]); + if (!isNaN(limitValue) && limitValue > 0) { + limit = limitValue; + } else { + console.log( + "❌ Invalid limit value. Please use --limit=N where N is a positive number.", + ); + return; + } + } + + // Parse assignment IDs parameter (--assignments=1,2,3) + let assignmentIds: number[] | undefined; + const assignmentsArg = process.argv.find((arg) => + arg.startsWith("--assignments="), + ); + if (assignmentsArg) { + const assignmentValues = assignmentsArg + .split("=")[1] + .split(",") + .map((id) => parseInt(id.trim())); + if (assignmentValues.every((id) => !isNaN(id) && id > 0)) { + assignmentIds = assignmentValues; + } else { + console.log( + "❌ Invalid assignment IDs. Please use --assignments=1,2,3 where all values are positive numbers.", + ); + return; + } + } + + if (isDebugMode) { + console.log("🐛 Debug mode enabled - showing detailed detection info\n"); + } + + if (includeAll) { + console.log( + "🌐 Include-all mode enabled - scanning ALL questions (active, deleted, draft versions)\n", + ); + } + + if (limit) { + console.log(`📏 Limit set to ${limit} records\n`); + } + + if (assignmentIds && assignmentIds.length > 0) { + console.log( + `🎯 Filtering to assignment IDs: ${assignmentIds.join(", ")}\n`, + ); + } + + switch (command) { + case "find-bad": { + try { + const badTranslations = await findBadTranslations( + isDebugMode, + limit, + assignmentIds, + includeAll, + ); + + console.log( + `\n📊 RESULTS: Found ${badTranslations.length} translation records with language mismatches\n`, + ); + + if (badTranslations.length > 0) { + console.log("💾 Bad Translation Record IDs:"); + console.log(badTranslations.map((bt) => bt.id).join(",")); + + console.log("\n📋 Detailed Report:"); + console.log("─".repeat(120)); + console.log( + "ID".padEnd(8) + + "Q ID".padEnd(8) + + "V ID".padEnd(8) + + "Expected".padEnd(12) + + "Detected".padEnd(12) + + "Issue", + ); + console.log("─".repeat(120)); + + for (const bt of badTranslations) { + console.log( + String(bt.id).padEnd(8) + + String(bt.questionId || "N/A").padEnd(8) + + String(bt.variantId || "N/A").padEnd(8) + + bt.languageCode.padEnd(12) + + bt.detectedLanguage.padEnd(12) + + bt.issue.slice(0, 60) + + (bt.issue.length > 60 ? "..." : ""), + ); + } + + if (badTranslations.length > 0) { + console.log(`\n💡 Next steps:`); + console.log(` 1. Review the bad translations above`); + console.log( + ` 2. Retranslate them: npm run translation-audit retranslate ${badTranslations + .map((bt) => bt.id) + .join(",")}`, + ); + console.log( + ` 3. Or delete and regenerate: npm run translation-audit delete-and-retranslate `, + ); + } + } else { + console.log( + "✅ No bad translations found! All translations match their expected language codes.", + ); + } + } catch (error) { + console.error("❌ Error finding bad translations:", error); + } + + break; + } + case "retranslate": { + const idsArgument = process.argv[3]; + if (!idsArgument) { + console.log( + "❌ Please provide comma-separated translation IDs to analyze for retranslation", + ); + console.log( + "Usage: npm run translation-audit retranslate 123,456,789 [--assignments=1,2,3]", + ); + return; + } + + try { + const translationIds = idsArgument + .split(",") + .map((id) => Number.parseInt(id.trim())) + .filter((id) => !isNaN(id)); + + if (translationIds.length === 0) { + console.log("❌ No valid translation IDs provided"); + return; + } + + // Extract single assignment ID for retranslation (use first one if multiple provided) + const singleAssignmentId = + assignmentIds && assignmentIds.length > 0 + ? assignmentIds[0] + : undefined; + + if (singleAssignmentId) { + console.log( + `🎯 Using assignment ID ${singleAssignmentId} for translation operations\n`, + ); + } + + // Use batch processing for better performance + await markForRetranslationBatch( + translationIds, + singleAssignmentId, + includeAll, + ); + } catch (error) { + console.error("❌ Error analyzing records for retranslation:", error); + } + + break; + } + case "delete-and-retranslate": { + const idsArgument = process.argv[3]; + if (!idsArgument) { + console.log( + "❌ Please provide comma-separated translation IDs to delete and retranslate", + ); + console.log( + "Usage: npm run translation-audit delete-and-retranslate 123,456,789", + ); + console.log( + "⚠️ WARNING: This will permanently delete the translation records!", + ); + return; + } + + // Safety confirmation + console.log( + "⚠️ WARNING: You are about to permanently delete translation records!", + ); + console.log( + " This action cannot be undone. The records will be regenerated automatically later.", + ); + console.log( + " Press Ctrl+C to cancel or wait 5 seconds to continue...\n", + ); + + await new Promise((resolve) => setTimeout(resolve, 5000)); + + try { + const translationIds = idsArgument + .split(",") + .map((id) => Number.parseInt(id.trim())) + .filter((id) => !isNaN(id)); + + if (translationIds.length === 0) { + console.log("❌ No valid translation IDs provided"); + return; + } + + await deleteAndRetranslate(translationIds, includeAll); + } catch (error) { + console.error("❌ Error deleting and retranslating records:", error); + } + + break; + } + case "delete-only": { + const idsArgument = process.argv[3]; + if (!idsArgument) { + console.log( + "❌ Please provide comma-separated translation IDs to delete", + ); + console.log("Usage: npm run translation-audit delete-only 123,456,789"); + console.log( + "⚠️ WARNING: This will permanently delete the translation records!", + ); + return; + } + + // Safety confirmation + console.log( + "⚠️ WARNING: You are about to permanently delete translation records!", + ); + console.log( + " This action cannot be undone. The records will be regenerated automatically later.", + ); + console.log( + " Press Ctrl+C to cancel or wait 5 seconds to continue...\n", + ); + + await new Promise((resolve) => setTimeout(resolve, 5000)); + + try { + const translationIds = idsArgument + .split(",") + .map((id) => Number.parseInt(id.trim())) + .filter((id) => !isNaN(id)); + + if (translationIds.length === 0) { + console.log("❌ No valid translation IDs provided"); + return; + } + + await deleteTranslationsOnly(translationIds, includeAll); + } catch (error) { + console.error("❌ Error deleting records:", error); + } + + break; + } + case "delete-assignment": { + const assignmentIdArg = process.argv[3]; + if (!assignmentIdArg) { + console.log("❌ Please provide an assignment ID"); + console.log( + "Usage: npm run translation-audit delete-assignment 123 [--include-all]", + ); + console.log( + "⚠️ WARNING: This will delete ALL translation records for the assignment!", + ); + return; + } + + const assignmentIdToDelete = Number.parseInt(assignmentIdArg.trim()); + if (isNaN(assignmentIdToDelete)) { + console.log("❌ Invalid assignment ID provided"); + return; + } + + // Safety confirmation + console.log( + "🚨 DANGER: You are about to delete ALL translation records for assignment " + + assignmentIdToDelete + + "!", + ); + console.log(" This includes:"); + console.log(" - All question translations"); + console.log(" - All variant translations"); + console.log(" - All assignment-level translations"); + console.log(" This action cannot be undone!"); + console.log( + " Press Ctrl+C to cancel or wait 10 seconds to continue...\n", + ); + + await new Promise((resolve) => setTimeout(resolve, 10000)); + + try { + await deleteAssignmentTranslations(assignmentIdToDelete, includeAll); + } catch (error) { + console.error("❌ Error deleting assignment translations:", error); + } + + break; + } + default: { + console.log(` +🔧 Translation Audit CLI Tool + +Usage: + npm run translation-audit find-bad [--debug] [--limit=N] [--assignments=1,2,3] [--include-all] - Find translation records with language mismatches + npm run translation-audit retranslate 123,456,789 [--assignments=N] [--include-all] - Retranslate specific record IDs using batch processing + npm run translation-audit delete-only 123,456,789 [--include-all] - Delete translation records only (they will regenerate) + npm run translation-audit delete-assignment 123 [--include-all] - Delete ALL translations for an assignment (DANGEROUS!) + npm run translation-audit delete-and-retranslate 123,456 [--assignments=1,2,3] [--include-all] - Delete bad translations (they will regenerate) + +Examples: + npm run translation-audit find-bad - Scan active translation records + npm run translation-audit find-bad --include-all - Scan ALL translation records (including deleted/draft) + npm run translation-audit find-bad --debug - Show detailed detection info for troubleshooting + npm run translation-audit find-bad --limit=100 - Scan only the first 100 records + npm run translation-audit find-bad --assignments=42,55 - Scan only translations from assignments 42 and 55 + npm run translation-audit find-bad --debug --limit=50 --assignments=42 - Debug mode with 50 record limit for assignment 42 + npm run translation-audit retranslate 45,67,89 - Actually retranslate and update database + npm run translation-audit retranslate 45,67,89 --assignments=42 - Retranslate using assignment 42's context for better quality + npm run translation-audit retranslate 45,67,89 --include-all - Retranslate including inactive questions/variants + npm run translation-audit delete-only 45,67,89 - Delete translation records only (no retranslation) + npm run translation-audit delete-assignment 42 - Delete ALL translations for assignment 42 (DANGEROUS!) + npm run translation-audit delete-assignment 42 --include-all - Delete ALL translations including inactive versions + npm run translation-audit delete-and-retranslate 45,67,89 + +Options: + --debug Show detailed language detection information for troubleshooting + --limit=N Limit scanning to first N translation records (useful for testing/sampling) + --assignments=1,2 Filter scanning to specific assignment IDs, or provide assignment context for retranslation + --include-all Include ALL questions (active, deleted, draft versions) - no filtering + +Workflow: + 1. Run 'find-bad' to identify problematic translations (active questions only by default) + 2. Use '--include-all' to scan deleted questions and draft assignment versions + 3. Use '--limit' for quick testing or large databases + 4. Use '--debug' flag to see detailed language detection info + 5. Run 'retranslate' to fix specific records using GPT-5-nano + 6. Run 'delete-and-retranslate' to remove bad translations (safe - they regenerate automatically) + +Environment Variables: + DATABASE_URL - PostgreSQL connection string (required) + OPENAI_API_KEY - Required for 'retranslate' command and enhanced language detection via project's TranslationService +`); + } + } + + await prisma.$disconnect(); +} + +// Handle graceful shutdown +process.on("SIGINT", async () => { + console.log("\n🛑 Shutting down gracefully..."); + await prisma.$disconnect(); + process.exit(0); +}); + +process.on("SIGTERM", async () => { + console.log("\n🛑 Shutting down gracefully..."); + await prisma.$disconnect(); + process.exit(0); +}); + +// Run the CLI +if (require.main === module) { + main().catch((error) => { + console.error("❌ Fatal error:", error); + process.exit(1); + }); +} diff --git a/apps/api/src/shared.module.ts b/apps/api/src/shared.module.ts index 9ff3a939..0330bcd9 100644 --- a/apps/api/src/shared.module.ts +++ b/apps/api/src/shared.module.ts @@ -1,9 +1,9 @@ -import { Global, Module } from "@nestjs/common"; import { HttpModule } from "@nestjs/axios"; +import { Global, Module } from "@nestjs/common"; import { PrismaService } from "src/prisma.service"; +import { JobStatusServiceV2 } from "./api/assignment/v2/services/job-status.service"; import { TranslationService } from "./api/assignment/v2/services/translation.service"; import { LlmModule } from "./api/llm/llm.module"; -import { JobStatusServiceV2 } from "./api/assignment/v2/services/job-status.service"; @Global() @Module({ diff --git a/apps/web/.eslintrc.cjs b/apps/web/.eslintrc.cjs index bdf79997..970a0419 100644 --- a/apps/web/.eslintrc.cjs +++ b/apps/web/.eslintrc.cjs @@ -15,14 +15,24 @@ module.exports = { }, }, settings: { - 'import/resolver': { + "import/resolver": { typescript: { - project: './tsconfig.json', + project: "./tsconfig.json", }, }, }, plugins: ["@typescript-eslint"], rules: { + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-unsafe-assignment": "off", + "@typescript-eslint/restrict-plus-operands": "off", + "@typescript-eslint/require-await": "off", + "@typescript-eslint/no-unsafe-member-access": "off", + "@typescript-eslint/no-unsafe-call": "off", + "@typescript-eslint/no-unsafe-return": "off", + "@typescript-eslint/no-floating-promises": "off", + "@typescript-eslint/restrict-template-expressions": "off", + "@typescript-eslint/no-unsafe-argument": "off", "@typescript-eslint/no-non-null-assertion": "off", "@typescript-eslint/no-misused-promises": [ "error", diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index f64a6d48..453d4172 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -3,7 +3,6 @@ ARG BASE_IMAGE=node:22-alpine FROM ${BASE_IMAGE} AS builder -ARG SN_GITHUB_NPM_TOKEN ARG DIR=/usr/src/app # Pruning using turbo for the Next.js app @@ -18,27 +17,32 @@ ARG DIR=/usr/src/app WORKDIR $DIR COPY --from=builder $DIR/ . COPY --from=builder /usr/local/share/.config/yarn/global /usr/local/share/.config/yarn/global -RUN yarn install --ignore-scripts --frozen-lockfile --network-timeout 600000 && yarn cache clean +RUN yarn install --ignore-scripts --frozen-lockfile && yarn cache clean # Running build using turbo for the Next.js app FROM installer AS sourcer WORKDIR $DIR COPY --from=builder /usr/local/share/.config/yarn/global /usr/local/share/.config/yarn/global -RUN yarn build --filter=web && yarn install --production --ignore-scripts --frozen-lockfile --network-timeout 600000 && yarn cache clean +RUN yarn build --filter=web && yarn install --production --ignore-scripts --frozen-lockfile && yarn cache clean # Production stage FROM ${BASE_IMAGE} AS production ARG DIR=/usr/src/app ENV NODE_ENV production + WORKDIR $DIR -RUN apk add --no-cache dumb-init +RUN ls + +RUN apk add --no-cache dumb-init~=1 COPY --chown=node:node --from=sourcer $DIR/apps/web/.next ./.next -COPY --chown=node:node --from=sourcer $DIR/package.json $DIR/package.json +COPY --chown=node:node --from=sourcer $DIR/apps/web/package.json ./package.json COPY --chown=node:node --from=sourcer $DIR/node_modules $DIR/node_modules COPY --chown=node:node --from=sourcer $DIR/apps/web/public ./public COPY --chown=node:node --from=sourcer $DIR/apps/web/entrypoint.sh ./entrypoint.sh +ENV NODE_OPTIONS="--require ./node_modules/@instana/collector/src/immediate" + ENTRYPOINT ["/usr/bin/dumb-init", "--"] -CMD ["./entrypoint.sh"] +CMD ["sh","-c","NODE_OPTIONS='--require ./node_modules/@instana/collector/src/immediate' ./entrypoint.sh"] EXPOSE 3000 # Patched stage with updates diff --git a/apps/web/app/Helpers/checkDiff.ts b/apps/web/app/Helpers/checkDiff.ts index d5fe0674..e5498a8d 100644 --- a/apps/web/app/Helpers/checkDiff.ts +++ b/apps/web/app/Helpers/checkDiff.ts @@ -73,6 +73,7 @@ export function useChangesSummary(): string { showQuestionScore, showAssignmentScore, showQuestions, + showCorrectAnswer, } = useAssignmentFeedbackConfig(); const changesSummary = useMemo(() => { @@ -112,6 +113,14 @@ export function useChangesSummary(): string { ) diffs.push("Changed assignment score visibility."); + if ( + !safeCompare( + showCorrectAnswer, + originalAssignment.showCorrectAnswer ?? true, + ) + ) + diffs.push("Changed correct answer visibility."); + // check if question order is different if (!safeArrayCompare(questionOrder, originalAssignment.questionOrder)) { diffs.push("Modified question order."); diff --git a/apps/web/app/Helpers/checkQuestionsReady.ts b/apps/web/app/Helpers/checkQuestionsReady.ts index d7d9daac..435e67f9 100644 --- a/apps/web/app/Helpers/checkQuestionsReady.ts +++ b/apps/web/app/Helpers/checkQuestionsReady.ts @@ -1,6 +1,3 @@ -import { useAssignmentConfig } from "@/stores/assignmentConfig"; -import { useAuthorStore } from "@/stores/author"; -import { useCallback } from "react"; import type { Choice, Question, @@ -9,6 +6,9 @@ import type { Scoring, } from "../../config/types"; import { useDebugLog } from "../../lib/utils"; +import { useAssignmentConfig } from "@/stores/assignmentConfig"; +import { useAuthorStore } from "@/stores/author"; +import { useCallback } from "react"; interface ValidationError { message: string; @@ -51,10 +51,7 @@ export const useQuestionsAreReadyToBePublished = ( }; } if (q.assignmentId == null) { - return { - message: `Question ${index + 1} assignment ID is missing.`, - step: 0, - }; + q.assignmentId = useAuthorStore.getState().activeAssignmentId; } return null; }; @@ -236,7 +233,6 @@ export const useQuestionsAreReadyToBePublished = ( for (let i = 0; i < questions.length; i++) { const q = questions[i]; - debugLog(`Checking question ${i + 1}:`, q); const basicError = validateBasicFields(q, i); if (basicError) { @@ -279,8 +275,6 @@ export const useQuestionsAreReadyToBePublished = ( break; } } - - debugLog(`Question ${i + 1} passed all checks.`); } if (isValid) { @@ -317,7 +311,7 @@ export const useQuestionsAreReadyToBePublished = ( } if ( !assignmentConfig.displayOrder && - assignmentConfig.numberOfQuestionsPerAttempt === null + assignmentConfig.numberOfQuestionsPerAttempt !== null ) { message = `Question order is required.`; debugLog(message); diff --git a/apps/web/app/Helpers/fileReader.ts b/apps/web/app/Helpers/fileReader.ts index 6f9fd46c..cf0406e9 100644 --- a/apps/web/app/Helpers/fileReader.ts +++ b/apps/web/app/Helpers/fileReader.ts @@ -1,5 +1,4 @@ /* eslint-disable */ - import JSZip from "jszip"; import mammoth from "mammoth"; import Papa, { ParseResult } from "papaparse"; diff --git a/apps/web/app/Helpers/getLanguageName.ts b/apps/web/app/Helpers/getLanguageName.ts index ddfa0109..1b323dcd 100644 --- a/apps/web/app/Helpers/getLanguageName.ts +++ b/apps/web/app/Helpers/getLanguageName.ts @@ -7,4 +7,8 @@ export const getLanguageName = (code: string) => { const language = languages.find((lang) => lang.code === code); return language ? language.name : code.toUpperCase(); }; +export const getLanguageCode = (name: string) => { + const language = languages.find((lang) => lang.name === name); + return language ? language.code : null; +}; export const AVAILABLE_LANGUAGES = languages.map((lang) => lang.code); diff --git a/apps/web/app/admin/components/AdminDashboard.tsx b/apps/web/app/admin/components/AdminDashboard.tsx new file mode 100644 index 00000000..54bfef9f --- /dev/null +++ b/apps/web/app/admin/components/AdminDashboard.tsx @@ -0,0 +1,1246 @@ +"use client"; + +import { useState } from "react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { ReactQueryDevtools } from "@tanstack/react-query-devtools"; +import { FeedbackTable } from "./FeedbackTable"; +import { ReportsTable } from "./ReportsTable"; +import { AssignmentAnalyticsTable } from "./AssignmentAnalyticsTable"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { cn } from "@/lib/utils"; +import { + useDashboardStats, + useCurrentPriceUpscaling, + useUpscalePricing, + useRemovePriceUpscaling, + useRefreshDashboard, +} from "@/hooks/useAdminDashboard"; +import { + Settings, + RefreshCw, + TrendingUp, + AlertTriangle, + Calculator, + RotateCcw, + Calendar, + ChevronDown, +} from "lucide-react"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import Link from "next/link"; +import { queryClient } from "@/lib/query-client"; + +interface AdminDashboardProps { + sessionToken?: string | null; + onLogout?: () => void; +} + +function AdminDashboardContent({ + sessionToken, + onLogout, +}: AdminDashboardProps) { + const [activeTab, setActiveTab] = useState< + "feedback" | "reports" | "assignments" + >("assignments"); + const [filters, setFilters] = useState<{ + startDate?: string; + endDate?: string; + assignmentId?: number; + assignmentName?: string; + userId?: string; + }>({}); + const [datePreset, setDatePreset] = useState("all"); + const [customDateRange, setCustomDateRange] = useState<{ + start: string; + end: string; + }>({ start: "", end: "" }); + const [showCustomDatePopover, setShowCustomDatePopover] = useState(false); + const [quickActionResults, setQuickActionResults] = useState( + null, + ); + const [quickActionTitle, setQuickActionTitle] = useState(""); + + // Price upscaling modal state + const [isPriceUpscalingModalOpen, setIsPriceUpscalingModalOpen] = + useState(false); + const [globalUpscalingFactor, setGlobalUpscalingFactor] = useState(""); + const [usageTypeUpscaling, setUsageTypeUpscaling] = useState({ + TRANSLATION: "", + QUESTION_GENERATION: "", + ASSIGNMENT_GENERATION: "", + LIVE_RECORDING_FEEDBACK: "", + GRADING_VALIDATION: "", + ASSIGNMENT_GRADING: "", + OTHER: "", + }); + + // TanStack Query hooks + const { + data: stats, + isLoading: loadingStats, + error: statsError, + } = useDashboardStats(sessionToken, filters); + + const { data: currentUpscaling, isLoading: loadingUpscaling } = + useCurrentPriceUpscaling(sessionToken); + + const upscalePricingMutation = useUpscalePricing(sessionToken); + const removePricingMutation = useRemovePriceUpscaling(sessionToken); + const refreshDashboard = useRefreshDashboard(sessionToken); + + const handleFiltersChange = (newFilters: typeof filters) => { + setFilters(newFilters); + }; + + const handleDatePresetChange = (preset: string) => { + setDatePreset(preset); + const now = new Date(); + const today = new Date(now.getFullYear(), now.getMonth(), now.getDate()); + + const newFilters = { ...filters }; + + switch (preset) { + case "today": { + newFilters.startDate = today.toISOString(); + newFilters.endDate = new Date( + today.getTime() + 24 * 60 * 60 * 1000, + ).toISOString(); + break; + } + case "yesterday": { + const yesterday = new Date(today.getTime() - 24 * 60 * 60 * 1000); + newFilters.startDate = yesterday.toISOString(); + newFilters.endDate = today.toISOString(); + break; + } + case "last7days": { + newFilters.startDate = new Date( + today.getTime() - 7 * 24 * 60 * 60 * 1000, + ).toISOString(); + newFilters.endDate = new Date( + today.getTime() + 24 * 60 * 60 * 1000, + ).toISOString(); + break; + } + case "last30days": { + newFilters.startDate = new Date( + today.getTime() - 30 * 24 * 60 * 60 * 1000, + ).toISOString(); + newFilters.endDate = new Date( + today.getTime() + 24 * 60 * 60 * 1000, + ).toISOString(); + break; + } + case "thisMonth": { + newFilters.startDate = new Date( + now.getFullYear(), + now.getMonth(), + 1, + ).toISOString(); + newFilters.endDate = new Date( + now.getFullYear(), + now.getMonth() + 1, + 0, + 23, + 59, + 59, + ).toISOString(); + break; + } + case "lastMonth": { + newFilters.startDate = new Date( + now.getFullYear(), + now.getMonth() - 1, + 1, + ).toISOString(); + newFilters.endDate = new Date( + now.getFullYear(), + now.getMonth(), + 0, + 23, + 59, + 59, + ).toISOString(); + break; + } + case "custom": { + setShowCustomDatePopover(true); + return; + } + case "all": { + break; + } + default: { + delete newFilters.startDate; + delete newFilters.endDate; + break; + } + } + + setFilters(newFilters); + }; + + const handleCustomDateApply = () => { + if (customDateRange.start && customDateRange.end) { + setFilters({ + ...filters, + startDate: new Date(customDateRange.start).toISOString(), + endDate: new Date(customDateRange.end).toISOString(), + }); + setShowCustomDatePopover(false); + } + }; + + const formatDateRange = () => { + if (!filters.startDate || !filters.endDate) return "All time"; + + const start = new Date(filters.startDate); + const end = new Date(filters.endDate); + + if (datePreset === "today") return "Today"; + if (datePreset === "yesterday") return "Yesterday"; + if (datePreset === "last7days") return "Last 7 days"; + if (datePreset === "last30days") return "Last 30 days"; + if (datePreset === "thisMonth") return "This month"; + if (datePreset === "lastMonth") return "Last month"; + + return `${start.toLocaleDateString()} - ${end.toLocaleDateString()}`; + }; + + const handleQuickActionComplete = (result: any) => { + setQuickActionResults(result.data); + setQuickActionTitle(result.title); + setActiveTab("assignments"); + }; + + const handleRefresh = () => { + refreshDashboard(); + }; + + const isAdmin = stats?.userRole === "admin"; + + const clearQuickActionResults = () => { + setQuickActionResults(null); + setQuickActionTitle(""); + }; + + const handleUsageTypeUpscalingChange = ( + usageType: keyof typeof usageTypeUpscaling, + value: string, + ) => { + setUsageTypeUpscaling((prev) => ({ + ...prev, + [usageType]: value, + })); + }; + + const resetUpscalingModal = () => { + setGlobalUpscalingFactor(""); + setUsageTypeUpscaling({ + TRANSLATION: "", + QUESTION_GENERATION: "", + ASSIGNMENT_GENERATION: "", + LIVE_RECORDING_FEEDBACK: "", + GRADING_VALIDATION: "", + ASSIGNMENT_GRADING: "", + OTHER: "", + }); + }; + + // Example calculation for price impact + const calculatePriceExample = () => { + // Use real dashboard data when available, otherwise fall back to sample data + const useRealData = stats && stats.costBreakdown; + + const exampleUsage = useRealData + ? { + // Map real dashboard data to usage types with estimated token usage + TRANSLATION: { + inputTokens: 1500, + outputTokens: 800, + currentCost: + stats.costBreakdown.translation / + Math.max(stats.publishedAssignments, 1), + }, + QUESTION_GENERATION: { + inputTokens: 2000, + outputTokens: 1200, + currentCost: + stats.costBreakdown.questionGeneration / + Math.max(stats.publishedAssignments, 1), + }, + ASSIGNMENT_GENERATION: { + inputTokens: 800, + outputTokens: 1500, + currentCost: + (stats.costBreakdown.questionGeneration / + Math.max(stats.publishedAssignments, 1)) * + 0.3, + }, + LIVE_RECORDING_FEEDBACK: { + inputTokens: 1200, + outputTokens: 900, + currentCost: + (stats.costBreakdown.grading / + Math.max(stats.publishedAssignments, 1)) * + 0.2, + }, + GRADING_VALIDATION: { + inputTokens: 600, + outputTokens: 400, + currentCost: + (stats.costBreakdown.grading / + Math.max(stats.publishedAssignments, 1)) * + 0.1, + }, + ASSIGNMENT_GRADING: { + inputTokens: 2200, + outputTokens: 1800, + currentCost: + (stats.costBreakdown.grading / + Math.max(stats.publishedAssignments, 1)) * + 0.7, + }, + OTHER: { + inputTokens: 300, + outputTokens: 200, + currentCost: + stats.costBreakdown.other / + Math.max(stats.publishedAssignments, 1), + }, + } + : { + // Fallback sample data for demonstration + TRANSLATION: { + inputTokens: 1500, + outputTokens: 800, + currentCost: 0.0085, + }, + QUESTION_GENERATION: { + inputTokens: 2000, + outputTokens: 1200, + currentCost: 0.0125, + }, + ASSIGNMENT_GENERATION: { + inputTokens: 800, + outputTokens: 1500, + currentCost: 0.0095, + }, + LIVE_RECORDING_FEEDBACK: { + inputTokens: 1200, + outputTokens: 900, + currentCost: 0.0105, + }, + GRADING_VALIDATION: { + inputTokens: 600, + outputTokens: 400, + currentCost: 0.0045, + }, + ASSIGNMENT_GRADING: { + inputTokens: 2200, + outputTokens: 1800, + currentCost: 0.0185, + }, + OTHER: { inputTokens: 300, outputTokens: 200, currentCost: 0.0025 }, + }; + + let totalCurrentCost = 0; + let totalNewCost = 0; + const breakdown: { + [key: string]: { current: number; new: number; factor: number }; + } = {}; + + for (const [usageType, usage] of Object.entries(exampleUsage)) { + totalCurrentCost += usage.currentCost; + + // Calculate scaling factor for this usage type + let scalingFactor = 1; + + // Apply global factor first + const globalFactor = parseFloat(globalUpscalingFactor); + if (globalFactor && globalFactor > 0) { + scalingFactor *= globalFactor; + } + + // Apply usage-specific factor + const usageFactorValue = + usageTypeUpscaling[usageType as keyof typeof usageTypeUpscaling]; + const usageFactor = parseFloat(usageFactorValue); + if (usageFactor && usageFactor > 0) { + scalingFactor *= usageFactor; + } + + const newCost = usage.currentCost * scalingFactor; + totalNewCost += newCost; + + breakdown[usageType] = { + current: usage.currentCost, + new: newCost, + factor: scalingFactor, + }; + } + + return { + totalCurrentCost, + totalNewCost, + breakdown, + percentageChange: + totalCurrentCost > 0 + ? ((totalNewCost - totalCurrentCost) / totalCurrentCost) * 100 + : 0, + }; + }; + + const handlePriceUpscaling = async () => { + if (!sessionToken) return; + + // Validate inputs + const globalFactor = parseFloat(globalUpscalingFactor); + if (globalUpscalingFactor && (isNaN(globalFactor) || globalFactor <= 0)) { + alert("Global upscaling factor must be a positive number"); + return; + } + + const usageFactors: { [key: string]: number } = {}; + for (const [usageType, value] of Object.entries(usageTypeUpscaling)) { + if (value.trim()) { + const factor = parseFloat(value); + if (isNaN(factor) || factor <= 0) { + alert(`${usageType} upscaling factor must be a positive number`); + return; + } + usageFactors[usageType] = factor; + } + } + + if (!globalUpscalingFactor && Object.keys(usageFactors).length === 0) { + alert("Please enter at least one upscaling factor"); + return; + } + + try { + await upscalePricingMutation.mutateAsync({ + globalFactor: globalFactor || undefined, + usageFactors: + Object.keys(usageFactors).length > 0 ? usageFactors : undefined, + reason: "Manual price upscaling via admin interface", + }); + + alert("Prices have been successfully upscaled!"); + setIsPriceUpscalingModalOpen(false); + resetUpscalingModal(); + } catch (error) { + console.error("Failed to upscale prices:", error); + alert( + `Failed to upscale prices: ${error instanceof Error ? error.message : "Please try again."}`, + ); + } + }; + + const handleRemoveUpscaling = async () => { + if (!sessionToken) return; + + const confirmRemoval = confirm( + "Are you sure you want to remove the current price upscaling? This will revert all pricing to base rates.", + ); + if (!confirmRemoval) return; + + try { + const result = await removePricingMutation.mutateAsync( + "Manual removal via admin interface", + ); + + if (result.success) { + alert( + "Price upscaling has been successfully removed. All pricing reverted to base rates.", + ); + } else { + alert(result.message || "No active price upscaling found to remove."); + } + } catch (error) { + console.error("Failed to remove upscaling:", error); + alert( + `Failed to remove price upscaling: ${error instanceof Error ? error.message : "Please try again."}`, + ); + } + }; + + // Show error state + if (statsError) { + return ( +
+ + + + + Error Loading Dashboard + + + +

+ {statsError instanceof Error + ? statsError.message + : "Failed to load dashboard data"} +

+ + + +
+ ); + } + + return ( +
+
+
+
+

+ {isAdmin ? "Admin Dashboard" : "Author Dashboard"} +

+ {stats && ( + + {isAdmin ? "Super Admin" : "Author"} + + )} +
+

+ {isAdmin + ? "Manage all assignments, feedback and reports" + : "Manage your assignments and feedback"} +

+
+
+ + {isAdmin && ( + <> + {/* Current Upscaling Status */} + {currentUpscaling && ( +
+ + Price Upscaling Active + +
+ )} + + + + + + + + + + Price Upscaling (Super Admin Only) + + + Apply upscaling factors to AI pricing. You can set a + global factor or specific factors for each usage type. + + + +
+ {/* Current Upscaling Status */} + {currentUpscaling && ( +
+ +
+

+ Current Active Upscaling +

+
+ {currentUpscaling.globalFactor && ( +
+ Global Factor: {currentUpscaling.globalFactor}x +
+ )} + {currentUpscaling.usageTypeFactors && ( +
Usage-specific factors applied
+ )} +
+ Applied:{" "} + {new Date( + currentUpscaling.effectiveDate, + ).toLocaleString()} +
+ {currentUpscaling.reason && ( +
+ Reason: {currentUpscaling.reason} +
+ )} +
+
+ +
+ )} + + {/* Global Upscaling */} +
+ + + setGlobalUpscalingFactor(e.target.value) + } + className="mt-1" + /> +

+ If set, this will be applied to all usage types + (multiplied with individual factors) +

+
+ + {/* Usage Type Specific Upscaling */} +
+ +
+ {Object.entries(usageTypeUpscaling).map( + ([usageType, value]) => ( +
+ + + handleUsageTypeUpscalingChange( + usageType as keyof typeof usageTypeUpscaling, + e.target.value, + ) + } + className="mt-1" + /> +
+ ), + )} +
+

+ Individual factors are applied after the global factor + (if set) +

+
+ + {/* Price Impact Example */} + {(globalUpscalingFactor || + Object.values(usageTypeUpscaling).some((v) => + v.trim(), + )) && ( +
+
+ + +
+
+ {(() => { + const example = calculatePriceExample(); + const useRealData = stats && stats.costBreakdown; + return ( + <> +

+ {useRealData + ? `Based on your current assignment data (average per assignment)` + : `Based on a typical assignment with average AI usage`} +

+
+ {/* Summary */} +
+
+
+ Total Assignment Cost +
+
+ Current → New +
+
+
+
+ ${example.totalCurrentCost.toFixed(4)} → + ${example.totalNewCost.toFixed(4)} +
+
0 ? "text-red-600" : example.percentageChange < 0 ? "text-green-600" : "text-gray-600"}`} + > + {example.percentageChange > 0 + ? "+" + : ""} + {example.percentageChange.toFixed(1)}% + change +
+
+
+ + {/* Detailed Breakdown */} +
+ {Object.entries(example.breakdown) + .filter(([, data]) => data.factor !== 1) + .map(([usageType, data]) => ( +
+
+ {usageType + .replace(/_/g, " ") + .toLowerCase() + .replace(/\b\w/g, (l) => + l.toUpperCase(), + )} +
+
+
+ ${data.current.toFixed(4)} → $ + {data.new.toFixed(4)} +
+
+ ×{data.factor.toFixed(1)} +
+
+
+ ))} +
+ + {Object.values(example.breakdown).every( + (data) => data.factor === 1, + ) && ( +
+ No changes applied with current factors +
+ )} +
+ + ); + })()} +
+
+ )} + + {/* Action Buttons */} +
+ +
+ + +
+
+
+ + + + + + + + )} + {onLogout && ( + + )} +
+
+ + {/* Date Filter */} +
+
+
+ + + + + + + + +
+
+ + + setCustomDateRange({ + ...customDateRange, + start: e.target.value, + }) + } + /> +
+
+ + + setCustomDateRange({ + ...customDateRange, + end: e.target.value, + }) + } + /> +
+
+ + +
+
+ + +
+ + {filters.startDate && ( + + )} +
+ +
+ {stats && filters.startDate && ( + Showing data from {formatDateRange()} + )} +
+
+ + {/* Stats Cards */} + {loadingStats ? ( +
+
+ + Loading dashboard data... + +
+ ) : stats ? ( + <> +
+ + + +
+ Assignments Created + + + +
+ {stats.totalAssignments.toLocaleString()} +
+

+ Total created +

+ + + + + + +
+ Assignments Published + + + +
+ {stats.publishedAssignments.toLocaleString()} +
+

+ Currently active +

+ + + + + + +
+ Total Unique Learners + + + +
+ {stats.totalLearners.toLocaleString()} +
+

+ Registered users +

+ + + + + + +
+ Avg Rating + + + +
+ {stats.averageAssignmentRating?.toFixed(1) || "0.0"} +
+

+ Out of 5 stars +

+ + + + + + +
+ AI Cost + + + +
+ ${stats.totalCost.toFixed(2)} +
+

+ {filters.startDate ? formatDateRange() : "Total spent"} +

+ + + {/* Reports stats only for admins */} + {isAdmin && ( + <> + + + +
+ Total Reports + + + +
+ {stats.totalReports.toLocaleString()} +
+

+ All reports +

+ + + + + + +
+ Open Reports + + + +
+ {stats.openReports.toLocaleString()} +
+

+ Need attention +

+ + + + )} +
+ + {/* Cost Breakdown */} + + + +
+ AI Cost Breakdown + +

+ {filters.startDate + ? `Cost distribution for ${formatDateRange()}` + : "Cost distribution across different AI services"} +

+ + +
+
+
+ Grading +
+
+ ${stats.costBreakdown.grading.toFixed(2)} +
+
+
+
+ Question Gen +
+
+ ${stats.costBreakdown.questionGeneration.toFixed(2)} +
+
+
+
+ Translation +
+
+ ${stats.costBreakdown.translation.toFixed(2)} +
+
+
+
+ Other +
+
+ ${stats.costBreakdown.other.toFixed(2)} +
+
+
+ + + + ) : ( +
+ No data available +
+ )} + + {/* Tab Navigation */} +
+
+ +
+
+ + {/* Tab Content */} + + + {activeTab === "assignments" && ( + + )} + {activeTab === "feedback" && ( + + )} + {/* Reports tab only for admins */} + {activeTab === "reports" && isAdmin && ( + + )} + + +
+ ); +} + +export function OptimizedAdminDashboard(props: AdminDashboardProps) { + return ( + + + {/* */} + + ); +} diff --git a/apps/web/app/admin/components/AdminLogin.tsx b/apps/web/app/admin/components/AdminLogin.tsx new file mode 100644 index 00000000..5c1aba9d --- /dev/null +++ b/apps/web/app/admin/components/AdminLogin.tsx @@ -0,0 +1,254 @@ +"use client"; + +import { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { + Loader2, + Mail, + Shield, + Info, + Users, + MessageSquare, +} from "lucide-react"; + +interface AdminLoginProps { + onAuthenticated: (sessionToken: string) => void; +} + +export function AdminLogin({ onAuthenticated }: AdminLoginProps) { + const [step, setStep] = useState<"email" | "code">("email"); + const [email, setEmail] = useState(""); + const [code, setCode] = useState(""); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(""); + const [success, setSuccess] = useState(""); + + const handleSendCode = async (e: React.FormEvent) => { + e.preventDefault(); + setLoading(true); + setError(""); + setSuccess(""); + + try { + const response = await fetch("/api/v1/auth/admin/send-code", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ email }), + }); + + const data = await response.json(); + + if (!response.ok) { + throw new Error(data.message || "Failed to send verification code"); + } + + setSuccess("Verification code sent to your email!"); + setStep("code"); + } catch (err) { + setError(err instanceof Error ? err.message : "An error occurred"); + } finally { + setLoading(false); + } + }; + + const handleVerifyCode = async (e: React.FormEvent) => { + e.preventDefault(); + setLoading(true); + setError(""); + + try { + const response = await fetch("/api/v1/auth/admin/verify-code", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ email, code }), + }); + + const data = await response.json(); + + if (!response.ok) { + throw new Error(data.message || "Failed to verify code"); + } + + // Store session token + localStorage.setItem("adminSessionToken", data.sessionToken); + localStorage.setItem("adminEmail", email); + localStorage.setItem("adminExpiresAt", data.expiresAt); + + onAuthenticated(data.sessionToken); + } catch (err) { + setError(err instanceof Error ? err.message : "An error occurred"); + } finally { + setLoading(false); + } + }; + + const handleBackToEmail = () => { + setStep("email"); + setCode(""); + setError(""); + setSuccess(""); + }; + + if (step === "email") { + return ( +
+ + +
+ +
+ Admin Access + + Enter your admin email to receive a verification code + + + + {/* Important Notes */} +
+ + + + Use the same email you used when publishing + assignments. + + + + + + + No access? You need to have activity in an + assignment to access that assignment's admin dashboard. + + + + + + + Super user privileges? Contact the developers + or stakeholders for elevated access. + + +
+ +
+
+ setEmail(e.target.value)} + required + disabled={loading} + /> +
+ + {error && ( + + Error + {error} + + )} + + {success && ( + + + Success + {success} + + )} + + +
+ + +
+ ); + } + + return ( +
+ + +
+ +
+ Enter Verification Code + We sent a 6-digit code to {email} + + +
+
+ + setCode(e.target.value.replace(/\D/g, "").slice(0, 6)) + } + required + disabled={loading} + className="text-center text-lg tracking-widest" + maxLength={6} + /> +
+ + {error && ( + + Error + {error} + + )} + +
+ + + +
+
+ +
+ +
+ + +
+ ); +} diff --git a/apps/web/app/admin/components/AssignmentAnalyticsTable.tsx b/apps/web/app/admin/components/AssignmentAnalyticsTable.tsx new file mode 100644 index 00000000..68c2438e --- /dev/null +++ b/apps/web/app/admin/components/AssignmentAnalyticsTable.tsx @@ -0,0 +1,1007 @@ +"use client"; + +import { useState, useEffect, useMemo } from "react"; +import { useRouter } from "next/navigation"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { + createColumnHelper, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, + type ColumnDef, + type SortingState, + type ColumnFiltersState, +} from "@tanstack/react-table"; +import { + getAssignmentAnalytics, + type AssignmentAnalyticsData, +} from "@/lib/talkToBackend"; +import { QuickActions } from "./QuickActions"; +import { + Search, + ChevronLeft, + ChevronRight, + DollarSign, + Users, + FileText, + Star, + ExternalLink, + CalendarIcon, + Filter, + X, + SortAsc, + SortDesc, + ChevronUp, + ChevronDown, +} from "lucide-react"; + +interface AssignmentAnalyticsTableProps { + sessionToken?: string | null; + isAdmin?: boolean; + quickActionResults?: any[] | null; + quickActionTitle?: string; + onClearQuickActionResults?: () => void; + onQuickActionComplete?: (result: any) => void; + filters?: { + startDate?: string; + endDate?: string; + assignmentId?: number; + assignmentName?: string; + userId?: string; + }; + onFiltersChange?: (filters: { + startDate?: string; + endDate?: string; + assignmentId?: number; + assignmentName?: string; + userId?: string; + }) => void; +} + +export function AssignmentAnalyticsTable({ + sessionToken, + isAdmin, + quickActionResults, + quickActionTitle, + onClearQuickActionResults, + onQuickActionComplete, + filters, + onFiltersChange, +}: AssignmentAnalyticsTableProps) { + const router = useRouter(); + const [data, setData] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [globalFilter, setGlobalFilter] = useState(""); + const [sorting, setSorting] = useState([]); + const [columnFilters, setColumnFilters] = useState([]); + const [tablePagination, setTablePagination] = useState({ + pageIndex: 0, + pageSize: 10, + }); + const [showFilters, setShowFilters] = useState(false); + const [localFilters, setLocalFilters] = useState(filters || {}); + + const fetchData = async () => { + if (!sessionToken) return; + + setLoading(true); + setError(null); + + try { + // Always fetch all data for tanstack table to handle pagination/filtering + const response = await getAssignmentAnalytics( + sessionToken, + 1, + 1000, // Get all data for client-side table operations + undefined, + ); + setData(response.data); + } catch (err) { + setError( + err instanceof Error + ? err.message + : "Failed to fetch assignment analytics", + ); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + if (!quickActionResults) { + fetchData(); + } + }, [sessionToken, quickActionResults]); + + useEffect(() => { + setLocalFilters(filters || {}); + }, [filters]); + + const handleClearQuickActionResults = () => { + if (onClearQuickActionResults) { + onClearQuickActionResults(); + } + // Also fetch fresh data + fetchData(); + }; + + const handleFilterChange = (key: string, value: string | number) => { + const newFilters = { + ...localFilters, + [key]: value === "" ? undefined : value, + }; + setLocalFilters(newFilters); + }; + + const applyFilters = () => { + if (onFiltersChange) { + onFiltersChange(localFilters); + } + }; + + // Use Quick Action results if available, otherwise use regular data + const rawData = quickActionResults || data; + const isShowingQuickActionResults = !!quickActionResults; + const currentQuickActionTitle = quickActionTitle; + + // Helper functions + const formatCurrency = (amount: number) => { + return new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + minimumFractionDigits: 2, + maximumFractionDigits: 4, + }).format(amount); + }; + + const formatPercentage = (value: number) => { + return `${Math.round(value)}%`; + }; + + const navigateToInsights = (assignmentId: number) => { + // open new window + window.open(`/admin/insights/${assignmentId}`, "_blank"); + }; + + // Create column helper for type safety + const columnHelper = createColumnHelper(); + + // Define table columns + const columns = useMemo[]>( + () => [ + columnHelper.accessor("name", { + header: "Assignment", + cell: ({ getValue, row }) => ( +
+
{getValue()}
+
+ ID: {row.original.id} +
+
+ ), + enableSorting: true, + }), + + columnHelper.accessor("published", { + header: "Status", + cell: ({ getValue }) => ( + + {getValue() ? "Published" : "Draft"} + + ), + enableSorting: true, + filterFn: "equals", + }), + columnHelper.display({ + id: "costPerAttempt", + header: "Cost/Attempt", + cell: ({ row }) => { + const assignment = row.original; + return ( +
+
+ {assignment.totalAttempts > 0 + ? formatCurrency( + assignment.totalCost / assignment.totalAttempts, + ) + : "N/A"} +
+ {assignment.totalAttempts > 0 && ( +
+ {assignment.totalAttempts} attempts +
+ )} +
+ ); + }, + enableSorting: true, + sortingFn: (rowA, rowB) => { + const a = rowA.original; + const b = rowB.original; + const aValue = + a.totalAttempts > 0 ? a.totalCost / a.totalAttempts : 0; + const bValue = + b.totalAttempts > 0 ? b.totalCost / b.totalAttempts : 0; + return aValue - bValue; + }, + }), + columnHelper.accessor("totalCost", { + header: "Total Cost", + cell: ({ getValue, row }) => ( +
+
+ {formatCurrency(getValue())} +
+ {row.original.insights?.costBreakdown && ( +
+ Grading:{" "} + {formatCurrency(row.original.insights.costBreakdown.grading)} +
+ )} +
+ ), + enableSorting: true, + }), + + columnHelper.accessor("uniqueLearners", { + header: "Learners", + cell: ({ getValue }) => ( +
+
+ + {getValue()} +
+
+ ), + enableSorting: true, + }), + + columnHelper.accessor("totalAttempts", { + header: "Attempts", + cell: ({ getValue, row }) => ( +
+
+
{getValue()}
+
+ {row.original.completedAttempts} completed +
+
+
+ ), + enableSorting: true, + }), + + columnHelper.display({ + id: "completion", + header: "Completion", + cell: ({ row }) => { + const assignment = row.original; + const completionRate = + assignment.totalAttempts > 0 + ? (assignment.completedAttempts / assignment.totalAttempts) * 100 + : 0; + return ( +
+
+
+ {formatPercentage(completionRate)} +
+
+
+
+
+
+ ); + }, + enableSorting: true, + sortingFn: (rowA, rowB) => { + const a = rowA.original; + const b = rowB.original; + const aCompletion = + a.totalAttempts > 0 + ? (a.completedAttempts / a.totalAttempts) * 100 + : 0; + const bCompletion = + b.totalAttempts > 0 + ? (b.completedAttempts / b.totalAttempts) * 100 + : 0; + return aCompletion - bCompletion; + }, + }), + + columnHelper.accessor("averageGrade", { + header: "Avg Grade", + cell: ({ getValue }) => { + const grade = getValue(); + return ( +
+
+
+ {grade > 0 ? formatPercentage(grade) : "N/A"} +
+ {grade > 0 && ( +
+
+
= 80 + ? "bg-green-500" + : grade >= 60 + ? "bg-yellow-500" + : "bg-red-500" + }`} + style={{ width: `${grade}%` }} + /> +
+
+ )} +
+
+ ); + }, + enableSorting: true, + }), + + columnHelper.accessor("averageRating", { + header: "Rating", + cell: ({ getValue }) => ( +
+
+ + + {getValue() > 0 ? getValue().toFixed(1) : "N/A"} + +
+
+ ), + enableSorting: true, + }), + + columnHelper.display({ + id: "actions", + header: "Actions", + cell: ({ row }) => ( +
+ +
+ ), + }), + ], + [formatCurrency, formatPercentage, navigateToInsights], + ); + + // Create table instance + const table = useReactTable({ + data: rawData, + columns, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + onGlobalFilterChange: setGlobalFilter, + onPaginationChange: setTablePagination, + state: { + sorting, + columnFilters, + globalFilter, + pagination: tablePagination, + }, + globalFilterFn: (row, _columnId, value) => { + // Search in assignment name and ID + const assignment = row.original; + const searchValue = value.toLowerCase(); + + // Search in assignment name + const nameMatch = assignment.name.toLowerCase().includes(searchValue); + + // Search in assignment ID (convert to string) + const idMatch = assignment.id.toString().includes(searchValue); + + return nameMatch || idMatch; + }, + }); + + const hasActiveFilters = Object.values(filters || {}).some( + (value) => value !== undefined && value !== "", + ); + + if (loading && !quickActionResults && data.length === 0) { + return ( +
+
+
+ Loading assignment analytics... +
+
+
+ ); + } + + if (error) { + return ( +
+
+
Error: {error}
+
+
+ ); + } + + return ( +
+ {/* Enhanced Filters */} + + +
+ Filters & Search +
+ {/* Quick Actions */} + {!isShowingQuickActionResults && ( + + )} + + {/* Quick Action Results */} + {isShowingQuickActionResults && ( +
+ + Quick Action Results: {currentQuickActionTitle} + + +
+ )} + + +
+
+ + + {/* Global Search */} +
+ +
+ + setGlobalFilter(e.target.value)} + className="pl-10" + /> + {globalFilter && ( + + )} +
+
+ + {/* Quick Filters Row */} +
+ {/* Status Filter */} +
+ + +
+ + {/* Page Size */} +
+ + +
+
+ + {showFilters && ( +
+
+ {/* Date Range */} +
+ +
+
+ + + handleFilterChange("startDate", e.target.value) + } + className="pl-10" + /> +
+
+ + + handleFilterChange("endDate", e.target.value) + } + className="pl-10" + /> +
+
+
+ + {/* Assignment Filters */} +
+ +
+ + handleFilterChange( + "assignmentId", + e.target.value ? parseInt(e.target.value) : "", + ) + } + /> + + handleFilterChange("assignmentName", e.target.value) + } + /> +
+
+ + {/* User Filter */} +
+ + + handleFilterChange("userId", e.target.value) + } + /> +
+
+ + {/* Action Buttons */} +
+ + +
+ + {/* Active Filters Summary */} + {hasActiveFilters && ( +
+
+ Active Advanced Filters: +
+
+ {filters?.startDate && ( + + From: {filters.startDate} + + )} + {filters?.endDate && ( + + To: {filters.endDate} + + )} + {filters?.assignmentId && ( + + Assignment ID: {filters.assignmentId} + + )} + {filters?.assignmentName && ( + + Assignment: {filters.assignmentName} + + )} + {filters?.userId && ( + + User: {filters.userId} + + )} +
+
+ )} +
+ )} + + {/* Active Filters */} + {(globalFilter || + table.getColumn("published")?.getFilterValue() !== undefined) && ( +
+
+ Active Filters: +
+
+ {globalFilter && ( + + Search: {globalFilter} + + )} + {table.getColumn("published")?.getFilterValue() !== + undefined && ( + + Status:{" "} + {table.getColumn("published")?.getFilterValue() + ? "Published" + : "Draft"} + + )} + +
+
+ )} + + + + {/* Summary Stats - Based on Filtered Data */} + {rawData && rawData.length > 0 && ( +
+ +
+ +
+
+ {table.getFilteredRowModel().rows.length} +
+
+ {globalFilter || + table.getColumn("published")?.getFilterValue() !== undefined + ? "Filtered Assignments" + : "Total Assignments"} +
+
+
+ + +
+ +
+
+ {formatCurrency( + table + .getFilteredRowModel() + .rows.reduce( + (sum, row) => sum + row.original.totalCost, + 0, + ), + ).replace("$", "")} +
+
+ {globalFilter || + table.getColumn("published")?.getFilterValue() !== undefined + ? "Filtered Cost" + : "Total Cost"} +
+
+
+ + +
+ +
+
+ {table + .getFilteredRowModel() + .rows.reduce( + (sum, row) => sum + row.original.uniqueLearners, + 0, + )} +
+
+ {globalFilter || + table.getColumn("published")?.getFilterValue() !== undefined + ? "Filtered Learner-Assignment Pairs" + : "Total Learner-Assignment Pairs"} +
+
+
+ + +
+ +
+
+ {(() => { + const filteredData = table + .getFilteredRowModel() + .rows.map((row) => row.original); + const validRatings = filteredData.filter( + (a) => a.averageRating > 0, + ); + if (validRatings.length === 0) return "N/A"; + const avgRating = + validRatings.reduce( + (sum, a) => sum + a.averageRating, + 0, + ) / validRatings.length; + return avgRating.toFixed(1); + })()} +
+
+ {globalFilter || + table.getColumn("published")?.getFilterValue() !== undefined + ? "Filtered Avg Rating" + : "Avg Rating"} +
+
+
+ +
+ )} + + {/* Analytics Table */} + + +
+ + + My Assignment Analytics + +
+ Showing {table.getFilteredRowModel().rows.length} of{" "} + {rawData.length} assignments +
+
+ + + {rawData.length === 0 ? ( +
+ {isShowingQuickActionResults + ? "No results found for this quick action" + : "No assignment analytics found"} +
+ ) : ( +
+ {/* Table */} +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + + {table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
+
+ + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + {header.column.getCanSort() && ( + + {{ + asc: ( + + ), + desc: ( + + ), + }[ + header.column.getIsSorted() as string + ] ?? ( +
+ + +
+ )} + + )} +
+
+ {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} +
+
+
+ + {/* Empty State */} + {table.getFilteredRowModel().rows.length === 0 && + rawData.length > 0 && ( +
+
+ +
+

+ No assignments match your filters +

+

+ Try adjusting your search terms or filters +

+
+ )} + + {/* Pagination */} + {table.getPageCount() > 1 && ( +
+
+ + Page {table.getState().pagination.pageIndex + 1} of{" "} + {table.getPageCount()} + +
+
+ + +
+
+ )} +
+ )} + + +
+ ); +} diff --git a/apps/web/app/admin/components/DashboardFilters.tsx b/apps/web/app/admin/components/DashboardFilters.tsx new file mode 100644 index 00000000..5f466358 --- /dev/null +++ b/apps/web/app/admin/components/DashboardFilters.tsx @@ -0,0 +1,245 @@ +"use client"; + +import { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { CalendarIcon, Filter, X } from "lucide-react"; +import { QuickActions } from "./QuickActions"; + +interface DashboardFiltersProps { + onFiltersChange: (filters: { + startDate?: string; + endDate?: string; + assignmentId?: number; + assignmentName?: string; + userId?: string; + }) => void; + onClearFilters: () => void; + currentFilters: { + startDate?: string; + endDate?: string; + assignmentId?: number; + assignmentName?: string; + userId?: string; + }; + sessionToken?: string | null; + onQuickActionComplete?: (result: any) => void; +} + +export function DashboardFilters({ + onFiltersChange, + onClearFilters, + currentFilters, + sessionToken, + onQuickActionComplete, +}: DashboardFiltersProps) { + const [showFilters, setShowFilters] = useState(false); + const [localFilters, setLocalFilters] = useState(currentFilters); + + const handleFilterChange = (key: string, value: string | number) => { + const newFilters = { + ...localFilters, + [key]: value === "" ? undefined : value, + }; + setLocalFilters(newFilters); + }; + + const applyFilters = () => { + onFiltersChange(localFilters); + }; + + const clearAllFilters = () => { + const emptyFilters = { + startDate: undefined, + endDate: undefined, + assignmentId: undefined, + assignmentName: undefined, + userId: undefined, + }; + setLocalFilters(emptyFilters); + onFiltersChange(emptyFilters); + onClearFilters(); + }; + + const hasActiveFilters = Object.values(currentFilters).some( + (value) => value !== undefined && value !== "", + ); + + return ( +
+ {/* Quick Actions */} + + + {/* Filters */} + + +
+ + + Dashboard Filters + {hasActiveFilters && ( + + Active + + )} + +
+ {hasActiveFilters && ( + + )} + +
+
+ + + {showFilters && ( + +
+ {/* Date Range */} +
+ +
+
+ + + handleFilterChange("startDate", e.target.value) + } + className="pl-10" + /> +
+
+ + + handleFilterChange("endDate", e.target.value) + } + className="pl-10" + /> +
+
+
+ + {/* Assignment Filters */} +
+ +
+ + handleFilterChange( + "assignmentId", + e.target.value ? parseInt(e.target.value) : "", + ) + } + /> + + handleFilterChange("assignmentName", e.target.value) + } + /> +
+
+ + {/* User Filter */} +
+ + handleFilterChange("userId", e.target.value)} + /> +
+
+ + {/* Action Buttons */} +
+ + +
+ + {/* Active Filters Summary */} + {hasActiveFilters && ( +
+
+ Active Filters: +
+
+ {currentFilters.startDate && ( + + From: {currentFilters.startDate} + + )} + {currentFilters.endDate && ( + + To: {currentFilters.endDate} + + )} + {currentFilters.assignmentId && ( + + Assignment ID: {currentFilters.assignmentId} + + )} + {currentFilters.assignmentName && ( + + Assignment: {currentFilters.assignmentName} + + )} + {currentFilters.userId && ( + + User: {currentFilters.userId} + + )} +
+
+ )} + + )} + +
+ ); +} diff --git a/apps/web/app/admin/components/FeedbackTable.tsx b/apps/web/app/admin/components/FeedbackTable.tsx new file mode 100644 index 00000000..f463169a --- /dev/null +++ b/apps/web/app/admin/components/FeedbackTable.tsx @@ -0,0 +1,792 @@ +"use client"; + +import { useState, useEffect, useMemo } from "react"; +import { + Search, + ChevronLeft, + ChevronRight, + Download, + SortAsc, + SortDesc, + ChevronUp, + ChevronDown, + X, + Star, + CalendarIcon, +} from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { + createColumnHelper, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, + type ColumnDef, + type SortingState, + type ColumnFiltersState, +} from "@tanstack/react-table"; +import { + getAdminFeedback, + type FeedbackData, + type FeedbackFilters, +} from "@/lib/talkToBackend"; +import { FeedbackModal } from "@/components/modals/FeedbackModal"; + +interface FeedbackTableProps { + sessionToken?: string | null; +} + +export function FeedbackTable({ sessionToken }: FeedbackTableProps) { + const [feedback, setFeedback] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [globalFilter, setGlobalFilter] = useState(""); + const [sorting, setSorting] = useState([]); + const [columnFilters, setColumnFilters] = useState([]); + const [tablePagination, setTablePagination] = useState({ + pageIndex: 0, + pageSize: 20, + }); + const [assignmentIdFilter, setAssignmentIdFilter] = useState(""); + const [startDate, setStartDate] = useState(""); + const [endDate, setEndDate] = useState(""); + + // Modal states + const [selectedFeedback, setSelectedFeedback] = useState( + null, + ); + const [isFeedbackModalOpen, setIsFeedbackModalOpen] = useState(false); + + // Helper functions + const getRatingStars = (rating: number) => { + return "★".repeat(rating) + "☆".repeat(5 - rating); + }; + + const openFeedbackModal = (feedbackItem: FeedbackData) => { + setSelectedFeedback(feedbackItem); + setIsFeedbackModalOpen(true); + }; + + const closeFeedbackModal = () => { + setIsFeedbackModalOpen(false); + setSelectedFeedback(null); + }; + + const fetchFeedback = async () => { + if (!sessionToken) return; + + setLoading(true); + setError(null); + + try { + // Fetch all feedback for client-side filtering/pagination + const data = await getAdminFeedback( + { page: 1, limit: 1000 }, + undefined, + sessionToken, + ); + + setFeedback(data.data || []); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to fetch feedback"); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchFeedback(); + }, [sessionToken]); + + // Create column helper + const columnHelper = createColumnHelper(); + + // Define table columns + const columns = useMemo[]>( + () => [ + columnHelper.display({ + id: "assignment", + header: "Assignment", + cell: ({ row }) => ( +
+
{row.original.assignment.name}
+
+ ID: {row.original.assignment.id} +
+
+ ), + enableSorting: true, + sortingFn: (rowA, rowB) => { + const a = rowA.original.assignment.name; + const b = rowB.original.assignment.name; + return a.localeCompare(b); + }, + }), + + columnHelper.accessor("userId", { + header: "User ID", + cell: ({ getValue }) => ( +
{getValue()}
+ ), + enableSorting: true, + }), + + columnHelper.accessor("comments", { + header: "Comments", + cell: ({ getValue }) => ( +
{getValue()}
+ ), + enableSorting: true, + }), + + columnHelper.display({ + id: "ratings", + header: "Ratings", + cell: ({ row }) => ( +
+
+ AI: +
+ {Array.from({ length: 5 }, (_, i) => ( + + ))} + + ({row.original.aiGradingRating}) + +
+
+
+ Assignment: +
+ {Array.from({ length: 5 }, (_, i) => ( + + ))} + + ({row.original.assignmentRating}) + +
+
+
+ ), + enableSorting: true, + sortingFn: (rowA, rowB) => { + const a = + (rowA.original.aiGradingRating + rowA.original.assignmentRating) / + 2; + const b = + (rowB.original.aiGradingRating + rowB.original.assignmentRating) / + 2; + return a - b; + }, + }), + + columnHelper.accessor("allowContact", { + header: "Contact Info", + cell: ({ getValue, row }) => { + if (getValue()) { + return ( +
+
+ {row.original.firstName} {row.original.lastName} +
+
+ {row.original.email} +
+
+ ); + } else { + return No contact; + } + }, + enableSorting: true, + filterFn: "equals", + }), + + columnHelper.display({ + id: "grade", + header: "Grade", + cell: ({ row }) => ( + + {(row.original.assignmentAttempt.grade * 100).toFixed(1)}% + + ), + enableSorting: true, + sortingFn: (rowA, rowB) => { + const a = rowA.original.assignmentAttempt.grade; + const b = rowB.original.assignmentAttempt.grade; + return a - b; + }, + }), + + columnHelper.accessor("createdAt", { + header: "Date", + cell: ({ getValue }) => ( +
+
+ {new Date(getValue()).toLocaleDateString()} +
+
+ {new Date(getValue()).toLocaleTimeString()} +
+
+ ), + enableSorting: true, + }), + + columnHelper.display({ + id: "actions", + header: "Actions", + cell: ({ row }) => ( + + ), + }), + ], + [getRatingStars, openFeedbackModal], + ); + + // Create global filter function with dependencies + const globalFilterFn = useMemo(() => { + return (row: any, _columnId: string, value: string) => { + const feedbackItem = row.original; + const searchValue = value?.toLowerCase() || ""; + + // Apply assignment ID filter + if ( + assignmentIdFilter && + feedbackItem.assignment.id.toString() !== assignmentIdFilter + ) { + return false; + } + + // Apply date range filter + const itemDate = new Date(feedbackItem.createdAt); + if (startDate && itemDate < new Date(startDate)) { + return false; + } + if (endDate && itemDate > new Date(endDate + "T23:59:59")) { + return false; + } + + // If no search value, return true (item passes filters) + if (!value) return true; + + // Search in comments, user ID, assignment name, and contact info + const commentsMatch = feedbackItem.comments + .toLowerCase() + .includes(searchValue); + const userMatch = feedbackItem.userId.toLowerCase().includes(searchValue); + const assignmentMatch = feedbackItem.assignment.name + .toLowerCase() + .includes(searchValue); + const assignmentIdMatch = feedbackItem.assignment.id + .toString() + .includes(searchValue); + const emailMatch = + feedbackItem.email?.toLowerCase().includes(searchValue) || false; + const nameMatch = + `${feedbackItem.firstName || ""} ${feedbackItem.lastName || ""}` + .toLowerCase() + .includes(searchValue); + + return ( + commentsMatch || + userMatch || + assignmentMatch || + assignmentIdMatch || + emailMatch || + nameMatch + ); + }; + }, [assignmentIdFilter, startDate, endDate]); + + // Create table instance + const table = useReactTable({ + data: feedback, + columns, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + onGlobalFilterChange: setGlobalFilter, + onPaginationChange: setTablePagination, + state: { + sorting, + columnFilters, + globalFilter, + pagination: tablePagination, + }, + globalFilterFn, + }); + + // Force re-filtering when custom filters change + useEffect(() => { + // Trigger a re-render by toggling the global filter + const currentFilter = globalFilter || ""; + table.setGlobalFilter(currentFilter + " "); + table.setGlobalFilter(currentFilter); + }, [assignmentIdFilter, startDate, endDate, table, globalFilter]); + + const exportToCSV = () => { + const filteredData = table + .getFilteredRowModel() + .rows.map((row) => row.original); + + const headers = [ + "ID", + "Assignment Name", + "User ID", + "Comments", + "AI Grading Rating", + "Assignment Rating", + "Allow Contact", + "First Name", + "Last Name", + "Email", + "Grade", + "Created At", + ]; + + const csvContent = [ + headers.join(","), + ...filteredData.map((item) => + [ + item.id, + `"${item.assignment.name}"`, + item.userId, + `"${item.comments}"`, + item.aiGradingRating, + item.assignmentRating, + item.allowContact, + item.firstName || "", + item.lastName || "", + item.email || "", + item.assignmentAttempt.grade, + new Date(item.createdAt).toLocaleString(), + ].join(","), + ), + ].join("\n"); + + const blob = new Blob([csvContent], { type: "text/csv" }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `feedback_${new Date().toISOString().split("T")[0]}.csv`; + a.click(); + window.URL.revokeObjectURL(url); + }; + + if (loading && feedback.length === 0) { + return ( +
+
+
Loading feedback...
+
+
+ ); + } + + if (error) { + return ( +
+
+
Error: {error}
+
+
+ ); + } + + return ( +
+ {/* Filters */} + + +
+ Feedback Management +
+ +
+
+ + + {/* Global Search */} +
+ +
+ + setGlobalFilter(e.target.value)} + className="pl-10" + /> + {globalFilter && ( + + )} +
+
+ + {/* Quick Filters Row 1 */} +
+ {/* Contact Filter */} +
+ + +
+ + {/* Assignment ID Filter */} +
+ + setAssignmentIdFilter(e.target.value)} + className="w-full" + /> +
+ + {/* Page Size */} +
+ + +
+
+ + {/* Date Range Filters */} +
+
+ +
+ + setStartDate(e.target.value)} + className="pl-10" + /> +
+
+
+ +
+ + setEndDate(e.target.value)} + className="pl-10" + /> +
+
+
+ + {/* Active Filters */} + {(globalFilter || + table.getColumn("allowContact")?.getFilterValue() !== undefined || + assignmentIdFilter || + startDate || + endDate) && ( +
+
+ Active Filters: +
+
+ {globalFilter && ( + + Search: {globalFilter} + + )} + {table.getColumn("allowContact")?.getFilterValue() !== + undefined && ( + + Contact:{" "} + {table.getColumn("allowContact")?.getFilterValue() + ? "Allow" + : "No Contact"} + + )} + {assignmentIdFilter && ( + + Assignment ID: {assignmentIdFilter} + + )} + {startDate && ( + + From: {startDate} + + )} + {endDate && ( + + To: {endDate} + + )} + +
+
+ )} + + + + {/* Table */} + + +
+ Feedback +
+ Showing {table.getFilteredRowModel().rows.length} of{" "} + {feedback.length} feedback entries +
+
+ + + {feedback.length === 0 ? ( +
+ No feedback found +
+ ) : ( +
+ {/* Table */} +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + + {table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
+
+ + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + {header.column.getCanSort() && ( + + {{ + asc: ( + + ), + desc: ( + + ), + }[ + header.column.getIsSorted() as string + ] ?? ( +
+ + +
+ )} + + )} +
+
+ {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} +
+
+
+ + {/* Empty State */} + {table.getFilteredRowModel().rows.length === 0 && + feedback.length > 0 && ( +
+
+ +
+

+ No feedback matches your filters +

+

+ Try adjusting your search terms or filters +

+
+ )} + + {/* Pagination */} + {table.getPageCount() > 1 && ( +
+
+ + Page {table.getState().pagination.pageIndex + 1} of{" "} + {table.getPageCount()} + +
+
+ + +
+
+ )} +
+ )} + + + + {/* Feedback Modal */} + +
+ ); +} diff --git a/apps/web/app/admin/components/OptimizedAdminDashboard.tsx b/apps/web/app/admin/components/OptimizedAdminDashboard.tsx new file mode 100644 index 00000000..5788040e --- /dev/null +++ b/apps/web/app/admin/components/OptimizedAdminDashboard.tsx @@ -0,0 +1,1005 @@ +"use client"; + +import { useState } from "react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { ReactQueryDevtools } from "@tanstack/react-query-devtools"; +import { FeedbackTable } from "./FeedbackTable"; +import { ReportsTable } from "./ReportsTable"; +import { AssignmentAnalyticsTable } from "./AssignmentAnalyticsTable"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { cn } from "@/lib/utils"; +import { + useDashboardStats, + useCurrentPriceUpscaling, + useUpscalePricing, + useRemovePriceUpscaling, + useRefreshDashboard, +} from "@/hooks/useAdminDashboard"; +import { + Settings, + RefreshCw, + TrendingUp, + AlertTriangle, + Calculator, + RotateCcw, +} from "lucide-react"; +import Link from "next/link"; +import { queryClient } from "@/lib/query-client"; + +interface AdminDashboardProps { + sessionToken?: string | null; + onLogout?: () => void; +} + +function AdminDashboardContent({ + sessionToken, + onLogout, +}: AdminDashboardProps) { + const [activeTab, setActiveTab] = useState< + "feedback" | "reports" | "assignments" + >("assignments"); + const [filters, setFilters] = useState<{ + startDate?: string; + endDate?: string; + assignmentId?: number; + assignmentName?: string; + userId?: string; + }>({}); + const [quickActionResults, setQuickActionResults] = useState( + null, + ); + const [quickActionTitle, setQuickActionTitle] = useState(""); + + // Price upscaling modal state + const [isPriceUpscalingModalOpen, setIsPriceUpscalingModalOpen] = + useState(false); + const [globalUpscalingFactor, setGlobalUpscalingFactor] = useState(""); + const [usageTypeUpscaling, setUsageTypeUpscaling] = useState({ + TRANSLATION: "", + QUESTION_GENERATION: "", + ASSIGNMENT_GENERATION: "", + LIVE_RECORDING_FEEDBACK: "", + GRADING_VALIDATION: "", + ASSIGNMENT_GRADING: "", + OTHER: "", + }); + + // TanStack Query hooks + const { + data: stats, + isLoading: loadingStats, + error: statsError, + } = useDashboardStats(sessionToken, filters); + + const { data: currentUpscaling, isLoading: loadingUpscaling } = + useCurrentPriceUpscaling(sessionToken); + + const upscalePricingMutation = useUpscalePricing(sessionToken); + const removePricingMutation = useRemovePriceUpscaling(sessionToken); + const refreshDashboard = useRefreshDashboard(sessionToken); + + const handleFiltersChange = (newFilters: typeof filters) => { + setFilters(newFilters); + }; + + const handleQuickActionComplete = (result: any) => { + setQuickActionResults(result.data); + setQuickActionTitle(result.title); + setActiveTab("assignments"); + }; + + const handleRefresh = () => { + refreshDashboard(); + }; + + const isAdmin = stats?.userRole === "admin"; + + const clearQuickActionResults = () => { + setQuickActionResults(null); + setQuickActionTitle(""); + }; + + const handleUsageTypeUpscalingChange = ( + usageType: keyof typeof usageTypeUpscaling, + value: string, + ) => { + setUsageTypeUpscaling((prev) => ({ + ...prev, + [usageType]: value, + })); + }; + + const resetUpscalingModal = () => { + setGlobalUpscalingFactor(""); + setUsageTypeUpscaling({ + TRANSLATION: "", + QUESTION_GENERATION: "", + ASSIGNMENT_GENERATION: "", + LIVE_RECORDING_FEEDBACK: "", + GRADING_VALIDATION: "", + ASSIGNMENT_GRADING: "", + OTHER: "", + }); + }; + + // Example calculation for price impact + const calculatePriceExample = () => { + // Use real dashboard data when available, otherwise fall back to sample data + const useRealData = stats && stats.costBreakdown; + + const exampleUsage = useRealData + ? { + // Map real dashboard data to usage types with estimated token usage + TRANSLATION: { + inputTokens: 1500, + outputTokens: 800, + currentCost: + stats.costBreakdown.translation / + Math.max(stats.publishedAssignments, 1), + }, + QUESTION_GENERATION: { + inputTokens: 2000, + outputTokens: 1200, + currentCost: + stats.costBreakdown.questionGeneration / + Math.max(stats.publishedAssignments, 1), + }, + ASSIGNMENT_GENERATION: { + inputTokens: 800, + outputTokens: 1500, + currentCost: + (stats.costBreakdown.questionGeneration / + Math.max(stats.publishedAssignments, 1)) * + 0.3, + }, + LIVE_RECORDING_FEEDBACK: { + inputTokens: 1200, + outputTokens: 900, + currentCost: + (stats.costBreakdown.grading / + Math.max(stats.publishedAssignments, 1)) * + 0.2, + }, + GRADING_VALIDATION: { + inputTokens: 600, + outputTokens: 400, + currentCost: + (stats.costBreakdown.grading / + Math.max(stats.publishedAssignments, 1)) * + 0.1, + }, + ASSIGNMENT_GRADING: { + inputTokens: 2200, + outputTokens: 1800, + currentCost: + (stats.costBreakdown.grading / + Math.max(stats.publishedAssignments, 1)) * + 0.7, + }, + OTHER: { + inputTokens: 300, + outputTokens: 200, + currentCost: + stats.costBreakdown.other / + Math.max(stats.publishedAssignments, 1), + }, + } + : { + // Fallback sample data for demonstration + TRANSLATION: { + inputTokens: 1500, + outputTokens: 800, + currentCost: 0.0085, + }, + QUESTION_GENERATION: { + inputTokens: 2000, + outputTokens: 1200, + currentCost: 0.0125, + }, + ASSIGNMENT_GENERATION: { + inputTokens: 800, + outputTokens: 1500, + currentCost: 0.0095, + }, + LIVE_RECORDING_FEEDBACK: { + inputTokens: 1200, + outputTokens: 900, + currentCost: 0.0105, + }, + GRADING_VALIDATION: { + inputTokens: 600, + outputTokens: 400, + currentCost: 0.0045, + }, + ASSIGNMENT_GRADING: { + inputTokens: 2200, + outputTokens: 1800, + currentCost: 0.0185, + }, + OTHER: { inputTokens: 300, outputTokens: 200, currentCost: 0.0025 }, + }; + + let totalCurrentCost = 0; + let totalNewCost = 0; + const breakdown: { + [key: string]: { current: number; new: number; factor: number }; + } = {}; + + for (const [usageType, usage] of Object.entries(exampleUsage)) { + totalCurrentCost += usage.currentCost; + + // Calculate scaling factor for this usage type + let scalingFactor = 1; + + // Apply global factor first + const globalFactor = parseFloat(globalUpscalingFactor); + if (globalFactor && globalFactor > 0) { + scalingFactor *= globalFactor; + } + + // Apply usage-specific factor + const usageFactorValue = + usageTypeUpscaling[usageType as keyof typeof usageTypeUpscaling]; + const usageFactor = parseFloat(usageFactorValue); + if (usageFactor && usageFactor > 0) { + scalingFactor *= usageFactor; + } + + const newCost = usage.currentCost * scalingFactor; + totalNewCost += newCost; + + breakdown[usageType] = { + current: usage.currentCost, + new: newCost, + factor: scalingFactor, + }; + } + + return { + totalCurrentCost, + totalNewCost, + breakdown, + percentageChange: + totalCurrentCost > 0 + ? ((totalNewCost - totalCurrentCost) / totalCurrentCost) * 100 + : 0, + }; + }; + + const handlePriceUpscaling = async () => { + if (!sessionToken) return; + + // Validate inputs + const globalFactor = parseFloat(globalUpscalingFactor); + if (globalUpscalingFactor && (isNaN(globalFactor) || globalFactor <= 0)) { + alert("Global upscaling factor must be a positive number"); + return; + } + + const usageFactors: { [key: string]: number } = {}; + for (const [usageType, value] of Object.entries(usageTypeUpscaling)) { + if (value.trim()) { + const factor = parseFloat(value); + if (isNaN(factor) || factor <= 0) { + alert(`${usageType} upscaling factor must be a positive number`); + return; + } + usageFactors[usageType] = factor; + } + } + + if (!globalUpscalingFactor && Object.keys(usageFactors).length === 0) { + alert("Please enter at least one upscaling factor"); + return; + } + + try { + await upscalePricingMutation.mutateAsync({ + globalFactor: globalFactor || undefined, + usageFactors: + Object.keys(usageFactors).length > 0 ? usageFactors : undefined, + reason: "Manual price upscaling via admin interface", + }); + + alert("Prices have been successfully upscaled!"); + setIsPriceUpscalingModalOpen(false); + resetUpscalingModal(); + } catch (error) { + console.error("Failed to upscale prices:", error); + alert( + `Failed to upscale prices: ${error instanceof Error ? error.message : "Please try again."}`, + ); + } + }; + + const handleRemoveUpscaling = async () => { + if (!sessionToken) return; + + const confirmRemoval = confirm( + "Are you sure you want to remove the current price upscaling? This will revert all pricing to base rates.", + ); + if (!confirmRemoval) return; + + try { + const result = await removePricingMutation.mutateAsync( + "Manual removal via admin interface", + ); + + if (result.success) { + alert( + "Price upscaling has been successfully removed. All pricing reverted to base rates.", + ); + } else { + alert(result.message || "No active price upscaling found to remove."); + } + } catch (error) { + console.error("Failed to remove upscaling:", error); + alert( + `Failed to remove price upscaling: ${error instanceof Error ? error.message : "Please try again."}`, + ); + } + }; + + // Show error state + if (statsError) { + return ( +
+ + + + + Error Loading Dashboard + + + +

+ {statsError instanceof Error + ? statsError.message + : "Failed to load dashboard data"} +

+ + + +
+ ); + } + + return ( +
+
+
+
+

+ {isAdmin ? "Admin Dashboard" : "Author Dashboard"} +

+ {stats && ( + + {isAdmin ? "Super Admin" : "Author"} + + )} +
+

+ {isAdmin + ? "Manage all assignments, feedback and reports" + : "Manage your assignments and feedback"} +

+
+
+ + {isAdmin && ( + <> + {/* Current Upscaling Status */} + {currentUpscaling && ( +
+ + Price Upscaling Active + +
+ )} + + + + + + + + + + Price Upscaling (Super Admin Only) + + + Apply upscaling factors to AI pricing. You can set a + global factor or specific factors for each usage type. + + + +
+ {/* Current Upscaling Status */} + {currentUpscaling && ( +
+ +
+

+ Current Active Upscaling +

+
+ {currentUpscaling.globalFactor && ( +
+ Global Factor: {currentUpscaling.globalFactor}x +
+ )} + {currentUpscaling.usageTypeFactors && ( +
Usage-specific factors applied
+ )} +
+ Applied:{" "} + {new Date( + currentUpscaling.effectiveDate, + ).toLocaleString()} +
+ {currentUpscaling.reason && ( +
+ Reason: {currentUpscaling.reason} +
+ )} +
+
+ +
+ )} + + {/* Global Upscaling */} +
+ + + setGlobalUpscalingFactor(e.target.value) + } + className="mt-1" + /> +

+ If set, this will be applied to all usage types + (multiplied with individual factors) +

+
+ + {/* Usage Type Specific Upscaling */} +
+ +
+ {Object.entries(usageTypeUpscaling).map( + ([usageType, value]) => ( +
+ + + handleUsageTypeUpscalingChange( + usageType as keyof typeof usageTypeUpscaling, + e.target.value, + ) + } + className="mt-1" + /> +
+ ), + )} +
+

+ Individual factors are applied after the global factor + (if set) +

+
+ + {/* Price Impact Example */} + {(globalUpscalingFactor || + Object.values(usageTypeUpscaling).some((v) => + v.trim(), + )) && ( +
+
+ + +
+
+ {(() => { + const example = calculatePriceExample(); + const useRealData = stats && stats.costBreakdown; + return ( + <> +

+ {useRealData + ? `Based on your current assignment data (average per assignment)` + : `Based on a typical assignment with average AI usage`} +

+
+ {/* Summary */} +
+
+
+ Total Assignment Cost +
+
+ Current → New +
+
+
+
+ ${example.totalCurrentCost.toFixed(4)} → + ${example.totalNewCost.toFixed(4)} +
+
0 ? "text-red-600" : example.percentageChange < 0 ? "text-green-600" : "text-gray-600"}`} + > + {example.percentageChange > 0 + ? "+" + : ""} + {example.percentageChange.toFixed(1)}% + change +
+
+
+ + {/* Detailed Breakdown */} +
+ {Object.entries(example.breakdown) + .filter(([, data]) => data.factor !== 1) + .map(([usageType, data]) => ( +
+
+ {usageType + .replace(/_/g, " ") + .toLowerCase() + .replace(/\b\w/g, (l) => + l.toUpperCase(), + )} +
+
+
+ ${data.current.toFixed(4)} → $ + {data.new.toFixed(4)} +
+
+ ×{data.factor.toFixed(1)} +
+
+
+ ))} +
+ + {Object.values(example.breakdown).every( + (data) => data.factor === 1, + ) && ( +
+ No changes applied with current factors +
+ )} +
+ + ); + })()} +
+
+ )} + + {/* Action Buttons */} +
+ +
+ + +
+
+
+ + + + + + + + )} + {onLogout && ( + + )} +
+
+ + {/* Stats Cards */} + {loadingStats ? ( +
+
+ + Loading dashboard data... + +
+ ) : stats ? ( + <> +
+ + + +
+ Assignments Created + + + +
+ {stats.totalAssignments.toLocaleString()} +
+

+ Total created +

+ + + + + + +
+ Assignments Published + + + +
+ {stats.publishedAssignments.toLocaleString()} +
+

+ Currently active +

+ + + + + + +
+ Total Unique Learners + + + +
+ {stats.totalLearners.toLocaleString()} +
+

+ Registered users +

+ + + + + + +
+ Avg Rating + + + +
+ {stats.averageAssignmentRating?.toFixed(1) || "0.0"} +
+

+ Out of 5 stars +

+ + + + + + +
+ AI Cost + + + +
+ ${stats.totalCost.toFixed(0)} +
+

+ Total spent +

+ + + {/* Reports stats only for admins */} + {isAdmin && ( + <> + + + +
+ Total Reports + + + +
+ {stats.totalReports.toLocaleString()} +
+

+ All reports +

+ + + + + + +
+ Open Reports + + + +
+ {stats.openReports.toLocaleString()} +
+

+ Need attention +

+ + + + )} +
+ + {/* Cost Breakdown */} + + + +
+ AI Cost Breakdown + +

+ Cost distribution across different AI services +

+ + +
+
+
+ Grading +
+
+ ${Math.round(stats.costBreakdown.grading)} +
+
+
+
+ Question Gen +
+
+ ${Math.round(stats.costBreakdown.questionGeneration)} +
+
+
+
+ Translation +
+
+ ${Math.round(stats.costBreakdown.translation)} +
+
+
+
+ Other +
+
+ ${Math.round(stats.costBreakdown.other)} +
+
+
+ + + + ) : ( +
+ No data available +
+ )} + + {/* Tab Navigation */} +
+
+ +
+
+ + {/* Tab Content */} + + + {activeTab === "assignments" && ( + + )} + {activeTab === "feedback" && ( + + )} + {/* Reports tab only for admins */} + {activeTab === "reports" && isAdmin && ( + + )} + + +
+ ); +} + +export function OptimizedAdminDashboard(props: AdminDashboardProps) { + return ( + + + + + ); +} diff --git a/apps/web/app/admin/components/QuickActions.tsx b/apps/web/app/admin/components/QuickActions.tsx new file mode 100644 index 00000000..3daae897 --- /dev/null +++ b/apps/web/app/admin/components/QuickActions.tsx @@ -0,0 +1,353 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Badge } from "@/components/ui/badge"; +import { + Zap, + TrendingUp, + Users, + DollarSign, + Star, + Activity, + BarChart, + Target, + AlertTriangle, + Award, + X, +} from "lucide-react"; +import { executeQuickAction } from "@/lib/shared"; + +interface QuickActionsProps { + sessionToken?: string | null; + onActionComplete?: (result: QuickActionResult) => void; +} + +interface QuickActionResult { + title: string; + data: Array>; +} + +const quickActions = [ + { + id: "top-assignments-by-cost", + name: "Top Assignments by AI Cost", + description: "Find assignments with highest AI processing costs", + icon: DollarSign, + category: "Cost Analysis", + color: "bg-red-100 text-red-800 border-red-200", + }, + { + id: "top-assignments-by-attempts", + name: "Most Attempted Assignments", + description: "Assignments with highest number of attempts", + icon: TrendingUp, + category: "Activity", + color: "bg-blue-100 text-blue-800 border-blue-200", + }, + { + id: "top-assignments-by-learners", + name: "Assignments with Most Learners", + description: "Find assignments with highest learner engagement", + icon: Users, + category: "Engagement", + color: "bg-green-100 text-green-800 border-green-200", + }, + { + id: "most-expensive-assignments", + name: "Most Expensive Assignments", + description: "Assignments with highest total costs", + icon: DollarSign, + category: "Cost Analysis", + color: "bg-red-100 text-red-800 border-red-200", + }, + { + id: "assignments-with-most-reports", + name: "Assignments with Most Reports", + description: "Find assignments generating most issue reports", + icon: AlertTriangle, + category: "Quality", + color: "bg-orange-100 text-orange-800 border-orange-200", + }, + { + id: "highest-rated-assignments", + name: "Highest Rated Assignments", + description: "Best performing assignments by learner ratings", + icon: Star, + category: "Quality", + color: "bg-yellow-100 text-yellow-800 border-yellow-200", + }, + { + id: "assignments-with-lowest-ratings", + name: "Lowest Rated Assignments", + description: "Assignments needing attention based on ratings", + icon: Star, + category: "Quality", + color: "bg-gray-100 text-gray-800 border-gray-200", + }, + { + id: "recent-high-activity", + name: "Recent High Activity", + description: "Assignments with high activity in last 7 days", + icon: Activity, + category: "Activity", + color: "bg-purple-100 text-purple-800 border-purple-200", + }, + { + id: "cost-per-learner-analysis", + name: "Cost Per Learner Analysis", + description: "Analyze cost efficiency per learner", + icon: BarChart, + category: "Cost Analysis", + color: "bg-indigo-100 text-indigo-800 border-indigo-200", + }, + { + id: "completion-rate-analysis", + name: "Completion Rate Analysis", + description: "Analyze assignment completion rates", + icon: Target, + category: "Performance", + color: "bg-teal-100 text-teal-800 border-teal-200", + }, +]; + +const categories = [ + "All", + "Cost Analysis", + "Activity", + "Engagement", + "Quality", + "Performance", +]; + +export function QuickActions({ + sessionToken, + onActionComplete, +}: QuickActionsProps) { + const [selectedAction, setSelectedAction] = useState(""); + const [selectedCategory, setSelectedCategory] = useState("All"); + const [limit, setLimit] = useState(10); + const [loading, setLoading] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + const [showResults, setShowResults] = useState(false); + const [executionTime, setExecutionTime] = useState(0); + const [isOpen, setIsOpen] = useState(false); + + const filteredActions = + selectedCategory === "All" + ? quickActions + : quickActions.filter((action) => action.category === selectedCategory); + + const handleExecuteAction = async () => { + if (!selectedAction || !sessionToken) return; + + const startTime = Date.now(); + setLoading(true); + setError(null); + setResult(null); + setShowResults(false); + + try { + const actionResult = (await executeQuickAction( + sessionToken, + selectedAction, + limit, + )) as QuickActionResult; + const endTime = Date.now(); + setExecutionTime(endTime - startTime); + + setResult(actionResult); + onActionComplete?.(actionResult); + setIsOpen(false); // Close modal after successful execution + + // Animate results appearance + setTimeout(() => { + setShowResults(true); + }, 300); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to execute action"); + } finally { + setLoading(false); + } + }; + + // Reset state when modal closes + const handleModalOpenChange = (open: boolean) => { + setIsOpen(open); + if (!open) { + setSelectedAction(""); + setError(null); + setResult(null); + setShowResults(false); + } + }; + + const selectedActionInfo = quickActions.find( + (action) => action.id === selectedAction, + ); + + return ( + + + + + + + + + Quick Actions + + + Run pre-built analytics queries to gain instant insights into your + data + + + +
+ {/* Category Filter */} +
+ + +
+ + {/* Quick Action Cards Grid */} +
+ {filteredActions.map((action) => { + const Icon = action.icon; + const isSelected = selectedAction === action.id; + return ( + setSelectedAction(action.id)} + > + + + + {action.name} + {isSelected && ( + + Selected + + )} + + + +

+ {action.description} +

+ + {action.category} + + + + ); + })} +
+ + {/* Action Controls */} + {selectedAction && ( + + + Execute Action + + + {/* Results Limit */} +
+ + +
+ + {/* Execute Button */} + + + {/* Error Display */} + {error && ( +
+
+ +
+

Error

+

{error}

+
+
+
+ )} + + + )} +
+ + + ); +} diff --git a/apps/web/app/admin/components/ReportsTable.tsx b/apps/web/app/admin/components/ReportsTable.tsx new file mode 100644 index 00000000..ff9d9298 --- /dev/null +++ b/apps/web/app/admin/components/ReportsTable.tsx @@ -0,0 +1,755 @@ +"use client"; + +import { useState, useEffect, useMemo } from "react"; +import { + Search, + ChevronLeft, + ChevronRight, + Download, + SortAsc, + SortDesc, + ChevronUp, + ChevronDown, + X, + CalendarIcon, +} from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { + createColumnHelper, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, + type ColumnDef, + type SortingState, + type ColumnFiltersState, +} from "@tanstack/react-table"; +import { + getAdminReports, + type ReportData, + type ReportsFilters, +} from "@/lib/talkToBackend"; +import { ReportModal } from "@/components/modals/ReportModal"; + +interface ReportsTableProps { + sessionToken?: string | null; +} + +export function ReportsTable({ sessionToken }: ReportsTableProps) { + const [reports, setReports] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [globalFilter, setGlobalFilter] = useState(""); + const [sorting, setSorting] = useState([]); + const [columnFilters, setColumnFilters] = useState([]); + const [tablePagination, setTablePagination] = useState({ + pageIndex: 0, + pageSize: 20, + }); + const [startDate, setStartDate] = useState(""); + const [endDate, setEndDate] = useState(""); + + // Modal states + const [selectedReport, setSelectedReport] = useState(null); + const [isReportModalOpen, setIsReportModalOpen] = useState(false); + + // Helper functions + const getStatusBadge = (status: string) => { + const statusVariants = { + OPEN: "destructive" as const, + IN_PROGRESS: "secondary" as const, + CLOSED: "default" as const, + RESOLVED: "outline" as const, + }; + + return ( + + {status.replace("_", " ")} + + ); + }; + + const getIssueTypeBadge = (issueType: string) => { + const typeVariants = { + BUG: "destructive" as const, + FEATURE_REQUEST: "default" as const, + QUESTION: "secondary" as const, + FEEDBACK: "outline" as const, + }; + + return ( + + {issueType.replace("_", " ")} + + ); + }; + + const openReportModal = (reportItem: ReportData) => { + setSelectedReport(reportItem); + setIsReportModalOpen(true); + }; + + const closeReportModal = () => { + setIsReportModalOpen(false); + setSelectedReport(null); + }; + + const fetchReports = async () => { + if (!sessionToken) return; + + setLoading(true); + setError(null); + + try { + // Fetch all reports for client-side filtering/pagination + const data = await getAdminReports( + { page: 1, limit: 1000 }, + undefined, + sessionToken, + ); + + setReports(data.data || []); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to fetch reports"); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + fetchReports(); + }, [sessionToken]); + + // Create column helper + const columnHelper = createColumnHelper(); + + // Define table columns + const columns = useMemo[]>( + () => [ + columnHelper.accessor("id", { + header: "ID", + cell: ({ getValue, row }) => ( +
+
#{getValue()}
+ {row.original.issueNumber && ( +
+ GitHub: #{row.original.issueNumber} +
+ )} +
+ ), + enableSorting: true, + }), + + columnHelper.accessor("reporterId", { + header: "Reporter", + cell: ({ getValue, row }) => ( +
+
{getValue()}
+ {row.original.author && ( + + Author + + )} +
+ ), + enableSorting: true, + }), + + columnHelper.display({ + id: "assignment", + header: "Assignment", + cell: ({ row }) => ( +
+
+ {row.original.assignment?.name || "N/A"} +
+ {row.original.assignment && ( +
+ ID: {row.original.assignment.id} +
+ )} +
+ ), + enableSorting: true, + sortingFn: (rowA, rowB) => { + const a = rowA.original.assignment?.name || ""; + const b = rowB.original.assignment?.name || ""; + return a.localeCompare(b); + }, + }), + + columnHelper.accessor("issueType", { + header: "Issue Type", + cell: ({ getValue }) => getIssueTypeBadge(getValue()), + enableSorting: true, + filterFn: "equals", + }), + + columnHelper.accessor("status", { + header: "Status", + cell: ({ getValue }) => getStatusBadge(getValue()), + enableSorting: true, + filterFn: "equals", + }), + + columnHelper.accessor("description", { + header: "Description", + cell: ({ getValue, row }) => ( +
+
{getValue()}
+ {row.original.statusMessage && ( +
+ Status: {row.original.statusMessage} +
+ )} +
+ ), + enableSorting: true, + }), + + columnHelper.accessor("createdAt", { + header: "Created", + cell: ({ getValue }) => ( +
+
+ {new Date(getValue()).toLocaleDateString()} +
+
+ {new Date(getValue()).toLocaleTimeString()} +
+
+ ), + enableSorting: true, + }), + + columnHelper.display({ + id: "actions", + header: "Actions", + cell: ({ row }) => ( + + ), + }), + ], + [getStatusBadge, getIssueTypeBadge, openReportModal], + ); + + // Create global filter function with dependencies + const globalFilterFn = useMemo(() => { + return (row: any, _columnId: string, value: string) => { + const report = row.original; + const searchValue = value?.toLowerCase() || ""; + + // Apply date range filter + const itemDate = new Date(report.createdAt); + if (startDate && itemDate < new Date(startDate)) { + return false; + } + if (endDate && itemDate > new Date(endDate + "T23:59:59")) { + return false; + } + + // If no search value, return true (item passes filters) + if (!value) return true; + + // Search in description, reporter ID, and assignment name + const descriptionMatch = report.description + .toLowerCase() + .includes(searchValue); + const reporterMatch = report.reporterId + .toLowerCase() + .includes(searchValue); + const assignmentMatch = + report.assignment?.name?.toLowerCase().includes(searchValue) || false; + const idMatch = report.id.toString().includes(searchValue); + + return descriptionMatch || reporterMatch || assignmentMatch || idMatch; + }; + }, [startDate, endDate]); + + // Create table instance + const table = useReactTable({ + data: reports, + columns, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + onGlobalFilterChange: setGlobalFilter, + onPaginationChange: setTablePagination, + state: { + sorting, + columnFilters, + globalFilter, + pagination: tablePagination, + }, + globalFilterFn, + }); + + // Force re-filtering when custom filters change + useEffect(() => { + // Trigger a re-render by toggling the global filter + const currentFilter = globalFilter || ""; + table.setGlobalFilter(currentFilter + " "); + table.setGlobalFilter(currentFilter); + }, [startDate, endDate, table, globalFilter]); + + const exportToCSV = () => { + const filteredData = table + .getFilteredRowModel() + .rows.map((row) => row.original); + + const headers = [ + "ID", + "Reporter ID", + "Assignment Name", + "Issue Type", + "Status", + "Description", + "Is Author", + "Issue Number", + "Status Message", + "Resolution", + "Comments", + "Closure Reason", + "Created At", + "Updated At", + ]; + + const csvContent = [ + headers.join(","), + ...filteredData.map((item) => + [ + item.id, + item.reporterId, + `"${item.assignment?.name || "N/A"}"`, + item.issueType, + item.status, + `"${item.description}"`, + item.author, + item.issueNumber || "", + `"${item.statusMessage || ""}"`, + `"${item.resolution || ""}"`, + `"${item.comments || ""}"`, + `"${item.closureReason || ""}"`, + new Date(item.createdAt).toLocaleString(), + new Date(item.updatedAt).toLocaleString(), + ].join(","), + ), + ].join("\n"); + + const blob = new Blob([csvContent], { type: "text/csv" }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `reports_${new Date().toISOString().split("T")[0]}.csv`; + a.click(); + window.URL.revokeObjectURL(url); + }; + + if (loading && reports.length === 0) { + return ( +
+
+
Loading reports...
+
+
+ ); + } + + if (error) { + return ( +
+
+
Error: {error}
+
+
+ ); + } + + return ( +
+ {/* Filters */} + + +
+ Reports Management +
+ +
+
+ + + {/* Global Search */} +
+ +
+ + setGlobalFilter(e.target.value)} + className="pl-10" + /> + {globalFilter && ( + + )} +
+
+ + {/* Quick Filters Row 1 */} +
+ {/* Status Filter */} +
+ + +
+ + {/* Issue Type Filter */} +
+ + +
+ + {/* Page Size */} +
+ + +
+
+ + {/* Date Range Filters */} +
+
+ +
+ + setStartDate(e.target.value)} + className="pl-10" + /> +
+
+
+ +
+ + setEndDate(e.target.value)} + className="pl-10" + /> +
+
+
+ + {/* Active Filters */} + {(globalFilter || + table.getColumn("status")?.getFilterValue() !== undefined || + table.getColumn("issueType")?.getFilterValue() !== undefined || + startDate || + endDate) && ( +
+
+ Active Filters: +
+
+ {globalFilter && ( + + Search: {globalFilter} + + )} + {table.getColumn("status")?.getFilterValue() !== undefined && ( + + Status:{" "} + {String(table.getColumn("status")?.getFilterValue())} + + )} + {table.getColumn("issueType")?.getFilterValue() !== + undefined && ( + + Type:{" "} + {String(table.getColumn("issueType")?.getFilterValue())} + + )} + {startDate && ( + + From: {startDate} + + )} + {endDate && ( + + To: {endDate} + + )} + +
+
+ )} + + + + {/* Table */} + + +
+ Reports +
+ Showing {table.getFilteredRowModel().rows.length} of{" "} + {reports.length} reports +
+
+ + + {reports.length === 0 ? ( +
+ No reports found +
+ ) : ( +
+ {/* Table */} +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + + {table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
+
+ + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + {header.column.getCanSort() && ( + + {{ + asc: ( + + ), + desc: ( + + ), + }[ + header.column.getIsSorted() as string + ] ?? ( +
+ + +
+ )} + + )} +
+
+ {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} +
+
+
+ + {/* Empty State */} + {table.getFilteredRowModel().rows.length === 0 && + reports.length > 0 && ( +
+
+ +
+

+ No reports match your filters +

+

+ Try adjusting your search terms or filters +

+
+ )} + + {/* Pagination */} + {table.getPageCount() > 1 && ( +
+
+ + Page {table.getState().pagination.pageIndex + 1} of{" "} + {table.getPageCount()} + +
+
+ + +
+
+ )} +
+ )} + + + + {/* Report Modal */} + +
+ ); +} diff --git a/apps/web/app/admin/insights/[id]/page.tsx b/apps/web/app/admin/insights/[id]/page.tsx new file mode 100644 index 00000000..5f4bca4c --- /dev/null +++ b/apps/web/app/admin/insights/[id]/page.tsx @@ -0,0 +1,1770 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { useRouter, useParams } from "next/navigation"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + ArrowLeft, + Users, + FileText, + DollarSign, + Star, + TrendingUp, + Clock, + AlertTriangle, + CheckCircle, + MessageSquare, + Activity, + BarChart3, + ChevronDown, + ChevronUp, + Search, + Filter, + X, + ArrowUpDown, + ArrowUp, + ArrowDown, +} from "lucide-react"; +import { + getCurrentAdminUser, + getDetailedAssignmentInsights, +} from "@/lib/shared"; +import { FeedbackModal } from "@/components/modals/FeedbackModal"; +import { ReportModal } from "@/components/modals/ReportModal"; +import { formatPricePerMillionTokens } from "@/config/constants"; + +interface DetailedInsightData { + assignment: { + id: number; + name: string; + type: string; + published: boolean; + introduction?: string; + instructions?: string; + timeEstimateMinutes?: number; + allotedTimeMinutes?: number; + passingGrade?: number; + createdAt: string; + updatedAt: string; + totalPoints: number; + }; + analytics: { + totalCost: number; + uniqueLearners: number; + totalAttempts: number; + completedAttempts: number; + averageGrade: number; + averageRating: number; + costBreakdown: { + grading: number; + questionGeneration: number; + translation: number; + other: number; + }; + performanceInsights: string[]; + }; + questions: Array<{ + id: number; + question: string; + type: string; + totalPoints: number; + correctPercentage: number; + averagePoints: number; + responseCount: number; + insight: string; + variants: number; + translations: Array<{ languageCode: string }>; + }>; + attempts: Array<{ + id: number; + userId: string; + submitted: boolean; + grade: number | null; + createdAt: string; + timeSpent?: number; + completionRate: number; + }>; + feedback: Array<{ + id: number; + userId: string; + assignmentRating: number | null; + aiGradingRating: number | null; + aiFeedbackRating: number | null; + comments: string | null; + createdAt: string; + }>; + reports: Array<{ + id: number; + issueType: string; + description: string; + status: string; + createdAt: string; + }>; + aiUsage: Array<{ + usageType: string; + tokensIn: number; + tokensOut: number; + usageCount: number; + inputCost: number; + outputCost: number; + totalCost: number; + modelUsed: string; + inputTokenPrice: number; + outputTokenPrice: number; + pricingEffectiveDate: string; + calculationSteps: { + inputCalculation: string; + outputCalculation: string; + totalCalculation: string; + }; + createdAt: string; + }>; + costCalculationDetails?: { + totalCost: number; + breakdown: Array<{ + usageType: string; + tokensIn: number; + tokensOut: number; + modelUsed: string; + inputTokenPrice: number; + outputTokenPrice: number; + inputCost: number; + outputCost: number; + totalCost: number; + pricingEffectiveDate: string; + usageDate: string; + calculationSteps: { + inputCalculation: string; + outputCalculation: string; + totalCalculation: string; + }; + }>; + summary: { + totalInputTokens: number; + totalOutputTokens: number; + totalInputCost: number; + totalOutputCost: number; + averageInputPrice: number; + averageOutputPrice: number; + modelDistribution: Record; + usageTypeDistribution: { + grading: number; + questionGeneration: number; + translation: number; + other: number; + }; + }; + }; + authorActivity?: { + totalAuthors: number; + authors: Array<{ + userId: string; + totalAssignments: number; + totalQuestions: number; + totalAttempts: number; + totalAIUsage: number; + totalFeedback: number; + averageAttemptsPerAssignment: number; + averageQuestionsPerAssignment: number; + recentActivityCount: number; + joinedAt: string; + isActiveContributor: boolean; + activityScore: number; + }>; + activityInsights: string[]; + }; +} + +export default function AssignmentInsightsPage() { + const router = useRouter(); + const params = useParams(); + const assignmentId = params?.id as string; + const [isUserAdmin, setIsUserAdmin] = useState(false); + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [activeTab, setActiveTab] = useState("overview"); + const [showDetailedUsage, setShowDetailedUsage] = useState(false); + + // Attempts filtering state + const [attemptSearch, setAttemptSearch] = useState(""); + const [attemptStatusFilter, setAttemptStatusFilter] = useState("all"); + const [attemptGradeFilter, setAttemptGradeFilter] = useState("all"); + const [attemptSortBy, setAttemptSortBy] = useState("createdAt"); + const [attemptSortOrder, setAttemptSortOrder] = useState("desc"); + + // Modal states + const [selectedFeedback, setSelectedFeedback] = useState(null); + const [selectedReport, setSelectedReport] = useState(null); + const [isFeedbackModalOpen, setIsFeedbackModalOpen] = useState(false); + const [isReportModalOpen, setIsReportModalOpen] = useState(false); + + useEffect(() => { + const fetchData = async () => { + if (!assignmentId) return; + + const sessionToken = localStorage.getItem("adminSessionToken"); + + if (!sessionToken) { + router.push( + `/admin?returnTo=${encodeURIComponent(window.location.pathname)}`, + ); + return; + } + + try { + const user = await getCurrentAdminUser(sessionToken); + setIsUserAdmin(user.isAdmin); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to fetch user"); + } + + try { + setLoading(true); + const response = await getDetailedAssignmentInsights( + sessionToken, + parseInt(assignmentId), + ); + setData(response); + } catch (err) { + setError( + err instanceof Error ? err.message : "Failed to fetch insights", + ); + } finally { + setLoading(false); + } + }; + + fetchData(); + }, [assignmentId, router]); + + const formatCurrency = (amount: number) => { + return new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + minimumFractionDigits: 2, + maximumFractionDigits: 4, + }).format(amount); + }; + + const formatDate = (dateString: string) => { + return new Date(dateString).toLocaleDateString("en-US", { + year: "numeric", + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + }); + }; + + // Modal handlers + const openFeedbackModal = (feedback: any) => { + setSelectedFeedback(feedback); + setIsFeedbackModalOpen(true); + }; + + const closeFeedbackModal = () => { + setIsFeedbackModalOpen(false); + setSelectedFeedback(null); + }; + + const openReportModal = (report: any) => { + setSelectedReport(report); + setIsReportModalOpen(true); + }; + + const closeReportModal = () => { + setIsReportModalOpen(false); + setSelectedReport(null); + }; + + const formatDuration = (minutes?: number) => { + if (!minutes) return "N/A"; + const hours = Math.floor(minutes / 60); + const mins = minutes % 60; + return hours > 0 ? `${hours}h ${mins}m` : `${mins}m`; + }; + + // Filter and sort attempts + const getFilteredAndSortedAttempts = () => { + if (!data?.attempts) return []; + + const filtered = data.attempts.filter((attempt) => { + // Search filter + const searchMatch = + attemptSearch === "" || + attempt.userId.toLowerCase().includes(attemptSearch.toLowerCase()); + + // Status filter + const statusMatch = + attemptStatusFilter === "all" || + (attemptStatusFilter === "submitted" && attempt.submitted) || + (attemptStatusFilter === "in-progress" && !attempt.submitted); + + // Grade filter + const gradeMatch = + attemptGradeFilter === "all" || + (attemptGradeFilter === "passed" && + attempt.grade !== null && + attempt.grade >= 0.6) || + (attemptGradeFilter === "failed" && + attempt.grade !== null && + attempt.grade < 0.6) || + (attemptGradeFilter === "ungraded" && attempt.grade === null); + + return searchMatch && statusMatch && gradeMatch; + }); + + // Sort attempts + filtered.sort((a, b) => { + let aValue: any, bValue: any; + + switch (attemptSortBy) { + case "userId": + aValue = a.userId.toLowerCase(); + bValue = b.userId.toLowerCase(); + break; + case "grade": + aValue = a.grade ?? -1; + bValue = b.grade ?? -1; + break; + case "timeSpent": + aValue = a.timeSpent ?? 0; + bValue = b.timeSpent ?? 0; + break; + case "createdAt": + default: + aValue = new Date(a.createdAt).getTime(); + bValue = new Date(b.createdAt).getTime(); + break; + } + + if (attemptSortOrder === "asc") { + return aValue > bValue ? 1 : aValue < bValue ? -1 : 0; + } else { + return aValue < bValue ? 1 : aValue > bValue ? -1 : 0; + } + }); + + return filtered; + }; + + const filteredAttempts = getFilteredAndSortedAttempts(); + + const clearAttemptFilters = () => { + setAttemptSearch(""); + setAttemptStatusFilter("all"); + setAttemptGradeFilter("all"); + setAttemptSortBy("createdAt"); + setAttemptSortOrder("desc"); + }; + + if (loading) { + return ( +
+
+
+ Loading detailed insights... +
+
+
+ ); + } + + if (error || !data) { + return ( +
+
+
Error: {error || "No data found"}
+
+
+ ); + } + + return ( +
+ {/* Header */} +
+
+
+

{data.assignment.name}

+ + {data.assignment.published ? "Published" : "Draft"} + +
+

+ Assignment ID: {data.assignment.id} +

+
+
+ + {/* Key Metrics */} +
+ + +
+
+

Total Cost

+

+ {formatCurrency(data.analytics.totalCost)} +

+
+ +
+ + + + + +
+
+

Authors

+

+ {data.authorActivity?.totalAuthors || 0} +

+
+ +
+ + + + + +
+
+

Learners

+

+ {data.analytics.uniqueLearners} +

+
+ +
+ + + + + +
+
+

Completion Rate

+

+ {data.analytics.totalAttempts > 0 + ? `${Math.round((data.analytics.completedAttempts / data.analytics.totalAttempts) * 100)}%` + : "N/A"} +

+
+ +
+ + + + + +
+
+

Avg Grade

+

+ {data.analytics.averageGrade > 0 + ? `${data.analytics.averageGrade.toFixed(2)}%` + : "N/A"} +

+
+ +
+ + + + + +
+
+

Rating

+

+ {data.analytics.averageRating > 0 + ? data.analytics.averageRating.toFixed(1) + : "N/A"} +

+
+ +
+ + +
+ + {/* Detailed Tabs */} + + + Overview + Authors + Questions + Attempts + Feedback + AI Usage + {isUserAdmin && Reports} + + + {/* Overview Tab */} + +
+ {/* Assignment Details */} + + + + + Assignment Details + + + +
+

Type

+

+ {data.assignment.type} +

+
+
+

Time Estimate

+

+ {formatDuration(data.assignment.timeEstimateMinutes)} +

+
+
+

Allotted Time

+

+ {formatDuration(data.assignment.allotedTimeMinutes)} +

+
+
+

Passing Grade

+

+ {data.assignment.passingGrade}% +

+
+
+

Total Points

+

+ {data.assignment.totalPoints} +

+
+
+

Created

+

+ {formatDate(data.assignment.createdAt)} +

+
+
+

Last Updated

+

+ {formatDate(data.assignment.updatedAt)} +

+
+ + + + {/* Advanced Cost Breakdown */} + + + + + Cost Analysis Breakdown + + + +
+ {/* Cost per Attempt */} +
+
+ +
+
+ Cost per Attempt +
+
+ {data.analytics.totalAttempts > 0 + ? formatCurrency( + data.analytics.totalCost / + data.analytics.totalAttempts, + ) + : "N/A"} +
+
+ {data.analytics.totalAttempts} total attempts +
+
+ + {/* Authoring Costs */} +
+
+ +
+
+ Authoring Costs +
+
+ {formatCurrency( + data.aiUsage + .filter((usage) => + [ + "TRANSLATION", + "QUESTION_GENERATION", + "ASSIGNMENT_GENERATION", + ].includes(usage.usageType), + ) + .reduce( + (sum, usage) => sum + (usage.totalCost || 0), + 0, + ), + )} +
+
+ Content creation & translation +
+
+ + {/* Learner Grading Costs */} +
+
+ +
+
+ Grading Costs +
+
+ {formatCurrency( + data.aiUsage + .filter((usage) => + [ + "LIVE_RECORDING_FEEDBACK", + "GRADING_VALIDATION", + "ASSIGNMENT_GRADING", + ].includes(usage.usageType), + ) + .reduce( + (sum, usage) => sum + (usage.totalCost || 0), + 0, + ), + )} +
+
+ Student feedback & validation +
+
+
+ + {/* Detailed Breakdown */} +
+
+ {/* Authoring Breakdown */} +
+

+ Authoring Details +

+
+ {data.aiUsage + .filter((usage) => + [ + "TRANSLATION", + "QUESTION_GENERATION", + "ASSIGNMENT_GENERATION", + ].includes(usage.usageType), + ) + .map((usage, index) => ( +
+ + {usage.usageType + .replace("_", " ") + .toLowerCase() + .replace(/\b\w/g, (l) => l.toUpperCase())} + + + {formatCurrency(usage.totalCost || 0)} + +
+ ))} +
+
+ + {/* Grading Breakdown */} +
+

+ Grading Details +

+
+ {data.aiUsage + .filter((usage) => + [ + "LIVE_RECORDING_FEEDBACK", + "GRADING_VALIDATION", + "ASSIGNMENT_GRADING", + ].includes(usage.usageType), + ) + .map((usage, index) => ( +
+ + {usage.usageType + .replace("_", " ") + .toLowerCase() + .replace(/\b\w/g, (l) => l.toUpperCase())} + + + {formatCurrency(usage.totalCost || 0)} + +
+ ))} +
+
+
+
+ + +
+ + {/* Performance Insights */} + {data.analytics.performanceInsights.length > 0 && ( + + + + + Performance Insights + + + +
    + {data.analytics.performanceInsights.map((insight, index) => ( +
  • + + {insight} +
    • + ))} +
+ + + )} + + + {/* Authors Tab */} + + {data.authorActivity && data.authorActivity.totalAuthors > 0 ? ( +
+ {/* Author Summary Cards */} +
+ + +
+
+

+ Total Authors +

+

+ {data.authorActivity.totalAuthors} +

+

+ { + data.authorActivity.authors.filter( + (a) => a.isActiveContributor, + ).length + }{" "} + active contributors +

+
+ +
+ + + + + +
+
+

+ Most Active +

+

+ {data.authorActivity.authors[0]?.userId || "N/A"} +

+

+ {data.authorActivity.authors[0]?.totalAssignments || + 0}{" "} + assignments +

+
+ +
+ + + + + +
+
+

+ Total Assignments +

+

+ {data.authorActivity.authors.reduce( + (sum, a) => sum + a.totalAssignments, + 0, + )} +

+

+ by all authors combined +

+
+ +
+ + +
+ + {/* Activity Insights */} + {data.authorActivity.activityInsights.length > 0 && ( + + + + + Author Activity Insights + + + +
    + {data.authorActivity.activityInsights.map( + (insight, index) => ( +
  • + + {insight} +
    • + ), + )} +
+ + + )} + + {/* Detailed Authors Table */} + + + Author Activity Details + + + + + + Author + + Activity Score + + + Assignments + + + Questions published + + AI Usage + Feedback + Joined + Status + + + + {data.authorActivity.authors.map((author) => ( + + +
+
+ + {author.userId + .split("@")[0] + ?.substring(0, 2) + .toUpperCase() || "AU"} + +
+
+
+ {author.userId.split("@")[0]} +
+
+ {author.userId.split("@")[1]} +
+
+
+ + + 10 + ? "default" + : "secondary" + } + > + {author.activityScore} + + + + {author.totalAssignments} + + + {author.totalQuestions} + + + {author.totalAIUsage > 1000 + ? "Ridiculous Usage!" + : author.totalAIUsage > 500 + ? "Very High Usage" + : author.totalAIUsage > 100 + ? "High Usage" + : author.totalAIUsage > 50 + ? "Moderate Usage" + : author.totalAIUsage > 0 + ? "Low Usage" + : "No Usage"} + + + {author.totalFeedback} + + + {formatDate(author.joinedAt)} + + + + {author.isActiveContributor + ? "Active" + : "Occasional"} + + + + ))} + +
+ + +
+ ) : ( + + +
+ No author information available for this assignment +
+ + + )} + + + {/* Questions Tab */} + + + + Question Performance Analysis + + + + + + Question + Type + Points + Pass Rate % + Avg Points + Responses + Variants + Languages + + + + {data.questions.map((question) => ( + + +
+ {question.question} +
+
+ {question.insight} +
+ + + {question.type} + + + {question.totalPoints} + + + + {Math.round(question.correctPercentage)}% + + + + {question.averagePoints.toFixed(1)} + + + {question.responseCount} + + + {question.variants} + + + {question.translations.length} + + + ))} + +
+ + + + + {/* Attempts Tab */} + + + +
+
+ Assignment Attempts +

+ {filteredAttempts.length} of {data.attempts.length} attempts +

+
+ +
+ + + {/* Search and Filters */} +
+ {/* Search */} +
+ + setAttemptSearch(e.target.value)} + className="pl-9" + /> + {attemptSearch && ( + + )} +
+ + {/* Status Filter */} + + + {/* Grade Filter */} + + + {/* Sort */} +
+ + +
+
+ + {/* Summary Stats */} +
+
+
+ {filteredAttempts.length} +
+
+ Total Shown +
+
+
+
+ {filteredAttempts.filter((a) => a.submitted).length} +
+
Submitted
+
+
+
+ {filteredAttempts.filter((a) => !a.submitted).length} +
+
+ In Progress +
+
+
+
+ { + filteredAttempts.filter( + (a) => a.grade !== null && a.grade >= 0.6, + ).length + } +
+
Passed
+
+
+ + {/* Table */} + + + + setAttemptSortBy("userId")} + > +
+ User ID + {attemptSortBy === "userId" && + (attemptSortOrder === "asc" ? ( + + ) : ( + + ))} +
+ + Status + setAttemptSortBy("grade")} + > +
+ Grade + {attemptSortBy === "grade" && + (attemptSortOrder === "asc" ? ( + + ) : ( + + ))} +
+ + setAttemptSortBy("createdAt")} + > +
+ Started + {attemptSortBy === "createdAt" && + (attemptSortOrder === "asc" ? ( + + ) : ( + + ))} +
+ + setAttemptSortBy("timeSpent")} + > +
+ Time Spent + {attemptSortBy === "timeSpent" && + (attemptSortOrder === "asc" ? ( + + ) : ( + + ))} +
+ + + + + {filteredAttempts.length === 0 ? ( + + + No attempts match the current filters + + + ) : ( + filteredAttempts.map((attempt) => ( + + + {attempt.userId} + + + + {attempt.submitted ? "Submitted" : "In Progress"} + + + + {attempt.grade !== null ? ( + = 0.6 + ? "text-green-600 font-semibold" + : "text-red-600 font-semibold" + } + > + {Math.round(attempt.grade * 100)}% + + ) : ( + "N/A" + )} + + {formatDate(attempt.createdAt)} + + {attempt.timeSpent !== null && + attempt.timeSpent !== undefined + ? formatDuration(attempt.timeSpent) + : "N/A"} + + + )) + )} + +
+ + + + + {/* Feedback Tab */} + + + + User Feedback + + + {data.feedback.length === 0 ? ( +
+ No feedback received yet +
+ ) : ( + + + + User + + Assignment Rating + + AI Grading + AI Feedback + Comments + Date + Actions + + + + {data.feedback.map((feedback) => ( + + + {feedback.userId} + + + {feedback.assignmentRating ? ( +
+ + {feedback.assignmentRating} +
+ ) : ( + "N/A" + )} + + + {feedback.aiGradingRating ? ( +
+ + {feedback.aiGradingRating} +
+ ) : ( + "N/A" + )} + + + {feedback.aiFeedbackRating ? ( +
+ + {feedback.aiFeedbackRating} +
+ ) : ( + "N/A" + )} + + +
+ {feedback.comments || "No comments"} +
+ + {formatDate(feedback.createdAt)} + + + + + ))} + +
+ )} + + + + + {/* AI Usage Tab */} + + {/* Cost Category Summary */} +
+ {/* Authoring Costs */} + + + + + Authoring Costs + + + +
+ {formatCurrency( + data.aiUsage + .filter((usage) => + [ + "TRANSLATION", + "QUESTION_GENERATION", + "ASSIGNMENT_GENERATION", + ].includes(usage.usageType), + ) + .reduce((sum, usage) => sum + (usage.totalCost || 0), 0), + )} +
+
+ {data.aiUsage + .filter((usage) => + [ + "TRANSLATION", + "QUESTION_GENERATION", + "ASSIGNMENT_GENERATION", + ].includes(usage.usageType), + ) + .map((usage, index) => ( +
+ + {usage.usageType + .replace("_", " ") + .toLowerCase() + .replace(/\b\w/g, (l) => l.toUpperCase())} + + + {formatCurrency(usage.totalCost || 0)} + +
+ ))} +
+ + + + {/* Grading Costs */} + + + + + Grading Costs + + + +
+ {formatCurrency( + data.aiUsage + .filter((usage) => + [ + "LIVE_RECORDING_FEEDBACK", + "GRADING_VALIDATION", + "ASSIGNMENT_GRADING", + ].includes(usage.usageType), + ) + .reduce((sum, usage) => sum + (usage.totalCost || 0), 0), + )} +
+
+ {data.aiUsage + .filter((usage) => + [ + "LIVE_RECORDING_FEEDBACK", + "GRADING_VALIDATION", + "ASSIGNMENT_GRADING", + ].includes(usage.usageType), + ) + .map((usage, index) => ( +
+ + {usage.usageType + .replace("_", " ") + .toLowerCase() + .replace(/\b\w/g, (l) => l.toUpperCase())} + + + {formatCurrency(usage.totalCost || 0)} + +
+ ))} +
+ + +
+ + {/* AI Usage Table */} + + +
+
+ AI Usage Details +

+ Detailed breakdown of AI usage by type and model +

+
+ +
+ + + + + + Usage Type + Model Used + Total Cost + {showDetailedUsage && ( + <> + Tokens In + + Tokens Out + + + Input Cost + + + Output Cost + + Last Used On + + )} + + + + {data.aiUsage.map((usage, index) => ( + + + + {usage.usageType.replace("_", " ")} + + + +
+ {usage.modelUsed} + {showDetailedUsage && ( +
+ In:{" "} + {formatPricePerMillionTokens( + usage.inputTokenPrice, + )} + /1M | Out:{" "} + {formatPricePerMillionTokens( + usage.outputTokenPrice, + )} + /1M +
+ )} +
+ + + {formatCurrency(usage.totalCost)} + + {showDetailedUsage && ( + <> + + {usage.tokensIn.toLocaleString()} + + + {usage.tokensOut.toLocaleString()} + + + {formatCurrency(usage.inputCost)} + + + {formatCurrency(usage.outputCost)} + + +
+
{formatDate(usage.createdAt)}
+
+ Pricing:{" "} + {formatDate(usage.pricingEffectiveDate)} +
+
+ + + )} + + ))} + +
+ + {/* Detailed Calculation Steps - only shown when details are expanded */} + {showDetailedUsage && ( +
+

Calculation Details

+ {data.aiUsage.map((usage, index) => ( +
+
+
+ {usage.usageType} + {usage.modelUsed} +
+ + {formatCurrency(usage.totalCost)} + +
+
+
+ {usage.calculationSteps.inputCalculation} +
+
+ {usage.calculationSteps.outputCalculation} +
+
+ {usage.calculationSteps.totalCalculation} +
+
+
+ ))} +
+ )} + + + + + {/* Reports Tab */} + {isUserAdmin && ( + + + + Issue Reports + + + {data.reports.length === 0 ? ( +
+ No reports submitted +
+ ) : ( + + + + Issue Type + Description + Status + Date + Actions + + + + {data.reports.map((report) => ( + + + {report.issueType} + + +
+ {report.description} +
+ + + + {report.status} + + + {formatDate(report.createdAt)} + + + + + ))} + +
+ )} + + + + )} + + + {/* Modals */} + + + +
+ ); +} diff --git a/apps/web/app/admin/llm-assignments/page.tsx b/apps/web/app/admin/llm-assignments/page.tsx new file mode 100644 index 00000000..971761c2 --- /dev/null +++ b/apps/web/app/admin/llm-assignments/page.tsx @@ -0,0 +1,484 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { useRouter } from "next/navigation"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + ArrowLeft, + Settings, + Zap, + CheckCircle, + AlertTriangle, + RefreshCw, + RotateCcw, +} from "lucide-react"; +import { formatPricePerMillionTokens } from "@/config/constants"; + +interface AIFeature { + id: number; + featureKey: string; + featureType: string; + displayName: string; + description?: string; + isActive: boolean; + requiresModel: boolean; + defaultModelKey?: string; + assignedModel?: { + id: number; + modelKey: string; + displayName: string; + provider: string; + priority: number; + assignedBy?: string; + assignedAt: Date; + }; +} + +interface LLMModel { + id: number; + modelKey: string; + displayName: string; + provider: string; + isActive: boolean; + currentPricing: { + inputTokenPrice: number; + outputTokenPrice: number; + } | null; + assignedFeatures: Array<{ + featureKey: string; + featureDisplayName: string; + priority: number; + }>; +} + +export default function LLMAssignmentsPage() { + const router = useRouter(); + const [features, setFeatures] = useState([]); + const [models, setModels] = useState([]); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(null); + const [changes, setChanges] = useState>(new Map()); + + useEffect(() => { + fetchData(); + }, []); + + const fetchData = async () => { + const sessionToken = localStorage.getItem("adminSessionToken"); + if (!sessionToken) { + router.push( + `/admin?returnTo=${encodeURIComponent(window.location.pathname)}`, + ); + return; + } + + try { + setLoading(true); + setError(null); + + const [featuresRes, modelsRes] = await Promise.all([ + fetch("/api/v1/llm-assignments/features", { + headers: { "x-admin-token": sessionToken }, + }), + fetch("/api/v1/llm-assignments/models", { + headers: { "x-admin-token": sessionToken }, + }), + ]); + + if (!featuresRes.ok || !modelsRes.ok) { + throw new Error("Failed to fetch data"); + } + + const featuresData = await featuresRes.json(); + const modelsData = await modelsRes.json(); + + setFeatures(featuresData.data || []); + setModels(modelsData.data || []); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to fetch data"); + } finally { + setLoading(false); + } + }; + + const handleModelChange = (featureKey: string, modelKey: string) => { + const newChanges = new Map(changes); + + // Find the current assignment + const feature = features.find((f) => f.featureKey === featureKey); + const currentModelKey = feature?.assignedModel?.modelKey; + + if (currentModelKey === modelKey) { + // No change from current assignment + newChanges.delete(featureKey); + } else { + newChanges.set(featureKey, modelKey); + } + + setChanges(newChanges); + }; + + const saveChanges = async () => { + if (changes.size === 0) return; + + const sessionToken = localStorage.getItem("adminSessionToken"); + if (!sessionToken) return; + + try { + setSaving(true); + setError(null); + setSuccess(null); + + const assignments = Array.from(changes.entries()).map( + ([featureKey, modelKey]) => ({ + featureKey, + modelKey, + }), + ); + + const response = await fetch("/api/v1/llm-assignments/bulk-assign", { + method: "PUT", + headers: { + "Content-Type": "application/json", + "x-admin-token": sessionToken, + }, + body: JSON.stringify({ assignments }), + }); + + if (!response.ok) { + throw new Error("Failed to save assignments"); + } + + const result = await response.json(); + + if (result.success) { + setSuccess( + `Successfully updated ${result.data.successful} assignments`, + ); + setChanges(new Map()); + await fetchData(); // Refresh data + } else { + throw new Error(result.message || "Failed to save assignments"); + } + } catch (err) { + setError( + err instanceof Error ? err.message : "Failed to save assignments", + ); + } finally { + setSaving(false); + } + }; + + const resetToDefaults = async () => { + const sessionToken = localStorage.getItem("adminSessionToken"); + if (!sessionToken) return; + + try { + setSaving(true); + setError(null); + setSuccess(null); + + const response = await fetch( + "/api/v1/llm-assignments/reset-to-defaults", + { + method: "POST", + headers: { "x-admin-token": sessionToken }, + }, + ); + + if (!response.ok) { + throw new Error("Failed to reset to defaults"); + } + + const result = await response.json(); + + if (result.success) { + setSuccess( + `Successfully reset ${result.data.resetCount} features to default models`, + ); + setChanges(new Map()); + await fetchData(); + } else { + throw new Error(result.message || "Failed to reset to defaults"); + } + } catch (err) { + setError( + err instanceof Error ? err.message : "Failed to reset to defaults", + ); + } finally { + setSaving(false); + } + }; + + const getEffectiveModel = (feature: AIFeature): string => { + const pendingChange = changes.get(feature.featureKey); + if (pendingChange) return pendingChange; + + return feature.assignedModel?.modelKey || feature.defaultModelKey || "none"; + }; + + const getAvailableModelsForFeature = (feature: AIFeature): LLMModel[] => { + const visionCapableFeatures = [ + "image_grading", + "presentation_grading", + "video_grading", + ]; + + const visionModels = ["gpt-4.1-mini"]; // Vision-capable models + + if (visionCapableFeatures.includes(feature.featureKey)) { + // For vision features, show all models (including vision models) + return models.filter((m) => m.isActive); + } else { + // For non-vision features, exclude vision-only models + return models.filter( + (m) => m.isActive && !visionModels.includes(m.modelKey), + ); + } + }; + + const hasChanges = changes.size > 0; + + if (loading) { + return ( +
+
+
+ Loading LLM assignments... +
+
+
+ ); + } + + return ( +
+ {/* Header */} +
+ +
+
+ +

LLM Feature Assignments

+
+

+ Manage which AI models are used for different features +

+
+
+ + +
+
+ + {/* Status Messages */} + {error && ( + + + {error} + + )} + + {success && ( + + + {success} + + )} + + {/* Feature Assignments Table */} + + + + + AI Feature Assignments + + + + + + + Feature + Type + Current Model + Assigned Model + Status + + + + {features.map((feature) => { + const effectiveModel = getEffectiveModel(feature); + const hasChange = changes.has(feature.featureKey); + const selectedModel = models.find( + (m) => m.modelKey === effectiveModel, + ); + + return ( + + +
+
{feature.displayName}
+
+ {feature.description} +
+
+ + + + {feature.featureType.replace("_", " ")} + + + + {feature.assignedModel ? ( +
+
+ {feature.assignedModel.displayName} +
+
+ {feature.assignedModel.provider} •{" "} + {feature.assignedModel.modelKey} +
+
+ ) : ( +
+ Default: {feature.defaultModelKey} +
+ )} + + + + + +
+ {feature.isActive ? ( + Active + ) : ( + Inactive + )} + {hasChange && ( + + Modified + + )} +
+ + + ); + })} + +
+ + + + {/* Available Models Summary */} + + + Available Models + + +
+ {models.map((model) => ( +
+
+

{model.displayName}

+ + {model.isActive ? "Active" : "Inactive"} + +
+
+
+ {model.provider} • {model.modelKey} +
+ {model.currentPricing && ( +
+ {formatPricePerMillionTokens( + model.currentPricing.inputTokenPrice, + )} + /1M input tokens •{" "} + {formatPricePerMillionTokens( + model.currentPricing.outputTokenPrice, + )} + /1M output tokens +
+ )} +
+
+ Used by {model.assignedFeatures.length} feature + {model.assignedFeatures.length !== 1 ? "s" : ""} +
+
+ ))} +
+ + +
+ ); +} diff --git a/apps/web/app/admin/page.tsx b/apps/web/app/admin/page.tsx new file mode 100644 index 00000000..b742a1ed --- /dev/null +++ b/apps/web/app/admin/page.tsx @@ -0,0 +1,144 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { useRouter, useSearchParams } from "next/navigation"; +import { getUser } from "@/lib/talkToBackend"; +import Loading from "@/components/Loading"; +import animationData from "@/animations/LoadSN.json"; +import { AdminLogin } from "./components/AdminLogin"; +import { OptimizedAdminDashboard } from "./components/AdminDashboard"; + +export default function AdminPage() { + const [isLoading, setIsLoading] = useState(true); + const [isAuthenticated, setIsAuthenticated] = useState(false); + const [sessionToken, setSessionToken] = useState(null); + const [userRole, setUserRole] = useState(null); + const router = useRouter(); + const searchParams = useSearchParams(); + const returnTo = searchParams.get("returnTo"); + + useEffect(() => { + const checkAdminAccess = async () => { + try { + // First, check if we have a valid admin session token + const adminToken = localStorage.getItem("adminSessionToken"); + const adminEmail = localStorage.getItem("adminEmail"); + const expiresAt = localStorage.getItem("adminExpiresAt"); + + if (adminToken && adminEmail && expiresAt) { + const expireDate = new Date(expiresAt); + + if (expireDate > new Date()) { + // Session appears valid, let's verify it with the backend + try { + const response = await fetch( + "/api/v1/reports/feedback?page=1&limit=1", + { + headers: { + "x-admin-token": adminToken, + }, + }, + ); + + if (response.ok) { + // Session is valid with backend + setSessionToken(adminToken); + setIsAuthenticated(true); + setUserRole("admin"); + setIsLoading(false); + + // If user already has valid session and there's a returnTo parameter, redirect + if (returnTo) { + router.push(returnTo); + } + return; + } else { + // Session invalid, clear it + localStorage.removeItem("adminSessionToken"); + localStorage.removeItem("adminEmail"); + localStorage.removeItem("adminExpiresAt"); + } + } catch (apiError) { + console.error("Error validating session with backend:", apiError); + // Clear potentially invalid session + localStorage.removeItem("adminSessionToken"); + localStorage.removeItem("adminEmail"); + localStorage.removeItem("adminExpiresAt"); + } + } else { + // Session expired, clear it + localStorage.removeItem("adminSessionToken"); + localStorage.removeItem("adminEmail"); + localStorage.removeItem("adminExpiresAt"); + } + } + } catch (error) { + console.error("Failed to check admin access:", error); + // Will show login form + } finally { + setIsLoading(false); + } + }; + + checkAdminAccess(); + }, [router, returnTo]); + + const handleAuthenticated = (token: string) => { + setSessionToken(token); + setIsAuthenticated(true); + setUserRole("admin"); + + // Redirect to the original destination if returnTo parameter exists + if (returnTo) { + router.push(returnTo); + } + }; + + const handleLogout = async () => { + const adminToken = localStorage.getItem("adminSessionToken"); + + if (adminToken) { + try { + await fetch("/api/v1/auth/admin/logout", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ sessionToken: adminToken }), + }); + } catch (error) { + console.error("Failed to logout:", error); + } + } + + // Clear local storage + localStorage.removeItem("adminSessionToken"); + localStorage.removeItem("adminEmail"); + localStorage.removeItem("adminExpiresAt"); + + // Reset state + setSessionToken(null); + setIsAuthenticated(false); + setUserRole(null); + + // Redirect to home page + router.push("/"); + }; + + if (isLoading) { + return ; + } + + if (!isAuthenticated) { + return ; + } + + return ( +
+ +
+ ); +} diff --git a/apps/web/app/api/markChat/route.ts b/apps/web/app/api/markChat/route.ts index efcc02bf..cbbf7dbb 100644 --- a/apps/web/app/api/markChat/route.ts +++ b/apps/web/app/api/markChat/route.ts @@ -1,8 +1,9 @@ +/* eslint-disable */ +import { authorTools, learnerTools } from "./stream/route"; import { searchKnowledgeBase } from "@/app/chatbot/lib/markChatFunctions"; -/* eslint-disable */ import { NextRequest, NextResponse } from "next/server"; +import { NextRequest, NextResponse } from "next/server"; import OpenAI from "openai"; import { z } from "zod"; -import { authorTools, learnerTools } from "./stream/route"; const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, diff --git a/apps/web/app/api/markChat/services/markChatService.ts b/apps/web/app/api/markChat/services/markChatService.ts index fc618948..878aabc1 100644 --- a/apps/web/app/api/markChat/services/markChatService.ts +++ b/apps/web/app/api/markChat/services/markChatService.ts @@ -1,10 +1,9 @@ /* eslint-disable */ - -import { executeAuthorStoreOperation } from "@/app/chatbot/store/authorStoreUtil"; -import { searchKnowledgeBase } from "@/app/chatbot/knowledgebase"; import { ReportingService } from "./reportingService"; -import { IssueSeverity } from "@/config/types"; +import { searchKnowledgeBase } from "@/app/chatbot/knowledgebase"; +import { executeAuthorStoreOperation } from "@/app/chatbot/store/authorStoreUtil"; import { getBaseApiPath } from "@/config/constants"; +import { IssueSeverity } from "@/config/types"; /** * Get or create a chat session for today @@ -112,7 +111,7 @@ export class MarkChatService { [key: string]: any; } = {}, cookieHeader?: string, - ): Promise { + ): Promise<{ content: string; reportId?: number; issueNumber?: number }> { try { const title = `[${details.userRole?.toUpperCase() || "USER"}] ${issueType.charAt(0).toUpperCase() + issueType.slice(1)} Issue Report`; @@ -131,12 +130,17 @@ export class MarkChatService { cookieHeader, ); - return ( - result.content || - `Thank you for your report. Our team will review it shortly.` - ); + return { + content: + result.content || + `Thank you for your report. Our team will review it shortly.`, + reportId: result.reportId, + issueNumber: result.issueNumber, + }; } catch (error) { - return `There was an error submitting your issue report. Please try again later. (Error: ${error.message})`; + return { + content: `There was an error submitting your issue report. Please try again later. (Error: ${error.message})`, + }; } } diff --git a/apps/web/app/api/markChat/services/reportingService.ts b/apps/web/app/api/markChat/services/reportingService.ts index a93a0293..19c80196 100644 --- a/apps/web/app/api/markChat/services/reportingService.ts +++ b/apps/web/app/api/markChat/services/reportingService.ts @@ -1,5 +1,4 @@ /* eslint-disable */ - import { getBaseApiPath } from "@/config/constants"; import { IssueSeverity } from "@/config/types"; import { BASE_API_ROUTES } from "@/lib/talkToBackend"; @@ -19,6 +18,7 @@ interface ReportResponse { content: string; issueId?: string | number; issueNumber?: number; + reportId?: number; error?: string; } @@ -51,7 +51,6 @@ export class ReportingService { category: details.category || "General Issue", ...details, }; - console.log("Forwarding cookies:", cookieHeader); const response = await fetch(`${getBaseApiPath("v1")}/reports`, { method: "POST", @@ -80,6 +79,7 @@ export class ReportingService { data.message || `Thank you for reporting this issue. Our team will review it shortly.`, issueNumber: data.issueNumber, + reportId: data.reportId, }; } catch (error) { return { diff --git a/apps/web/app/api/markChat/stream/route.ts b/apps/web/app/api/markChat/stream/route.ts index 2120777a..ddb840c7 100644 --- a/apps/web/app/api/markChat/stream/route.ts +++ b/apps/web/app/api/markChat/stream/route.ts @@ -1,9 +1,4 @@ /* eslint-disable */ - -import { streamText } from "ai"; -import { openai } from "@ai-sdk/openai"; -import { z } from "zod"; - import { MarkChatService } from "../services/markChatService"; import { getAssignmentRubric, @@ -12,6 +7,9 @@ import { searchKnowledgeBase, submitFeedbackQuestion, } from "@/app/chatbot/lib/markChatFunctions"; +import { openai } from "@ai-sdk/openai"; +import { streamText } from "ai"; +import { z } from "zod"; const STANDARD_ERROR_MESSAGE = "Sorry for the inconvenience, I am still new around here and this capability is not there yet, my developers are working on it!"; @@ -51,12 +49,17 @@ export function learnerTools(cookieHeader: string) { }), }, reportIssue: { - description: "Report a technical issue or bug with the platform", + description: + "Report a technical issue or bug with the platform. Extract the user's issue description and use it to prefill the form.", parameters: z.object({ issueType: z .enum(["technical", "content", "grading", "other"]) .describe("The type of issue being reported"), - description: z.string().describe("Detailed description of the issue"), + description: z + .string() + .describe( + "Detailed description of the issue - extract this from the user's message to prefill the form", + ), assignmentId: z .number() .optional() @@ -68,19 +71,131 @@ export function learnerTools(cookieHeader: string) { .optional() .describe("The severity of the issue"), }), - execute: withErrorHandling( - async ({ issueType, description, assignmentId, severity }) => { - // const res = await MarkChatService.reportIssue(issueType, description, { - // assignmentId, - // userRole: "learner", - // severity: severity || "info", - // category: "Learner Issue", - // cookieHeader, - // }); - - return STANDARD_ERROR_MESSAGE; - }, - ), + execute: async ({ issueType, description, assignmentId, severity }) => { + // Return a client execution request for form preview instead of immediate submission + return JSON.stringify({ + clientExecution: true, + function: "showReportPreview", + params: { + type: "report", + issueType, + description, + assignmentId, + severity: severity || "info", + userRole: "learner", + category: "Learner Issue", + }, + }); + }, + }, + provideFeedback: { + description: + "Provide general feedback about the learning experience or platform. Extract the user's feedback text and use it as the description to prefill the form.", + parameters: z.object({ + feedbackType: z + .enum(["general", "assignment", "grading", "experience"]) + .describe("The type of feedback being provided"), + description: z + .string() + .describe( + "Detailed feedback comments - extract this from the user's message to prefill the form", + ), + assignmentId: z + .number() + .optional() + .describe( + "The ID of the assignment (if feedback is assignment-specific)", + ), + rating: z + .number() + .min(1) + .max(5) + .optional() + .describe("Optional rating from 1-5 stars"), + }), + execute: async ({ feedbackType, description, assignmentId, rating }) => { + return JSON.stringify({ + clientExecution: true, + function: "showReportPreview", + params: { + type: "feedback", + issueType: "FEEDBACK", + description, + assignmentId, + rating, + userRole: "learner", + category: "Learner Feedback", + }, + }); + }, + }, + submitSuggestion: { + description: + "Submit suggestions for improving the platform or assignments. Extract the user's suggestion text and use it as the description to prefill the form.", + parameters: z.object({ + suggestionType: z + .enum(["feature", "content", "ui", "general"]) + .describe("The type of suggestion being made"), + description: z + .string() + .describe( + "Detailed suggestion or improvement idea - extract this from the user's message to prefill the form", + ), + assignmentId: z + .number() + .optional() + .describe( + "The ID of the assignment (if suggestion is assignment-specific)", + ), + }), + execute: async ({ suggestionType, description, assignmentId }) => { + return JSON.stringify({ + clientExecution: true, + function: "showReportPreview", + params: { + type: "suggestion", + issueType: "SUGGESTION", + description, + assignmentId, + userRole: "learner", + category: "Learner Suggestion", + }, + }); + }, + }, + submitInquiry: { + description: + "Submit general questions or inquiries about the platform or assignments. Extract the user's question text and use it as the description to prefill the form.", + parameters: z.object({ + inquiryType: z + .enum(["general", "technical", "academic", "other"]) + .describe("The type of inquiry being made"), + description: z + .string() + .describe( + "The question or inquiry details - extract this from the user's message to prefill the form", + ), + assignmentId: z + .number() + .optional() + .describe( + "The ID of the assignment (if inquiry is assignment-specific)", + ), + }), + execute: async ({ inquiryType, description, assignmentId }) => { + return JSON.stringify({ + clientExecution: true, + function: "showReportPreview", + params: { + type: "inquiry", + issueType: "OTHER", + description, + assignmentId, + userRole: "learner", + category: "Learner Inquiry", + }, + }); + }, }, getQuestionDetails: { description: @@ -362,12 +477,17 @@ export function authorTools(cookieHeader: string) { }), }, reportIssue: { - description: "Report a technical issue or bug with the platform", + description: + "Report a technical issue or bug with the platform. Extract the user's issue description and use it to prefill the form.", parameters: z.object({ issueType: z .enum(["technical", "content", "grading", "other"]) .describe("The type of issue being reported"), - description: z.string().describe("Detailed description of the issue"), + description: z + .string() + .describe( + "Detailed description of the issue - extract this from the user's message to prefill the form", + ), assignmentId: z .number() .optional() @@ -379,20 +499,130 @@ export function authorTools(cookieHeader: string) { .optional() .describe("The severity of the issue"), }), - execute: withErrorHandling( - async ({ issueType, description, assignmentId, severity }) => { - // const res = await MarkChatService.reportIssue(issueType, description, { - // assignmentId, - // userRole: "author", - // severity: severity || "info", - // category: "Author Issue", - // cookieHeader, - // }); - - // return typeof res === "string" ? res : res || STANDARD_ERROR_MESSAGE; - return STANDARD_ERROR_MESSAGE; - }, - ), + execute: async ({ issueType, description, assignmentId, severity }) => { + // Return a client execution request for form preview instead of immediate submission + return JSON.stringify({ + clientExecution: true, + function: "showReportPreview", + params: { + issueType, + description, + assignmentId, + severity: severity || "info", + userRole: "author", + category: "Author Issue", + }, + }); + }, + }, + provideFeedback: { + description: + "Provide general feedback about the teaching experience or platform. Extract the user's feedback text and use it as the description to prefill the form.", + parameters: z.object({ + feedbackType: z + .enum(["general", "assignment", "grading", "experience"]) + .describe("The type of feedback being provided"), + description: z + .string() + .describe( + "Detailed feedback comments - extract this from the user's message to prefill the form", + ), + assignmentId: z + .number() + .optional() + .describe( + "The ID of the assignment (if feedback is assignment-specific)", + ), + rating: z + .number() + .min(1) + .max(5) + .optional() + .describe("Optional rating from 1-5 stars"), + }), + execute: async ({ feedbackType, description, assignmentId, rating }) => { + return JSON.stringify({ + clientExecution: true, + function: "showReportPreview", + params: { + type: "feedback", + issueType: "FEEDBACK", + description, + assignmentId, + rating, + userRole: "author", + category: "Author Feedback", + }, + }); + }, + }, + submitSuggestion: { + description: + "Submit suggestions for improving the platform or teaching tools. Extract the user's suggestion text and use it as the description to prefill the form.", + parameters: z.object({ + suggestionType: z + .enum(["feature", "content", "ui", "general"]) + .describe("The type of suggestion being made"), + description: z + .string() + .describe( + "Detailed suggestion or improvement idea - extract this from the user's message to prefill the form", + ), + assignmentId: z + .number() + .optional() + .describe( + "The ID of the assignment (if suggestion is assignment-specific)", + ), + }), + execute: async ({ suggestionType, description, assignmentId }) => { + return JSON.stringify({ + clientExecution: true, + function: "showReportPreview", + params: { + type: "suggestion", + issueType: "SUGGESTION", + description, + assignmentId, + userRole: "author", + category: "Author Suggestion", + }, + }); + }, + }, + submitInquiry: { + description: + "Submit general questions or inquiries about the platform or assignments. Extract the user's question text and use it as the description to prefill the form.", + parameters: z.object({ + inquiryType: z + .enum(["general", "technical", "academic", "other"]) + .describe("The type of inquiry being made"), + description: z + .string() + .describe( + "The question or inquiry details - extract this from the user's message to prefill the form", + ), + assignmentId: z + .number() + .optional() + .describe( + "The ID of the assignment (if inquiry is assignment-specific)", + ), + }), + execute: async ({ inquiryType, description, assignmentId }) => { + return JSON.stringify({ + clientExecution: true, + function: "showReportPreview", + params: { + type: "inquiry", + issueType: "OTHER", + description, + assignmentId, + userRole: "author", + category: "Author Inquiry", + }, + }); + }, }, }; } @@ -400,6 +630,7 @@ export function authorTools(cookieHeader: string) { function generateSystemPrompt(userRole, assignmentInfo) { const assignmentMode = assignmentInfo?.mode || "unknown"; const isSubmitted = assignmentInfo?.submitted === true; + const assignmentId = assignmentInfo?.assignmentId; const systemPrompts = { author: `You are Mark, an AI assistant for assignment authors on an educational platform. Your primary purpose is to help instructors create high-quality educational content through direct action. @@ -459,7 +690,12 @@ TOOL USAGE: - Use deleteQuestion for removing questions - Use generateQuestionsFromObjectives for AI-generated content - Use updateLearningObjectives for curriculum planning -- Use reportIssue only after exhausting troubleshooting options +- Use reportIssue only for technical issues after exhausting troubleshooting options +- Use provideFeedback for sharing general feedback about teaching experience +- Use submitSuggestion for platform or teaching tool improvement ideas +- Use submitInquiry for general questions or inquiries + +IMPORTANT: ${assignmentId ? `When calling tools that require assignmentId, always use ${assignmentId}` : "Assignment ID information is not available in the current context"} RESPONSE STYLE: - Be conversational and encouraging @@ -573,11 +809,16 @@ EMOTIONAL SUPPORT & ENCOURAGEMENT: TOOL USAGE: - Use searchKnowledgeBase for platform help -- Use reportIssue ONLY after troubleshooting +- Use reportIssue ONLY for technical issues after troubleshooting - Use getQuestionDetails for question information - Use getAssignmentRubric for grading criteria - Use submitFeedbackQuestion for feedback concerns - Use requestRegrading for regrade requests +- Use provideFeedback for sharing general feedback about learning experience +- Use submitSuggestion for platform improvement ideas +- Use submitInquiry for general questions or inquiries + +IMPORTANT: ${assignmentId ? `When calling tools that require assignmentId, always use ${assignmentId}` : "Assignment ID information is not available in the current context"} RESPONSE STYLE: - Warm, encouraging, and patient @@ -680,6 +921,10 @@ export async function POST(req) { const systemPrompt = generateSystemPrompt(userRole, { mode: assignmentMode, submitted: isSubmitted, + assignmentId: + userRole === "learner" + ? parseInt(assignmentInfo?.assignmentId || "0") + : undefined, }); const result = await streamText({ @@ -757,18 +1002,49 @@ export async function POST(req) { const toolResults = (await result.toolResults) || []; for (const toolResult of toolResults) { if (toolResult && toolResult.result) { + // Handle report, feedback, suggestion, and inquiry tool results if ( - !fullContent.includes(toolResult.result) && - toolResult.toolName === "reportIssue" + [ + "reportIssue", + "provideFeedback", + "submitSuggestion", + "submitInquiry", + ].includes(toolResult.toolName) ) { - const toolResponse = `\n\n${toolResult.result}`; - fullContent += toolResponse; - await writer.write(new TextEncoder().encode(toolResponse)); + try { + const parsedResult = JSON.parse(toolResult.result); + if ( + parsedResult.clientExecution && + parsedResult.function === "showReportPreview" + ) { + // Add to client executions for report preview + trackedClientExecutions.push({ + function: parsedResult.function, + params: parsedResult.params, + }); + } else { + // Regular tool result - add to content if not already there + if (!fullContent.includes(toolResult.result)) { + const toolResponse = `\n\n${toolResult.result}`; + fullContent += toolResponse; + await writer.write( + new TextEncoder().encode(toolResponse), + ); + } + } + } catch (e) { + // If parsing fails, treat as regular result + if (!fullContent.includes(toolResult.result)) { + const toolResponse = `\n\n${toolResult.result}`; + fullContent += toolResponse; + await writer.write(new TextEncoder().encode(toolResponse)); + } + } } } } - if (trackedClientExecutions.length > 0 && userRole === "author") { + if (trackedClientExecutions.length > 0) { const marker = `\n\n`; diff --git a/apps/web/app/author/(components)/(questionComponents)/CriteriaTable.tsx b/apps/web/app/author/(components)/(questionComponents)/CriteriaTable.tsx index 4c52ec9f..918ada71 100644 --- a/apps/web/app/author/(components)/(questionComponents)/CriteriaTable.tsx +++ b/apps/web/app/author/(components)/(questionComponents)/CriteriaTable.tsx @@ -1,9 +1,10 @@ "use client"; import Tooltip from "@/components/Tooltip"; +import { useAuthorStore } from "@/stores/author"; import { PlusIcon } from "@heroicons/react/24/outline"; import { PencilIcon, SparklesIcon } from "@heroicons/react/24/solid"; -import React, { FC, useEffect, useState } from "react"; +import React, { FC, useEffect, useRef, useState } from "react"; interface CriteriaRowProps { initialPoints: number; diff --git a/apps/web/app/author/(components)/(questionComponents)/QuestionWrapper.tsx b/apps/web/app/author/(components)/(questionComponents)/QuestionWrapper.tsx index 92dd3044..d5124e4f 100644 --- a/apps/web/app/author/(components)/(questionComponents)/QuestionWrapper.tsx +++ b/apps/web/app/author/(components)/(questionComponents)/QuestionWrapper.tsx @@ -4,15 +4,19 @@ import MarkdownViewer from "@/components/MarkdownViewer"; import WarningAlert from "@/components/WarningAlert"; import type { Choice, + Criteria, QuestionAuthorStore, QuestionType, ResponseType, + Rubric, + RubricType, UpdateQuestionStateParams, } from "@/config/types"; import { expandMarkingRubric, generateRubric } from "@/lib/talkToBackend"; import { useAuthorStore, useQuestionStore } from "@/stores/author"; import MarkdownEditor from "@components/MarkDownEditor"; import { InformationCircleIcon } from "@heroicons/react/24/outline"; +import { ArrowDownIcon, PlusIcon } from "@heroicons/react/24/solid"; import React, { FC, useEffect, diff --git a/apps/web/app/author/(components)/(questionComponents)/RubricSwitcher.tsx b/apps/web/app/author/(components)/(questionComponents)/RubricSwitcher.tsx index ecf9f064..a10ad9b6 100644 --- a/apps/web/app/author/(components)/(questionComponents)/RubricSwitcher.tsx +++ b/apps/web/app/author/(components)/(questionComponents)/RubricSwitcher.tsx @@ -1,9 +1,15 @@ "use client"; +import MarkdownEditor from "@/components/MarkDownEditor"; import { QuestionAuthorStore, Rubric } from "@/config/types"; import { useAuthorStore } from "@/stores/author"; import { TrashIcon } from "@heroicons/react/24/outline"; -import { PencilIcon, PlusIcon, SparklesIcon } from "@heroicons/react/24/solid"; +import { + MinusIcon, + PencilIcon, + PlusIcon, + SparklesIcon, +} from "@heroicons/react/24/solid"; import React, { useState } from "react"; import CriteriaTable from "./CriteriaTable"; diff --git a/apps/web/app/author/(components)/AuthorQuestionsPage/Question.tsx b/apps/web/app/author/(components)/AuthorQuestionsPage/Question.tsx index 87af5959..e23b6a18 100644 --- a/apps/web/app/author/(components)/AuthorQuestionsPage/Question.tsx +++ b/apps/web/app/author/(components)/AuthorQuestionsPage/Question.tsx @@ -1,7 +1,9 @@ +import MarkdownEditor from "@/components/MarkDownEditor"; import MultipleChoiceSVG from "@/components/svgs/MC"; import Tooltip from "@/components/Tooltip"; import type { CreateQuestionRequest, + Criteria, QuestionAuthorStore, QuestionType, QuestionVariants, diff --git a/apps/web/app/author/(components)/AuthorQuestionsPage/index.tsx b/apps/web/app/author/(components)/AuthorQuestionsPage/index.tsx index 7c7841ef..256dcbc1 100644 --- a/apps/web/app/author/(components)/AuthorQuestionsPage/index.tsx +++ b/apps/web/app/author/(components)/AuthorQuestionsPage/index.tsx @@ -16,9 +16,12 @@ import type { Scoring, } from "@/config/types"; import useBeforeUnload from "@/hooks/use-before-unload"; +import { useVersionControl } from "@/hooks/useVersionControl"; import { generateQuestionVariant, getAssignment } from "@/lib/talkToBackend"; import { generateTempQuestionId } from "@/lib/utils"; import { useAuthorStore } from "@/stores/author"; +import { useAssignmentConfig } from "@/stores/assignmentConfig"; +import { useAssignmentFeedbackConfig } from "@/stores/assignmentFeedbackConfig"; import { closestCenter, DndContext, @@ -69,6 +72,89 @@ import { FooterNavigation } from "../StepOne/FooterNavigation"; import Question from "./Question"; import { handleJumpToQuestionTitle } from "@/app/Helpers/handleJumpToQuestion"; import ImportModal from "../ImportModal"; +type ClientSnapshot = { + browser: { name?: string; version?: string; ua?: string }; + os?: string; + deviceType: "mobile" | "tablet" | "desktop" | "unknown"; + isMobile: boolean; + screen: { width: number | null; height: number | null; dpr: number | null }; + hardware: { cores: number | null; memoryGB: number | null }; + network: { + downlinkMbps: number | null; + effectiveType: string | null; + rttMs: number | null; + }; + timezone: string | null; +}; + +function parseBrowser(ua: string) { + const pairs = [ + [/Edg\/([\d.]+)/i, "Edge"], + [/Chrome\/([\d.]+)/i, "Chrome"], + [/Version\/([\d.]+).*Safari/i, "Safari"], + [/Firefox\/([\d.]+)/i, "Firefox"], + ]; + for (const [re, name] of pairs) { + const m = ua.match(re as RegExp); + if (m) return { name, version: m[1] }; + } + return { name: undefined, version: undefined }; +} +function detectDeviceType(ua: string): ClientSnapshot["deviceType"] { + const s = ua.toLowerCase(); + if (/mobile|iphone|ipod|android(?!.*tablet)/.test(s)) return "mobile"; + if (/ipad|tablet|kindle|silk/.test(s)) return "tablet"; + if (/cros|macintosh|windows|linux|x11/.test(s)) return "desktop"; + return "unknown"; +} +function getOS(ua: string) { + if (/Windows NT/i.test(ua)) return "Windows"; + if (/Mac OS X/i.test(ua)) return "macOS"; + if (/Android/i.test(ua)) return "Android"; + if (/iPhone|iPad|iPod/i.test(ua)) return "iOS/iPadOS"; + if (/CrOS/i.test(ua)) return "ChromeOS"; + if (/Linux/i.test(ua)) return "Linux"; + return undefined; +} +function getNetworkInfo() { + const c: any = + (navigator as any).connection || + (navigator as any).mozConnection || + (navigator as any).webkitConnection; + return { + downlinkMbps: c?.downlink ?? null, + effectiveType: c?.effectiveType ?? null, + rttMs: c?.rtt ?? null, + }; +} + +export async function buildClientSnapshot(): Promise { + const uaData: any = (navigator as any).userAgentData; + const ua = navigator.userAgent ?? ""; + const { name, version } = uaData + ? { name: uaData.brands?.[0]?.brand, version: uaData.brands?.[0]?.version } + : parseBrowser(ua); + + const deviceType = uaData?.mobile ? "mobile" : detectDeviceType(ua); + + return { + browser: { name, version, ua: ua || undefined }, + os: uaData?.platform || getOS(ua), + deviceType, + isMobile: deviceType === "mobile", + screen: { + width: screen?.width ?? null, + height: screen?.height ?? null, + dpr: devicePixelRatio ?? null, + }, + hardware: { + cores: navigator.hardwareConcurrency ?? null, + memoryGB: (navigator as any).deviceMemory ?? null, + }, + network: getNetworkInfo(), + timezone: Intl.DateTimeFormat().resolvedOptions?.().timeZone ?? null, + }; +} interface Props { assignmentId: number; @@ -96,6 +182,7 @@ const AuthorQuestionsPage: FC = ({ //questions are previously loaded into global state through backend call const [handleToggleTable, setHandleToggleTable] = useState(true); // State to toggle the table of contents const questions = useAuthorStore((state) => state.questions, shallow); + const setQuestions = useAuthorStore((state) => state.setQuestions); const addQuestion = useAuthorStore((state) => state.addQuestion); const activeAssignmentId = useAuthorStore( @@ -104,6 +191,11 @@ const AuthorQuestionsPage: FC = ({ const setActiveAssignmentId = useAuthorStore( (state) => state.setActiveAssignmentId, ); + const checkedOutVersion = useAuthorStore((state) => state.checkedOutVersion); + + // Get version control hook to ensure version synchronization + const { loadVersions } = useVersionControl(); + const [isMassVariationLoading, setIsMassVariationLoading] = useState(false); const [questionVariationNumber, setQuestionVariationNumber] = useState(null); @@ -114,6 +206,12 @@ const AuthorQuestionsPage: FC = ({ const [fileUploadModalOpen, setFileUploadModalOpen] = useState(false); const [isReportModalOpen, setIsReportModalOpen] = useState(false); const [isImportModalOpen, setIsImportModalOpen] = useState(false); + const [client, setClient] = useState(null); + + async function testSnapshot() { + const snap = await buildClientSnapshot(); + setClient(snap); // show on page + } const questionTypes = useMemo( () => [ { @@ -157,7 +255,21 @@ const AuthorQuestionsPage: FC = ({ [], ); + // Ensure versions are always fresh on page load + useEffect(() => { + if (assignmentId) { + loadVersions().catch(console.error); + } + }, [assignmentId, loadVersions]); + useEffect(() => { + if (checkedOutVersion) { + if (assignmentId !== activeAssignmentId) { + setActiveAssignmentId(assignmentId); + } + return; + } + if (assignmentId !== activeAssignmentId) { const fetchAssignment = async () => { try { @@ -302,6 +414,7 @@ const AuthorQuestionsPage: FC = ({ setName, setQuestions, setFocusedQuestionId, + checkedOutVersion, ]); useEffect(() => { @@ -748,7 +861,22 @@ const AuthorQuestionsPage: FC = ({ */ const handleImportQuestions = ( importedQuestions: QuestionAuthorStore[], - options: { replaceExisting: boolean }, + options: { + replaceExisting: boolean; + appendToExisting: boolean; + validateQuestions: boolean; + importChoices: boolean; + importRubrics: boolean; + importConfig: boolean; + importAssignmentSettings: boolean; + }, + assignmentData?: { + questions?: QuestionAuthorStore[]; + assignment?: any; + config?: any; + feedbackConfig?: any; + gradingCriteria?: any; + }, ) => { try { // Process imported questions with proper total points calculation @@ -843,14 +971,132 @@ const AuthorQuestionsPage: FC = ({ .getState() .setQuestionOrder(updatedQuestions.map((q) => q.id)); + // Handle assignment settings if requested + let importMessage = `Successfully imported ${importedQuestions.length} question(s)!`; + if (options.importAssignmentSettings && assignmentData) { + let settingsUpdated = false; + + if (assignmentData.assignment) { + // Update assignment metadata if available + if ( + assignmentData.assignment.name && + assignmentData.assignment.name !== "Imported Assignment" + ) { + setName(assignmentData.assignment.name); + settingsUpdated = true; + } + if (assignmentData.assignment.introduction) { + useAuthorStore + .getState() + .setIntroduction(assignmentData.assignment.introduction); + settingsUpdated = true; + } + if (assignmentData.assignment.instructions) { + useAuthorStore + .getState() + .setInstructions(assignmentData.assignment.instructions); + settingsUpdated = true; + } + // Check for gradingCriteria in assignment object + if (assignmentData.assignment.gradingCriteria) { + useAuthorStore + .getState() + .setGradingCriteriaOverview( + assignmentData.assignment.gradingCriteria, + ); + settingsUpdated = true; + } + } + + if (assignmentData.config) { + // Update assignment configuration if available + const configStore = useAssignmentConfig.getState(); + + if (assignmentData.config.graded !== undefined) { + configStore.setGraded(assignmentData.config.graded); + settingsUpdated = true; + } + if (assignmentData.config.numAttempts !== undefined) { + configStore.setNumAttempts(assignmentData.config.numAttempts); + settingsUpdated = true; + } + if (assignmentData.config.allotedTimeMinutes !== undefined) { + configStore.setAllotedTimeMinutes(assignmentData.config.allotedTimeMinutes); + settingsUpdated = true; + } + if (assignmentData.config.timeEstimateMinutes !== undefined) { + configStore.setTimeEstimateMinutes(assignmentData.config.timeEstimateMinutes); + settingsUpdated = true; + } + if (assignmentData.config.passingGrade !== undefined) { + configStore.setPassingGrade(assignmentData.config.passingGrade); + settingsUpdated = true; + } + if (assignmentData.config.numberOfQuestionsPerAttempt !== undefined) { + configStore.setNumberOfQuestionsPerAttempt( + assignmentData.config.numberOfQuestionsPerAttempt, + ); + settingsUpdated = true; + } + if (assignmentData.config.displayOrder !== undefined) { + configStore.setDisplayOrder(assignmentData.config.displayOrder as "DEFINED" | "RANDOM"); + settingsUpdated = true; + } + if (assignmentData.config.questionDisplay !== undefined) { + configStore.setQuestionDisplay(assignmentData.config.questionDisplay as any); + settingsUpdated = true; + } + if (assignmentData.config.strictTimeLimit !== undefined) { + configStore.setStrictTimeLimit(assignmentData.config.strictTimeLimit); + settingsUpdated = true; + } + } + + // Check for root-level gradingCriteria + if (assignmentData.gradingCriteria) { + useAuthorStore + .getState() + .setGradingCriteriaOverview(assignmentData.gradingCriteria); + settingsUpdated = true; + } + + // Import feedback configuration if available + if (assignmentData.feedbackConfig) { + const feedbackConfigStore = useAssignmentFeedbackConfig.getState(); + + if (assignmentData.feedbackConfig.verbosityLevel !== undefined) { + feedbackConfigStore.setVerbosityLevel(assignmentData.feedbackConfig.verbosityLevel as any); + settingsUpdated = true; + } + if (assignmentData.feedbackConfig.showSubmissionFeedback !== undefined) { + feedbackConfigStore.setShowSubmissionFeedback(assignmentData.feedbackConfig.showSubmissionFeedback); + settingsUpdated = true; + } + if (assignmentData.feedbackConfig.showQuestionScore !== undefined) { + feedbackConfigStore.setShowQuestionScore(assignmentData.feedbackConfig.showQuestionScore); + settingsUpdated = true; + } + if (assignmentData.feedbackConfig.showAssignmentScore !== undefined) { + feedbackConfigStore.setShowAssignmentScore(assignmentData.feedbackConfig.showAssignmentScore); + settingsUpdated = true; + } + if (assignmentData.feedbackConfig.showQuestions !== undefined) { + feedbackConfigStore.setShowQuestion(assignmentData.feedbackConfig.showQuestions); + settingsUpdated = true; + } + } + + if (settingsUpdated) { + importMessage += " Assignment settings have also been imported."; + } + } + // Focus on the first imported question if (processedQuestions.length > 0) { setFocusedQuestionId(processedQuestions[0].id); } - toast.success( - `Successfully imported ${importedQuestions.length} question(s)!`, - ); + toast.success(importMessage); } catch (error) { console.error("Import failed:", error); toast.error("Failed to import questions. Please try again."); @@ -991,7 +1237,7 @@ const AuthorQuestionsPage: FC = ({ > - Import Questions (Beta){" "} + Import Assignment (Beta){" "}
@@ -1020,7 +1266,7 @@ const AuthorQuestionsPage: FC = ({
- Mass Variations (Beta) + Mass Variations = (props) => { showQuestionScore, showAssignmentScore, showQuestions, + showCorrectAnswer, ] = useAssignmentFeedbackConfig((state) => [ state.showSubmissionFeedback, state.showQuestionScore, state.showAssignmentScore, state.showQuestions, + state.showCorrectAnswer, ]); const assignmentConfig = { questionDisplay: assignmentConfigstate.questionDisplay, @@ -43,6 +45,7 @@ const CheckLearnerSideButton: FC = (props) => { showQuestions: showQuestions, showAssignmentScore: showAssignmentScore, showQuestionScore: showQuestionScore, + showCorrectAnswer: showCorrectAnswer, instructions: authorState.instructions ?? "", gradingCriteriaOverview: authorState.gradingCriteriaOverview ?? "", showSubmissionFeedback: showSubmissionFeedback, diff --git a/apps/web/app/author/(components)/Header/SaveAndPublishButton.tsx b/apps/web/app/author/(components)/Header/SaveAndPublishButton.tsx new file mode 100644 index 00000000..2e38b8ad --- /dev/null +++ b/apps/web/app/author/(components)/Header/SaveAndPublishButton.tsx @@ -0,0 +1,621 @@ +"use client"; + +import TooltipMessage from "@/app/components/ToolTipMessage"; +import { useChangesSummary } from "@/app/Helpers/checkDiff"; +import Spinner from "@/components/svgs/Spinner"; +import { useAuthorStore } from "@/stores/author"; +import { useRouter } from "next/navigation"; +import { useEffect, useState, type FC } from "react"; +import { + ChevronRightIcon, + ExclamationTriangleIcon, + XMarkIcon, +} from "@heroicons/react/24/outline"; +import { handleScrollToFirstErrorField } from "@/app/Helpers/handleJumpToErrors"; +import Tooltip from "@/components/Tooltip"; +import { VersionSelectionModal } from "@/components/version-control/VersionSelectionModal"; +import { VersionConflictModal } from "@/components/version-control/VersionConflictModal"; +import { useVersionControl } from "@/hooks/useVersionControl"; +import { VersionComparison } from "@/types/version-types"; +import { useAssignmentConfig } from "@/stores/assignmentConfig"; +import { decodeFields } from "@/app/Helpers/decoder"; +import { getAssignment, getUser } from "@/lib/shared"; +import { mergeData } from "@/lib/utils"; +import { useAssignmentFeedbackConfig } from "@/stores/assignmentFeedbackConfig"; + +interface Props { + submitting: boolean; + questionsAreReadyToBePublished: () => { + isValid: boolean; + message: string; + step: number | null; + invalidQuestionId: number; + }; + handlePublishButton: ( + description?: string, + publishImmediately?: boolean, + versionNumber?: string, + ) => void; + currentStepId?: number; +} + +const SaveAndPublishButton: FC = ({ + submitting, + questionsAreReadyToBePublished, + handlePublishButton, + currentStepId = 0, +}) => { + const router = useRouter(); + const validateAssignmentSetup = useAuthorStore((state) => state.validate); + const [showErrorModal, setShowErrorModal] = useState(false); + const [showPublishModal, setShowPublishModal] = useState(false); + const [showConflictModal, setShowConflictModal] = useState(false); + const [versionComparison, setVersionComparison] = + useState(null); + const [conflictDetails, setConflictDetails] = useState<{ + existingVersion: any; + requestedVersion: string; + description: string; + isDraft: boolean; + } | null>(null); + const [setAuthorStore, activeAssignmentId, setQuestionOrder] = useAuthorStore( + (state) => [ + state.setAuthorStore, + state.activeAssignmentId, + state.setQuestionOrder, + ], + ); + const [setAssignmentConfigStore] = useAssignmentConfig((state) => [ + state.setAssignmentConfigStore, + ]); + const [setAssignmentFeedbackConfigStore] = useAssignmentFeedbackConfig( + (state) => [state.setAssignmentFeedbackConfigStore], + ); + const fetchAssignment = async () => { + const assignment = await getAssignment(activeAssignmentId); + if (assignment) { + const decodedFields = decodeFields({ + introduction: assignment.introduction, + instructions: assignment.instructions, + gradingCriteriaOverview: assignment.gradingCriteriaOverview, + }); + + const decodedAssignment = { + ...assignment, + ...decodedFields, + }; + + useAuthorStore.getState().setOriginalAssignment(decodedAssignment); + + const mergedAuthorData = mergeData( + useAuthorStore.getState(), + decodedAssignment, + ); + const { updatedAt, ...cleanedAuthorData } = mergedAuthorData; + setAuthorStore({ + ...cleanedAuthorData, + }); + if (decodedAssignment.questionOrder) { + setQuestionOrder(decodedAssignment.questionOrder); + } else { + setQuestionOrder(questions.map((question) => question.id)); + } + const mergedAssignmentConfigData = mergeData( + useAssignmentConfig.getState(), + decodedAssignment, + ); + if (decodedAssignment.questionVariationNumber !== undefined) { + setAssignmentConfigStore({ + questionVariationNumber: decodedAssignment.questionVariationNumber, + }); + } + const { + updatedAt: authorStoreUpdatedAt, + ...cleanedAssignmentConfigData + } = mergedAssignmentConfigData; + setAssignmentConfigStore({ + ...cleanedAssignmentConfigData, + }); + + const mergedAssignmentFeedbackData = mergeData( + useAssignmentFeedbackConfig.getState(), + decodedAssignment, + ); + const { + updatedAt: assignmentFeedbackUpdatedAt, + ...cleanedAssignmentFeedbackData + } = mergedAssignmentFeedbackData; + setAssignmentFeedbackConfigStore({ + ...cleanedAssignmentFeedbackData, + }); + + useAuthorStore.getState().setName(decodedAssignment.name); + } + }; + + const versionControlHook = useVersionControl(); + const { + versions, + currentVersion, + compareVersions, + createVersion, + updateExistingVersion, + } = versionControlHook; + + const { isValid, message, step, invalidQuestionId } = + questionsAreReadyToBePublished(); + const questions = useAuthorStore((state) => state.questions); + const setFocusedQuestionId = useAuthorStore( + (state) => state.setFocusedQuestionId, + ); + const isLoading = !questions; + const hasEmptyQuestion = questions?.some((q) => q.type === "EMPTY"); + const { assignmentId } = useAuthorStore((state) => ({ + assignmentId: state.activeAssignmentId, + })); + const changesSummary = useChangesSummary(); + const hasChanges = changesSummary !== "No changes detected."; + const isLastStep = currentStepId === 3; + + const pageRouterUsingSteps = (step: number | null) => { + switch (true) { + case step === 0: + return `/author/${assignmentId}/questions`; + case step === 1: + return `/author/${assignmentId}/config`; + case step === 2: + return `/author/${assignmentId}/review`; + default: + return `/author/${assignmentId}`; + } + }; + + function handleNavigate() { + setShowErrorModal(false); + + // Navigate first + if (step !== null && step !== undefined) { + const nextPage = pageRouterUsingSteps(step); + + if (nextPage) { + router.push(nextPage); + + // If we have an invalidQuestionId, set it and scroll after navigation + if (invalidQuestionId) { + setFocusedQuestionId(invalidQuestionId); + + // Wait for navigation and rendering to complete + setTimeout(() => { + const element = document.getElementById( + `question-title-${invalidQuestionId}`, + ); + if (element) { + element.scrollIntoView({ + behavior: "smooth", + block: "center", + inline: "center", + }); + } else { + // If question-title element doesn't exist, try the question element + const questionElement = document.getElementById( + `question-${invalidQuestionId}`, + ); + if (questionElement) { + questionElement.scrollIntoView({ + behavior: "smooth", + block: "start", + inline: "nearest", + }); + } + } + }, 500); // Give time for navigation and rendering + } + } else { + router.push(`/author/${assignmentId}`); + } + } else if (invalidQuestionId) { + // If no step but we have an invalid question ID, we're already on the right page + setFocusedQuestionId(invalidQuestionId); + + setTimeout(() => { + const element = document.getElementById( + `question-title-${invalidQuestionId}`, + ); + if (element) { + element.scrollIntoView({ + behavior: "smooth", + block: "center", + inline: "center", + }); + } + }, 100); + } + } + + const handleButtonClick = async () => { + if (disableButton && message !== "") { + setShowErrorModal(true); + return; + } + + await handleShowPublishModal(); + }; + + const disableButton = + submitting || + isLoading || + questions?.length === 0 || + hasEmptyQuestion || + !isValid || + !hasChanges; + + const getStatusMessage = () => { + if (isLoading) return { text: "Loading questions...", type: "loading" }; + if (questions?.length === 0 && step === 2) + return { text: "You need to add at least one question", type: "error" }; + if (hasEmptyQuestion) + return { text: "Some questions have incomplete fields", type: "error" }; + if (!isValid) return { text: message, type: "error", hasAction: true }; + if (submitting) + return { text: "Mark is publishing your questions...", type: "loading" }; + if (!hasChanges) return { text: "No changes detected.", type: "warning" }; + return { text: "Ready to publish version", type: "success" }; + }; + + const statusMessage = getStatusMessage(); + + const handleShowPublishModal = async () => { + try { + if (!currentVersion) { + // No current version - assume this is a new assignment or first version + const defaultComparison: VersionComparison = { + fromVersion: { + id: 0, + versionNumber: "0.0.0", + versionDescription: "Previous", + isDraft: false, + isActive: false, + published: true, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + toVersion: { + id: 1, + versionNumber: "1.0.0", + versionDescription: "New version", + isDraft: false, + isActive: true, + published: false, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + assignmentChanges: [ + { + field: "name", + fromValue: null, + toValue: "new assignment", + changeType: "added", + }, + ], + questionChanges: [], + }; + setVersionComparison(defaultComparison); + setShowPublishModal(true); + return; + } + + // For now, provide a reasonable default comparison since we don't have + // a way to compare against current working state + const defaultComparison: VersionComparison = { + fromVersion: { + ...currentVersion, + createdAt: currentVersion.createdAt, + versionNumber: currentVersion.versionNumber.toString(), + }, + toVersion: { + ...currentVersion, + createdAt: currentVersion.createdAt, + versionNumber: "next", + versionDescription: "Updated version", + }, + assignmentChanges: [ + { + field: "instructions", + fromValue: "previous", + toValue: "updated", + changeType: "modified", + }, + ], + questionChanges: [], + }; + setVersionComparison(defaultComparison); + setShowPublishModal(true); + } catch (error) { + console.error("Failed to analyze changes:", error); + // Provide fallback comparison for when comparison fails + const fallbackComparison: VersionComparison = { + fromVersion: currentVersion + ? { + ...currentVersion, + createdAt: currentVersion.createdAt, + versionNumber: currentVersion.versionNumber.toString(), + } + : { + id: 0, + versionNumber: "0.0.0", + versionDescription: "Previous", + isDraft: false, + isActive: false, + published: true, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + toVersion: { + id: 1, + versionNumber: "next", + versionDescription: "Updated version", + isDraft: false, + isActive: true, + published: false, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + assignmentChanges: [ + { + field: "instructions", + fromValue: "previous", + toValue: "updated", + changeType: "modified", + }, + ], + questionChanges: [], + }; + setVersionComparison(fallbackComparison); + setShowPublishModal(true); + } + }; + + const handleVersionSave = async ( + versionNumber: string, + description: string, + isDraft: boolean, + shouldUpdate?: boolean, + versionId?: number, + ) => { + try { + if (shouldUpdate && versionId) { + // Use the dedicated update function for existing versions + if (updateExistingVersion) { + await updateExistingVersion( + versionId, + versionNumber, + description, + false, // Force publish instead of draft + ); + setShowPublishModal(false); + + await fetchAssignment(); + router.push(`/author/${assignmentId}/version-tree`); + } else { + throw new Error("updateExistingVersion function not available"); + } + } else { + setShowPublishModal(false); + void handlePublishButton(description, true, versionNumber); + await fetchAssignment(); + router.push(`/author/${assignmentId}/version-tree`); + } + } catch (error: any) { + console.error("Failed to save version:", error); + + // Check if this is a version conflict error + if ( + error.response?.status === 409 && + error.response?.data?.versionExists + ) { + const conflictData = error.response.data; + setConflictDetails({ + existingVersion: conflictData.existingVersion, + requestedVersion: versionNumber, + description, + isDraft: false, // Always publish + }); + setShowPublishModal(false); + setShowConflictModal(true); + return; + } + + throw error; + } + }; + + const handleUpdateExistingVersion = async () => { + if (!conflictDetails) return; + + try { + void handlePublishButton( + conflictDetails.description, + true, + conflictDetails.requestedVersion, + ); + + setShowConflictModal(false); + setConflictDetails(null); + } catch (error) { + console.error("Failed to update existing version:", error); + throw error; + } + }; + + const handleCreateNewVersion = () => { + setShowConflictModal(false); + setConflictDetails(null); + // Reopen the version selection modal to let user pick a different version number + setShowPublishModal(true); + }; + + // Hide modal when conditions change and button becomes enabled + useEffect(() => { + if (!disableButton) { + setShowErrorModal(false); + } + }, [disableButton]); + + return ( + <> +
+ {/* Button */} + <> + + + + +
+ + {/* Error Modal */} + {showErrorModal && ( +
+
+ {/* Backdrop */} +
setShowErrorModal(false)} + /> + + {/* Modal Content */} +
+ {/* Close button */} + + + {/* Modal Header */} +
+ {statusMessage.type === "error" && ( + + )} + {statusMessage.type === "warning" && ( + + )} +

+ {statusMessage.type === "error" ? "Error" : "Warning"} +

+
+ + {/* Modal Body */} +
+ +
+ + {/* Modal Footer */} +
+ + {statusMessage.hasAction && ( + + )} +
+
+
+
+ )} + + {/* Version Selection Modal */} + setShowPublishModal(false)} + onSave={handleVersionSave} + currentVersions={versions.map((v) => ({ + versionNumber: v.versionNumber?.toString() || "0.0.0", + id: v.id, + isDraft: v.isDraft, + isActive: v.isActive, + published: v.published, + }))} + comparison={versionComparison} + isLoading={submitting} + workingVersion={ + currentVersion + ? { + versionNumber: + currentVersion.versionNumber?.toString() || "0.0.0", + id: currentVersion.id, + isDraft: currentVersion.isDraft, + isActive: currentVersion.isActive, + published: currentVersion.published, + } + : undefined + } + forcePublish={true} + /> + + {/* Version Conflict Modal */} + {conflictDetails && ( + { + setShowConflictModal(false); + setConflictDetails(null); + }} + onUpdate={handleUpdateExistingVersion} + onCreateNew={handleCreateNewVersion} + existingVersion={conflictDetails.existingVersion} + requestedVersion={conflictDetails.requestedVersion} + /> + )} + + ); +}; + +export default SaveAndPublishButton; diff --git a/apps/web/app/author/(components)/Header/SubmitQuestionsButton.tsx b/apps/web/app/author/(components)/Header/SubmitQuestionsButton.tsx index e0692d53..45cc59ac 100644 --- a/apps/web/app/author/(components)/Header/SubmitQuestionsButton.tsx +++ b/apps/web/app/author/(components)/Header/SubmitQuestionsButton.tsx @@ -5,6 +5,7 @@ import { useChangesSummary } from "@/app/Helpers/checkDiff"; import Spinner from "@/components/svgs/Spinner"; import { useAuthorStore } from "@/stores/author"; import { useRouter } from "next/navigation"; +import { toast } from "sonner"; import { useEffect, useState, type FC } from "react"; import { ChevronRightIcon, @@ -13,6 +14,19 @@ import { } from "@heroicons/react/24/outline"; import { handleScrollToFirstErrorField } from "@/app/Helpers/handleJumpToErrors"; import Tooltip from "@/components/Tooltip"; +import { VersionSelectionModal } from "@/components/version-control/VersionSelectionModal"; +import { VersionConflictModal } from "@/components/version-control/VersionConflictModal"; +import { useVersionControl } from "@/hooks/useVersionControl"; +import { VersionComparison } from "@/types/version-types"; +import { + SemanticVersion, + VersionSuggestion, + formatSemanticVersion, + parseSemanticVersion, + suggestNextVersion, + analyzeChanges, + getLatestVersion, +} from "@/lib/semantic-versioning"; interface Props { submitting: boolean; @@ -22,7 +36,11 @@ interface Props { step: number | null; invalidQuestionId: number; }; - handlePublishButton: () => void; + handlePublishButton: ( + description?: string, + publishImmediately?: boolean, + versionNumber?: string, + ) => void; currentStepId?: number; } @@ -35,6 +53,25 @@ const SubmitQuestionsButton: FC = ({ const router = useRouter(); const validateAssignmentSetup = useAuthorStore((state) => state.validate); const [showErrorModal, setShowErrorModal] = useState(false); + const [showVersionModal, setShowVersionModal] = useState(false); + const [showConflictModal, setShowConflictModal] = useState(false); + const [versionComparison, setVersionComparison] = + useState(null); + const [conflictDetails, setConflictDetails] = useState<{ + existingVersion: any; + requestedVersion: string; + description: string; + isDraft: boolean; + } | null>(null); + + const versionControlHook = useVersionControl(); + const { + versions, + currentVersion, + compareVersions, + createVersion, + updateExistingVersion, + } = versionControlHook; const { isValid, message, step, invalidQuestionId } = questionsAreReadyToBePublished(); @@ -52,7 +89,6 @@ const SubmitQuestionsButton: FC = ({ const isLastStep = currentStepId === 3; const pageRouterUsingSteps = (step: number | null) => { - console.log("pageRouterUsingSteps", step); switch (true) { case step === 0: return `/author/${assignmentId}/questions`; @@ -70,12 +106,9 @@ const SubmitQuestionsButton: FC = ({ // Navigate first if (step !== null && step !== undefined) { - console.log("navigating to step", step); const nextPage = pageRouterUsingSteps(step); - console.log("nextPage", nextPage); if (nextPage) { - console.log("pushing to nextPage", nextPage); router.push(nextPage); // If we have an invalidQuestionId, set it and scroll after navigation @@ -130,31 +163,153 @@ const SubmitQuestionsButton: FC = ({ } } - const handleButtonClick = () => { + const handleButtonClick = async () => { if (disableButton && message !== "") { setShowErrorModal(true); return; } - if (isLastStep) { - handlePublishButton(); - } else { - goToNextStep(); + // Create draft immediately without showing modal + await handleCreateDraftImmediately(); + }; + + const handleCreateDraftImmediately = async () => { + try { + // Generate version comparison for change analysis + const versionComparison = await generateVersionComparison(); + + // Get recommended version using same logic as VersionSelectionModal + const recommendedVersion = getRecommendedDraftVersion(versionComparison); + const defaultDescription = `Draft created on ${new Date().toLocaleString()}`; + + // Use createVersion from author store to save draft snapshot without triggering publish job + if (createVersion) { + const result = await createVersion( + defaultDescription, + true, + recommendedVersion, + false, + ); + + if (result) { + toast.success("Draft saved successfully!"); + // Navigate to version-tree after draft creation + router.push(`/author/${assignmentId}/version-tree`); + } else { + toast.error("Failed to save draft. Please try again."); + } + } else { + throw new Error("createVersion function not available"); + } + } catch (error: any) { + console.error("Failed to create draft:", error); + + // If there's an error, fall back to showing the modal + await handleShowVersionModal(); } }; - const goToNextStep = () => { - const isValid = validateAssignmentSetup(); - if (!isValid) { - handleScrollToFirstErrorField(); - return; + const generateVersionComparison = async (): Promise => { + if (!currentVersion) { + // No current version - assume this is a new assignment or first version + return { + fromVersion: { + id: 0, + versionNumber: "0.0.0", + versionDescription: "Previous", + isDraft: false, + isActive: false, + published: true, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + toVersion: { + id: 1, + versionNumber: "1.0.0", + versionDescription: "New version", + isDraft: false, + isActive: true, + published: false, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + assignmentChanges: [ + { + field: "name", + fromValue: null, + toValue: "new assignment", + changeType: "added", + }, + ], + questionChanges: [], + }; } - const nextPage = pageRouterUsingSteps(currentStepId); - console.log("currentStep", currentStepId); - if (nextPage) { - router.push(nextPage); - } else { - router.push(`/author/${assignmentId}`); + + // For now, provide a reasonable default comparison since we don't have + // a way to compare against current working state + return { + fromVersion: { + ...currentVersion, + createdAt: currentVersion.createdAt, + versionNumber: currentVersion.versionNumber.toString(), + }, + toVersion: { + ...currentVersion, + createdAt: currentVersion.createdAt, + versionNumber: "next", + versionDescription: "Updated version", + }, + assignmentChanges: [ + { + field: "instructions", + fromValue: "previous", + toValue: "updated", + changeType: "modified", + }, + ], + questionChanges: [], + }; + }; + + const getRecommendedDraftVersion = ( + comparison: VersionComparison, + ): string => { + try { + // Get the latest version from existing versions + const latestVersion = getLatestVersion(versions); + const currentVersionString = latestVersion + ? formatSemanticVersion(latestVersion) + : "1.0.0"; + + // Analyze changes to determine version suggestion + const changeAnalysis = analyzeChanges(comparison); + + // Get version suggestions with isDraft=false to skip -rc suffix + const suggestions = suggestNextVersion( + currentVersionString, + changeAnalysis, + false, + ); + + // Return the first (recommended) suggestion + if (suggestions.length > 0) { + return formatSemanticVersion(suggestions[0]); + } + + // Fallback to patch version without rc suffix + const current = parseSemanticVersion(currentVersionString); + return formatSemanticVersion({ + ...current, + patch: current.patch + 1, + }); + } catch (error) { + console.error("Error generating recommended version:", error); + // Final fallback without rc suffix + return "1.0.0"; } }; @@ -176,11 +331,215 @@ const SubmitQuestionsButton: FC = ({ if (submitting) return { text: "Mark is analyzing your questions...", type: "loading" }; if (!hasChanges) return { text: "No changes detected.", type: "warning" }; - return { text: "Ready to publish", type: "success" }; + return { text: "Ready to create draft", type: "success" }; }; const statusMessage = getStatusMessage(); + const handleShowVersionModal = async () => { + try { + if (!currentVersion) { + // No current version - assume this is a new assignment or first version + const defaultComparison: VersionComparison = { + fromVersion: { + id: 0, + versionNumber: "0.0.0", + versionDescription: "Previous", + isDraft: false, + isActive: false, + published: true, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + toVersion: { + id: 1, + versionNumber: "1.0.0", + versionDescription: "New version", + isDraft: false, + isActive: true, + published: false, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + assignmentChanges: [ + { + field: "name", + fromValue: null, + toValue: "new assignment", + changeType: "added", + }, + ], + questionChanges: [], + }; + setVersionComparison(defaultComparison); + setShowVersionModal(true); + return; + } + + // For now, provide a reasonable default comparison since we don't have + // a way to compare against current working state + const defaultComparison: VersionComparison = { + fromVersion: { + ...currentVersion, + createdAt: currentVersion.createdAt, + versionNumber: currentVersion.versionNumber.toString(), + }, + toVersion: { + ...currentVersion, + createdAt: currentVersion.createdAt, + versionNumber: "next", + versionDescription: "Updated version", + }, + assignmentChanges: [ + { + field: "instructions", + fromValue: "previous", + toValue: "updated", + changeType: "modified", + }, + ], + questionChanges: [], + }; + setVersionComparison(defaultComparison); + setShowVersionModal(true); + } catch (error) { + console.error("Failed to analyze changes:", error); + // Provide fallback comparison for when comparison fails + const fallbackComparison: VersionComparison = { + fromVersion: currentVersion + ? { + ...currentVersion, + createdAt: currentVersion.createdAt, + versionNumber: currentVersion.versionNumber.toString(), + } + : { + id: 0, + versionNumber: "0.0.0", + versionDescription: "Previous", + isDraft: false, + isActive: false, + published: true, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + toVersion: { + id: 1, + versionNumber: "next", + versionDescription: "Updated version", + isDraft: false, + isActive: true, + published: false, + createdBy: "system", + createdAt: new Date().toISOString(), + questionCount: 0, + wasAutoIncremented: false, + }, + assignmentChanges: [ + { + field: "instructions", + fromValue: "previous", + toValue: "updated", + changeType: "modified", + }, + ], + questionChanges: [], + }; + setVersionComparison(fallbackComparison); + setShowVersionModal(true); + } + }; + + const handleVersionSave = async ( + versionNumber: string, + description: string, + isDraft: boolean, + shouldUpdate?: boolean, + versionId?: number, + ) => { + try { + if (shouldUpdate && versionId) { + if (updateExistingVersion) { + await updateExistingVersion( + versionId, + versionNumber, + description, + true, // Force draft when updating + ); + setShowVersionModal(false); + + // Navigate to version-tree after update + router.push(`/author/${assignmentId}/version-tree`); + } else { + throw new Error("updateExistingVersion function not available"); + } + } else if (createVersion) { + await createVersion(description, true, versionNumber, shouldUpdate); + setShowVersionModal(false); + + // Navigate to version-tree after draft creation + router.push(`/author/${assignmentId}/version-tree`); + } else { + console.error("createVersion function not available"); + throw new Error("createVersion function not available"); + } + } catch (error: any) { + console.error("Failed to save version:", error); + + // Check if this is a version conflict error + if ( + error.response?.status === 409 && + error.response?.data?.versionExists + ) { + const conflictData = error.response.data; + setConflictDetails({ + existingVersion: conflictData.existingVersion, + requestedVersion: versionNumber, + description, + isDraft, + }); + setShowVersionModal(false); + setShowConflictModal(true); + return; + } + + throw error; + } + }; + + const handleUpdateExistingVersion = async () => { + if (!conflictDetails || !createVersion) return; + + try { + // Call createVersion with updateExisting flag + await createVersion( + conflictDetails.description, + conflictDetails.isDraft, + conflictDetails.requestedVersion, + true, // updateExisting flag + ); + + setShowConflictModal(false); + setConflictDetails(null); + // Don't call handlePublishButton when updating - createVersion with updateExisting handles the complete process + } catch (error) { + console.error("Failed to update existing version:", error); + throw error; + } + }; + + const handleCreateNewVersion = () => { + setShowConflictModal(false); + setConflictDetails(null); + // Reopen the version selection modal to let user pick a different version number + setShowVersionModal(true); + }; + // Hide modal when conditions change and button becomes enabled useEffect(() => { if (!disableButton) { @@ -192,40 +551,26 @@ const SubmitQuestionsButton: FC = ({ <>
{/* Button */} - {isLastStep ? ( - <> - - - - - ) : ( - - )} + + +
{/* Error Modal */} @@ -299,6 +644,49 @@ const SubmitQuestionsButton: FC = ({
)} + + {/* Version Selection Modal */} + setShowVersionModal(false)} + onSave={handleVersionSave} + currentVersions={versions.map((v) => ({ + versionNumber: v.versionNumber?.toString() || "0.0.0", + id: v.id, + isDraft: v.isDraft, + isActive: v.isActive, + published: v.published, + }))} + comparison={versionComparison} + isLoading={submitting} + workingVersion={ + currentVersion + ? { + versionNumber: + currentVersion.versionNumber?.toString() || "0.0.0", + id: currentVersion.id, + isDraft: currentVersion.isDraft, + isActive: currentVersion.isActive, + published: currentVersion.published, + } + : undefined + } + /> + + {/* Version Conflict Modal */} + {conflictDetails && ( + { + setShowConflictModal(false); + setConflictDetails(null); + }} + onUpdate={handleUpdateExistingVersion} + onCreateNew={handleCreateNewVersion} + existingVersion={conflictDetails.existingVersion} + requestedVersion={conflictDetails.requestedVersion} + /> + )} ); }; diff --git a/apps/web/app/author/(components)/Header/index.tsx b/apps/web/app/author/(components)/Header/index.tsx index 91fc6fb2..633e1f87 100644 --- a/apps/web/app/author/(components)/Header/index.tsx +++ b/apps/web/app/author/(components)/Header/index.tsx @@ -3,6 +3,7 @@ import CheckLearnerSideButton from "@/app/author/(components)/Header/CheckLearnerSideButton"; import { useMarkChatStore } from "@/app/chatbot/store/useMarkChatStore"; import { useChangesSummary } from "@/app/Helpers/checkDiff"; +import { useChatbot } from "@/hooks/useChatbot"; import { decodeFields } from "@/app/Helpers/decoder"; import { encodeFields } from "@/app/Helpers/encoder"; import { processQuestions } from "@/app/Helpers/processQuestionsBeforePublish"; @@ -32,13 +33,14 @@ import { useAssignmentFeedbackConfig } from "@/stores/assignmentFeedbackConfig"; import { useAuthorStore } from "@/stores/author"; import SNIcon from "@components/SNIcon"; import Title from "@components/Title"; -import { IconRefresh } from "@tabler/icons-react"; import { usePathname, useRouter } from "next/navigation"; -import { useEffect, useState } from "react"; +import { BarChart3 } from "lucide-react"; +import React, { useEffect, useState } from "react"; import { toast } from "sonner"; import { useQuestionsAreReadyToBePublished } from "../../../Helpers/checkQuestionsReady"; import { Nav } from "./Nav"; import SubmitQuestionsButton from "./SubmitQuestionsButton"; +import SaveAndPublishButton from "./SaveAndPublishButton"; function maybeDecodeString(str: string | null | undefined): string | null { if (!str) return str; @@ -87,6 +89,7 @@ function fixScoringAndDecode(assignment: Assignment): Assignment { function AuthorHeader() { const router = useRouter(); const pathname = usePathname(); + const { isOpen: isChatbotOpen } = useChatbot(); const assignmentId = extractAssignmentId(pathname); const [currentStepId, setCurrentStepId] = useState(0); const setQuestions = useAuthorStore((state) => state.setQuestions); @@ -109,6 +112,8 @@ function AuthorHeader() { state.activeAssignmentId, state.name, ]); + + const loadVersions = useAuthorStore((state) => state.loadVersions); const questionsAreReadyToBePublished = useQuestionsAreReadyToBePublished( questions as Question[], ); @@ -157,19 +162,20 @@ function AuthorHeader() { showQuestionScore, showAssignmentScore, showQuestions, + showCorrectAnswer, ] = useAssignmentFeedbackConfig((state) => [ state.showSubmissionFeedback, state.showQuestionScore, state.showAssignmentScore, state.showQuestions, + state.showCorrectAnswer, ]); - const [role, setRole] = useAuthorStore((state) => [ - state.role, - state.setRole, - ]); + const role = useAuthorStore((state) => state.role); const [showAreYouSureModal, setShowAreYouSureModal] = useState(false); + const [showDraftModal, setShowDraftModal] = useState(false); + const [draftName, setDraftName] = useState(""); const deleteAuthorStore = useAuthorStore((state) => state.deleteStore); const deleteAssignmentConfigStore = useAssignmentConfig( @@ -273,6 +279,7 @@ function AuthorHeader() { showSubmissionFeedback: newAssignment.showSubmissionFeedback, showQuestionScore: newAssignment.showQuestionScore, showAssignmentScore: newAssignment.showAssignmentScore, + showCorrectAnswer: newAssignment.showCorrectAnswer, }); useAuthorStore.getState().setName(newAssignment.name); @@ -285,6 +292,8 @@ function AuthorHeader() { }; const fetchAssignment = async () => { + // For now, just load the regular assignment + // TODO: Re-enable draft loading once basic version control is working const assignment = await getAssignment(parseInt(assignmentId, 10)); if (assignment) { const decodedFields = decodeFields({ @@ -371,6 +380,47 @@ function AuthorHeader() { void fetchData(); }, [assignmentId, router]); + // Listen for draft activation publishing events from VersionTreeView + useEffect(() => { + const handleTriggerHeaderPublish = (event: any) => { + const { + description, + publishImmediately, + versionNumber, + updateExisting, + afterPublish, + } = event.detail; + + // Store the afterPublish callback for later use + const originalAfterPublish = afterPublish; + + // Call handlePublishButton with the provided parameters + handlePublishButton(description, publishImmediately) + .then(() => { + // After successful publishing, execute the callback if provided + if ( + originalAfterPublish && + typeof originalAfterPublish === "function" + ) { + originalAfterPublish(); + } + }) + .catch((error) => { + console.error("Header publishing failed:", error); + toast.error("Failed to publish version through header"); + }); + }; + + window.addEventListener("triggerHeaderPublish", handleTriggerHeaderPublish); + + return () => { + window.removeEventListener( + "triggerHeaderPublish", + handleTriggerHeaderPublish, + ); + }; + }, [handlePublishButton]); + function calculateTotalPoints(questions: QuestionAuthorStore[]) { return questions.map((question: QuestionAuthorStore) => { const totalPoints = question.scoring?.rubrics @@ -388,10 +438,16 @@ function AuthorHeader() { }); } - async function handlePublishButton() { + async function handlePublishButton( + description?: string, + publishImmediately = true, + versionNumber?: string, + ): Promise { setSubmitting(true); setJobProgress(0); - setCurrentMessage("Initializing publishing..."); + setCurrentMessage( + publishImmediately ? "Initializing publishing..." : "Creating version...", + ); setProgressStatus("In Progress"); const role = await getUserRole(); @@ -454,18 +510,25 @@ function AuthorHeader() { updatedAt, questionOrder, timeEstimateMinutes: timeEstimateMinutes, - published: true, + published: publishImmediately, showSubmissionFeedback, showQuestions, showQuestionScore, showAssignmentScore, + showCorrectAnswer, numberOfQuestionsPerAttempt, questions: questionsAreDifferent ? processQuestions(clonedCurrentQuestions) : null, + versionDescription: description, + versionNumber: versionNumber, }; if (assignmentData.introduction === null) { - toast.error("Introduction is required to publish the assignment."); + toast.error( + publishImmediately + ? "Introduction is required to publish the assignment." + : "Introduction is required to create a version.", + ); setSubmitting(false); return; } @@ -484,24 +547,40 @@ function AuthorHeader() { }, setQuestions, ); - toast.success("Questions published successfully!"); + if (publishImmediately) { + toast.success("Questions published successfully!"); + } else { + toast.success("Version saved successfully!"); + } setProgressStatus("Completed"); - setTimeout(() => { - router.push( - `/author/${activeAssignmentId}?submissionTime=${Date.now()}`, - ); - }, 300); + + // Reload versions to reflect the new version + try { + await loadVersions(); + } catch (error) { + console.error("Failed to reload versions after publish:", error); + } } else { toast.error( - "Failed to start the publishing process. Please try again.", + publishImmediately + ? "Failed to start the publishing process. Please try again." + : "Failed to create version. Please try again.", ); setProgressStatus("Failed"); } } catch (error: unknown) { if (error instanceof Error) { - toast.error(`Error during publishing: ${error.message}`); + toast.error( + publishImmediately + ? `Error during publishing: ${error.message}` + : `Error creating version: ${error.message}`, + ); } else { - toast.error("An unknown error occurred during publishing."); + toast.error( + publishImmediately + ? "An unknown error occurred during publishing." + : "An unknown error occurred while creating version.", + ); } setProgressStatus("Failed"); } finally { @@ -528,9 +607,59 @@ function AuthorHeader() { toast.success("Synced with latest published version."); }; + const handleSaveChanges = async ( + customDraftName?: string, + ): Promise => { + if (!activeAssignmentId) { + toast.error("No assignment selected"); + return false; + } + + try { + const { saveDraft } = await import("@/lib/author"); + const draftData = { + draftName: + customDraftName || `Manual save - ${new Date().toLocaleString()}`, + assignmentData: { + name, + introduction, + instructions, + gradingCriteriaOverview, + }, + questionsData: questions, + }; + + const result = await saveDraft(activeAssignmentId, draftData); + return !!result; + } catch (error) { + console.error("Save error:", error); + return false; + } + }; + + const handleConfirmSaveDraft = async () => { + setShowDraftModal(false); + const success = await handleSaveChanges(draftName || undefined); + if (success) { + toast.success("Draft saved successfully!"); + } else { + toast.error("Failed to save draft. Please try again."); + } + setDraftName(""); + }; + + const handleCancelSaveDraft = () => { + setShowDraftModal(false); + setDraftName(""); + }; + return ( <> -
+
@@ -551,29 +680,41 @@ function AuthorHeader() { />
- - - - + {/* Admin Insights Button - Only show for admins/authors when assignment exists */} + {(role === "admin" || role === "author") && + activeAssignmentId && ( + + )} + + +
@@ -616,6 +757,49 @@ function AuthorHeader() {
)} + + {showDraftModal && ( + +
+

+ Enter a name for this draft to help you identify it later. +

+
+ + setDraftName(e.target.value)} + placeholder={`Draft - ${new Date().toLocaleDateString()}`} + className="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" + /> +

+ If left empty, a default name with timestamp will be used. +

+
+
+ + +
+
+ + )} ); } diff --git a/apps/web/app/author/(components)/ImportModal.tsx b/apps/web/app/author/(components)/ImportModal.tsx index aa9e12d9..a4b7a0ea 100644 --- a/apps/web/app/author/(components)/ImportModal.tsx +++ b/apps/web/app/author/(components)/ImportModal.tsx @@ -15,7 +15,11 @@ import { ResponseType } from "@/config/types"; interface ImportModalProps { isOpen: boolean; onClose: () => void; - onImport: (questions: QuestionAuthorStore[], options: ImportOptions) => void; + onImport: ( + questions: QuestionAuthorStore[], + options: ImportOptions, + assignmentData?: ParsedData, + ) => void; } interface ImportOptions { @@ -25,6 +29,7 @@ interface ImportOptions { importChoices: boolean; importRubrics: boolean; importConfig: boolean; + importAssignmentSettings: boolean; } interface ParsedData { @@ -59,6 +64,7 @@ const ImportModal: React.FC = ({ importChoices: true, importRubrics: true, importConfig: false, + importAssignmentSettings: false, }); const [isProcessing, setIsProcessing] = useState(false); const [importStep, setImportStep] = useState< @@ -445,10 +451,6 @@ const ImportModal: React.FC = ({ "No questions found in the file. Please check the file format and content.", ); } - - console.log( - `Successfully parsed ${data.questions.length} questions from ${file.name}`, - ); setParsedData(data); if (importOptions.validateQuestions && data.questions) { @@ -1273,7 +1275,7 @@ const ImportModal: React.FC = ({ })); } - onImport(questionsToImport, importOptions); + onImport(questionsToImport, importOptions, parsedData); handleClose(); }; @@ -1421,6 +1423,18 @@ const ImportModal: React.FC = ({ {selectedFile?.name.split(".").pop()?.toUpperCase()}
+ {parsedData.assignment && ( +
+ Assignment data: + Available +
+ )} + {parsedData.config && ( +
+ Configuration: + Available +
+ )}
@@ -1481,6 +1495,12 @@ const ImportModal: React.FC = ({ label: "Import rubrics and scoring", description: "Include grading criteria", }, + { + id: "importAssignmentSettings", + label: "Import assignment settings", + description: + "Include assignment metadata, config, and instructions", + }, { id: "validateQuestions", label: "Validate imported questions", diff --git a/apps/web/app/author/(components)/StepOne/MainContent.tsx b/apps/web/app/author/(components)/StepOne/MainContent.tsx index ddd923f3..bcc6812b 100644 --- a/apps/web/app/author/(components)/StepOne/MainContent.tsx +++ b/apps/web/app/author/(components)/StepOne/MainContent.tsx @@ -5,6 +5,7 @@ import LoadingPage from "@/app/loading"; import MarkdownEditor from "@/components/MarkDownEditor"; import { useAuthorStore } from "@/stores/author"; import SectionWithTitle from "../ReusableSections/SectionWithTitle"; +import React from "react"; const stepOneSections = { introduction: { diff --git a/apps/web/app/author/(components)/StepTwo/AssignmentFeedback.tsx b/apps/web/app/author/(components)/StepTwo/AssignmentFeedback.tsx index e442aa24..488b7606 100644 --- a/apps/web/app/author/(components)/StepTwo/AssignmentFeedback.tsx +++ b/apps/web/app/author/(components)/StepTwo/AssignmentFeedback.tsx @@ -64,6 +64,7 @@ const Component: FC = () => { setShowSubmissionFeedback, setShowQuestionScore, setShowQuestion, + setShowCorrectAnswer, ] = useAssignmentFeedbackConfig((state) => [ state.verbosityLevel, state.setVerbosityLevel, @@ -71,17 +72,20 @@ const Component: FC = () => { state.setShowSubmissionFeedback, state.setShowQuestionScore, state.setShowQuestion, + state.setShowCorrectAnswer, ]); const [ showAssignmentScore, showSubmissionFeedback, showQuestionScore, showQuestions, + showCorrectAnswer, ] = useAssignmentFeedbackConfig((state) => [ state.showAssignmentScore, state.showSubmissionFeedback, state.showQuestionScore, state.showQuestions, + state.showCorrectAnswer, ]); const handleButtonClick = (verbosity: VerbosityLevels) => { setVerbosityLevel(verbosity); @@ -91,36 +95,53 @@ const Component: FC = () => { setShowSubmissionFeedback(true); setShowQuestionScore(true); setShowQuestion(true); + setShowCorrectAnswer(true); break; case "Custom": setShowAssignmentScore(true); setShowSubmissionFeedback(false); setShowQuestionScore(true); setShowQuestion(true); + setShowCorrectAnswer(true); break; case "None": setShowAssignmentScore(false); setShowSubmissionFeedback(false); setShowQuestionScore(false); setShowQuestion(false); + setShowCorrectAnswer(false); break; default: break; } }; useEffect(() => { - if (showAssignmentScore && showSubmissionFeedback && showQuestionScore) { + if ( + showAssignmentScore && + showSubmissionFeedback && + showQuestionScore && + showQuestions && + showCorrectAnswer + ) { setVerbosityLevel("Full"); } else if ( !showAssignmentScore && !showSubmissionFeedback && - !showQuestionScore + !showQuestionScore && + !showQuestions && + !showCorrectAnswer ) { setVerbosityLevel("None"); } else { setVerbosityLevel("Custom"); } - }, [showAssignmentScore, showSubmissionFeedback, showQuestionScore]); + }, [ + showAssignmentScore, + showSubmissionFeedback, + showQuestionScore, + showQuestions, + showCorrectAnswer, + ]); return ( ; diff --git a/apps/web/app/author/(components)/StepTwo/FeedbackSettings.tsx b/apps/web/app/author/(components)/StepTwo/FeedbackSettings.tsx index 508ca259..f26bc591 100644 --- a/apps/web/app/author/(components)/StepTwo/FeedbackSettings.tsx +++ b/apps/web/app/author/(components)/StepTwo/FeedbackSettings.tsx @@ -57,10 +57,12 @@ const SettingsContainer: React.FC = () => { toggleShowSubmissionFeedback, toggleShowQuestionScore, toggleShowQuestions, + toggleShowCorrectAnswer, showAssignmentScore, showSubmissionFeedback, showQuestionScore, showQuestions, + showCorrectAnswer, } = useAssignmentFeedbackConfig(); const settingsData = [ @@ -90,6 +92,13 @@ const SettingsContainer: React.FC = () => { value: showQuestions, toggleValue: toggleShowQuestions, }, + { + title: "Show Correct Answers on Pass", + description: + "The correct answer will be visible to the learner only if they pass the assignment.", + value: showCorrectAnswer, + toggleValue: toggleShowCorrectAnswer, + }, ] as const; return (
diff --git a/apps/web/app/author/(components)/WarningBeforeUnload.tsx b/apps/web/app/author/(components)/WarningBeforeUnload.tsx index d0e5aafb..f1bc96ed 100644 --- a/apps/web/app/author/(components)/WarningBeforeUnload.tsx +++ b/apps/web/app/author/(components)/WarningBeforeUnload.tsx @@ -1,5 +1,7 @@ "use client"; +import WarningModal from "@/components/WarningModal"; +import useBeforeUnload from "@/hooks/use-before-unload"; import { usePathname } from "next/navigation"; const WarningBeforeUnload = () => { diff --git a/apps/web/app/author/[assignmentId]/review/page.tsx b/apps/web/app/author/[assignmentId]/review/page.tsx index c7cd9421..57cbacc5 100644 --- a/apps/web/app/author/[assignmentId]/review/page.tsx +++ b/apps/web/app/author/[assignmentId]/review/page.tsx @@ -27,10 +27,224 @@ import { PencilIcon, StarIcon, DocumentArrowDownIcon, + DocumentArrowUpIcon, + XMarkIcon, + WrenchScrewdriverIcon, } from "@heroicons/react/24/solid"; import { QuestionAuthorStore } from "@/config/types"; import ExportModal, { ExportOptions } from "../../(components)/ExportModal"; +// Helper function to determine if a validation error is question-related +const isQuestionRelatedValidationError = (message: string): boolean => { + const questionRelatedErrors = [ + "question", + "rubric", + "choice", + "variant", + "description", + "text", + ]; + const hasQuestionIssue = questionRelatedErrors.some((error) => + message.toLowerCase().includes(error.toLowerCase()), + ); + return hasQuestionIssue; +}; + +// Issues Modal Component +const IssuesModal = ({ + isOpen, + onClose, + questionIssues, + questions, + isValid, + message, + invalidQuestionId, + onNavigateToFix, + onAutoFix, + onNavigateToConfig, +}: { + isOpen: boolean; + onClose: () => void; + questionIssues: Record; + questions: QuestionAuthorStore[]; + isValid: boolean; + message: string; + invalidQuestionId: number | null; + onNavigateToFix: (questionId: number) => void; + onAutoFix: (questionId: number, issue: string) => void; + onNavigateToConfig: () => void; +}) => { + if (!isOpen) return null; + + const totalIssues = Object.keys(questionIssues).length; + const hasValidationError = !isValid && message; + const totalAllIssues = totalIssues + (hasValidationError ? 1 : 0); + + return ( +
+
+ {/* Header */} +
+
+ +

+ Issues Found ({totalAllIssues} total) +

+
+ +
+ + {/* Content */} +
+ {/* Show configuration error only if it's not a question-specific issue */} + {!isValid && + message && + !isQuestionRelatedValidationError(message) && ( +
+
+
+ +
+

+ Configuration Error +

+

{message}

+
+
+ +
+
+ )} + + {/* Show note about question-related validation errors */} + {!isValid && + message && + Object.keys(questionIssues).length > 0 && + isQuestionRelatedValidationError(message) && ( +
+
+ +
+

Note

+

+ The validation system also detected this issue, but it's + shown below as a question-specific issue with fix options. +

+
+
+
+ )} + +
+ {Object.entries(questionIssues).map(([questionId, issues]) => { + const question = questions.find( + (q) => q.id === parseInt(questionId), + ); + const questionIndex = + questions.findIndex((q) => q.id === parseInt(questionId)) + 1; + + return ( +
+
+

+ Question {questionIndex}:{" "} + + {question?.question || "Untitled Question"} + +

+ +
+ +
+ {issues.map((issue, index) => ( +
+
+ + {issue} +
+ + {/* Auto-fix button for fixable issues */} + {canAutoFix(issue) && ( + + )} +
+ ))} +
+
+ ); + })} +
+
+ + {/* Footer */} +
+
+

+ Fix these issues to ensure your assignment works properly for + learners. +

+ +
+
+
+
+ ); +}; + +// Helper function to determine if an issue can be auto-fixed +const canAutoFix = (issue: string): boolean => { + const autoFixableIssues = [ + "Question type not selected", + "Question title is empty", + "No choices added", + "No rubrics defined", + "has no criteria defined", + "description is empty", + "question is empty", + ]; + return autoFixableIssues.some((fixable) => issue.includes(fixable)); +}; + // Component to show before/after comparison const ChangeComparison = ({ label, @@ -528,6 +742,7 @@ const QuestionChanges = ({ function Component() { const [viewMode, setViewMode] = useState<"changes" | "full">("changes"); const [isExportModalOpen, setIsExportModalOpen] = useState(false); + const [isIssuesModalOpen, setIsIssuesModalOpen] = useState(false); const [ graded, @@ -560,6 +775,8 @@ function Component() { learningObjectives, name, questionOrder, + replaceQuestion, + addQuestion, ] = useAuthorStore((state) => [ state.introduction, state.instructions, @@ -569,6 +786,8 @@ function Component() { state.learningObjectives, state.name, state.questionOrder, + state.replaceQuestion, + state.addQuestion, ]); const [ @@ -577,12 +796,14 @@ function Component() { showQuestionScore, showAssignmentScore, showQuestions, + showCorrectAnswer, ] = useAssignmentFeedbackConfig((state) => [ state.verbosityLevel, state.showSubmissionFeedback, state.showQuestionScore, state.showAssignmentScore, state.showQuestions, + state.showCorrectAnswer, ]); const router = useRouter(); @@ -623,6 +844,9 @@ function Component() { showAssignmentScore: filteredChanges.some((c) => c.includes("Changed assignment score visibility"), ), + showCorrectAnswer: filteredChanges.some((c) => + c.includes("Changed correct answer visibility"), + ), questionOrder: filteredChanges.some((c) => c.includes("Modified question order"), ), @@ -711,13 +935,34 @@ function Component() { qIssues.push("Some choices are empty"); } - if ( - (q.type === "TEXT" || q.type === "URL" || q.type === "UPLOAD") && - (!q.scoring?.rubrics || - q.scoring.rubrics.length === 0 || - q.scoring.rubrics.some((r) => !r.criteria || r.criteria.length === 0)) - ) { - qIssues.push("No rubric criteria defined"); + if (q.type === "TEXT" || q.type === "URL" || q.type === "UPLOAD") { + if (!q.scoring?.rubrics || q.scoring.rubrics.length === 0) { + qIssues.push("No rubrics defined"); + } else { + // Check each rubric + q.scoring.rubrics.forEach((rubric, rubricIndex) => { + // Check if rubric question is empty + if (!rubric.rubricQuestion || rubric.rubricQuestion.trim() === "") { + qIssues.push(`Rubric ${rubricIndex + 1} question is empty`); + } + + if (!rubric.criteria || rubric.criteria.length === 0) { + qIssues.push(`Rubric ${rubricIndex + 1} has no criteria defined`); + } else { + // Check each criteria in the rubric + rubric.criteria.forEach((criteria, criteriaIndex) => { + if ( + !criteria.description || + criteria.description.trim() === "" + ) { + qIssues.push( + `Rubric ${rubricIndex + 1} criteria ${criteriaIndex + 1} description is empty`, + ); + } + }); + } + }); + } } if (qIssues.length > 0) { @@ -733,6 +978,126 @@ function Component() { return changes.details.filter((d) => d.includes(`question ${questionId}`)); }; + // Handle navigation to fix a specific question + const handleNavigateToFix = (questionId: number) => { + setIsIssuesModalOpen(false); + router.push(`/author/${activeAssignmentId}/questions`); + setTimeout(() => { + const element = document.getElementById(`question-${questionId}`); + if (element) { + element.scrollIntoView({ + behavior: "smooth", + block: "center", + }); + } + }, 500); + }; + + // Handle navigation to config page + const handleNavigateToConfig = () => { + router.push(`/author/${activeAssignmentId}/config`); + }; + + // Handle auto-fix functionality + const handleAutoFix = (questionId: number, issue: string) => { + const question = questions?.find((q) => q.id === questionId); + if (!question) return; + + let updatedQuestion = { ...question }; + + // Auto-fix based on issue type + if (issue.includes("Question type not selected")) { + updatedQuestion.type = "TEXT"; // Default to text question + } + + if (issue.includes("Question title is empty")) { + updatedQuestion.question = "Untitled Question"; + } + + if ( + issue.includes("No choices added") && + (question.type === "MULTIPLE_CORRECT" || + question.type === "SINGLE_CORRECT") + ) { + updatedQuestion.choices = [ + { choice: "Option 1", isCorrect: true, points: 1 }, + { choice: "Option 2", isCorrect: false, points: 0 }, + ]; + } + + // Handle rubric-related issues + if (issue.includes("No rubrics defined")) { + updatedQuestion.scoring = { + ...updatedQuestion.scoring, + rubrics: [ + { + rubricQuestion: "Default Rubric", + criteria: [ + { + description: "Default criteria description", + points: 1, + id: 1, + }, + ], + }, + ], + }; + } + + if (issue.includes("has no criteria defined")) { + // Extract rubric index from issue text like "Rubric 1 has no criteria defined" + const rubricMatch = issue.match(/Rubric (\d+)/); + if (rubricMatch && updatedQuestion.scoring?.rubrics) { + const rubricIndex = parseInt(rubricMatch[1]) - 1; + if (updatedQuestion.scoring.rubrics[rubricIndex]) { + updatedQuestion.scoring.rubrics[rubricIndex].criteria = [ + { + description: "Default criteria description", + points: 1, + id: 1, + }, + ]; + } + } + } + + if (issue.includes("description is empty")) { + // Extract rubric and criteria indices from issue text like "Rubric 1 criteria 1 description is empty" + const match = issue.match(/Rubric (\d+) criteria (\d+)/); + if (match && updatedQuestion.scoring?.rubrics) { + const rubricIndex = parseInt(match[1]) - 1; + const criteriaIndex = parseInt(match[2]) - 1; + if ( + updatedQuestion.scoring.rubrics[rubricIndex]?.criteria?.[ + criteriaIndex + ] + ) { + updatedQuestion.scoring.rubrics[rubricIndex].criteria[ + criteriaIndex + ].description = "Default criteria description"; + } + } + } + + if (issue.includes("question is empty")) { + // Extract rubric index from issue text like "Rubric 1 question is empty" + const rubricMatch = issue.match(/Rubric (\d+)/); + if (rubricMatch && updatedQuestion.scoring?.rubrics) { + const rubricIndex = parseInt(rubricMatch[1]) - 1; + if (updatedQuestion.scoring.rubrics[rubricIndex]) { + updatedQuestion.scoring.rubrics[rubricIndex].rubricQuestion = + "Default Rubric Question"; + } + } + } + + // Update the question in the store + replaceQuestion(questionId, updatedQuestion); + + // Show a success message (you could replace this with a toast notification) + alert(`Auto-fixed: ${issue}`); + }; + // Handle export functionality const handleExport = async (exportOptions: ExportOptions) => { try { @@ -1090,10 +1455,35 @@ function Component() { )} {!isValid && ( - + )}
@@ -1102,7 +1492,7 @@ function Component() { onClick={() => setIsExportModalOpen(true)} className="px-4 py-2 text-sm font-medium text-white bg-purple-600 border border-transparent rounded-md hover:bg-purple-700 transition-colors flex items-center gap-2" > - + Export @@ -1646,6 +2036,20 @@ function Component() { onClose={() => setIsExportModalOpen(false)} onExport={handleExport} /> + + {/* Issues Modal */} + setIsIssuesModalOpen(false)} + questionIssues={questionIssues} + questions={questions || []} + isValid={isValid} + message={message} + invalidQuestionId={invalidQuestionId} + onNavigateToFix={handleNavigateToFix} + onAutoFix={handleAutoFix} + onNavigateToConfig={handleNavigateToConfig} + /> ); } diff --git a/apps/web/app/author/[assignmentId]/version-tree/page.tsx b/apps/web/app/author/[assignmentId]/version-tree/page.tsx new file mode 100644 index 00000000..21c2dec0 --- /dev/null +++ b/apps/web/app/author/[assignmentId]/version-tree/page.tsx @@ -0,0 +1,11 @@ +"use client"; + +import { VersionTreeView } from "@/components/version-control/VersionTreeView"; + +interface Props { + params: { assignmentId: string }; +} + +export default function VersionTreePage({ params }: Props) { + return ; +} diff --git a/apps/web/app/author/layout.tsx b/apps/web/app/author/layout.tsx index 876c5cdb..4158fafb 100644 --- a/apps/web/app/author/layout.tsx +++ b/apps/web/app/author/layout.tsx @@ -1,4 +1,5 @@ import Header from "./(components)/Header"; +import { BottomVersionBar } from "@/components/version-control/BottomVersionBar"; export default function RootLayout({ children, @@ -8,9 +9,10 @@ export default function RootLayout({ return ( <>
-
+
{children}
+ ); } diff --git a/apps/web/app/chatbot/components/MarkChat.tsx b/apps/web/app/chatbot/components/MarkChat.tsx index dcb95043..c6e735c1 100644 --- a/apps/web/app/chatbot/components/MarkChat.tsx +++ b/apps/web/app/chatbot/components/MarkChat.tsx @@ -15,10 +15,10 @@ import { ClockIcon, ArchiveBoxIcon, BellIcon, + ChatBubbleBottomCenterTextIcon, } from "@heroicons/react/24/outline"; import { ArrowPathIcon, - ChevronDownIcon, PaperAirplaneIcon, SparklesIcon, XMarkIcon, @@ -26,12 +26,14 @@ import { AdjustmentsHorizontalIcon, } from "@heroicons/react/24/solid"; import { AnimatePresence, motion } from "framer-motion"; +import Draggable from "react-draggable"; import Image from "next/image"; -import React, { useEffect, useRef, useState, useCallback } from "react"; +import React, { useEffect, useRef, useState } from "react"; import ReactMarkdown from "react-markdown"; import { useAuthorContext } from "../store/useAuthorContext"; import { useLearnerContext } from "../store/useLearnerContext"; import { ChatRole, useMarkChatStore } from "../store/useMarkChatStore"; +import { useAuthorStore } from "@/stores/author"; import { toast } from "sonner"; import Tippy from "@tippyjs/react"; import "tippy.js/dist/tippy.css"; @@ -42,10 +44,152 @@ import { addMessageToChat, endChat, getUser, + directUpload, + getFileType, } from "@/lib/shared"; +import { getBaseApiPath } from "@/config/constants"; import UserReportsPanel from "./UserReportsPanel"; -// import { NotificationsPanel } from "./NotificationPanel"; -// import { getUserNotifications, markNotificationAsRead } from "@/lib/author"; +import ReportPreviewModal from "@/components/ReportPreviewModal"; +import { useChatbot } from "../../../hooks/useChatbot"; +import { useMarkSpeech } from "../../../hooks/useMarkSpeech"; +import { useUserBehaviorMonitor } from "../../../hooks/useUserBehaviorMonitor"; +import { useDropzone } from "react-dropzone"; +import { useCallback } from "react"; +import SpeechBubble from "../../../components/SpeechBubble"; +import { NotificationsPanel } from "./NotificationPanel"; +import { getUserNotifications, markNotificationAsRead } from "@/lib/author"; + +interface ScreenshotDropzoneProps { + file: File | null | undefined; + onFileSelect: (file: File) => void; + onFileRemove: () => void; +} + +const ScreenshotDropzone: React.FC = ({ + file, + onFileSelect, + onFileRemove, +}) => { + const onDrop = useCallback( + (acceptedFiles: File[]) => { + if (acceptedFiles.length > 0) { + onFileSelect(acceptedFiles[0]); + } + }, + [onFileSelect], + ); + + const { getRootProps, getInputProps, isDragActive, fileRejections } = + useDropzone({ + onDrop, + accept: { + "image/*": [".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".svg"], + }, + multiple: false, + maxSize: 10 * 1024 * 1024, // 10MB + onDropRejected: (rejectedFiles) => { + const error = rejectedFiles[0]?.errors[0]; + if (error?.code === "file-too-large") { + toast.error("File is too large. Maximum size is 10MB."); + } else if (error?.code === "file-invalid-type") { + toast.error("Invalid file type. Please select an image file."); + } else { + toast.error("File rejected. Please try another file."); + } + }, + }); + + if (file) { + return ( +
+
+ + + + {file.name} + + {Math.round(file.size / 1024)}KB + +
+ +
+ ); + } + + const rootProps = getRootProps(); + + return ( +
+ +
+ {isDragActive ? ( + <> + + + +

+ Drop screenshot here +

+ + ) : ( + <> + + + +

+ Drop screenshot here or click to select +

+

+ PNG, JPG, GIF up to 10MB +

+ + )} +
+
+ ); +}; const SuggestionsPanel = ({ suggestions, @@ -293,6 +437,10 @@ const SpecialActionUI = ({ handleRegradeRequest, handleIssueReport, handleCreateQuestion, + handleReportPreview, + handleFeedback, + handleSuggestion, + handleInquiry, }) => { if (!specialActions.show) return null; return ( @@ -379,6 +527,57 @@ const SpecialActionUI = ({
+ ) : specialActions.type === "feedback" ? ( +
+
+ +

Share Your Feedback

+
+

+ I'd love to hear about your experience with the platform. Your + feedback helps us improve! +

+ +
+ ) : specialActions.type === "suggestion" ? ( +
+
+ +

Share Your Suggestion

+
+

+ I'd love to hear your ideas for improving the platform or adding new + features! +

+ +
+ ) : specialActions.type === "inquiry" ? ( +
+
+ +

Ask a Question

+
+

+ I'm here to help answer your questions and provide assistance with + whatever you need. +

+ +
) : null} ); @@ -524,38 +723,70 @@ const ChatMessages = ({ chatBubbleVariants, getAccentColor, renderTypingIndicator, + onClientExecution, }) => { + const filteredMessages = React.useMemo( + () => messages.filter((msg) => msg.role !== "system"), + [messages], + ); + + const [processedMessageIds, setProcessedMessageIds] = React.useState( + new Set(), + ); + + // Handle client executions outside of the render loop + React.useEffect(() => { + if (onClientExecution) { + filteredMessages.forEach((msg) => { + // Only process messages that haven't been processed yet + if ( + !processedMessageIds.has(msg.id) && + msg.role === "assistant" && + msg.toolCalls && + Array.isArray(msg.toolCalls) + ) { + msg.toolCalls.forEach((toolCall) => { + if (toolCall.function === "showReportPreview") { + onClientExecution(toolCall); + } + }); + // Mark this message as processed + setProcessedMessageIds((prev) => new Set(prev).add(msg.id)); + } + }); + } + }, [filteredMessages, onClientExecution, processedMessageIds]); + return ( <> - {messages - .filter((msg) => msg.role !== "system") - .map((msg, index) => { - const messageContent = msg.content; - return ( - { + const messageContent = msg.content; + + return ( + +
-
-
- {messageContent} -
+
+ {messageContent}
- - ); - })} +
+ + ); + })} {renderTypingIndicator()} ); @@ -587,7 +818,7 @@ const ChatHistoryDrawer = ({ transition={{ type: "spring", damping: 25, stiffness: 300 }} className="fixed left-0 top-0 h-full w-80 bg-white dark:bg-gray-900 shadow-xl z-[999999] overflow-y-auto" > -
+

Chat History

- {/* */} - {/* */} - - - - - - + {isChatbotOpen && ( + + {/* Header */} +
+
+ {MarkFace && ( + Mark AI Assistant + )} +
+

+ Mark AI Assistant +

+

+ Your AI learning companion +

+ +
-
+ {/* Action buttons bar */} +
+
+ + + + + {/* handleCheckReports */} +
+ +
+ + {/* Chat content */} +
+ {/* Settings panel */} + + {showSettings && ( + + )} + {showNotifications && ( + { + markNotificationRead(notification.id); + setShowNotifications(false); + }} + onClose={() => setShowNotifications(false)} + /> + )} + + + {/* Context indicators */} +
+ {/* Messages area */}
- - {showSettings && ( - +
+ {userInput.trim() !== "" && ( + + + )} - - - {/* {showNotifications && ( - setShowNotifications(false)} - onMarkRead={markNotificationRead} - onClickNotification={handleNotificationClick} - /> - )} */} - - +
- - {specialActions.show && ( - - )} - - {isInitializing ? (
@@ -1984,15 +2842,14 @@ Can you help me complete and implement this question?`; chatBubbleVariants={chatBubbleVariants} getAccentColor={getAccentColor} renderTypingIndicator={renderTypingIndicator} + onClientExecution={handleClientExecution} /> )}
- + {/* Input area */} +
{showSuggestions && ( )} -
+ + {/* Report Preview Form - positioned right above input */} + + {specialActions.show && ( + + )} + + +