From 277fdba258b98031563753c46c66faf7f7336e0b Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Tue, 23 Sep 2025 15:20:13 +1000 Subject: [PATCH 01/23] Check reviewers are assigned before opening review phase --- .gitignore | 3 + src/autopilot/autopilot.module.ts | 8 +- src/autopilot/constants/review.constants.ts | 12 + src/autopilot/services/autopilot.service.ts | 149 +++++++--- .../services/phase-review.service.ts | 12 +- .../services/review-assignment.service.ts | 280 ++++++++++++++++++ src/config/sections/review.config.ts | 17 +- src/config/validation.ts | 4 + 8 files changed, 426 insertions(+), 59 deletions(-) create mode 100644 src/autopilot/constants/review.constants.ts create mode 100644 src/autopilot/services/review-assignment.service.ts diff --git a/.gitignore b/.gitignore index 3502ef7..81f046b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ yarn-error.log* lerna-debug.log* .pnpm-debug.log* +# IDE +.vscode/ + # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json diff --git a/src/autopilot/autopilot.module.ts b/src/autopilot/autopilot.module.ts index 6c25a0f..b214fc8 100644 --- a/src/autopilot/autopilot.module.ts +++ b/src/autopilot/autopilot.module.ts @@ -7,6 +7,7 @@ import { ChallengeModule } from '../challenge/challenge.module'; import { ReviewModule } from '../review/review.module'; import { ResourcesModule } from '../resources/resources.module'; import { PhaseReviewService } from './services/phase-review.service'; +import { ReviewAssignmentService } from './services/review-assignment.service'; @Module({ imports: [ @@ -18,7 +19,12 @@ import { PhaseReviewService } from './services/phase-review.service'; ReviewModule, ResourcesModule, ], - providers: [AutopilotService, SchedulerService, PhaseReviewService], + providers: [ + AutopilotService, + SchedulerService, + PhaseReviewService, + ReviewAssignmentService, + ], exports: [AutopilotService, SchedulerService], }) export class AutopilotModule {} diff --git a/src/autopilot/constants/review.constants.ts b/src/autopilot/constants/review.constants.ts new file mode 100644 index 0000000..a09d0d9 --- /dev/null +++ b/src/autopilot/constants/review.constants.ts @@ -0,0 +1,12 @@ +export const REVIEW_PHASE_NAMES = new Set(['Review', 'Iterative Review']); + +const DEFAULT_PHASE_ROLES = ['Reviewer', 'Iterative Reviewer']; + +export const PHASE_ROLE_MAP: Record = { + Review: ['Reviewer'], + 'Iterative Review': ['Iterative Reviewer'], +}; + +export function getRoleNamesForPhase(phaseName: string): string[] { + return PHASE_ROLE_MAP[phaseName] ?? DEFAULT_PHASE_ROLES; +} diff --git a/src/autopilot/services/autopilot.service.ts b/src/autopilot/services/autopilot.service.ts index 9f851fd..2d06c29 100644 --- a/src/autopilot/services/autopilot.service.ts +++ b/src/autopilot/services/autopilot.service.ts @@ -1,5 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { SchedulerService } from './scheduler.service'; +import { PhaseReviewService } from './phase-review.service'; +import { ReviewAssignmentService } from './review-assignment.service'; import { PhaseTransitionPayload, ChallengeUpdatePayload, @@ -9,6 +11,7 @@ import { import { ChallengeApiService } from '../../challenge/challenge-api.service'; import { IPhase } from '../../challenge/interfaces/challenge.interface'; import { AUTOPILOT_COMMANDS } from '../../common/constants/commands.constants'; +import { REVIEW_PHASE_NAMES } from '../constants/review.constants'; @Injectable() export class AutopilotService { @@ -19,6 +22,8 @@ export class AutopilotService { constructor( private readonly schedulerService: SchedulerService, private readonly challengeApiService: ChallengeApiService, + private readonly phaseReviewService: PhaseReviewService, + private readonly reviewAssignmentService: ReviewAssignmentService, ) { // Set up the phase chain callback to handle next phase opening and scheduling this.schedulerService.setPhaseChainCallback( @@ -540,79 +545,127 @@ export class AutopilotService { ); let processedCount = 0; + let deferredCount = 0; + for (const nextPhase of nextPhases) { - try { - // Step 1: Open the phase first - this.logger.log( - `[PHASE CHAIN] Opening phase ${nextPhase.name} (${nextPhase.id}) for challenge ${challengeId}`, + const openPhaseCallback = async () => + await this.openPhaseAndSchedule( + challengeId, + projectId, + projectStatus, + nextPhase, ); - const openResult = await this.challengeApiService.advancePhase( + try { + const canOpenNow = await this.reviewAssignmentService.ensureAssignmentsOrSchedule( challengeId, - nextPhase.id, - 'open', + nextPhase, + openPhaseCallback, ); - if (!openResult.success) { - this.logger.error( - `[PHASE CHAIN] Failed to open phase ${nextPhase.name} (${nextPhase.id}) for challenge ${challengeId}: ${openResult.message}`, - ); + if (!canOpenNow) { + deferredCount++; continue; } - this.logger.log( - `[PHASE CHAIN] Successfully opened phase ${nextPhase.name} (${nextPhase.id}) for challenge ${challengeId}`, + const opened = await openPhaseCallback(); + if (opened) { + processedCount++; + } + } catch (error) { + const err = error as Error; + this.logger.error( + `[PHASE CHAIN] Failed to open and schedule phase ${nextPhase.name} (${nextPhase.id}) for challenge ${challengeId}: ${err.message}`, + err.stack, ); + } + } - // Step 2: Schedule the phase for closure (use updated phase data from API response) - const updatedPhase = - openResult.updatedPhases?.find((p) => p.id === nextPhase.id) || - nextPhase; + const summaryParts = [`opened and scheduled ${processedCount} out of ${nextPhases.length}`]; + if (deferredCount > 0) { + summaryParts.push(`deferred ${deferredCount} awaiting reviewer assignments`); + } - if (!updatedPhase.scheduledEndDate) { - this.logger.warn( - `[PHASE CHAIN] Opened phase ${nextPhase.name} (${nextPhase.id}) has no scheduled end date, skipping scheduling`, - ); - continue; - } + this.logger.log( + `[PHASE CHAIN] ${summaryParts.join(', ')} for challenge ${challengeId}`, + ); + } - // Check if this phase is already scheduled to avoid duplicates - const phaseKey = `${challengeId}:${nextPhase.id}`; - if (this.activeSchedules.has(phaseKey)) { - this.logger.log( - `[PHASE CHAIN] Phase ${nextPhase.name} (${nextPhase.id}) is already scheduled, skipping`, - ); - continue; - } + private async openPhaseAndSchedule( + challengeId: string, + projectId: number, + projectStatus: string, + phase: IPhase, + ): Promise { + this.logger.log( + `[PHASE CHAIN] Opening phase ${phase.name} (${phase.id}) for challenge ${challengeId}`, + ); - const nextPhaseData: PhaseTransitionPayload = { - projectId, - challengeId, - phaseId: updatedPhase.id, - phaseTypeName: updatedPhase.name, - state: 'END', - operator: AutopilotOperator.SYSTEM_PHASE_CHAIN, - projectStatus, - date: updatedPhase.scheduledEndDate, - }; + const openResult = await this.challengeApiService.advancePhase( + challengeId, + phase.id, + 'open', + ); - const jobId = this.schedulePhaseTransition(nextPhaseData); - processedCount++; - this.logger.log( - `[PHASE CHAIN] Scheduled opened phase ${updatedPhase.name} (${updatedPhase.id}) for closure at ${updatedPhase.scheduledEndDate} with job ID: ${jobId}`, - ); + if (!openResult.success) { + this.logger.error( + `[PHASE CHAIN] Failed to open phase ${phase.name} (${phase.id}) for challenge ${challengeId}: ${openResult.message}`, + ); + return false; + } + + this.reviewAssignmentService.clearPolling(challengeId, phase.id); + + this.logger.log( + `[PHASE CHAIN] Successfully opened phase ${phase.name} (${phase.id}) for challenge ${challengeId}`, + ); + + if (REVIEW_PHASE_NAMES.has(phase.name)) { + try { + await this.phaseReviewService.handlePhaseOpened(challengeId, phase.id); } catch (error) { const err = error as Error; this.logger.error( - `[PHASE CHAIN] Failed to open and schedule phase ${nextPhase.name} (${nextPhase.id}) for challenge ${challengeId}: ${err.message}`, + `[PHASE CHAIN] Failed to prepare review records for phase ${phase.name} (${phase.id}) on challenge ${challengeId}: ${err.message}`, err.stack, ); } } + const updatedPhase = + openResult.updatedPhases?.find((p) => p.id === phase.id) || phase; + + if (!updatedPhase.scheduledEndDate) { + this.logger.warn( + `[PHASE CHAIN] Opened phase ${phase.name} (${phase.id}) has no scheduled end date, skipping scheduling`, + ); + return false; + } + + const phaseKey = `${challengeId}:${phase.id}`; + if (this.activeSchedules.has(phaseKey)) { + this.logger.log( + `[PHASE CHAIN] Phase ${phase.name} (${phase.id}) is already scheduled, skipping`, + ); + return false; + } + + const nextPhaseData: PhaseTransitionPayload = { + projectId, + challengeId, + phaseId: updatedPhase.id, + phaseTypeName: updatedPhase.name, + state: 'END', + operator: AutopilotOperator.SYSTEM_PHASE_CHAIN, + projectStatus, + date: updatedPhase.scheduledEndDate, + }; + + const jobId = this.schedulePhaseTransition(nextPhaseData); this.logger.log( - `[PHASE CHAIN] Successfully opened and scheduled ${processedCount} out of ${nextPhases.length} next phases for challenge ${challengeId}`, + `[PHASE CHAIN] Scheduled opened phase ${updatedPhase.name} (${updatedPhase.id}) for closure at ${updatedPhase.scheduledEndDate} with job ID: ${jobId}`, ); + return true; } /** diff --git a/src/autopilot/services/phase-review.service.ts b/src/autopilot/services/phase-review.service.ts index b355c09..05f484d 100644 --- a/src/autopilot/services/phase-review.service.ts +++ b/src/autopilot/services/phase-review.service.ts @@ -3,12 +3,10 @@ import { ChallengeApiService } from '../../challenge/challenge-api.service'; import { ReviewService } from '../../review/review.service'; import { ResourcesService } from '../../resources/resources.service'; import { IChallengeReviewer } from '../../challenge/interfaces/challenge.interface'; - -const REVIEW_PHASE_NAMES = new Set(['Review', 'Iterative Review']); -const PHASE_ROLE_MAP: Record = { - Review: ['Reviewer'], - 'Iterative Review': ['Iterative Reviewer'], -}; +import { + getRoleNamesForPhase, + REVIEW_PHASE_NAMES, +} from '../constants/review.constants'; @Injectable() export class PhaseReviewService { @@ -55,7 +53,7 @@ export class PhaseReviewService { return; } - const roleNames = PHASE_ROLE_MAP[phase.name] ?? ['Reviewer', 'Iterative Reviewer']; + const roleNames = getRoleNamesForPhase(phase.name); const reviewerResources = await this.resourcesService.getReviewerResources( challengeId, roleNames, diff --git a/src/autopilot/services/review-assignment.service.ts b/src/autopilot/services/review-assignment.service.ts new file mode 100644 index 0000000..8628edb --- /dev/null +++ b/src/autopilot/services/review-assignment.service.ts @@ -0,0 +1,280 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { ConfigService } from '@nestjs/config'; +import { ChallengeApiService } from '../../challenge/challenge-api.service'; +import { ResourcesService } from '../../resources/resources.service'; +import { + IChallenge, + IChallengeReviewer, + IPhase, +} from '../../challenge/interfaces/challenge.interface'; +import { + REVIEW_PHASE_NAMES, + getRoleNamesForPhase, +} from '../constants/review.constants'; + +interface PhaseSummary { + id: string; + phaseId: string; + name: string; +} + +interface PollerContext { + processing: boolean; + interval: NodeJS.Timeout; +} + +interface AssignmentStatus { + ready: boolean; + required: number; + assigned: number; + phaseMissing: boolean; + phaseOpen: boolean; +} + +@Injectable() +export class ReviewAssignmentService { + private readonly logger = new Logger(ReviewAssignmentService.name); + private readonly pollers = new Map(); + private readonly pollIntervalMs: number; + + constructor( + private readonly schedulerRegistry: SchedulerRegistry, + private readonly configService: ConfigService, + private readonly challengeApiService: ChallengeApiService, + private readonly resourcesService: ResourcesService, + ) { + const configuredInterval = this.configService.get( + 'review.assignmentPollIntervalMs', + ); + + const defaultInterval = 5 * 60 * 1000; // 5 minutes + this.pollIntervalMs = configuredInterval && configuredInterval > 0 + ? configuredInterval + : defaultInterval; + } + + async ensureAssignmentsOrSchedule( + challengeId: string, + phase: IPhase, + openPhaseCallback: () => Promise, + ): Promise { + if (!REVIEW_PHASE_NAMES.has(phase.name)) { + return true; + } + + const status = await this.evaluateAssignmentStatus(challengeId, phase); + const phaseKey = this.buildKey(challengeId, phase.id); + + if (status.phaseMissing) { + this.logger.warn( + `Review phase ${phase.id} not found when verifying assignments for challenge ${challengeId}. Proceeding with opening to avoid blocking phase chain.`, + ); + this.clearPolling(challengeId, phase.id); + return true; + } + + if (status.phaseOpen || status.ready) { + this.clearPolling(challengeId, phase.id); + return true; + } + + this.logger.warn( + `Insufficient reviewers for challenge ${challengeId} phase ${phase.id} (${phase.name}). Required: ${status.required}, Assigned: ${status.assigned}. Deferring phase opening and polling every ${Math.round(this.pollIntervalMs / 1000)}s.`, + ); + + if (!this.pollers.has(phaseKey)) { + this.startPolling(challengeId, phase, openPhaseCallback); + } + + return false; + } + + clearPolling(challengeId: string, phaseId: string): void { + const key = this.buildKey(challengeId, phaseId); + const context = this.pollers.get(key); + + if (!context) { + return; + } + + if (this.schedulerRegistry.doesExist('interval', key)) { + this.schedulerRegistry.deleteInterval(key); + } + + clearInterval(context.interval); + this.pollers.delete(key); + this.logger.debug( + `Stopped reviewer assignment polling for challenge ${challengeId}, phase ${phaseId}.`, + ); + } + + private startPolling( + challengeId: string, + phase: PhaseSummary, + openPhaseCallback: () => Promise, + ): void { + const key = this.buildKey(challengeId, phase.id); + + if (this.pollers.has(key)) { + return; + } + + const context: PollerContext = { + processing: false, + interval: setInterval(async () => { + if (context.processing) { + return; + } + + context.processing = true; + try { + const status = await this.evaluateAssignmentStatus(challengeId, phase); + + if (status.phaseMissing) { + this.logger.warn( + `Review phase ${phase.id} no longer exists on challenge ${challengeId}. Stopping assignment polling.`, + ); + this.clearPolling(challengeId, phase.id); + return; + } + + if (status.phaseOpen) { + this.logger.log( + `Review phase ${phase.id} for challenge ${challengeId} is already open. Stopping assignment polling.`, + ); + this.clearPolling(challengeId, phase.id); + return; + } + + if (!status.ready) { + return; + } + + this.logger.log( + `Required reviewers detected for challenge ${challengeId}, phase ${phase.id}. Opening review phase automatically.`, + ); + await openPhaseCallback(); + } catch (error) { + const err = error as Error; + this.logger.error( + `Error while polling reviewer assignments for challenge ${challengeId}, phase ${phase.id}: ${err.message}`, + err.stack, + ); + } finally { + context.processing = false; + } + }, this.pollIntervalMs), + }; + + this.schedulerRegistry.addInterval(key, context.interval); + this.pollers.set(key, context); + } + + private async evaluateAssignmentStatus( + challengeId: string, + phase: PhaseSummary, + ): Promise { + let challenge: IChallenge; + + try { + challenge = await this.challengeApiService.getChallengeById(challengeId); + } catch (error) { + const err = error as Error; + this.logger.error( + `Failed to fetch challenge ${challengeId} while verifying reviewer assignments: ${err.message}`, + err.stack, + ); + return { + ready: false, + required: 0, + assigned: 0, + phaseMissing: true, + phaseOpen: false, + }; + } + + const phaseDetails = challenge.phases.find((p) => p.id === phase.id); + + if (!phaseDetails) { + return { + ready: false, + required: 0, + assigned: 0, + phaseMissing: true, + phaseOpen: false, + }; + } + + if (phaseDetails.isOpen) { + return { + ready: true, + required: 0, + assigned: 0, + phaseMissing: false, + phaseOpen: true, + }; + } + + const reviewerConfigs = this.getReviewerConfigsForPhase( + challenge.reviewers, + phaseDetails.phaseId, + ); + + if (reviewerConfigs.length === 0) { + return { + ready: true, + required: 0, + assigned: 0, + phaseMissing: false, + phaseOpen: false, + }; + } + + const required = reviewerConfigs.reduce((total, config) => { + const count = config.memberReviewerCount ?? 1; + return total + Math.max(count, 0); + }, 0); + + if (required === 0) { + return { + ready: true, + required: 0, + assigned: 0, + phaseMissing: false, + phaseOpen: false, + }; + } + + const roleNames = getRoleNamesForPhase(phaseDetails.name); + const assignedReviewers = await this.resourcesService.getReviewerResources( + challengeId, + roleNames, + ); + + const assigned = assignedReviewers.length; + const ready = assigned >= required; + + return { + ready, + required, + assigned, + phaseMissing: false, + phaseOpen: false, + }; + } + + private getReviewerConfigsForPhase( + reviewers: IChallengeReviewer[], + phaseTemplateId: string, + ): IChallengeReviewer[] { + return reviewers.filter( + (reviewer) => + reviewer.isMemberReview && reviewer.phaseId === phaseTemplateId, + ); + } + + private buildKey(challengeId: string, phaseId: string): string { + return `${challengeId}:${phaseId}:reviewer-assignment`; + } +} diff --git a/src/config/sections/review.config.ts b/src/config/sections/review.config.ts index c595ef0..1e6bc98 100644 --- a/src/config/sections/review.config.ts +++ b/src/config/sections/review.config.ts @@ -1,5 +1,16 @@ import { registerAs } from '@nestjs/config'; -export default registerAs('review', () => ({ - dbUrl: process.env.REVIEW_DB_URL, -})); +const DEFAULT_POLL_INTERVAL_MS = 5 * 60 * 1000; + +export default registerAs('review', () => { + const pollIntervalEnv = process.env.REVIEWER_POLL_INTERVAL_MS; + const pollInterval = Number(pollIntervalEnv); + + return { + dbUrl: process.env.REVIEW_DB_URL, + assignmentPollIntervalMs: + Number.isFinite(pollInterval) && pollInterval > 0 + ? pollInterval + : DEFAULT_POLL_INTERVAL_MS, + }; +}); diff --git a/src/config/validation.ts b/src/config/validation.ts index e743274..575e823 100644 --- a/src/config/validation.ts +++ b/src/config/validation.ts @@ -42,6 +42,10 @@ export const validationSchema = Joi.object({ then: Joi.optional().default('postgresql://localhost:5432/resources'), otherwise: Joi.required(), }), + REVIEWER_POLL_INTERVAL_MS: Joi.number() + .integer() + .positive() + .default(5 * 60 * 1000), // Auth0 Configuration (optional in test environment) AUTH0_URL: Joi.string() From 2014f79656580a1e0f88315297c379033269d85f Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Tue, 23 Sep 2025 15:22:24 +1000 Subject: [PATCH 02/23] Deploy branch --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cd43e41..085f93a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,6 +64,7 @@ workflows: branches: only: - develop + - reviews # Production builds are exectuted only on tagged commits to the # master branch. From 7d6295410d900c8c09f2271bfdba2e9f0269b829 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Tue, 23 Sep 2025 15:39:27 +1000 Subject: [PATCH 03/23] Ignore challenges not active --- src/autopilot/services/autopilot.service.ts | 46 +++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/autopilot/services/autopilot.service.ts b/src/autopilot/services/autopilot.service.ts index 2d06c29..344871c 100644 --- a/src/autopilot/services/autopilot.service.ts +++ b/src/autopilot/services/autopilot.service.ts @@ -43,6 +43,10 @@ export class AutopilotService { ); } + private isChallengeActive(status?: string): boolean { + return (status ?? '').toUpperCase() === 'ACTIVE'; + } + schedulePhaseTransition(phaseData: PhaseTransitionPayload): string { try { const phaseKey = `${phaseData.challengeId}:${phaseData.phaseId}`; @@ -141,6 +145,13 @@ export class AutopilotService { `Consumed phase transition event: ${JSON.stringify(message)}`, ); + if (!this.isChallengeActive(message.projectStatus)) { + this.logger.log( + `Ignoring phase transition for challenge ${message.challengeId} with status ${message.projectStatus}; only ACTIVE challenges are processed.`, + ); + return; + } + if (message.state === 'START') { // Advance the phase (open it) using the scheduler service void (async () => { @@ -197,6 +208,13 @@ export class AutopilotService { challenge.id, ); + if (!this.isChallengeActive(challengeDetails.status)) { + this.logger.log( + `Skipping challenge ${challenge.id} with status ${challengeDetails.status}; only ACTIVE challenges are processed.`, + ); + return; + } + if (!challengeDetails.phases) { this.logger.warn( `Challenge ${challenge.id} has no phases to schedule.`, @@ -266,6 +284,13 @@ export class AutopilotService { message.id, ); + if (!this.isChallengeActive(challengeDetails.status)) { + this.logger.log( + `Skipping challenge ${message.id} update with status ${challengeDetails.status}; only ACTIVE challenges are processed.`, + ); + return; + } + if (!challengeDetails.phases) { this.logger.warn( `Updated challenge ${message.id} has no phases to process.`, @@ -397,6 +422,13 @@ export class AutopilotService { return; } + if (!this.isChallengeActive(challengeDetails.status)) { + this.logger.log( + `${AUTOPILOT_COMMANDS.RESCHEDULE_PHASE}: ignoring challenge ${challengeId} with status ${challengeDetails.status}; only ACTIVE challenges are processed.`, + ); + return; + } + const phaseTypeName = await this.challengeApiService.getPhaseTypeName( challengeDetails.id, @@ -533,6 +565,13 @@ export class AutopilotService { projectStatus: string, nextPhases: IPhase[], ): Promise { + if (!this.isChallengeActive(projectStatus)) { + this.logger.log( + `[PHASE CHAIN] Challenge ${challengeId} is not ACTIVE (status: ${projectStatus}), skipping phase chain processing.`, + ); + return; + } + if (!nextPhases || nextPhases.length === 0) { this.logger.log( `[PHASE CHAIN] No next phases to open for challenge ${challengeId}`, @@ -597,6 +636,13 @@ export class AutopilotService { projectStatus: string, phase: IPhase, ): Promise { + if (!this.isChallengeActive(projectStatus)) { + this.logger.log( + `[PHASE CHAIN] Challenge ${challengeId} is not ACTIVE (status: ${projectStatus}); skipping phase ${phase.name} (${phase.id}).`, + ); + return false; + } + this.logger.log( `[PHASE CHAIN] Opening phase ${phase.name} (${phase.id}) for challenge ${challengeId}`, ); From 012c67d7b84a093f10346c08d3eed7a0947c2a39 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Tue, 23 Sep 2025 16:04:49 +1000 Subject: [PATCH 04/23] Better handling of phases with the same start and end dates --- src/autopilot/services/autopilot.service.ts | 242 ++++++++++-------- .../services/phase-review.service.ts | 21 +- .../services/review-assignment.service.ts | 12 +- src/challenge/challenge-api.service.ts | 10 +- src/resources/resources.service.ts | 3 +- 5 files changed, 170 insertions(+), 118 deletions(-) diff --git a/src/autopilot/services/autopilot.service.ts b/src/autopilot/services/autopilot.service.ts index 344871c..b5c52bb 100644 --- a/src/autopilot/services/autopilot.service.ts +++ b/src/autopilot/services/autopilot.service.ts @@ -222,49 +222,66 @@ export class AutopilotService { return; } - // Find the next phase that should be scheduled (similar to PhaseAdvancer logic) - const nextPhase = this.findNextPhaseToSchedule(challengeDetails.phases); + // Find the phases that should be scheduled (similar logic to PhaseAdvancer) + const phasesToSchedule = this.findPhasesToSchedule( + challengeDetails.phases, + ); - if (!nextPhase) { + if (phasesToSchedule.length === 0) { this.logger.log( `No phase needs to be scheduled for new challenge ${challenge.id}`, ); return; } - // Determine if we should schedule for start or end based on phase state const now = new Date(); - const shouldOpen = - !nextPhase.isOpen && - !nextPhase.actualEndDate && - new Date(nextPhase.scheduledStartDate) <= now; + const scheduledSummaries: string[] = []; + + for (const nextPhase of phasesToSchedule) { + const shouldOpen = + !nextPhase.isOpen && + !nextPhase.actualEndDate && + nextPhase.scheduledStartDate && + new Date(nextPhase.scheduledStartDate) <= now; + + const scheduleDate = shouldOpen + ? nextPhase.scheduledStartDate + : nextPhase.scheduledEndDate; + const state = shouldOpen ? 'START' : 'END'; + + if (!scheduleDate) { + this.logger.warn( + `Next phase ${nextPhase.id} for new challenge ${challenge.id} has no scheduled ${shouldOpen ? 'start' : 'end'} date. Skipping.`, + ); + continue; + } - const scheduleDate = shouldOpen - ? nextPhase.scheduledStartDate - : nextPhase.scheduledEndDate; - const state = shouldOpen ? 'START' : 'END'; + const phaseData: PhaseTransitionPayload = { + projectId: challengeDetails.projectId, + challengeId: challengeDetails.id, + phaseId: nextPhase.id, + phaseTypeName: nextPhase.name, + state, + operator: AutopilotOperator.SYSTEM_NEW_CHALLENGE, + projectStatus: challengeDetails.status, + date: scheduleDate, + }; + + this.schedulePhaseTransition(phaseData); + scheduledSummaries.push( + `${nextPhase.name} (${nextPhase.id}) -> ${state} @ ${scheduleDate}`, + ); + } - if (!scheduleDate) { + if (scheduledSummaries.length === 0) { this.logger.warn( - `Next phase ${nextPhase.id} for new challenge ${challenge.id} has no scheduled ${shouldOpen ? 'start' : 'end'} date. Skipping.`, + `Unable to schedule any phases for new challenge ${challenge.id} due to missing schedule data.`, ); return; } - const phaseData: PhaseTransitionPayload = { - projectId: challengeDetails.projectId, - challengeId: challengeDetails.id, - phaseId: nextPhase.id, - phaseTypeName: nextPhase.name, - state, - operator: AutopilotOperator.SYSTEM_NEW_CHALLENGE, - projectStatus: challengeDetails.status, - date: scheduleDate, - }; - - this.schedulePhaseTransition(phaseData); this.logger.log( - `Scheduled next phase ${nextPhase.name} (${nextPhase.id}) for new challenge ${challenge.id}`, + `Scheduled ${scheduledSummaries.length} phase(s) for new challenge ${challenge.id}: ${scheduledSummaries.join('; ')}`, ); } catch (error) { const err = error as Error; @@ -298,50 +315,66 @@ export class AutopilotService { return; } - // Find the next phase that should be scheduled (similar to PhaseAdvancer logic) - const nextPhase = this.findNextPhaseToSchedule(challengeDetails.phases); + const phasesToSchedule = this.findPhasesToSchedule( + challengeDetails.phases, + ); - if (!nextPhase) { + if (phasesToSchedule.length === 0) { this.logger.log( `No phase needs to be rescheduled for updated challenge ${message.id}`, ); return; } - // Determine if we should schedule for start or end based on phase state const now = new Date(); - const shouldOpen = - !nextPhase.isOpen && - !nextPhase.actualEndDate && - new Date(nextPhase.scheduledStartDate) <= now; + const rescheduledSummaries: string[] = []; + + for (const nextPhase of phasesToSchedule) { + const shouldOpen = + !nextPhase.isOpen && + !nextPhase.actualEndDate && + nextPhase.scheduledStartDate && + new Date(nextPhase.scheduledStartDate) <= now; + + const scheduleDate = shouldOpen + ? nextPhase.scheduledStartDate + : nextPhase.scheduledEndDate; + const state = shouldOpen ? 'START' : 'END'; + + if (!scheduleDate) { + this.logger.warn( + `Next phase ${nextPhase.id} for updated challenge ${message.id} has no scheduled ${shouldOpen ? 'start' : 'end'} date. Skipping.`, + ); + continue; + } - const scheduleDate = shouldOpen - ? nextPhase.scheduledStartDate - : nextPhase.scheduledEndDate; - const state = shouldOpen ? 'START' : 'END'; + const payload: PhaseTransitionPayload = { + projectId: challengeDetails.projectId, + challengeId: challengeDetails.id, + phaseId: nextPhase.id, + phaseTypeName: nextPhase.name, + operator: message.operator, + projectStatus: challengeDetails.status, + date: scheduleDate, + state, + }; + + this.reschedulePhaseTransition(challengeDetails.id, payload); + rescheduledSummaries.push( + `${nextPhase.name} (${nextPhase.id}) -> ${state} @ ${scheduleDate}`, + ); + } - if (!scheduleDate) { + if (rescheduledSummaries.length === 0) { this.logger.warn( - `Next phase ${nextPhase.id} for updated challenge ${message.id} has no scheduled ${shouldOpen ? 'start' : 'end'} date. Skipping.`, + `Unable to reschedule any phases for updated challenge ${message.id} due to missing schedule data.`, ); return; } - const payload: PhaseTransitionPayload = { - projectId: challengeDetails.projectId, - challengeId: challengeDetails.id, - phaseId: nextPhase.id, - phaseTypeName: nextPhase.name, - operator: message.operator, - projectStatus: challengeDetails.status, - date: scheduleDate, - state, - }; - this.logger.log( - `Rescheduling next phase ${nextPhase.name} (${nextPhase.id}) for updated challenge ${message.id}`, + `Rescheduled ${rescheduledSummaries.length} phase(s) for updated challenge ${message.id}: ${rescheduledSummaries.join('; ')}`, ); - this.reschedulePhaseTransition(challengeDetails.id, payload); } catch (error) { const err = error as Error; this.logger.error( @@ -468,54 +501,56 @@ export class AutopilotService { } /** - * Find the next phase that should be scheduled based on current phase state. - * Similar logic to PhaseAdvancer.js - only schedule the next phase that should advance. + * Find the phases that should be scheduled based on current phase state. + * Similar logic to PhaseAdvancer.js - ensure every phase that needs attention is handled. */ - private findNextPhaseToSchedule(phases: IPhase[]): IPhase | null { + private findPhasesToSchedule(phases: IPhase[]): IPhase[] { const now = new Date(); // First, check for phases that should be open but aren't - const phasesToOpen = phases.filter((phase) => { - if (phase.isOpen || phase.actualEndDate) { - return false; // Already open or already ended - } - - const startTime = new Date(phase.scheduledStartDate); - if (startTime > now) { - return false; // Not time to start yet - } + const phasesToOpen = phases + .filter((phase) => { + if (phase.isOpen || phase.actualEndDate) { + return false; // Already open or already ended + } - // Check if predecessor requirements are met - if (!phase.predecessor) { - return true; // No predecessor, ready to start - } + const startTime = new Date(phase.scheduledStartDate); + if (startTime > now) { + return false; // Not time to start yet + } - const predecessor = phases.find( - (p) => p.phaseId === phase.predecessor || p.id === phase.predecessor, - ); + // Check if predecessor requirements are met + if (!phase.predecessor) { + return true; // No predecessor, ready to start + } - return Boolean(predecessor?.actualEndDate); // Predecessor has ended - }); + const predecessor = phases.find( + (p) => p.phaseId === phase.predecessor || p.id === phase.predecessor, + ); - if (phasesToOpen.length > 0) { - // Return the earliest phase that should be opened - return phasesToOpen.sort( + return Boolean(predecessor?.actualEndDate); // Predecessor has ended + }) + .sort( (a, b) => new Date(a.scheduledStartDate).getTime() - new Date(b.scheduledStartDate).getTime(), - )[0]; + ); + + if (phasesToOpen.length > 0) { + return phasesToOpen; } // Next, check for open phases that should be closed - const openPhases = phases.filter((phase) => phase.isOpen); - - if (openPhases.length > 0) { - // Return the earliest phase that should end - return openPhases.sort( + const openPhases = phases + .filter((phase) => phase.isOpen) + .sort( (a, b) => new Date(a.scheduledEndDate).getTime() - new Date(b.scheduledEndDate).getTime(), - )[0]; + ); + + if (openPhases.length > 0) { + return openPhases; } // Finally, look for future phases that need to be scheduled @@ -527,10 +562,6 @@ export class AutopilotService { new Date(phase.scheduledStartDate) > now, // starts in the future ); - if (futurePhases.length === 0) { - return null; - } - // Find phases that are ready to start (no predecessor or predecessor is closed) const readyPhases = futurePhases.filter((phase) => { if (!phase.predecessor) { @@ -545,15 +576,21 @@ export class AutopilotService { }); if (readyPhases.length === 0) { - return null; + return []; } - // Return the earliest scheduled phase - return readyPhases.sort( + // Return the phases with the earliest scheduled start (handle identical start times) + const sortedReady = readyPhases.sort( (a, b) => new Date(a.scheduledStartDate).getTime() - new Date(b.scheduledStartDate).getTime(), - )[0]; + ); + + const earliest = new Date(sortedReady[0].scheduledStartDate).getTime(); + + return sortedReady.filter( + (phase) => new Date(phase.scheduledStartDate).getTime() === earliest, + ); } /** @@ -596,11 +633,12 @@ export class AutopilotService { ); try { - const canOpenNow = await this.reviewAssignmentService.ensureAssignmentsOrSchedule( - challengeId, - nextPhase, - openPhaseCallback, - ); + const canOpenNow = + await this.reviewAssignmentService.ensureAssignmentsOrSchedule( + challengeId, + nextPhase, + openPhaseCallback, + ); if (!canOpenNow) { deferredCount++; @@ -620,9 +658,13 @@ export class AutopilotService { } } - const summaryParts = [`opened and scheduled ${processedCount} out of ${nextPhases.length}`]; + const summaryParts = [ + `opened and scheduled ${processedCount} out of ${nextPhases.length}`, + ]; if (deferredCount > 0) { - summaryParts.push(`deferred ${deferredCount} awaiting reviewer assignments`); + summaryParts.push( + `deferred ${deferredCount} awaiting reviewer assignments`, + ); } this.logger.log( diff --git a/src/autopilot/services/phase-review.service.ts b/src/autopilot/services/phase-review.service.ts index 05f484d..0f35154 100644 --- a/src/autopilot/services/phase-review.service.ts +++ b/src/autopilot/services/phase-review.service.ts @@ -18,11 +18,9 @@ export class PhaseReviewService { private readonly resourcesService: ResourcesService, ) {} - async handlePhaseOpened( - challengeId: string, - phaseId: string, - ): Promise { - const challenge = await this.challengeApiService.getChallengeById(challengeId); + async handlePhaseOpened(challengeId: string, phaseId: string): Promise { + const challenge = + await this.challengeApiService.getChallengeById(challengeId); const phase = challenge.phases.find((p) => p.id === phaseId); if (!phase) { @@ -48,7 +46,11 @@ export class PhaseReviewService { return; } - const scorecardId = this.pickScorecardId(reviewerConfigs, challengeId, phase.id); + const scorecardId = this.pickScorecardId( + reviewerConfigs, + challengeId, + phase.id, + ); if (!scorecardId) { return; } @@ -66,7 +68,8 @@ export class PhaseReviewService { return; } - const submissionIds = await this.reviewService.getActiveSubmissionIds(challengeId); + const submissionIds = + await this.reviewService.getActiveSubmissionIds(challengeId); if (!submissionIds.length) { this.logger.log( `No submissions found for challenge ${challengeId}; skipping review creation for phase ${phase.name}`, @@ -74,7 +77,9 @@ export class PhaseReviewService { return; } - const existingPairs = await this.reviewService.getExistingReviewPairs(phase.id); + const existingPairs = await this.reviewService.getExistingReviewPairs( + phase.id, + ); let createdCount = 0; for (const resource of reviewerResources) { diff --git a/src/autopilot/services/review-assignment.service.ts b/src/autopilot/services/review-assignment.service.ts index 8628edb..3c16bb6 100644 --- a/src/autopilot/services/review-assignment.service.ts +++ b/src/autopilot/services/review-assignment.service.ts @@ -49,9 +49,10 @@ export class ReviewAssignmentService { ); const defaultInterval = 5 * 60 * 1000; // 5 minutes - this.pollIntervalMs = configuredInterval && configuredInterval > 0 - ? configuredInterval - : defaultInterval; + this.pollIntervalMs = + configuredInterval && configuredInterval > 0 + ? configuredInterval + : defaultInterval; } async ensureAssignmentsOrSchedule( @@ -129,7 +130,10 @@ export class ReviewAssignmentService { context.processing = true; try { - const status = await this.evaluateAssignmentStatus(challengeId, phase); + const status = await this.evaluateAssignmentStatus( + challengeId, + phase, + ); if (status.phaseMissing) { this.logger.warn( diff --git a/src/challenge/challenge-api.service.ts b/src/challenge/challenge-api.service.ts index 7eafb55..49327e1 100644 --- a/src/challenge/challenge-api.service.ts +++ b/src/challenge/challenge-api.service.ts @@ -143,8 +143,8 @@ export class ChallengeApiService { this.logger.debug( `Attempting to ${operation} phase ${phaseId} for challenge ${challengeId}`, - ); - + ); + if (!challenge) { this.logger.warn( `Challenge ${challengeId} not found when advancing phase.`, @@ -312,9 +312,9 @@ export class ChallengeApiService { updatedBy: challenge.updatedBy, metadata: [], phases: challenge.phases.map((phase) => this.mapPhase(phase)), - reviewers: challenge.reviewers?.map((reviewer) => - this.mapReviewer(reviewer), - ) || [], + reviewers: + challenge.reviewers?.map((reviewer) => this.mapReviewer(reviewer)) || + [], discussions: [], events: [], prizeSets: [], diff --git a/src/resources/resources.service.ts b/src/resources/resources.service.ts index 298f793..44d37fa 100644 --- a/src/resources/resources.service.ts +++ b/src/resources/resources.service.ts @@ -34,7 +34,8 @@ export class ResourcesService { AND rr."name" IN (${roleList}) `; - const reviewers = await this.prisma.$queryRaw(query); + const reviewers = + await this.prisma.$queryRaw(query); return reviewers; } } From 1309ac1d7ee3da7d35559efeda3e9299ed794031 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 24 Sep 2025 08:03:33 +1000 Subject: [PATCH 05/23] Add winners at the end of the challenge --- src/autopilot/autopilot.module.ts | 2 + .../services/challenge-completion.service.ts | 85 +++++++++++++++++++ .../services/review-assignment.service.ts | 83 +++++++++--------- src/autopilot/services/scheduler.service.ts | 28 ++++++ src/challenge/challenge-api.service.ts | 48 ++++++++++- .../interfaces/challenge.interface.ts | 8 ++ src/resources/resources.service.ts | 37 ++++++++ src/review/review.service.ts | 82 ++++++++++++++++++ 8 files changed, 331 insertions(+), 42 deletions(-) create mode 100644 src/autopilot/services/challenge-completion.service.ts diff --git a/src/autopilot/autopilot.module.ts b/src/autopilot/autopilot.module.ts index b214fc8..d4a47e4 100644 --- a/src/autopilot/autopilot.module.ts +++ b/src/autopilot/autopilot.module.ts @@ -8,6 +8,7 @@ import { ReviewModule } from '../review/review.module'; import { ResourcesModule } from '../resources/resources.module'; import { PhaseReviewService } from './services/phase-review.service'; import { ReviewAssignmentService } from './services/review-assignment.service'; +import { ChallengeCompletionService } from './services/challenge-completion.service'; @Module({ imports: [ @@ -24,6 +25,7 @@ import { ReviewAssignmentService } from './services/review-assignment.service'; SchedulerService, PhaseReviewService, ReviewAssignmentService, + ChallengeCompletionService, ], exports: [AutopilotService, SchedulerService], }) diff --git a/src/autopilot/services/challenge-completion.service.ts b/src/autopilot/services/challenge-completion.service.ts new file mode 100644 index 0000000..e4eb175 --- /dev/null +++ b/src/autopilot/services/challenge-completion.service.ts @@ -0,0 +1,85 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ChallengeApiService } from '../../challenge/challenge-api.service'; +import { ReviewService } from '../../review/review.service'; +import { ResourcesService } from '../../resources/resources.service'; +import { IChallengeWinner } from '../../challenge/interfaces/challenge.interface'; + +@Injectable() +export class ChallengeCompletionService { + private readonly logger = new Logger(ChallengeCompletionService.name); + + constructor( + private readonly challengeApiService: ChallengeApiService, + private readonly reviewService: ReviewService, + private readonly resourcesService: ResourcesService, + ) {} + + async finalizeChallenge(challengeId: string): Promise { + const challenge = + await this.challengeApiService.getChallengeById(challengeId); + + if ( + challenge.status === 'COMPLETED' && + challenge.winners && + challenge.winners.length + ) { + this.logger.log( + `Challenge ${challengeId} is already completed with winners; skipping finalization.`, + ); + return; + } + + const scoreRows = await this.reviewService.getTopFinalReviewScores( + challengeId, + 3, + ); + + if (!scoreRows.length) { + this.logger.warn( + `No final review scores found for challenge ${challengeId}; marking completed without winners.`, + ); + await this.challengeApiService.completeChallenge(challengeId, []); + return; + } + + const memberIds = scoreRows.map((row) => row.memberId); + const handleMap = await this.resourcesService.getMemberHandleMap( + challengeId, + memberIds, + ); + + const winners: IChallengeWinner[] = []; + for (const [index, row] of scoreRows.entries()) { + const numericMemberId = Number(row.memberId); + if (!Number.isFinite(numericMemberId)) { + this.logger.warn( + `Skipping winner placement ${index + 1} for challenge ${challengeId} because memberId ${row.memberId} is not numeric.`, + ); + continue; + } + + winners.push({ + userId: numericMemberId, + handle: handleMap.get(row.memberId) ?? row.memberId, + placement: winners.length + 1, + }); + + if (winners.length >= 3) { + break; + } + } + + if (!winners.length) { + this.logger.warn( + `Unable to derive any numeric winners for challenge ${challengeId}; marking completed without winners.`, + ); + await this.challengeApiService.completeChallenge(challengeId, []); + return; + } + + await this.challengeApiService.completeChallenge(challengeId, winners); + this.logger.log( + `Marked challenge ${challengeId} as COMPLETED with ${winners.length} winner(s).`, + ); + } +} diff --git a/src/autopilot/services/review-assignment.service.ts b/src/autopilot/services/review-assignment.service.ts index 3c16bb6..20e025c 100644 --- a/src/autopilot/services/review-assignment.service.ts +++ b/src/autopilot/services/review-assignment.service.ts @@ -123,54 +123,57 @@ export class ReviewAssignmentService { const context: PollerContext = { processing: false, - interval: setInterval(async () => { - if (context.processing) { - return; - } + interval: null as unknown as NodeJS.Timeout, + }; - context.processing = true; - try { - const status = await this.evaluateAssignmentStatus( - challengeId, - phase, - ); + const executePoll = async () => { + if (context.processing) { + return; + } - if (status.phaseMissing) { - this.logger.warn( - `Review phase ${phase.id} no longer exists on challenge ${challengeId}. Stopping assignment polling.`, - ); - this.clearPolling(challengeId, phase.id); - return; - } - - if (status.phaseOpen) { - this.logger.log( - `Review phase ${phase.id} for challenge ${challengeId} is already open. Stopping assignment polling.`, - ); - this.clearPolling(challengeId, phase.id); - return; - } - - if (!status.ready) { - return; - } + context.processing = true; + try { + const status = await this.evaluateAssignmentStatus(challengeId, phase); - this.logger.log( - `Required reviewers detected for challenge ${challengeId}, phase ${phase.id}. Opening review phase automatically.`, + if (status.phaseMissing) { + this.logger.warn( + `Review phase ${phase.id} no longer exists on challenge ${challengeId}. Stopping assignment polling.`, ); - await openPhaseCallback(); - } catch (error) { - const err = error as Error; - this.logger.error( - `Error while polling reviewer assignments for challenge ${challengeId}, phase ${phase.id}: ${err.message}`, - err.stack, + this.clearPolling(challengeId, phase.id); + return; + } + + if (status.phaseOpen) { + this.logger.log( + `Review phase ${phase.id} for challenge ${challengeId} is already open. Stopping assignment polling.`, ); - } finally { - context.processing = false; + this.clearPolling(challengeId, phase.id); + return; } - }, this.pollIntervalMs), + + if (!status.ready) { + return; + } + + this.logger.log( + `Required reviewers detected for challenge ${challengeId}, phase ${phase.id}. Opening review phase automatically.`, + ); + await openPhaseCallback(); + } catch (error) { + const err = error as Error; + this.logger.error( + `Error while polling reviewer assignments for challenge ${challengeId}, phase ${phase.id}: ${err.message}`, + err.stack, + ); + } finally { + context.processing = false; + } }; + context.interval = setInterval(() => { + void executePoll(); + }, this.pollIntervalMs); + this.schedulerRegistry.addInterval(key, context.interval); this.pollers.set(key, context); } diff --git a/src/autopilot/services/scheduler.service.ts b/src/autopilot/services/scheduler.service.ts index 9691447..60d2359 100644 --- a/src/autopilot/services/scheduler.service.ts +++ b/src/autopilot/services/scheduler.service.ts @@ -3,6 +3,7 @@ import { SchedulerRegistry } from '@nestjs/schedule'; import { KafkaService } from '../../kafka/kafka.service'; import { ChallengeApiService } from '../../challenge/challenge-api.service'; import { PhaseReviewService } from './phase-review.service'; +import { ChallengeCompletionService } from './challenge-completion.service'; import { PhaseTransitionMessage, PhaseTransitionPayload, @@ -28,6 +29,7 @@ export class SchedulerService { private readonly kafkaService: KafkaService, private readonly challengeApiService: ChallengeApiService, private readonly phaseReviewService: PhaseReviewService, + private readonly challengeCompletionService: ChallengeCompletionService, ) {} setPhaseChainCallback( @@ -294,6 +296,32 @@ export class SchedulerService { } } + if (operation === 'close') { + try { + let phases = result.updatedPhases; + if (!phases || !phases.length) { + phases = await this.challengeApiService.getChallengePhases( + data.challengeId, + ); + } + + const hasOpenPhases = phases?.some((phase) => phase.isOpen) ?? true; + const hasNextPhases = Boolean(result.next?.phases?.length); + + if (!hasOpenPhases && !hasNextPhases) { + await this.challengeCompletionService.finalizeChallenge( + data.challengeId, + ); + } + } catch (error) { + const err = error as Error; + this.logger.error( + `Failed to finalize challenge ${data.challengeId} after closing phase ${data.phaseId}: ${err.message}`, + err.stack, + ); + } + } + // Handle phase transition chain - open and schedule next phases if they exist if ( result.next?.operation === 'open' && diff --git a/src/challenge/challenge-api.service.ts b/src/challenge/challenge-api.service.ts index 49327e1..2299f3b 100644 --- a/src/challenge/challenge-api.service.ts +++ b/src/challenge/challenge-api.service.ts @@ -1,7 +1,11 @@ import { Injectable, Logger, NotFoundException } from '@nestjs/common'; -import { Prisma, ChallengeStatusEnum } from '@prisma/client'; +import { Prisma, ChallengeStatusEnum, PrizeSetTypeEnum } from '@prisma/client'; import { ChallengePrismaService } from './challenge-prisma.service'; -import { IPhase, IChallenge } from './interfaces/challenge.interface'; +import { + IPhase, + IChallenge, + IChallengeWinner, +} from './interfaces/challenge.interface'; // DTO for filtering challenges interface ChallengeFiltersDto { @@ -315,6 +319,7 @@ export class ChallengeApiService { reviewers: challenge.reviewers?.map((reviewer) => this.mapReviewer(reviewer)) || [], + winners: challenge.winners?.map((winner) => this.mapWinner(winner)) || [], discussions: [], events: [], prizeSets: [], @@ -394,6 +399,45 @@ export class ChallengeApiService { }; } + private mapWinner( + winner: ChallengeWithRelations['winners'][number], + ): IChallengeWinner { + return { + userId: winner.userId, + handle: winner.handle, + placement: winner.placement, + type: winner.type, + }; + } + + async completeChallenge( + challengeId: string, + winners: IChallengeWinner[], + ): Promise { + await this.prisma.$transaction(async (tx) => { + await tx.challenge.update({ + where: { id: challengeId }, + data: { status: ChallengeStatusEnum.COMPLETED }, + }); + + await tx.challengeWinner.deleteMany({ where: { challengeId } }); + + if (winners.length) { + await tx.challengeWinner.createMany({ + data: winners.map((winner) => ({ + challengeId, + userId: winner.userId, + handle: winner.handle, + placement: winner.placement, + type: PrizeSetTypeEnum.PLACEMENT, + createdBy: 'Autopilot', + updatedBy: 'Autopilot', + })), + }); + } + }); + } + private ensureTimestamp(date?: Date | null): string { return date ? date.toISOString() : new Date(0).toISOString(); } diff --git a/src/challenge/interfaces/challenge.interface.ts b/src/challenge/interfaces/challenge.interface.ts index 601322e..fa0b0e3 100644 --- a/src/challenge/interfaces/challenge.interface.ts +++ b/src/challenge/interfaces/challenge.interface.ts @@ -33,6 +33,13 @@ export interface IChallengeReviewer { aiWorkflowId: string | null; } +export interface IChallengeWinner { + userId: number; + handle: string; + placement: number; + type?: string; +} + /** * Represents a full challenge object from the Challenge API. */ @@ -61,6 +68,7 @@ export interface IChallenge { metadata: any[]; phases: IPhase[]; reviewers: IChallengeReviewer[]; + winners: IChallengeWinner[]; discussions: any[]; events: any[]; prizeSets: any[]; diff --git a/src/resources/resources.service.ts b/src/resources/resources.service.ts index 44d37fa..b7eaa59 100644 --- a/src/resources/resources.service.ts +++ b/src/resources/resources.service.ts @@ -16,6 +16,43 @@ export class ResourcesService { constructor(private readonly prisma: ResourcesPrismaService) {} + async getMemberHandleMap( + challengeId: string, + memberIds: string[], + ): Promise> { + if (!challengeId || !memberIds.length) { + return new Map(); + } + + const uniqueIds = Array.from( + new Set(memberIds.map((id) => id?.trim()).filter(Boolean)), + ); + + if (!uniqueIds.length) { + return new Map(); + } + + const idList = Prisma.join(uniqueIds.map((id) => Prisma.sql`${id}`)); + + const rows = await this.prisma.$queryRaw< + Array<{ memberId: string; memberHandle: string | null }> + >(Prisma.sql` + SELECT r."memberId", r."memberHandle" + FROM ${ResourcesService.RESOURCE_TABLE} r + WHERE r."challengeId" = ${challengeId} + AND r."memberId" IN (${idList}) + `); + + return new Map( + rows + .filter((row) => Boolean(row.memberId)) + .map((row) => [ + String(row.memberId), + row.memberHandle?.trim() || String(row.memberId), + ]), + ); + } + async getReviewerResources( challengeId: string, roleNames: string[], diff --git a/src/review/review.service.ts b/src/review/review.service.ts index d4d1a26..a227ae4 100644 --- a/src/review/review.service.ts +++ b/src/review/review.service.ts @@ -15,9 +15,91 @@ interface ReviewRecord { export class ReviewService { private static readonly REVIEW_TABLE = Prisma.sql`"review"`; private static readonly SUBMISSION_TABLE = Prisma.sql`"submission"`; + private static readonly REVIEW_SUMMATION_TABLE = Prisma.sql`"reviewSummation"`; constructor(private readonly prisma: ReviewPrismaService) {} + async getTopFinalReviewScores( + challengeId: string, + limit = 3, + ): Promise< + Array<{ + memberId: string; + submissionId: string; + aggregateScore: number; + }> + > { + if (!challengeId) { + return []; + } + + const fetchLimit = Math.max(limit, 1) * 5; + const limitClause = Prisma.sql`LIMIT ${fetchLimit}`; + + const rows = await this.prisma.$queryRaw< + Array<{ + memberId: string | null; + submissionId: string; + aggregateScore: number | string | null; + }> + >(Prisma.sql` + SELECT + s."memberId" AS "memberId", + s."id" AS "submissionId", + rs."aggregateScore" AS "aggregateScore" + FROM ${ReviewService.REVIEW_SUMMATION_TABLE} rs + INNER JOIN ${ReviewService.SUBMISSION_TABLE} s + ON s."id" = rs."submissionId" + WHERE s."challengeId" = ${challengeId} + AND rs."isFinal" = true + AND rs."aggregateScore" IS NOT NULL + AND s."memberId" IS NOT NULL + AND s."type" = 'CONTEST_SUBMISSION' + ORDER BY rs."aggregateScore" DESC, s."submittedDate" ASC, s."id" ASC + ${limitClause} + `); + + if (!rows.length) { + return []; + } + + const seenMembers = new Set(); + const winners: Array<{ + memberId: string; + submissionId: string; + aggregateScore: number; + }> = []; + + for (const row of rows) { + const memberId = row.memberId?.trim(); + if (!memberId || seenMembers.has(memberId)) { + continue; + } + + const aggregateScore = + typeof row.aggregateScore === 'string' + ? Number(row.aggregateScore) + : (row.aggregateScore ?? 0); + + if (Number.isNaN(aggregateScore)) { + continue; + } + + winners.push({ + memberId, + submissionId: row.submissionId, + aggregateScore, + }); + seenMembers.add(memberId); + + if (winners.length >= limit) { + break; + } + } + + return winners; + } + async getActiveSubmissionIds(challengeId: string): Promise { const query = Prisma.sql` SELECT "id" From 18d952e13c24b6ce67c67df95332217d37c620f7 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 24 Sep 2025 09:13:47 +1000 Subject: [PATCH 06/23] Logging to DB for easier queries and tracking --- .gitignore | 1 + package.json | 5 +- prisma/autopilot.schema.prisma | 25 + src/app.module.ts | 2 + src/autopilot/autopilot-logging.module.ts | 10 + .../services/autopilot-db-logger.service.ts | 133 +++++ .../services/autopilot-prisma.service.ts | 39 ++ .../services/phase-review.service.ts | 2 + src/challenge/challenge-api.service.ts | 524 ++++++++++++------ src/config/configuration.ts | 2 + src/config/sections/autopilot.config.ts | 6 + src/config/validation.ts | 8 + src/resources/resources.service.ts | 85 ++- src/review/review.service.ts | 237 +++++--- 14 files changed, 828 insertions(+), 251 deletions(-) create mode 100644 prisma/autopilot.schema.prisma create mode 100644 src/autopilot/autopilot-logging.module.ts create mode 100644 src/autopilot/services/autopilot-db-logger.service.ts create mode 100644 src/autopilot/services/autopilot-prisma.service.ts create mode 100644 src/config/sections/autopilot.config.ts diff --git a/.gitignore b/.gitignore index 81f046b..fc157d1 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ build/Release # Dependency directories node_modules/ jspm_packages/ +src/autopilot/generated/ # Snowpack dependency directory (https://snowpack.dev/) web_modules/ diff --git a/package.json b/package.json index fec45f9..25f107f 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,9 @@ "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", - "prisma:generate": "prisma generate", - "postinstall": "prisma generate" + "prisma:generate": "prisma generate --schema prisma/challenge.schema.prisma && prisma generate --schema prisma/autopilot.schema.prisma", + "postinstall": "prisma generate --schema prisma/challenge.schema.prisma && prisma generate --schema prisma/autopilot.schema.prisma", + "prisma:pushautopilot": "prisma db push --schema prisma/autopilot.schema.prisma" }, "prisma": { "schema": "prisma/challenge.schema.prisma" diff --git a/prisma/autopilot.schema.prisma b/prisma/autopilot.schema.prisma new file mode 100644 index 0000000..e38ca70 --- /dev/null +++ b/prisma/autopilot.schema.prisma @@ -0,0 +1,25 @@ +datasource autopilot { + provider = "postgresql" + url = env("AUTOPILOT_DB_URL") + schemas = ["autopilot"] +} + +generator autopilotClient { + provider = "prisma-client-js" + output = "../src/autopilot/generated/autopilot-client" +} + +/// Records detailed database interactions performed by the autopilot service. +model AutopilotAction { + id String @id @default(uuid()) + challengeId String? + action String + status String @default("SUCCESS") + source String? + details Json? + createdAt DateTime @default(now()) + + @@index([challengeId]) + @@map("actions") + @@schema("autopilot") +} diff --git a/src/app.module.ts b/src/app.module.ts index 842e739..6e80c56 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -6,11 +6,13 @@ import { HealthModule } from './health/health.module'; import { ScheduleModule } from '@nestjs/schedule'; import { RecoveryModule } from './recovery/recovery.module'; import { SyncModule } from './sync/sync.module'; +import { AutopilotLoggingModule } from './autopilot/autopilot-logging.module'; @Module({ imports: [ ScheduleModule.forRoot(), AppConfigModule, + AutopilotLoggingModule, KafkaModule, AutopilotModule, HealthModule, diff --git a/src/autopilot/autopilot-logging.module.ts b/src/autopilot/autopilot-logging.module.ts new file mode 100644 index 0000000..78cd4c0 --- /dev/null +++ b/src/autopilot/autopilot-logging.module.ts @@ -0,0 +1,10 @@ +import { Global, Module } from '@nestjs/common'; +import { AutopilotDbLoggerService } from './services/autopilot-db-logger.service'; +import { AutopilotPrismaService } from './services/autopilot-prisma.service'; + +@Global() +@Module({ + providers: [AutopilotPrismaService, AutopilotDbLoggerService], + exports: [AutopilotDbLoggerService], +}) +export class AutopilotLoggingModule {} diff --git a/src/autopilot/services/autopilot-db-logger.service.ts b/src/autopilot/services/autopilot-db-logger.service.ts new file mode 100644 index 0000000..f74a985 --- /dev/null +++ b/src/autopilot/services/autopilot-db-logger.service.ts @@ -0,0 +1,133 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Prisma } from '@prisma/client'; +import { randomUUID } from 'node:crypto'; +import { AutopilotPrismaService } from './autopilot-prisma.service'; + +export interface AutopilotDbLogPayload { + challengeId?: string | null; + source?: string; + status?: 'SUCCESS' | 'ERROR' | 'INFO'; + details?: Record | Array | null; +} + +@Injectable() +export class AutopilotDbLoggerService { + private readonly logger = new Logger(AutopilotDbLoggerService.name); + private readonly dbDebugEnabled: boolean; + private schemaReady = false; + private initializing?: Promise; + + constructor( + private readonly configService: ConfigService, + private readonly prisma: AutopilotPrismaService, + ) { + this.dbDebugEnabled = + this.configService.get('autopilot.dbDebug') ?? false; + } + + async logAction( + action: string, + payload: AutopilotDbLogPayload = {}, + ): Promise { + if (!this.dbDebugEnabled) { + return; + } + + const databaseUrl = this.configService.get('autopilot.dbUrl'); + if (!databaseUrl) { + this.logger.warn( + `DB_DEBUG is enabled but AUTOPILOT_DB_URL is not configured. Skipping DB debug log for action "${action}".`, + ); + return; + } + + try { + await this.ensureSchema(); + } catch (error) { + const err = error as Error; + this.logger.warn( + `Failed to ensure autopilot debug schema while logging action "${action}": ${err.message}`, + ); + return; + } + + const { challengeId = null, source, status = 'SUCCESS', details } = payload; + const serializedDetails = + details === undefined || details === null ? null : JSON.stringify(details); + + try { + await this.prisma.$executeRaw( + Prisma.sql` + INSERT INTO "autopilot"."actions" ( + "id", + "challengeId", + "action", + "status", + "source", + "details", + "createdAt" + ) VALUES ( + ${randomUUID()}, + ${challengeId}, + ${action}, + ${status}, + ${source ?? null}, + ${serializedDetails}, + NOW() + ) + `, + ); + } catch (error) { + const err = error as Error; + this.logger.warn( + `Failed to write DB debug action "${action}": ${err.message}`, + ); + } + } + + private async ensureSchema(): Promise { + if (this.schemaReady) { + return; + } + + if (!this.initializing) { + this.initializing = this.initializeSchema(); + } + + await this.initializing; + } + + private async initializeSchema(): Promise { + try { + await this.prisma.$executeRaw( + Prisma.sql`CREATE SCHEMA IF NOT EXISTS "autopilot"` + ); + + await this.prisma.$executeRaw( + Prisma.sql` + CREATE TABLE IF NOT EXISTS "autopilot"."actions" ( + "id" UUID PRIMARY KEY, + "challengeId" TEXT NULL, + "action" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'SUCCESS', + "source" TEXT NULL, + "details" JSONB NULL, + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + `, + ); + + await this.prisma.$executeRaw( + Prisma.sql` + CREATE INDEX IF NOT EXISTS "idx_autopilot_actions_challenge" + ON "autopilot"."actions" ("challengeId") + `, + ); + + this.schemaReady = true; + } finally { + this.initializing = undefined; + } + } +} diff --git a/src/autopilot/services/autopilot-prisma.service.ts b/src/autopilot/services/autopilot-prisma.service.ts new file mode 100644 index 0000000..b281e87 --- /dev/null +++ b/src/autopilot/services/autopilot-prisma.service.ts @@ -0,0 +1,39 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { PrismaClient } from '@prisma/client'; + +@Injectable() +export class AutopilotPrismaService + extends PrismaClient + implements OnModuleDestroy +{ + private readonly logger = new Logger(AutopilotPrismaService.name); + + constructor(configService: ConfigService) { + const databaseUrl = configService.get('autopilot.dbUrl'); + + super( + databaseUrl + ? { + datasources: { + db: { + url: databaseUrl, + }, + }, + } + : undefined, + ); + + if (!databaseUrl) { + Logger.warn( + 'AUTOPILOT_DB_URL is not configured. Prisma client will rely on the default environment resolution.', + AutopilotPrismaService.name, + ); + } + } + + async onModuleDestroy(): Promise { + await this.$disconnect(); + this.logger.debug('Disconnected Prisma client for Autopilot DB.'); + } +} diff --git a/src/autopilot/services/phase-review.service.ts b/src/autopilot/services/phase-review.service.ts index 0f35154..2583b3d 100644 --- a/src/autopilot/services/phase-review.service.ts +++ b/src/autopilot/services/phase-review.service.ts @@ -79,6 +79,7 @@ export class PhaseReviewService { const existingPairs = await this.reviewService.getExistingReviewPairs( phase.id, + challengeId, ); let createdCount = 0; @@ -95,6 +96,7 @@ export class PhaseReviewService { resource.id, phase.id, scorecardId, + challengeId, ); existingPairs.add(key); createdCount++; diff --git a/src/challenge/challenge-api.service.ts b/src/challenge/challenge-api.service.ts index 2299f3b..cf0e479 100644 --- a/src/challenge/challenge-api.service.ts +++ b/src/challenge/challenge-api.service.ts @@ -1,6 +1,7 @@ import { Injectable, Logger, NotFoundException } from '@nestjs/common'; import { Prisma, ChallengeStatusEnum, PrizeSetTypeEnum } from '@prisma/client'; import { ChallengePrismaService } from './challenge-prisma.service'; +import { AutopilotDbLoggerService } from '../autopilot/services/autopilot-db-logger.service'; import { IPhase, IChallenge, @@ -53,7 +54,10 @@ export class ChallengeApiService { private readonly logger = new Logger(ChallengeApiService.name); private readonly defaultPageSize = 50; - constructor(private readonly prisma: ChallengePrismaService) {} + constructor( + private readonly prisma: ChallengePrismaService, + private readonly dbLogger: AutopilotDbLoggerService, + ) {} async getAllActiveChallenges( filters: ChallengeFiltersDto = {}, @@ -70,34 +74,98 @@ export class ChallengeApiService { typeof filters.page === 'number' || typeof filters.perPage === 'number'; const perPage = filters.perPage ?? this.defaultPageSize; const page = filters.page ?? 1; + try { + const challenges = await this.prisma.challenge.findMany({ + ...challengeWithRelationsArgs, + where, + ...(shouldPaginate + ? { + skip: Math.max(page - 1, 0) * perPage, + take: perPage, + } + : {}), + orderBy: { updatedAt: 'desc' }, + }); - const challenges = await this.prisma.challenge.findMany({ - ...challengeWithRelationsArgs, - where, - ...(shouldPaginate - ? { - skip: Math.max(page - 1, 0) * perPage, - take: perPage, - } - : {}), - orderBy: { updatedAt: 'desc' }, - }); + const mapped = challenges.map((challenge) => this.mapChallenge(challenge)); + + void this.dbLogger.logAction('challenge.getAllActiveChallenges', { + status: 'SUCCESS', + source: ChallengeApiService.name, + details: { + filters: { + status: filters.status ?? null, + isLightweight: filters.isLightweight ?? null, + page, + perPage, + }, + resultCount: mapped.length, + }, + }); - return challenges.map((challenge) => this.mapChallenge(challenge)); + return mapped; + } catch (error) { + const err = error as Error; + + void this.dbLogger.logAction('challenge.getAllActiveChallenges', { + status: 'ERROR', + source: ChallengeApiService.name, + details: { + filters: { + status: filters.status ?? null, + isLightweight: filters.isLightweight ?? null, + page, + perPage, + }, + error: err.message, + }, + }); + + throw err; + } } async getChallenge(challengeId: string): Promise { this.logger.debug(`Fetching challenge with ID: ${challengeId}`); - const challenge = await this.prisma.challenge.findUnique({ - ...challengeWithRelationsArgs, - where: { id: challengeId }, - }); + try { + const challenge = await this.prisma.challenge.findUnique({ + ...challengeWithRelationsArgs, + where: { id: challengeId }, + }); - if (!challenge) { - return null; - } + if (!challenge) { + void this.dbLogger.logAction('challenge.getChallenge', { + challengeId, + status: 'SUCCESS', + source: ChallengeApiService.name, + details: { found: false }, + }); + return null; + } + + const mapped = this.mapChallenge(challenge); + + void this.dbLogger.logAction('challenge.getChallenge', { + challengeId, + status: 'SUCCESS', + source: ChallengeApiService.name, + details: { + found: true, + phaseCount: mapped.phases?.length ?? 0, + }, + }); - return this.mapChallenge(challenge); + return mapped; + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('challenge.getChallenge', { + challengeId, + status: 'ERROR', + source: ChallengeApiService.name, + details: { error: err.message }, + }); + throw err; + } } /** @@ -116,7 +184,16 @@ export class ChallengeApiService { async getChallengePhases(challengeId: string): Promise { const challenge = await this.getChallenge(challengeId); - return challenge?.phases || []; + const phases = challenge?.phases || []; + + void this.dbLogger.logAction('challenge.getChallengePhases', { + challengeId, + status: 'SUCCESS', + source: ChallengeApiService.name, + details: { phaseCount: phases.length }, + }); + + return phases; } async getPhaseDetails( @@ -124,7 +201,19 @@ export class ChallengeApiService { phaseId: string, ): Promise { const phases = await this.getChallengePhases(challengeId); - return phases.find((p) => p.id === phaseId) || null; + const phase = phases.find((p) => p.id === phaseId) || null; + + void this.dbLogger.logAction('challenge.getPhaseDetails', { + challengeId, + status: 'SUCCESS', + source: ChallengeApiService.name, + details: { + phaseId, + found: Boolean(phase), + }, + }); + + return phase; } async getPhaseTypeName( @@ -132,7 +221,19 @@ export class ChallengeApiService { phaseId: string, ): Promise { const phase = await this.getPhaseDetails(challengeId, phaseId); - return phase?.name || 'Unknown'; + const name = phase?.name || 'Unknown'; + + void this.dbLogger.logAction('challenge.getPhaseTypeName', { + challengeId, + status: 'SUCCESS', + source: ChallengeApiService.name, + details: { + phaseId, + phaseName: name, + }, + }); + + return name; } async advancePhase( @@ -140,153 +241,232 @@ export class ChallengeApiService { phaseId: string, operation: 'open' | 'close', ): Promise { - const challenge = await this.prisma.challenge.findUnique({ - ...challengeWithRelationsArgs, - where: { id: challengeId }, - }); - this.logger.debug( `Attempting to ${operation} phase ${phaseId} for challenge ${challengeId}`, ); + try { + const challenge = await this.prisma.challenge.findUnique({ + ...challengeWithRelationsArgs, + where: { id: challengeId }, + }); - if (!challenge) { - this.logger.warn( - `Challenge ${challengeId} not found when advancing phase.`, - ); - return { - success: false, - message: `Challenge ${challengeId} not found`, - }; - } - - if (challenge.status !== ChallengeStatusEnum.ACTIVE) { - return { - success: false, - message: `Challenge ${challengeId} is not active (status: ${challenge.status}).`, - }; - } + if (!challenge) { + this.logger.warn( + `Challenge ${challengeId} not found when advancing phase.`, + ); + const result: PhaseAdvanceResponseDto = { + success: false, + message: `Challenge ${challengeId} not found`, + }; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'INFO', + source: ChallengeApiService.name, + details: { phaseId, operation, result }, + }); + return result; + } - const targetPhase = challenge.phases.find((phase) => phase.id === phaseId); + if (challenge.status !== ChallengeStatusEnum.ACTIVE) { + const result: PhaseAdvanceResponseDto = { + success: false, + message: `Challenge ${challengeId} is not active (status: ${challenge.status}).`, + }; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'INFO', + source: ChallengeApiService.name, + details: { phaseId, operation, result }, + }); + return result; + } - if (!targetPhase) { - this.logger.warn( - `Phase ${phaseId} not found in challenge ${challengeId} while attempting to ${operation}.`, - ); - return { - success: false, - message: `Phase ${phaseId} not found in challenge ${challengeId}`, - }; - } + const targetPhase = challenge.phases.find((phase) => phase.id === phaseId); + + if (!targetPhase) { + this.logger.warn( + `Phase ${phaseId} not found in challenge ${challengeId} while attempting to ${operation}.`, + ); + const result: PhaseAdvanceResponseDto = { + success: false, + message: `Phase ${phaseId} not found in challenge ${challengeId}`, + }; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'INFO', + source: ChallengeApiService.name, + details: { phaseId, operation, result }, + }); + return result; + } - if (operation === 'open' && targetPhase.isOpen) { - return { - success: false, - message: `Phase ${targetPhase.name} is already open`, - }; - } + if (operation === 'open' && targetPhase.isOpen) { + const result: PhaseAdvanceResponseDto = { + success: false, + message: `Phase ${targetPhase.name} is already open`, + }; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'INFO', + source: ChallengeApiService.name, + details: { phaseId, operation, result }, + }); + return result; + } - if (operation === 'close' && !targetPhase.isOpen) { - return { - success: false, - message: `Phase ${targetPhase.name} is already closed`, - }; - } + if (operation === 'close' && !targetPhase.isOpen) { + const result: PhaseAdvanceResponseDto = { + success: false, + message: `Phase ${targetPhase.name} is already closed`, + }; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'INFO', + source: ChallengeApiService.name, + details: { phaseId, operation, result }, + }); + return result; + } - const now = new Date(); + const now = new Date(); + const currentPhaseNames = new Set( + challenge.currentPhaseNames || [], + ); - const currentPhaseNames = new Set( - challenge.currentPhaseNames || [], - ); + try { + await this.prisma.$transaction(async (tx) => { + if (operation === 'open') { + currentPhaseNames.add(targetPhase.name); + await tx.challengePhase.update({ + where: { id: targetPhase.id }, + data: { + isOpen: true, + actualStartDate: targetPhase.actualStartDate ?? now, + actualEndDate: null, + }, + }); + } else { + currentPhaseNames.delete(targetPhase.name); + await tx.challengePhase.update({ + where: { id: targetPhase.id }, + data: { + isOpen: false, + actualEndDate: targetPhase.actualEndDate ?? now, + }, + }); + } - try { - await this.prisma.$transaction(async (tx) => { - if (operation === 'open') { - currentPhaseNames.add(targetPhase.name); - await tx.challengePhase.update({ - where: { id: targetPhase.id }, - data: { - isOpen: true, - actualStartDate: targetPhase.actualStartDate ?? now, - actualEndDate: null, - }, - }); - } else { - currentPhaseNames.delete(targetPhase.name); - await tx.challengePhase.update({ - where: { id: targetPhase.id }, + await tx.challenge.update({ + where: { id: challengeId }, data: { - isOpen: false, - actualEndDate: targetPhase.actualEndDate ?? now, + currentPhaseNames: Array.from(currentPhaseNames), }, }); - } - - await tx.challenge.update({ - where: { id: challengeId }, - data: { - currentPhaseNames: Array.from(currentPhaseNames), - }, }); + } catch (error) { + const err = error as Error; + this.logger.error( + `Failed to ${operation} phase ${phaseId} for challenge ${challengeId}: ${err.message}`, + err.stack, + ); + const result: PhaseAdvanceResponseDto = { + success: false, + message: `Failed to ${operation} phase`, + }; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'ERROR', + source: ChallengeApiService.name, + details: { phaseId, operation, error: err.message }, + }); + return result; + } + + const updatedChallenge = await this.prisma.challenge.findUnique({ + ...challengeWithRelationsArgs, + where: { id: challengeId }, }); - } catch (error) { - const err = error as Error; - this.logger.error( - `Failed to ${operation} phase ${phaseId} for challenge ${challengeId}: ${err.message}`, - err.stack, - ); - return { success: false, message: `Failed to ${operation} phase` }; - } - const updatedChallenge = await this.prisma.challenge.findUnique({ - ...challengeWithRelationsArgs, - where: { id: challengeId }, - }); + if (!updatedChallenge) { + const result: PhaseAdvanceResponseDto = { + success: true, + message: `Phase ${targetPhase.name} ${operation}d but failed to reload challenge`, + }; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'INFO', + source: ChallengeApiService.name, + details: { phaseId, operation, result }, + }); + return result; + } - if (!updatedChallenge) { - return { - success: true, - message: `Phase ${targetPhase.name} ${operation}d but failed to reload challenge`, - }; - } + const hasWinningSubmission = (updatedChallenge.winners || []).length > 0; + + const updatedPhases = updatedChallenge.phases.map((phase) => + this.mapPhase(phase), + ); - const hasWinningSubmission = (updatedChallenge.winners || []).length > 0; + let nextPhases: IPhase[] | undefined; - const updatedPhases = updatedChallenge.phases.map((phase) => - this.mapPhase(phase), - ); + if (operation === 'close') { + const successors = updatedChallenge.phases.filter((phase) => { + if (!phase.predecessor) { + return false; + } + + const predecessorMatches = + phase.predecessor === targetPhase.phaseId || + phase.predecessor === targetPhase.id; - let nextPhases: IPhase[] | undefined; + return predecessorMatches && !phase.actualEndDate && !phase.isOpen; + }); - if (operation === 'close') { - const successors = updatedChallenge.phases.filter((phase) => { - if (!phase.predecessor) { - return false; + if (successors.length > 0) { + nextPhases = successors.map((phase) => this.mapPhase(phase)); } + } - const predecessorMatches = - phase.predecessor === targetPhase.phaseId || - phase.predecessor === targetPhase.id; + const result: PhaseAdvanceResponseDto = { + success: true, + hasWinningSubmission, + message: `Successfully ${operation}d phase ${targetPhase.name} for challenge ${challengeId}`, + updatedPhases, + next: nextPhases + ? { + operation: 'open', + phases: nextPhases, + } + : undefined, + }; - return predecessorMatches && !phase.actualEndDate && !phase.isOpen; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'SUCCESS', + source: ChallengeApiService.name, + details: { + phaseId, + operation, + hasWinningSubmission, + nextPhaseCount: nextPhases?.length ?? 0, + }, }); - if (successors.length > 0) { - nextPhases = successors.map((phase) => this.mapPhase(phase)); - } + return result; + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('challenge.advancePhase', { + challengeId, + status: 'ERROR', + source: ChallengeApiService.name, + details: { + phaseId, + operation, + error: err.message, + }, + }); + throw err; } - - return { - success: true, - hasWinningSubmission, - message: `Successfully ${operation}d phase ${targetPhase.name} for challenge ${challengeId}`, - updatedPhases, - next: nextPhases - ? { - operation: 'open', - phases: nextPhases, - } - : undefined, - }; } private mapChallenge(challenge: ChallengeWithRelations): IChallenge { @@ -414,28 +594,46 @@ export class ChallengeApiService { challengeId: string, winners: IChallengeWinner[], ): Promise { - await this.prisma.$transaction(async (tx) => { - await tx.challenge.update({ - where: { id: challengeId }, - data: { status: ChallengeStatusEnum.COMPLETED }, + try { + await this.prisma.$transaction(async (tx) => { + await tx.challenge.update({ + where: { id: challengeId }, + data: { status: ChallengeStatusEnum.COMPLETED }, + }); + + await tx.challengeWinner.deleteMany({ where: { challengeId } }); + + if (winners.length) { + await tx.challengeWinner.createMany({ + data: winners.map((winner) => ({ + challengeId, + userId: winner.userId, + handle: winner.handle, + placement: winner.placement, + type: PrizeSetTypeEnum.PLACEMENT, + createdBy: 'Autopilot', + updatedBy: 'Autopilot', + })), + }); + } }); - await tx.challengeWinner.deleteMany({ where: { challengeId } }); - - if (winners.length) { - await tx.challengeWinner.createMany({ - data: winners.map((winner) => ({ - challengeId, - userId: winner.userId, - handle: winner.handle, - placement: winner.placement, - type: PrizeSetTypeEnum.PLACEMENT, - createdBy: 'Autopilot', - updatedBy: 'Autopilot', - })), - }); - } - }); + void this.dbLogger.logAction('challenge.completeChallenge', { + challengeId, + status: 'SUCCESS', + source: ChallengeApiService.name, + details: { winnersCount: winners.length }, + }); + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('challenge.completeChallenge', { + challengeId, + status: 'ERROR', + source: ChallengeApiService.name, + details: { winnersCount: winners.length, error: err.message }, + }); + throw err; + } } private ensureTimestamp(date?: Date | null): string { diff --git a/src/config/configuration.ts b/src/config/configuration.ts index 70a8a51..3b079a2 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -4,6 +4,7 @@ import challengeConfig from './sections/challenge.config'; import reviewConfig from './sections/review.config'; import resourcesConfig from './sections/resources.config'; import auth0Config from './sections/auth0.config'; +import autopilotConfig from './sections/autopilot.config'; export default () => ({ app: appConfig(), @@ -12,4 +13,5 @@ export default () => ({ review: reviewConfig(), resources: resourcesConfig(), auth0: auth0Config(), + autopilot: autopilotConfig(), }); diff --git a/src/config/sections/autopilot.config.ts b/src/config/sections/autopilot.config.ts new file mode 100644 index 0000000..76a8cdd --- /dev/null +++ b/src/config/sections/autopilot.config.ts @@ -0,0 +1,6 @@ +import { registerAs } from '@nestjs/config'; + +export default registerAs('autopilot', () => ({ + dbUrl: process.env.AUTOPILOT_DB_URL, + dbDebug: process.env.DB_DEBUG === 'true', +})); diff --git a/src/config/validation.ts b/src/config/validation.ts index 575e823..bb8d9d9 100644 --- a/src/config/validation.ts +++ b/src/config/validation.ts @@ -12,6 +12,7 @@ export const validationSchema = Joi.object({ .default('info'), LOG_DIR: Joi.string().default('logs'), ENABLE_FILE_LOGGING: Joi.boolean().default(false), + DB_DEBUG: Joi.boolean().default(false), // Kafka Configuration KAFKA_BROKERS: Joi.string().required(), @@ -42,6 +43,13 @@ export const validationSchema = Joi.object({ then: Joi.optional().default('postgresql://localhost:5432/resources'), otherwise: Joi.required(), }), + AUTOPILOT_DB_URL: Joi.string() + .uri() + .when('DB_DEBUG', { + is: true, + then: Joi.required(), + otherwise: Joi.optional(), + }), REVIEWER_POLL_INTERVAL_MS: Joi.number() .integer() .positive() diff --git a/src/resources/resources.service.ts b/src/resources/resources.service.ts index b7eaa59..7c5b968 100644 --- a/src/resources/resources.service.ts +++ b/src/resources/resources.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { ResourcesPrismaService } from './resources-prisma.service'; +import { AutopilotDbLoggerService } from '../autopilot/services/autopilot-db-logger.service'; export interface ReviewerResourceRecord { id: string; @@ -14,7 +15,10 @@ export class ResourcesService { private static readonly RESOURCE_TABLE = Prisma.sql`"Resource"`; private static readonly RESOURCE_ROLE_TABLE = Prisma.sql`"ResourceRole"`; - constructor(private readonly prisma: ResourcesPrismaService) {} + constructor( + private readonly prisma: ResourcesPrismaService, + private readonly dbLogger: AutopilotDbLoggerService, + ) {} async getMemberHandleMap( challengeId: string, @@ -34,23 +38,43 @@ export class ResourcesService { const idList = Prisma.join(uniqueIds.map((id) => Prisma.sql`${id}`)); - const rows = await this.prisma.$queryRaw< - Array<{ memberId: string; memberHandle: string | null }> - >(Prisma.sql` - SELECT r."memberId", r."memberHandle" - FROM ${ResourcesService.RESOURCE_TABLE} r - WHERE r."challengeId" = ${challengeId} - AND r."memberId" IN (${idList}) - `); - - return new Map( - rows - .filter((row) => Boolean(row.memberId)) - .map((row) => [ - String(row.memberId), - row.memberHandle?.trim() || String(row.memberId), - ]), - ); + try { + const rows = await this.prisma.$queryRaw< + Array<{ memberId: string; memberHandle: string | null }> + >(Prisma.sql` + SELECT r."memberId", r."memberHandle" + FROM ${ResourcesService.RESOURCE_TABLE} r + WHERE r."challengeId" = ${challengeId} + AND r."memberId" IN (${idList}) + `); + + const handleMap = new Map( + rows + .filter((row) => Boolean(row.memberId)) + .map((row) => [ + String(row.memberId), + row.memberHandle?.trim() || String(row.memberId), + ]), + ); + + void this.dbLogger.logAction('resources.getMemberHandleMap', { + challengeId, + status: 'SUCCESS', + source: ResourcesService.name, + details: { inputCount: uniqueIds.length, matchedCount: handleMap.size }, + }); + + return handleMap; + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('resources.getMemberHandleMap', { + challengeId, + status: 'ERROR', + source: ResourcesService.name, + details: { inputCount: uniqueIds.length, error: err.message }, + }); + throw err; + } } async getReviewerResources( @@ -71,8 +95,27 @@ export class ResourcesService { AND rr."name" IN (${roleList}) `; - const reviewers = - await this.prisma.$queryRaw(query); - return reviewers; + try { + const reviewers = + await this.prisma.$queryRaw(query); + + void this.dbLogger.logAction('resources.getReviewerResources', { + challengeId, + status: 'SUCCESS', + source: ResourcesService.name, + details: { roleCount: roleNames.length, reviewerCount: reviewers.length }, + }); + + return reviewers; + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('resources.getReviewerResources', { + challengeId, + status: 'ERROR', + source: ResourcesService.name, + details: { roleCount: roleNames.length, error: err.message }, + }); + throw err; + } } } diff --git a/src/review/review.service.ts b/src/review/review.service.ts index a227ae4..a27a053 100644 --- a/src/review/review.service.ts +++ b/src/review/review.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { ReviewPrismaService } from './review-prisma.service'; +import { AutopilotDbLoggerService } from '../autopilot/services/autopilot-db-logger.service'; interface SubmissionRecord { id: string; @@ -17,7 +18,10 @@ export class ReviewService { private static readonly SUBMISSION_TABLE = Prisma.sql`"submission"`; private static readonly REVIEW_SUMMATION_TABLE = Prisma.sql`"reviewSummation"`; - constructor(private readonly prisma: ReviewPrismaService) {} + constructor( + private readonly prisma: ReviewPrismaService, + private readonly dbLogger: AutopilotDbLoggerService, + ) {} async getTopFinalReviewScores( challengeId: string, @@ -36,68 +40,96 @@ export class ReviewService { const fetchLimit = Math.max(limit, 1) * 5; const limitClause = Prisma.sql`LIMIT ${fetchLimit}`; - const rows = await this.prisma.$queryRaw< - Array<{ - memberId: string | null; + try { + const rows = await this.prisma.$queryRaw< + Array<{ + memberId: string | null; + submissionId: string; + aggregateScore: number | string | null; + }> + >(Prisma.sql` + SELECT + s."memberId" AS "memberId", + s."id" AS "submissionId", + rs."aggregateScore" AS "aggregateScore" + FROM ${ReviewService.REVIEW_SUMMATION_TABLE} rs + INNER JOIN ${ReviewService.SUBMISSION_TABLE} s + ON s."id" = rs."submissionId" + WHERE s."challengeId" = ${challengeId} + AND rs."isFinal" = true + AND rs."aggregateScore" IS NOT NULL + AND s."memberId" IS NOT NULL + AND s."type" = 'CONTEST_SUBMISSION' + ORDER BY rs."aggregateScore" DESC, s."submittedDate" ASC, s."id" ASC + ${limitClause} + `); + + if (!rows.length) { + void this.dbLogger.logAction('review.getTopFinalReviewScores', { + challengeId, + status: 'SUCCESS', + source: ReviewService.name, + details: { limit, rowsExamined: 0, winnersCount: 0 }, + }); + return []; + } + + const seenMembers = new Set(); + const winners: Array<{ + memberId: string; submissionId: string; - aggregateScore: number | string | null; - }> - >(Prisma.sql` - SELECT - s."memberId" AS "memberId", - s."id" AS "submissionId", - rs."aggregateScore" AS "aggregateScore" - FROM ${ReviewService.REVIEW_SUMMATION_TABLE} rs - INNER JOIN ${ReviewService.SUBMISSION_TABLE} s - ON s."id" = rs."submissionId" - WHERE s."challengeId" = ${challengeId} - AND rs."isFinal" = true - AND rs."aggregateScore" IS NOT NULL - AND s."memberId" IS NOT NULL - AND s."type" = 'CONTEST_SUBMISSION' - ORDER BY rs."aggregateScore" DESC, s."submittedDate" ASC, s."id" ASC - ${limitClause} - `); - - if (!rows.length) { - return []; - } + aggregateScore: number; + }> = []; - const seenMembers = new Set(); - const winners: Array<{ - memberId: string; - submissionId: string; - aggregateScore: number; - }> = []; + for (const row of rows) { + const memberId = row.memberId?.trim(); + if (!memberId || seenMembers.has(memberId)) { + continue; + } - for (const row of rows) { - const memberId = row.memberId?.trim(); - if (!memberId || seenMembers.has(memberId)) { - continue; - } + const aggregateScore = + typeof row.aggregateScore === 'string' + ? Number(row.aggregateScore) + : (row.aggregateScore ?? 0); + + if (Number.isNaN(aggregateScore)) { + continue; + } - const aggregateScore = - typeof row.aggregateScore === 'string' - ? Number(row.aggregateScore) - : (row.aggregateScore ?? 0); + winners.push({ + memberId, + submissionId: row.submissionId, + aggregateScore, + }); + seenMembers.add(memberId); - if (Number.isNaN(aggregateScore)) { - continue; + if (winners.length >= limit) { + break; + } } - winners.push({ - memberId, - submissionId: row.submissionId, - aggregateScore, + void this.dbLogger.logAction('review.getTopFinalReviewScores', { + challengeId, + status: 'SUCCESS', + source: ReviewService.name, + details: { + limit, + rowsExamined: rows.length, + winnersCount: winners.length, + }, }); - seenMembers.add(memberId); - if (winners.length >= limit) { - break; - } + return winners; + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('review.getTopFinalReviewScores', { + challengeId, + status: 'ERROR', + source: ReviewService.name, + details: { limit, error: err.message }, + }); + throw err; } - - return winners; } async getActiveSubmissionIds(challengeId: string): Promise { @@ -108,28 +140,75 @@ export class ReviewService { AND ("status" = 'ACTIVE' OR "status" IS NULL) `; - const submissions = await this.prisma.$queryRaw(query); - return submissions.map((record) => record.id).filter(Boolean); + try { + const submissions = await this.prisma.$queryRaw(query); + const submissionIds = submissions.map((record) => record.id).filter(Boolean); + + void this.dbLogger.logAction('review.getActiveSubmissionIds', { + challengeId, + status: 'SUCCESS', + source: ReviewService.name, + details: { submissionCount: submissionIds.length }, + }); + + return submissionIds; + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('review.getActiveSubmissionIds', { + challengeId, + status: 'ERROR', + source: ReviewService.name, + details: { error: err.message }, + }); + throw err; + } } - async getExistingReviewPairs(phaseId: string): Promise> { + async getExistingReviewPairs( + phaseId: string, + challengeId?: string, + ): Promise> { const query = Prisma.sql` SELECT "submissionId", "resourceId" FROM ${ReviewService.REVIEW_TABLE} WHERE "phaseId" = ${phaseId} `; - const existing = await this.prisma.$queryRaw(query); - const result = new Set(); + try { + const existing = await this.prisma.$queryRaw(query); + const result = new Set(); - for (const record of existing) { - if (!record.submissionId) { - continue; + for (const record of existing) { + if (!record.submissionId) { + continue; + } + result.add(this.composeKey(record.resourceId, record.submissionId)); } - result.add(this.composeKey(record.resourceId, record.submissionId)); - } - return result; + void this.dbLogger.logAction('review.getExistingReviewPairs', { + challengeId: challengeId ?? null, + status: 'SUCCESS', + source: ReviewService.name, + details: { + phaseId, + pairCount: result.size, + }, + }); + + return result; + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('review.getExistingReviewPairs', { + challengeId: challengeId ?? null, + status: 'ERROR', + source: ReviewService.name, + details: { + phaseId, + error: err.message, + }, + }); + throw err; + } } async createPendingReview( @@ -137,6 +216,7 @@ export class ReviewService { resourceId: string, phaseId: string, scorecardId: string, + challengeId: string, ): Promise { const insert = Prisma.sql` INSERT INTO ${ReviewService.REVIEW_TABLE} ( @@ -158,7 +238,34 @@ export class ReviewService { ) `; - await this.prisma.$executeRaw(insert); + try { + await this.prisma.$executeRaw(insert); + + void this.dbLogger.logAction('review.createPendingReview', { + challengeId, + status: 'SUCCESS', + source: ReviewService.name, + details: { + resourceId, + submissionId, + phaseId, + }, + }); + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('review.createPendingReview', { + challengeId, + status: 'ERROR', + source: ReviewService.name, + details: { + resourceId, + submissionId, + phaseId, + error: err.message, + }, + }); + throw err; + } } private composeKey(resourceId: string, submissionId: string): string { From 52d511b1b227bf90a4625b1f531817330ea0fc0c Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 24 Sep 2025 10:07:23 +1000 Subject: [PATCH 07/23] Fix JSONb usage --- src/autopilot/services/autopilot-db-logger.service.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/autopilot/services/autopilot-db-logger.service.ts b/src/autopilot/services/autopilot-db-logger.service.ts index f74a985..3067b47 100644 --- a/src/autopilot/services/autopilot-db-logger.service.ts +++ b/src/autopilot/services/autopilot-db-logger.service.ts @@ -53,8 +53,10 @@ export class AutopilotDbLoggerService { } const { challengeId = null, source, status = 'SUCCESS', details } = payload; - const serializedDetails = - details === undefined || details === null ? null : JSON.stringify(details); + const detailsFragment = + details === undefined || details === null + ? Prisma.sql`NULL` + : Prisma.sql`${JSON.stringify(details)}::jsonb`; try { await this.prisma.$executeRaw( @@ -73,7 +75,7 @@ export class AutopilotDbLoggerService { ${action}, ${status}, ${source ?? null}, - ${serializedDetails}, + ${detailsFragment}, NOW() ) `, From 71e6def4877230f0bbf91da47cc761bb776a21c2 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 24 Sep 2025 10:34:17 +1000 Subject: [PATCH 08/23] Tweaks to not complete challenge too early --- src/autopilot/services/scheduler.service.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/autopilot/services/scheduler.service.ts b/src/autopilot/services/scheduler.service.ts index 60d2359..0ee21b6 100644 --- a/src/autopilot/services/scheduler.service.ts +++ b/src/autopilot/services/scheduler.service.ts @@ -306,12 +306,21 @@ export class SchedulerService { } const hasOpenPhases = phases?.some((phase) => phase.isOpen) ?? true; + const hasIncompletePhases = + phases?.some((phase) => !phase.actualEndDate) ?? true; const hasNextPhases = Boolean(result.next?.phases?.length); - if (!hasOpenPhases && !hasNextPhases) { + if (!hasOpenPhases && !hasNextPhases && !hasIncompletePhases) { await this.challengeCompletionService.finalizeChallenge( data.challengeId, ); + } else { + const pendingCount = phases?.reduce((pending, phase) => { + return pending + (phase.isOpen || !phase.actualEndDate ? 1 : 0); + }, 0); + this.logger.debug?.( + `Challenge ${data.challengeId} not ready for completion after closing phase ${data.phaseId}. Pending phases: ${pendingCount ?? 'unknown'}, next phases: ${result.next?.phases?.length ?? 0}.`, + ); } } catch (error) { const err = error as Error; From 182f976c5684a8a4e062a70bd999d79462b6bea2 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 24 Sep 2025 17:35:06 +1000 Subject: [PATCH 09/23] Better copmletion tracking --- .nvmrc | 1 + .../services/challenge-completion.service.ts | 18 ++- src/autopilot/services/scheduler.service.ts | 115 +++++++++++++++++- 3 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 .nvmrc diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..2edeafb --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +20 \ No newline at end of file diff --git a/src/autopilot/services/challenge-completion.service.ts b/src/autopilot/services/challenge-completion.service.ts index e4eb175..48cf2d8 100644 --- a/src/autopilot/services/challenge-completion.service.ts +++ b/src/autopilot/services/challenge-completion.service.ts @@ -14,7 +14,7 @@ export class ChallengeCompletionService { private readonly resourcesService: ResourcesService, ) {} - async finalizeChallenge(challengeId: string): Promise { + async finalizeChallenge(challengeId: string): Promise { const challenge = await this.challengeApiService.getChallengeById(challengeId); @@ -26,7 +26,7 @@ export class ChallengeCompletionService { this.logger.log( `Challenge ${challengeId} is already completed with winners; skipping finalization.`, ); - return; + return true; } const scoreRows = await this.reviewService.getTopFinalReviewScores( @@ -35,11 +35,18 @@ export class ChallengeCompletionService { ); if (!scoreRows.length) { + if ((challenge.numOfSubmissions ?? 0) > 0) { + this.logger.warn( + `Final review scores are not yet available for challenge ${challengeId}. Will retry finalization later.`, + ); + return false; + } + this.logger.warn( - `No final review scores found for challenge ${challengeId}; marking completed without winners.`, + `No submissions found for challenge ${challengeId}; marking completed without winners.`, ); await this.challengeApiService.completeChallenge(challengeId, []); - return; + return true; } const memberIds = scoreRows.map((row) => row.memberId); @@ -74,12 +81,13 @@ export class ChallengeCompletionService { `Unable to derive any numeric winners for challenge ${challengeId}; marking completed without winners.`, ); await this.challengeApiService.completeChallenge(challengeId, []); - return; + return true; } await this.challengeApiService.completeChallenge(challengeId, winners); this.logger.log( `Marked challenge ${challengeId} as COMPLETED with ${winners.length} winner(s).`, ); + return true; } } diff --git a/src/autopilot/services/scheduler.service.ts b/src/autopilot/services/scheduler.service.ts index 0ee21b6..ef6ce7e 100644 --- a/src/autopilot/services/scheduler.service.ts +++ b/src/autopilot/services/scheduler.service.ts @@ -23,6 +23,11 @@ export class SchedulerService { nextPhases: any[], ) => Promise | void) | null = null; + private finalizationRetryJobs = new Map(); + private finalizationAttempts = new Map(); + private readonly finalizationRetryBaseDelayMs = 60_000; + private readonly finalizationRetryMaxAttempts = 10; + private readonly finalizationRetryMaxDelayMs = 10 * 60 * 1000; constructor( private schedulerRegistry: SchedulerRegistry, @@ -311,9 +316,7 @@ export class SchedulerService { const hasNextPhases = Boolean(result.next?.phases?.length); if (!hasOpenPhases && !hasNextPhases && !hasIncompletePhases) { - await this.challengeCompletionService.finalizeChallenge( - data.challengeId, - ); + await this.attemptChallengeFinalization(data.challengeId); } else { const pendingCount = phases?.reduce((pending, phase) => { return pending + (phase.isOpen || !phase.actualEndDate ? 1 : 0); @@ -385,4 +388,110 @@ export class SchedulerService { ); } } + + private async attemptChallengeFinalization( + challengeId: string, + ): Promise { + const attempt = (this.finalizationAttempts.get(challengeId) ?? 0) + 1; + this.finalizationAttempts.set(challengeId, attempt); + + try { + const completed = + await this.challengeCompletionService.finalizeChallenge(challengeId); + + if (completed) { + this.logger.log( + `Successfully finalized challenge ${challengeId} after ${attempt} attempt(s).`, + ); + this.clearFinalizationRetry(challengeId); + this.finalizationAttempts.delete(challengeId); + return; + } + + this.logger.log( + `Final review scores not yet available for challenge ${challengeId}; scheduling retry attempt ${attempt + 1}.`, + ); + this.scheduleFinalizationRetry(challengeId, attempt); + } catch (error) { + const err = error as Error; + this.logger.error( + `Attempt ${attempt} to finalize challenge ${challengeId} failed: ${err.message}`, + err.stack, + ); + this.scheduleFinalizationRetry(challengeId, attempt); + } + } + + private scheduleFinalizationRetry( + challengeId: string, + attempt: number, + ): void { + const existingJobId = this.finalizationRetryJobs.get(challengeId); + if ( + existingJobId && + this.schedulerRegistry.doesExist('timeout', existingJobId) + ) { + this.logger.debug?.( + `Finalization retry already scheduled for challenge ${challengeId}; skipping duplicate schedule.`, + ); + return; + } + + if (attempt >= this.finalizationRetryMaxAttempts) { + this.logger.error( + `Reached maximum finalization attempts (${this.finalizationRetryMaxAttempts}) for challenge ${challengeId}. Manual intervention required to resolve winners.`, + ); + this.clearFinalizationRetry(challengeId); + this.finalizationAttempts.delete(challengeId); + return; + } + + const delay = this.computeFinalizationDelay(attempt); + const jobId = `challenge-finalize:${challengeId}:${attempt + 1}`; + + const timeout = setTimeout(() => { + try { + if (this.schedulerRegistry.doesExist('timeout', jobId)) { + this.schedulerRegistry.deleteTimeout(jobId); + } + } catch (cleanupError) { + this.logger.error( + `Failed to clean up finalization retry job ${jobId}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`, + ); + } finally { + this.finalizationRetryJobs.delete(challengeId); + } + + void this.attemptChallengeFinalization(challengeId); + }, delay); + + try { + this.schedulerRegistry.addTimeout(jobId, timeout); + this.finalizationRetryJobs.set(challengeId, jobId); + this.logger.log( + `Scheduled finalization retry ${attempt + 1} for challenge ${challengeId} in ${Math.round(delay / 1000)} second(s).`, + ); + } catch (error) { + const err = error as Error; + this.logger.error( + `Failed to schedule finalization retry for challenge ${challengeId}: ${err.message}`, + err.stack, + ); + clearTimeout(timeout); + } + } + + private computeFinalizationDelay(attempt: number): number { + const multiplier = Math.max(attempt, 1); + const delay = this.finalizationRetryBaseDelayMs * multiplier; + return Math.min(delay, this.finalizationRetryMaxDelayMs); + } + + private clearFinalizationRetry(challengeId: string): void { + const jobId = this.finalizationRetryJobs.get(challengeId); + if (jobId && this.schedulerRegistry.doesExist('timeout', jobId)) { + this.schedulerRegistry.deleteTimeout(jobId); + } + this.finalizationRetryJobs.delete(challengeId); + } } From 4160579277aa27bda54f0ad791a43a320a9ce3cf Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 24 Sep 2025 20:26:45 +1000 Subject: [PATCH 10/23] Completed status check tweaks --- src/review/review.service.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/review/review.service.ts b/src/review/review.service.ts index a27a053..edafead 100644 --- a/src/review/review.service.ts +++ b/src/review/review.service.ts @@ -59,7 +59,10 @@ export class ReviewService { AND rs."isFinal" = true AND rs."aggregateScore" IS NOT NULL AND s."memberId" IS NOT NULL - AND s."type" = 'CONTEST_SUBMISSION' + AND ( + s."type" IS NULL + OR upper(s."type") = 'CONTEST_SUBMISSION' + ) ORDER BY rs."aggregateScore" DESC, s."submittedDate" ASC, s."id" ASC ${limitClause} `); From c9a887c29184500ae0a2146c7fddca39c21b96a7 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 25 Sep 2025 16:33:48 +1000 Subject: [PATCH 11/23] Better Iterative Review handling --- .../interfaces/autopilot.interface.ts | 21 ++ .../services/autopilot.service.spec.ts | 186 ++++++++++++++++++ src/autopilot/services/autopilot.service.ts | 109 ++++++++++ src/kafka/constants/topics.ts | 1 + src/kafka/consumers/autopilot.consumer.ts | 10 + src/kafka/types/topic-payload-map.type.ts | 2 + 6 files changed, 329 insertions(+) create mode 100644 src/autopilot/services/autopilot.service.spec.ts diff --git a/src/autopilot/interfaces/autopilot.interface.ts b/src/autopilot/interfaces/autopilot.interface.ts index 1f69db9..31ecc33 100644 --- a/src/autopilot/interfaces/autopilot.interface.ts +++ b/src/autopilot/interfaces/autopilot.interface.ts @@ -65,3 +65,24 @@ export interface ChallengeUpdateMessage extends BaseMessage { export interface CommandMessage extends BaseMessage { payload: CommandPayload; } + +export interface SubmissionAggregatePayload { + resource: string; + id: string; + type?: string; + memberId?: number; + challengeId?: number; + legacyChallengeId?: number; + v5ChallengeId?: string; + submissionPhaseId?: string; + fileType?: string; + submittedDate?: string; + url?: string; + isFileSubmission?: boolean; + created?: string; + updated?: string; + createdBy?: string; + updatedBy?: string; + originalTopic?: string; + [key: string]: unknown; +} diff --git a/src/autopilot/services/autopilot.service.spec.ts b/src/autopilot/services/autopilot.service.spec.ts new file mode 100644 index 0000000..a580952 --- /dev/null +++ b/src/autopilot/services/autopilot.service.spec.ts @@ -0,0 +1,186 @@ +import { AutopilotService } from './autopilot.service'; +import { SchedulerService } from './scheduler.service'; +import { ChallengeApiService } from '../../challenge/challenge-api.service'; +import { PhaseReviewService } from './phase-review.service'; +import { ReviewAssignmentService } from './review-assignment.service'; +import { SubmissionAggregatePayload } from '../interfaces/autopilot.interface'; +import { IChallenge, IPhase } from '../../challenge/interfaces/challenge.interface'; + +describe('AutopilotService - handleSubmissionNotificationAggregate', () => { + const isoNow = new Date().toISOString(); + + const createPhase = (overrides: Partial = {}): IPhase => ({ + id: 'phase-1', + phaseId: 'phase-1', + name: 'Iterative Review', + description: null, + isOpen: false, + duration: 0, + scheduledStartDate: isoNow, + scheduledEndDate: isoNow, + actualStartDate: null, + actualEndDate: null, + predecessor: null, + constraints: [], + ...overrides, + }); + + const createChallenge = (overrides: Partial = {}): IChallenge => ({ + id: 'challenge-123', + name: 'Test Challenge', + description: null, + descriptionFormat: 'markdown', + projectId: 123, + typeId: 'type-id', + trackId: 'track-id', + timelineTemplateId: 'template-id', + currentPhaseNames: [], + tags: [], + groups: [], + submissionStartDate: isoNow, + submissionEndDate: isoNow, + registrationStartDate: isoNow, + registrationEndDate: isoNow, + startDate: isoNow, + endDate: null, + legacyId: null, + status: 'ACTIVE', + createdBy: 'tester', + updatedBy: 'tester', + metadata: [], + phases: [createPhase()], + reviewers: [], + winners: [], + discussions: [], + events: [], + prizeSets: [], + terms: [], + skills: [], + attachments: [], + track: 'Development', + type: 'First2Finish', + legacy: {}, + task: {}, + created: isoNow, + updated: isoNow, + overview: {}, + numOfSubmissions: 0, + numOfCheckpointSubmissions: 0, + numOfRegistrants: 0, + ...overrides, + }); + + const createPayload = ( + overrides: Partial = {}, + ): SubmissionAggregatePayload => ({ + resource: 'submission', + id: 'submission-1', + originalTopic: 'submission.notification.create', + v5ChallengeId: 'challenge-123', + ...overrides, + }); + + let schedulerService: Partial; + let challengeApiService: { + getChallengeById: jest.Mock; + advancePhase: jest.Mock; + }; + let autopilotService: AutopilotService; + + beforeEach(() => { + schedulerService = { + setPhaseChainCallback: jest.fn(), + schedulePhaseTransition: jest.fn(), + cancelScheduledTransition: jest.fn(), + getScheduledTransition: jest.fn(), + }; + + challengeApiService = { + getChallengeById: jest.fn(), + advancePhase: jest.fn(), + }; + + autopilotService = new AutopilotService( + schedulerService as SchedulerService, + challengeApiService as unknown as ChallengeApiService, + {} as PhaseReviewService, + {} as ReviewAssignmentService, + ); + + jest.clearAllMocks(); + }); + + it('ignores messages that are not submission.create aggregates', async () => { + await autopilotService.handleSubmissionNotificationAggregate( + createPayload({ originalTopic: 'submission.notification.update' }), + ); + + expect(challengeApiService.getChallengeById).not.toHaveBeenCalled(); + expect(challengeApiService.advancePhase).not.toHaveBeenCalled(); + }); + + it('ignores messages without a v5 challenge id', async () => { + await autopilotService.handleSubmissionNotificationAggregate( + createPayload({ v5ChallengeId: undefined }), + ); + + expect(challengeApiService.getChallengeById).not.toHaveBeenCalled(); + }); + + it('opens iterative review phase for First2Finish challenge', async () => { + const iterativeReviewPhase = createPhase({ id: 'iterative-phase' }); + challengeApiService.getChallengeById.mockResolvedValue( + createChallenge({ phases: [iterativeReviewPhase] }), + ); + challengeApiService.advancePhase.mockResolvedValue({ + success: true, + message: 'opened', + }); + + await autopilotService.handleSubmissionNotificationAggregate( + createPayload({ id: 'submission-123' }), + ); + + expect(challengeApiService.getChallengeById).toHaveBeenCalledWith( + 'challenge-123', + ); + expect(challengeApiService.advancePhase).toHaveBeenCalledWith( + 'challenge-123', + 'iterative-phase', + 'open', + ); + }); + + it('skips non-First2Finish challenges', async () => { + challengeApiService.getChallengeById.mockResolvedValue( + createChallenge({ type: 'Design Challenge' }), + ); + + await autopilotService.handleSubmissionNotificationAggregate(createPayload()); + + expect(challengeApiService.advancePhase).not.toHaveBeenCalled(); + }); + + it('skips when iterative review phase is already open', async () => { + const iterativeReviewPhase = createPhase({ isOpen: true }); + challengeApiService.getChallengeById.mockResolvedValue( + createChallenge({ phases: [iterativeReviewPhase] }), + ); + + await autopilotService.handleSubmissionNotificationAggregate(createPayload()); + + expect(challengeApiService.advancePhase).not.toHaveBeenCalled(); + }); + + it('skips when iterative review phase is not present', async () => { + challengeApiService.getChallengeById.mockResolvedValue( + createChallenge({ + phases: [createPhase({ name: 'Submission', id: 'submission-phase' })], + }), + ); + + await autopilotService.handleSubmissionNotificationAggregate(createPayload()); + + expect(challengeApiService.advancePhase).not.toHaveBeenCalled(); + }); +}); diff --git a/src/autopilot/services/autopilot.service.ts b/src/autopilot/services/autopilot.service.ts index b5c52bb..6ee02d0 100644 --- a/src/autopilot/services/autopilot.service.ts +++ b/src/autopilot/services/autopilot.service.ts @@ -7,12 +7,17 @@ import { ChallengeUpdatePayload, CommandPayload, AutopilotOperator, + SubmissionAggregatePayload, } from '../interfaces/autopilot.interface'; import { ChallengeApiService } from '../../challenge/challenge-api.service'; import { IPhase } from '../../challenge/interfaces/challenge.interface'; import { AUTOPILOT_COMMANDS } from '../../common/constants/commands.constants'; import { REVIEW_PHASE_NAMES } from '../constants/review.constants'; +const SUBMISSION_NOTIFICATION_CREATE_TOPIC = 'submission.notification.create'; +const ITERATIVE_REVIEW_PHASE_NAME = 'Iterative Review'; +const FIRST2FINISH_TYPE = 'first2finish'; + @Injectable() export class AutopilotService { private readonly logger = new Logger(AutopilotService.name); @@ -47,6 +52,10 @@ export class AutopilotService { return (status ?? '').toUpperCase() === 'ACTIVE'; } + private isFirst2FinishChallenge(type?: string): boolean { + return (type ?? '').toLowerCase() === FIRST2FINISH_TYPE; + } + schedulePhaseTransition(phaseData: PhaseTransitionPayload): string { try { const phaseKey = `${phaseData.challengeId}:${phaseData.phaseId}`; @@ -384,6 +393,106 @@ export class AutopilotService { } } + async handleSubmissionNotificationAggregate( + payload: SubmissionAggregatePayload, + ): Promise { + const { id: submissionId } = payload; + const challengeId = payload.v5ChallengeId; + + if (payload.originalTopic !== SUBMISSION_NOTIFICATION_CREATE_TOPIC) { + this.logger.debug( + 'Ignoring submission aggregate message with non-create original topic', + { + submissionId, + originalTopic: payload.originalTopic, + }, + ); + return; + } + + if (!challengeId) { + this.logger.warn( + 'Submission aggregate message missing v5ChallengeId; unable to process', + { submissionId }, + ); + return; + } + + try { + const challenge = await this.challengeApiService.getChallengeById( + challengeId, + ); + + if (!this.isFirst2FinishChallenge(challenge.type)) { + this.logger.debug( + 'Skipping submission aggregate for non-First2Finish challenge', + { + submissionId, + challengeId, + challengeType: challenge.type, + }, + ); + return; + } + + const iterativeReviewPhase = challenge.phases?.find( + (phase) => phase.name === ITERATIVE_REVIEW_PHASE_NAME, + ); + + if (!iterativeReviewPhase) { + this.logger.warn( + 'No Iterative Review phase found for First2Finish challenge', + { submissionId, challengeId }, + ); + return; + } + + if (iterativeReviewPhase.isOpen) { + this.logger.debug( + 'Iterative Review phase already open; skipping advance', + { + submissionId, + challengeId, + phaseId: iterativeReviewPhase.id, + }, + ); + return; + } + + this.logger.log( + `Opening Iterative Review phase ${iterativeReviewPhase.id} for challenge ${challengeId} in response to submission ${submissionId}.`, + ); + + const advanceResult = await this.challengeApiService.advancePhase( + challenge.id, + iterativeReviewPhase.id, + 'open', + ); + + if (!advanceResult.success) { + this.logger.warn( + 'Advance phase operation reported failure for Iterative Review phase', + { + submissionId, + challengeId, + phaseId: iterativeReviewPhase.id, + message: advanceResult.message, + }, + ); + } else { + this.logger.log( + `Iterative Review phase ${iterativeReviewPhase.id} opened for challenge ${challengeId}.`, + ); + } + } catch (error) { + const err = error as Error; + this.logger.error( + `Failed processing submission aggregate for challenge ${challengeId}: ${err.message}`, + err.stack, + ); + } + } + handleCommand(message: CommandPayload): void { const { command, operator, projectId, date, phaseId } = message; diff --git a/src/kafka/constants/topics.ts b/src/kafka/constants/topics.ts index 6936f6f..7bbfa28 100644 --- a/src/kafka/constants/topics.ts +++ b/src/kafka/constants/topics.ts @@ -4,6 +4,7 @@ export const KAFKA_TOPICS = { CHALLENGE_CREATED: 'challenge.notification.create', CHALLENGE_UPDATED: 'challenge.notification.update', COMMAND: 'autopilot.command', + SUBMISSION_NOTIFICATION_AGGREGATE: 'submission.notification.aggregate', } as const; export type KafkaTopic = (typeof KAFKA_TOPICS)[keyof typeof KAFKA_TOPICS]; diff --git a/src/kafka/consumers/autopilot.consumer.ts b/src/kafka/consumers/autopilot.consumer.ts index d339771..22405b5 100644 --- a/src/kafka/consumers/autopilot.consumer.ts +++ b/src/kafka/consumers/autopilot.consumer.ts @@ -8,6 +8,7 @@ import { ChallengeUpdatePayload, CommandPayload, PhaseTransitionPayload, + SubmissionAggregatePayload, } from 'src/autopilot/interfaces/autopilot.interface'; @Injectable() @@ -44,6 +45,10 @@ export class AutopilotConsumer { [KAFKA_TOPICS.COMMAND]: this.autopilotService.handleCommand.bind( this.autopilotService, ) as (message: CommandPayload) => Promise, + [KAFKA_TOPICS.SUBMISSION_NOTIFICATION_AGGREGATE]: + this.autopilotService.handleSubmissionNotificationAggregate.bind( + this.autopilotService, + ) as (message: SubmissionAggregatePayload) => Promise, }; } @@ -73,6 +78,11 @@ export class AutopilotConsumer { case KAFKA_TOPICS.COMMAND: this.autopilotService.handleCommand(payload as CommandPayload); break; + case KAFKA_TOPICS.SUBMISSION_NOTIFICATION_AGGREGATE: + await this.autopilotService.handleSubmissionNotificationAggregate( + payload as SubmissionAggregatePayload, + ); + break; default: throw new Error(`Unexpected topic: ${topic as string}`); } diff --git a/src/kafka/types/topic-payload-map.type.ts b/src/kafka/types/topic-payload-map.type.ts index 2678e6e..b30ea18 100644 --- a/src/kafka/types/topic-payload-map.type.ts +++ b/src/kafka/types/topic-payload-map.type.ts @@ -3,6 +3,7 @@ import { ChallengeUpdatePayload, CommandPayload, PhaseTransitionPayload, + SubmissionAggregatePayload, } from 'src/autopilot/interfaces/autopilot.interface'; export type TopicPayloadMap = { @@ -11,4 +12,5 @@ export type TopicPayloadMap = { [KAFKA_TOPICS.CHALLENGE_CREATED]: ChallengeUpdatePayload; [KAFKA_TOPICS.CHALLENGE_UPDATED]: ChallengeUpdatePayload; [KAFKA_TOPICS.COMMAND]: CommandPayload; + [KAFKA_TOPICS.SUBMISSION_NOTIFICATION_AGGREGATE]: SubmissionAggregatePayload; }; From 37df54a148d15ccf3d35ea81ddd05e408349c3e4 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 09:30:17 +1000 Subject: [PATCH 12/23] Use bullmq for scheduling instead of native, due to overflow and lack of backup for restarts --- docs/SCHEDULER.md | 36 +- package.json | 3 +- pnpm-lock.yaml | 183 ++++++++++ .../services/autopilot-db-logger.service.ts | 2 +- .../services/autopilot.service.spec.ts | 25 +- src/autopilot/services/autopilot.service.ts | 96 ++++-- src/autopilot/services/scheduler.service.ts | 315 +++++++++++------- src/challenge/challenge-api.service.ts | 8 +- src/config/validation.ts | 15 +- src/kafka/consumers/autopilot.consumer.ts | 4 +- src/recovery/recovery.service.ts | 2 +- src/resources/resources.service.ts | 5 +- src/review/review.service.ts | 7 +- src/sync/sync.service.ts | 9 +- 14 files changed, 516 insertions(+), 194 deletions(-) diff --git a/docs/SCHEDULER.md b/docs/SCHEDULER.md index 6c572b3..c96c209 100644 --- a/docs/SCHEDULER.md +++ b/docs/SCHEDULER.md @@ -34,22 +34,14 @@ The Autopilot Scheduler is an event-based scheduling system that automatically t ### 1. Event-Based Scheduling Mechanism -The system uses NestJS's `@nestjs/schedule` module with `SchedulerRegistry` for dynamic job management: - -```typescript -// Key Dependencies Added -"@nestjs/schedule": "^6.0.0" - -// Module Configuration -ScheduleModule.forRoot() // Added to AppModule -``` +The system uses BullMQ (Redis-backed queues) for dynamic job management and durable delayed execution. #### Core Scheduling Features -- **Dynamic Job Registration**: Jobs are created and registered at runtime based on phase end times -- **Unique Job Identification**: Each job uses format `{projectId}:{phaseId}` -- **Automatic Cleanup**: Jobs are automatically removed from the registry after execution or cancellation. -- **Timeout-Based Execution**: Uses `setTimeout` for precise, one-time execution, which is ideal for phase deadlines. +- **Dynamic Job Registration**: Jobs are queued at runtime based on phase end times +- **Unique Job Identification**: Each BullMQ job uses format `{challengeId}:{phaseId}` +- **Automatic Cleanup**: Jobs are automatically removed from the queue after execution or cancellation +- **Durable Delays**: Redis stores the delay, so jobs survive restarts and are not subject to Node.js timer limits ### 2. Event Generation @@ -180,9 +172,9 @@ Cancels a scheduled phase transition. - `projectId`: Challenge project ID - `phaseId`: Phase ID to cancel -- **Returns:** `true` if cancelled successfully +- **Returns:** `Promise` resolving to `true` if cancelled successfully (false when the job has already run) -#### reschedulePhaseTransition(projectId: number, newPhaseData: PhaseTransitionPayload): string +#### reschedulePhaseTransition(projectId: number, newPhaseData: PhaseTransitionPayload): Promise Updates an existing schedule with new timing information. @@ -190,17 +182,17 @@ Updates an existing schedule with new timing information. - `projectId`: Challenge project ID - `newPhaseData`: Updated phase information -- **Returns:** New job ID +- **Returns:** `Promise` resolving to the new BullMQ job ID ### SchedulerService -#### schedulePhaseTransition(phaseData: PhaseTransitionPayload) +#### schedulePhaseTransition(phaseData: PhaseTransitionPayload): Promise -Low-level job scheduling using NestJS SchedulerRegistry. +Queues a delayed phase transition using BullMQ (backed by Redis) and returns the job ID once scheduled. -#### cancelScheduledTransition(jobId: string): boolean +#### cancelScheduledTransition(jobId: string): Promise -Removes a scheduled job by ID. +Removes a scheduled BullMQ job by ID. #### getAllScheduledTransitions(): string[] @@ -226,7 +218,7 @@ const phaseData = { date: '2025-06-20T23:59:59Z' }; -const jobId = autopilotService.schedulePhaseTransition(phaseData); +const jobId = await autopilotService.schedulePhaseTransition(phaseData); // Job scheduled, will trigger automatically at specified time ``` @@ -239,7 +231,7 @@ const updatedData = { date: '2025-06-21T23:59:59Z' // New end time }; -const newJobId = autopilotService.reschedulePhaseTransition(101, updatedData); +const newJobId = await autopilotService.reschedulePhaseTransition(101, updatedData); // Old job cancelled, new job scheduled with updated time ``` diff --git a/package.json b/package.json index 25f107f..c81d444 100644 --- a/package.json +++ b/package.json @@ -41,11 +41,12 @@ "@prisma/client": "^6.4.1", "@types/express": "^5.0.0", "axios": "^1.9.0", + "bullmq": "^5.58.8", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", "joi": "^17.13.3", - "node-rdkafka": "^2.16.0", "nest-winston": "^1.10.2", + "node-rdkafka": "^2.16.0", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "prisma": "^6.4.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5a60eab..beea867 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: axios: specifier: ^1.9.0 version: 1.10.0 + bullmq: + specifier: ^5.58.8 + version: 5.58.8 class-transformer: specifier: ^0.5.1 version: 0.5.1 @@ -583,6 +586,9 @@ packages: '@types/node': optional: true + '@ioredis/commands@1.4.0': + resolution: {integrity: sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==} + '@isaacs/balanced-match@4.0.1': resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==} engines: {node: 20 || >=22} @@ -695,6 +701,36 @@ packages: '@microsoft/tsdoc@0.15.1': resolution: {integrity: sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw==} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] + '@napi-rs/nice-android-arm-eabi@1.0.4': resolution: {integrity: sha512-OZFMYUkih4g6HCKTjqJHhMUlgvPiDuSLZPbPBWHLjKmFTv74COzRlq/gwHtmEVaR39mJQ6ZyttDl2HNMUbLVoA==} engines: {node: '>= 10'} @@ -1672,6 +1708,9 @@ packages: buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bullmq@5.58.8: + resolution: {integrity: sha512-j7h2JlWs7rOzsLePKtNK+zLOyrH6PRurLLZ6SriSpt9w5fHR128IFSd4gHGwYUb41ycnWmbLDcggp64zNW/p/Q==} + busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} engines: {node: '>=10.16.0'} @@ -1790,6 +1829,10 @@ packages: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -1909,6 +1952,10 @@ packages: create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + cron-parser@4.9.0: + resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} + engines: {node: '>=12.0.0'} + cron@4.3.0: resolution: {integrity: sha512-ciiYNLfSlF9MrDqnbMdRWFiA6oizSF7kA1osPP9lRzNu0Uu+AWog1UKy7SkckiDY2irrNjeO6qLyKnXC8oxmrw==} engines: {node: '>=18.x'} @@ -1967,6 +2014,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -1974,6 +2025,10 @@ packages: destr@2.0.5: resolution: {integrity: sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==} + detect-libc@2.1.1: + resolution: {integrity: sha512-ecqj/sy1jcK1uWrwpR67UhYrIFQ+5WlGxth34WquCbamhFA6hkkwiu37o6J5xCHdo1oixJRfVRw+ywV+Hq/0Aw==} + engines: {node: '>=8'} + detect-newline@3.1.0: resolution: {integrity: sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==} engines: {node: '>=8'} @@ -2520,6 +2575,10 @@ packages: inspect-with-kind@1.0.5: resolution: {integrity: sha512-MAQUJuIo7Xqk8EVNP+6d3CKq9c80hi4tjIbIAT6lmGW9W6WzlHiu9PS8uSuUYU+Do+j1baiFp3H25XEVxDIG2g==} + ioredis@5.8.0: + resolution: {integrity: sha512-AUXbKn9gvo9hHKvk6LbZJQSKn/qIfkWXrnsyL9Yrf+oeXmla9Nmf6XEumOddyhM8neynpK5oAV6r9r99KBuwzA==} + engines: {node: '>=12.22.0'} + ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -2857,9 +2916,15 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + lodash.includes@4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.isboolean@3.0.3: resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} @@ -3025,6 +3090,13 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@1.11.5: + resolution: {integrity: sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==} + multer@2.0.1: resolution: {integrity: sha512-Ug8bXeTIUlxurg8xLTEskKShvcKDZALo1THEX5E41pYCD2sCVub5/kIRIGqWNoqV6szyLyQKV6mD4QUrWE5GCQ==} engines: {node: '>= 10.16.0'} @@ -3061,6 +3133,10 @@ packages: node-fetch-native@1.6.7: resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -3343,6 +3419,14 @@ packages: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + reflect-metadata@0.2.2: resolution: {integrity: sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==} @@ -3542,6 +3626,9 @@ packages: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} @@ -3884,6 +3971,10 @@ packages: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} engines: {node: '>= 0.4.0'} + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} hasBin: true @@ -4465,6 +4556,8 @@ snapshots: optionalDependencies: '@types/node': 22.16.4 + '@ioredis/commands@1.4.0': {} + '@isaacs/balanced-match@4.0.1': {} '@isaacs/brace-expansion@5.0.0': @@ -4680,6 +4773,24 @@ snapshots: '@microsoft/tsdoc@0.15.1': {} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + '@napi-rs/nice-android-arm-eabi@1.0.4': optional: true @@ -5740,6 +5851,18 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bullmq@5.58.8: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.8.0 + msgpackr: 1.11.5 + node-abort-controller: 3.1.1 + semver: 7.7.2 + tslib: 2.8.1 + uuid: 11.1.0 + transitivePeerDependencies: + - supports-color + busboy@1.6.0: dependencies: streamsearch: 1.1.0 @@ -5860,6 +5983,8 @@ snapshots: clone@1.0.4: {} + cluster-key-slot@1.1.2: {} + co@4.6.0: {} collect-v8-coverage@1.0.2: {} @@ -5977,6 +6102,10 @@ snapshots: create-require@1.1.1: {} + cron-parser@4.9.0: + dependencies: + luxon: 3.6.1 + cron@4.3.0: dependencies: '@types/luxon': 3.6.2 @@ -6018,10 +6147,15 @@ snapshots: delayed-stream@1.0.0: {} + denque@2.1.0: {} + depd@2.0.0: {} destr@2.0.5: {} + detect-libc@2.1.1: + optional: true + detect-newline@3.1.0: {} dezalgo@1.0.4: @@ -6616,6 +6750,20 @@ snapshots: dependencies: kind-of: 6.0.3 + ioredis@5.8.0: + dependencies: + '@ioredis/commands': 1.4.0 + cluster-key-slot: 1.1.2 + debug: 4.4.1(supports-color@5.5.0) + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ipaddr.js@1.9.1: {} is-arrayish@0.2.1: {} @@ -7125,8 +7273,12 @@ snapshots: dependencies: p-locate: 5.0.0 + lodash.defaults@4.2.0: {} + lodash.includes@4.3.0: {} + lodash.isarguments@3.1.0: {} + lodash.isboolean@3.0.3: {} lodash.isinteger@4.0.4: {} @@ -7254,6 +7406,22 @@ snapshots: ms@2.1.3: {} + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@1.11.5: + optionalDependencies: + msgpackr-extract: 3.0.3 + multer@2.0.1: dependencies: append-field: 1.0.0 @@ -7288,6 +7456,11 @@ snapshots: node-fetch-native@1.6.7: {} + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.1 + optional: true + node-int64@0.4.0: {} node-rdkafka@2.18.0: @@ -7553,6 +7726,12 @@ snapshots: readdirp@4.1.2: {} + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + reflect-metadata@0.2.2: {} repeat-string@1.6.1: {} @@ -7760,6 +7939,8 @@ snapshots: dependencies: escape-string-regexp: 2.0.0 + standard-as-callback@2.1.0: {} + statuses@2.0.1: {} statuses@2.0.2: {} @@ -8099,6 +8280,8 @@ snapshots: utils-merge@1.0.1: {} + uuid@11.1.0: {} + uuid@9.0.1: {} v8-compile-cache-lib@3.0.1: {} diff --git a/src/autopilot/services/autopilot-db-logger.service.ts b/src/autopilot/services/autopilot-db-logger.service.ts index 3067b47..71d0166 100644 --- a/src/autopilot/services/autopilot-db-logger.service.ts +++ b/src/autopilot/services/autopilot-db-logger.service.ts @@ -103,7 +103,7 @@ export class AutopilotDbLoggerService { private async initializeSchema(): Promise { try { await this.prisma.$executeRaw( - Prisma.sql`CREATE SCHEMA IF NOT EXISTS "autopilot"` + Prisma.sql`CREATE SCHEMA IF NOT EXISTS "autopilot"`, ); await this.prisma.$executeRaw( diff --git a/src/autopilot/services/autopilot.service.spec.ts b/src/autopilot/services/autopilot.service.spec.ts index a580952..e3542c4 100644 --- a/src/autopilot/services/autopilot.service.spec.ts +++ b/src/autopilot/services/autopilot.service.spec.ts @@ -4,7 +4,10 @@ import { ChallengeApiService } from '../../challenge/challenge-api.service'; import { PhaseReviewService } from './phase-review.service'; import { ReviewAssignmentService } from './review-assignment.service'; import { SubmissionAggregatePayload } from '../interfaces/autopilot.interface'; -import { IChallenge, IPhase } from '../../challenge/interfaces/challenge.interface'; +import { + IChallenge, + IPhase, +} from '../../challenge/interfaces/challenge.interface'; describe('AutopilotService - handleSubmissionNotificationAggregate', () => { const isoNow = new Date().toISOString(); @@ -25,7 +28,9 @@ describe('AutopilotService - handleSubmissionNotificationAggregate', () => { ...overrides, }); - const createChallenge = (overrides: Partial = {}): IChallenge => ({ + const createChallenge = ( + overrides: Partial = {}, + ): IChallenge => ({ id: 'challenge-123', name: 'Test Challenge', description: null, @@ -90,8 +95,8 @@ describe('AutopilotService - handleSubmissionNotificationAggregate', () => { beforeEach(() => { schedulerService = { setPhaseChainCallback: jest.fn(), - schedulePhaseTransition: jest.fn(), - cancelScheduledTransition: jest.fn(), + schedulePhaseTransition: jest.fn().mockResolvedValue('job-id'), + cancelScheduledTransition: jest.fn().mockResolvedValue(true), getScheduledTransition: jest.fn(), }; @@ -156,7 +161,9 @@ describe('AutopilotService - handleSubmissionNotificationAggregate', () => { createChallenge({ type: 'Design Challenge' }), ); - await autopilotService.handleSubmissionNotificationAggregate(createPayload()); + await autopilotService.handleSubmissionNotificationAggregate( + createPayload(), + ); expect(challengeApiService.advancePhase).not.toHaveBeenCalled(); }); @@ -167,7 +174,9 @@ describe('AutopilotService - handleSubmissionNotificationAggregate', () => { createChallenge({ phases: [iterativeReviewPhase] }), ); - await autopilotService.handleSubmissionNotificationAggregate(createPayload()); + await autopilotService.handleSubmissionNotificationAggregate( + createPayload(), + ); expect(challengeApiService.advancePhase).not.toHaveBeenCalled(); }); @@ -179,7 +188,9 @@ describe('AutopilotService - handleSubmissionNotificationAggregate', () => { }), ); - await autopilotService.handleSubmissionNotificationAggregate(createPayload()); + await autopilotService.handleSubmissionNotificationAggregate( + createPayload(), + ); expect(challengeApiService.advancePhase).not.toHaveBeenCalled(); }); diff --git a/src/autopilot/services/autopilot.service.ts b/src/autopilot/services/autopilot.service.ts index 6ee02d0..43b28c1 100644 --- a/src/autopilot/services/autopilot.service.ts +++ b/src/autopilot/services/autopilot.service.ts @@ -56,7 +56,9 @@ export class AutopilotService { return (type ?? '').toLowerCase() === FIRST2FINISH_TYPE; } - schedulePhaseTransition(phaseData: PhaseTransitionPayload): string { + async schedulePhaseTransition( + phaseData: PhaseTransitionPayload, + ): Promise { try { const phaseKey = `${phaseData.challengeId}:${phaseData.phaseId}`; @@ -65,11 +67,18 @@ export class AutopilotService { this.logger.log( `Canceling existing schedule for phase ${phaseKey} before rescheduling.`, ); - this.schedulerService.cancelScheduledTransition(existingJobId); + const canceled = + await this.schedulerService.cancelScheduledTransition(existingJobId); + if (!canceled) { + this.logger.warn( + `Failed to cancel existing schedule ${existingJobId} for phase ${phaseKey}`, + ); + } this.activeSchedules.delete(phaseKey); } - const jobId = this.schedulerService.schedulePhaseTransition(phaseData); + const jobId = + await this.schedulerService.schedulePhaseTransition(phaseData); this.activeSchedules.set(phaseKey, jobId); this.logger.log( @@ -86,7 +95,10 @@ export class AutopilotService { } } - cancelPhaseTransition(challengeId: string, phaseId: string): boolean { + async cancelPhaseTransition( + challengeId: string, + phaseId: string, + ): Promise { const phaseKey = `${challengeId}:${phaseId}`; const jobId = this.activeSchedules.get(phaseKey); @@ -95,19 +107,25 @@ export class AutopilotService { return false; } - const canceled = this.schedulerService.cancelScheduledTransition(jobId); + const canceled = + await this.schedulerService.cancelScheduledTransition(jobId); if (canceled) { this.activeSchedules.delete(phaseKey); this.logger.log(`Canceled scheduled transition for phase ${phaseKey}`); + return true; } - return canceled; + this.logger.warn( + `Unable to cancel scheduled transition for phase ${phaseKey}; job may have already executed. Removing stale reference.`, + ); + this.activeSchedules.delete(phaseKey); + return false; } - reschedulePhaseTransition( + async reschedulePhaseTransition( challengeId: string, newPhaseData: PhaseTransitionPayload, - ): string { + ): Promise { const phaseKey = `${challengeId}:${newPhaseData.phaseId}`; const existingJobId = this.activeSchedules.get(phaseKey); let wasRescheduled = false; @@ -138,7 +156,7 @@ export class AutopilotService { } } - const newJobId = this.schedulePhaseTransition(newPhaseData); + const newJobId = await this.schedulePhaseTransition(newPhaseData); if (wasRescheduled) { this.logger.log( @@ -187,7 +205,7 @@ export class AutopilotService { ); // Clean up the scheduled job after closing the phase - const canceled = this.cancelPhaseTransition( + const canceled = await this.cancelPhaseTransition( message.challengeId, message.phaseId, ); @@ -276,7 +294,7 @@ export class AutopilotService { date: scheduleDate, }; - this.schedulePhaseTransition(phaseData); + await this.schedulePhaseTransition(phaseData); scheduledSummaries.push( `${nextPhase.name} (${nextPhase.id}) -> ${state} @ ${scheduleDate}`, ); @@ -368,7 +386,7 @@ export class AutopilotService { state, }; - this.reschedulePhaseTransition(challengeDetails.id, payload); + await this.reschedulePhaseTransition(challengeDetails.id, payload); rescheduledSummaries.push( `${nextPhase.name} (${nextPhase.id}) -> ${state} @ ${scheduleDate}`, ); @@ -419,9 +437,8 @@ export class AutopilotService { } try { - const challenge = await this.challengeApiService.getChallengeById( - challengeId, - ); + const challenge = + await this.challengeApiService.getChallengeById(challengeId); if (!this.isFirst2FinishChallenge(challenge.type)) { this.logger.debug( @@ -493,7 +510,7 @@ export class AutopilotService { } } - handleCommand(message: CommandPayload): void { + async handleCommand(message: CommandPayload): Promise { const { command, operator, projectId, date, phaseId } = message; this.logger.log(`[COMMAND RECEIVED] ${command} from ${operator}`); @@ -516,16 +533,7 @@ export class AutopilotService { ); return; } - const canceled = this.cancelPhaseTransition(challengeId, phaseId); - if (canceled) { - this.logger.log( - `Canceled scheduled transition for phase ${challengeId}:${phaseId}`, - ); - } else { - this.logger.warn( - `No active schedule found for phase ${challengeId}:${phaseId}`, - ); - } + await this.handleSinglePhaseCancellation(challengeId, phaseId); } else { const challengeId = message.challengeId; if (!challengeId) { @@ -534,12 +542,7 @@ export class AutopilotService { ); return; } - for (const key of this.activeSchedules.keys()) { - if (key.startsWith(`${challengeId}:`)) { - const phaseIdFromKey = key.split(':')[1]; - this.cancelPhaseTransition(challengeId, phaseIdFromKey); - } - } + await this.cancelAllPhasesForChallenge(challengeId); } break; @@ -588,7 +591,10 @@ export class AutopilotService { date, }; - this.reschedulePhaseTransition(challengeDetails.id, payload); + await this.reschedulePhaseTransition( + challengeDetails.id, + payload, + ); } catch (error) { const err = error as Error; this.logger.error( @@ -609,6 +615,28 @@ export class AutopilotService { } } + private async handleSinglePhaseCancellation( + challengeId: string, + phaseId: string, + ): Promise { + await this.cancelPhaseTransition(challengeId, phaseId); + } + + private async cancelAllPhasesForChallenge( + challengeId: string, + ): Promise { + const phaseKeys = Array.from(this.activeSchedules.keys()).filter((key) => + key.startsWith(`${challengeId}:`), + ); + + for (const key of phaseKeys) { + const [, phaseId] = key.split(':'); + if (phaseId) { + await this.handleSinglePhaseCancellation(challengeId, phaseId); + } + } + } + /** * Find the phases that should be scheduled based on current phase state. * Similar logic to PhaseAdvancer.js - ensure every phase that needs attention is handled. @@ -858,7 +886,7 @@ export class AutopilotService { date: updatedPhase.scheduledEndDate, }; - const jobId = this.schedulePhaseTransition(nextPhaseData); + const jobId = await this.schedulePhaseTransition(nextPhaseData); this.logger.log( `[PHASE CHAIN] Scheduled opened phase ${updatedPhase.name} (${updatedPhase.id}) for closure at ${updatedPhase.scheduledEndDate} with job ID: ${jobId}`, ); diff --git a/src/autopilot/services/scheduler.service.ts b/src/autopilot/services/scheduler.service.ts index ef6ce7e..6a13c5c 100644 --- a/src/autopilot/services/scheduler.service.ts +++ b/src/autopilot/services/scheduler.service.ts @@ -1,5 +1,9 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { SchedulerRegistry } from '@nestjs/schedule'; +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; import { KafkaService } from '../../kafka/kafka.service'; import { ChallengeApiService } from '../../challenge/challenge-api.service'; import { PhaseReviewService } from './phase-review.service'; @@ -10,9 +14,12 @@ import { AutopilotOperator, } from '../interfaces/autopilot.interface'; import { KAFKA_TOPICS } from '../../kafka/constants/topics'; +import { Job, Queue, RedisOptions, Worker } from 'bullmq'; + +const PHASE_QUEUE_NAME = 'autopilot-phase-transitions'; @Injectable() -export class SchedulerService { +export class SchedulerService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(SchedulerService.name); private scheduledJobs = new Map(); private phaseChainCallback: @@ -23,20 +30,118 @@ export class SchedulerService { nextPhases: any[], ) => Promise | void) | null = null; - private finalizationRetryJobs = new Map(); + private finalizationRetryTimers = new Map(); private finalizationAttempts = new Map(); private readonly finalizationRetryBaseDelayMs = 60_000; private readonly finalizationRetryMaxAttempts = 10; private readonly finalizationRetryMaxDelayMs = 10 * 60 * 1000; + private redisConnection?: RedisOptions; + private phaseQueue?: Queue; + private phaseQueueWorker?: Worker; + private initializationPromise?: Promise; + constructor( - private schedulerRegistry: SchedulerRegistry, private readonly kafkaService: KafkaService, private readonly challengeApiService: ChallengeApiService, private readonly phaseReviewService: PhaseReviewService, private readonly challengeCompletionService: ChallengeCompletionService, ) {} + private async ensureInitialized(): Promise { + if (!this.initializationPromise) { + this.initializationPromise = this.initializeBullQueue(); + } + await this.initializationPromise; + } + + private async initializeBullQueue(): Promise { + const redisUrl = process.env.REDIS_URL || 'redis://127.0.0.1:6379'; + this.redisConnection = { url: redisUrl }; + + this.phaseQueue = new Queue(PHASE_QUEUE_NAME, { + connection: this.redisConnection, + }); + + this.phaseQueueWorker = new Worker( + PHASE_QUEUE_NAME, + async (job) => await this.handlePhaseTransitionJob(job), + { + connection: this.redisConnection, + concurrency: 1, + }, + ); + + this.phaseQueueWorker.on('failed', (job, error) => { + if (!job) { + return; + } + this.logger.error( + `BullMQ job ${job.id} failed for challenge ${job.data.challengeId}, phase ${job.data.phaseId}: ${error.message}`, + error.stack, + ); + }); + + this.phaseQueueWorker.on('error', (error) => { + this.logger.error(`BullMQ worker error: ${error.message}`, error.stack); + }); + + await this.phaseQueueWorker.waitUntilReady(); + this.logger.log( + `BullMQ scheduler initialized using ${redisUrl} for queue ${PHASE_QUEUE_NAME}`, + ); + } + + private async handlePhaseTransitionJob( + job: Job, + ): Promise { + await this.runScheduledTransition(String(job.id), job.data); + } + + async onModuleInit(): Promise { + if (!this.initializationPromise) { + this.initializationPromise = this.initializeBullQueue(); + } + + try { + await this.initializationPromise; + } catch (error) { + const err = error as Error; + this.logger.error( + `Failed to initialize BullMQ scheduler: ${err.message}`, + err.stack, + ); + throw error; + } + } + + async onModuleDestroy(): Promise { + const disposables: Promise[] = []; + + if (this.phaseQueueWorker) { + disposables.push(this.phaseQueueWorker.close()); + } + + if (this.phaseQueue) { + disposables.push(this.phaseQueue.close()); + } + + try { + await Promise.all(disposables); + } catch (error) { + const err = error as Error; + this.logger.error( + `Error while shutting down BullMQ resources: ${err.message}`, + err.stack, + ); + } + + for (const timeout of this.finalizationRetryTimers.values()) { + clearTimeout(timeout); + } + this.finalizationRetryTimers.clear(); + } + setPhaseChainCallback( callback: ( challengeId: string, @@ -48,7 +153,9 @@ export class SchedulerService { this.phaseChainCallback = callback; } - schedulePhaseTransition(phaseData: PhaseTransitionPayload): string { + async schedulePhaseTransition( + phaseData: PhaseTransitionPayload, + ): Promise { const { challengeId, phaseId, date: endTime } = phaseData; const jobId = `${challengeId}:${phaseId}`; @@ -62,69 +169,24 @@ export class SchedulerService { } try { - const timeoutDuration = new Date(endTime).getTime() - Date.now(); - - // Corrected: Ensure the timeout is never negative to prevent the TimeoutNegativeWarning. - // If the time is in the past, it will execute on the next tick (timeout of 0). - const timeout = setTimeout( - () => { - void (async () => { - try { - // Before triggering the event, check if the phase still needs the transition - const phaseDetails = - await this.challengeApiService.getPhaseDetails( - phaseData.challengeId, - phaseData.phaseId, - ); - - if (!phaseDetails) { - this.logger.warn( - `Phase ${phaseData.phaseId} not found in challenge ${phaseData.challengeId}, skipping scheduled transition`, - ); - return; - } + await this.ensureInitialized(); - // Check if the phase is in the expected state for the transition - if (phaseData.state === 'END' && !phaseDetails.isOpen) { - this.logger.warn( - `Scheduled END transition for phase ${phaseData.phaseId} but it's already closed, skipping`, - ); - return; - } - - if (phaseData.state === 'START' && phaseDetails.isOpen) { - this.logger.warn( - `Scheduled START transition for phase ${phaseData.phaseId} but it's already open, skipping`, - ); - return; - } - - await this.triggerKafkaEvent(phaseData); + const delayMs = Math.max(0, new Date(endTime).getTime() - Date.now()); + if (!this.phaseQueue) { + throw new Error('Phase queue not initialized'); + } - // Call advancePhase method when phase transition is triggered - await this.advancePhase(phaseData); - } catch (e) { - this.logger.error( - `Failed to trigger Kafka event for job ${jobId}`, - e, - ); - } finally { - if (this.schedulerRegistry.doesExist('timeout', jobId)) { - this.schedulerRegistry.deleteTimeout(jobId); - this.logger.log( - `Removed job for phase ${jobId} from registry after execution`, - ); - } - this.scheduledJobs.delete(jobId); - } - })(); - }, - Math.max(0, timeoutDuration), - ); + await this.phaseQueue.add('phase-transition', phaseData, { + jobId, + delay: delayMs, + removeOnComplete: true, + attempts: 1, + }); - this.schedulerRegistry.addTimeout(jobId, timeout); this.scheduledJobs.set(jobId, phaseData); - this.logger.log(`Successfully scheduled job ${jobId} for ${endTime}`); + this.logger.log( + `Successfully scheduled job ${jobId} for ${endTime} with delay ${delayMs} ms`, + ); return jobId; } catch (error) { this.logger.error( @@ -135,22 +197,77 @@ export class SchedulerService { } } - cancelScheduledTransition(jobId: string): boolean { + private async runScheduledTransition( + jobId: string, + phaseData: PhaseTransitionPayload, + ): Promise { try { - if (this.schedulerRegistry.doesExist('timeout', jobId)) { - this.schedulerRegistry.deleteTimeout(jobId); - this.scheduledJobs.delete(jobId); - this.logger.log(`Canceled scheduled transition for phase ${jobId}`); - return true; - } else { + // Before triggering the event, check if the phase still needs the transition + const phaseDetails = await this.challengeApiService.getPhaseDetails( + phaseData.challengeId, + phaseData.phaseId, + ); + + if (!phaseDetails) { this.logger.warn( - `No timeout found for phase ${jobId}, skipping cancellation`, + `Phase ${phaseData.phaseId} not found in challenge ${phaseData.challengeId}, skipping scheduled transition`, + ); + return; + } + + // Check if the phase is in the expected state for the transition + if (phaseData.state === 'END' && !phaseDetails.isOpen) { + this.logger.warn( + `Scheduled END transition for phase ${phaseData.phaseId} but it's already closed, skipping`, + ); + return; + } + + if (phaseData.state === 'START' && phaseDetails.isOpen) { + this.logger.warn( + `Scheduled START transition for phase ${phaseData.phaseId} but it's already open, skipping`, + ); + return; + } + + await this.triggerKafkaEvent(phaseData); + + // Call advancePhase method when phase transition is triggered + await this.advancePhase(phaseData); + } catch (error) { + this.logger.error( + `Failed to trigger Kafka event for job ${jobId}`, + error, + ); + } finally { + this.scheduledJobs.delete(jobId); + } + } + + async cancelScheduledTransition(jobId: string): Promise { + try { + await this.ensureInitialized(); + + if (!this.phaseQueue) { + throw new Error('Phase queue not initialized'); + } + + const job = await this.phaseQueue.getJob(jobId); + if (!job) { + this.logger.warn( + `No BullMQ job found for phase ${jobId}, skipping cancellation`, ); return false; } + + await job.remove(); + this.scheduledJobs.delete(jobId); + this.logger.log(`Canceled scheduled transition for phase ${jobId}`); + return true; } catch (error) { this.logger.error( - `Error canceling scheduled transition: ${error instanceof Error ? error.message : String(error)}`, + `Error canceling scheduled transition ${jobId}: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error.stack : undefined, ); return false; } @@ -426,11 +543,8 @@ export class SchedulerService { challengeId: string, attempt: number, ): void { - const existingJobId = this.finalizationRetryJobs.get(challengeId); - if ( - existingJobId && - this.schedulerRegistry.doesExist('timeout', existingJobId) - ) { + const existingTimer = this.finalizationRetryTimers.get(challengeId); + if (existingTimer) { this.logger.debug?.( `Finalization retry already scheduled for challenge ${challengeId}; skipping duplicate schedule.`, ); @@ -447,38 +561,15 @@ export class SchedulerService { } const delay = this.computeFinalizationDelay(attempt); - const jobId = `challenge-finalize:${challengeId}:${attempt + 1}`; - - const timeout = setTimeout(() => { - try { - if (this.schedulerRegistry.doesExist('timeout', jobId)) { - this.schedulerRegistry.deleteTimeout(jobId); - } - } catch (cleanupError) { - this.logger.error( - `Failed to clean up finalization retry job ${jobId}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`, - ); - } finally { - this.finalizationRetryJobs.delete(challengeId); - } - + const timer = setTimeout(() => { + this.finalizationRetryTimers.delete(challengeId); void this.attemptChallengeFinalization(challengeId); }, delay); - try { - this.schedulerRegistry.addTimeout(jobId, timeout); - this.finalizationRetryJobs.set(challengeId, jobId); - this.logger.log( - `Scheduled finalization retry ${attempt + 1} for challenge ${challengeId} in ${Math.round(delay / 1000)} second(s).`, - ); - } catch (error) { - const err = error as Error; - this.logger.error( - `Failed to schedule finalization retry for challenge ${challengeId}: ${err.message}`, - err.stack, - ); - clearTimeout(timeout); - } + this.finalizationRetryTimers.set(challengeId, timer); + this.logger.log( + `Scheduled finalization retry ${attempt + 1} for challenge ${challengeId} in ${Math.round(delay / 1000)} second(s).`, + ); } private computeFinalizationDelay(attempt: number): number { @@ -488,10 +579,10 @@ export class SchedulerService { } private clearFinalizationRetry(challengeId: string): void { - const jobId = this.finalizationRetryJobs.get(challengeId); - if (jobId && this.schedulerRegistry.doesExist('timeout', jobId)) { - this.schedulerRegistry.deleteTimeout(jobId); + const timer = this.finalizationRetryTimers.get(challengeId); + if (timer) { + clearTimeout(timer); } - this.finalizationRetryJobs.delete(challengeId); + this.finalizationRetryTimers.delete(challengeId); } } diff --git a/src/challenge/challenge-api.service.ts b/src/challenge/challenge-api.service.ts index cf0e479..d852b6d 100644 --- a/src/challenge/challenge-api.service.ts +++ b/src/challenge/challenge-api.service.ts @@ -87,7 +87,9 @@ export class ChallengeApiService { orderBy: { updatedAt: 'desc' }, }); - const mapped = challenges.map((challenge) => this.mapChallenge(challenge)); + const mapped = challenges.map((challenge) => + this.mapChallenge(challenge), + ); void this.dbLogger.logAction('challenge.getAllActiveChallenges', { status: 'SUCCESS', @@ -281,7 +283,9 @@ export class ChallengeApiService { return result; } - const targetPhase = challenge.phases.find((phase) => phase.id === phaseId); + const targetPhase = challenge.phases.find( + (phase) => phase.id === phaseId, + ); if (!targetPhase) { this.logger.warn( diff --git a/src/config/validation.ts b/src/config/validation.ts index bb8d9d9..a121fd0 100644 --- a/src/config/validation.ts +++ b/src/config/validation.ts @@ -13,6 +13,9 @@ export const validationSchema = Joi.object({ LOG_DIR: Joi.string().default('logs'), ENABLE_FILE_LOGGING: Joi.boolean().default(false), DB_DEBUG: Joi.boolean().default(false), + REDIS_URL: Joi.string() + .uri({ scheme: ['redis', 'rediss'] }) + .default('redis://127.0.0.1:6379'), // Kafka Configuration KAFKA_BROKERS: Joi.string().required(), @@ -43,13 +46,11 @@ export const validationSchema = Joi.object({ then: Joi.optional().default('postgresql://localhost:5432/resources'), otherwise: Joi.required(), }), - AUTOPILOT_DB_URL: Joi.string() - .uri() - .when('DB_DEBUG', { - is: true, - then: Joi.required(), - otherwise: Joi.optional(), - }), + AUTOPILOT_DB_URL: Joi.string().uri().when('DB_DEBUG', { + is: true, + then: Joi.required(), + otherwise: Joi.optional(), + }), REVIEWER_POLL_INTERVAL_MS: Joi.number() .integer() .positive() diff --git a/src/kafka/consumers/autopilot.consumer.ts b/src/kafka/consumers/autopilot.consumer.ts index 22405b5..2f98e56 100644 --- a/src/kafka/consumers/autopilot.consumer.ts +++ b/src/kafka/consumers/autopilot.consumer.ts @@ -76,7 +76,9 @@ export class AutopilotConsumer { ); break; case KAFKA_TOPICS.COMMAND: - this.autopilotService.handleCommand(payload as CommandPayload); + await this.autopilotService.handleCommand( + payload as CommandPayload, + ); break; case KAFKA_TOPICS.SUBMISSION_NOTIFICATION_AGGREGATE: await this.autopilotService.handleSubmissionNotificationAggregate( diff --git a/src/recovery/recovery.service.ts b/src/recovery/recovery.service.ts index d4bb025..d45e258 100644 --- a/src/recovery/recovery.service.ts +++ b/src/recovery/recovery.service.ts @@ -109,7 +109,7 @@ export class RecoveryService implements OnApplicationBootstrap { this.logger.log( `Scheduling phase ${phase.name} (${phase.id}) for challenge ${challenge.id} to ${state} at ${scheduleDate}`, ); - this.autopilotService.schedulePhaseTransition(phaseData); + await this.autopilotService.schedulePhaseTransition(phaseData); } } } diff --git a/src/resources/resources.service.ts b/src/resources/resources.service.ts index 7c5b968..d48c2a0 100644 --- a/src/resources/resources.service.ts +++ b/src/resources/resources.service.ts @@ -103,7 +103,10 @@ export class ResourcesService { challengeId, status: 'SUCCESS', source: ResourcesService.name, - details: { roleCount: roleNames.length, reviewerCount: reviewers.length }, + details: { + roleCount: roleNames.length, + reviewerCount: reviewers.length, + }, }); return reviewers; diff --git a/src/review/review.service.ts b/src/review/review.service.ts index edafead..cfc08ab 100644 --- a/src/review/review.service.ts +++ b/src/review/review.service.ts @@ -144,8 +144,11 @@ export class ReviewService { `; try { - const submissions = await this.prisma.$queryRaw(query); - const submissionIds = submissions.map((record) => record.id).filter(Boolean); + const submissions = + await this.prisma.$queryRaw(query); + const submissionIds = submissions + .map((record) => record.id) + .filter(Boolean); void this.dbLogger.logAction('review.getActiveSubmissionIds', { challengeId, diff --git a/src/sync/sync.service.ts b/src/sync/sync.service.ts index ccced5d..467e7ed 100644 --- a/src/sync/sync.service.ts +++ b/src/sync/sync.service.ts @@ -113,7 +113,7 @@ export class SyncService { this.logger.log( `New active phase found: ${phaseKey} (${state}). Scheduling for ${scheduleDate}...`, ); - this.autopilotService.schedulePhaseTransition(phaseData); + await this.autopilotService.schedulePhaseTransition(phaseData); added++; } else if ( scheduledJob.date && @@ -124,7 +124,7 @@ export class SyncService { this.logger.log( `Phase ${phaseKey} has updated timing or state. Rescheduling from ${scheduledJob.state} at ${scheduledJob.date} to ${state} at ${scheduleDate}...`, ); - void this.autopilotService.reschedulePhaseTransition( + await this.autopilotService.reschedulePhaseTransition( challenge.id, phaseData, ); @@ -140,7 +140,10 @@ export class SyncService { `Obsolete schedule found: ${scheduledPhaseKey}. Cancelling...`, ); const [challengeId, phaseId] = scheduledPhaseKey.split(':'); - this.autopilotService.cancelPhaseTransition(challengeId, phaseId); + await this.autopilotService.cancelPhaseTransition( + challengeId, + phaseId, + ); removed++; } } From 7289d3d422c0baa794c45b0d0d203b067dd6a898 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 10:06:15 +1000 Subject: [PATCH 13/23] Switch to prismatic kafka client and fix an error with bullmq --- .nvmrc | 2 +- Dockerfile | 8 +- package.json | 2 +- pnpm-lock.yaml | 465 ++++++++- src/autopilot/services/scheduler.service.ts | 3 + src/kafka/kafka.service.ts | 1037 +++++++------------ yarn.lock | 169 ++- 7 files changed, 988 insertions(+), 698 deletions(-) diff --git a/.nvmrc b/.nvmrc index 2edeafb..8fdd954 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -20 \ No newline at end of file +22 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 32ad319..17d1422 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # ---- Base Stage ---- -FROM node:20-bookworm AS base +FROM node:22-alpine AS base WORKDIR /usr/src/app # ---- Dependencies Stage ---- @@ -17,10 +17,10 @@ COPY prisma ./prisma RUN yarn install # ---- Build Stage ---- -FROM deps AS build +FROM pnpm AS build COPY . . -RUN yarn prisma:generate -RUN yarn build +RUN pnpm prisma:generate +RUN pnpm build # ---- Production Stage ---- FROM base AS production diff --git a/package.json b/package.json index c81d444..3ed96ca 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "class-validator": "^0.14.2", "joi": "^17.13.3", "nest-winston": "^1.10.2", - "node-rdkafka": "^2.16.0", + "@platformatic/kafka": "^1.14.0", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "prisma": "^6.4.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index beea867..873c672 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: '@nestjs/terminus': specifier: ^11.0.0 version: 11.0.0(@nestjs/axios@4.0.1(@nestjs/common@11.1.4(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(axios@1.10.0)(rxjs@7.8.2))(@nestjs/common@11.1.4(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.4)(@prisma/client@6.16.2(prisma@6.16.2(typescript@5.8.3))(typescript@5.8.3))(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@platformatic/kafka': + specifier: ^1.14.0 + version: 1.14.0 '@prisma/client': specifier: ^6.4.1 version: 6.16.2(prisma@6.16.2(typescript@5.8.3))(typescript@5.8.3) @@ -68,9 +71,6 @@ importers: nest-winston: specifier: ^1.10.2 version: 1.10.2(@nestjs/common@11.1.4(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(winston@3.17.0) - node-rdkafka: - specifier: ^2.16.0 - version: 2.18.0 passport: specifier: ^0.7.0 version: 0.7.0 @@ -212,6 +212,84 @@ packages: resolution: {integrity: sha512-QsmFuYdAyeCyg9WF/AJBhFXDUfCwmDFTEbsv5t5KPSP6slhk0GoLNZApniiFytU2siRlSxVNpve2uATyYuAYkQ==} engines: {node: ^18.19.1 || ^20.11.1 || >=22.0.0, npm: ^6.11.0 || ^7.5.6 || >=8.0.0, yarn: '>= 1.13.0'} + '@antoniomuso/lz4-napi-android-arm-eabi@2.9.0': + resolution: {integrity: sha512-aeT/9SoWq7rnmzssWuCKUPaxVt3fzE9q+xq/ZHbnUSmrm8/EhLOACMvQeCOnL0IZsmPh8EpuwIE1TZyM9iQPRA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@antoniomuso/lz4-napi-android-arm64@2.9.0': + resolution: {integrity: sha512-ibQ0qiEvmljXAM97IgOZfh+PeiSQ0Rqf2HErJlZPVm2v4GVJoB67v21v1TUydqNNV5L8bwufVoZ90nheL8X9ZA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@antoniomuso/lz4-napi-darwin-arm64@2.9.0': + resolution: {integrity: sha512-1su4K1MWa4bcWoZlHajv+luGmFDV1JwIsvjtDF+0HhUveSDPP+8A4Z34zOZidURIr08Sl7M7ViPth6ZQ9SqnAA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@antoniomuso/lz4-napi-darwin-x64@2.9.0': + resolution: {integrity: sha512-8Lnbm2MkdJtiJ/nbcRS9zRyGp3G0sG6D+Y/x1vTP8nZs3/f8tBwYNsjxCQyyXNNyHcYWwVGbk68onP/pyDljOA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@antoniomuso/lz4-napi-freebsd-x64@2.9.0': + resolution: {integrity: sha512-k04EMVOjntKDPrdR4Tf8WyNseuk9PTtSGw8WHyp4CTjoR1s+YJxtp9SMnThe5o2q0TATwk8WGYb/Howrp5OMxw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@antoniomuso/lz4-napi-linux-arm-gnueabihf@2.9.0': + resolution: {integrity: sha512-H92F8zPZmgy2r8IhCWh3qIBfLp2BQ5cp18RoDXhtGFWwkh+5gVWrZp11IVznrsdgB0QeW0VR7dAMMHg3WLOPfA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@antoniomuso/lz4-napi-linux-arm64-gnu@2.9.0': + resolution: {integrity: sha512-25crh0qs/3Rj3fMI8ulYD0DoaKsidUhMBki2aeO69ZK+F8bmQ/e2++FlgJ6f3EgMP5CNxJtnZXKhPOraQWjwAw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@antoniomuso/lz4-napi-linux-arm64-musl@2.9.0': + resolution: {integrity: sha512-eJtHp38zuLaYI0/cOV/BKcNQiXUBo4GPx53FTf0Y307yUjLsn48LNeN0vD28Ct9YrbUae3bQvMD5AD86She0ww==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@antoniomuso/lz4-napi-linux-x64-gnu@2.9.0': + resolution: {integrity: sha512-mDjS4dyjRKaZQcAP71SphkYH5r3kufB30ih/VETVu/br2toCfBk6Zr1xhL1r+L7FaVAFzF62B7h30CiqrN0Awg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@antoniomuso/lz4-napi-linux-x64-musl@2.9.0': + resolution: {integrity: sha512-pvU7Z7qjkjn17NkddBtBQ7C2iRqjtZ7WJ3Jqrjtj4XxolY3Q0HaYMvWjkWhzb9AKGZbj5y+EHYtbVoZJ2TSQhQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@antoniomuso/lz4-napi-win32-arm64-msvc@2.9.0': + resolution: {integrity: sha512-aioLlbpJl0QPEXLXhh2bzyitc3T7Jot3f1ap6WdKiRa+CIjMHXw1nxJXy07MLXif10r+qVZr86ic8dvwErgqEQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@antoniomuso/lz4-napi-win32-ia32-msvc@2.9.0': + resolution: {integrity: sha512-VaF4XMTdYb59TsPsiqnWwsNaWKHhgxF33z5p4zg4n0tp20eWozl76hn8B+aXthSs40W0W1N97QhxxV4oXGd8cg==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@antoniomuso/lz4-napi-win32-x64-msvc@2.9.0': + resolution: {integrity: sha512-wfA8ShO3eGLxJ1LDwXJo87XL2D4NkMJV1pfHPvLZpD0MWb9u8VfgS+gKK5YhT7XKjzVdeIna9jgFdn2HBnZBxA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + '@babel/code-frame@7.27.1': resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==} engines: {node: '>=6.9.0'} @@ -392,6 +470,15 @@ packages: '@dabh/diagnostics@2.0.3': resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} + '@emnapi/core@1.5.0': + resolution: {integrity: sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==} + + '@emnapi/runtime@1.5.0': + resolution: {integrity: sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==} + + '@emnapi/wasi-threads@1.1.0': + resolution: {integrity: sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==} + '@eslint-community/eslint-utils@4.7.0': resolution: {integrity: sha512-dyybb3AcajC7uha6CvhdVRJqaKyn7w2YKqKyAN37NKYgZT36w+iRb0Dymmc5qEJ549c/S31cMMSFd75bteCpCw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -831,6 +918,119 @@ packages: resolution: {integrity: sha512-Sqih1YARrmMoHlXGgI9JrrgkzxcaaEso0AH+Y7j8NHonUs+xe4iDsgC3IBIDNdzEewbNpccNN6hip+b5vmyRLw==} engines: {node: '>= 10'} + '@napi-rs/snappy-android-arm-eabi@7.3.3': + resolution: {integrity: sha512-d4vUFFzNBvazGfB/KU8MnEax6itTIgRWXodPdZDnWKHy9HwVBndpCiedQDcSNHcZNYV36rx034rpn7SAuTL2NA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/snappy-android-arm64@7.3.3': + resolution: {integrity: sha512-Uh+w18dhzjVl85MGhRnojb7OLlX2ErvMsYIunO/7l3Frvc2zQvfqsWsFJanu2dwqlE2YDooeNP84S+ywgN9sxg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/snappy-darwin-arm64@7.3.3': + resolution: {integrity: sha512-AmJn+6yOu/0V0YNHLKmRUNYkn93iv/1wtPayC7O1OHtfY6YqHQ31/MVeeRBiEYtQW9TwVZxXrDirxSB1PxRdtw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/snappy-darwin-x64@7.3.3': + resolution: {integrity: sha512-biLTXBmPjPmO7HIpv+5BaV9Gy/4+QJSUNJW8Pjx1UlWAVnocPy7um+zbvAWStZssTI5sfn/jOClrAegD4w09UA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/snappy-freebsd-x64@7.3.3': + resolution: {integrity: sha512-E3R3ewm8Mrjm0yL2TC3VgnphDsQaCPixNJqBbGiz3NTshVDhlPlOgPKF0NGYqKiKaDGdD9PKtUgOR4vagUtn7g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/snappy-linux-arm-gnueabihf@7.3.3': + resolution: {integrity: sha512-ZuNgtmk9j0KyT7TfLyEnvZJxOhbkyNR761nk04F0Q4NTHMICP28wQj0xgEsnCHUsEeA9OXrRL4R7waiLn+rOQA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/snappy-linux-arm64-gnu@7.3.3': + resolution: {integrity: sha512-KIzwtq0dAzshzpqZWjg0Q9lUx93iZN7wCCUzCdLYIQ+mvJZKM10VCdn0RcuQze1R3UJTPwpPLXQIVskNMBYyPA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/snappy-linux-arm64-musl@7.3.3': + resolution: {integrity: sha512-AAED4cQS74xPvktsyVmz5sy8vSxG/+3d7Rq2FDBZzj3Fv6v5vux6uZnECPCAqpALCdTtJ61unqpOyqO7hZCt1Q==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/snappy-linux-ppc64-gnu@7.3.3': + resolution: {integrity: sha512-pofO5eSLg8ZTBwVae4WHHwJxJGZI8NEb4r5Mppvq12J/1/Hq1HecClXmfY3A7bdT2fsS2Td+Q7CI9VdBOj2sbA==} + engines: {node: '>= 10'} + cpu: [ppc64] + os: [linux] + + '@napi-rs/snappy-linux-riscv64-gnu@7.3.3': + resolution: {integrity: sha512-OiHYdeuwj0TVBXADUmmQDQ4lL1TB+8EwmXnFgOutoDVXHaUl0CJFyXLa6tYUXe+gRY8hs1v7eb0vyE97LKY06Q==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + + '@napi-rs/snappy-linux-s390x-gnu@7.3.3': + resolution: {integrity: sha512-66QdmuV9CTq/S/xifZXlMy3PsZTviAgkqqpZ+7vPCmLtuP+nqhaeupShOFf/sIDsS0gZePazPosPTeTBbhkLHg==} + engines: {node: '>= 10'} + cpu: [s390x] + os: [linux] + + '@napi-rs/snappy-linux-x64-gnu@7.3.3': + resolution: {integrity: sha512-g6KURjOxrgb8yXDEZMuIcHkUr/7TKlDwSiydEQtMtP3n4iI4sNjkcE/WNKlR3+t9bZh1pFGAq7NFRBtouQGHpQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/snappy-linux-x64-musl@7.3.3': + resolution: {integrity: sha512-6UvOyczHknpaKjrlKKSlX3rwpOrfJwiMG6qA0NRKJFgbcCAEUxmN9A8JvW4inP46DKdQ0bekdOxwRtAhFiTDfg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/snappy-openharmony-arm64@7.3.3': + resolution: {integrity: sha512-I5mak/5rTprobf7wMCk0vFhClmWOL/QiIJM4XontysnadmP/R9hAcmuFmoMV2GaxC9MblqLA7Z++gy8ou5hJVw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [openharmony] + + '@napi-rs/snappy-wasm32-wasi@7.3.3': + resolution: {integrity: sha512-+EroeygVYo9RksOchjF206frhMkfD2PaIun3yH4Zp5j/Y0oIEgs/+VhAYx/f+zHRylQYUIdLzDRclcoepvlR8Q==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@napi-rs/snappy-win32-arm64-msvc@7.3.3': + resolution: {integrity: sha512-rxqfntBsCfzgOha/OlG8ld2hs6YSMGhpMUbFjeQLyVDbooY041fRXv3S7yk52DfO6H4QQhLT5+p7cW0mYdhyiQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/snappy-win32-ia32-msvc@7.3.3': + resolution: {integrity: sha512-joRV16DsRtqjGt0CdSpxGCkO0UlHGeTZ/GqvdscoALpRKbikR2Top4C61dxEchmOd3lSYsXutuwWWGg3Nr++WA==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/snappy-win32-x64-msvc@7.3.3': + resolution: {integrity: sha512-cEnQwcsdJyOU7HSZODWsHpKuQoSYM4jaqw/hn9pOXYbRN1+02WxYppD3fdMuKN6TOA6YG5KA5PHRNeVilNX86Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/triples@1.2.0': + resolution: {integrity: sha512-HAPjR3bnCsdXBsATpDIP5WCrw0JcACwhhrwIAQhiR46n+jm+a2F8kBsfseAuWtSyQ+H3Yebt2k43B5dy+04yMA==} + + '@napi-rs/wasm-runtime@1.0.5': + resolution: {integrity: sha512-TBr9Cf9onSAS2LQ2+QHx6XcC6h9+RIzJgbqG3++9TUZSH204AwEy5jg3BTQ0VATsyoGj4ee49tN/y6rvaOOtcg==} + '@nestjs/axios@4.0.1': resolution: {integrity: sha512-68pFJgu+/AZbWkGu65Z3r55bTsCPlgyKaV4BSG8yUAD72q1PPuyVRgUwFv6BxdnibTUHlyxm06FmYWNC+bjN7A==} peerDependencies: @@ -1019,6 +1219,9 @@ packages: resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} engines: {node: ^14.21.3 || >=16} + '@node-rs/helper@1.6.0': + resolution: {integrity: sha512-2OTh/tokcLA1qom1zuCJm2gQzaZljCCbtX1YCrwRVd/toz7KxaDRFeLTAPwhs8m9hWgzrBn5rShRm6IaZofCPw==} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1043,6 +1246,10 @@ packages: resolution: {integrity: sha512-YLT9Zo3oNPJoBjBc4q8G2mjU4tqIbf5CEOORbUUr48dCD9q3umJ3IPlVqOqDakPfd2HuwccBaqlGhN4Gmr5OWg==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + '@platformatic/kafka@1.14.0': + resolution: {integrity: sha512-7DVRU1sqYo8r9Hh5rEJaCVjc9GSdb50xGAvUwS9TMKuMY9IZEec5TRkiZ22H63bFgC/aBUhy2IfKpLpwlv8cPw==} + engines: {node: '>= 20.19.4 || >= 22.18.0 || >= 24.6.0'} + '@prisma/client@6.16.2': resolution: {integrity: sha512-E00PxBcalMfYO/TWnXobBVUai6eW/g5OsifWQsQDzJYm7yaY+IRLo7ZLsaefi0QkTpxfuhFcQ/w180i6kX3iJw==} engines: {node: '>=18.18'} @@ -1210,6 +1417,9 @@ packages: '@tsconfig/node16@1.0.4': resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + '@tybys/wasm-util@0.10.1': + resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==} + '@types/babel__core@7.20.5': resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} @@ -1660,9 +1870,6 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} - bindings@1.5.0: - resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} - bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -1973,6 +2180,15 @@ packages: supports-color: optional: true + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + decompress-response@6.0.0: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} @@ -2312,9 +2528,6 @@ packages: resolution: {integrity: sha512-ek5xNX2YBYlXhiUXui3D/BXa3LdqPmoLJ7rqEx2bKJ7EAUEfmXgW0Das7Dc6Nr9MvqaOnIqiPV0mZk/r/UpNAg==} engines: {node: '>=20'} - file-uri-to-path@1.0.0: - resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} - filelist@1.0.4: resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} @@ -2975,6 +3188,10 @@ packages: resolution: {integrity: sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==} engines: {node: '>=12'} + lz4-napi@2.9.0: + resolution: {integrity: sha512-ZOWqxBMIK5768aD20tYn5B6Pp9WPM9UG/LHk8neG9p0gC1DtjdzhTtlkxhAjvTRpmJvMtnnqLKlT+COlqAt9cQ==} + engines: {node: '>= 10'} + magic-string@0.30.17: resolution: {integrity: sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==} @@ -3087,6 +3304,9 @@ packages: engines: {node: '>=10'} hasBin: true + mnemonist@0.40.3: + resolution: {integrity: sha512-Vjyr90sJ23CKKH/qPAgUKicw/v6pRoamxIEDFOF8uSgFME7DqPRpHgRTejWVjkdGg5dXj0/NyxZHZ9bcjH+2uQ==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -3105,9 +3325,6 @@ packages: resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==} engines: {node: ^18.17.0 || >=20.5.0} - nan@2.23.0: - resolution: {integrity: sha512-1UxuyYGdoQHcGg87Lkqm3FzefucTa0NAiOcuRsDmysep3c1LVCRK2krrUDafMWtjSG04htvAmvg96+SDknOmgQ==} - natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -3140,10 +3357,6 @@ packages: node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} - node-rdkafka@2.18.0: - resolution: {integrity: sha512-jYkmO0sPvjesmzhv1WFOO4z7IMiAFpThR6/lcnFDWgSPkYL95CtcuVNo/R5PpjujmqSgS22GMkL1qvU4DTAvEQ==} - engines: {node: '>=6.0.0'} - node-releases@2.0.19: resolution: {integrity: sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==} @@ -3177,6 +3390,9 @@ packages: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} + obliterator@2.0.5: + resolution: {integrity: sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==} + ohash@2.0.11: resolution: {integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==} @@ -3514,6 +3730,9 @@ packages: resolution: {integrity: sha512-Gn/JaSk/Mt9gYubxTtSn/QCV4em9mpAPiR1rqy/Ocu19u/G9J5WWdNoUT4SiV6mFC3y6cxyFcFwdzPM3FgxGAQ==} engines: {node: '>= 10.13.0'} + scule@1.3.0: + resolution: {integrity: sha512-6FtHJEvt+pVMIB9IBY+IcCJ6Z5f1iQnytgyfKMhDKgmzYG+TeH/wx1y3l27rshSbLiSanrR9ffZDrEsmjlQF2g==} + seek-bzip@2.0.0: resolution: {integrity: sha512-SMguiTnYrhpLdk3PwfzHeotrcwi8bNV4iemL9tx9poR/yeaMYwB9VzR1w7b57DuWpuqR8n6oZboi0hj3AxZxQg==} hasBin: true @@ -3594,6 +3813,10 @@ packages: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} + snappy@7.3.3: + resolution: {integrity: sha512-UDJVCunvgblRpfTOjo/uT7pQzfrTsSICJ4yVS4aq7SsGBaUSpJwaVP15nF//jqinSLpN7boe/BqbUmtWMTQ5MQ==} + engines: {node: '>= 10'} + sort-keys-length@1.0.1: resolution: {integrity: sha512-GRbEOUqCxemTAk/b32F2xa8wDTs+Z1QHOkbhJDQTvv/6G3ZkbJ+frYWsTcc7cBB3Fu4wy4XlLCuNtJuMn7Gsvw==} engines: {node: '>=0.10.0'} @@ -4158,6 +4381,45 @@ snapshots: transitivePeerDependencies: - chokidar + '@antoniomuso/lz4-napi-android-arm-eabi@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-android-arm64@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-darwin-arm64@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-darwin-x64@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-freebsd-x64@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-linux-arm-gnueabihf@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-linux-arm64-gnu@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-linux-arm64-musl@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-linux-x64-gnu@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-linux-x64-musl@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-win32-arm64-msvc@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-win32-ia32-msvc@2.9.0': + optional: true + + '@antoniomuso/lz4-napi-win32-x64-msvc@2.9.0': + optional: true + '@babel/code-frame@7.27.1': dependencies: '@babel/helper-validator-identifier': 7.27.1 @@ -4362,6 +4624,22 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 + '@emnapi/core@1.5.0': + dependencies: + '@emnapi/wasi-threads': 1.1.0 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.5.0': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.1.0': + dependencies: + tslib: 2.8.1 + optional: true + '@eslint-community/eslint-utils@4.7.0(eslint@9.31.0(jiti@2.5.1))': dependencies: eslint: 9.31.0(jiti@2.5.1) @@ -4859,6 +5137,72 @@ snapshots: '@napi-rs/nice-win32-x64-msvc': 1.0.4 optional: true + '@napi-rs/snappy-android-arm-eabi@7.3.3': + optional: true + + '@napi-rs/snappy-android-arm64@7.3.3': + optional: true + + '@napi-rs/snappy-darwin-arm64@7.3.3': + optional: true + + '@napi-rs/snappy-darwin-x64@7.3.3': + optional: true + + '@napi-rs/snappy-freebsd-x64@7.3.3': + optional: true + + '@napi-rs/snappy-linux-arm-gnueabihf@7.3.3': + optional: true + + '@napi-rs/snappy-linux-arm64-gnu@7.3.3': + optional: true + + '@napi-rs/snappy-linux-arm64-musl@7.3.3': + optional: true + + '@napi-rs/snappy-linux-ppc64-gnu@7.3.3': + optional: true + + '@napi-rs/snappy-linux-riscv64-gnu@7.3.3': + optional: true + + '@napi-rs/snappy-linux-s390x-gnu@7.3.3': + optional: true + + '@napi-rs/snappy-linux-x64-gnu@7.3.3': + optional: true + + '@napi-rs/snappy-linux-x64-musl@7.3.3': + optional: true + + '@napi-rs/snappy-openharmony-arm64@7.3.3': + optional: true + + '@napi-rs/snappy-wasm32-wasi@7.3.3': + dependencies: + '@napi-rs/wasm-runtime': 1.0.5 + optional: true + + '@napi-rs/snappy-win32-arm64-msvc@7.3.3': + optional: true + + '@napi-rs/snappy-win32-ia32-msvc@7.3.3': + optional: true + + '@napi-rs/snappy-win32-x64-msvc@7.3.3': + optional: true + + '@napi-rs/triples@1.2.0': + optional: true + + '@napi-rs/wasm-runtime@1.0.5': + dependencies: + '@emnapi/core': 1.5.0 + '@emnapi/runtime': 1.5.0 + '@tybys/wasm-util': 0.10.1 + optional: true + '@nestjs/axios@4.0.1(@nestjs/common@11.1.4(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2))(axios@1.10.0)(rxjs@7.8.2)': dependencies: '@nestjs/common': 11.1.4(class-transformer@0.5.1)(class-validator@0.14.2)(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -5024,6 +5368,11 @@ snapshots: '@noble/hashes@1.8.0': {} + '@node-rs/helper@1.6.0': + dependencies: + '@napi-rs/triples': 1.2.0 + optional: true + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -5046,6 +5395,19 @@ snapshots: '@pkgr/core@0.2.7': {} + '@platformatic/kafka@1.14.0': + dependencies: + ajv: 8.17.1 + debug: 4.4.3 + fastq: 1.19.1 + mnemonist: 0.40.3 + scule: 1.3.0 + optionalDependencies: + lz4-napi: 2.9.0 + snappy: 7.3.3 + transitivePeerDependencies: + - supports-color + '@prisma/client@6.16.2(prisma@6.16.2(typescript@5.8.3))(typescript@5.8.3)': optionalDependencies: prisma: 6.16.2(typescript@5.8.3) @@ -5194,6 +5556,11 @@ snapshots: '@tsconfig/node16@1.0.4': {} + '@tybys/wasm-util@0.10.1': + dependencies: + tslib: 2.8.1 + optional: true + '@types/babel__core@7.20.5': dependencies: '@babel/parser': 7.28.0 @@ -5777,10 +6144,6 @@ snapshots: binary-extensions@2.3.0: {} - bindings@1.5.0: - dependencies: - file-uri-to-path: 1.0.0 - bl@4.1.0: dependencies: buffer: 5.7.1 @@ -6123,6 +6486,10 @@ snapshots: optionalDependencies: supports-color: 5.5.0 + debug@4.4.3: + dependencies: + ms: 2.1.3 + decompress-response@6.0.0: dependencies: mimic-response: 3.1.0 @@ -6475,8 +6842,6 @@ snapshots: transitivePeerDependencies: - supports-color - file-uri-to-path@1.0.0: {} - filelist@1.0.4: dependencies: minimatch: 5.1.6 @@ -7321,6 +7686,25 @@ snapshots: luxon@3.6.1: {} + lz4-napi@2.9.0: + dependencies: + '@node-rs/helper': 1.6.0 + optionalDependencies: + '@antoniomuso/lz4-napi-android-arm-eabi': 2.9.0 + '@antoniomuso/lz4-napi-android-arm64': 2.9.0 + '@antoniomuso/lz4-napi-darwin-arm64': 2.9.0 + '@antoniomuso/lz4-napi-darwin-x64': 2.9.0 + '@antoniomuso/lz4-napi-freebsd-x64': 2.9.0 + '@antoniomuso/lz4-napi-linux-arm-gnueabihf': 2.9.0 + '@antoniomuso/lz4-napi-linux-arm64-gnu': 2.9.0 + '@antoniomuso/lz4-napi-linux-arm64-musl': 2.9.0 + '@antoniomuso/lz4-napi-linux-x64-gnu': 2.9.0 + '@antoniomuso/lz4-napi-linux-x64-musl': 2.9.0 + '@antoniomuso/lz4-napi-win32-arm64-msvc': 2.9.0 + '@antoniomuso/lz4-napi-win32-ia32-msvc': 2.9.0 + '@antoniomuso/lz4-napi-win32-x64-msvc': 2.9.0 + optional: true + magic-string@0.30.17: dependencies: '@jridgewell/sourcemap-codec': 1.5.4 @@ -7404,6 +7788,10 @@ snapshots: mkdirp@1.0.4: {} + mnemonist@0.40.3: + dependencies: + obliterator: 2.0.5 + ms@2.1.3: {} msgpackr-extract@3.0.3: @@ -7434,8 +7822,6 @@ snapshots: mute-stream@2.0.0: {} - nan@2.23.0: {} - natural-compare@1.4.0: {} negotiator@1.0.0: {} @@ -7463,11 +7849,6 @@ snapshots: node-int64@0.4.0: {} - node-rdkafka@2.18.0: - dependencies: - bindings: 1.5.0 - nan: 2.23.0 - node-releases@2.0.19: {} nodemon@3.1.10: @@ -7503,6 +7884,8 @@ snapshots: object-inspect@1.13.4: {} + obliterator@2.0.5: {} + ohash@2.0.11: {} on-finished@2.4.1: @@ -7814,6 +8197,8 @@ snapshots: ajv-formats: 2.1.1(ajv@8.17.1) ajv-keywords: 5.1.0(ajv@8.17.1) + scule@1.3.0: {} + seek-bzip@2.0.0: dependencies: commander: 6.2.1 @@ -7909,6 +8294,28 @@ snapshots: slash@3.0.0: {} + snappy@7.3.3: + optionalDependencies: + '@napi-rs/snappy-android-arm-eabi': 7.3.3 + '@napi-rs/snappy-android-arm64': 7.3.3 + '@napi-rs/snappy-darwin-arm64': 7.3.3 + '@napi-rs/snappy-darwin-x64': 7.3.3 + '@napi-rs/snappy-freebsd-x64': 7.3.3 + '@napi-rs/snappy-linux-arm-gnueabihf': 7.3.3 + '@napi-rs/snappy-linux-arm64-gnu': 7.3.3 + '@napi-rs/snappy-linux-arm64-musl': 7.3.3 + '@napi-rs/snappy-linux-ppc64-gnu': 7.3.3 + '@napi-rs/snappy-linux-riscv64-gnu': 7.3.3 + '@napi-rs/snappy-linux-s390x-gnu': 7.3.3 + '@napi-rs/snappy-linux-x64-gnu': 7.3.3 + '@napi-rs/snappy-linux-x64-musl': 7.3.3 + '@napi-rs/snappy-openharmony-arm64': 7.3.3 + '@napi-rs/snappy-wasm32-wasi': 7.3.3 + '@napi-rs/snappy-win32-arm64-msvc': 7.3.3 + '@napi-rs/snappy-win32-ia32-msvc': 7.3.3 + '@napi-rs/snappy-win32-x64-msvc': 7.3.3 + optional: true + sort-keys-length@1.0.1: dependencies: sort-keys: 1.1.2 diff --git a/src/autopilot/services/scheduler.service.ts b/src/autopilot/services/scheduler.service.ts index 6a13c5c..b7819ed 100644 --- a/src/autopilot/services/scheduler.service.ts +++ b/src/autopilot/services/scheduler.service.ts @@ -17,6 +17,7 @@ import { KAFKA_TOPICS } from '../../kafka/constants/topics'; import { Job, Queue, RedisOptions, Worker } from 'bullmq'; const PHASE_QUEUE_NAME = 'autopilot-phase-transitions'; +const PHASE_QUEUE_PREFIX = '{autopilot-phase-transitions}'; @Injectable() export class SchedulerService implements OnModuleInit, OnModuleDestroy { @@ -61,6 +62,7 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { this.phaseQueue = new Queue(PHASE_QUEUE_NAME, { connection: this.redisConnection, + prefix: PHASE_QUEUE_PREFIX, }); this.phaseQueueWorker = new Worker( @@ -69,6 +71,7 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { { connection: this.redisConnection, concurrency: 1, + prefix: PHASE_QUEUE_PREFIX, }, ); diff --git a/src/kafka/kafka.service.ts b/src/kafka/kafka.service.ts index 3457436..77aedb4 100644 --- a/src/kafka/kafka.service.ts +++ b/src/kafka/kafka.service.ts @@ -4,37 +4,51 @@ import { OnModuleInit, } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import * as Kafka from 'node-rdkafka'; +import { + Consumer, + MessagesStream, + ProduceAcks, + Producer, + jsonDeserializer, + jsonSerializer, + stringDeserializer, + stringSerializer, +} from '@platformatic/kafka'; +import { v4 as uuidv4 } from 'uuid'; + import { KafkaConnectionException, - KafkaProducerException, KafkaConsumerException, + KafkaProducerException, } from '../common/exceptions/kafka.exception'; +import { CONFIG } from '../common/constants/config.constants'; import { LoggerService } from '../common/services/logger.service'; import { CircuitBreaker } from '../common/utils/circuit-breaker'; -import { v4 as uuidv4 } from 'uuid'; -import { CONFIG } from '../common/constants/config.constants'; import { IKafkaConfig } from '../common/types/kafka.types'; +type KafkaProducer = Producer; +type KafkaConsumer = Consumer; +type KafkaStream = MessagesStream; + @Injectable() export class KafkaService implements OnApplicationShutdown, OnModuleInit { - private readonly producer: Kafka.Producer; - private readonly consumers: Map; - private readonly consumerLoops: Map; - private readonly logger: LoggerService; - private readonly circuitBreaker: CircuitBreaker; - private readonly retryDelayAfterReconnectMs = 5000; + private readonly logger = new LoggerService(KafkaService.name); + private readonly circuitBreaker = new CircuitBreaker({ + failureThreshold: CONFIG.CIRCUIT_BREAKER.DEFAULT_FAILURE_THRESHOLD, + resetTimeout: CONFIG.CIRCUIT_BREAKER.DEFAULT_RESET_TIMEOUT, + }); private readonly kafkaConfig: IKafkaConfig; - private producerReady = false; - private producerConnecting?: Promise; + private readonly producer: KafkaProducer; + private readonly consumers = new Map(); + private readonly consumerStreams = new Map(); + private readonly consumerLoops = new Map>(); + private shuttingDown = false; constructor(private readonly configService: ConfigService) { - this.logger = new LoggerService(KafkaService.name); - try { - const brokersValue = this.configService.get< - string | string[] | undefined - >('kafka.brokers'); + const brokersValue = this.configService.get( + 'kafka.brokers', + ); const kafkaBrokers = Array.isArray(brokersValue) ? brokersValue : brokersValue?.split(',') || CONFIG.KAFKA.DEFAULT_BROKERS; @@ -57,31 +71,10 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { }, }; - const producerConfig: Kafka.ProducerGlobalConfig = { - 'client.id': this.kafkaConfig.clientId, - 'metadata.broker.list': this.kafkaConfig.brokers.join(','), - dr_cb: true, - 'enable.idempotence': true, - }; - - const producerTopicConfig: Kafka.ProducerTopicConfig = { - 'request.required.acks': -1, - }; - - this.producer = new Kafka.Producer(producerConfig, producerTopicConfig); - this.registerProducerEvents(); - - this.consumers = new Map(); - this.consumerLoops = new Map(); - this.circuitBreaker = new CircuitBreaker({ - failureThreshold: CONFIG.CIRCUIT_BREAKER.DEFAULT_FAILURE_THRESHOLD, - resetTimeout: CONFIG.CIRCUIT_BREAKER.DEFAULT_RESET_TIMEOUT, - }); + this.producer = this.createProducer(); } catch (error) { - const err = error as Error; - this.logger.error('Failed to initialize Kafka service', { - error: err.stack || err.message, - }); + const err = this.normalizeError(error, 'Failed to initialize Kafka service'); + this.logger.error(err.message, { error: err.stack || err.message }); throw new KafkaConnectionException({ error: err.stack || err.message, }); @@ -90,378 +83,96 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { async onModuleInit(): Promise { try { - await this.ensureProducerConnected(); + await this.producer.metadata({ topics: [] }); this.logger.info('Kafka service initialized successfully'); } catch (error) { - const err = error as Error; - this.logger.error('Failed to initialize Kafka service', { - error: err.stack || err.message, - }); + const err = this.normalizeError( + error, + 'Failed to initialize Kafka producer metadata request', + ); + this.logger.error(err.message, { error: err.stack || err.message }); throw new KafkaConnectionException({ error: err.stack || err.message, }); } } - private registerProducerEvents(): void { - this.producer.on('event.error', (err: Kafka.LibrdKafkaError) => { - this.logger.error('Kafka producer error event received', { - error: err.message, - code: err.code, - }); - }); - - this.producer.on('ready', () => { - this.logger.info('Kafka producer connected'); - }); - - this.producer.on('disconnected', () => { - this.producerReady = false; - this.producerConnecting = undefined; - this.logger.warn('Kafka producer disconnected'); - }); - } - - private async ensureProducerConnected(): Promise { - if (this.producerReady && this.producer.isConnected()) { - return; - } - - if (!this.producerConnecting) { - this.producerConnecting = new Promise((resolve, reject) => { - const cleanup = () => { - this.producer.removeListener('event.error', errorListener); - }; - - const errorListener = (err: Kafka.LibrdKafkaError) => { - cleanup(); - this.producerConnecting = undefined; - reject(new Error(err.message)); - }; - - this.producer.once('event.error', errorListener); - - try { - this.producer.connect(undefined, (err) => { - if (err) { - cleanup(); - this.producerConnecting = undefined; - reject( - this.normalizeError( - err, - 'Kafka producer connection callback error', - ), - ); - } else { - this.producerReady = true; - this.producer.setPollInterval(100); - cleanup(); - this.producerConnecting = undefined; - resolve(); - } - }); - } catch (connectError) { - cleanup(); - this.producerConnecting = undefined; - reject(connectError as Error); - } - }); - } - - await this.producerConnecting; - - if (!this.producerReady) { - throw new Error('Kafka producer not ready after connection attempt'); - } - } - - private async reconnectProducer(): Promise { - this.producerReady = false; - this.producerConnecting = undefined; - - if (this.producer.isConnected()) { - try { - await new Promise((resolve, reject) => { - this.producer.disconnect((err) => { - if (err) { - reject( - this.normalizeError(err, 'Kafka producer disconnect error'), - ); - } else { - resolve(); - } - }); - }); - } catch (error) { - const err = error as Error; - this.logger.warn('Failed to disconnect producer during reconnect', { - error: err.stack || err.message, - }); - } - } - - await this.ensureProducerConnected(); - } - - private async flushProducer(timeoutMs = 1000): Promise { - await new Promise((resolve, reject) => { - this.producer.flush(timeoutMs, (err) => { - if (err) { - reject(this.normalizeError(err, 'Kafka producer flush error')); - } else { - resolve(); - } - }); - }); - } - - private async delay(ms: number): Promise { - await new Promise((resolve) => setTimeout(resolve, ms)); - } - - private normalizeError(error: unknown, fallbackMessage: string): Error { - if (error instanceof Error) { - return error; - } - - if (typeof error === 'string') { - return new Error(`${fallbackMessage}: ${error}`); - } + async produce(topic: string, message: unknown): Promise { + const correlationId = uuidv4(); + const timestamp = Date.now(); try { - return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`); - } catch (serializationError) { - const serializationMessage = - serializationError instanceof Error - ? serializationError.message - : 'unknown serialization error'; - return new Error( - `${fallbackMessage}; failed to serialize original error: ${serializationMessage}`, + await this.circuitBreaker.execute(async () => + this.sendRecords(topic, [message], correlationId, timestamp), ); - } - } - private async sendWithReconnect( - sendAction: () => void, - metadata: { topic: string; correlationId?: string; messageCount?: number }, - ): Promise { - await this.ensureProducerConnected(); - - try { - sendAction(); - await this.flushProducer(); - return; + this.logger.info(`[KAFKA-PRODUCER] Message produced to ${topic}`, { + correlationId, + topic, + timestamp: new Date(timestamp).toISOString(), + }); } catch (error) { - const err = error as Error; - this.logger.warn('Kafka producer send failed, attempting reconnect', { - ...metadata, + const err = this.normalizeError(error, `Failed to produce message to ${topic}`); + this.logger.error(err.message, { + correlationId, error: err.stack || err.message, }); - } - - await this.reconnectProducer(); - - this.logger.info('Retrying Kafka send after reconnect delay', { - ...metadata, - delayMs: this.retryDelayAfterReconnectMs, - }); - - await this.delay(this.retryDelayAfterReconnectMs); - - try { - sendAction(); - await this.flushProducer(); - } catch (retryError) { - const retryErr = retryError as Error; - this.logger.error('Kafka producer retry failed after reconnect', { - ...metadata, - error: retryErr.stack || retryErr.message, - }); - throw retryErr; - } - } - - private buildHeaders( - correlationId: string, - timestamp: number, - ): Array<{ key: string; value: Buffer }> { - return [ - { key: 'correlation-id', value: Buffer.from(correlationId, 'utf8') }, - { key: 'timestamp', value: Buffer.from(timestamp.toString(), 'utf8') }, - { key: 'content-type', value: Buffer.from('application/json', 'utf8') }, - ]; - } - - private getHeaderValue(headers: unknown, key: string): string | undefined { - if (!Array.isArray(headers)) { - return undefined; - } - - for (const header of headers as Array<{ - key?: unknown; - value?: unknown; - }>) { - if (header?.key !== key) { - continue; - } - - const value = header.value; - if (typeof value === 'string') { - return value; - } - - if (Buffer.isBuffer(value)) { - return value.toString('utf8'); - } - } - - return undefined; - } - - async sendMessage(topic: string, message: unknown): Promise { - try { - const encodedMessage = this.encodeMessage(message); - await this.sendWithReconnect( - () => { - this.producer.produce( - topic, - null, - encodedMessage, - undefined, - Date.now(), - ); - }, - { topic }, - ); - this.logger.log(`Message sent to topic ${topic}`); - } catch (error) { - const err = error as Error; - this.logger.error( - `Failed to send message to topic ${topic}: ${err.message}`, - ); throw new KafkaProducerException( - `Failed to send message to topic ${topic}: ${err.message}`, + `Failed to produce message to ${topic}: ${err.message}`, ); } } - private encodeMessage(message: unknown): Buffer { - try { - const jsonString = JSON.stringify(message); - return Buffer.from(jsonString, 'utf8'); - } catch (error) { - const err = error as Error; - this.logger.error('Failed to encode message as JSON', { - error: err.stack || err.message, - }); - throw new Error(`Failed to encode message as JSON: ${err.message}`); - } - } - - private decodeMessage(buffer: Buffer): unknown { - try { - const jsonString = buffer.toString('utf8'); - return JSON.parse(jsonString); - } catch (error) { - const err = error as Error; - this.logger.error('Failed to decode JSON message', { - error: err.stack || err.message, - }); - throw new Error(`Failed to decode JSON message: ${err.message}`); - } - } - - async produce(topic: string, message: unknown): Promise { + async produceBatch(topic: string, messages: unknown[]): Promise { const correlationId = uuidv4(); const timestamp = Date.now(); try { - await this.circuitBreaker.execute(async () => { - const encodedValue = this.encodeMessage(message); - const headers = this.buildHeaders(correlationId, timestamp); - - await this.sendWithReconnect( - () => { - this.producer.produce( - topic, - null, - encodedValue, - undefined, - timestamp, - headers as any, - ); - }, - { topic, correlationId }, - ); - - this.logger.info(`[KAFKA-PRODUCER] Message produced to ${topic}`, { - correlationId, - topic, - timestamp: new Date(timestamp).toISOString(), - }); + await this.circuitBreaker.execute(async () => + this.sendRecords(topic, messages, correlationId, timestamp), + ); + + this.logger.info(`[KAFKA-PRODUCER] Batch produced to ${topic}`, { + correlationId, + count: messages.length, + topic, + timestamp: new Date(timestamp).toISOString(), }); } catch (error) { - const err = error as Error; - this.logger.error(`Failed to produce message to ${topic}`, { + const err = this.normalizeError( + error, + `Failed to produce batch to ${topic}`, + ); + this.logger.error(err.message, { correlationId, + topic, + count: messages.length, error: err.stack || err.message, }); throw new KafkaProducerException( - `Failed to produce message to ${topic}: ${err.message}`, + `Failed to produce batch to ${topic}: ${err.message}`, ); } } - async produceBatch(topic: string, messages: unknown[]): Promise { + async sendMessage(topic: string, message: unknown): Promise { const correlationId = uuidv4(); - const startTime = Date.now(); + const timestamp = Date.now(); try { - await this.circuitBreaker.execute(async () => { - this.logger.info(`Producing batch to ${topic}`, { - correlationId, - count: messages.length, - }); - - const timestamp = Date.now(); - const headers = this.buildHeaders(correlationId, timestamp); - - await this.sendWithReconnect( - () => { - messages.forEach((message) => { - const encoded = this.encodeMessage(message); - - this.producer.produce( - topic, - null, - encoded, - undefined, - timestamp, - headers as any, - ); - }); - }, - { - topic, - correlationId, - messageCount: messages.length, - }, - ); - this.logger.info(`Batch produced to ${topic}`, { - correlationId, - count: messages.length, - latency: Date.now() - startTime, - }); - }); + await this.sendRecords(topic, [message], correlationId, timestamp); + this.logger.log(`Message sent to topic ${topic}`); } catch (error) { - const err = error as Error; - this.logger.error(`Failed to produce batch to ${topic}`, { - correlationId, + const err = this.normalizeError( + error, + `Failed to send message to topic ${topic}`, + ); + this.logger.error(err.message, { + topic, error: err.stack || err.message, - count: messages.length, }); throw new KafkaProducerException( - `Failed to produce batch to ${topic}: ${err.message}`, + `Failed to send message to topic ${topic}: ${err.message}`, ); } } @@ -471,347 +182,367 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { topics: string[], onMessage: (message: unknown) => Promise, ): Promise { - const correlationId = uuidv4(); - try { await this.circuitBreaker.execute(async () => { - let consumer = this.consumers.get(groupId); - if (!consumer) { - consumer = this.createConsumer(groupId); - this.consumers.set(groupId, consumer); + const consumer = this.getOrCreateConsumer(groupId); + + if (this.consumerStreams.has(groupId)) { + await this.closeStream(groupId); } - await this.connectConsumer(consumer, groupId, topics); - this.startConsumerLoop(consumer, groupId, topics, onMessage); + const stream = await consumer.consume({ + topics, + autocommit: true, + }); + + this.consumerStreams.set(groupId, stream); + const loop = this.startConsumerLoop(groupId, topics, stream, onMessage); + this.consumerLoops.set(groupId, loop); }); } catch (error) { - const err = error as Error; - this.logger.error(`Failed to start consumer for group ${groupId}`, { - error: err.stack, - correlationId, + const err = this.normalizeError( + error, + `Failed to start consumer for group ${groupId}`, + ); + this.logger.error(err.message, { + groupId, topics, + error: err.stack || err.message, }); throw new KafkaConsumerException( `Failed to start consumer for group ${groupId}`, - { - error: err.stack || err.message, - }, + { error: err.stack || err.message }, ); } } - private createConsumer(groupId: string): Kafka.KafkaConsumer { - const consumerConfig: Kafka.ConsumerGlobalConfig = { - 'group.id': groupId, - 'metadata.broker.list': this.kafkaConfig.brokers.join(','), - 'client.id': `${this.kafkaConfig.clientId}-${groupId}`, - 'enable.auto.commit': true, - 'auto.commit.interval.ms': CONFIG.KAFKA.DEFAULT_AUTO_COMMIT_INTERVAL, - 'queued.min.messages': 1, - }; + async onApplicationShutdown(signal?: string): Promise { + this.logger.info('Starting Kafka graceful shutdown', { signal }); + this.shuttingDown = true; - const topicConfig: Kafka.ConsumerTopicConfig = { - 'auto.offset.reset': 'latest', - }; + try { + this.logger.info('Closing consumer streams...'); + await Promise.all( + Array.from(this.consumerStreams.keys()).map((groupId) => + this.closeStream(groupId).catch((error) => { + const err = this.normalizeError( + error, + `Failed closing stream for consumer ${groupId}`, + ); + this.logger.warn(err.message, { + groupId, + error: err.stack || err.message, + }); + }), + ), + ); + + this.logger.info('Waiting for consumer loops to finish...'); + await Promise.allSettled(this.consumerLoops.values()); + + this.logger.info('Closing Kafka consumers...'); + await Promise.all( + Array.from(this.consumers.entries()).map(async ([groupId, consumer]) => { + try { + await consumer.close(); + this.logger.info(`Consumer ${groupId} closed successfully`); + } catch (error) { + const err = this.normalizeError( + error, + `Error closing consumer ${groupId}`, + ); + this.logger.error(err.message, { + groupId, + error: err.stack || err.message, + }); + } + }), + ); + + this.logger.info('Closing Kafka producer...'); + await this.producer.close(); + this.logger.info('Kafka connections closed successfully'); + } catch (error) { + const err = this.normalizeError(error, 'Error during Kafka shutdown'); + this.logger.error(err.message, { signal, error: err.stack || err.message }); + throw err; + } finally { + this.consumerLoops.clear(); + this.consumerStreams.clear(); + this.consumers.clear(); + } + } + + async isConnected(): Promise { + try { + return ( + this.producer.isConnected() && + Array.from(this.consumers.values()).every((consumer) => consumer.isConnected()) + ); + } catch (error) { + const err = this.normalizeError( + error, + 'Failed to check Kafka connection status', + ); + this.logger.error(err.message, { + error: err.stack || err.message, + timestamp: new Date().toISOString(), + }); + return false; + } + } + + private createProducer(): KafkaProducer { + return new Producer({ + clientId: this.kafkaConfig.clientId, + bootstrapBrokers: this.kafkaConfig.brokers, + idempotent: true, + acks: ProduceAcks.ALL, + retries: this.kafkaConfig.retry.retries, + retryDelay: this.kafkaConfig.retry.initialRetryTime, + timeout: this.kafkaConfig.retry.maxRetryTime, + maxInflights: CONFIG.KAFKA.DEFAULT_MAX_IN_FLIGHT_REQUESTS, + serializers: { + key: stringSerializer, + value: jsonSerializer, + headerKey: stringSerializer, + headerValue: stringSerializer, + }, + }); + } + + private getOrCreateConsumer(groupId: string): KafkaConsumer { + const existing = this.consumers.get(groupId); + if (existing) { + return existing; + } - const consumer = new Kafka.KafkaConsumer(consumerConfig, topicConfig); + const consumer = new Consumer({ + clientId: `${this.kafkaConfig.clientId}-${groupId}`, + groupId, + bootstrapBrokers: this.kafkaConfig.brokers, + autocommit: true, + retries: this.kafkaConfig.retry.retries, + retryDelay: this.kafkaConfig.retry.initialRetryTime, + timeout: this.kafkaConfig.retry.maxRetryTime, + maxWaitTime: CONFIG.KAFKA.DEFAULT_MAX_WAIT_TIME, + maxBytes: CONFIG.KAFKA.DEFAULT_MAX_BYTES, + deserializers: { + key: stringDeserializer, + value: jsonDeserializer, + headerKey: stringDeserializer, + headerValue: stringDeserializer, + }, + }); + + consumer.on('consumer:group:rebalance', (info) => { + this.logger.info(`Kafka consumer ${groupId} rebalanced`, { info }); + }); - consumer.on('event.error', (err: Kafka.LibrdKafkaError) => { - this.logger.error(`Kafka consumer error for group ${groupId}`, { - error: err.message, - code: err.code, + consumer.on('client:broker:disconnect', (details) => { + this.logger.warn(`Kafka consumer ${groupId} disconnected from broker`, { + details, }); }); - consumer.on('disconnected', () => { - this.consumerLoops.delete(groupId); - this.logger.warn(`Kafka consumer ${groupId} disconnected`); + consumer.on('client:broker:failed', (details) => { + this.logger.error(`Kafka consumer ${groupId} broker failure`, { + details, + }); }); + this.consumers.set(groupId, consumer); return consumer; } - private async connectConsumer( - consumer: Kafka.KafkaConsumer, + private async startConsumerLoop( groupId: string, topics: string[], + stream: KafkaStream, + onMessage: (message: unknown) => Promise, ): Promise { - if (consumer.isConnected()) { - consumer.unsubscribe(); - consumer.subscribe(topics); - this.logger.info(`Kafka consumer ${groupId} re-subscribed`, { topics }); - return; - } + try { + for await (const message of stream) { + const correlationId = + this.getHeaderValue(message.headers, 'correlation-id') || uuidv4(); + const messageTimestamp = Number(message.timestamp ?? BigInt(Date.now())); - await new Promise((resolve, reject) => { - const onReady = () => { try { - consumer.subscribe(topics); - this.logger.info(`Kafka consumer ${groupId} connected`, { topics }); - resolve(); - } catch (subscribeError) { - reject(subscribeError as Error); - } finally { - cleanup(); - } - }; - - const onError = (err: Kafka.LibrdKafkaError) => { - cleanup(); - reject(new Error(err.message)); - }; + if (message.value === undefined) { + throw new Error('Message value is undefined'); + } - const cleanup = () => { - consumer.removeListener('ready', onReady); - consumer.removeListener('event.error', onError); - }; + this.logger.info( + `[KAFKA-CONSUMER] Starting to process message from ${message.topic}`, + { + correlationId, + topic: message.topic, + partition: message.partition, + timestamp: new Date(messageTimestamp).toISOString(), + }, + ); - consumer.once('ready', onReady); - consumer.once('event.error', onError); + await onMessage(message.value); - try { - consumer.connect(); - } catch (error) { - cleanup(); - reject(error as Error); + this.logger.info( + `[KAFKA-CONSUMER] Completed processing message from ${message.topic}`, + { + correlationId, + topic: message.topic, + partition: message.partition, + timestamp: new Date().toISOString(), + }, + ); + } catch (processingError) { + const err = this.normalizeError( + processingError, + `Error processing message from topic ${message.topic}`, + ); + this.logger.error(err.message, { + correlationId, + topic: message.topic, + partition: message.partition, + error: err.stack || err.message, + }); + await this.sendToDLQ(message.topic, message.value).catch((dlqError) => { + const dlqErr = this.normalizeError( + dlqError, + `Failed to send message to DLQ for topic ${message.topic}`, + ); + this.logger.error(dlqErr.message, { + correlationId, + topic: message.topic, + error: dlqErr.stack || dlqErr.message, + }); + }); + } } - }); + } catch (error) { + if (!this.shuttingDown) { + const err = this.normalizeError(error, 'Kafka consumer loop error'); + this.logger.error(err.message, { + groupId, + topics, + error: err.stack || err.message, + }); + } + } finally { + this.consumerStreams.delete(groupId); + this.consumerLoops.delete(groupId); + if (!this.shuttingDown) { + this.logger.warn(`Kafka consumer loop for group ${groupId} ended`); + } + } } - private startConsumerLoop( - consumer: Kafka.KafkaConsumer, - groupId: string, - topics: string[], - onMessage: (message: unknown) => Promise, - ): void { - if (this.consumerLoops.get(groupId)) { + private async closeStream(groupId: string): Promise { + const stream = this.consumerStreams.get(groupId); + if (!stream) { return; } - this.consumerLoops.set(groupId, true); - - const consumeNext = () => { - if (!consumer.isConnected()) { - this.logger.warn( - `Kafka consumer ${groupId} is not connected, retrying consume`, - { - groupId, - topics, - }, - ); - setTimeout(consumeNext, this.retryDelayAfterReconnectMs); - return; - } + await stream.close(); + this.consumerStreams.delete(groupId); + } - consumer.consume(1, (err, messages) => { - if (err) { - this.logger.error(`Kafka consumer error for group ${groupId}`, { - error: err.message, - code: err.code, - }); - setTimeout(consumeNext, this.retryDelayAfterReconnectMs); - return; - } + private buildHeaders( + correlationId: string, + timestamp: number, + ): Record { + return { + 'correlation-id': correlationId, + timestamp: timestamp.toString(), + 'content-type': 'application/json', + }; + } - if (!messages || messages.length === 0) { - setTimeout(consumeNext, 100); - return; - } + private getHeaderValue( + headers: Map | undefined, + key: string, + ): string | undefined { + if (!headers) { + return undefined; + } - const [message] = messages; + const value = headers.get(key); + if (typeof value === 'string') { + return value; + } - void (async () => { - const messageCorrelationId = - this.getHeaderValue(message.headers, 'correlation-id') || uuidv4(); + return undefined; + } - try { - if (!message.value) { - throw new Error('Message value is null or undefined'); - } - - const decodedMessage = this.decodeMessage(message.value); - - if (!decodedMessage) { - throw new Error('Decoded message is null or undefined'); - } - - this.logger.info( - `[KAFKA-CONSUMER] Starting to process message from ${message.topic}`, - { - correlationId: messageCorrelationId, - topic: message.topic, - partition: message.partition, - timestamp: new Date().toISOString(), - }, - ); + private async sendRecords( + topic: string, + values: unknown[], + correlationId: string, + timestamp: number, + ): Promise { + const headers = this.buildHeaders(correlationId, timestamp); + + await this.producer.send({ + messages: values.map((value) => ({ + topic, + value, + headers, + })), + acks: ProduceAcks.ALL, + }); + } - await onMessage(decodedMessage); + private async sendToDLQ(originalTopic: string, message: unknown): Promise { + const dlqTopic = `${originalTopic}.dlq`; - this.logger.info( - `[KAFKA-CONSUMER] Completed processing message from ${message.topic}`, - { - correlationId: messageCorrelationId, - topic: message.topic, - partition: message.partition, - timestamp: new Date().toISOString(), - }, - ); - } catch (error) { - const err = error as Error; - this.logger.error( - `Error processing message from topic ${message.topic}`, - { - error: err.stack, - correlationId: messageCorrelationId, - topic: message.topic, - partition: message.partition, - }, - ); - if (message.value) { - await this.sendToDLQ(message.topic, message.value); - } - } finally { - consumeNext(); - } - })(); - }); - }; + const serializedMessage = this.serializeForDlq(message); - consumeNext(); + await this.produce(dlqTopic, { + originalTopic, + originalMessage: serializedMessage, + error: 'Failed to process message', + timestamp: new Date().toISOString(), + }); } - private async sendToDLQ( - originalTopic: string, - message: Buffer, - ): Promise { - const dlqTopic = `${originalTopic}.dlq`; + private serializeForDlq(message: unknown): string { try { - await this.produce(dlqTopic, { - originalTopic, - originalMessage: message.toString('base64'), - error: 'Failed to process message', - timestamp: new Date().toISOString(), - }); + if (Buffer.isBuffer(message)) { + return message.toString('base64'); + } + + if (message === undefined) { + return Buffer.from('null', 'utf8').toString('base64'); + } + + return Buffer.from(JSON.stringify(message), 'utf8').toString('base64'); } catch (error) { - const err = error as Error; - this.logger.error('Failed to send message to DLQ', { - error: err.stack, - topic: dlqTopic, + const fallback = this.normalizeError(error, 'Failed to serialize DLQ message'); + this.logger.warn(fallback.message, { + error: fallback.stack || fallback.message, }); + return Buffer.from(String(message), 'utf8').toString('base64'); } } - async onApplicationShutdown(signal?: string): Promise { - this.logger.info('Starting Kafka graceful shutdown', { signal }); - const shutdownTimeout = 30000; - - try { - this.logger.info('Stopping producer...'); - await Promise.race([ - new Promise((resolve, reject) => { - this.producer.disconnect((err) => { - if (err) { - reject( - this.normalizeError( - err, - 'Kafka producer shutdown disconnect error', - ), - ); - } else { - resolve(); - } - }); - }), - new Promise((_, reject) => - setTimeout( - () => reject(new Error('Producer disconnect timeout')), - shutdownTimeout, - ), - ), - ]); - this.logger.info('Producer disconnected successfully'); - - this.logger.info('Stopping consumers...'); - const consumerDisconnectPromises = Array.from( - this.consumers.entries(), - ).map(async ([groupId, consumer]) => { - try { - await Promise.race([ - new Promise((resolve, reject) => { - consumer.disconnect((err) => { - if (err) { - reject( - this.normalizeError( - err, - `Kafka consumer ${groupId} shutdown disconnect error`, - ), - ); - } else { - resolve(); - } - }); - }), - new Promise((_, reject) => - setTimeout( - () => - reject(new Error(`Consumer ${groupId} disconnect timeout`)), - shutdownTimeout, - ), - ), - ]); - this.logger.info(`Consumer ${groupId} disconnected successfully`); - } catch (error) { - const err = error as Error; - this.logger.error(`Error disconnecting consumer ${groupId}`, { - error: err.stack, - groupId, - }); - } finally { - this.consumerLoops.delete(groupId); - } - }); + private normalizeError(error: unknown, fallbackMessage: string): Error { + if (error instanceof Error) { + return error; + } - await Promise.all(consumerDisconnectPromises); - this.logger.info('All Kafka connections closed successfully.'); - } catch (error) { - const err = error as Error; - this.logger.error('Error during Kafka shutdown', { - error: err.stack, - signal, - }); - throw err; - } finally { - this.consumers.clear(); + if (typeof error === 'string') { + return new Error(`${fallbackMessage}: ${error}`); } - } - async isConnected(): Promise { try { - await this.sendWithReconnect( - () => { - this.producer.produce( - '__kafka_health_check', - null, - Buffer.from('health_check'), - undefined, - Date.now(), - ); - }, - { topic: '__kafka_health_check' }, - ); - - const consumersConnected = Array.from(this.consumers.values()).every( - (consumer) => consumer.isConnected(), + return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`); + } catch (serializationError) { + const serializationMessage = + serializationError instanceof Error + ? serializationError.message + : 'unknown serialization error'; + return new Error( + `${fallbackMessage}; failed to serialize original error: ${serializationMessage}`, ); - - return this.producer.isConnected() && consumersConnected; - } catch (error) { - const err = error as Error; - this.logger.error('Failed to check Kafka connection status', { - error: err.stack, - timestamp: new Date().toISOString(), - }); - return false; } } } + diff --git a/yarn.lock b/yarn.lock index 473c811..44e1fae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -598,6 +598,11 @@ resolved "https://registry.yarnpkg.com/@inquirer/type/-/type-3.0.8.tgz#efc293ba0ed91e90e6267f1aacc1c70d20b8b4e8" integrity sha512-lg9Whz8onIHRthWaN1Q9EGLa/0LFJjyM8mEUbL1eTi6yMGvBf8gvyDLtxSXztQsxMvhxxNpJYrwa1YHdq+w4Jw== +"@ioredis/commands@1.4.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.4.0.tgz#9f657d51cdd5d2fdb8889592aa4a355546151f25" + integrity sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ== + "@isaacs/balanced-match@^4.0.1": version "4.0.1" resolved "https://registry.yarnpkg.com/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz#3081dadbc3460661b751e7591d7faea5df39dd29" @@ -890,6 +895,36 @@ resolved "https://registry.yarnpkg.com/@microsoft/tsdoc/-/tsdoc-0.15.1.tgz#d4f6937353bc4568292654efb0a0e0532adbcba2" integrity sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw== +"@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz#9edec61b22c3082018a79f6d1c30289ddf3d9d11" + integrity sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw== + +"@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz#33677a275204898ad8acbf62734fc4dc0b6a4855" + integrity sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw== + +"@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz#19edf7cdc2e7063ee328403c1d895a86dd28f4bb" + integrity sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg== + +"@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz#94fb0543ba2e28766c3fc439cabbe0440ae70159" + integrity sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw== + +"@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz#4a0609ab5fe44d07c9c60a11e4484d3c38bbd6e3" + integrity sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg== + +"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz#0aa5502d547b57abfc4ac492de68e2006e417242" + integrity sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ== + "@napi-rs/nice-android-arm-eabi@1.1.1": version "1.1.1" resolved "https://registry.yarnpkg.com/@napi-rs/nice-android-arm-eabi/-/nice-android-arm-eabi-1.1.1.tgz#4ebd966821cd6c2cc7cc020eb468de397bb9b40f" @@ -1132,6 +1167,17 @@ boxen "5.1.2" check-disk-space "3.4.0" +"@platformatic/kafka@^1.14.0": + version "1.14.0" + resolved "https://registry.npmjs.org/@platformatic/kafka/-/kafka-1.14.0.tgz#77804d31f5c34af393cd4e65877c119f856c057e" + integrity sha512-7DVRU1sqYo8r9Hh5rEJaCVjc9GSdb50xGAvUwS9TMKuMY9IZEec5TRkiZ22H63bFgC/aBUhy2IfKpLpwlv8cPw== + dependencies: + ajv "^8.17.1" + debug "^4.4.3" + fastq "^1.19.1" + mnemonist "^0.40.3" + scule "^1.3.0" + "@nestjs/testing@^11.0.1": version "11.1.6" resolved "https://registry.yarnpkg.com/@nestjs/testing/-/testing-11.1.6.tgz#7f172a8024948dee4cb318acccfff31c1356f338" @@ -2461,6 +2507,19 @@ buffer@^5.2.1, buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" +bullmq@^5.58.8: + version "5.58.8" + resolved "https://registry.yarnpkg.com/bullmq/-/bullmq-5.58.8.tgz#e06b1bb3f9be5b20e9136bf62d251071f0b86f13" + integrity sha512-j7h2JlWs7rOzsLePKtNK+zLOyrH6PRurLLZ6SriSpt9w5fHR128IFSd4gHGwYUb41ycnWmbLDcggp64zNW/p/Q== + dependencies: + cron-parser "^4.9.0" + ioredis "^5.4.1" + msgpackr "^1.11.2" + node-abort-controller "^3.1.1" + semver "^7.5.4" + tslib "^2.0.0" + uuid "^11.1.0" + busboy@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" @@ -2671,6 +2730,11 @@ clone@^1.0.2: resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" integrity sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg== +cluster-key-slot@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -2877,6 +2941,13 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== +cron-parser@^4.9.0: + version "4.9.0" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5" + integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q== + dependencies: + luxon "^3.2.1" + cron@4.3.3: version "4.3.3" resolved "https://registry.yarnpkg.com/cron/-/cron-4.3.3.tgz#d37cfcbc73ba34a50d9d9ce9b653ae60837377d7" @@ -2955,6 +3026,11 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0, depd@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -2965,6 +3041,11 @@ destr@^2.0.3: resolved "https://registry.yarnpkg.com/destr/-/destr-2.0.5.tgz#7d112ff1b925fb8d2079fac5bdb4a90973b51fdb" integrity sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA== +detect-libc@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.1.tgz#9f1e511ace6bb525efea4651345beac424dac7b9" + integrity sha512-ecqj/sy1jcK1uWrwpR67UhYrIFQ+5WlGxth34WquCbamhFA6hkkwiu37o6J5xCHdo1oixJRfVRw+ywV+Hq/0Aw== + detect-newline@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" @@ -3937,6 +4018,21 @@ inspect-with-kind@^1.0.5: dependencies: kind-of "^6.0.2" +ioredis@^5.4.1: + version "5.8.0" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.8.0.tgz#a1c4ef6be2e274cc8e99c9e22794ef1ef06dc24a" + integrity sha512-AUXbKn9gvo9hHKvk6LbZJQSKn/qIfkWXrnsyL9Yrf+oeXmla9Nmf6XEumOddyhM8neynpK5oAV6r9r99KBuwzA== + dependencies: + "@ioredis/commands" "1.4.0" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -4642,11 +4738,21 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" +lodash.defaults@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" + integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ== + lodash.includes@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" integrity sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w== +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== + lodash.isboolean@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6" @@ -4729,7 +4835,7 @@ lru-cache@^5.1.1: dependencies: yallist "^3.0.2" -luxon@~3.7.0: +luxon@^3.2.1, luxon@~3.7.0: version "3.7.2" resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.7.2.tgz#d697e48f478553cca187a0f8436aff468e3ba0ba" integrity sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew== @@ -4902,6 +5008,27 @@ ms@^2.1.1, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== +msgpackr-extract@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz#e9d87023de39ce714872f9e9504e3c1996d61012" + integrity sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA== + dependencies: + node-gyp-build-optional-packages "5.2.2" + optionalDependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.3" + +msgpackr@^1.11.2: + version "1.11.5" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.11.5.tgz#edf0b9d9cb7d8ed6897dd0e42cfb865a2f4b602e" + integrity sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA== + optionalDependencies: + msgpackr-extract "^3.0.2" + multer@2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/multer/-/multer-2.0.2.tgz#08a8aa8255865388c387aaf041426b0c87bf58dd" @@ -4947,7 +5074,7 @@ nest-winston@^1.10.2: dependencies: fast-safe-stringify "^2.1.1" -node-abort-controller@^3.0.1: +node-abort-controller@^3.0.1, node-abort-controller@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== @@ -4964,18 +5091,18 @@ node-fetch-native@^1.6.6: resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.7.tgz#9d09ca63066cc48423211ed4caf5d70075d76a71" integrity sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q== +node-gyp-build-optional-packages@5.2.2: + version "5.2.2" + resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz#522f50c2d53134d7f3a76cd7255de4ab6c96a3a4" + integrity sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw== + dependencies: + detect-libc "^2.0.1" + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== -node-rdkafka@^2.16.0: - version "2.18.0" - resolved "https://registry.yarnpkg.com/node-rdkafka/-/node-rdkafka-2.18.0.tgz#116950e49dfe804932c8bc6dbc68949793e72ee2" - integrity sha512-jYkmO0sPvjesmzhv1WFOO4z7IMiAFpThR6/lcnFDWgSPkYL95CtcuVNo/R5PpjujmqSgS22GMkL1qvU4DTAvEQ== - dependencies: - bindings "^1.3.1" - nan "^2.17.0" node-releases@^2.0.21: version "2.0.21" @@ -5442,6 +5569,18 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" + integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w== + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" + integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A== + dependencies: + redis-errors "^1.0.0" + reflect-metadata@^0.2.2: version "0.2.2" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.2.2.tgz#400c845b6cba87a21f2c65c4aeb158f4fa4d9c5b" @@ -5807,6 +5946,11 @@ stack-utils@^2.0.3: dependencies: escape-string-regexp "^2.0.0" +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + statuses@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" @@ -6203,7 +6347,7 @@ tsconfig@^7.0.0: strip-bom "^3.0.0" strip-json-comments "^2.0.0" -tslib@2.8.1, tslib@^2.1.0: +tslib@2.8.1, tslib@^2.0.0, tslib@^2.1.0: version "2.8.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== @@ -6352,6 +6496,11 @@ utils-merge@^1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== +uuid@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.1.0.tgz#9549028be1753bb934fc96e2bca09bb4105ae912" + integrity sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A== + uuid@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" From 3968c8040dcd3f7a1ecebfe9cfb430e54175f018 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 10:15:12 +1000 Subject: [PATCH 14/23] Build fix --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 17d1422..11e0ddf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,10 +14,10 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* COPY package.json ./ COPY prisma ./prisma -RUN yarn install +RUN pnpm install # ---- Build Stage ---- -FROM pnpm AS build +FROM deps AS build COPY . . RUN pnpm prisma:generate RUN pnpm build From becc408bdc315909ff0f077498bfffccadc17b00 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 10:18:04 +1000 Subject: [PATCH 15/23] Build fix --- Dockerfile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 11e0ddf..a9742a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,14 +4,6 @@ WORKDIR /usr/src/app # ---- Dependencies Stage ---- FROM base AS deps -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 \ - make \ - g++ \ - pkg-config \ - librdkafka-dev \ - && rm -rf /var/lib/apt/lists/* COPY package.json ./ COPY prisma ./prisma RUN pnpm install From 1f5d81df65ca0177df94d4683a705c1d115b9af1 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 10:20:24 +1000 Subject: [PATCH 16/23] Build fix --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index a9742a4..e5fc8b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,10 +17,6 @@ RUN pnpm build # ---- Production Stage ---- FROM base AS production ENV NODE_ENV production -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - librdkafka1 \ - && rm -rf /var/lib/apt/lists/* COPY --from=build /usr/src/app/dist ./dist COPY --from=deps /usr/src/app/node_modules ./node_modules EXPOSE 3000 From 09c816318adfb867c57d8b4f0e1e238b4da428df Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 10:22:37 +1000 Subject: [PATCH 17/23] Minor cleanup and text tweaks --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e5fc8b1..b32d708 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ WORKDIR /usr/src/app FROM base AS deps COPY package.json ./ COPY prisma ./prisma +RUN npm install -g pnpm RUN pnpm install # ---- Build Stage ---- From 4c44d55da331cb1a48f55ee36b8fe7476873540a Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 11:19:04 +1000 Subject: [PATCH 18/23] Patch kafka library until https://github.com/platformatic/kafka/issues/120 is fixed --- package.json | 5 +- patches/@platformatic+kafka+1.14.0.patch | 55 ++++++++ pnpm-lock.yaml | 168 +++++++++++++++++++++++ 3 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 patches/@platformatic+kafka+1.14.0.patch diff --git a/package.json b/package.json index 3ed96ca..741a921 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", "prisma:generate": "prisma generate --schema prisma/challenge.schema.prisma && prisma generate --schema prisma/autopilot.schema.prisma", - "postinstall": "prisma generate --schema prisma/challenge.schema.prisma && prisma generate --schema prisma/autopilot.schema.prisma", + "postinstall": "prisma generate --schema prisma/challenge.schema.prisma && prisma generate --schema prisma/autopilot.schema.prisma && patch-package", "prisma:pushautopilot": "prisma db push --schema prisma/autopilot.schema.prisma" }, "prisma": { @@ -38,6 +38,7 @@ "@nestjs/schedule": "^6.0.0", "@nestjs/swagger": "^7.4.0", "@nestjs/terminus": "^11.0.0", + "@platformatic/kafka": "^1.14.0", "@prisma/client": "^6.4.1", "@types/express": "^5.0.0", "axios": "^1.9.0", @@ -46,7 +47,6 @@ "class-validator": "^0.14.2", "joi": "^17.13.3", "nest-winston": "^1.10.2", - "@platformatic/kafka": "^1.14.0", "passport": "^0.7.0", "passport-jwt": "^4.0.1", "prisma": "^6.4.1", @@ -73,6 +73,7 @@ "globals": "^16.0.0", "jest": "^29.7.0", "nodemon": "^3.0.0", + "patch-package": "^8.0.0", "prettier": "^3.4.2", "source-map-support": "^0.5.21", "supertest": "^7.0.0", diff --git a/patches/@platformatic+kafka+1.14.0.patch b/patches/@platformatic+kafka+1.14.0.patch new file mode 100644 index 0000000..d780049 --- /dev/null +++ b/patches/@platformatic+kafka+1.14.0.patch @@ -0,0 +1,55 @@ +--- a/node_modules/.pnpm/@platformatic+kafka@1.14.0/node_modules/@platformatic/kafka/dist/clients/consumer/consumer.js ++++ b/node_modules/.pnpm/@platformatic+kafka@1.14.0/node_modules/@platformatic/kafka/dist/clients/consumer/consumer.js +@@ -842,14 +842,44 @@ + .appendInt32(0).buffer; // No user data + } + #decodeProtocolAssignment(buffer) { +- const reader = Reader.from(buffer); +- reader.skip(2); // Ignore Version information +- return reader.readArray(r => { +- return { +- topic: r.readString(false), +- partitions: r.readArray(r => r.readInt32(), false, false) +- }; +- }, false, false); ++ if (!buffer || typeof buffer.length !== 'number' || buffer.length === 0) { ++ return []; ++ } ++ const totalLength = typeof buffer.length === 'number' ? buffer.length : 0; ++ if (totalLength < 2) { ++ return []; ++ } ++ const decode = (compact) => { ++ const reader = Reader.from(buffer); ++ reader.skip(2); // Ignore Version information ++ return reader.readArray(r => { ++ return { ++ topic: r.readString(compact), ++ partitions: r.readArray(r => r.readInt32(), compact, compact) ++ }; ++ }, compact, compact); ++ }; ++ const shouldFallback = (error) => error?.code === 'PLT_KFK_USER' && error?.message === 'Out of bounds.'; ++ const preferCompact = totalLength - 2 < 4; ++ if (!preferCompact) { ++ try { ++ return decode(false); ++ } ++ catch (error) { ++ if (!shouldFallback(error)) { ++ throw error; ++ } ++ } ++ } ++ try { ++ return decode(true); ++ } ++ catch (error) { ++ if (shouldFallback(error)) { ++ return []; ++ } ++ throw error; ++ } + } + #createAssignments(metadata) { + const partitionTracker = new Map(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 873c672..0412ca9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -144,6 +144,9 @@ importers: nodemon: specifier: ^3.0.0 version: 3.1.10 + patch-package: + specifier: ^8.0.0 + version: 8.0.0 prettier: specifier: ^3.4.2 version: 3.6.2 @@ -1702,6 +1705,9 @@ packages: '@xtuc/long@4.2.2': resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} + '@yarnpkg/lockfile@1.1.0': + resolution: {integrity: sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -1818,6 +1824,10 @@ packages: asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + at-least-node@1.0.0: + resolution: {integrity: sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==} + engines: {node: '>= 4.0.0'} + axios@1.10.0: resolution: {integrity: sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw==} @@ -1946,6 +1956,10 @@ packages: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} + call-bind@1.0.8: + resolution: {integrity: sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==} + engines: {node: '>= 0.4'} + call-bound@1.0.4: resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} engines: {node: '>= 0.4'} @@ -2223,6 +2237,10 @@ packages: resolution: {integrity: sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==} engines: {node: '>=10'} + define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} @@ -2559,6 +2577,9 @@ packages: resolution: {integrity: sha512-+iwzCJ7C5v5KgcBuueqVoNiHVoQpwiUK5XFLjf0affFTep+Wcw93tPvmb8tqujDNmzhBDPddnWV/qgWSXgq+Hg==} engines: {node: '>=12'} + find-yarn-workspace-root@2.0.0: + resolution: {integrity: sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==} + flat-cache@4.0.1: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} @@ -2613,6 +2634,10 @@ packages: resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} engines: {node: '>=12'} + fs-extra@9.1.0: + resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} + engines: {node: '>=10'} + fs-monkey@1.0.6: resolution: {integrity: sha512-b1FMfwetIKymC0eioW7mTywihSQE4oLzQn1dB6rZB5fx/3NpNEdAWeCSMB+60/AeT0TCXsxzAlcYVEFCTAksWg==} @@ -2713,6 +2738,9 @@ packages: resolution: {integrity: sha512-Pq0h+hvsVm6dDEa8x82GnLSYHOzNDt7f0ddFa3FqcQlgzEiptPqL+XrOJNavjOzSYiYWIrgeVYYgGlLmnxwilQ==} engines: {node: '>=8'} + has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + has-symbols@1.1.0: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} @@ -2810,6 +2838,11 @@ packages: resolution: {integrity: sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==} engines: {node: '>= 0.4'} + is-docker@2.2.1: + resolution: {integrity: sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==} + engines: {node: '>=8'} + hasBin: true + is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -2853,6 +2886,13 @@ packages: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} + is-wsl@2.2.0: + resolution: {integrity: sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==} + engines: {node: '>=8'} + + isarray@2.0.5: + resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -3064,6 +3104,10 @@ packages: json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + json-stable-stringify@1.3.0: + resolution: {integrity: sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg==} + engines: {node: '>= 0.4'} + json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -3075,6 +3119,9 @@ packages: jsonfile@6.1.0: resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} + jsonify@0.0.1: + resolution: {integrity: sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==} + jsonwebtoken@9.0.2: resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} engines: {node: '>=12', npm: '>=6'} @@ -3092,6 +3139,9 @@ packages: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} + klaw-sync@6.0.0: + resolution: {integrity: sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==} + kleur@3.0.3: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} engines: {node: '>=6'} @@ -3390,6 +3440,10 @@ packages: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} + object-keys@1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + obliterator@2.0.5: resolution: {integrity: sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==} @@ -3410,6 +3464,10 @@ packages: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} + open@7.4.2: + resolution: {integrity: sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==} + engines: {node: '>=8'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -3472,6 +3530,11 @@ packages: resolution: {integrity: sha512-cPLl+qZpSc+ireUvt+IzqbED1cHHkDoVYMo30jbJIdOOjQ1MQYZBPiNvmi8UM6lJuOpTPXJGZQk0DtC4y61MYQ==} engines: {node: '>= 0.4.0'} + patch-package@8.0.0: + resolution: {integrity: sha512-da8BVIhzjtgScwDJ2TtKsfT5JFWz1hYoBl9rUQ1f38MC2HwnEIkK8VN3dKMKcP7P7bvvgzNDbfNHtx3MsQb5vA==} + engines: {node: '>=14', npm: '>5'} + hasBin: true + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -3765,6 +3828,10 @@ packages: resolution: {integrity: sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==} engines: {node: '>= 18'} + set-function-length@1.2.2: + resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==} + engines: {node: '>= 0.4'} + setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -3809,6 +3876,10 @@ packages: sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + slash@2.0.0: + resolution: {integrity: sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==} + engines: {node: '>=6'} + slash@3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} @@ -4296,6 +4367,11 @@ packages: yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yaml@2.8.1: + resolution: {integrity: sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==} + engines: {node: '>= 14.6'} + hasBin: true + yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} @@ -5964,6 +6040,8 @@ snapshots: '@xtuc/long@4.2.2': {} + '@yarnpkg/lockfile@1.1.0': {} + accepts@2.0.0: dependencies: mime-types: 3.0.1 @@ -6059,6 +6137,8 @@ snapshots: asynckit@0.4.0: {} + at-least-node@1.0.0: {} + axios@1.10.0: dependencies: follow-redirects: 1.15.9 @@ -6264,6 +6344,13 @@ snapshots: es-errors: 1.3.0 function-bind: 1.1.2 + call-bind@1.0.8: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + get-intrinsic: 1.3.0 + set-function-length: 1.2.2 + call-bound@1.0.4: dependencies: call-bind-apply-helpers: 1.0.2 @@ -6510,6 +6597,12 @@ snapshots: defer-to-connect@2.0.1: {} + define-data-property@1.1.4: + dependencies: + es-define-property: 1.0.1 + es-errors: 1.3.0 + gopd: 1.2.0 + defu@6.1.4: {} delayed-stream@1.0.0: {} @@ -6881,6 +6974,10 @@ snapshots: dependencies: semver-regex: 4.0.5 + find-yarn-workspace-root@2.0.0: + dependencies: + micromatch: 4.0.8 + flat-cache@4.0.1: dependencies: flatted: 3.3.3 @@ -6940,6 +7037,13 @@ snapshots: jsonfile: 6.1.0 universalify: 2.0.1 + fs-extra@9.1.0: + dependencies: + at-least-node: 1.0.0 + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.1 + fs-monkey@1.0.6: {} fs.realpath@1.0.0: {} @@ -7047,6 +7151,10 @@ snapshots: has-own-prop@2.0.0: {} + has-property-descriptors@1.0.2: + dependencies: + es-define-property: 1.0.1 + has-symbols@1.1.0: {} has-tostringtag@1.0.2: @@ -7143,6 +7251,8 @@ snapshots: dependencies: hasown: 2.0.2 + is-docker@2.2.1: {} + is-extglob@2.1.1: {} is-fullwidth-code-point@3.0.0: {} @@ -7167,6 +7277,12 @@ snapshots: is-unicode-supported@0.1.0: {} + is-wsl@2.2.0: + dependencies: + is-docker: 2.2.1 + + isarray@2.0.5: {} + isexe@2.0.0: {} istanbul-lib-coverage@3.2.2: {} @@ -7571,6 +7687,14 @@ snapshots: json-stable-stringify-without-jsonify@1.0.1: {} + json-stable-stringify@1.3.0: + dependencies: + call-bind: 1.0.8 + call-bound: 1.0.4 + isarray: 2.0.5 + jsonify: 0.0.1 + object-keys: 1.1.1 + json5@2.2.3: {} jsonc-parser@3.3.1: {} @@ -7581,6 +7705,8 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 + jsonify@0.0.1: {} + jsonwebtoken@9.0.2: dependencies: jws: 3.2.2 @@ -7611,6 +7737,10 @@ snapshots: kind-of@6.0.3: {} + klaw-sync@6.0.0: + dependencies: + graceful-fs: 4.2.11 + kleur@3.0.3: {} kuler@2.0.0: {} @@ -7884,6 +8014,8 @@ snapshots: object-inspect@1.13.4: {} + object-keys@1.1.1: {} + obliterator@2.0.5: {} ohash@2.0.11: {} @@ -7904,6 +8036,11 @@ snapshots: dependencies: mimic-fn: 2.1.0 + open@7.4.2: + dependencies: + is-docker: 2.2.1 + is-wsl: 2.2.0 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -7975,6 +8112,24 @@ snapshots: pause: 0.0.1 utils-merge: 1.0.1 + patch-package@8.0.0: + dependencies: + '@yarnpkg/lockfile': 1.1.0 + chalk: 4.1.2 + ci-info: 3.9.0 + cross-spawn: 7.0.6 + find-yarn-workspace-root: 2.0.0 + fs-extra: 9.1.0 + json-stable-stringify: 1.3.0 + klaw-sync: 6.0.0 + minimist: 1.2.8 + open: 7.4.2 + rimraf: 2.7.1 + semver: 7.7.2 + slash: 2.0.0 + tmp: 0.0.33 + yaml: 2.8.1 + path-exists@4.0.0: {} path-is-absolute@1.0.1: {} @@ -8242,6 +8397,15 @@ snapshots: transitivePeerDependencies: - supports-color + set-function-length@1.2.2: + dependencies: + define-data-property: 1.1.4 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.3.0 + gopd: 1.2.0 + has-property-descriptors: 1.0.2 + setprototypeof@1.2.0: {} shebang-command@2.0.0: @@ -8292,6 +8456,8 @@ snapshots: sisteransi@1.0.5: {} + slash@2.0.0: {} + slash@3.0.0: {} snappy@7.3.3: @@ -8811,6 +8977,8 @@ snapshots: yallist@3.1.1: {} + yaml@2.8.1: {} + yargs-parser@21.1.1: {} yargs@17.7.2: From 335089ae0cc6a11ce8338349f3a7b7c3250d0417 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 13:11:05 +1000 Subject: [PATCH 19/23] Build fix --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b32d708..61e9f06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,11 @@ WORKDIR /usr/src/app # ---- Dependencies Stage ---- FROM base AS deps COPY package.json ./ +COPY pnpm-lock.yaml ./ +COPY patches ./patches COPY prisma ./prisma RUN npm install -g pnpm -RUN pnpm install +RUN pnpm install # ---- Build Stage ---- FROM deps AS build From e2bba8402a27c224fef56148d95bc772fb3bc9ac Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 13:26:48 +1000 Subject: [PATCH 20/23] Fix scheduler IDs so bullmq doesn't complain --- src/autopilot/services/scheduler.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autopilot/services/scheduler.service.ts b/src/autopilot/services/scheduler.service.ts index b7819ed..1cfadf7 100644 --- a/src/autopilot/services/scheduler.service.ts +++ b/src/autopilot/services/scheduler.service.ts @@ -160,7 +160,7 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { phaseData: PhaseTransitionPayload, ): Promise { const { challengeId, phaseId, date: endTime } = phaseData; - const jobId = `${challengeId}:${phaseId}`; + const jobId = `${challengeId}|${phaseId}`; // BullMQ rejects ':' in custom IDs, use pipe instead if (!endTime || endTime === '' || isNaN(new Date(endTime).getTime())) { this.logger.error( From 332ed0d4c7c5924ab9f38e59ada996e2c7219ab7 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 18:38:41 +1000 Subject: [PATCH 21/23] Pause review phase if reviews are still pending, instead of closing automatically --- .../services/scheduler.service.spec.ts | 137 ++++++++++++++++++ src/autopilot/services/scheduler.service.ts | 79 ++++++++++ src/review/review.service.ts | 49 +++++++ 3 files changed, 265 insertions(+) create mode 100644 src/autopilot/services/scheduler.service.spec.ts diff --git a/src/autopilot/services/scheduler.service.spec.ts b/src/autopilot/services/scheduler.service.spec.ts new file mode 100644 index 0000000..f80b3f0 --- /dev/null +++ b/src/autopilot/services/scheduler.service.spec.ts @@ -0,0 +1,137 @@ +jest.mock('../../kafka/kafka.service', () => ({ + KafkaService: jest.fn().mockImplementation(() => ({ + produce: jest.fn(), + })), +})); + +import { SchedulerService } from './scheduler.service'; +import { KafkaService } from '../../kafka/kafka.service'; +import { ChallengeApiService } from '../../challenge/challenge-api.service'; +import { PhaseReviewService } from './phase-review.service'; +import { ChallengeCompletionService } from './challenge-completion.service'; +import { ReviewService } from '../../review/review.service'; +import { AutopilotOperator, PhaseTransitionPayload } from '../interfaces/autopilot.interface'; + +const createPayload = ( + overrides: Partial = {}, +): PhaseTransitionPayload => ({ + projectId: 123, + challengeId: 'challenge-1', + phaseId: 'phase-1', + phaseTypeName: 'Review', + state: 'END', + operator: AutopilotOperator.SYSTEM_PHASE_CHAIN, + projectStatus: 'ACTIVE', + ...overrides, +}); + +describe('SchedulerService (review phase deferral)', () => { + let scheduler: SchedulerService; + let kafkaService: jest.Mocked; + let challengeApiService: jest.Mocked; + let phaseReviewService: jest.Mocked; + let challengeCompletionService: jest.Mocked; + let reviewService: jest.Mocked; + + beforeEach(() => { + kafkaService = { + produce: jest.fn(), + } as unknown as jest.Mocked; + + challengeApiService = { + getPhaseDetails: jest.fn(), + advancePhase: jest.fn(), + getChallengePhases: jest.fn(), + } as unknown as jest.Mocked; + + phaseReviewService = { + handlePhaseOpened: jest.fn(), + } as unknown as jest.Mocked; + + challengeCompletionService = { + finalizeChallenge: jest.fn().mockResolvedValue(true), + } as unknown as jest.Mocked; + + reviewService = { + getPendingReviewCount: jest.fn(), + } as unknown as jest.Mocked; + + scheduler = new SchedulerService( + kafkaService, + challengeApiService, + phaseReviewService, + challengeCompletionService, + reviewService, + ); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('defers closing review phases when pending reviews exist', async () => { + const payload = createPayload(); + const phaseDetails = { + id: payload.phaseId, + name: 'Review', + isOpen: true, + }; + + challengeApiService.getPhaseDetails.mockResolvedValue(phaseDetails as any); + reviewService.getPendingReviewCount.mockResolvedValue(2); + + const scheduleSpy = jest + .spyOn(scheduler, 'schedulePhaseTransition') + .mockResolvedValue('rescheduled'); + + await scheduler.advancePhase(payload); + + expect(challengeApiService.advancePhase).not.toHaveBeenCalled(); + expect(reviewService.getPendingReviewCount).toHaveBeenCalledWith( + payload.phaseId, + payload.challengeId, + ); + expect(scheduleSpy).toHaveBeenCalledTimes(1); + + const [rescheduledPayload] = scheduleSpy.mock.calls[0]; + expect(rescheduledPayload.state).toBe('END'); + expect(rescheduledPayload.phaseId).toBe(payload.phaseId); + expect(rescheduledPayload.challengeId).toBe(payload.challengeId); + expect(rescheduledPayload.date).toBeDefined(); + expect(new Date(rescheduledPayload.date as string).getTime()).toBeGreaterThan( + Date.now(), + ); + }); + + it('closes review phases when no pending reviews remain', async () => { + const payload = createPayload(); + const phaseDetails = { + id: payload.phaseId, + name: 'Review', + isOpen: true, + }; + + challengeApiService.getPhaseDetails.mockResolvedValue(phaseDetails as any); + reviewService.getPendingReviewCount.mockResolvedValue(0); + challengeApiService.advancePhase.mockResolvedValue({ + success: true, + message: 'closed', + updatedPhases: [ + { + id: payload.phaseId, + isOpen: false, + actualEndDate: new Date().toISOString(), + }, + ], + } as any); + + await scheduler.advancePhase(payload); + + expect(reviewService.getPendingReviewCount).toHaveBeenCalled(); + expect(challengeApiService.advancePhase).toHaveBeenCalledWith( + payload.challengeId, + payload.phaseId, + 'close', + ); + }); +}); diff --git a/src/autopilot/services/scheduler.service.ts b/src/autopilot/services/scheduler.service.ts index 1cfadf7..bcdab85 100644 --- a/src/autopilot/services/scheduler.service.ts +++ b/src/autopilot/services/scheduler.service.ts @@ -15,6 +15,8 @@ import { } from '../interfaces/autopilot.interface'; import { KAFKA_TOPICS } from '../../kafka/constants/topics'; import { Job, Queue, RedisOptions, Worker } from 'bullmq'; +import { ReviewService } from '../../review/review.service'; +import { REVIEW_PHASE_NAMES } from '../constants/review.constants'; const PHASE_QUEUE_NAME = 'autopilot-phase-transitions'; const PHASE_QUEUE_PREFIX = '{autopilot-phase-transitions}'; @@ -36,6 +38,9 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { private readonly finalizationRetryBaseDelayMs = 60_000; private readonly finalizationRetryMaxAttempts = 10; private readonly finalizationRetryMaxDelayMs = 10 * 60 * 1000; + private readonly reviewCloseRetryAttempts = new Map(); + private readonly reviewCloseRetryBaseDelayMs = 10 * 60 * 1000; + private readonly reviewCloseRetryMaxDelayMs = 60 * 60 * 1000; private redisConnection?: RedisOptions; private phaseQueue?: Queue; @@ -47,6 +52,7 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { private readonly challengeApiService: ChallengeApiService, private readonly phaseReviewService: PhaseReviewService, private readonly challengeCompletionService: ChallengeCompletionService, + private readonly reviewService: ReviewService, ) {} private async ensureInitialized(): Promise { @@ -391,6 +397,30 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { return; } + const isReviewPhase = + REVIEW_PHASE_NAMES.has(phaseDetails.name) || + REVIEW_PHASE_NAMES.has(data.phaseTypeName); + + if (operation === 'close' && isReviewPhase) { + try { + const pendingReviews = await this.reviewService.getPendingReviewCount( + data.phaseId, + data.challengeId, + ); + + if (pendingReviews > 0) { + await this.deferReviewPhaseClosure(data, pendingReviews); + return; + } + } catch (error) { + const err = error as Error; + this.logger.error( + `[REVIEW LATE] Unable to verify pending reviews for phase ${data.phaseId} on challenge ${data.challengeId}: ${err.message}`, + err.stack, + ); + } + } + this.logger.log( `Phase ${data.phaseId} is currently ${phaseDetails.isOpen ? 'open' : 'closed'}, will ${operation} it`, ); @@ -406,6 +436,12 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { `Successfully advanced phase ${data.phaseId} for challenge ${data.challengeId}: ${result.message}`, ); + if (operation === 'close' && isReviewPhase) { + this.reviewCloseRetryAttempts.delete( + this.buildReviewPhaseKey(data.challengeId, data.phaseId), + ); + } + if (operation === 'open') { try { await this.phaseReviewService.handlePhaseOpened( @@ -588,4 +624,47 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { } this.finalizationRetryTimers.delete(challengeId); } + + private async deferReviewPhaseClosure( + data: PhaseTransitionPayload, + pendingCount: number, + ): Promise { + const key = this.buildReviewPhaseKey(data.challengeId, data.phaseId); + const attempt = (this.reviewCloseRetryAttempts.get(key) ?? 0) + 1; + this.reviewCloseRetryAttempts.set(key, attempt); + + const delay = this.computeReviewCloseRetryDelay(attempt); + const nextRun = new Date(Date.now() + delay).toISOString(); + + const payload: PhaseTransitionPayload = { + ...data, + date: nextRun, + operator: data.operator ?? AutopilotOperator.SYSTEM_SCHEDULER, + }; + + try { + await this.schedulePhaseTransition(payload); + this.logger.warn( + `[REVIEW LATE] Deferred closing review phase ${data.phaseId} for challenge ${data.challengeId}; ${pendingCount} incomplete review(s) detected. Retrying in ${Math.round(delay / 60000)} minute(s).`, + ); + } catch (error) { + const err = error as Error; + this.logger.error( + `[REVIEW LATE] Failed to reschedule close for review phase ${data.phaseId} on challenge ${data.challengeId}: ${err.message}`, + err.stack, + ); + this.reviewCloseRetryAttempts.delete(key); + throw err; + } + } + + private computeReviewCloseRetryDelay(attempt: number): number { + const multiplier = Math.max(attempt, 1); + const delay = this.reviewCloseRetryBaseDelayMs * multiplier; + return Math.min(delay, this.reviewCloseRetryMaxDelayMs); + } + + private buildReviewPhaseKey(challengeId: string, phaseId: string): string { + return `${challengeId}|${phaseId}`; + } } diff --git a/src/review/review.service.ts b/src/review/review.service.ts index cfc08ab..40de90d 100644 --- a/src/review/review.service.ts +++ b/src/review/review.service.ts @@ -12,6 +12,10 @@ interface ReviewRecord { resourceId: string; } +interface PendingCountRecord { + count: number | string; +} + @Injectable() export class ReviewService { private static readonly REVIEW_TABLE = Prisma.sql`"review"`; @@ -217,6 +221,51 @@ export class ReviewService { } } + async getPendingReviewCount( + phaseId: string, + challengeId?: string, + ): Promise { + const query = Prisma.sql` + SELECT COUNT(*)::int AS count + FROM ${ReviewService.REVIEW_TABLE} + WHERE "phaseId" = ${phaseId} + AND ( + "status" IS NULL + OR UPPER("status") NOT IN ('COMPLETED', 'NO_REVIEW') + ) + `; + + try { + const [record] = await this.prisma.$queryRaw(query); + const rawCount = Number(record?.count ?? 0); + const count = Number.isFinite(rawCount) ? rawCount : 0; + + void this.dbLogger.logAction('review.getPendingReviewCount', { + challengeId: challengeId ?? null, + status: 'SUCCESS', + source: ReviewService.name, + details: { + phaseId, + pendingCount: count, + }, + }); + + return count; + } catch (error) { + const err = error as Error; + void this.dbLogger.logAction('review.getPendingReviewCount', { + challengeId: challengeId ?? null, + status: 'ERROR', + source: ReviewService.name, + details: { + phaseId, + error: err.message, + }, + }); + throw err; + } + } + async createPendingReview( submissionId: string, resourceId: string, From d46e214b838a1b98612e2d1cf7e29714881d66b9 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 26 Sep 2025 21:36:54 +1000 Subject: [PATCH 22/23] Fix up problem where review closes early --- .../services/scheduler.service.spec.ts | 11 ++- src/autopilot/services/scheduler.service.ts | 12 ++- src/kafka/kafka.service.ts | 96 ++++++++++++------- src/review/review.service.ts | 2 +- 4 files changed, 77 insertions(+), 44 deletions(-) diff --git a/src/autopilot/services/scheduler.service.spec.ts b/src/autopilot/services/scheduler.service.spec.ts index f80b3f0..cc5ac30 100644 --- a/src/autopilot/services/scheduler.service.spec.ts +++ b/src/autopilot/services/scheduler.service.spec.ts @@ -10,7 +10,10 @@ import { ChallengeApiService } from '../../challenge/challenge-api.service'; import { PhaseReviewService } from './phase-review.service'; import { ChallengeCompletionService } from './challenge-completion.service'; import { ReviewService } from '../../review/review.service'; -import { AutopilotOperator, PhaseTransitionPayload } from '../interfaces/autopilot.interface'; +import { + AutopilotOperator, + PhaseTransitionPayload, +} from '../interfaces/autopilot.interface'; const createPayload = ( overrides: Partial = {}, @@ -98,9 +101,9 @@ describe('SchedulerService (review phase deferral)', () => { expect(rescheduledPayload.phaseId).toBe(payload.phaseId); expect(rescheduledPayload.challengeId).toBe(payload.challengeId); expect(rescheduledPayload.date).toBeDefined(); - expect(new Date(rescheduledPayload.date as string).getTime()).toBeGreaterThan( - Date.now(), - ); + expect( + new Date(rescheduledPayload.date as string).getTime(), + ).toBeGreaterThan(Date.now()); }); it('closes review phases when no pending reviews remain', async () => { diff --git a/src/autopilot/services/scheduler.service.ts b/src/autopilot/services/scheduler.service.ts index bcdab85..52f4743 100644 --- a/src/autopilot/services/scheduler.service.ts +++ b/src/autopilot/services/scheduler.service.ts @@ -418,6 +418,9 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { `[REVIEW LATE] Unable to verify pending reviews for phase ${data.phaseId} on challenge ${data.challengeId}: ${err.message}`, err.stack, ); + + await this.deferReviewPhaseClosure(data); + return; } } @@ -627,7 +630,7 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { private async deferReviewPhaseClosure( data: PhaseTransitionPayload, - pendingCount: number, + pendingCount?: number, ): Promise { const key = this.buildReviewPhaseKey(data.challengeId, data.phaseId); const attempt = (this.reviewCloseRetryAttempts.get(key) ?? 0) + 1; @@ -644,8 +647,13 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { try { await this.schedulePhaseTransition(payload); + const pendingDescription = + typeof pendingCount === 'number' && pendingCount >= 0 + ? pendingCount + : 'unknown'; + this.logger.warn( - `[REVIEW LATE] Deferred closing review phase ${data.phaseId} for challenge ${data.challengeId}; ${pendingCount} incomplete review(s) detected. Retrying in ${Math.round(delay / 60000)} minute(s).`, + `[REVIEW LATE] Deferred closing review phase ${data.phaseId} for challenge ${data.challengeId}; ${pendingDescription} incomplete review(s) detected. Retrying in ${Math.round(delay / 60000)} minute(s).`, ); } catch (error) { const err = error as Error; diff --git a/src/kafka/kafka.service.ts b/src/kafka/kafka.service.ts index 77aedb4..e38d4cf 100644 --- a/src/kafka/kafka.service.ts +++ b/src/kafka/kafka.service.ts @@ -46,9 +46,9 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { constructor(private readonly configService: ConfigService) { try { - const brokersValue = this.configService.get( - 'kafka.brokers', - ); + const brokersValue = this.configService.get< + string | string[] | undefined + >('kafka.brokers'); const kafkaBrokers = Array.isArray(brokersValue) ? brokersValue : brokersValue?.split(',') || CONFIG.KAFKA.DEFAULT_BROKERS; @@ -73,7 +73,10 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { this.producer = this.createProducer(); } catch (error) { - const err = this.normalizeError(error, 'Failed to initialize Kafka service'); + const err = this.normalizeError( + error, + 'Failed to initialize Kafka service', + ); this.logger.error(err.message, { error: err.stack || err.message }); throw new KafkaConnectionException({ error: err.stack || err.message, @@ -112,7 +115,10 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { timestamp: new Date(timestamp).toISOString(), }); } catch (error) { - const err = this.normalizeError(error, `Failed to produce message to ${topic}`); + const err = this.normalizeError( + error, + `Failed to produce message to ${topic}`, + ); this.logger.error(err.message, { correlationId, error: err.stack || err.message, @@ -242,21 +248,23 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { this.logger.info('Closing Kafka consumers...'); await Promise.all( - Array.from(this.consumers.entries()).map(async ([groupId, consumer]) => { - try { - await consumer.close(); - this.logger.info(`Consumer ${groupId} closed successfully`); - } catch (error) { - const err = this.normalizeError( - error, - `Error closing consumer ${groupId}`, - ); - this.logger.error(err.message, { - groupId, - error: err.stack || err.message, - }); - } - }), + Array.from(this.consumers.entries()).map( + async ([groupId, consumer]) => { + try { + await consumer.close(); + this.logger.info(`Consumer ${groupId} closed successfully`); + } catch (error) { + const err = this.normalizeError( + error, + `Error closing consumer ${groupId}`, + ); + this.logger.error(err.message, { + groupId, + error: err.stack || err.message, + }); + } + }, + ), ); this.logger.info('Closing Kafka producer...'); @@ -264,7 +272,10 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { this.logger.info('Kafka connections closed successfully'); } catch (error) { const err = this.normalizeError(error, 'Error during Kafka shutdown'); - this.logger.error(err.message, { signal, error: err.stack || err.message }); + this.logger.error(err.message, { + signal, + error: err.stack || err.message, + }); throw err; } finally { this.consumerLoops.clear(); @@ -277,7 +288,9 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { try { return ( this.producer.isConnected() && - Array.from(this.consumers.values()).every((consumer) => consumer.isConnected()) + Array.from(this.consumers.values()).every((consumer) => + consumer.isConnected(), + ) ); } catch (error) { const err = this.normalizeError( @@ -365,7 +378,9 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { for await (const message of stream) { const correlationId = this.getHeaderValue(message.headers, 'correlation-id') || uuidv4(); - const messageTimestamp = Number(message.timestamp ?? BigInt(Date.now())); + const messageTimestamp = Number( + message.timestamp ?? BigInt(Date.now()), + ); try { if (message.value === undefined) { @@ -404,17 +419,19 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { partition: message.partition, error: err.stack || err.message, }); - await this.sendToDLQ(message.topic, message.value).catch((dlqError) => { - const dlqErr = this.normalizeError( - dlqError, - `Failed to send message to DLQ for topic ${message.topic}`, - ); - this.logger.error(dlqErr.message, { - correlationId, - topic: message.topic, - error: dlqErr.stack || dlqErr.message, - }); - }); + await this.sendToDLQ(message.topic, message.value).catch( + (dlqError) => { + const dlqErr = this.normalizeError( + dlqError, + `Failed to send message to DLQ for topic ${message.topic}`, + ); + this.logger.error(dlqErr.message, { + correlationId, + topic: message.topic, + error: dlqErr.stack || dlqErr.message, + }); + }, + ); } } } catch (error) { @@ -490,7 +507,10 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { }); } - private async sendToDLQ(originalTopic: string, message: unknown): Promise { + private async sendToDLQ( + originalTopic: string, + message: unknown, + ): Promise { const dlqTopic = `${originalTopic}.dlq`; const serializedMessage = this.serializeForDlq(message); @@ -515,7 +535,10 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { return Buffer.from(JSON.stringify(message), 'utf8').toString('base64'); } catch (error) { - const fallback = this.normalizeError(error, 'Failed to serialize DLQ message'); + const fallback = this.normalizeError( + error, + 'Failed to serialize DLQ message', + ); this.logger.warn(fallback.message, { error: fallback.stack || fallback.message, }); @@ -545,4 +568,3 @@ export class KafkaService implements OnApplicationShutdown, OnModuleInit { } } } - diff --git a/src/review/review.service.ts b/src/review/review.service.ts index 40de90d..04d9232 100644 --- a/src/review/review.service.ts +++ b/src/review/review.service.ts @@ -231,7 +231,7 @@ export class ReviewService { WHERE "phaseId" = ${phaseId} AND ( "status" IS NULL - OR UPPER("status") NOT IN ('COMPLETED', 'NO_REVIEW') + OR UPPER(("status")::text) NOT IN ('COMPLETED', 'NO_REVIEW') ) `; From bfbdbdaa89938861643200e119d49ea9f14ff5ed Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Sat, 27 Sep 2025 09:09:44 +1000 Subject: [PATCH 23/23] Fix issue with duplicate pending reviews getting created --- .../services/phase-review.service.ts | 6 +++-- src/review/review.service.ts | 24 ++++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/autopilot/services/phase-review.service.ts b/src/autopilot/services/phase-review.service.ts index 2583b3d..4929d5a 100644 --- a/src/autopilot/services/phase-review.service.ts +++ b/src/autopilot/services/phase-review.service.ts @@ -91,7 +91,7 @@ export class PhaseReviewService { } try { - await this.reviewService.createPendingReview( + const created = await this.reviewService.createPendingReview( submissionId, resource.id, phase.id, @@ -99,7 +99,9 @@ export class PhaseReviewService { challengeId, ); existingPairs.add(key); - createdCount++; + if (created) { + createdCount++; + } } catch (error) { const err = error as Error; this.logger.error( diff --git a/src/review/review.service.ts b/src/review/review.service.ts index 04d9232..c356bd4 100644 --- a/src/review/review.service.ts +++ b/src/review/review.service.ts @@ -272,7 +272,7 @@ export class ReviewService { phaseId: string, scorecardId: string, challengeId: string, - ): Promise { + ): Promise { const insert = Prisma.sql` INSERT INTO ${ReviewService.REVIEW_TABLE} ( "resourceId", @@ -282,7 +282,8 @@ export class ReviewService { "status", "createdAt", "updatedAt" - ) VALUES ( + ) + SELECT ${resourceId}, ${phaseId}, ${submissionId}, @@ -290,11 +291,25 @@ export class ReviewService { 'PENDING', NOW(), NOW() + WHERE NOT EXISTS ( + SELECT 1 + FROM ${ReviewService.REVIEW_TABLE} existing + WHERE existing."resourceId" = ${resourceId} + AND existing."phaseId" = ${phaseId} + AND existing."submissionId" = ${submissionId} + AND ( + existing."scorecardId" = ${scorecardId} + OR ( + existing."scorecardId" IS NULL + AND ${scorecardId} IS NULL + ) + ) ) `; try { - await this.prisma.$executeRaw(insert); + const rowsInserted = await this.prisma.$executeRaw(insert); + const created = rowsInserted > 0; void this.dbLogger.logAction('review.createPendingReview', { challengeId, @@ -304,8 +319,11 @@ export class ReviewService { resourceId, submissionId, phaseId, + created, }, }); + + return created; } catch (error) { const err = error as Error; void this.dbLogger.logAction('review.createPendingReview', {