From 6f1ef837d5018de5281fe43fbb80089845136a64 Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Thu, 9 Oct 2025 18:58:29 +0300 Subject: [PATCH 01/48] PM-2222 - send notification when ai workflow run has compelted --- src/shared/config/common.config.ts | 3 + .../modules/global/workflow-queue.handler.ts | 86 ++++++++++++++++++- 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/src/shared/config/common.config.ts b/src/shared/config/common.config.ts index 7ef2dca..d3b2f0a 100644 --- a/src/shared/config/common.config.ts +++ b/src/shared/config/common.config.ts @@ -70,5 +70,8 @@ export const CommonConfig = { contactManagersEmailTemplate: process.env.SENDGRID_CONTACT_MANAGERS_TEMPLATE ?? 'd-00000000000000000000000000000000', + aiWorkflowRunCompletedEmailTemplate: + process.env.SENDGRID_AI_WORKFLOW_RUN_COMPLETED_TEMPLATE ?? + 'd-7d14d986ba0a4317b449164b73939910', }, }; diff --git a/src/shared/modules/global/workflow-queue.handler.ts b/src/shared/modules/global/workflow-queue.handler.ts index 166e827..3676e87 100644 --- a/src/shared/modules/global/workflow-queue.handler.ts +++ b/src/shared/modules/global/workflow-queue.handler.ts @@ -3,7 +3,9 @@ import { GiteaService } from './gitea.service'; import { PrismaService } from './prisma.service'; import { QueueSchedulerService } from './queue-scheduler.service'; import { Job } from 'pg-boss'; -import { aiWorkflowRun } from '@prisma/client'; +import { aiWorkflow, aiWorkflowRun } from '@prisma/client'; +import { EventBusSendEmailPayload, EventBusService } from './eventBus.service'; +import { CommonConfig } from 'src/shared/config/common.config'; @Injectable() export class WorkflowQueueHandler implements OnModuleInit { @@ -13,6 +15,7 @@ export class WorkflowQueueHandler implements OnModuleInit { private readonly prisma: PrismaService, private readonly scheduler: QueueSchedulerService, private readonly giteaService: GiteaService, + private readonly eventBusService: EventBusService, ) {} async onModuleInit() { @@ -150,7 +153,7 @@ export class WorkflowQueueHandler implements OnModuleInit { return; } - let [aiWorkflowRun]: (aiWorkflowRun | null)[] = aiWorkflowRuns; + let [aiWorkflowRun]: ((typeof aiWorkflowRuns)[0] | null)[] = aiWorkflowRuns; if ( !aiWorkflowRun && @@ -299,6 +302,7 @@ export class WorkflowQueueHandler implements OnModuleInit { } } catch (e) { this.logger.log(aiWorkflowRun.id, e.message); + return; } this.logger.log({ @@ -309,9 +313,87 @@ export class WorkflowQueueHandler implements OnModuleInit { status: conclusion, timestamp: new Date().toISOString(), }); + + try { + await this.sendWorkflowRunCompletedNotification(aiWorkflowRun); + } catch (e) { + this.logger.log( + `Failed to send workflowRun compelted notification for aiWorkflowRun ${aiWorkflowRun.id}. Got error ${e.message ?? e}!`, + ); + } break; default: break; } } + + async sendWorkflowRunCompletedNotification( + aiWorkflowRun: aiWorkflowRun & { workflow: aiWorkflow }, + ) { + const submission = await this.prisma.submission.findUnique({ + where: { id: aiWorkflowRun.submissionId }, + }); + + if (!submission) { + this.logger.log( + `Failed to send workflowRun compelted notification for aiWorkflowRun ${aiWorkflowRun.id}. Submission ${aiWorkflowRun.submissionId} is missing!`, + ); + return; + } + + const [challenge] = await this.prisma.$queryRaw<{ name: string }[]>` + SELECT + id, + name + FROM challenges."Challenge" c + WHERE c.id=${submission.challengeId} + `; + + if (!challenge) { + this.logger.log( + `Failed to send workflowRun compelted notification for aiWorkflowRun ${aiWorkflowRun.id}. Challenge ${submission.challengeId} couldn't be fetched!`, + ); + return; + } + + const [user] = await this.prisma.$queryRaw< + { + handle: string; + email: string; + firstName?: string; + lastName?: string; + }[] + >` + SELECT + handle, + email, + "firstName", + "lastName" + FROM members.member u + WHERE u."userId"::text=${submission.memberId} + `; + + if (!user) { + this.logger.log( + `Failed to send workflowRun compelted notification for aiWorkflowRun ${aiWorkflowRun.id}. User ${submission.memberId} couldn't be fetched!`, + ); + return; + } + + await this.eventBusService.sendEmail({ + ...new EventBusSendEmailPayload(), + sendgrid_template_id: + CommonConfig.sendgridConfig.aiWorkflowRunCompletedEmailTemplate, + recipients: [user.email], + data: { + userName: + [user.firstName, user.lastName].filter(Boolean).join(' ') ?? + user.handle, + aiWorkflowName: aiWorkflowRun.workflow.name, + reviewLink: '', + submissionId: submission.id, + challengeName: challenge.name, + }, + }); + } } From a6bbd9198a152b3d9be62610668ff326e24c506f Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Fri, 10 Oct 2025 08:33:30 +0300 Subject: [PATCH 02/48] add review link for ai run completed notification --- src/shared/modules/global/workflow-queue.handler.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/shared/modules/global/workflow-queue.handler.ts b/src/shared/modules/global/workflow-queue.handler.ts index 3676e87..871cfae 100644 --- a/src/shared/modules/global/workflow-queue.handler.ts +++ b/src/shared/modules/global/workflow-queue.handler.ts @@ -341,7 +341,9 @@ export class WorkflowQueueHandler implements OnModuleInit { return; } - const [challenge] = await this.prisma.$queryRaw<{ name: string }[]>` + const [challenge] = await this.prisma.$queryRaw< + { id: string; name: string }[] + >` SELECT id, name @@ -390,7 +392,7 @@ export class WorkflowQueueHandler implements OnModuleInit { [user.firstName, user.lastName].filter(Boolean).join(' ') ?? user.handle, aiWorkflowName: aiWorkflowRun.workflow.name, - reviewLink: '', + reviewLink: `${CommonConfig.ui.reviewUIUrl}/review/active-challenges/${challenge.id}/scorecard-details/${submission.id}#aiWorkflowRunId=${aiWorkflowRun.id}`, submissionId: submission.id, challengeName: challenge.name, }, From c0a0e3ab6cacdcd5014aec404d287f6a02353a5f Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Sun, 12 Oct 2025 22:07:09 +0300 Subject: [PATCH 03/48] typo fix & common config --- src/shared/config/common.config.ts | 4 ++++ src/shared/modules/global/workflow-queue.handler.ts | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/shared/config/common.config.ts b/src/shared/config/common.config.ts index d3b2f0a..ad844a4 100644 --- a/src/shared/config/common.config.ts +++ b/src/shared/config/common.config.ts @@ -74,4 +74,8 @@ export const CommonConfig = { process.env.SENDGRID_AI_WORKFLOW_RUN_COMPLETED_TEMPLATE ?? 'd-7d14d986ba0a4317b449164b73939910', }, + ui: { + reviewUIUrl: + process.env.REVIEW_UI_URL ?? 'https://review-v6.topcoder-dev.com', + }, }; diff --git a/src/shared/modules/global/workflow-queue.handler.ts b/src/shared/modules/global/workflow-queue.handler.ts index 871cfae..4c9bd29 100644 --- a/src/shared/modules/global/workflow-queue.handler.ts +++ b/src/shared/modules/global/workflow-queue.handler.ts @@ -318,7 +318,7 @@ export class WorkflowQueueHandler implements OnModuleInit { await this.sendWorkflowRunCompletedNotification(aiWorkflowRun); } catch (e) { this.logger.log( - `Failed to send workflowRun compelted notification for aiWorkflowRun ${aiWorkflowRun.id}. Got error ${e.message ?? e}!`, + `Failed to send workflowRun completed notification for aiWorkflowRun ${aiWorkflowRun.id}. Got error ${e.message ?? e}!`, ); } break; @@ -336,7 +336,7 @@ export class WorkflowQueueHandler implements OnModuleInit { if (!submission) { this.logger.log( - `Failed to send workflowRun compelted notification for aiWorkflowRun ${aiWorkflowRun.id}. Submission ${aiWorkflowRun.submissionId} is missing!`, + `Failed to send workflowRun completed notification for aiWorkflowRun ${aiWorkflowRun.id}. Submission ${aiWorkflowRun.submissionId} is missing!`, ); return; } @@ -353,7 +353,7 @@ export class WorkflowQueueHandler implements OnModuleInit { if (!challenge) { this.logger.log( - `Failed to send workflowRun compelted notification for aiWorkflowRun ${aiWorkflowRun.id}. Challenge ${submission.challengeId} couldn't be fetched!`, + `Failed to send workflowRun completed notification for aiWorkflowRun ${aiWorkflowRun.id}. Challenge ${submission.challengeId} couldn't be fetched!`, ); return; } @@ -377,7 +377,7 @@ export class WorkflowQueueHandler implements OnModuleInit { if (!user) { this.logger.log( - `Failed to send workflowRun compelted notification for aiWorkflowRun ${aiWorkflowRun.id}. User ${submission.memberId} couldn't be fetched!`, + `Failed to send workflowRun completed notification for aiWorkflowRun ${aiWorkflowRun.id}. User ${submission.memberId} couldn't be fetched!`, ); return; } @@ -389,7 +389,7 @@ export class WorkflowQueueHandler implements OnModuleInit { recipients: [user.email], data: { userName: - [user.firstName, user.lastName].filter(Boolean).join(' ') ?? + [user.firstName, user.lastName].filter(Boolean).join(' ') || user.handle, aiWorkflowName: aiWorkflowRun.workflow.name, reviewLink: `${CommonConfig.ui.reviewUIUrl}/review/active-challenges/${challenge.id}/scorecard-details/${submission.id}#aiWorkflowRunId=${aiWorkflowRun.id}`, From b4cdbf63ed2f96a9c121d992145a7e1e26f4080c Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Mon, 13 Oct 2025 14:21:06 +0300 Subject: [PATCH 04/48] use the correct db connection --- src/shared/modules/global/workflow-queue.handler.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/shared/modules/global/workflow-queue.handler.ts b/src/shared/modules/global/workflow-queue.handler.ts index 4c9bd29..70c67e1 100644 --- a/src/shared/modules/global/workflow-queue.handler.ts +++ b/src/shared/modules/global/workflow-queue.handler.ts @@ -6,6 +6,8 @@ import { Job } from 'pg-boss'; import { aiWorkflow, aiWorkflowRun } from '@prisma/client'; import { EventBusSendEmailPayload, EventBusService } from './eventBus.service'; import { CommonConfig } from 'src/shared/config/common.config'; +import { ChallengePrismaService } from './challenge-prisma.service'; +import { MemberPrismaService } from './member-prisma.service'; @Injectable() export class WorkflowQueueHandler implements OnModuleInit { @@ -13,6 +15,8 @@ export class WorkflowQueueHandler implements OnModuleInit { constructor( private readonly prisma: PrismaService, + private readonly challengesPrisma: ChallengePrismaService, + private readonly membersPrisma: MemberPrismaService, private readonly scheduler: QueueSchedulerService, private readonly giteaService: GiteaService, private readonly eventBusService: EventBusService, @@ -341,7 +345,7 @@ export class WorkflowQueueHandler implements OnModuleInit { return; } - const [challenge] = await this.prisma.$queryRaw< + const [challenge] = await this.challengesPrisma.$queryRaw< { id: string; name: string }[] >` SELECT @@ -358,7 +362,7 @@ export class WorkflowQueueHandler implements OnModuleInit { return; } - const [user] = await this.prisma.$queryRaw< + const [user] = await this.membersPrisma.$queryRaw< { handle: string; email: string; From 3d9eb9286bafbc67e81c08b28154a693fd3186e5 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 15 Oct 2025 12:12:03 +1100 Subject: [PATCH 05/48] Better handling of "isLatest" for unlimited submission type challenges --- src/api/review/review.service.spec.ts | 20 ++ src/api/review/review.service.ts | 27 +-- src/api/submission/submission.service.spec.ts | 178 ++++++++++++++++++ src/api/submission/submission.service.ts | 166 +++++++++++++++- src/dto/submission.dto.ts | 3 +- 5 files changed, 377 insertions(+), 17 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index e043322..a41c36e 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -757,6 +757,26 @@ describe('ReviewService.getReview authorization checks', () => { ).resolves.toMatchObject({ id: 'review-1', resourceId: 'resource-1' }); }); + it('allows checkpoint screeners to access their own reviews before completion', async () => { + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + { + ...baseReviewerResource, + roleName: 'Checkpoint Screener', + }, + ]); + + prismaMock.review.findUniqueOrThrow.mockImplementation(() => + Promise.resolve({ + ...defaultReviewData(), + resourceId: 'resource-1', + }), + ); + + await expect( + service.getReview(baseAuthUser, 'review-1'), + ).resolves.toMatchObject({ id: 'review-1', resourceId: 'resource-1' }); + }); + it('blocks submitters from accessing reviews before appeals or iterative review completion', async () => { const submitterUser: JwtUser = { userId: 'submitter-1', diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 9931e70..bd7b00d 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -3028,8 +3028,10 @@ export class ReviewService { uid, ); reviewerResources = resources.filter((r) => { - const rn = (r.roleName || '').toLowerCase(); - return rn.includes('reviewer'); + const roleName = (r.roleName || '').toLowerCase(); + return REVIEW_ACCESS_ROLE_KEYWORDS.some((keyword) => + roleName.includes(keyword), + ); }); hasCopilotRole = resources.some((r) => (r.roleName || '').toLowerCase().includes('copilot'), @@ -3278,14 +3280,14 @@ export class ReviewService { return false; } - const enforceableNames = new Set([ + const names = new Set([ 'review', 'iterative review', 'screening', 'checkpoint screening', ]); - if (enforceableNames.has(normalized)) { + if (names.has(normalized)) { return true; } @@ -3296,12 +3298,12 @@ export class ReviewService { challenge: ChallengeData | null | undefined, ): boolean { if (!challenge?.metadata) { - return false; + return true; } const rawValue = challenge.metadata['submissionLimit']; if (rawValue == null) { - return false; + return true; } let parsed: unknown = rawValue; @@ -3309,7 +3311,7 @@ export class ReviewService { if (typeof rawValue === 'string') { const trimmed = rawValue.trim(); if (!trimmed) { - return false; + return true; } try { @@ -3323,7 +3325,7 @@ export class ReviewService { if (['unlimited', 'false', '0', 'no', 'none'].includes(normalized)) { return false; } - return false; + return true; } } @@ -3340,7 +3342,7 @@ export class ReviewService { if (['unlimited', 'false', '0', 'no', 'none'].includes(normalized)) { return false; } - return false; + return true; } if (parsed && typeof parsed === 'object') { @@ -3371,10 +3373,13 @@ export class ReviewService { if (limitFlag === true) { return true; } - return false; + if (limitFlag === false) { + return false; + } + return true; } - return false; + return true; } private extractSubmissionLimitUnlimited(value: unknown): boolean | null { diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index 45e5be5..d7b3113 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -1,4 +1,5 @@ import { ForbiddenException } from '@nestjs/common'; +import { SubmissionStatus, SubmissionType } from '@prisma/client'; import { Readable } from 'stream'; import { SubmissionService } from './submission.service'; import { UserRole } from 'src/shared/enums/userRole.enum'; @@ -268,4 +269,181 @@ describe('SubmissionService', () => { expect(s3Send).not.toHaveBeenCalled(); }); }); + + describe('listSubmission', () => { + let prismaMock: { + submission: { + findMany: jest.Mock; + count: jest.Mock; + findFirst: jest.Mock; + }; + }; + let prismaErrorServiceMock: { handleError: jest.Mock }; + let challengePrismaMock: { + challengeMetadata: { findMany: jest.Mock }; + }; + let listService: SubmissionService; + + beforeEach(() => { + prismaMock = { + submission: { + findMany: jest.fn(), + count: jest.fn(), + findFirst: jest.fn(), + }, + }; + prismaErrorServiceMock = { + handleError: jest.fn(), + }; + challengePrismaMock = { + challengeMetadata: { + findMany: jest.fn(), + }, + }; + challengePrismaMock.challengeMetadata.findMany.mockResolvedValue([]); + listService = new SubmissionService( + prismaMock as any, + prismaErrorServiceMock as any, + challengePrismaMock as any, + {} as any, + { + validateSubmitterRegistration: jest.fn(), + getMemberResourcesRoles: jest.fn(), + } as any, + {} as any, + {} as any, + { member: { findMany: jest.fn() } } as any, + ); + }); + + it('applies default ordering and marks the newest submission as latest', async () => { + const submissions = [ + { + id: 'submission-old', + challengeId: 'challenge-1', + memberId: 'member-1', + submittedDate: new Date('2024-01-01T10:00:00Z'), + createdAt: new Date('2024-01-01T10:00:00Z'), + updatedAt: new Date('2024-01-01T10:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + { + id: 'submission-new', + challengeId: 'challenge-1', + memberId: 'member-1', + submittedDate: new Date('2024-01-02T12:00:00Z'), + createdAt: new Date('2024-01-02T12:00:00Z'), + updatedAt: new Date('2024-01-02T12:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + ]; + + prismaMock.submission.findMany.mockResolvedValue( + submissions.map((entry) => ({ ...entry })), + ); + prismaMock.submission.count.mockResolvedValue(submissions.length); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'submission-new', + }); + + const result = await listService.listSubmission( + { isMachine: false } as any, + { challengeId: 'challenge-1' } as any, + { page: 1, perPage: 50 } as any, + ); + + expect(prismaMock.submission.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + orderBy: [ + { submittedDate: 'desc' }, + { createdAt: 'desc' }, + { updatedAt: 'desc' }, + { id: 'desc' }, + ], + }), + ); + + expect( + challengePrismaMock.challengeMetadata.findMany, + ).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + challengeId: { in: ['challenge-1'] }, + }), + }), + ); + + const latestEntries = result.data.filter((entry) => entry.isLatest); + expect(latestEntries.map((entry) => entry.id)).toEqual([ + 'submission-new', + ]); + }); + + it('omits isLatest when submission metadata indicates unlimited submissions', async () => { + challengePrismaMock.challengeMetadata.findMany.mockResolvedValue([ + { + challengeId: 'challenge-1', + value: '{"unlimited":"true","limit":"false","count":""}', + }, + ]); + + const submissions = [ + { + id: 'submission-old', + challengeId: 'challenge-1', + memberId: 'member-1', + submittedDate: new Date('2024-01-01T10:00:00Z'), + createdAt: new Date('2024-01-01T10:00:00Z'), + updatedAt: new Date('2024-01-01T10:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + { + id: 'submission-new', + challengeId: 'challenge-1', + memberId: 'member-1', + submittedDate: new Date('2024-01-02T12:00:00Z'), + createdAt: new Date('2024-01-02T12:00:00Z'), + updatedAt: new Date('2024-01-02T12:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + ]; + + prismaMock.submission.findMany.mockResolvedValue( + submissions.map((entry) => ({ ...entry })), + ); + prismaMock.submission.count.mockResolvedValue(submissions.length); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'submission-new', + }); + + const result = await listService.listSubmission( + { isMachine: false } as any, + { challengeId: 'challenge-1' } as any, + { page: 1, perPage: 50 } as any, + ); + + expect(result.data[0]).not.toHaveProperty('isLatest'); + expect(result.data[1]).not.toHaveProperty('isLatest'); + }); + }); }); diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index 90947f9..bde63e5 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -40,6 +40,7 @@ import { Upload } from '@aws-sdk/lib-storage'; import { Readable, PassThrough } from 'stream'; import { EventBusService } from 'src/shared/modules/global/eventBus.service'; import { SubmissionAccessAuditResponseDto } from 'src/dto/submission-access-audit.dto'; +import { Prisma } from '@prisma/client'; type SubmissionMinimal = { id: string; @@ -1374,7 +1375,7 @@ export class SubmissionService { } } await this.populateLatestSubmissionFlags([data]); - await this.populateLatestSubmissionFlags([data]); + await this.stripIsLatestForUnlimitedChallenges([data]); return this.buildResponse(data); } catch (error) { const errorResponse = this.prismaErrorService.handleError( @@ -1399,12 +1400,30 @@ export class SubmissionService { try { const { page = 1, perPage = 10 } = paginationDto || {}; const skip = (page - 1) * perPage; - let orderBy; + type OrderByClause = Record; + + const defaultOrderBy: OrderByClause[] = [ + { submittedDate: 'desc' as const }, + { createdAt: 'desc' as const }, + { updatedAt: 'desc' as const }, + { id: 'desc' as const }, + ]; + + let orderBy: OrderByClause[] = [...defaultOrderBy]; if (sortDto && sortDto.orderBy && sortDto.sortBy) { - orderBy = { - [sortDto.sortBy]: sortDto.orderBy.toLowerCase(), + const direction = + sortDto.orderBy.toLowerCase() === 'asc' ? 'asc' : 'desc'; + const primaryOrder: OrderByClause = { + [sortDto.sortBy]: direction, }; + + const fallbackOrder = defaultOrderBy.filter((entry) => { + const [key] = Object.keys(entry); + return key !== sortDto.sortBy; + }); + + orderBy = [primaryOrder, ...fallbackOrder]; } const requestedMemberId = queryDto.memberId @@ -1522,6 +1541,7 @@ export class SubmissionService { }); await this.populateLatestSubmissionFlags(submissions); + await this.stripIsLatestForUnlimitedChallenges(submissions); this.logger.log( `Found ${submissions.length} submissions (page ${page} of ${Math.ceil(totalCount / perPage)})`, @@ -1608,6 +1628,7 @@ export class SubmissionService { async getSubmission(submissionId: string): Promise { const data = await this.checkSubmission(submissionId); await this.populateLatestSubmissionFlags([data]); + await this.stripIsLatestForUnlimitedChallenges([data]); return this.buildResponse(data); } @@ -1946,6 +1967,139 @@ export class SubmissionService { } } + private async stripIsLatestForUnlimitedChallenges( + submissions: Array< + { challengeId?: string | null } & Record + >, + ): Promise { + if (!submissions.length) { + return; + } + + const challengeId = Array.from( + new Set( + submissions + .map((submission) => + submission.challengeId !== undefined && + submission.challengeId !== null + ? String(submission.challengeId) + : null, + ) + .filter((id): id is string => !!id), + ), + )[0]; + + let metadataEntries: Array<{ value: string }> = []; + try { + metadataEntries = await this.challengePrisma.$queryRaw(Prisma.sql` + SELECT \"value\" from \"ChallengeMetadata\" WHERE \"challengeId\"= ${challengeId} AND name = 'submissionLimit' + `); + } catch (error) { + this.logger.warn( + `Failed to load submissionLimit metadata for challenge ${challengeId}: ${(error as Error)?.message}`, + ); + return; + } + + if (!metadataEntries.length) { + return; + } + + let challengeSubmissionsAreUnlimited = false; + for (const entry of metadataEntries) { + const unlimited = this.extractSubmissionLimitUnlimited(entry.value); + if (unlimited === true) { + challengeSubmissionsAreUnlimited = true; + } + } + + for (const submission of submissions) { + if (challengeSubmissionsAreUnlimited) { + delete (submission as any).isLatest; + } + } + } + + private extractSubmissionLimitUnlimited(value: unknown): boolean | null { + if (value == null) { + return null; + } + + if (typeof value === 'string') { + const trimmed = value.trim(); + if (!trimmed) { + return null; + } + + try { + const parsed = JSON.parse(trimmed); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return this.coerceLooseBoolean( + (parsed as Record).unlimited, + ); + } + return this.coerceLooseBoolean(parsed); + } catch { + return this.coerceLooseBoolean(trimmed); + } + } + + if (typeof value === 'object' && !Array.isArray(value)) { + return this.coerceLooseBoolean( + (value as Record).unlimited, + ); + } + + return this.coerceLooseBoolean(value); + } + + private coerceLooseBoolean(value: unknown): boolean | null { + if (value == null) { + return null; + } + + if (value === true || value === false) { + return value; + } + + if (value instanceof Boolean) { + return value.valueOf(); + } + + if (value instanceof Number) { + return this.coerceLooseBoolean(value.valueOf()); + } + + let candidate: string; + + if (typeof value === 'string') { + candidate = value; + } else if (value instanceof String) { + candidate = value.valueOf(); + } else if (typeof value === 'number') { + if (!Number.isFinite(value)) { + return null; + } + candidate = value.toString(); + } else if (typeof value === 'bigint') { + candidate = value.toString(); + } else { + return null; + } + + const normalized = candidate.trim().toLowerCase(); + if (!normalized) { + return null; + } + if (normalized === 'true') { + return true; + } + if (normalized === 'false') { + return false; + } + return null; + } + private buildResponse(data: any): SubmissionResponseDto { const dto: SubmissionResponseDto = { ...data, @@ -1958,7 +2112,9 @@ export class SubmissionService { if (data.reviewSummation) { dto.reviewSummation = data.reviewSummation; } - dto.isLatest = Boolean(data.isLatest); + if (Object.prototype.hasOwnProperty.call(data, 'isLatest')) { + dto.isLatest = Boolean(data.isLatest); + } return dto; } } diff --git a/src/dto/submission.dto.ts b/src/dto/submission.dto.ts index c3878d7..3d19097 100644 --- a/src/dto/submission.dto.ts +++ b/src/dto/submission.dto.ts @@ -356,6 +356,7 @@ export class SubmissionResponseDto { description: 'Indicates whether this is the most recent submission for the member on this challenge', example: true, + required: false, }) - isLatest: boolean; + isLatest?: boolean; } From a84d5851323e549fcaed65f34c26623691ac4ebb Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 15 Oct 2025 18:32:35 +1100 Subject: [PATCH 06/48] Updated data for F2F iterative reviews for trimmed data for reviews / submissions that aren't owned by the current submitter --- src/api/review/review.service.spec.ts | 324 ++++++++++++++++++++++++++ src/api/review/review.service.ts | 113 +++++++-- 2 files changed, 422 insertions(+), 15 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index a41c36e..51a16ef 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -875,6 +875,125 @@ describe('ReviewService.getReview authorization checks', () => { service.getReview(submitterUser, 'review-1'), ).resolves.toMatchObject({ id: 'review-1' }); }); + + it('allows First2Finish submitters to access their review while iterative review is open', async () => { + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [UserRole.Submitter], + isMachine: false, + }; + + prismaMock.review.findUniqueOrThrow.mockResolvedValue({ + ...defaultReviewData(), + submission: { + id: 'submission-1', + challengeId: 'challenge-1', + memberId: submitterUser.userId, + }, + }); + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + { + ...baseReviewerResource, + id: 'resource-submit', + memberId: submitterUser.userId, + roleName: 'Submitter', + roleId: CommonConfig.roles.submitterRoleId, + }, + ]); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([{ id: 'submission-1' }]); + } + return Promise.resolve([{ id: 'submission-1' }]); + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + type: 'First2Finish', + phases: [ + { id: 'phase-sub', name: 'Submission', isOpen: false }, + { id: 'phase-iter', name: 'Iterative Review', isOpen: true }, + ], + }); + + await expect( + service.getReview(submitterUser, 'review-1'), + ).resolves.toMatchObject({ id: 'review-1' }); + }); + + it('trims iterative review details for other submitters on First2Finish challenges', async () => { + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [UserRole.Submitter], + isMachine: false, + }; + + const now = new Date(); + + prismaMock.review.findUniqueOrThrow.mockResolvedValue({ + ...defaultReviewData(), + resourceId: 'resource-reviewer', + phaseId: 'phase-iter', + reviewItems: [ + { + id: 'item-1', + scorecardQuestionId: 'q-1', + initialAnswer: 'Yes', + finalAnswer: 'Yes', + managerComment: 'detail', + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [ + { + id: 'comment-1', + comment: 'test comment', + appeal: { id: 'appeal-1' }, + }, + ], + }, + ], + submission: { + id: 'submission-2', + challengeId: 'challenge-1', + memberId: 'other-submitter', + }, + }); + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + { + ...baseReviewerResource, + id: 'resource-submit', + memberId: submitterUser.userId, + roleName: 'Submitter', + roleId: CommonConfig.roles.submitterRoleId, + }, + ]); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([{ id: 'submission-1' }]); + } + return Promise.resolve([{ id: 'submission-1' }]); + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + status: ChallengeStatus.COMPLETED, + type: 'First2Finish', + phases: [{ id: 'phase-iter', name: 'Iterative Review', isOpen: false }], + }); + + const response = await service.getReview(submitterUser, 'review-1'); + + expect(response.reviewItems).toEqual([]); + expect(response.appeals).toEqual([]); + expect(response.phaseName).toBe('Iterative Review'); + }); }); describe('ReviewService.getReviews reviewer visibility', () => { @@ -1337,6 +1456,211 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(response.data[0].reviewItems).toHaveLength(1); }); + it('allows First2Finish submitters to view their reviews while iterative review is open', async () => { + const now = new Date(); + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [UserRole.Submitter], + isMachine: false, + }; + + const submitterResource = { + ...buildResource('Submitter', submitterUser.userId), + roleId: CommonConfig.roles.submitterRoleId, + memberId: submitterUser.userId, + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + submitterResource, + ]); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId && where?.challengeId) { + return Promise.resolve([{ id: 'submission-1' }]); + } + if (where?.memberId) { + return Promise.resolve([{ id: 'submission-1' }]); + } + if (where?.challengeId) { + return Promise.resolve([ + { id: 'submission-1' }, + { id: 'submission-2' }, + ]); + } + return Promise.resolve([]); + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + type: 'First2Finish', + phases: [ + { id: 'phase-sub', name: 'Submission', isOpen: false }, + { id: 'phase-iter', name: 'Iterative Review', isOpen: true }, + ], + }); + + prismaMock.review.findMany.mockResolvedValue([ + { + id: 'review-1', + resourceId: 'resource-reviewer', + submissionId: 'submission-1', + phaseId: 'phase-iter', + scorecardId: 'scorecard-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + reviewItems: [ + { + id: 'item-1', + scorecardQuestionId: 'question-1', + initialAnswer: 'Yes', + finalAnswer: 'Yes', + managerComment: null, + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 95, + initialScore: 90, + submission: { + id: 'submission-1', + memberId: submitterUser.userId, + challengeId: 'challenge-1', + }, + }, + ]); + prismaMock.review.count.mockResolvedValue(1); + + const response = await service.getReviews( + submitterUser, + undefined, + 'challenge-1', + ); + + expect(response.data).toHaveLength(1); + expect(response.data[0].finalScore).toBe(95); + expect(response.data[0].initialScore).toBe(90); + expect(response.data[0].reviewItems).toHaveLength(1); + + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; + expect(callArgs.where.submissionId).toEqual({ + in: ['submission-1', 'submission-2'], + }); + }); + + it('trims iterative review details for other submitters on First2Finish challenges', async () => { + const now = new Date(); + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [UserRole.Submitter], + isMachine: false, + }; + + const submitterResource = { + ...buildResource('Submitter', submitterUser.userId), + roleId: CommonConfig.roles.submitterRoleId, + memberId: submitterUser.userId, + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + submitterResource, + ]); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId && where?.challengeId) { + return Promise.resolve([{ id: 'submission-own' }]); + } + if (where?.memberId) { + return Promise.resolve([{ id: 'submission-own' }]); + } + if (where?.challengeId === 'challenge-1') { + return Promise.resolve([ + { id: 'submission-own' }, + { id: 'submission-other' }, + ]); + } + return Promise.resolve([]); + }); + + const challengeDetail = { + id: 'challenge-1', + status: ChallengeStatus.COMPLETED, + type: 'First2Finish', + phases: [{ id: 'phase-iter', name: 'Iterative Review', isOpen: false }], + }; + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue( + challengeDetail, + ); + challengeApiServiceMock.getChallenges.mockResolvedValue([challengeDetail]); + + const reviewRecord = { + id: 'review-1', + resourceId: 'resource-reviewer', + submissionId: 'submission-other', + phaseId: 'phase-iter', + scorecardId: 'scorecard-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + reviewItems: [ + { + id: 'item-1', + scorecardQuestionId: 'question-1', + initialAnswer: 'Yes', + finalAnswer: 'Yes', + managerComment: null, + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [ + { + id: 'comment-1', + comment: 'detail', + appeal: { id: 'appeal-1' }, + }, + ], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 88, + initialScore: 80, + submission: { + id: 'submission-other', + memberId: 'second-submitter', + challengeId: 'challenge-1', + }, + }; + + prismaMock.review.findMany.mockResolvedValue([reviewRecord]); + prismaMock.review.count.mockResolvedValue(1); + + const response = await service.getReviews( + submitterUser, + undefined, + 'challenge-1', + ); + + expect(response.data).toHaveLength(1); + expect(response.data[0].reviewItems).toEqual([]); + expect((response.data[0] as any).appeals).toEqual([]); + expect(response.data[0].finalScore).toBe(88); + }); + it('enriches reviewer metadata when member data is available', async () => { const now = new Date(); diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index bd7b00d..d8800c1 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -2469,6 +2469,7 @@ export class ReviewService { await this.challengeApiService.getChallengeDetail(challengeId); const challenge = challengeDetail; const phases = challenge.phases || []; + const isFirst2Finish = this.isFirst2FinishChallenge(challenge); const appealsOpen = phases.some( (p) => p.name === 'Appeals' && p.isOpen, ); @@ -2492,6 +2493,9 @@ export class ReviewService { if (challenge.status === ChallengeStatus.COMPLETED) { // Allowed to see all reviews on this challenge // reviewWhereClause already limited to submissions on this challenge + } else if (isFirst2Finish) { + // First2Finish submitters can view all reviews; details for other submissions + // will be trimmed later in the response mapping. } else if (appealsOpen || appealsResponseOpen) { // Restrict to own reviews (own submissions only) restrictToSubmissionIds(mySubmissionIds); @@ -2887,8 +2891,26 @@ export class ReviewService { finalScore: shouldMaskReviewDetails ? null : review.finalScore, }; + const reviewChallengeId = + review.submission?.challengeId ?? challengeId ?? null; + const challengeForReview = reviewChallengeId + ? (challengeCache.get(reviewChallengeId) ?? null) + : null; + const normalizedPhaseName = (phaseName ?? '').trim().toLowerCase(); + const shouldTrimIterativeReviewForOtherSubmitters = + !isPrivilegedRequester && + submitterSubmissionIdSet.size > 0 && + !hasCopilotRoleForChallenge && + !isReviewerForReview && + !isOwnSubmission && + normalizedPhaseName === 'iterative review' && + this.isFirst2FinishChallenge(challengeForReview); + const shouldStripReviewItems = + shouldMaskReviewDetails || + shouldTrimIterativeReviewForOtherSubmitters; + if (!isThin) { - sanitizedReview.reviewItems = shouldMaskReviewDetails + sanitizedReview.reviewItems = shouldStripReviewItems ? [] : (review.reviewItems ?? []); } else { @@ -3000,6 +3022,12 @@ export class ReviewService { }); const challengeCache = new Map(); + const isPrivilegedRequester = authUser?.isMachine || isAdmin(authUser); + let challengeDetail: ChallengeData | null = null; + let hasCopilotRole = false; + let reviewerResourceIds = new Set(); + let isReviewerForReview = false; + let isSubmitterForChallenge = false; // Authorization for non-M2M, non-admin users if (!authUser?.isMachine && !isAdmin(authUser)) { @@ -3018,9 +3046,9 @@ export class ReviewService { const challenge = await this.challengeApiService.getChallengeDetail(challengeId); challengeCache.set(challengeId, challenge); + challengeDetail = challenge; let reviewerResources: ResourceInfo[] = []; - let hasCopilotRole = false; try { const resources = await this.resourceApiService.getMemberResourcesRoles( @@ -3043,12 +3071,13 @@ export class ReviewService { ); } - if (reviewerResources.length > 0) { - const reviewerResourceIds = new Set( - reviewerResources.map((r) => String(r.id)), - ); - const reviewResourceId = String(data.resourceId ?? ''); + reviewerResourceIds = new Set( + reviewerResources.map((r) => String(r.id)), + ); + const reviewResourceId = String(data.resourceId ?? ''); + isReviewerForReview = reviewerResourceIds.has(reviewResourceId); + if (reviewerResources.length > 0) { if ( challenge.status !== ChallengeStatus.COMPLETED && !reviewerResourceIds.has(reviewResourceId) @@ -3066,7 +3095,8 @@ export class ReviewService { where: { challengeId, memberId: uid }, select: { id: true }, }); - if (mySubs.length === 0) { + isSubmitterForChallenge = mySubs.length > 0; + if (!isSubmitterForChallenge) { throw new ForbiddenException({ message: 'You must have submitted to this challenge to access this review', @@ -3087,7 +3117,8 @@ export class ReviewService { }); } - if (!visibility.allowAny && !isOwnSubmission) { + const isFirst2Finish = this.isFirst2FinishChallenge(challenge); + if (!visibility.allowAny && !isOwnSubmission && !isFirst2Finish) { throw new ForbiddenException({ message: 'Only reviews of your own submission are accessible during Appeals or Appeals Response', @@ -3115,11 +3146,37 @@ export class ReviewService { // ignore } result.appeals = flattenedAppeals; - result.phaseName = await this.resolvePhaseNameFromChallenge({ + const resolvedPhaseName = await this.resolvePhaseNameFromChallenge({ challengeId: data.submission?.challengeId ?? null, phaseId: data.phaseId ?? null, challengeCache, }); + result.phaseName = resolvedPhaseName; + + const normalizedPhaseName = (resolvedPhaseName ?? '') + .trim() + .toLowerCase(); + const requesterId = String(authUser?.userId ?? ''); + const submissionOwnerId = String(data.submission?.memberId ?? ''); + const challengeForReview = data.submission?.challengeId + ? (challengeCache.get(data.submission.challengeId) ?? challengeDetail) + : challengeDetail; + const shouldTrimIterativeReviewForOtherSubmitters = + !isPrivilegedRequester && + isSubmitterForChallenge && + !hasCopilotRole && + !isReviewerForReview && + submissionOwnerId.length > 0 && + requesterId.length > 0 && + submissionOwnerId !== requesterId && + normalizedPhaseName === 'iterative review' && + this.isFirst2FinishChallenge(challengeForReview); + + if (shouldTrimIterativeReviewForOtherSubmitters) { + result.reviewItems = []; + result.appeals = []; + } + return result as ReviewResponseDto; } catch (error) { if (error instanceof ForbiddenException) { @@ -3202,16 +3259,16 @@ export class ReviewService { return matchedPhase?.name ?? null; } - private getSubmitterVisibilityForChallenge(challenge?: { - status?: ChallengeStatus; - phases?: Array<{ name?: string | null; isOpen?: boolean | null }>; - }): { allowAny: boolean; allowOwn: boolean } { + private getSubmitterVisibilityForChallenge( + challenge?: ChallengeData | null, + ): { allowAny: boolean; allowOwn: boolean } { if (!challenge) { return { allowAny: false, allowOwn: false }; } const status = challenge.status; const phases = challenge.phases || []; + const isFirst2Finish = this.isFirst2FinishChallenge(challenge); const normalizeName = (phaseName: string | null | undefined) => String(phaseName ?? '').toLowerCase(); @@ -3239,11 +3296,37 @@ export class ReviewService { allowAny || appealsOpen || appealsResponseOpen || - (!hasAppealsPhases && iterativeReviewClosed); + (!hasAppealsPhases && iterativeReviewClosed) || + isFirst2Finish; return { allowAny, allowOwn }; } + private isFirst2FinishChallenge(challenge?: ChallengeData | null): boolean { + if (!challenge) { + return false; + } + + const typeName = (challenge.type ?? '').trim().toLowerCase(); + if ( + typeName === 'first2finish' || + typeName === 'first 2 finish' || + typeName === 'topgear task' + ) { + return true; + } + + const legacySubTrack = (challenge.legacy?.subTrack ?? '') + .trim() + .toLowerCase(); + + if (legacySubTrack === 'first_2_finish') { + return true; + } + + return false; + } + private shouldEnforceLatestSubmissionForReview( reviewTypeName: string | null | undefined, challenge: ChallengeData | null | undefined, From 0582798173c3b3452fe51b7052fd170e7e8d52f7 Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Wed, 15 Oct 2025 15:53:04 +0300 Subject: [PATCH 07/48] Update challenge reviewer payment fields --- prisma/challenge-schema.prisma | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/prisma/challenge-schema.prisma b/prisma/challenge-schema.prisma index 8f9ba54..a426689 100644 --- a/prisma/challenge-schema.prisma +++ b/prisma/challenge-schema.prisma @@ -589,8 +589,9 @@ model ChallengeReviewer { isMemberReview Boolean memberReviewerCount Int? phaseId String - basePayment Float? - incrementalPayment Float? + fixedAmount Float? @default(0) + baseCoefficient Float? + incrementalCoefficient Float? type ReviewOpportunityTypeEnum? aiWorkflowId String? @db.VarChar(14) @@ -622,8 +623,9 @@ model DefaultChallengeReviewer { isMemberReview Boolean memberReviewerCount Int? phaseName String - basePayment Float? - incrementalPayment Float? + fixedAmount Float? @default(0) + baseCoefficient Float? + incrementalCoefficient Float? opportunityType ReviewOpportunityTypeEnum? isAIReviewer Boolean From 45fa146448cf09c09fb24a99166f3ba00bba55f1 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 16 Oct 2025 10:43:42 +1100 Subject: [PATCH 08/48] Updates to return filtered information on reviews in a challenge to non-submitter owned submissions, allowing us to better show data in the UI. --- src/api/review/review.service.spec.ts | 162 +++++++++++++++++++++++++- src/api/review/review.service.ts | 41 +++++-- 2 files changed, 194 insertions(+), 9 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index 51a16ef..c59b1cf 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -175,6 +175,8 @@ describe('ReviewService.createReview authorization checks', () => { roleName: 'Reviewer', }, ]); + resourcePrismaMock.resource.findMany.mockResolvedValue([]); + memberPrismaMock.member.findMany.mockResolvedValue([]); }); it('throws when resource does not belong to non-admin user', async () => { @@ -1133,7 +1135,7 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(callArgs.where.resourceId).toBeUndefined(); }); - it('restricts submitters to their own submissions when submission phase is closed on an active challenge', async () => { + it('includes non-owned submissions with limited visibility when submission phase is closed on an active challenge', async () => { const submitterResource = { ...buildResource('Submitter'), roleId: CommonConfig.roles.submitterRoleId, @@ -1163,7 +1165,9 @@ describe('ReviewService.getReviews reviewer visibility', () => { await service.getReviews(baseAuthUser, undefined, 'challenge-1'); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.submissionId).toEqual({ in: [baseSubmission.id] }); + expect(callArgs.where.submissionId).toEqual({ + in: [baseSubmission.id, 'submission-other'], + }); }); it('masks scores and review items for submitters before appeals open', async () => { @@ -1326,6 +1330,160 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(response.data[0].reviewItems).toHaveLength(1); }); + it('returns limited review data for non-owned submissions when appeals are open', async () => { + const now = new Date(); + const submitterResource = { + ...buildResource('Submitter'), + roleId: CommonConfig.roles.submitterRoleId, + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + submitterResource, + ]); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Active Challenge', + status: ChallengeStatus.ACTIVE, + phases: [ + { id: 'phase-sub', name: 'Submission', isOpen: false }, + { id: 'phase-appeals', name: 'Appeals', isOpen: true }, + ], + }); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([baseSubmission]); + } + return Promise.resolve([baseSubmission, { id: 'submission-other' }]); + }); + + const ownReview = { + id: 'review-1', + resourceId: 'resource-reviewer', + submissionId: baseSubmission.id, + phaseId: 'phase-appeals', + scorecardId: 'scorecard-1', + typeId: 'type-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: { notes: 'own submission' }, + reviewItems: [ + { + id: 'item-1', + scorecardQuestionId: 'question-1', + initialAnswer: 'Yes', + finalAnswer: 'Yes', + managerComment: null, + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 95.5, + initialScore: 93.2, + submission: { + id: baseSubmission.id, + memberId: baseAuthUser.userId, + challengeId: 'challenge-1', + }, + }; + + const otherReview = { + id: 'review-2', + resourceId: 'resource-reviewer-2', + submissionId: 'submission-other', + phaseId: 'phase-appeals', + scorecardId: 'scorecard-2', + typeId: 'type-2', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: { notes: 'other submission' }, + reviewItems: [ + { + id: 'item-2', + scorecardQuestionId: 'question-2', + initialAnswer: 'No', + finalAnswer: 'No', + managerComment: 'Needs work', + createdAt: now, + createdBy: 'reviewer-2', + updatedAt: now, + updatedBy: 'reviewer-2', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator-2', + updatedAt: now, + updatedBy: 'updater-2', + finalScore: 70.0, + initialScore: 68.0, + submission: { + id: 'submission-other', + memberId: 'other-user', + challengeId: 'challenge-1', + }, + }; + + prismaMock.review.findMany.mockResolvedValue([ownReview, otherReview]); + prismaMock.review.count.mockResolvedValue(2); + + const response = await service.getReviews( + baseAuthUser, + undefined, + 'challenge-1', + ); + + expect(response.data).toHaveLength(2); + const own = response.data.find( + (entry) => entry.submissionId === baseSubmission.id, + ); + const other = response.data.find( + (entry) => entry.submissionId === 'submission-other', + ); + + expect(own).toBeDefined(); + expect(other).toBeDefined(); + expect(own?.finalScore).toBe(95.5); + expect(own?.initialScore).toBe(93.2); + expect(own?.reviewItems).toHaveLength(1); + + const otherResult = other as any; + expect(otherResult && 'finalScore' in otherResult).toBe(false); + expect(otherResult && 'initialScore' in otherResult).toBe(false); + expect(other?.reviewItems).toEqual([]); + expect(other?.reviewDate).toEqual(now); + expect(other?.phaseName).toBe('Appeals'); + expect(other).toMatchObject({ + id: 'review-2', + resourceId: 'resource-reviewer-2', + submissionId: 'submission-other', + scorecardId: 'scorecard-2', + status: ReviewStatus.COMPLETED, + phaseId: 'phase-appeals', + createdAt: now, + createdBy: 'creator-2', + updatedAt: now, + updatedBy: 'updater-2', + }); + expect(other?.reviewerHandle).toBeNull(); + expect(other?.reviewerMaxRating).toBeNull(); + expect(other?.submitterHandle).toBeNull(); + expect(other?.submitterMaxRating).toBeNull(); + expect(other && 'metadata' in other).toBe(false); + expect(other && 'typeId' in other).toBe(false); + expect(other && 'committed' in other).toBe(false); + }); + it('omits nested review details when thin flag is set', async () => { const now = new Date(); const machineUser: JwtUser = { diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index d8800c1..353b0ea 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -2313,6 +2313,7 @@ export class ReviewService { allowAny: false, allowOwn: false, }; + let allowLimitedVisibilityForOtherSubmissions = false; // Utility to merge an allowed set of submission IDs into where clause const restrictToSubmissionIds = (allowedIds: string[]) => { @@ -2497,15 +2498,15 @@ export class ReviewService { // First2Finish submitters can view all reviews; details for other submissions // will be trimmed later in the response mapping. } else if (appealsOpen || appealsResponseOpen) { - // Restrict to own reviews (own submissions only) - restrictToSubmissionIds(mySubmissionIds); + // Allow limited visibility into other submissions; details will be redacted later. + allowLimitedVisibilityForOtherSubmissions = true; } else if ( challenge.status === ChallengeStatus.ACTIVE && submissionPhaseClosed && hasSubmitterRoleForChallenge ) { - // Submitters can access their own submissions once submission phase closes - restrictToSubmissionIds(mySubmissionIds); + // Allow limited visibility into other submissions once submission phase closes. + allowLimitedVisibilityForOtherSubmissions = true; } else { // No access for non-completed, non-appeals phases throw new ForbiddenException({ @@ -2877,18 +2878,29 @@ export class ReviewService { !isReviewerForReview && isOwnSubmission && !submitterVisibilityState.allowOwn; + const shouldLimitNonOwnerVisibility = + allowLimitedVisibilityForOtherSubmissions && + !isPrivilegedRequester && + hasSubmitterRoleForChallenge && + submitterSubmissionIdSet.size > 0 && + !hasCopilotRoleForChallenge && + !isReviewerForReview && + !isOwnSubmission && + !submitterVisibilityState.allowAny; const reviewPhaseId = review.phaseId ? String(review.phaseId) : ''; const phaseName = reviewPhaseId ? (phaseNameCache.get(reviewPhaseId) ?? null) : null; + const sanitizeScores = + shouldMaskReviewDetails || shouldLimitNonOwnerVisibility; const sanitizedReview: typeof review & { reviewItems?: typeof review.reviewItems; } = { ...review, - initialScore: shouldMaskReviewDetails ? null : review.initialScore, - finalScore: shouldMaskReviewDetails ? null : review.finalScore, + initialScore: sanitizeScores ? null : review.initialScore, + finalScore: sanitizeScores ? null : review.finalScore, }; const reviewChallengeId = @@ -2907,7 +2919,8 @@ export class ReviewService { this.isFirst2FinishChallenge(challengeForReview); const shouldStripReviewItems = shouldMaskReviewDetails || - shouldTrimIterativeReviewForOtherSubmitters; + shouldTrimIterativeReviewForOtherSubmitters || + shouldLimitNonOwnerVisibility; if (!isThin) { sanitizedReview.reviewItems = shouldStripReviewItems @@ -2967,6 +2980,20 @@ export class ReviewService { result.submitterHandle = submitterProfile?.handle ?? null; result.submitterMaxRating = submitterProfile?.maxRating ?? null; } + if (shouldLimitNonOwnerVisibility) { + delete (result as any).finalScore; + delete (result as any).initialScore; + result.submitterHandle = null; + result.submitterMaxRating = null; + if (!isThin) { + (result as any).appeals = []; + } else { + delete (result as any).appeals; + } + delete (result as any).metadata; + delete (result as any).typeId; + delete (result as any).committed; + } return result; }); From 6ee41d27e02a0e9b9f617d5e5439ee259cd3a394 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 16 Oct 2025 11:48:30 +1100 Subject: [PATCH 09/48] Fix sending contact manager messages --- src/api/review-application/reviewApplication.service.ts | 4 +++- src/shared/config/common.config.ts | 2 +- src/shared/modules/global/eventBus.service.ts | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/api/review-application/reviewApplication.service.ts b/src/api/review-application/reviewApplication.service.ts index 7fa08c6..8c3119f 100644 --- a/src/api/review-application/reviewApplication.service.ts +++ b/src/api/review-application/reviewApplication.service.ts @@ -545,7 +545,9 @@ export class ReviewApplicationService { // prepare challenge data const challengeName = challengeData.name; const challengeUrl = - CommonConfig.apis.onlineReviewUrlBase + challengeData.legacyId; + CommonConfig.apis.onlineReviewUrlBase + + challengeData.id + + '/challenge-details'; // build event bus message payload const eventBusPayloads: EventBusSendEmailPayload[] = []; for (const entity of entityList) { diff --git a/src/shared/config/common.config.ts b/src/shared/config/common.config.ts index 7ef2dca..751ae0b 100644 --- a/src/shared/config/common.config.ts +++ b/src/shared/config/common.config.ts @@ -50,7 +50,7 @@ export const CommonConfig = { v6ApiUrl: process.env.V6_API_URL ?? 'https://api.topcoder-dev.com/v6', memberApiUrl: process.env.MEMBER_API_URL ?? 'http://localhost:4000/members', onlineReviewUrlBase: - 'https://software.topcoder.com/review/actions/ViewProjectDetails?pid=', + 'https://review.topcoder.com/review/active-challenges/', }, // Resource role configuration roles: { diff --git a/src/shared/modules/global/eventBus.service.ts b/src/shared/modules/global/eventBus.service.ts index 2177726..687a7b0 100644 --- a/src/shared/modules/global/eventBus.service.ts +++ b/src/shared/modules/global/eventBus.service.ts @@ -24,9 +24,8 @@ class EventBusMessage { export class EventBusSendEmailPayload { // Template-specific variables payload. Structure depends on the sendgrid template. data: Record; - from: Record = { - email: 'Topcoder ', - }; + from: string = 'no-reply@topcoder.com'; + replyTo: string = 'no-reply@topcoder.com'; version: string = 'v3'; sendgrid_template_id: string; recipients: string[]; @@ -84,6 +83,7 @@ export class EventBusService { * @param payload send email payload */ async sendEmail(payload: EventBusSendEmailPayload): Promise { + console.log(`${JSON.stringify(payload, null, 2)}`); await this.postMessage('external.action.email', payload); } From 1682350375c2cbd003868c9e3777d4a09b4ee3f2 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 16 Oct 2025 15:06:53 +1100 Subject: [PATCH 10/48] Handle returning screening review details at the appropriate time --- src/api/review/review.service.spec.ts | 169 ++++++++++++++++++++++++++ src/api/review/review.service.ts | 150 ++++++++++++++++++----- 2 files changed, 291 insertions(+), 28 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index c59b1cf..3144734 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -831,6 +831,80 @@ describe('ReviewService.getReview authorization checks', () => { }); }); + it('allows submitters to access their own screening review once the screening phase completes', async () => { + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [UserRole.Submitter], + isMachine: false, + }; + + const now = new Date(); + + prismaMock.review.findUniqueOrThrow.mockResolvedValue({ + ...defaultReviewData(), + phaseId: 'phase-screening', + finalScore: 88.2, + initialScore: 85.4, + reviewItems: [ + { + id: 'item-1', + scorecardQuestionId: 'q-1', + initialAnswer: 'Yes', + finalAnswer: 'No', + managerComment: 'detailed', + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [], + }, + ], + submission: { + id: 'submission-1', + challengeId: 'challenge-1', + memberId: submitterUser.userId, + }, + }); + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + { + ...baseReviewerResource, + id: 'resource-submit', + memberId: submitterUser.userId, + roleName: 'Submitter', + roleId: CommonConfig.roles.submitterRoleId, + }, + ]); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([{ id: 'submission-1' }]); + } + return Promise.resolve([{ id: 'submission-1' }]); + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + phases: [ + { + id: 'phase-screening', + name: 'Screening', + isOpen: false, + actualEndTime: now.toISOString(), + }, + { id: 'phase-appeals', name: 'Appeals', isOpen: false }, + ], + }); + + const response = await service.getReview(submitterUser, 'review-1'); + + expect(response.reviewItems).toHaveLength(1); + expect(response.finalScore).toBe(88.2); + expect(response.initialScore).toBe(85.4); + expect(response.phaseName).toBe('Screening'); + }); + it('allows submitters to access their review once iterative review closes without appeals', async () => { const submitterUser: JwtUser = { userId: 'submitter-1', @@ -1250,6 +1324,101 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(response.data[0].reviewItems).toEqual([]); }); + it('returns full screening review details for submitters once the screening phase completes', async () => { + const now = new Date(); + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [], + isMachine: false, + }; + + const submitterResource = { + ...buildResource('Submitter', submitterUser.userId), + roleId: CommonConfig.roles.submitterRoleId, + memberId: submitterUser.userId, + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + submitterResource, + ]); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Active Challenge', + status: ChallengeStatus.ACTIVE, + phases: [ + { + id: 'phase-screening', + name: 'Screening', + isOpen: false, + actualEndTime: now.toISOString(), + }, + { id: 'phase-appeals', name: 'Appeals', isOpen: false }, + ], + }); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([{ id: baseSubmission.id }]); + } + return Promise.resolve([{ id: baseSubmission.id }]); + }); + + const reviewRecord = { + id: 'review-1', + resourceId: 'resource-reviewer', + submissionId: baseSubmission.id, + phaseId: 'phase-screening', + scorecardId: 'scorecard-1', + typeId: 'type-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + reviewItems: [ + { + id: 'item-1', + scorecardQuestionId: 'question-1', + initialAnswer: 'Yes', + finalAnswer: 'No', + managerComment: 'detail', + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 91.1, + initialScore: 89.9, + submission: { + id: baseSubmission.id, + memberId: submitterUser.userId, + challengeId: 'challenge-1', + }, + }; + + prismaMock.review.findMany.mockResolvedValue([reviewRecord]); + prismaMock.review.count.mockResolvedValue(1); + + const response = await service.getReviews( + submitterUser, + undefined, + 'challenge-1', + ); + + expect(response.data).toHaveLength(1); + const [review] = response.data; + expect(review.finalScore).toBe(91.1); + expect(review.initialScore).toBe(89.9); + expect(review.reviewItems).toHaveLength(1); + expect(review.phaseName).toBe('Screening'); + }); + it('returns scores for submitters once appeals are open', async () => { const now = new Date(); const submitterResource = { diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 353b0ea..72a6f0a 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -2507,6 +2507,12 @@ export class ReviewService { ) { // Allow limited visibility into other submissions once submission phase closes. allowLimitedVisibilityForOtherSubmissions = true; + } else if ( + hasSubmitterRoleForChallenge && + this.hasChallengePhaseCompleted(challenge, ['screening']) + ) { + // Allow submitters to access their own screening reviews once screening phase completes. + restrictToSubmissionIds(mySubmissionIds); } else { // No access for non-completed, non-appeals phases throw new ForbiddenException({ @@ -2870,7 +2876,32 @@ export class ReviewService { const isReviewerForReview = reviewerResourceIdSet.has(reviewResourceId); const isOwnSubmission = submitterSubmissionIdSet.has(reviewSubmissionId); + + const reviewPhaseId = review.phaseId ? String(review.phaseId) : ''; + const phaseName = reviewPhaseId + ? (phaseNameCache.get(reviewPhaseId) ?? null) + : null; + const normalizedPhaseName = this.normalizePhaseName(phaseName); + + const reviewChallengeId = + review.submission?.challengeId ?? challengeId ?? null; + const challengeForReview = reviewChallengeId + ? (challengeCache.get(reviewChallengeId) ?? null) + : null; + const screeningPhaseCompleted = this.hasChallengePhaseCompleted( + challengeForReview, + ['screening'], + ); + const allowOwnScreeningVisibility = + !isPrivilegedRequester && + hasSubmitterRoleForChallenge && + !hasCopilotRoleForChallenge && + !isReviewerForReview && + isOwnSubmission && + screeningPhaseCompleted && + normalizedPhaseName === 'screening'; const shouldMaskReviewDetails = + !allowOwnScreeningVisibility && !isPrivilegedRequester && hasSubmitterRoleForChallenge && submitterSubmissionIdSet.size > 0 && @@ -2888,11 +2919,6 @@ export class ReviewService { !isOwnSubmission && !submitterVisibilityState.allowAny; - const reviewPhaseId = review.phaseId ? String(review.phaseId) : ''; - const phaseName = reviewPhaseId - ? (phaseNameCache.get(reviewPhaseId) ?? null) - : null; - const sanitizeScores = shouldMaskReviewDetails || shouldLimitNonOwnerVisibility; const sanitizedReview: typeof review & { @@ -2903,12 +2929,6 @@ export class ReviewService { finalScore: sanitizeScores ? null : review.finalScore, }; - const reviewChallengeId = - review.submission?.challengeId ?? challengeId ?? null; - const challengeForReview = reviewChallengeId - ? (challengeCache.get(reviewChallengeId) ?? null) - : null; - const normalizedPhaseName = (phaseName ?? '').trim().toLowerCase(); const shouldTrimIterativeReviewForOtherSubmitters = !isPrivilegedRequester && submitterSubmissionIdSet.size > 0 && @@ -3050,6 +3070,7 @@ export class ReviewService { const challengeCache = new Map(); const isPrivilegedRequester = authUser?.isMachine || isAdmin(authUser); + let resolvedPhaseName: string | null = null; let challengeDetail: ChallengeData | null = null; let hasCopilotRole = false; let reviewerResourceIds = new Set(); @@ -3135,7 +3156,19 @@ export class ReviewService { const visibility = this.getSubmitterVisibilityForChallenge(challenge); const isOwnSubmission = !!uid && data.submission?.memberId === uid; - if (!visibility.allowOwn) { + resolvedPhaseName = await this.resolvePhaseNameFromChallenge({ + challengeId, + phaseId: data.phaseId ?? null, + challengeCache, + }); + const normalizedPhaseNameForAccess = + this.normalizePhaseName(resolvedPhaseName); + const canSeeOwnScreeningReview = + isOwnSubmission && + normalizedPhaseNameForAccess === 'screening' && + this.hasChallengePhaseCompleted(challenge, ['screening']); + + if (!visibility.allowOwn && !canSeeOwnScreeningReview) { throw new ForbiddenException({ message: 'Reviews are not accessible for this challenge at the current phase', @@ -3173,16 +3206,16 @@ export class ReviewService { // ignore } result.appeals = flattenedAppeals; - const resolvedPhaseName = await this.resolvePhaseNameFromChallenge({ - challengeId: data.submission?.challengeId ?? null, - phaseId: data.phaseId ?? null, - challengeCache, - }); + if (!resolvedPhaseName) { + resolvedPhaseName = await this.resolvePhaseNameFromChallenge({ + challengeId: data.submission?.challengeId ?? null, + phaseId: data.phaseId ?? null, + challengeCache, + }); + } result.phaseName = resolvedPhaseName; - const normalizedPhaseName = (resolvedPhaseName ?? '') - .trim() - .toLowerCase(); + const normalizedPhaseName = this.normalizePhaseName(resolvedPhaseName); const requesterId = String(authUser?.userId ?? ''); const submissionOwnerId = String(data.submission?.memberId ?? ''); const challengeForReview = data.submission?.challengeId @@ -3286,6 +3319,67 @@ export class ReviewService { return matchedPhase?.name ?? null; } + private normalizePhaseName(phaseName: string | null | undefined): string { + return String(phaseName ?? '') + .trim() + .toLowerCase(); + } + + private hasChallengePhaseCompleted( + challenge: ChallengeData | null | undefined, + phaseNames: string[], + ): boolean { + if (!challenge?.phases?.length) { + return false; + } + + const normalizedTargets = new Set( + (phaseNames ?? []) + .map((name) => this.normalizePhaseName(name)) + .filter((name) => name.length > 0), + ); + + if (!normalizedTargets.size) { + return false; + } + + return (challenge.phases ?? []).some((phase) => { + if (!phase) { + return false; + } + + const normalizedName = this.normalizePhaseName((phase as any).name); + if (!normalizedTargets.has(normalizedName)) { + return false; + } + + if ((phase as any).isOpen === true) { + return false; + } + + const actualEnd = + (phase as any).actualEndTime ?? (phase as any).actualEndDate ?? null; + + if (actualEnd == null) { + return false; + } + + if (typeof actualEnd === 'string') { + return actualEnd.trim().length > 0; + } + + if (actualEnd instanceof Date) { + return true; + } + + try { + return String(actualEnd).trim().length > 0; + } catch { + return false; + } + }); + } + private getSubmitterVisibilityForChallenge( challenge?: ChallengeData | null, ): { allowAny: boolean; allowOwn: boolean } { @@ -3297,25 +3391,25 @@ export class ReviewService { const phases = challenge.phases || []; const isFirst2Finish = this.isFirst2FinishChallenge(challenge); - const normalizeName = (phaseName: string | null | undefined) => - String(phaseName ?? '').toLowerCase(); - const appealsOpen = phases.some( - (phase) => normalizeName(phase.name) === 'appeals' && phase.isOpen, + (phase) => + this.normalizePhaseName(phase?.name) === 'appeals' && + phase?.isOpen === true, ); const appealsResponseOpen = phases.some( (phase) => - normalizeName(phase.name) === 'appeals response' && phase.isOpen, + this.normalizePhaseName(phase?.name) === 'appeals response' && + phase?.isOpen === true, ); const hasAppealsPhases = phases.some((phase) => { - const name = normalizeName(phase.name); + const name = this.normalizePhaseName(phase?.name); return name === 'appeals' || name === 'appeals response'; }); const iterativeReviewClosed = phases.some((phase) => { - const name = normalizeName(phase.name); - return name === 'iterative review' && phase.isOpen === false; + const name = this.normalizePhaseName(phase?.name); + return name === 'iterative review' && phase?.isOpen === false; }); const allowAny = status === ChallengeStatus.COMPLETED; From bb35775864af7b680faf4cef16dc6fdf0ba5f323 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 17 Oct 2025 07:10:45 +1100 Subject: [PATCH 11/48] Fix copilot / admin reopening review scorecard, and don't allow reopen review unless review phase is open --- src/api/contact/contactRequests.service.ts | 33 +++- src/api/review/review.service.spec.ts | 210 ++++++++++++++++++++- src/api/review/review.service.ts | 112 ++++++++++- 3 files changed, 347 insertions(+), 8 deletions(-) diff --git a/src/api/contact/contactRequests.service.ts b/src/api/contact/contactRequests.service.ts index 8f8b74f..a2bdcc0 100644 --- a/src/api/contact/contactRequests.service.ts +++ b/src/api/contact/contactRequests.service.ts @@ -167,10 +167,23 @@ export class ContactRequestsService { return; } - // Get emails for recipients - const memberInfos = await this.memberService.getUserEmails(memberIds); + // Get emails for recipients and requester + const requesterId = authUser?.userId ? String(authUser.userId) : undefined; + const lookupIds = requesterId + ? Array.from(new Set([...memberIds, requesterId])) + : memberIds; + + const memberInfos = await this.memberService.getUserEmails(lookupIds); + const memberInfoById = new Map( + memberInfos.map((info) => [String(info.userId), info]), + ); + const recipients = Array.from( - new Set(memberInfos.map((m) => m.email).filter(Boolean)), + new Set( + memberIds + .map((id) => memberInfoById.get(id)?.email) + .filter((email): email is string => Boolean(email)), + ), ); if (recipients.length === 0) { @@ -180,6 +193,16 @@ export class ContactRequestsService { return; } + const requesterEmail = requesterId + ? memberInfoById.get(requesterId)?.email + : undefined; + + if (!requesterEmail && requesterId) { + this.logger.warn( + `No email found for requester ${requesterId} when notifying challenge ${challengeId}`, + ); + } + const payload: EventBusSendEmailPayload = new EventBusSendEmailPayload(); payload.sendgrid_template_id = CommonConfig.sendgridConfig.contactManagersEmailTemplate; @@ -189,6 +212,10 @@ export class ContactRequestsService { challengeName: challenge.name, message: message ?? '', }; + if (requesterEmail) { + payload.from = requesterEmail; + payload.replyTo = requesterEmail; + } await this.eventBusService.sendEmail(payload); } diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index 3144734..dcfe209 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -2910,6 +2910,99 @@ describe('ReviewService.updateReview challenge status enforcement', () => { ); }); + it('rejects reopening a completed review when the associated phase has closed', async () => { + prismaMock.review.findUnique.mockResolvedValueOnce({ + id: 'review-1', + resourceId: 'resource-1', + status: ReviewStatus.COMPLETED, + phaseId: 'phase-review', + submission: { + challengeId: 'challenge-1', + }, + reviewItems: [], + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValueOnce({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + phases: [ + { + id: 'phase-review', + name: 'Review', + isOpen: false, + actualEndTime: '2024-01-31T00:00:00.000Z', + }, + ], + }); + + await expect( + service.updateReview(nonPrivilegedUser, 'review-1', { + status: ReviewStatus.PENDING, + }), + ).rejects.toMatchObject({ + response: expect.objectContaining({ + code: 'REVIEW_UPDATE_FORBIDDEN_PHASE_CLOSED', + }), + status: 403, + }); + + expect(prismaMock.review.update).not.toHaveBeenCalled(); + expect(recomputeSpy).not.toHaveBeenCalled(); + expect(challengeApiServiceMock.getChallengeDetail).toHaveBeenCalledWith( + 'challenge-1', + ); + }); + + it('rejects reopening a completed review for admins when the associated phase has closed', async () => { + prismaMock.review.findUnique.mockResolvedValueOnce({ + id: 'review-1', + resourceId: 'resource-1', + status: ReviewStatus.COMPLETED, + phaseId: 'phase-review', + submission: { + challengeId: 'challenge-1', + }, + reviewItems: [], + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValueOnce({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + phases: [ + { + id: 'phase-review', + name: 'Review', + isOpen: false, + actualEndTime: '2024-01-31T00:00:00.000Z', + }, + ], + }); + + const adminUser: JwtUser = { + userId: 'admin-1', + roles: [UserRole.Admin], + isMachine: false, + }; + + await expect( + service.updateReview(adminUser, 'review-1', { + status: ReviewStatus.IN_PROGRESS, + }), + ).rejects.toMatchObject({ + response: expect.objectContaining({ + code: 'REVIEW_UPDATE_FORBIDDEN_PHASE_CLOSED', + }), + status: 403, + }); + + expect(prismaMock.review.update).not.toHaveBeenCalled(); + expect(resourceApiServiceMock.getResources).not.toHaveBeenCalled(); + expect(recomputeSpy).not.toHaveBeenCalled(); + expect(challengeApiServiceMock.getChallengeDetail).toHaveBeenCalledWith( + 'challenge-1', + ); + }); + it('publishes completion event when review status transitions to COMPLETED', async () => { const completionDate = new Date('2024-02-01T10:00:00Z'); @@ -3015,6 +3108,104 @@ describe('ReviewService.updateReview challenge status enforcement', () => { isMachine: false, }; + const existingReview = { + id: 'review-1', + resourceId: 'resource-1', + status: ReviewStatus.COMPLETED, + committed: true, + reviewDate: new Date('2024-01-15T10:00:00Z'), + submission: { + challengeId: 'challenge-1', + }, + reviewItems: [], + }; + prismaMock.review.findUnique.mockResolvedValueOnce(existingReview); + prismaMock.review.update.mockResolvedValueOnce({ + ...existingReview, + status: ReviewStatus.PENDING, + committed: false, + reviewDate: null, + }); + + resourceApiServiceMock.getResources.mockResolvedValue([ + { + id: 'resource-9', + challengeId: 'challenge-1', + memberId: 'copilot-1', + }, + ]); + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + { + id: 'resource-9', + challengeId: 'challenge-1', + memberId: 'copilot-1', + memberHandle: 'copilotHandle', + roleId: 'copilot-role', + created: new Date().toISOString(), + createdBy: 'tc', + roleName: 'Copilot', + }, + ]); + + const result = await service.updateReview(copilotUser, 'review-1', { + status: ReviewStatus.PENDING, + committed: false, + } as any); + + expect(result).toEqual( + expect.objectContaining({ + id: 'review-1', + status: ReviewStatus.PENDING, + committed: false, + reviewDate: null, + appeals: [], + phaseName: null, + }), + ); + expect(prismaMock.review.update).toHaveBeenCalledTimes(1); + const updateArgs = prismaMock.review.update.mock.calls[0][0]; + expect(updateArgs.data).toMatchObject({ + status: ReviewStatus.PENDING, + committed: false, + reviewDate: null, + }); + expect(resourceApiServiceMock.getResources).toHaveBeenCalledWith({ + challengeId: 'challenge-1', + memberId: 'copilot-1', + }); + expect(resourceApiServiceMock.getMemberResourcesRoles).toHaveBeenCalledWith( + 'challenge-1', + 'copilot-1', + ); + expect(recomputeSpy).toHaveBeenCalledWith('review-1'); + }); + + it('defaults committed to false and clears reviewDate when reopening with only status provided', async () => { + const copilotUser: JwtUser = { + userId: 'copilot-1', + roles: [UserRole.Copilot], + isMachine: false, + }; + + const existingReview = { + id: 'review-1', + resourceId: 'resource-1', + status: ReviewStatus.COMPLETED, + committed: true, + reviewDate: new Date('2024-01-20T12:00:00Z'), + submission: { + challengeId: 'challenge-1', + }, + reviewItems: [], + }; + prismaMock.review.findUnique.mockResolvedValueOnce(existingReview); + prismaMock.review.update.mockResolvedValueOnce({ + ...existingReview, + status: ReviewStatus.IN_PROGRESS, + committed: false, + reviewDate: null, + }); + resourceApiServiceMock.getResources.mockResolvedValue([ { id: 'resource-9', @@ -3039,7 +3230,22 @@ describe('ReviewService.updateReview challenge status enforcement', () => { status: ReviewStatus.IN_PROGRESS, } as any); - expect(result).toEqual({ id: 'review-1', appeals: [], phaseName: null }); + expect(result).toEqual( + expect.objectContaining({ + id: 'review-1', + status: ReviewStatus.IN_PROGRESS, + committed: false, + reviewDate: null, + appeals: [], + phaseName: null, + }), + ); + const updateArgs = prismaMock.review.update.mock.calls[0][0]; + expect(updateArgs.data).toMatchObject({ + status: ReviewStatus.IN_PROGRESS, + committed: false, + reviewDate: null, + }); expect(resourceApiServiceMock.getResources).toHaveBeenCalledWith({ challengeId: 'challenge-1', memberId: 'copilot-1', @@ -3048,7 +3254,7 @@ describe('ReviewService.updateReview challenge status enforcement', () => { 'challenge-1', 'copilot-1', ); - expect(prismaMock.review.update).toHaveBeenCalledTimes(1); + expect(recomputeSpy).toHaveBeenCalledWith('review-1'); }); it('records an audit entry when an admin updates review status and answers', async () => { diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 72a6f0a..a67390a 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -1672,6 +1672,7 @@ export class ReviewService { const isPrivileged = isAdmin(requester); const challengeId = existingReview.submission?.challengeId; const challengeCache = new Map(); + let challengeDetailForPhase: ChallengeData | null = null; const isMemberRequester = !requester?.isMachine; const normalizedRoles = Array.isArray(requester.roles) ? requester.roles.map((role) => String(role).trim().toLowerCase()) @@ -1689,6 +1690,33 @@ export class ReviewService { .map(([key]) => key); const isStatusOnlyUpdate = definedBodyKeys.length === 1 && definedBodyKeys[0] === 'status'; + const requestedStatusValue = (body as ReviewPatchRequestDto).status; + const requestedStatus = + typeof requestedStatusValue === 'string' && + (Object.values(ReviewStatus) as string[]).includes(requestedStatusValue) + ? requestedStatusValue + : undefined; + const requestedCommittedValue = (body as ReviewPatchRequestDto).committed; + const requestedCommitted = + typeof requestedCommittedValue === 'boolean' + ? requestedCommittedValue + : undefined; + const reopenStatuses = new Set([ + ReviewStatus.IN_PROGRESS, + ReviewStatus.PENDING, + ]); + const isReopenStatusRequested = + requestedStatus !== undefined && reopenStatuses.has(requestedStatus); + const isReopenTransition = + existingReview.status === ReviewStatus.COMPLETED && + isReopenStatusRequested; + const isCopilotReopenPayload = + isReopenTransition && + definedBodyKeys.includes('status') && + definedBodyKeys.every( + (field) => field === 'status' || field === 'committed', + ) && + (requestedCommitted === undefined || requestedCommitted === false); if (isMemberRequester && !isPrivileged) { const requesterMemberId = String(requester?.userId ?? ''); @@ -1732,7 +1760,9 @@ export class ReviewService { ); const allowCopilotStatusPatch = - hasCopilotRole && isStatusOnlyUpdate && !ownsReview; + hasCopilotRole && + !ownsReview && + (isStatusOnlyUpdate || isCopilotReopenPayload); if (!ownsReview) { if (allowCopilotStatusPatch) { @@ -1810,7 +1840,6 @@ export class ReviewService { } if (!isPrivileged && challengeId) { - let challengeDetailForPhase: ChallengeData; try { challengeDetailForPhase = await this.challengeApiService.getChallengeDetail(challengeId); @@ -1827,7 +1856,10 @@ export class ReviewService { }); } - if (challengeDetailForPhase.status === ChallengeStatus.COMPLETED) { + if ( + challengeDetailForPhase && + challengeDetailForPhase.status === ChallengeStatus.COMPLETED + ) { throw new ForbiddenException({ message: 'Reviews for challenges in COMPLETED status cannot be updated. Only an admin can update a review once the challenge is complete.', @@ -1837,6 +1869,71 @@ export class ReviewService { } } + if ( + isReopenTransition && + challengeId && + existingReview.phaseId !== undefined && + existingReview.phaseId !== null + ) { + const normalizedPhaseId = String(existingReview.phaseId); + + if (!challengeDetailForPhase) { + if (challengeCache.has(challengeId)) { + challengeDetailForPhase = challengeCache.get(challengeId) ?? null; + } else { + try { + challengeDetailForPhase = + await this.challengeApiService.getChallengeDetail(challengeId); + challengeCache.set(challengeId, challengeDetailForPhase); + } catch (error) { + this.logger.error( + `[updateReview] Unable to verify phase ${normalizedPhaseId} for review ${id} on challenge ${challengeId}`, + error, + ); + throw new InternalServerErrorException({ + message: + 'Unable to verify the phase status for this review. Please try again later.', + code: 'CHALLENGE_PHASE_STATUS_UNAVAILABLE', + details: { + reviewId: id, + challengeId, + phaseId: normalizedPhaseId, + }, + }); + } + } + } + + const matchingPhase = + challengeDetailForPhase?.phases?.find((phase) => { + if (!phase) { + return false; + } + const candidateIds = [ + String((phase as any).id ?? ''), + String((phase as any).phaseId ?? ''), + ].filter((value) => value.length > 0); + return candidateIds.includes(normalizedPhaseId); + }) ?? null; + + if ( + matchingPhase && + matchingPhase.actualEndTime && + matchingPhase.isOpen === false + ) { + throw new ForbiddenException({ + message: + 'Reviews associated with closed challenge phases cannot be reopened to Pending or In Progress.', + code: 'REVIEW_UPDATE_FORBIDDEN_PHASE_CLOSED', + details: { + reviewId: id, + challengeId, + phaseId: normalizedPhaseId, + }, + }); + } + } + const incomingReviewItems = 'reviewItems' in body ? (body.reviewItems ?? undefined) : undefined; @@ -1901,6 +1998,15 @@ export class ReviewService { ...(reviewItemsUpdate ?? {}), } as Prisma.reviewUpdateInput; + if (isReopenTransition) { + if (updateData.committed === undefined) { + updateData.committed = false; + } + if (updateData.reviewDate === undefined) { + updateData.reviewDate = null; + } + } + try { const data = await this.prisma.review.update({ where: { id }, From b520e44acf3b545a79503406c06324e3d9381148 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 17 Oct 2025 11:44:32 +1100 Subject: [PATCH 12/48] Allow reviews to be pulled for challenge that failed review (PM-2361) --- src/api/review/review.service.spec.ts | 37 +++++++++++++++++++++++++++ src/api/review/review.service.ts | 25 +++++++++++++++--- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index dcfe209..8e47f39 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -1244,6 +1244,43 @@ describe('ReviewService.getReviews reviewer visibility', () => { }); }); + it('allows submitters to access reviews when the challenge failed review cancellation', async () => { + const submitterResource = { + ...buildResource('Submitter'), + roleId: CommonConfig.roles.submitterRoleId, + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + submitterResource, + ]); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Cancelled Failed Review Challenge', + status: ChallengeStatus.CANCELLED_FAILED_REVIEW, + phases: [], + }); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([baseSubmission]); + } + return Promise.resolve([baseSubmission, { id: 'submission-other' }]); + }); + + const result = await service.getReviews( + baseAuthUser, + undefined, + 'challenge-1', + ); + + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; + expect(callArgs.where.submissionId).toEqual({ + in: [baseSubmission.id, 'submission-other'], + }); + expect(result.data).toEqual([]); + }); + it('masks scores and review items for submitters before appeals open', async () => { const now = new Date(); const submitterResource = { diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index a67390a..d70bc71 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -2597,7 +2597,12 @@ export class ReviewService { requesterIsChallengeResource = true; } - if (challenge.status === ChallengeStatus.COMPLETED) { + if ( + [ + ChallengeStatus.COMPLETED, + ChallengeStatus.CANCELLED_FAILED_REVIEW, + ].includes(challenge.status) + ) { // Allowed to see all reviews on this challenge // reviewWhereClause already limited to submissions on this challenge } else if (isFirst2Finish) { @@ -2660,7 +2665,12 @@ export class ReviewService { const challenges = await this.challengeApiService.getChallenges(myChallengeIds); const completedIds = challenges - .filter((c) => c.status === ChallengeStatus.COMPLETED) + .filter((c) => + [ + ChallengeStatus.COMPLETED, + ChallengeStatus.CANCELLED_FAILED_REVIEW, + ].includes(c.status), + ) .map((c) => c.id); const appealsAllowedIds = challenges .filter((c) => { @@ -3232,8 +3242,12 @@ export class ReviewService { isReviewerForReview = reviewerResourceIds.has(reviewResourceId); if (reviewerResources.length > 0) { + const challengeInFinalState = [ + ChallengeStatus.COMPLETED, + ChallengeStatus.CANCELLED_FAILED_REVIEW, + ].includes(challenge.status); if ( - challenge.status !== ChallengeStatus.COMPLETED && + !challengeInFinalState && !reviewerResourceIds.has(reviewResourceId) ) { throw new ForbiddenException({ @@ -3518,7 +3532,10 @@ export class ReviewService { return name === 'iterative review' && phase?.isOpen === false; }); - const allowAny = status === ChallengeStatus.COMPLETED; + const allowAny = [ + ChallengeStatus.COMPLETED, + ChallengeStatus.CANCELLED_FAILED_REVIEW, + ].includes(status); const allowOwn = allowAny || appealsOpen || From 8060c795cd416184e6e8f0f0b58ebb591a781734 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 17 Oct 2025 17:58:05 +1100 Subject: [PATCH 13/48] Better handling of post-mortem reviews that aren't tied to a submission --- src/api/review/review.service.spec.ts | 130 ++++++++ src/api/review/review.service.ts | 283 +++++++++++++----- src/api/submission/submission.service.spec.ts | 141 ++++++++- src/api/submission/submission.service.ts | 59 +++- src/dto/review.dto.ts | 5 +- 5 files changed, 523 insertions(+), 95 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index 8e47f39..516f43f 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -47,6 +47,7 @@ describe('ReviewService.createReview authorization checks', () => { const resourcePrismaMock = { resource: { findMany: jest.fn().mockResolvedValue([]), + findUnique: jest.fn(), }, } as unknown as any; @@ -59,6 +60,7 @@ describe('ReviewService.createReview authorization checks', () => { const challengeApiServiceMock = { validateReviewSubmission: jest.fn(), getChallengeDetail: jest.fn(), + isPhaseOpen: jest.fn(), } as unknown as any; const eventBusServiceMock = { @@ -82,6 +84,12 @@ describe('ReviewService.createReview authorization checks', () => { isMachine: false, }; + const machineAuthUser: JwtUser = { + userId: 'machine-1', + roles: [], + isMachine: true, + }; + const buildReviewRequest = ( overrides: Partial = {}, ): ReviewRequestDto => @@ -167,6 +175,7 @@ describe('ReviewService.createReview authorization checks', () => { challengeApiServiceMock.getChallengeDetail.mockResolvedValue( baseChallengeDetail, ); + challengeApiServiceMock.isPhaseOpen.mockResolvedValue(true); resourceApiServiceMock.getResources.mockResolvedValue([baseResource]); resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ @@ -176,6 +185,7 @@ describe('ReviewService.createReview authorization checks', () => { }, ]); resourcePrismaMock.resource.findMany.mockResolvedValue([]); + resourcePrismaMock.resource.findUnique.mockResolvedValue(null); memberPrismaMock.member.findMany.mockResolvedValue([]); }); @@ -247,6 +257,75 @@ describe('ReviewService.createReview authorization checks', () => { } }); + it('allows Post-Mortem review creation without submission when phase is open', async () => { + const postMortemChallenge = { + ...baseChallengeDetail, + phases: [ + { + id: 'phase-post-mortem', + name: 'Post-Mortem', + isOpen: true, + }, + ], + }; + + const request = buildReviewRequest({ + submissionId: undefined, + resourceId: 'resource-1', + } as Partial); + + prismaMock.$queryRaw.mockReset(); + prismaMock.reviewType.findUnique.mockResolvedValue({ + id: 'type-1', + name: 'Post-Mortem Review', + }); + challengeApiServiceMock.validateReviewSubmission.mockReset(); + challengeApiServiceMock.getChallengeDetail.mockResolvedValue( + postMortemChallenge, + ); + challengeApiServiceMock.isPhaseOpen.mockResolvedValue(true); + resourceApiServiceMock.getResources.mockResolvedValue([]); + + const postMortemResourceRecord = { + ...baseResource, + phaseId: 'phase-post-mortem', + }; + resourcePrismaMock.resource.findUnique.mockResolvedValue( + postMortemResourceRecord, + ); + + const reviewCreateResult = { + ...buildReviewModel(request), + phaseId: 'phase-post-mortem', + }; + + prismaMock.review.create.mockResolvedValue(reviewCreateResult); + prismaMock.review.update.mockResolvedValue(reviewCreateResult); + + const computeSpy = jest + .spyOn(service as any, 'computeScoresFromItems') + .mockResolvedValue({ initialScore: null, finalScore: null }); + + try { + await expect( + service.createReview(machineAuthUser, request), + ).resolves.toMatchObject({ + phaseName: 'Post-Mortem', + }); + } finally { + computeSpy.mockRestore(); + } + + expect(prismaMock.$queryRaw).not.toHaveBeenCalled(); + expect( + challengeApiServiceMock.validateReviewSubmission, + ).not.toHaveBeenCalled(); + expect(challengeApiServiceMock.isPhaseOpen).toHaveBeenCalledWith( + 'challenge-1', + 'Post-Mortem', + ); + }); + it('allows reviews for non-latest submissions when submissionLimit.unlimited is the string "true"', async () => { const submissionId = 'submission-older'; prismaMock.$queryRaw.mockResolvedValue([ @@ -1209,6 +1288,57 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(callArgs.where.resourceId).toBeUndefined(); }); + it('returns empty results for submitters when reviews are not yet accessible', async () => { + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [], + isMachine: false, + }; + + const submitterResource = { + ...buildResource('Submitter', submitterUser.userId), + roleId: CommonConfig.roles.submitterRoleId, + memberId: submitterUser.userId, + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + submitterResource, + ]); + + prismaMock.submission.findMany.mockImplementation(({ where }: any = {}) => { + if (where?.challengeId) { + return Promise.resolve([{ id: baseSubmission.id }]); + } + if (where?.memberId) { + return Promise.resolve([{ id: baseSubmission.id }]); + } + return Promise.resolve([{ id: baseSubmission.id }]); + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Active Challenge', + status: ChallengeStatus.ACTIVE, + phases: [{ id: 'phase-sub', name: 'Submission', isOpen: true }], + }); + + const response = await service.getReviews( + submitterUser, + undefined, + 'challenge-1', + ); + + expect(response.data).toEqual([]); + expect(response.meta.totalCount).toBe(0); + expect(response.meta.totalPages).toBe(0); + expect(prismaMock.review.findMany).toHaveBeenCalledTimes(1); + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; + expect(callArgs.where.submissionId).toEqual({ in: ['__none__'] }); + expect(prismaMock.review.count).toHaveBeenCalledWith({ + where: callArgs.where, + }); + }); + it('includes non-owned submissions with limited visibility when submission phase is closed on an active challenge', async () => { const submitterResource = { ...buildResource('Submitter'), diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index d70bc71..7658954 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -965,7 +965,9 @@ export class ReviewService { authUser: JwtUser, body: ReviewRequestDto, ): Promise { - this.logger.log(`Creating review for submissionId: ${body.submissionId}`); + this.logger.log( + `Creating review for submissionId: ${body.submissionId ?? 'N/A'}`, + ); try { const scorecard = await this.prisma.scorecard.findUnique({ where: { id: body.scorecardId }, @@ -980,18 +982,34 @@ export class ReviewService { }); } - const submissions = await this.prisma.$queryRaw< - Array<{ - id: string; - challengeId: string | null; - memberId: string | null; - isLatest: boolean | null; - }> - >(Prisma.sql` + const normalizedSubmissionId = body.submissionId + ? String(body.submissionId).trim() + : undefined; + body.submissionId = normalizedSubmissionId; + + let submission: { + id: string; + challengeId: string | null; + memberId: string | null; + isLatest: boolean | null; + } | null = null; + let challengeId: string | undefined; + let submissionMemberId: string | null = null; + let submissionIsLatest = false; + + if (normalizedSubmissionId) { + const submissions = await this.prisma.$queryRaw< + Array<{ + id: string; + challengeId: string | null; + memberId: string | null; + isLatest: boolean | null; + }> + >(Prisma.sql` WITH target AS ( SELECT s."challengeId" FROM "submission" s - WHERE s."id" = ${body.submissionId} + WHERE s."id" = ${normalizedSubmissionId} ) SELECT ranked."id", @@ -1019,31 +1037,32 @@ export class ReviewService { SELECT target."challengeId" FROM target ) ) ranked - WHERE ranked."id" = ${body.submissionId} + WHERE ranked."id" = ${normalizedSubmissionId} `); - const submission = submissions[0] ?? null; + submission = submissions[0] ?? null; - if (!submission) { - throw new NotFoundException({ - message: `Submission with ID ${body.submissionId} was not found. Please verify the submissionId and try again.`, - code: 'SUBMISSION_NOT_FOUND', - details: { submissionId: body.submissionId }, - }); - } + if (!submission) { + throw new NotFoundException({ + message: `Submission with ID ${normalizedSubmissionId} was not found. Please verify the submissionId and try again.`, + code: 'SUBMISSION_NOT_FOUND', + details: { submissionId: normalizedSubmissionId }, + }); + } - if (!submission.challengeId) { - throw new BadRequestException({ - message: `Submission ${body.submissionId} does not have an associated challengeId`, - code: 'MISSING_CHALLENGE_ID', - }); - } + if (!submission.challengeId) { + throw new BadRequestException({ + message: `Submission ${normalizedSubmissionId} does not have an associated challengeId`, + code: 'MISSING_CHALLENGE_ID', + }); + } - const challengeId = submission.challengeId; - const submissionMemberId = submission.memberId - ? String(submission.memberId) - : null; - const submissionIsLatest = Boolean(submission.isLatest); + challengeId = submission.challengeId; + submissionMemberId = submission.memberId + ? String(submission.memberId) + : null; + submissionIsLatest = Boolean(submission.isLatest); + } const reviewType = await this.prisma.reviewType.findUnique({ where: { id: body.typeId }, @@ -1058,16 +1077,90 @@ export class ReviewService { }); } - await this.challengeApiService.validateReviewSubmission(challengeId); - - const challengeResources = await this.resourceApiService.getResources({ - challengeId, - }); + const reviewTypeName = (reviewType.name ?? '').trim(); + const isPostMortemReview = + /post[\s-]?mortem/i.test(reviewTypeName) || + reviewTypeName === 'Post Mortem'; const providedResourceId = body.resourceId ? String(body.resourceId).trim() : undefined; + let resourceRecord: + | (Awaited< + ReturnType + > & { roleName?: string | null }) + | null = null; + + if (!challengeId) { + if (!isPostMortemReview) { + throw new BadRequestException({ + message: + 'submissionId is required unless creating a Post-Mortem review without submissions.', + code: 'SUBMISSION_ID_REQUIRED', + }); + } + + if (!providedResourceId) { + throw new BadRequestException({ + message: + 'resourceId must be provided when creating a Post-Mortem review without a submission.', + code: 'RESOURCE_ID_REQUIRED', + }); + } + + resourceRecord = await this.resourcePrisma.resource.findUnique({ + where: { id: providedResourceId }, + }); + + if (!resourceRecord) { + throw new NotFoundException({ + message: `Resource with ID ${providedResourceId} was not found.`, + code: 'RESOURCE_NOT_FOUND', + details: { resourceId: providedResourceId }, + }); + } + + challengeId = String(resourceRecord.challengeId ?? '').trim(); + + if (!challengeId) { + throw new BadRequestException({ + message: `Resource ${providedResourceId} is not associated with a challenge.`, + code: 'MISSING_CHALLENGE_ID', + details: { resourceId: providedResourceId }, + }); + } + } + + if (!challengeId) { + throw new BadRequestException({ + message: + 'Unable to determine the challenge associated with this review request.', + code: 'MISSING_CHALLENGE_ID', + }); + } + + if (isPostMortemReview) { + const postMortemOpen = await this.challengeApiService.isPhaseOpen( + challengeId, + 'Post-Mortem', + ); + + if (!postMortemOpen) { + throw new BadRequestException({ + message: `Post-Mortem phase is not currently open for challenge ${challengeId}.`, + code: 'POST_MORTEM_PHASE_CLOSED', + details: { challengeId }, + }); + } + } else { + await this.challengeApiService.validateReviewSubmission(challengeId); + } + + const challengeResources = await this.resourceApiService.getResources({ + challengeId, + }); + let resource: ResourceInfo | undefined; if (providedResourceId) { resource = challengeResources.find( @@ -1075,14 +1168,58 @@ export class ReviewService { ); if (!resource) { - throw new NotFoundException({ - message: `Resource with ID ${providedResourceId} was not found for challenge ${challengeId}.`, - code: 'RESOURCE_NOT_FOUND', - details: { - resourceId: providedResourceId, - challengeId, - }, - }); + if (!resourceRecord) { + resourceRecord = await this.resourcePrisma.resource.findUnique({ + where: { id: providedResourceId }, + }); + } + + if (!resourceRecord) { + throw new NotFoundException({ + message: `Resource with ID ${providedResourceId} was not found for challenge ${challengeId}.`, + code: 'RESOURCE_NOT_FOUND', + details: { + resourceId: providedResourceId, + challengeId, + }, + }); + } + + const resourceRecordAny = resourceRecord as Record; + + const phaseIdValue = + resourceRecordAny?.phaseId !== undefined + ? resourceRecordAny.phaseId + : undefined; + + let phaseIdString: string | undefined; + if (typeof phaseIdValue === 'string') { + phaseIdString = phaseIdValue; + } else if (typeof phaseIdValue === 'number') { + phaseIdString = phaseIdValue.toString(); + } else { + phaseIdString = undefined; + } + + const createdValue = (resourceRecordAny?.created ?? + (resourceRecord as { createdAt?: Date; created?: Date }) + .createdAt ?? + new Date()) as Date | string; + + resource = { + id: resourceRecord.id, + challengeId: String(resourceRecord.challengeId ?? ''), + memberId: String(resourceRecord.memberId ?? ''), + memberHandle: String(resourceRecord.memberHandle ?? ''), + roleId: String(resourceRecord.roleId ?? ''), + phaseId: phaseIdString, + createdBy: String(resourceRecord.createdBy ?? ''), + created: createdValue, + roleName: + resourceRecordAny.roleName !== undefined + ? (resourceRecordAny.roleName as string | undefined) + : undefined, + }; } } @@ -1177,6 +1314,7 @@ export class ReviewService { await this.challengeApiService.getChallengeDetail(challengeId); if ( + submission && this.shouldEnforceLatestSubmissionForReview( reviewType.name, challenge, @@ -1202,32 +1340,43 @@ export class ReviewService { const resolvePhaseId = (phase: (typeof challengePhases)[number]) => String((phase as any)?.id ?? (phase as any)?.phaseId ?? ''); - const matchPhaseByName = (name: string) => - challengePhases.find((phase) => { - const phaseName = (phase?.name ?? '').trim().toLowerCase(); - return phaseName === name; - }); + const normalized = (value: string | undefined | null) => + (value ?? '').toLowerCase().replace(/[\s_-]+/g, ''); + + const targetPhaseKeys = isPostMortemReview + ? ['postmortem'] + : ['review', 'iterativereview']; - const reviewPhase = - matchPhaseByName('review') ?? matchPhaseByName('iterative review'); - const reviewPhaseName = reviewPhase?.name ?? null; + const targetPhase = challengePhases.find((phase) => + targetPhaseKeys.some((key) => normalized(phase?.name) === key), + ); + + const reviewPhaseName = targetPhase?.name ?? null; - if (!reviewPhase) { + if (!targetPhase) { throw new BadRequestException({ - message: `Challenge ${challengeId} does not have a Review phase.`, - code: 'REVIEW_PHASE_NOT_FOUND', + message: isPostMortemReview + ? `Challenge ${challengeId} does not have a Post-Mortem phase.` + : `Challenge ${challengeId} does not have a Review phase.`, + code: isPostMortemReview + ? 'POST_MORTEM_PHASE_NOT_FOUND' + : 'REVIEW_PHASE_NOT_FOUND', details: { challengeId, }, }); } - const reviewPhaseId = resolvePhaseId(reviewPhase); + const reviewPhaseId = resolvePhaseId(targetPhase); if (!reviewPhaseId) { throw new BadRequestException({ - message: `Review phase for challenge ${challengeId} is missing an identifier.`, - code: 'REVIEW_PHASE_NOT_FOUND', + message: isPostMortemReview + ? `Post-Mortem phase for challenge ${challengeId} is missing an identifier.` + : `Review phase for challenge ${challengeId} is missing an identifier.`, + code: isPostMortemReview + ? 'POST_MORTEM_PHASE_NOT_FOUND' + : 'REVIEW_PHASE_NOT_FOUND', details: { challengeId, }, @@ -1239,7 +1388,7 @@ export class ReviewService { : undefined; if (resourcePhaseId && resourcePhaseId !== reviewPhaseId) { throw new BadRequestException({ - message: `Resource ${resource.id} is associated with phase ${resourcePhaseId}, which does not match the Review phase ${reviewPhaseId}.`, + message: `Resource ${resource.id} is associated with phase ${resourcePhaseId}, which does not match the ${reviewPhaseName ?? 'target'} phase ${reviewPhaseId}.`, code: 'RESOURCE_PHASE_MISMATCH', details: { resourceId: resource.id, @@ -1378,6 +1527,7 @@ export class ReviewService { const prismaBody = mapReviewRequestToDto(body) as any; prismaBody.phaseId = reviewPhaseId; prismaBody.resourceId = reviewerResource.id; + prismaBody.submissionId = body.submissionId ?? null; const createdReview = await this.prisma.review.create({ data: prismaBody, include: { @@ -1457,7 +1607,7 @@ export class ReviewService { const errorResponse = this.prismaErrorService.handleError( error, - `creating review for submissionId: ${body.submissionId}`, + `creating review for submissionId: ${body.submissionId ?? 'N/A'}`, body, ); @@ -2625,16 +2775,11 @@ export class ReviewService { // Allow submitters to access their own screening reviews once screening phase completes. restrictToSubmissionIds(mySubmissionIds); } else { - // No access for non-completed, non-appeals phases - throw new ForbiddenException({ - message: - 'Reviews are not accessible for this challenge at the current phase', - code: 'FORBIDDEN_REVIEW_ACCESS', - details: { - challengeId, - status: challenge.status, - }, - }); + // Reviews exist but the phase does not allow visibility yet; respond with no results. + this.logger.debug( + `[getReviews] Challenge ${challengeId} is in status ${challenge.status}. Returning empty review list for requester ${uid}.`, + ); + restrictToSubmissionIds([]); } } } else { diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index d7b3113..a44e46b 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -3,6 +3,7 @@ import { SubmissionStatus, SubmissionType } from '@prisma/client'; import { Readable } from 'stream'; import { SubmissionService } from './submission.service'; import { UserRole } from 'src/shared/enums/userRole.enum'; +import { ChallengeStatus } from 'src/shared/enums/challengeStatus.enum'; jest.mock('nanoid', () => ({ __esModule: true, @@ -23,9 +24,11 @@ describe('SubmissionService', () => { { Key: `${submission.id}/internal-notes.txt` }, ]; let originalBucket: string | undefined; + let originalCleanBucket: string | undefined; beforeAll(() => { originalBucket = process.env.ARTIFACTS_S3_BUCKET; + originalCleanBucket = process.env.SUBMISSION_CLEAN_S3_BUCKET; }); beforeEach(() => { @@ -56,6 +59,7 @@ describe('SubmissionService', () => { }); process.env.ARTIFACTS_S3_BUCKET = 'unit-test-bucket'; + process.env.SUBMISSION_CLEAN_S3_BUCKET = 'unit-test-clean-bucket'; }); afterEach(() => { @@ -68,6 +72,11 @@ describe('SubmissionService', () => { } else { process.env.ARTIFACTS_S3_BUCKET = originalBucket; } + if (originalCleanBucket === undefined) { + delete process.env.SUBMISSION_CLEAN_S3_BUCKET; + } else { + process.env.SUBMISSION_CLEAN_S3_BUCKET = originalCleanBucket; + } }); describe('listArtifacts', () => { @@ -270,6 +279,118 @@ describe('SubmissionService', () => { }); }); + describe('getSubmissionFileStream', () => { + let prismaMock: { submission: { findFirst: jest.Mock } }; + let challengeApiServiceMock: { getChallengeDetail: jest.Mock }; + + beforeEach(() => { + prismaMock = { + submission: { + findFirst: jest.fn(), + }, + }; + challengeApiServiceMock = { + getChallengeDetail: jest.fn(), + }; + resourceApiService = { + getMemberResourcesRoles: jest.fn(), + }; + service = new SubmissionService( + prismaMock as any, + {} as any, + {} as any, + challengeApiServiceMock as any, + resourceApiService as any, + {} as any, + {} as any, + {} as any, + ); + jest.spyOn(service as any, 'checkSubmission').mockResolvedValue({ + id: 'sub-123', + memberId: 'owner-user', + challengeId: 'challenge-xyz', + url: 'https://s3.amazonaws.com/dummy/submission.zip', + }); + jest + .spyOn(service as any, 'parseS3Url') + .mockReturnValue({ key: 'dummy/submission.zip' }); + jest + .spyOn(service as any, 'recordSubmissionDownload') + .mockResolvedValue(undefined); + s3Send = jest + .fn() + .mockResolvedValueOnce({ ContentType: 'application/zip' }) + .mockResolvedValueOnce({ Body: Readable.from(['payload']) }); + jest.spyOn(service as any, 'getS3Client').mockReturnValue({ + send: s3Send, + }); + }); + + it('allows submitters with passing reviews to download when challenge is completed', async () => { + resourceApiService.getMemberResourcesRoles.mockResolvedValue([ + { roleName: 'Submitter' }, + ]); + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + status: ChallengeStatus.COMPLETED, + }); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'passing-sub', + }); + + const result = await service.getSubmissionFileStream( + { + userId: 'submitter-user', + isMachine: false, + roles: [], + } as any, + 'sub-123', + ); + + expect(result.fileName).toBe('submission-sub-123.zip'); + expect(resourceApiService.getMemberResourcesRoles).toHaveBeenCalledWith( + 'challenge-xyz', + 'submitter-user', + ); + expect(challengeApiServiceMock.getChallengeDetail).toHaveBeenCalledWith( + 'challenge-xyz', + ); + expect(prismaMock.submission.findFirst).toHaveBeenCalledWith({ + where: { + challengeId: 'challenge-xyz', + memberId: 'submitter-user', + reviewSummation: { + some: { + isPassing: true, + }, + }, + }, + select: { id: true }, + }); + expect(s3Send).toHaveBeenCalledTimes(2); + }); + + it('denies submitters when the challenge is not completed', async () => { + resourceApiService.getMemberResourcesRoles.mockResolvedValue([ + { roleName: 'Submitter' }, + ]); + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + status: ChallengeStatus.ACTIVE, + }); + + await expect( + service.getSubmissionFileStream( + { + userId: 'submitter-user', + isMachine: false, + roles: [], + } as any, + 'sub-123', + ), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(prismaMock.submission.findFirst).not.toHaveBeenCalled(); + }); + }); + describe('listSubmission', () => { let prismaMock: { submission: { @@ -280,7 +401,7 @@ describe('SubmissionService', () => { }; let prismaErrorServiceMock: { handleError: jest.Mock }; let challengePrismaMock: { - challengeMetadata: { findMany: jest.Mock }; + $queryRaw: jest.Mock; }; let listService: SubmissionService; @@ -296,11 +417,8 @@ describe('SubmissionService', () => { handleError: jest.fn(), }; challengePrismaMock = { - challengeMetadata: { - findMany: jest.fn(), - }, + $queryRaw: jest.fn().mockResolvedValue([]), }; - challengePrismaMock.challengeMetadata.findMany.mockResolvedValue([]); listService = new SubmissionService( prismaMock as any, prismaErrorServiceMock as any, @@ -373,15 +491,7 @@ describe('SubmissionService', () => { }), ); - expect( - challengePrismaMock.challengeMetadata.findMany, - ).toHaveBeenCalledWith( - expect.objectContaining({ - where: expect.objectContaining({ - challengeId: { in: ['challenge-1'] }, - }), - }), - ); + expect(challengePrismaMock.$queryRaw).toHaveBeenCalledTimes(1); const latestEntries = result.data.filter((entry) => entry.isLatest); expect(latestEntries.map((entry) => entry.id)).toEqual([ @@ -390,9 +500,8 @@ describe('SubmissionService', () => { }); it('omits isLatest when submission metadata indicates unlimited submissions', async () => { - challengePrismaMock.challengeMetadata.findMany.mockResolvedValue([ + challengePrismaMock.$queryRaw.mockResolvedValue([ { - challengeId: 'challenge-1', value: '{"unlimited":"true","limit":"false","count":""}', }, ]); diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index bde63e5..b5b5795 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -18,6 +18,7 @@ import { } from 'src/dto/submission.dto'; import { JwtUser, isAdmin } from 'src/shared/modules/global/jwt.service'; import { UserRole } from 'src/shared/enums/userRole.enum'; +import { ChallengeStatus } from 'src/shared/enums/challengeStatus.enum'; import { PrismaService } from 'src/shared/modules/global/prisma.service'; import { ChallengePrismaService } from 'src/shared/modules/global/challenge-prisma.service'; import { MemberPrismaService } from 'src/shared/modules/global/member-prisma.service'; @@ -480,7 +481,8 @@ export class SubmissionService { const isOwner = !!uid && submission.memberId === uid; let isReviewer = false; let isCopilot = false; - if (!isOwner && submission.challengeId) { + let isSubmitter = false; + if (!isOwner && submission.challengeId && uid) { try { const resources = await this.resourceApiService.getMemberResourcesRoles( @@ -489,18 +491,57 @@ export class SubmissionService { ); for (const r of resources) { const rn = (r.roleName || '').toLowerCase(); - if (rn.includes('reviewer')) isReviewer = true; - if (rn.includes('copilot')) isCopilot = true; - if (isReviewer || isCopilot) break; + if (rn.includes('reviewer')) { + isReviewer = true; + } + if (rn.includes('copilot')) { + isCopilot = true; + } + if (rn.includes('submitter')) { + isSubmitter = true; + } + if (isReviewer && isCopilot && isSubmitter) { + break; + } } - } catch { - // If we cannot confirm roles, deny access - isReviewer = false; - isCopilot = false; + } catch (err) { + // If we cannot confirm roles, deny access unless other checks succeed + this.logger.warn( + `Failed to load member roles for challenge ${submission.challengeId} and member ${uid}: ${(err as Error)?.message}`, + ); + } + } + + let canDownload = isOwner || isReviewer || isCopilot; + + if (!canDownload && isSubmitter && submission.challengeId && uid) { + try { + const challenge = await this.challengeApiService.getChallengeDetail( + submission.challengeId, + ); + if (challenge.status === ChallengeStatus.COMPLETED) { + const passingSubmission = await this.prisma.submission.findFirst({ + where: { + challengeId: submission.challengeId, + memberId: uid, + reviewSummation: { + some: { + isPassing: true, + }, + }, + }, + select: { id: true }, + }); + canDownload = !!passingSubmission; + } + } catch (err) { + this.logger.warn( + `Failed to validate submitter download eligibility for challenge ${submission.challengeId} and member ${uid}: ${(err as Error)?.message}`, + ); } } - if (!isOwner && !isReviewer && !isCopilot) { + if (!canDownload) { throw new ForbiddenException({ message: 'Only the submission owner, a challenge reviewer/copilot, or an admin can download the submission', diff --git a/src/dto/review.dto.ts b/src/dto/review.dto.ts index 402a6fd..862c153 100644 --- a/src/dto/review.dto.ts +++ b/src/dto/review.dto.ts @@ -222,10 +222,13 @@ export class ReviewCommonDto { @ApiProperty({ description: 'Submission ID being reviewed', example: 'submission789', + required: false, + nullable: true, }) + @IsOptional() @IsString() @IsNotEmpty() - submissionId: string; + submissionId?: string; @ApiProperty({ description: 'Scorecard ID used for the review', From c70c70a16b32c19537ae303b3daee54da7ad0b38 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Sat, 18 Oct 2025 08:07:56 +1100 Subject: [PATCH 14/48] Fixes for filtering by challenge status --- src/api/my-review/myReview.controller.ts | 7 +++++ src/api/my-review/myReview.service.spec.ts | 17 ++++++++++++ src/api/my-review/myReview.service.ts | 32 ++++++++++++++++++---- src/dto/my-review.dto.ts | 12 +++++++- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/api/my-review/myReview.controller.ts b/src/api/my-review/myReview.controller.ts index a7d1dd1..711f10a 100644 --- a/src/api/my-review/myReview.controller.ts +++ b/src/api/my-review/myReview.controller.ts @@ -18,6 +18,7 @@ import { Roles } from 'src/shared/guards/tokenRoles.guard'; import { UserRole } from 'src/shared/enums/userRole.enum'; import { Scopes } from 'src/shared/decorators/scopes.decorator'; import { Scope } from 'src/shared/enums/scopes.enum'; +import { ChallengeStatus } from 'src/shared/enums/challengeStatus.enum'; @ApiTags('My Reviews') @ApiBearerAuth() @@ -61,6 +62,12 @@ export class MyReviewController { required: false, description: 'Filter by challenge track identifier', }) + @ApiQuery({ + name: 'challengeStatus', + required: false, + description: 'Filter by challenge status', + enum: ChallengeStatus, + }) @ApiQuery({ name: 'past', required: false, diff --git a/src/api/my-review/myReview.service.spec.ts b/src/api/my-review/myReview.service.spec.ts index f037815..8c1bf48 100644 --- a/src/api/my-review/myReview.service.spec.ts +++ b/src/api/my-review/myReview.service.spec.ts @@ -5,6 +5,7 @@ jest.mock('nanoid', () => ({ import { UnauthorizedException } from '@nestjs/common'; import { MyReviewService } from './myReview.service'; +import { ChallengeStatus } from 'src/shared/enums/challengeStatus.enum'; describe('MyReviewService', () => { let service: MyReviewService; @@ -200,6 +201,22 @@ describe('MyReviewService', () => { expect(queryDetails.values).toEqual(expect.arrayContaining(pastStatuses)); }); + it('filters by specific challenge status when provided for past reviews', async () => { + challengePrismaMock.$queryRaw.mockResolvedValueOnce([{ total: 0n }]); + + await service.getMyReviews( + { isMachine: false, userId: '123' }, + { past: 'true', challengeStatus: ChallengeStatus.COMPLETED }, + ); + + const query = challengePrismaMock.$queryRaw.mock.calls[0][0]; + const queryDetails = query.inspect(); + + expect(queryDetails.sql).toMatch(/c\.status = \?::"ChallengeStatusEnum"/); + expect(queryDetails.sql).not.toContain('c.status IN'); + expect(queryDetails.values).toContain(ChallengeStatus.COMPLETED); + }); + it('reports negative time left when a phase is overdue and uses signed ordering', async () => { const now = Date.now(); const past = new Date(now - 10_000); diff --git a/src/api/my-review/myReview.service.ts b/src/api/my-review/myReview.service.ts index 8bdb0ac..680844e 100644 --- a/src/api/my-review/myReview.service.ts +++ b/src/api/my-review/myReview.service.ts @@ -95,6 +95,9 @@ export class MyReviewService { const challengeTrackId = filters.challengeTrackId?.trim(); const challengeTypeName = filters.challengeTypeName?.trim(); const challengeName = filters.challengeName?.trim(); + const normalizedChallengeStatus = filters.challengeStatus + ? filters.challengeStatus.trim().toUpperCase() + : undefined; const shouldFetchPastChallenges = typeof filters.past === 'string' @@ -124,12 +127,31 @@ export class MyReviewService { const whereFragments: Prisma.Sql[] = []; if (shouldFetchPastChallenges) { - const statusFragments = PAST_CHALLENGE_STATUSES.map( - (status) => Prisma.sql`${status}::"ChallengeStatusEnum"`, - ); - const statusList = joinSqlFragments(statusFragments, Prisma.sql`, `); - whereFragments.push(Prisma.sql`c.status IN (${statusList})`); + if (normalizedChallengeStatus) { + const pastStatusSet = new Set(PAST_CHALLENGE_STATUSES); + if (pastStatusSet.has(normalizedChallengeStatus)) { + whereFragments.push( + Prisma.sql`c.status = ${normalizedChallengeStatus}::"ChallengeStatusEnum"`, + ); + } else { + this.logger.warn( + `Challenge status ${normalizedChallengeStatus} is not allowed for past reviews; returning empty result set.`, + ); + whereFragments.push(Prisma.sql`1 = 0`); + } + } else { + const statusFragments = PAST_CHALLENGE_STATUSES.map( + (status) => Prisma.sql`${status}::"ChallengeStatusEnum"`, + ); + const statusList = joinSqlFragments(statusFragments, Prisma.sql`, `); + whereFragments.push(Prisma.sql`c.status IN (${statusList})`); + } } else { + if (normalizedChallengeStatus && normalizedChallengeStatus !== 'ACTIVE') { + this.logger.warn( + `Challenge status filter ${normalizedChallengeStatus} is not supported for active reviews and will be ignored.`, + ); + } whereFragments.push(Prisma.sql`c.status = 'ACTIVE'`); } diff --git a/src/dto/my-review.dto.ts b/src/dto/my-review.dto.ts index 5390059..47af339 100644 --- a/src/dto/my-review.dto.ts +++ b/src/dto/my-review.dto.ts @@ -1,5 +1,6 @@ import { ApiProperty } from '@nestjs/swagger'; -import { IsIn, IsOptional, IsString } from 'class-validator'; +import { IsEnum, IsIn, IsOptional, IsString } from 'class-validator'; +import { ChallengeStatus } from 'src/shared/enums/challengeStatus.enum'; export const ACTIVE_MY_REVIEW_SORT_FIELDS = [ 'challengeName', @@ -76,6 +77,15 @@ export class MyReviewFilterDto { @IsString() challengeName?: string; + @ApiProperty({ + description: 'Filter results to a specific challenge status', + required: false, + enum: ChallengeStatus, + }) + @IsOptional() + @IsEnum(ChallengeStatus) + challengeStatus?: ChallengeStatus; + @ApiProperty({ description: 'Whether or not to include current challenges or past challenges', From a639d461a8d9a8a353572fb8626c3bc1a7bbcb08 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Sat, 18 Oct 2025 09:54:29 +1100 Subject: [PATCH 15/48] Better handling of checkpoints and visibility after their related phases end --- src/api/review/review.service.ts | 126 ++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 12 deletions(-) diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 7658954..c67e76f 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -3153,6 +3153,10 @@ export class ReviewService { challengeForReview, ['screening'], ); + const checkpointReviewPhaseCompleted = + this.hasChallengePhaseClosedWithActualDates(challengeForReview, [ + 'checkpoint review', + ]); const allowOwnScreeningVisibility = !isPrivilegedRequester && hasSubmitterRoleForChallenge && @@ -3161,8 +3165,17 @@ export class ReviewService { isOwnSubmission && screeningPhaseCompleted && normalizedPhaseName === 'screening'; + const allowOwnCheckpointReviewVisibility = + !isPrivilegedRequester && + hasSubmitterRoleForChallenge && + !hasCopilotRoleForChallenge && + !isReviewerForReview && + isOwnSubmission && + checkpointReviewPhaseCompleted && + normalizedPhaseName === 'checkpoint review'; const shouldMaskReviewDetails = !allowOwnScreeningVisibility && + !allowOwnCheckpointReviewVisibility && !isPrivilegedRequester && hasSubmitterRoleForChallenge && submitterSubmissionIdSet.size > 0 && @@ -3432,8 +3445,18 @@ export class ReviewService { isOwnSubmission && normalizedPhaseNameForAccess === 'screening' && this.hasChallengePhaseCompleted(challenge, ['screening']); + const canSeeOwnCheckpointReview = + isOwnSubmission && + normalizedPhaseNameForAccess === 'checkpoint review' && + this.hasChallengePhaseClosedWithActualDates(challenge, [ + 'checkpoint review', + ]); - if (!visibility.allowOwn && !canSeeOwnScreeningReview) { + if ( + !visibility.allowOwn && + !canSeeOwnScreeningReview && + !canSeeOwnCheckpointReview + ) { throw new ForbiddenException({ message: 'Reviews are not accessible for this challenge at the current phase', @@ -3590,6 +3613,49 @@ export class ReviewService { .toLowerCase(); } + private hasTimestampValue(value: unknown): boolean { + if (value == null) { + return false; + } + if (value instanceof Date) { + return true; + } + switch (typeof value) { + case 'string': + return value.trim().length > 0; + case 'number': + return Number.isFinite(value); + case 'bigint': + case 'boolean': + return true; + case 'symbol': + case 'function': + return false; + case 'object': { + const valueWithToISOString = value as { + toISOString?: (() => string) | undefined; + valueOf?: (() => unknown) | undefined; + }; + if (typeof valueWithToISOString.toISOString === 'function') { + try { + return valueWithToISOString.toISOString().trim().length > 0; + } catch { + return false; + } + } + if (typeof valueWithToISOString.valueOf === 'function') { + const primitiveValue = valueWithToISOString.valueOf(); + if (primitiveValue !== value) { + return this.hasTimestampValue(primitiveValue); + } + } + return false; + } + default: + return false; + } + } + private hasChallengePhaseCompleted( challenge: ChallengeData | null | undefined, phaseNames: string[], @@ -3623,25 +3689,61 @@ export class ReviewService { } const actualEnd = - (phase as any).actualEndTime ?? (phase as any).actualEndDate ?? null; + (phase as any).actualEndTime ?? + (phase as any).actualEndDate ?? + (phase as any).actualEnd ?? + null; - if (actualEnd == null) { - return false; - } + return this.hasTimestampValue(actualEnd); + }); + } + + private hasChallengePhaseClosedWithActualDates( + challenge: ChallengeData | null | undefined, + phaseNames: string[], + ): boolean { + if (!challenge?.phases?.length) { + return false; + } - if (typeof actualEnd === 'string') { - return actualEnd.trim().length > 0; + const normalizedTargets = new Set( + (phaseNames ?? []) + .map((name) => this.normalizePhaseName(name)) + .filter((name) => name.length > 0), + ); + + if (!normalizedTargets.size) { + return false; + } + + return (challenge.phases ?? []).some((phase) => { + if (!phase) { + return false; } - if (actualEnd instanceof Date) { - return true; + const normalizedName = this.normalizePhaseName((phase as any).name); + if (!normalizedTargets.has(normalizedName)) { + return false; } - try { - return String(actualEnd).trim().length > 0; - } catch { + if ((phase as any).isOpen === true) { return false; } + + const actualStart = + (phase as any).actualStartTime ?? + (phase as any).actualStartDate ?? + (phase as any).actualStart ?? + null; + const actualEnd = + (phase as any).actualEndTime ?? + (phase as any).actualEndDate ?? + (phase as any).actualEnd ?? + null; + + return ( + this.hasTimestampValue(actualStart) && this.hasTimestampValue(actualEnd) + ); }); } From 2cff0e00f624bcbe007b24ac27df4f466bcffba8 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Sat, 18 Oct 2025 11:33:54 +1100 Subject: [PATCH 16/48] Approval phase handling updates --- src/api/review/review.service.spec.ts | 163 ++++++++++++++++++++++++++ src/api/review/review.service.ts | 45 ++++++- 2 files changed, 205 insertions(+), 3 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index 516f43f..dc179cf 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -910,6 +910,62 @@ describe('ReviewService.getReview authorization checks', () => { }); }); + it('allows submitters to access their review once the associated phase completes with actual dates', async () => { + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [UserRole.Submitter], + isMachine: false, + }; + + const now = new Date().toISOString(); + + prismaMock.review.findUniqueOrThrow.mockResolvedValue({ + ...defaultReviewData(), + phaseId: 'phase-review', + submission: { + id: 'submission-1', + challengeId: 'challenge-1', + memberId: submitterUser.userId, + }, + }); + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + { + ...baseReviewerResource, + id: 'resource-submit', + memberId: submitterUser.userId, + roleName: 'Submitter', + roleId: CommonConfig.roles.submitterRoleId, + }, + ]); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([{ id: 'submission-1' }]); + } + return Promise.resolve([{ id: 'submission-1' }]); + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + phases: [ + { + id: 'phase-review', + name: 'Review', + isOpen: false, + actualStartTime: now, + actualEndTime: now, + }, + { id: 'phase-appeals', name: 'Appeals', isOpen: false }, + ], + }); + + await expect( + service.getReview(submitterUser, 'review-1'), + ).resolves.toMatchObject({ id: 'review-1' }); + }); + it('allows submitters to access their own screening review once the screening phase completes', async () => { const submitterUser: JwtUser = { userId: 'submitter-1', @@ -1255,6 +1311,17 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(callArgs.where.submissionId).toEqual({ in: [baseSubmission.id] }); }); + it('filters reviews by the resource id when the requester is an approver', async () => { + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + buildResource('Approver'), + ]); + + await service.getReviews(baseAuthUser, undefined, 'challenge-1'); + + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; + expect(callArgs.where.resourceId).toEqual({ in: ['resource-1'] }); + }); + it('filters reviews by the resource id when the requester is a checkpoint screener', async () => { resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ buildResource('Checkpoint Screener'), @@ -1491,6 +1558,102 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(response.data[0].reviewItems).toEqual([]); }); + it('returns full review details for submitters once the associated review phase completes with actual dates', async () => { + const now = new Date(); + const submitterUser: JwtUser = { + userId: 'submitter-1', + roles: [], + isMachine: false, + }; + + const submitterResource = { + ...buildResource('Submitter', submitterUser.userId), + roleId: CommonConfig.roles.submitterRoleId, + memberId: submitterUser.userId, + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + submitterResource, + ]); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Active Challenge', + status: ChallengeStatus.ACTIVE, + phases: [ + { + id: 'phase-review', + name: 'Review', + isOpen: false, + actualStartTime: now.toISOString(), + actualEndTime: now.toISOString(), + }, + { id: 'phase-appeals', name: 'Appeals', isOpen: false }, + ], + }); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([{ id: baseSubmission.id }]); + } + return Promise.resolve([{ id: baseSubmission.id }]); + }); + + const reviewRecord = { + id: 'review-1', + resourceId: 'resource-reviewer', + submissionId: baseSubmission.id, + phaseId: 'phase-review', + scorecardId: 'scorecard-1', + typeId: 'type-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + reviewItems: [ + { + id: 'item-1', + scorecardQuestionId: 'question-1', + initialAnswer: 'Yes', + finalAnswer: 'No', + managerComment: 'detail', + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 88.5, + initialScore: 85.0, + submission: { + id: baseSubmission.id, + memberId: submitterUser.userId, + challengeId: 'challenge-1', + }, + }; + + prismaMock.review.findMany.mockResolvedValue([reviewRecord]); + prismaMock.review.count.mockResolvedValue(1); + + const response = await service.getReviews( + submitterUser, + undefined, + 'challenge-1', + ); + + expect(response.data).toHaveLength(1); + const [review] = response.data; + expect(review.finalScore).toBe(88.5); + expect(review.initialScore).toBe(85.0); + expect(review.reviewItems).toHaveLength(1); + expect(review.phaseName).toBe('Review'); + }); + it('returns full screening review details for submitters once the screening phase completes', async () => { const now = new Date(); const submitterUser: JwtUser = { diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index c67e76f..05106e2 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -51,8 +51,13 @@ const REVIEW_ITEM_COMMENTS_INCLUDE = { } as const; // Roles containing any of these keywords are treated as review-capable resources. -// This includes standard reviewers plus checkpoint screeners/reviewers. -const REVIEW_ACCESS_ROLE_KEYWORDS = ['reviewer', 'screener']; +// This includes standard reviewers plus checkpoint screeners/reviewers/approvers. +const REVIEW_ACCESS_ROLE_KEYWORDS = [ + 'reviewer', + 'screener', + 'approver', + 'approval', +]; type ReviewItemAccessMode = 'machine' | 'admin' | 'reviewer-owner' | 'copilot'; @@ -3157,6 +3162,18 @@ export class ReviewService { this.hasChallengePhaseClosedWithActualDates(challengeForReview, [ 'checkpoint review', ]); + const phaseNamesForCompletionCheck: string[] = []; + if (phaseName && phaseName.trim().length > 0) { + phaseNamesForCompletionCheck.push(phaseName); + } else if (normalizedPhaseName.length > 0) { + phaseNamesForCompletionCheck.push(normalizedPhaseName); + } + const phaseClosedWithActualDates = + phaseNamesForCompletionCheck.length > 0 && + this.hasChallengePhaseClosedWithActualDates( + challengeForReview, + phaseNamesForCompletionCheck, + ); const allowOwnScreeningVisibility = !isPrivilegedRequester && hasSubmitterRoleForChallenge && @@ -3173,9 +3190,17 @@ export class ReviewService { isOwnSubmission && checkpointReviewPhaseCompleted && normalizedPhaseName === 'checkpoint review'; + const allowOwnClosedPhaseVisibility = + !isPrivilegedRequester && + hasSubmitterRoleForChallenge && + !hasCopilotRoleForChallenge && + !isReviewerForReview && + isOwnSubmission && + phaseClosedWithActualDates; const shouldMaskReviewDetails = !allowOwnScreeningVisibility && !allowOwnCheckpointReviewVisibility && + !allowOwnClosedPhaseVisibility && !isPrivilegedRequester && hasSubmitterRoleForChallenge && submitterSubmissionIdSet.size > 0 && @@ -3441,6 +3466,12 @@ export class ReviewService { }); const normalizedPhaseNameForAccess = this.normalizePhaseName(resolvedPhaseName); + const phaseNamesForAccessCheck: string[] = []; + if (resolvedPhaseName && resolvedPhaseName.trim().length > 0) { + phaseNamesForAccessCheck.push(resolvedPhaseName); + } else if (normalizedPhaseNameForAccess.length > 0) { + phaseNamesForAccessCheck.push(normalizedPhaseNameForAccess); + } const canSeeOwnScreeningReview = isOwnSubmission && normalizedPhaseNameForAccess === 'screening' && @@ -3451,11 +3482,19 @@ export class ReviewService { this.hasChallengePhaseClosedWithActualDates(challenge, [ 'checkpoint review', ]); + const canSeeOwnClosedPhaseReview = + isOwnSubmission && + phaseNamesForAccessCheck.length > 0 && + this.hasChallengePhaseClosedWithActualDates( + challenge, + phaseNamesForAccessCheck, + ); if ( !visibility.allowOwn && !canSeeOwnScreeningReview && - !canSeeOwnCheckpointReview + !canSeeOwnCheckpointReview && + !canSeeOwnClosedPhaseReview ) { throw new ForbiddenException({ message: From 54464e9134910f3b49362ead068b4ce568512fa1 Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Mon, 20 Oct 2025 14:37:36 +0300 Subject: [PATCH 17/48] PM-1904 - expose if challenge has AI review assigned --- src/api/my-review/myReview.service.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/api/my-review/myReview.service.ts b/src/api/my-review/myReview.service.ts index 680844e..2d8ae5c 100644 --- a/src/api/my-review/myReview.service.ts +++ b/src/api/my-review/myReview.service.ts @@ -19,6 +19,7 @@ interface ChallengeSummaryRow { challengeName: string; challengeTypeId: string | null; challengeTypeName: string | null; + hasAsAIReview: boolean; currentPhaseName: string | null; currentPhaseScheduledEnd: Date | null; currentPhaseActualEnd: Date | null; @@ -297,6 +298,16 @@ export class MyReviewService { LIMIT 1 ) appeals_response_phase ON TRUE `, + Prisma.sql` + LEFT JOIN LATERAL ( + SELECT + CASE WHEN COUNT(*)::bigint > 0 THEN TRUE else FALSE END AS "hasAsAIReview" + FROM challenges."ChallengeReviewer" cr + WHERE cr."challengeId" = c.id + AND cr."isMemberReview" = false + LIMIT 1 + ) cr ON TRUE + `, ); if (challengeTypeId) { @@ -434,6 +445,7 @@ export class MyReviewService { c."typeId" AS "challengeTypeId", ct.name AS "challengeTypeName", cp.name AS "currentPhaseName", + cr."hasAsAIReview" as "hasAsAIReview", cp."scheduledEndDate" AS "currentPhaseScheduledEnd", cp."actualEndDate" AS "currentPhaseActualEnd", rr.name AS "resourceRoleName", @@ -539,6 +551,7 @@ export class MyReviewService { challengeName: row.challengeName, challengeTypeId: row.challengeTypeId, challengeTypeName: row.challengeTypeName, + hasAsAIReview: row.hasAsAIReview, challengeEndDate: row.challengeEndDate ? row.challengeEndDate.toISOString() : null, From d977c40b9d10fb060d5887c1af19ed9647ccb9c9 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 22 Oct 2025 07:55:10 +1100 Subject: [PATCH 18/48] Allow marathon matches to return all reviewSummations to registered submitters, for display in CA. --- .../review-summation.controller.ts | 14 +++- .../review-summation.service.ts | 84 ++++++++++++++++++- 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/src/api/review-summation/review-summation.controller.ts b/src/api/review-summation/review-summation.controller.ts index 70f634d..b6b1228 100644 --- a/src/api/review-summation/review-summation.controller.ts +++ b/src/api/review-summation/review-summation.controller.ts @@ -138,11 +138,12 @@ export class ReviewSummationController { } @Get() - @Roles(UserRole.Copilot, UserRole.Admin) + @Roles(UserRole.Copilot, UserRole.Admin, UserRole.Submitter) @Scopes(Scope.ReadReviewSummation) @ApiOperation({ summary: 'Search for review summations', - description: 'Roles: Copilot, Admin. | Scopes: read:review_summation', + description: + 'Roles: Copilot, Admin, Submitter. | Scopes: read:review_summation', }) @ApiResponse({ status: 200, @@ -150,6 +151,7 @@ export class ReviewSummationController { type: [ReviewSummationResponseDto], }) async listReviewSummations( + @Req() req: Request, @Query() queryDto: ReviewSummationQueryDto, @Query() paginationDto?: PaginationDto, @Query() sortDto?: SortDto, @@ -157,7 +159,13 @@ export class ReviewSummationController { this.logger.log( `Getting review summations with filters - ${JSON.stringify(queryDto)}`, ); - return this.service.searchSummation(queryDto, paginationDto, sortDto); + const authUser: JwtUser = req['user'] as JwtUser; + return this.service.searchSummation( + authUser, + queryDto, + paginationDto, + sortDto, + ); } @Get('/:reviewSummationId') diff --git a/src/api/review-summation/review-summation.service.ts b/src/api/review-summation/review-summation.service.ts index 548dce5..a3c362a 100644 --- a/src/api/review-summation/review-summation.service.ts +++ b/src/api/review-summation/review-summation.service.ts @@ -4,6 +4,7 @@ import { NotFoundException, InternalServerErrorException, BadRequestException, + ForbiddenException, } from '@nestjs/common'; import { PaginationDto } from 'src/dto/pagination.dto'; import { @@ -14,7 +15,7 @@ import { ReviewSummationUpdateRequestDto, } from 'src/dto/reviewSummation.dto'; import { SortDto } from 'src/dto/sort.dto'; -import { JwtUser } from 'src/shared/modules/global/jwt.service'; +import { JwtUser, isAdmin } from 'src/shared/modules/global/jwt.service'; import { PrismaService } from 'src/shared/modules/global/prisma.service'; import { PrismaErrorService } from 'src/shared/modules/global/prisma-error.service'; import { @@ -22,6 +23,7 @@ import { ChallengeData, } from 'src/shared/modules/global/challenge.service'; import { MemberPrismaService } from 'src/shared/modules/global/member-prisma.service'; +import { UserRole } from 'src/shared/enums/userRole.enum'; @Injectable() export class ReviewSummationService { @@ -63,6 +65,20 @@ export class ReviewSummationService { ); } + private isMarathonMatchChallenge(challenge: ChallengeData): boolean { + const type = (challenge.type ?? '').trim().toLowerCase(); + if (type === 'marathon match') { + return true; + } + + const legacyTrack = (challenge.legacy?.subTrack ?? '').trim().toLowerCase(); + if (legacyTrack.includes('marathon')) { + return true; + } + + return false; + } + private roundScore(value: number): number { return Math.round(value * 100) / 100; } @@ -494,6 +510,7 @@ export class ReviewSummationService { } async searchSummation( + authUser: JwtUser, queryDto: ReviewSummationQueryDto, paginationDto?: PaginationDto, sortDto?: SortDto, @@ -509,6 +526,61 @@ export class ReviewSummationService { }; } + const normalizedRoles = new Set( + (authUser?.roles ?? []) + .map((role) => + String(role ?? '') + .trim() + .toLowerCase(), + ) + .filter((role) => role.length > 0), + ); + + const hasSubmitterRole = normalizedRoles.has( + String(UserRole.Submitter).trim().toLowerCase(), + ); + const hasCopilotRole = normalizedRoles.has( + String(UserRole.Copilot).trim().toLowerCase(), + ); + const isPrivileged = + (authUser?.isMachine ?? false) || isAdmin(authUser) || hasCopilotRole; + const isSubmitterOnly = hasSubmitterRole && !isPrivileged; + + const rawChallengeId = queryDto.challengeId + ? String(queryDto.challengeId).trim() + : undefined; + const challengeIdFilter = + rawChallengeId && rawChallengeId.length ? rawChallengeId : undefined; + + if (isSubmitterOnly) { + if (!challengeIdFilter) { + throw new ForbiddenException({ + message: + 'Submitters must specify a challengeId when listing review summations.', + code: 'FORBIDDEN_REVIEW_SUMMATION_ACCESS', + details: { + reason: 'CHALLENGE_ID_REQUIRED', + }, + }); + } + + const challenge = + await this.challengeApiService.getChallengeDetail(challengeIdFilter); + + if (!this.isMarathonMatchChallenge(challenge)) { + throw new ForbiddenException({ + message: + 'Submitters can only view review summations for Marathon Match challenges.', + code: 'FORBIDDEN_REVIEW_SUMMATION_ACCESS', + details: { + challengeId: challengeIdFilter, + challengeType: challenge.type ?? null, + legacySubTrack: challenge.legacy?.subTrack ?? null, + }, + }); + } + } + // Build the where clause for review summations based on available filter parameters const reviewSummationWhereClause: any = {}; if (queryDto.submissionId) { @@ -540,8 +612,8 @@ export class ReviewSummationService { } const submissionWhereClause: Record = {}; - if (queryDto.challengeId) { - submissionWhereClause.challengeId = queryDto.challengeId; + if (challengeIdFilter) { + submissionWhereClause.challengeId = challengeIdFilter; } const whereClause = { @@ -551,7 +623,7 @@ export class ReviewSummationService { : {}), }; - const shouldEnrichSubmitterMetadata = Boolean(queryDto.challengeId); + const shouldEnrichSubmitterMetadata = Boolean(challengeIdFilter); const summations = await this.prisma.reviewSummation.findMany({ where: whereClause, @@ -679,6 +751,10 @@ export class ReviewSummationService { }, }; } catch (error) { + if (error instanceof ForbiddenException) { + throw error; + } + const errorResponse = this.prismaErrorService.handleError( error, `searching review summations with filters - submissionId: ${queryDto.submissionId}, scorecardId: ${queryDto.scorecardId}, challengeId: ${queryDto.challengeId}`, From 634a0cc0f8414edd2bf07ce4540441340590c509 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 22 Oct 2025 08:35:17 +1100 Subject: [PATCH 19/48] Further fixes for MMs --- .../review-summation.controller.ts | 2 +- .../review-summation.service.ts | 67 ++++++++++++++++++- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/src/api/review-summation/review-summation.controller.ts b/src/api/review-summation/review-summation.controller.ts index b6b1228..7b8f5ba 100644 --- a/src/api/review-summation/review-summation.controller.ts +++ b/src/api/review-summation/review-summation.controller.ts @@ -138,7 +138,7 @@ export class ReviewSummationController { } @Get() - @Roles(UserRole.Copilot, UserRole.Admin, UserRole.Submitter) + @Roles(UserRole.Copilot, UserRole.Admin, UserRole.Submitter, UserRole.User) @Scopes(Scope.ReadReviewSummation) @ApiOperation({ summary: 'Search for review summations', diff --git a/src/api/review-summation/review-summation.service.ts b/src/api/review-summation/review-summation.service.ts index a3c362a..4c458d6 100644 --- a/src/api/review-summation/review-summation.service.ts +++ b/src/api/review-summation/review-summation.service.ts @@ -23,6 +23,7 @@ import { ChallengeData, } from 'src/shared/modules/global/challenge.service'; import { MemberPrismaService } from 'src/shared/modules/global/member-prisma.service'; +import { ResourceApiService } from 'src/shared/modules/global/resource.service'; import { UserRole } from 'src/shared/enums/userRole.enum'; @Injectable() @@ -34,6 +35,7 @@ export class ReviewSummationService { private readonly prismaErrorService: PrismaErrorService, private readonly challengeApiService: ChallengeApiService, private readonly memberPrisma: MemberPrismaService, + private readonly resourceApiService: ResourceApiService, ) {} private readonly systemActor = 'ReviewSummationService'; @@ -542,9 +544,13 @@ export class ReviewSummationService { const hasCopilotRole = normalizedRoles.has( String(UserRole.Copilot).trim().toLowerCase(), ); + const hasGeneralUserRole = normalizedRoles.has( + String(UserRole.User).trim().toLowerCase(), + ); const isPrivileged = (authUser?.isMachine ?? false) || isAdmin(authUser) || hasCopilotRole; - const isSubmitterOnly = hasSubmitterRole && !isPrivileged; + const isSubmitterOnly = + !isPrivileged && (hasSubmitterRole || hasGeneralUserRole); const rawChallengeId = queryDto.challengeId ? String(queryDto.challengeId).trim() @@ -552,14 +558,37 @@ export class ReviewSummationService { const challengeIdFilter = rawChallengeId && rawChallengeId.length ? rawChallengeId : undefined; + let enforcedMemberId: string | undefined; + if (isSubmitterOnly) { + const userId = + authUser?.userId !== undefined && authUser?.userId !== null + ? String(authUser.userId) + : ''; + if (!userId) { + throw new ForbiddenException({ + message: + 'Authenticated user information is required to view review summations.', + code: 'SUBMITTER_USER_MISSING', + details: { + reason: 'USER_ID_MISSING', + roles: Array.from(normalizedRoles), + }, + }); + } + if (!challengeIdFilter) { throw new ForbiddenException({ message: 'Submitters must specify a challengeId when listing review summations.', - code: 'FORBIDDEN_REVIEW_SUMMATION_ACCESS', + code: 'SUBMITTER_CHALLENGE_ID_REQUIRED', details: { reason: 'CHALLENGE_ID_REQUIRED', + guidance: + 'Pass a challengeId query parameter when requesting review summations as a submitter.', + submitterUserId: authUser?.userId ?? null, + submitterHandle: authUser?.handle ?? null, + roles: Array.from(normalizedRoles), }, }); } @@ -571,14 +600,43 @@ export class ReviewSummationService { throw new ForbiddenException({ message: 'Submitters can only view review summations for Marathon Match challenges.', - code: 'FORBIDDEN_REVIEW_SUMMATION_ACCESS', + code: 'SUBMITTER_NON_MARATHON_FORBIDDEN', details: { challengeId: challengeIdFilter, challengeType: challenge.type ?? null, + legacyTrack: challenge.track ?? null, legacySubTrack: challenge.legacy?.subTrack ?? null, + allowedChallengeTypes: ['Marathon Match'], + submitterUserId: authUser?.userId ?? null, + submitterHandle: authUser?.handle ?? null, + roles: Array.from(normalizedRoles), }, }); } + + try { + await this.resourceApiService.validateSubmitterRegistration( + challengeIdFilter, + userId, + ); + } catch (validationError) { + const details = + validationError instanceof Error + ? validationError.message + : String(validationError); + throw new ForbiddenException({ + message: + 'Submitter access requires active registration for this challenge.', + code: 'SUBMITTER_NOT_REGISTERED', + details: { + challengeId: challengeIdFilter, + memberId: userId, + info: details, + }, + }); + } + + enforcedMemberId = userId; } // Build the where clause for review summations based on available filter parameters @@ -615,6 +673,9 @@ export class ReviewSummationService { if (challengeIdFilter) { submissionWhereClause.challengeId = challengeIdFilter; } + if (enforcedMemberId) { + submissionWhereClause.memberId = enforcedMemberId; + } const whereClause = { ...reviewSummationWhereClause, From 2f702f1f67f8c4a9b71bf5d108c6955ffba8040a Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 22 Oct 2025 10:05:19 +1100 Subject: [PATCH 20/48] Add submitter ID value to review summations to make it easier to close the MM for the system-admin app / challenge API --- src/api/review-summation/review-summation.service.ts | 12 ++++++++++++ src/dto/reviewSummation.dto.ts | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/api/review-summation/review-summation.service.ts b/src/api/review-summation/review-summation.service.ts index 4c458d6..16d16b5 100644 --- a/src/api/review-summation/review-summation.service.ts +++ b/src/api/review-summation/review-summation.service.ts @@ -779,12 +779,23 @@ export class ReviewSummationService { submission?: { memberId: string | null }; }; + let submitterId: number | null = null; let submitterHandle: string | null = null; let submitterMaxRating: number | null = null; if (submission && typeof submission.memberId === 'string') { const memberId = submission.memberId.trim(); if (memberId.length) { + const numericMemberId = Number.parseInt(memberId, 10); + if ( + Number.isNaN(numericMemberId) || + !Number.isFinite(numericMemberId) || + !Number.isSafeInteger(numericMemberId) + ) { + submitterId = null; + } else { + submitterId = numericMemberId; + } const profile = submitterInfoByMemberId.get(memberId); submitterHandle = profile?.handle ?? null; submitterMaxRating = profile?.maxRating ?? null; @@ -793,6 +804,7 @@ export class ReviewSummationService { return { ...rest, + submitterId, submitterHandle, submitterMaxRating, } as ReviewSummationResponseDto; diff --git a/src/dto/reviewSummation.dto.ts b/src/dto/reviewSummation.dto.ts index 4b995f7..1448c95 100644 --- a/src/dto/reviewSummation.dto.ts +++ b/src/dto/reviewSummation.dto.ts @@ -294,6 +294,16 @@ export class ReviewSummationResponseDto { }) updatedBy: string | null; + @ApiProperty({ + description: + 'Numeric member ID of the submitter associated with this review summation', + example: 305643, + required: false, + nullable: true, + type: Number, + }) + submitterId?: number | null; + @ApiProperty({ description: 'Handle of the submitter associated with this review summation', From cdc27550c99ca0ef4d6613347712573eb8c0793b Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 22 Oct 2025 14:23:46 +1100 Subject: [PATCH 21/48] Clear out scores if a scorecard is reopened. Fixes up some permission issues with screeners / reviewers (PM-2506) --- src/api/review/review.service.spec.ts | 150 +++++++++++++- src/api/review/review.service.ts | 189 ++++++++++++++++-- src/api/submission/submission.service.spec.ts | 64 +++++- src/api/submission/submission.service.ts | 15 +- 4 files changed, 388 insertions(+), 30 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index dc179cf..3deab93 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -806,6 +806,30 @@ describe('ReviewService.getReview authorization checks', () => { }); }); + it('allows reviewers to access screening reviews that are not their own before completion', async () => { + prismaMock.review.findUniqueOrThrow.mockResolvedValue({ + ...defaultReviewData(), + resourceId: 'resource-other', + phaseId: 'phase-screening', + }); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + phases: [ + { id: 'phase-screening', name: 'Screening', isOpen: false }, + { id: 'phase-review', name: 'Review', isOpen: true }, + ], + }); + + await expect( + service.getReview(baseAuthUser, 'review-1'), + ).resolves.toMatchObject({ + id: 'review-1', + resourceId: 'resource-other', + }); + }); + it('allows reviewers to access non-owned reviews once the challenge is completed', async () => { challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ id: 'challenge-1', @@ -1225,6 +1249,48 @@ describe('ReviewService.getReviews reviewer visibility', () => { const baseSubmission = { id: 'submission-1' }; + type PrismaWhereClause = { + AND?: PrismaWhereClause | PrismaWhereClause[]; + OR?: PrismaWhereClause[]; + [key: string]: unknown; + }; + + const flattenWhereClauses = ( + input: PrismaWhereClause | PrismaWhereClause[] | undefined | null, + ): PrismaWhereClause[] => { + if (!input) { + return []; + } + if (Array.isArray(input)) { + return input.flatMap((item) => flattenWhereClauses(item)); + } + if (input.AND) { + return flattenWhereClauses(input.AND); + } + return [input]; + }; + + const findClauseWithKey = ( + clauses: PrismaWhereClause[], + key: string, + ): PrismaWhereClause | undefined => { + for (const clause of clauses) { + if (!clause) { + continue; + } + if (typeof clause[key] !== 'undefined') { + return clause; + } + if (clause.OR?.length) { + const nested = findClauseWithKey(clause.OR, key); + if (nested) { + return nested; + } + } + } + return undefined; + }; + const buildResource = (roleName: string, memberId = baseAuthUser.userId) => ({ id: 'resource-1', challengeId: 'challenge-1', @@ -1290,6 +1356,15 @@ describe('ReviewService.getReviews reviewer visibility', () => { eventBusServiceMock, ); + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Active Challenge', + status: ChallengeStatus.ACTIVE, + phases: [ + { id: 'phase-review', name: 'Review', isOpen: false }, + { id: 'phase-screening', name: 'Screening', isOpen: false }, + ], + }); challengeApiServiceMock.getChallenges.mockResolvedValue([]); }); @@ -1307,8 +1382,20 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(prismaMock.review.findMany).toHaveBeenCalledTimes(1); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.resourceId).toEqual({ in: ['resource-1'] }); - expect(callArgs.where.submissionId).toEqual({ in: [baseSubmission.id] }); + const whereClauses = flattenWhereClauses(callArgs.where); + const resourceOrClause = whereClauses.find( + (clause) => + clause.OR?.some((entry) => typeof entry.resourceId !== 'undefined') ?? + false, + ); + expect(resourceOrClause?.OR).toEqual([ + { resourceId: { in: ['resource-1'] } }, + { phaseId: { in: ['phase-screening'] } }, + ]); + const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); + expect(submissionClause?.submissionId).toEqual({ + in: [baseSubmission.id], + }); }); it('filters reviews by the resource id when the requester is an approver', async () => { @@ -1319,7 +1406,11 @@ describe('ReviewService.getReviews reviewer visibility', () => { await service.getReviews(baseAuthUser, undefined, 'challenge-1'); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.resourceId).toEqual({ in: ['resource-1'] }); + const whereClauses = flattenWhereClauses(callArgs.where); + const resourceClause = whereClauses.find( + (clause) => typeof clause.resourceId !== 'undefined', + ); + expect(resourceClause?.resourceId).toEqual({ in: ['resource-1'] }); }); it('filters reviews by the resource id when the requester is a checkpoint screener', async () => { @@ -1330,7 +1421,11 @@ describe('ReviewService.getReviews reviewer visibility', () => { await service.getReviews(baseAuthUser, undefined, 'challenge-1'); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.resourceId).toEqual({ in: ['resource-1'] }); + const whereClauses = flattenWhereClauses(callArgs.where); + const resourceClause = whereClauses.find( + (clause) => typeof clause.resourceId !== 'undefined', + ); + expect(resourceClause?.resourceId).toEqual({ in: ['resource-1'] }); }); it('filters reviews by the resource id when the requester is a checkpoint reviewer', async () => { @@ -1341,7 +1436,16 @@ describe('ReviewService.getReviews reviewer visibility', () => { await service.getReviews(baseAuthUser, undefined, 'challenge-1'); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.resourceId).toEqual({ in: ['resource-1'] }); + const whereClauses = flattenWhereClauses(callArgs.where); + const resourceOrClause = whereClauses.find( + (clause) => + clause.OR?.some((entry) => typeof entry.resourceId !== 'undefined') ?? + false, + ); + expect(resourceOrClause?.OR).toEqual([ + { resourceId: { in: ['resource-1'] } }, + { phaseId: { in: ['phase-screening'] } }, + ]); }); it('does not restrict resource visibility for copilots', async () => { @@ -1400,7 +1504,9 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(response.meta.totalPages).toBe(0); expect(prismaMock.review.findMany).toHaveBeenCalledTimes(1); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.submissionId).toEqual({ in: ['__none__'] }); + const whereClauses = flattenWhereClauses(callArgs.where); + const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); + expect(submissionClause?.submissionId).toEqual({ in: ['__none__'] }); expect(prismaMock.review.count).toHaveBeenCalledWith({ where: callArgs.where, }); @@ -1436,7 +1542,9 @@ describe('ReviewService.getReviews reviewer visibility', () => { await service.getReviews(baseAuthUser, undefined, 'challenge-1'); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.submissionId).toEqual({ + const whereClauses = flattenWhereClauses(callArgs.where); + const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); + expect(submissionClause?.submissionId).toEqual({ in: [baseSubmission.id, 'submission-other'], }); }); @@ -1472,7 +1580,9 @@ describe('ReviewService.getReviews reviewer visibility', () => { ); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.submissionId).toEqual({ + const whereClauses = flattenWhereClauses(callArgs.where); + const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); + expect(submissionClause?.submissionId).toEqual({ in: [baseSubmission.id, 'submission-other'], }); expect(result.data).toEqual([]); @@ -2209,7 +2319,9 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(response.data[0].reviewItems).toHaveLength(1); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - expect(callArgs.where.submissionId).toEqual({ + const whereClauses = flattenWhereClauses(callArgs.where); + const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); + expect(submissionClause?.submissionId).toEqual({ in: ['submission-1', 'submission-2'], }); }); @@ -3443,6 +3555,8 @@ describe('ReviewService.updateReview challenge status enforcement', () => { resourceId: 'resource-1', status: ReviewStatus.COMPLETED, committed: true, + initialScore: 95, + finalScore: 90, reviewDate: new Date('2024-01-15T10:00:00Z'), submission: { challengeId: 'challenge-1', @@ -3454,6 +3568,8 @@ describe('ReviewService.updateReview challenge status enforcement', () => { ...existingReview, status: ReviewStatus.PENDING, committed: false, + initialScore: null, + finalScore: null, reviewDate: null, }); @@ -3487,6 +3603,8 @@ describe('ReviewService.updateReview challenge status enforcement', () => { id: 'review-1', status: ReviewStatus.PENDING, committed: false, + initialScore: null, + finalScore: null, reviewDate: null, appeals: [], phaseName: null, @@ -3497,6 +3615,8 @@ describe('ReviewService.updateReview challenge status enforcement', () => { expect(updateArgs.data).toMatchObject({ status: ReviewStatus.PENDING, committed: false, + initialScore: null, + finalScore: null, reviewDate: null, }); expect(resourceApiServiceMock.getResources).toHaveBeenCalledWith({ @@ -3507,7 +3627,7 @@ describe('ReviewService.updateReview challenge status enforcement', () => { 'challenge-1', 'copilot-1', ); - expect(recomputeSpy).toHaveBeenCalledWith('review-1'); + expect(recomputeSpy).not.toHaveBeenCalled(); }); it('defaults committed to false and clears reviewDate when reopening with only status provided', async () => { @@ -3522,6 +3642,8 @@ describe('ReviewService.updateReview challenge status enforcement', () => { resourceId: 'resource-1', status: ReviewStatus.COMPLETED, committed: true, + initialScore: 90, + finalScore: 95, reviewDate: new Date('2024-01-20T12:00:00Z'), submission: { challengeId: 'challenge-1', @@ -3533,6 +3655,8 @@ describe('ReviewService.updateReview challenge status enforcement', () => { ...existingReview, status: ReviewStatus.IN_PROGRESS, committed: false, + initialScore: null, + finalScore: null, reviewDate: null, }); @@ -3565,6 +3689,8 @@ describe('ReviewService.updateReview challenge status enforcement', () => { id: 'review-1', status: ReviewStatus.IN_PROGRESS, committed: false, + initialScore: null, + finalScore: null, reviewDate: null, appeals: [], phaseName: null, @@ -3574,6 +3700,8 @@ describe('ReviewService.updateReview challenge status enforcement', () => { expect(updateArgs.data).toMatchObject({ status: ReviewStatus.IN_PROGRESS, committed: false, + initialScore: null, + finalScore: null, reviewDate: null, }); expect(resourceApiServiceMock.getResources).toHaveBeenCalledWith({ @@ -3584,7 +3712,7 @@ describe('ReviewService.updateReview challenge status enforcement', () => { 'challenge-1', 'copilot-1', ); - expect(recomputeSpy).toHaveBeenCalledWith('review-1'); + expect(recomputeSpy).not.toHaveBeenCalled(); }); it('records an audit entry when an admin updates review status and answers', async () => { diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 05106e2..77bb3ee 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -2154,9 +2154,9 @@ export class ReviewService { } as Prisma.reviewUpdateInput; if (isReopenTransition) { - if (updateData.committed === undefined) { - updateData.committed = false; - } + updateData.committed = false; + updateData.initialScore = null; + updateData.finalScore = null; if (updateData.reviewDate === undefined) { updateData.reviewDate = null; } @@ -2173,7 +2173,9 @@ export class ReviewService { }, }); // Recalculate scores based on current review items - const recomputedScores = await this.recomputeAndUpdateReviewScores(id); + const recomputedScores = isReopenTransition + ? null + : await this.recomputeAndUpdateReviewScores(id); if ( existingReview.status !== ReviewStatus.COMPLETED && data.status === ReviewStatus.COMPLETED @@ -2565,10 +2567,12 @@ export class ReviewService { try { const reviewWhereClause: any = {}; let challengeDetail: ChallengeData | null = null; + let challengeScopedFilter: Prisma.reviewWhereInput | null = null; let requesterIsChallengeResource = false; const reviewerResourceIdSet = new Set(); const submitterSubmissionIdSet = new Set(); let hasCopilotRoleForChallenge = false; + let hasReviewerRoleForChallenge = false; let hasSubmitterRoleForChallenge = false; let submitterVisibilityState = { allowAny: false, @@ -2642,16 +2646,57 @@ export class ReviewService { if (challengeId) { this.logger.debug(`Fetching reviews by challengeId: ${challengeId}`); + + const hasDirectSubmissionFilter = Boolean(submissionId); + const submissions = await this.prisma.submission.findMany({ where: { challengeId }, select: { id: true }, }); const submissionIds = submissions.map((s) => s.id); + const challengeFilters: Prisma.reviewWhereInput[] = []; - if (submissionIds.length > 0) { - reviewWhereClause.submissionId = { in: submissionIds }; - } else { + if (!hasDirectSubmissionFilter && submissionIds.length > 0) { + challengeFilters.push({ submissionId: { in: submissionIds } }); + } + + if (!challengeDetail) { + try { + challengeDetail = + await this.challengeApiService.getChallengeDetail(challengeId); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.debug( + `[getReviews] Unable to fetch challenge detail for phase filtering: ${message}`, + ); + challengeDetail = null; + } + } + + if (challengeDetail?.phases?.length) { + const phaseIds = new Set(); + for (const phase of challengeDetail.phases ?? []) { + if (!phase) { + continue; + } + const candidates = [ + String((phase as any).id ?? '').trim(), + String((phase as any).phaseId ?? '').trim(), + ].filter((value) => value.length > 0); + + for (const candidate of candidates) { + phaseIds.add(candidate); + } + } + + if (phaseIds.size > 0) { + challengeFilters.push({ phaseId: { in: Array.from(phaseIds) } }); + } + } + + if (!hasDirectSubmissionFilter && challengeFilters.length === 0) { return { data: [], meta: { @@ -2662,6 +2707,12 @@ export class ReviewService { }, }; } + + if (challengeFilters.length === 1) { + challengeScopedFilter = challengeFilters[0]; + } else if (challengeFilters.length > 1) { + challengeScopedFilter = { OR: challengeFilters }; + } } // Authorization filtering for non-admin member tokens @@ -2689,6 +2740,9 @@ export class ReviewService { ) { reviewerResourceIdSet.add(r.id); } + if (roleName.includes('reviewer')) { + hasReviewerRoleForChallenge = true; + } }); hasCopilotRoleForChallenge = normalized.some((r) => (r.roleName || '').toLowerCase().includes('copilot'), @@ -2710,7 +2764,42 @@ export class ReviewService { if (hasCopilotRoleForChallenge) { // Copilots retain full visibility for the challenge } else if (reviewerResourceIdSet.size) { - restrictToResourceIds(Array.from(reviewerResourceIdSet)); + const reviewerResourceIds = Array.from(reviewerResourceIdSet); + if (hasReviewerRoleForChallenge && challengeId) { + let screeningPhaseIds: string[] = []; + if (!challengeDetail) { + try { + challengeDetail = + await this.challengeApiService.getChallengeDetail( + challengeId, + ); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.debug( + `[getReviews] Unable to fetch challenge ${challengeId} for reviewer screening access: ${message}`, + ); + } + } + + if (challengeDetail) { + screeningPhaseIds = this.getPhaseIdsForNames(challengeDetail, [ + 'screening', + 'checkpoint screening', + ]); + } + + if (screeningPhaseIds.length) { + reviewWhereClause.OR = [ + { resourceId: { in: reviewerResourceIds } }, + { phaseId: { in: screeningPhaseIds } }, + ]; + } else { + restrictToResourceIds(reviewerResourceIds); + } + } else { + restrictToResourceIds(reviewerResourceIds); + } } else { // Confirm the user has actually submitted to this challenge const mySubs = await this.prisma.submission.findMany({ @@ -2878,8 +2967,22 @@ export class ReviewService { ].includes(challengeDetail.status) && (isAdmin(authUser) || requesterIsChallengeResource); + const whereAndClauses: Prisma.reviewWhereInput[] = []; + if (Object.keys(reviewWhereClause).length > 0) { + whereAndClauses.push(reviewWhereClause); + } + if (challengeScopedFilter) { + whereAndClauses.push(challengeScopedFilter); + } + const finalWhereClause: Prisma.reviewWhereInput = + whereAndClauses.length === 0 + ? {} + : whereAndClauses.length === 1 + ? whereAndClauses[0] + : { AND: whereAndClauses }; + this.logger.debug(`Fetching reviews with where clause:`); - this.logger.debug(reviewWhereClause); + this.logger.debug(finalWhereClause); const reviewInclude: Prisma.reviewInclude = { submission: { @@ -2894,7 +2997,7 @@ export class ReviewService { } const reviews = await this.prisma.review.findMany({ - where: reviewWhereClause, + where: finalWhereClause, skip, take: perPage, include: reviewInclude, @@ -3317,7 +3420,7 @@ export class ReviewService { }); const totalCount = await this.prisma.review.count({ - where: reviewWhereClause, + where: finalWhereClause, }); this.logger.log( @@ -3372,6 +3475,7 @@ export class ReviewService { let resolvedPhaseName: string | null = null; let challengeDetail: ChallengeData | null = null; let hasCopilotRole = false; + let hasReviewerRoleResource = false; let reviewerResourceIds = new Set(); let isReviewerForReview = false; let isSubmitterForChallenge = false; @@ -3408,6 +3512,9 @@ export class ReviewService { roleName.includes(keyword), ); }); + hasReviewerRoleResource = reviewerResources.some((resource) => + (resource.roleName || '').toLowerCase().includes('reviewer'), + ); hasCopilotRole = resources.some((r) => (r.roleName || '').toLowerCase().includes('copilot'), ); @@ -3429,9 +3536,23 @@ export class ReviewService { ChallengeStatus.COMPLETED, ChallengeStatus.CANCELLED_FAILED_REVIEW, ].includes(challenge.status); + if (!resolvedPhaseName && data.phaseId && challengeId) { + resolvedPhaseName = await this.resolvePhaseNameFromChallenge({ + challengeId, + phaseId: data.phaseId, + challengeCache, + }); + } + const normalizedReviewerPhase = + this.normalizePhaseName(resolvedPhaseName); + const isScreeningPhaseForReviewer = + hasReviewerRoleResource && + (normalizedReviewerPhase === 'screening' || + normalizedReviewerPhase === 'checkpoint screening'); if ( !challengeInFinalState && - !reviewerResourceIds.has(reviewResourceId) + !reviewerResourceIds.has(reviewResourceId) && + !isScreeningPhaseForReviewer ) { throw new ForbiddenException({ message: @@ -3786,6 +3907,50 @@ export class ReviewService { }); } + private getPhaseIdsForNames( + challenge: ChallengeData | null | undefined, + phaseNames: string[], + ): string[] { + if (!challenge?.phases?.length) { + return []; + } + + const normalizedTargets = new Set( + (phaseNames ?? []) + .map((name) => this.normalizePhaseName(name)) + .filter((name) => name.length > 0), + ); + + if (!normalizedTargets.size) { + return []; + } + + const phaseIds: string[] = []; + + for (const phase of challenge.phases ?? []) { + if (!phase) { + continue; + } + + const normalizedName = this.normalizePhaseName((phase as any).name); + if (!normalizedTargets.has(normalizedName)) { + continue; + } + + const candidateIds = [(phase as any).id, (phase as any).phaseId] + .map((value) => String(value ?? '').trim()) + .filter((value) => value.length > 0); + + for (const candidate of candidateIds) { + if (!phaseIds.includes(candidate)) { + phaseIds.push(candidate); + } + } + } + + return phaseIds; + } + private getSubmitterVisibilityForChallenge( challenge?: ChallengeData | null, ): { allowAny: boolean; allowOwn: boolean } { diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index a44e46b..94f16cf 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -282,6 +282,7 @@ describe('SubmissionService', () => { describe('getSubmissionFileStream', () => { let prismaMock: { submission: { findFirst: jest.Mock } }; let challengeApiServiceMock: { getChallengeDetail: jest.Mock }; + let checkSubmissionSpy: jest.SpyInstance; beforeEach(() => { prismaMock = { @@ -305,12 +306,14 @@ describe('SubmissionService', () => { {} as any, {} as any, ); - jest.spyOn(service as any, 'checkSubmission').mockResolvedValue({ - id: 'sub-123', - memberId: 'owner-user', - challengeId: 'challenge-xyz', - url: 'https://s3.amazonaws.com/dummy/submission.zip', - }); + checkSubmissionSpy = jest + .spyOn(service as any, 'checkSubmission') + .mockResolvedValue({ + id: 'sub-123', + memberId: 'owner-user', + challengeId: 'challenge-xyz', + url: 'https://s3.amazonaws.com/dummy/submission.zip', + }); jest .spyOn(service as any, 'parseS3Url') .mockReturnValue({ key: 'dummy/submission.zip' }); @@ -326,6 +329,55 @@ describe('SubmissionService', () => { }); }); + it('allows screeners to download submissions', async () => { + resourceApiService.getMemberResourcesRoles.mockResolvedValue([ + { roleName: 'Screener' }, + ]); + + const result = await service.getSubmissionFileStream( + { + userId: 'screener-user', + isMachine: false, + roles: [], + } as any, + 'sub-123', + ); + + expect(result.fileName).toBe('submission-sub-123.zip'); + expect(resourceApiService.getMemberResourcesRoles).toHaveBeenCalledWith( + 'challenge-xyz', + 'screener-user', + ); + }); + + it('allows checkpoint screeners to download checkpoint submissions', async () => { + checkSubmissionSpy.mockResolvedValueOnce({ + id: 'checkpoint-sub-123', + memberId: 'owner-user', + challengeId: 'challenge-xyz', + url: 'https://s3.amazonaws.com/dummy/checkpoint.zip', + type: SubmissionType.CHECKPOINT_SUBMISSION, + }); + resourceApiService.getMemberResourcesRoles.mockResolvedValue([ + { roleName: 'Checkpoint Screener' }, + ]); + + const result = await service.getSubmissionFileStream( + { + userId: 'checkpoint-screener-user', + isMachine: false, + roles: [], + } as any, + 'checkpoint-sub-123', + ); + + expect(result.fileName).toBe('submission-checkpoint-sub-123.zip'); + expect(resourceApiService.getMemberResourcesRoles).toHaveBeenCalledWith( + 'challenge-xyz', + 'checkpoint-screener-user', + ); + }); + it('allows submitters with passing reviews to download when challenge is completed', async () => { resourceApiService.getMemberResourcesRoles.mockResolvedValue([ { roleName: 'Submitter' }, diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index b5b5795..a82e241 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -482,6 +482,8 @@ export class SubmissionService { let isReviewer = false; let isCopilot = false; let isSubmitter = false; + const isCheckpointSubmission = + submission.type === SubmissionType.CHECKPOINT_SUBMISSION; if (!isOwner && submission.challengeId && uid) { try { const resources = @@ -494,6 +496,15 @@ export class SubmissionService { if (rn.includes('reviewer')) { isReviewer = true; } + if (rn.includes('screener')) { + const roleIsCheckpoint = rn.includes('checkpoint'); + if ( + (isCheckpointSubmission && roleIsCheckpoint) || + (!isCheckpointSubmission && !roleIsCheckpoint) + ) { + isReviewer = true; + } + } if (rn.includes('copilot')) { isCopilot = true; } @@ -688,7 +699,9 @@ export class SubmissionService { for (const r of resources) { const rn = (r.roleName || '').toLowerCase(); if (rn.includes('copilot')) isAdminOrCopilot = true; - if (rn.includes('reviewer')) isReviewer = true; + if (rn.includes('reviewer') || rn.includes('screener')) { + isReviewer = true; + } } } catch { // Fall through; if we can't confirm roles, deny From 172449f7bd97e29f7d3d90fdab50c2bfaeda190e Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Wed, 22 Oct 2025 16:28:03 +1100 Subject: [PATCH 22/48] Fix for MM review summation pulling to allow submitters to see each other's scores --- .../review-summation.service.ts | 73 ++++++++++++++----- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/src/api/review-summation/review-summation.service.ts b/src/api/review-summation/review-summation.service.ts index 16d16b5..15a010a 100644 --- a/src/api/review-summation/review-summation.service.ts +++ b/src/api/review-summation/review-summation.service.ts @@ -614,29 +614,64 @@ export class ReviewSummationService { }); } + let memberResources: unknown[] = []; + let resourceLookupFailed = false; try { - await this.resourceApiService.validateSubmitterRegistration( - challengeIdFilter, - userId, - ); - } catch (validationError) { - const details = - validationError instanceof Error - ? validationError.message - : String(validationError); - throw new ForbiddenException({ - message: - 'Submitter access requires active registration for this challenge.', - code: 'SUBMITTER_NOT_REGISTERED', - details: { - challengeId: challengeIdFilter, - memberId: userId, - info: details, - }, + memberResources = await this.resourceApiService.getResources({ + challengeId: challengeIdFilter, + memberId: userId, }); + } catch (resourceLookupError) { + resourceLookupFailed = true; + const message = + resourceLookupError instanceof Error + ? resourceLookupError.message + : String(resourceLookupError); + this.logger.warn( + `[searchSummation] Unable to load member resources for challenge ${challengeIdFilter} and member ${userId}: ${message}`, + ); } - enforcedMemberId = userId; + if (!resourceLookupFailed) { + const hasAnyResource = + Array.isArray(memberResources) && memberResources.length > 0; + if (!hasAnyResource) { + throw new ForbiddenException({ + message: + 'Submitter access requires active registration for this challenge.', + code: 'SUBMITTER_NOT_REGISTERED', + details: { + challengeId: challengeIdFilter, + memberId: userId, + info: 'Member does not have any resources on this challenge.', + }, + }); + } + } else { + try { + await this.resourceApiService.validateSubmitterRegistration( + challengeIdFilter, + userId, + ); + } catch (validationError) { + const details = + validationError instanceof Error + ? validationError.message + : String(validationError); + throw new ForbiddenException({ + message: + 'Submitter access requires active registration for this challenge.', + code: 'SUBMITTER_NOT_REGISTERED', + details: { + challengeId: challengeIdFilter, + memberId: userId, + info: details, + }, + }); + } + + enforcedMemberId = userId; + } } // Build the where clause for review summations based on available filter parameters From e0bc50ab14903b632b679536879a39bc821d9cde Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 23 Oct 2025 08:19:14 +1100 Subject: [PATCH 23/48] Metadata for storing review details, and remove payments endpoint that move to the finance API --- .../migration.sql | 3 + prisma/schema.prisma | 1 + src/api/api.module.ts | 2 - src/api/payments/payments.controller.ts | 133 --------------- .../review-summation.service.ts | 18 ++- src/dto/reviewSummation.dto.ts | 39 +++++ .../modules/global/finance-prisma.service.ts | 151 ------------------ .../modules/global/globalProviders.module.ts | 3 - 8 files changed, 57 insertions(+), 293 deletions(-) create mode 100644 prisma/migrations/20251010110000_add_metadata_to_review_summation/migration.sql delete mode 100644 src/api/payments/payments.controller.ts delete mode 100644 src/shared/modules/global/finance-prisma.service.ts diff --git a/prisma/migrations/20251010110000_add_metadata_to_review_summation/migration.sql b/prisma/migrations/20251010110000_add_metadata_to_review_summation/migration.sql new file mode 100644 index 0000000..3736504 --- /dev/null +++ b/prisma/migrations/20251010110000_add_metadata_to_review_summation/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "reviewSummation" + ADD COLUMN "metadata" JSONB; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index cd65a46..1518323 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -368,6 +368,7 @@ model reviewSummation { createdBy String? updatedAt DateTime @updatedAt updatedBy String? + metadata Json? submission submission @relation(fields: [submissionId], references: [id], onDelete: Cascade) diff --git a/src/api/api.module.ts b/src/api/api.module.ts index b091777..6b9c106 100644 --- a/src/api/api.module.ts +++ b/src/api/api.module.ts @@ -35,7 +35,6 @@ import { ProjectResultService } from './project-result/projectResult.service'; import { ProjectResultController } from './project-result/projectResult.controller'; import { MyReviewController } from './my-review/myReview.controller'; import { MyReviewService } from './my-review/myReview.service'; -import { PaymentsController } from './payments/payments.controller'; @Module({ imports: [ @@ -60,7 +59,6 @@ import { PaymentsController } from './payments/payments.controller'; WebhookController, AiWorkflowController, ProjectResultController, - PaymentsController, ], providers: [ ReviewService, diff --git a/src/api/payments/payments.controller.ts b/src/api/payments/payments.controller.ts deleted file mode 100644 index 7ea943c..0000000 --- a/src/api/payments/payments.controller.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { - Controller, - Get, - Param, - Req, - Query, - UnauthorizedException, -} from '@nestjs/common'; -import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger'; -import { JwtUser, isAdmin } from 'src/shared/modules/global/jwt.service'; -import { ResourceApiService } from 'src/shared/modules/global/resource.service'; -import { FinancePrismaService } from 'src/shared/modules/global/finance-prisma.service'; - -@ApiTags('Payments') -@Controller('payments') -export class PaymentsController { - constructor( - private readonly financeDb: FinancePrismaService, - private readonly resourceApi: ResourceApiService, - ) {} - - @Get('challenges/:challengeId') - @ApiBearerAuth() - @ApiOperation({ - summary: - 'List payments (winnings) for a challenge with role-aware filtering', - }) - async getPaymentsForChallenge( - @Param('challengeId') challengeId: string, - @Req() req: any, - @Query('winnerOnly') winnerOnly?: string, - ) { - const authUser: JwtUser | undefined = req['user'] as JwtUser; - if (!authUser) { - throw new UnauthorizedException('Missing or invalid token'); - } - - // Defaults - let allowAllForChallenge = false; - let filterWinnerId: string | undefined = undefined; - - // Admins (and M2M tokens) can see all payments for the challenge - if (authUser.isMachine || isAdmin(authUser)) { - allowAllForChallenge = true; - } else { - const requesterId = String(authUser.userId ?? '').trim(); - if (!requesterId) { - throw new UnauthorizedException( - 'Authenticated user is missing required identifier', - ); - } - - // If explicitly requested to see only own winnings, enforce winner filter - if ((winnerOnly || '').toLowerCase() === 'true') { - filterWinnerId = requesterId; - } - - // Check copilot assignment for this challenge - try { - const roles = await this.resourceApi.getMemberResourcesRoles( - challengeId, - requesterId, - ); - const hasCopilotRole = roles.some((r) => - String(r.roleName ?? '') - .toLowerCase() - .includes('copilot'), - ); - if (hasCopilotRole) { - allowAllForChallenge = true; - } - } catch { - // If resource API returns 403/404, we still allow submitter visibility. - } - - // If not admin nor copilot, limit to winner_id = requester - if (!allowAllForChallenge) { - filterWinnerId = requesterId; - } - } - - // Query finance DB - const rows = await this.financeDb.getWinningsByExternalId( - challengeId, - filterWinnerId, - ); - - // Shape the response similar to Wallet Admin winnings - const data = rows.map((w) => ({ - id: w.winning_id, - type: 'PAYMENT', - handle: '', // handle to be resolved by the caller if needed - winnerId: w.winner_id, - origin: '', - category: w.category ?? 'CONTEST_PAYMENT', - title: w.title ?? undefined, - description: w.description ?? '', - externalId: w.external_id ?? challengeId, - attributes: { url: '' }, - details: (w.details || []).map((d) => ({ - id: d.id, - netAmount: d.net_amount ?? '0', - grossAmount: d.gross_amount ?? '0', - totalAmount: d.total_amount ?? '0', - installmentNumber: d.installment_number ?? 1, - status: d.status ?? 'OWED', - currency: d.currency ?? 'USD', - datePaid: d.date_paid - ? new Date(d.date_paid as any).toISOString() - : null, - })), - createdAt: w.created_at - ? new Date(w.created_at as any).toISOString() - : new Date().toISOString(), - releaseDate: (w.details?.[0]?.release_date as any) - ? new Date(w.details?.[0]?.release_date as any).toISOString() - : new Date().toISOString(), - datePaid: (w.details?.[0]?.date_paid as any) - ? new Date(w.details?.[0]?.date_paid as any).toISOString() - : null, - })); - - return { - winnings: data, - pagination: { - totalItems: data.length, - totalPages: 1, - pageSize: data.length, - currentPage: 1, - }, - }; - } -} diff --git a/src/api/review-summation/review-summation.service.ts b/src/api/review-summation/review-summation.service.ts index 15a010a..3597ecf 100644 --- a/src/api/review-summation/review-summation.service.ts +++ b/src/api/review-summation/review-summation.service.ts @@ -557,6 +557,8 @@ export class ReviewSummationService { : undefined; const challengeIdFilter = rawChallengeId && rawChallengeId.length ? rawChallengeId : undefined; + const includeMetadata = + (queryDto.metadata ?? '').toLowerCase() === 'true'; let enforcedMemberId: string | undefined; @@ -810,9 +812,11 @@ export class ReviewSummationService { }); const data: ReviewSummationResponseDto[] = summations.map((summation) => { - const { submission, ...rest } = summation as typeof summation & { - submission?: { memberId: string | null }; - }; + const { submission, metadata, ...rest } = + summation as typeof summation & { + submission?: { memberId: string | null }; + metadata?: unknown; + }; let submitterId: number | null = null; let submitterHandle: string | null = null; @@ -837,12 +841,18 @@ export class ReviewSummationService { } } - return { + const base: ReviewSummationResponseDto = { ...rest, submitterId, submitterHandle, submitterMaxRating, } as ReviewSummationResponseDto; + + if (includeMetadata) { + base.metadata = metadata ?? null; + } + + return base; }); this.logger.log( diff --git a/src/dto/reviewSummation.dto.ts b/src/dto/reviewSummation.dto.ts index 1448c95..a3bbe2d 100644 --- a/src/dto/reviewSummation.dto.ts +++ b/src/dto/reviewSummation.dto.ts @@ -77,6 +77,15 @@ export class ReviewSummationQueryDto { @IsString() @IsNotEmpty() challengeId?: string; + + @ApiProperty({ + description: + 'When true, include the metadata payload for each review summation in responses', + required: false, + }) + @IsOptional() + @IsBooleanString() + metadata?: string; } export class ReviewSummationBaseRequestDto { @@ -140,6 +149,16 @@ export class ReviewSummationBaseRequestDto { @IsOptional() @IsDateString() reviewedDate?: string; + + @ApiProperty({ + description: + 'Auxiliary metadata for the review summation (test scores, etc.)', + required: false, + type: Object, + additionalProperties: true, + }) + @IsOptional() + metadata?: unknown; } export class ReviewSummationRequestDto extends ReviewSummationBaseRequestDto {} @@ -215,6 +234,16 @@ export class ReviewSummationUpdateRequestDto { @IsOptional() @IsDateString() reviewedDate?: string; + + @ApiProperty({ + description: + 'Auxiliary metadata for the review summation (test scores, etc.)', + required: false, + type: Object, + additionalProperties: true, + }) + @IsOptional() + metadata?: unknown; } export class ReviewSummationResponseDto { @@ -321,6 +350,16 @@ export class ReviewSummationResponseDto { type: Number, }) submitterMaxRating?: number | null; + + @ApiProperty({ + description: + 'Auxiliary metadata for the review summation (test scores, etc.)', + required: false, + nullable: true, + type: Object, + additionalProperties: true, + }) + metadata?: Record | null; } export class ReviewSummationBatchResponseDto { diff --git a/src/shared/modules/global/finance-prisma.service.ts b/src/shared/modules/global/finance-prisma.service.ts deleted file mode 100644 index 8898cde..0000000 --- a/src/shared/modules/global/finance-prisma.service.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; -import { PrismaClient, Prisma } from '@prisma/client'; - -type WinningDetail = { - id: string; - net_amount: string | null; - gross_amount: string | null; - total_amount: string | null; - installment_number: number | null; - status: string | null; - currency: string | null; - date_paid: Date | null; - release_date: Date | null; -}; - -type WinningRow = { - winning_id: string; - winner_id: string; - category: string | null; - title: string | null; - description: string | null; - external_id: string | null; - created_at: Date | null; - details: WinningDetail[]; -}; - -/** - * Lightweight Prisma client targeting the Finance DB via FINANCE_DB_URL. - * Uses raw SQL queries, so no finance models are required in the generated client. - */ -@Injectable() -export class FinancePrismaService implements OnModuleInit, OnModuleDestroy { - private client: PrismaClient; - - constructor() { - const url = process.env.FINANCE_DB_URL || process.env.FINANCE_DATABASE_URL; - if (!url) { - // Intentionally not throwing here to allow app to boot; service methods will throw if used without URL - // This helps other features to work when payments are not configured. - - console.warn( - '[FinancePrismaService] FINANCE_DB_URL not set; payments features disabled.', - ); - } - - this.client = new PrismaClient({ - datasources: url ? { db: { url } } : undefined, - }); - } - - async onModuleInit(): Promise { - // Best-effort connect; if url is missing we skip connect and throw later on query - try { - await this.client.$connect(); - } catch (err) { - console.error( - '[FinancePrismaService] Failed to connect to Finance DB', - err, - ); - } - } - - async onModuleDestroy(): Promise { - try { - await this.client.$disconnect(); - } catch { - // ignore - } - } - - /** - * Fetch winnings for a challenge by `external_id`. When winnerId is provided, results are filtered to the user. - */ - async getWinningsByExternalId( - challengeId: string, - winnerId?: string, - ): Promise { - const hasUrl = Boolean( - process.env.FINANCE_DB_URL || process.env.FINANCE_DATABASE_URL, - ); - if (!hasUrl) { - throw new Error('FINANCE_DB_URL is not configured'); - } - - // Query base winnings rows - const baseRows: Array<{ - winning_id: string; - winner_id: string; - category: string | null; - title: string | null; - description: string | null; - external_id: string | null; - created_at: Date | null; - }> = await this.client.$queryRaw( - Prisma.sql`SELECT w.winning_id, w.winner_id, w.category, w.title, w.description, w.external_id, w.created_at - FROM winnings w - WHERE w.external_id = ${challengeId} - AND w.type = 'PAYMENT' - ${winnerId ? Prisma.sql`AND w.winner_id = ${winnerId}` : Prisma.sql``}`, - ); - - if (!baseRows.length) { - return [] as WinningRow[]; - } - - const ids = baseRows.map((r) => r.winning_id); - const idsList = Prisma.join(ids); - - // Query payment details for these winnings - const paymentRows: Array<{ - payment_id: string; - winnings_id: string; - net_amount: any; - gross_amount: any; - total_amount: any; - installment_number: number | null; - status: string | null; - currency: string | null; - date_paid: Date | null; - release_date: Date | null; - }> = await this.client.$queryRaw( - Prisma.sql`SELECT p.payment_id, p.winnings_id, p.net_amount, p.gross_amount, p.total_amount, p.installment_number, p.payment_status as status, p.currency, p.date_paid, p.release_date - FROM payment p - WHERE p.winnings_id = ANY(ARRAY[${idsList}]::uuid[])`, - ); - - const detailMap = new Map(); - for (const p of paymentRows) { - const arr = detailMap.get(p.winnings_id) || []; - arr.push({ - id: p.payment_id, - net_amount: p.net_amount?.toString?.() ?? p.net_amount, - gross_amount: p.gross_amount?.toString?.() ?? p.gross_amount, - total_amount: p.total_amount?.toString?.() ?? p.total_amount, - installment_number: p.installment_number ?? null, - status: p.status ?? null, - currency: p.currency ?? null, - date_paid: p.date_paid ?? null, - release_date: p.release_date ?? null, - }); - detailMap.set(p.winnings_id, arr); - } - - // Attach details - const result: WinningRow[] = baseRows.map((w) => ({ - ...w, - details: detailMap.get(w.winning_id) || [], - })); - return result; - } -} diff --git a/src/shared/modules/global/globalProviders.module.ts b/src/shared/modules/global/globalProviders.module.ts index 7a1eeff..930900b 100644 --- a/src/shared/modules/global/globalProviders.module.ts +++ b/src/shared/modules/global/globalProviders.module.ts @@ -22,7 +22,6 @@ import { ChallengePrismaService } from './challenge-prisma.service'; import { MemberPrismaService } from './member-prisma.service'; import { QueueSchedulerService } from './queue-scheduler.service'; import { WorkflowQueueHandler } from './workflow-queue.handler'; -import { FinancePrismaService } from './finance-prisma.service'; // Global module for providing global providers // Add any provider you want to be global here @@ -58,7 +57,6 @@ import { FinancePrismaService } from './finance-prisma.service'; SubmissionService, QueueSchedulerService, WorkflowQueueHandler, - FinancePrismaService, ], exports: [ PrismaService, @@ -79,7 +77,6 @@ import { FinancePrismaService } from './finance-prisma.service'; SubmissionScanCompleteOrchestrator, QueueSchedulerService, WorkflowQueueHandler, - FinancePrismaService, ], }) export class GlobalProvidersModule {} From 032ed4eded6e08231a8d810cb1dbe8ac971245b3 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 23 Oct 2025 08:30:57 +1100 Subject: [PATCH 24/48] Build fixes --- .../review-summation.service.ts | 43 ++++++++++++++++--- src/dto/reviewSummation.dto.ts | 7 +-- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/api/review-summation/review-summation.service.ts b/src/api/review-summation/review-summation.service.ts index 3597ecf..2e7ddac 100644 --- a/src/api/review-summation/review-summation.service.ts +++ b/src/api/review-summation/review-summation.service.ts @@ -25,6 +25,7 @@ import { import { MemberPrismaService } from 'src/shared/modules/global/member-prisma.service'; import { ResourceApiService } from 'src/shared/modules/global/resource.service'; import { UserRole } from 'src/shared/enums/userRole.enum'; +import { Prisma } from '@prisma/client'; @Injectable() export class ReviewSummationService { @@ -40,6 +41,20 @@ export class ReviewSummationService { private readonly systemActor = 'ReviewSummationService'; + private prepareMetadata( + metadata?: Prisma.JsonValue | null, + ): Prisma.InputJsonValue | Prisma.NullableJsonNullValueInput | undefined { + if (metadata === undefined) { + return undefined; + } + + if (metadata === null) { + return Prisma.JsonNull; + } + + return metadata as Prisma.InputJsonValue; + } + private phaseNameEquals( phaseName: string | null | undefined, target: string, @@ -451,10 +466,17 @@ export class ReviewSummationService { } } + const { metadata, ...rest } = body; + const createData: Prisma.reviewSummationUncheckedCreateInput = { + ...rest, + }; + const normalizedMetadata = this.prepareMetadata(metadata); + if (normalizedMetadata !== undefined) { + createData.metadata = normalizedMetadata; + } + const data = await this.prisma.reviewSummation.create({ - data: { - ...body, - }, + data: createData, }); this.logger.log(`Review summation created with ID: ${data.id}`); return data as ReviewSummationResponseDto; @@ -815,7 +837,7 @@ export class ReviewSummationService { const { submission, metadata, ...rest } = summation as typeof summation & { submission?: { memberId: string | null }; - metadata?: unknown; + metadata?: Prisma.JsonValue | null; }; let submitterId: number | null = null; @@ -941,11 +963,18 @@ export class ReviewSummationService { } } + const { metadata, ...rest } = body; + const updateData: Prisma.reviewSummationUncheckedUpdateInput = { + ...rest, + }; + const normalizedMetadata = this.prepareMetadata(metadata); + if (normalizedMetadata !== undefined) { + updateData.metadata = normalizedMetadata; + } + const data = await this.prisma.reviewSummation.update({ where: { id }, - data: { - ...body, - }, + data: updateData, }); this.logger.log(`Review summation updated successfully: ${id}`); return data as ReviewSummationResponseDto; diff --git a/src/dto/reviewSummation.dto.ts b/src/dto/reviewSummation.dto.ts index a3bbe2d..60dcdc1 100644 --- a/src/dto/reviewSummation.dto.ts +++ b/src/dto/reviewSummation.dto.ts @@ -1,4 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; +import { Prisma } from '@prisma/client'; import { IsString, IsNumber, @@ -158,7 +159,7 @@ export class ReviewSummationBaseRequestDto { additionalProperties: true, }) @IsOptional() - metadata?: unknown; + metadata?: Prisma.JsonValue; } export class ReviewSummationRequestDto extends ReviewSummationBaseRequestDto {} @@ -243,7 +244,7 @@ export class ReviewSummationUpdateRequestDto { additionalProperties: true, }) @IsOptional() - metadata?: unknown; + metadata?: Prisma.JsonValue; } export class ReviewSummationResponseDto { @@ -359,7 +360,7 @@ export class ReviewSummationResponseDto { type: Object, additionalProperties: true, }) - metadata?: Record | null; + metadata?: Prisma.JsonValue | null; } export class ReviewSummationBatchResponseDto { From f77ae4091334f1e97c37d37d24a3c1fd8ff0c143 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 23 Oct 2025 10:58:54 +1100 Subject: [PATCH 25/48] Better block for switching scorecards --- src/api/review/review.service.ts | 46 +++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 77bb3ee..662be92 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -3470,6 +3470,40 @@ export class ReviewService { }, }); + const reviewResourceId = String(data.resourceId ?? '').trim(); + let challengeId: string | null = null; + + if (data.submission?.challengeId) { + const normalizedChallengeId = String( + data.submission.challengeId, + ).trim(); + challengeId = normalizedChallengeId.length + ? normalizedChallengeId + : null; + } + + if (!challengeId && reviewResourceId.length) { + try { + const resourceRecord = await this.resourcePrisma.resource.findUnique({ + where: { id: reviewResourceId }, + select: { challengeId: true }, + }); + + const resourceChallengeId = String( + resourceRecord?.challengeId ?? '', + ).trim(); + if (resourceChallengeId.length) { + challengeId = resourceChallengeId; + } + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.debug( + `[getReview] Failed to resolve challengeId via resource ${reviewResourceId}: ${message}`, + ); + } + } + const challengeCache = new Map(); const isPrivilegedRequester = authUser?.isMachine || isAdmin(authUser); let resolvedPhaseName: string | null = null; @@ -3483,7 +3517,6 @@ export class ReviewService { // Authorization for non-M2M, non-admin users if (!authUser?.isMachine && !isAdmin(authUser)) { const uid = String(authUser?.userId ?? ''); - const challengeId = data.submission?.challengeId; if (!challengeId) { throw new ForbiddenException({ @@ -3526,9 +3559,10 @@ export class ReviewService { } reviewerResourceIds = new Set( - reviewerResources.map((r) => String(r.id)), + reviewerResources + .map((r) => String(r.id ?? '').trim()) + .filter((id) => id.length > 0), ); - const reviewResourceId = String(data.resourceId ?? ''); isReviewerForReview = reviewerResourceIds.has(reviewResourceId); if (reviewerResources.length > 0) { @@ -3656,7 +3690,7 @@ export class ReviewService { result.appeals = flattenedAppeals; if (!resolvedPhaseName) { resolvedPhaseName = await this.resolvePhaseNameFromChallenge({ - challengeId: data.submission?.challengeId ?? null, + challengeId, phaseId: data.phaseId ?? null, challengeCache, }); @@ -3666,8 +3700,8 @@ export class ReviewService { const normalizedPhaseName = this.normalizePhaseName(resolvedPhaseName); const requesterId = String(authUser?.userId ?? ''); const submissionOwnerId = String(data.submission?.memberId ?? ''); - const challengeForReview = data.submission?.challengeId - ? (challengeCache.get(data.submission.challengeId) ?? challengeDetail) + const challengeForReview = challengeId + ? (challengeCache.get(challengeId) ?? challengeDetail) : challengeDetail; const shouldTrimIterativeReviewForOtherSubmitters = !isPrivilegedRequester && From f9f1229edf3842c75f796598ab7335c0400c8c6f Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 23 Oct 2025 12:38:18 +1100 Subject: [PATCH 26/48] Allow F2F submitters to see other submissions once the challenge is complete --- src/api/submission/submission.service.spec.ts | 42 +++++++++++++ src/api/submission/submission.service.ts | 63 +++++++++++++++---- 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index 94f16cf..1cd25ca 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -384,6 +384,7 @@ describe('SubmissionService', () => { ]); challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ status: ChallengeStatus.COMPLETED, + type: 'Something Else', }); prismaMock.submission.findFirst.mockResolvedValue({ id: 'passing-sub', @@ -441,6 +442,47 @@ describe('SubmissionService', () => { ).rejects.toBeInstanceOf(ForbiddenException); expect(prismaMock.submission.findFirst).not.toHaveBeenCalled(); }); + + it('allows First2Finish submitters to download any submission when challenge is completed', async () => { + resourceApiService.getMemberResourcesRoles.mockResolvedValue([ + { roleName: 'Submitter' }, + ]); + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + status: ChallengeStatus.COMPLETED, + type: 'First2Finish', + legacy: { subTrack: 'first_2_finish' }, + }); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'own-submission', + }); + + const result = await service.getSubmissionFileStream( + { + userId: 'submitter-user', + isMachine: false, + roles: [], + } as any, + 'sub-123', + ); + + expect(result.fileName).toBe('submission-sub-123.zip'); + expect(resourceApiService.getMemberResourcesRoles).toHaveBeenCalledWith( + 'challenge-xyz', + 'submitter-user', + ); + expect(challengeApiServiceMock.getChallengeDetail).toHaveBeenCalledWith( + 'challenge-xyz', + ); + expect(prismaMock.submission.findFirst).toHaveBeenCalledTimes(1); + expect(prismaMock.submission.findFirst).toHaveBeenCalledWith({ + where: { + challengeId: 'challenge-xyz', + memberId: 'submitter-user', + }, + select: { id: true }, + }); + expect(s3Send).toHaveBeenCalledTimes(2); + }); }); describe('listSubmission', () => { diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index a82e241..d25a471 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -24,7 +24,10 @@ import { ChallengePrismaService } from 'src/shared/modules/global/challenge-pris import { MemberPrismaService } from 'src/shared/modules/global/member-prisma.service'; import { Utils } from 'src/shared/modules/global/utils.service'; import { PrismaErrorService } from 'src/shared/modules/global/prisma-error.service'; -import { ChallengeApiService } from 'src/shared/modules/global/challenge.service'; +import { + ChallengeApiService, + ChallengeData, +} from 'src/shared/modules/global/challenge.service'; import { ChallengeCatalogService } from 'src/shared/modules/global/challenge-catalog.service'; import { ResourceApiService } from 'src/shared/modules/global/resource.service'; import { ArtifactsCreateResponseDto } from 'src/dto/artifacts.dto'; @@ -531,19 +534,30 @@ export class SubmissionService { submission.challengeId, ); if (challenge.status === ChallengeStatus.COMPLETED) { - const passingSubmission = await this.prisma.submission.findFirst({ - where: { - challengeId: submission.challengeId, - memberId: uid, - reviewSummation: { - some: { - isPassing: true, + if (this.isFirst2FinishChallenge(challenge)) { + const memberSubmission = await this.prisma.submission.findFirst({ + where: { + challengeId: submission.challengeId, + memberId: uid, + }, + select: { id: true }, + }); + canDownload = !!memberSubmission; + } else { + const passingSubmission = await this.prisma.submission.findFirst({ + where: { + challengeId: submission.challengeId, + memberId: uid, + reviewSummation: { + some: { + isPassing: true, + }, }, }, - }, - select: { id: true }, - }); - canDownload = !!passingSubmission; + select: { id: true }, + }); + canDownload = !!passingSubmission; + } } } catch (err) { this.logger.warn( @@ -658,6 +672,31 @@ export class SubmissionService { } } + private isFirst2FinishChallenge(challenge?: ChallengeData | null): boolean { + if (!challenge) { + return false; + } + + const typeName = (challenge.type ?? '').trim().toLowerCase(); + if ( + typeName === 'first2finish' || + typeName === 'first 2 finish' || + typeName === 'topgear task' + ) { + return true; + } + + const legacySubTrack = (challenge.legacy?.subTrack ?? '') + .trim() + .toLowerCase(); + + if (legacySubTrack === 'first_2_finish') { + return true; + } + + return false; + } + /** * Streams a ZIP file containing all submissions for a challenge. * Inside the big zip are the individual submission .zip files from the clean bucket. From f7b9f9ae3954abe7eb4c00c823896ac1626a9316 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 23 Oct 2025 17:29:41 +1100 Subject: [PATCH 27/48] Performance indices --- .../migration.sql | 26 ++ prisma/schema.prisma | 10 + src/api/review/review.service.spec.ts | 101 ++++--- src/api/review/review.service.ts | 99 +++++-- src/api/submission/submission.service.spec.ts | 248 +++++++++++++++++- src/api/submission/submission.service.ts | 214 +++++++++++++++ 6 files changed, 632 insertions(+), 66 deletions(-) create mode 100644 prisma/migrations/20251023062919_performance_indices/migration.sql diff --git a/prisma/migrations/20251023062919_performance_indices/migration.sql b/prisma/migrations/20251023062919_performance_indices/migration.sql new file mode 100644 index 0000000..68b6633 --- /dev/null +++ b/prisma/migrations/20251023062919_performance_indices/migration.sql @@ -0,0 +1,26 @@ +-- CreateIndex +CREATE INDEX "aiWorkflowRun_submissionId_status_idx" ON "aiWorkflowRun"("submissionId", "status"); + +-- CreateIndex +CREATE INDEX "aiWorkflowRun_workflowId_idx" ON "aiWorkflowRun"("workflowId"); + +-- CreateIndex +CREATE INDEX "review_status_phaseId_idx" ON "review"("status", "phaseId"); + +-- CreateIndex +CREATE INDEX "review_resourceId_status_idx" ON "review"("resourceId", "status"); + +-- CreateIndex +CREATE INDEX "reviewOpportunity_status_challengeId_type_idx" ON "reviewOpportunity"("status", "challengeId", "type"); + +-- CreateIndex +CREATE INDEX "reviewSummation_submissionId_isPassing_idx" ON "reviewSummation"("submissionId", "isPassing"); + +-- CreateIndex +CREATE INDEX "submission_challengeId_memberId_status_idx" ON "submission"("challengeId", "memberId", "status"); + +-- CreateIndex +CREATE INDEX "submission_submittedDate_idx" ON "submission"("submittedDate"); + +-- CreateIndex +CREATE INDEX "upload_projectId_resourceId_idx" ON "upload"("projectId", "resourceId"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 1518323..d628d49 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -183,6 +183,8 @@ model review { @@index([phaseId]) // Index for filtering by phase @@index([scorecardId]) // Index for joining with scorecard table @@index([status]) // Index for filtering by review status + @@index([status, phaseId]) + @@index([resourceId, status]) @@unique([resourceId, submissionId, scorecardId]) } @@ -374,6 +376,7 @@ model reviewSummation { @@index([submissionId]) // Index for joining with submission table @@index([scorecardId]) // Index for joining with scorecard table + @@index([submissionId, isPassing]) } model submission { @@ -425,6 +428,8 @@ model submission { @@index([memberId]) @@index([challengeId]) @@index([legacySubmissionId]) + @@index([challengeId, memberId, status]) + @@index([submittedDate]) } enum ReviewOpportunityStatus { @@ -481,6 +486,7 @@ model reviewOpportunity { @@unique([challengeId, type]) @@index([id]) // Index for direct ID lookups @@index([challengeId]) // Index for filtering by challenge + @@index([status, challengeId, type]) } model reviewApplication { @@ -554,6 +560,7 @@ model upload { @@index([projectId]) @@index([legacyId]) + @@index([projectId, resourceId]) } model resourceSubmission { @@ -655,6 +662,9 @@ model aiWorkflowRun { workflow aiWorkflow @relation(fields: [workflowId], references: [id]) submission submission @relation(fields: [submissionId], references: [id]) items aiWorkflowRunItem[] + + @@index([submissionId, status]) + @@index([workflowId]) } model aiWorkflowRunItem { diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index 3deab93..fbcb904 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -1512,7 +1512,7 @@ describe('ReviewService.getReviews reviewer visibility', () => { }); }); - it('includes non-owned submissions with limited visibility when submission phase is closed on an active challenge', async () => { + it('restricts submitters to their own submissions when submission phase is closed on an active challenge', async () => { const submitterResource = { ...buildResource('Submitter'), roleId: CommonConfig.roles.submitterRoleId, @@ -1541,6 +1541,45 @@ describe('ReviewService.getReviews reviewer visibility', () => { await service.getReviews(baseAuthUser, undefined, 'challenge-1'); + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; + const whereClauses = flattenWhereClauses(callArgs.where); + const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); + expect(submissionClause?.submissionId).toEqual({ + in: [baseSubmission.id], + }); + }); + + it('allows limited visibility for marathon matches during appeals', async () => { + const submitterResource = { + ...buildResource('Submitter'), + roleId: CommonConfig.roles.submitterRoleId, + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + submitterResource, + ]); + + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Marathon Challenge', + status: ChallengeStatus.ACTIVE, + type: 'Marathon Match', + legacy: { subTrack: 'MARATHON_MATCH' }, + phases: [ + { id: 'phase-appeals', name: 'Appeals', isOpen: true }, + { id: 'phase-sub', name: 'Submission', isOpen: false }, + ], + }); + + prismaMock.submission.findMany.mockImplementation(({ where }: any) => { + if (where?.memberId) { + return Promise.resolve([baseSubmission]); + } + return Promise.resolve([baseSubmission, { id: 'submission-other' }]); + }); + + await service.getReviews(baseAuthUser, undefined, 'challenge-1'); + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; const whereClauses = flattenWhereClauses(callArgs.where); const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); @@ -1939,7 +1978,7 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(response.data[0].reviewItems).toHaveLength(1); }); - it('returns limited review data for non-owned submissions when appeals are open', async () => { + it('returns only owned reviews for submitters when appeals are open on non-marathon challenges', async () => { const now = new Date(); const submitterResource = { ...buildResource('Submitter'), @@ -2043,8 +2082,28 @@ describe('ReviewService.getReviews reviewer visibility', () => { }, }; - prismaMock.review.findMany.mockResolvedValue([ownReview, otherReview]); - prismaMock.review.count.mockResolvedValue(2); + prismaMock.review.findMany.mockImplementation((args: any) => { + const whereClauses = flattenWhereClauses(args.where); + const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); + const allowedIds: string[] = + (submissionClause?.submissionId as any)?.in ?? []; + return Promise.resolve( + [ownReview, otherReview].filter((review) => + allowedIds.includes(review.submissionId), + ), + ); + }); + prismaMock.review.count.mockImplementation((args: any) => { + const whereClauses = flattenWhereClauses(args.where); + const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); + const allowedIds: string[] = + (submissionClause?.submissionId as any)?.in ?? []; + return Promise.resolve( + [ownReview, otherReview].filter((review) => + allowedIds.includes(review.submissionId), + ).length, + ); + }); const response = await service.getReviews( baseAuthUser, @@ -2052,7 +2111,7 @@ describe('ReviewService.getReviews reviewer visibility', () => { 'challenge-1', ); - expect(response.data).toHaveLength(2); + expect(response.data).toHaveLength(1); const own = response.data.find( (entry) => entry.submissionId === baseSubmission.id, ); @@ -2061,36 +2120,10 @@ describe('ReviewService.getReviews reviewer visibility', () => { ); expect(own).toBeDefined(); - expect(other).toBeDefined(); + expect(other).toBeUndefined(); expect(own?.finalScore).toBe(95.5); expect(own?.initialScore).toBe(93.2); expect(own?.reviewItems).toHaveLength(1); - - const otherResult = other as any; - expect(otherResult && 'finalScore' in otherResult).toBe(false); - expect(otherResult && 'initialScore' in otherResult).toBe(false); - expect(other?.reviewItems).toEqual([]); - expect(other?.reviewDate).toEqual(now); - expect(other?.phaseName).toBe('Appeals'); - expect(other).toMatchObject({ - id: 'review-2', - resourceId: 'resource-reviewer-2', - submissionId: 'submission-other', - scorecardId: 'scorecard-2', - status: ReviewStatus.COMPLETED, - phaseId: 'phase-appeals', - createdAt: now, - createdBy: 'creator-2', - updatedAt: now, - updatedBy: 'updater-2', - }); - expect(other?.reviewerHandle).toBeNull(); - expect(other?.reviewerMaxRating).toBeNull(); - expect(other?.submitterHandle).toBeNull(); - expect(other?.submitterMaxRating).toBeNull(); - expect(other && 'metadata' in other).toBe(false); - expect(other && 'typeId' in other).toBe(false); - expect(other && 'committed' in other).toBe(false); }); it('omits nested review details when thin flag is set', async () => { @@ -2223,7 +2256,7 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(response.data[0].reviewItems).toHaveLength(1); }); - it('allows First2Finish submitters to view their reviews while iterative review is open', async () => { + it('restricts First2Finish submitters to their own reviews while iterative review is open', async () => { const now = new Date(); const submitterUser: JwtUser = { userId: 'submitter-1', @@ -2322,7 +2355,7 @@ describe('ReviewService.getReviews reviewer visibility', () => { const whereClauses = flattenWhereClauses(callArgs.where); const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); expect(submissionClause?.submissionId).toEqual({ - in: ['submission-1', 'submission-2'], + in: ['submission-1'], }); }); diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 662be92..522d7cc 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -2820,7 +2820,6 @@ export class ReviewService { await this.challengeApiService.getChallengeDetail(challengeId); const challenge = challengeDetail; const phases = challenge.phases || []; - const isFirst2Finish = this.isFirst2FinishChallenge(challenge); const appealsOpen = phases.some( (p) => p.name === 'Appeals' && p.isOpen, ); @@ -2832,6 +2831,13 @@ export class ReviewService { (p.name || '').toLowerCase() === 'submission' && p.isOpen === false, ); + const screeningPhaseCompleted = this.hasChallengePhaseCompleted( + challenge, + ['screening'], + ); + const challengeCompletedOrCancelled = + this.isCompletedOrCancelledStatus(challenge.status); + const isMarathonMatch = this.isMarathonMatchChallenge(challenge); const mySubmissionIds = mySubs.map((s) => s.id); mySubmissionIds.forEach((id) => submitterSubmissionIdSet.add(id)); submitterVisibilityState = @@ -2841,32 +2847,40 @@ export class ReviewService { requesterIsChallengeResource = true; } - if ( - [ - ChallengeStatus.COMPLETED, - ChallengeStatus.CANCELLED_FAILED_REVIEW, - ].includes(challenge.status) - ) { + if (challengeCompletedOrCancelled) { // Allowed to see all reviews on this challenge // reviewWhereClause already limited to submissions on this challenge - } else if (isFirst2Finish) { - // First2Finish submitters can view all reviews; details for other submissions - // will be trimmed later in the response mapping. - } else if (appealsOpen || appealsResponseOpen) { - // Allow limited visibility into other submissions; details will be redacted later. - allowLimitedVisibilityForOtherSubmissions = true; - } else if ( - challenge.status === ChallengeStatus.ACTIVE && - submissionPhaseClosed && - hasSubmitterRoleForChallenge - ) { - // Allow limited visibility into other submissions once submission phase closes. - allowLimitedVisibilityForOtherSubmissions = true; + } else if (isMarathonMatch) { + if (appealsOpen || appealsResponseOpen) { + // Marathon matches retain limited visibility for other submissions during appeals phases. + allowLimitedVisibilityForOtherSubmissions = true; + } else if ( + challenge.status === ChallengeStatus.ACTIVE && + submissionPhaseClosed && + hasSubmitterRoleForChallenge + ) { + // Allow limited visibility once marathon match submission phase closes. + allowLimitedVisibilityForOtherSubmissions = true; + } else if ( + hasSubmitterRoleForChallenge && + screeningPhaseCompleted + ) { + // Marathon submitters can still inspect their own reviews once screening completes. + restrictToSubmissionIds(mySubmissionIds); + } else { + this.logger.debug( + `[getReviews] Challenge ${challengeId} is in status ${challenge.status}. Returning empty review list for requester ${uid}.`, + ); + restrictToSubmissionIds([]); + } } else if ( hasSubmitterRoleForChallenge && - this.hasChallengePhaseCompleted(challenge, ['screening']) + (appealsOpen || + appealsResponseOpen || + submissionPhaseClosed || + screeningPhaseCompleted) ) { - // Allow submitters to access their own screening reviews once screening phase completes. + // Non-marathon submitters can only inspect their own submissions until completion/cancellation. restrictToSubmissionIds(mySubmissionIds); } else { // Reviews exist but the phase does not allow visibility yet; respond with no results. @@ -4017,10 +4031,7 @@ export class ReviewService { return name === 'iterative review' && phase?.isOpen === false; }); - const allowAny = [ - ChallengeStatus.COMPLETED, - ChallengeStatus.CANCELLED_FAILED_REVIEW, - ].includes(status); + const allowAny = this.isCompletedOrCancelledStatus(status); const allowOwn = allowAny || appealsOpen || @@ -4056,6 +4067,42 @@ export class ReviewService { return false; } + private isMarathonMatchChallenge(challenge?: ChallengeData | null): boolean { + if (!challenge) { + return false; + } + + const typeName = (challenge.type ?? '').trim().toLowerCase(); + if (typeName === 'marathon match') { + return true; + } + + const legacySubTrack = (challenge.legacy?.subTrack ?? '') + .trim() + .toLowerCase(); + if (legacySubTrack.includes('marathon')) { + return true; + } + + const legacyTrack = (challenge.legacy?.track ?? '').trim().toLowerCase(); + return legacyTrack.includes('marathon'); + } + + private isCompletedOrCancelledStatus( + status?: ChallengeStatus | null, + ): boolean { + if (!status) { + return false; + } + if (status === ChallengeStatus.COMPLETED) { + return true; + } + if (status === ChallengeStatus.CANCELLED) { + return true; + } + return String(status).startsWith('CANCELLED_'); + } + private shouldEnforceLatestSubmissionForReview( reviewTypeName: string | null | undefined, challenge: ChallengeData | null | undefined, diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index 1cd25ca..8592c15 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -4,6 +4,7 @@ import { Readable } from 'stream'; import { SubmissionService } from './submission.service'; import { UserRole } from 'src/shared/enums/userRole.enum'; import { ChallengeStatus } from 'src/shared/enums/challengeStatus.enum'; +import { CommonConfig } from 'src/shared/config/common.config'; jest.mock('nanoid', () => ({ __esModule: true, @@ -497,6 +498,15 @@ describe('SubmissionService', () => { let challengePrismaMock: { $queryRaw: jest.Mock; }; + let challengeApiServiceMock: { + getChallengeDetail: jest.Mock; + getChallenges: jest.Mock; + }; + let resourceApiServiceListMock: { + validateSubmitterRegistration: jest.Mock; + getMemberResourcesRoles: jest.Mock; + }; + let memberPrismaMock: { member: { findMany: jest.Mock } }; let listService: SubmissionService; beforeEach(() => { @@ -513,18 +523,30 @@ describe('SubmissionService', () => { challengePrismaMock = { $queryRaw: jest.fn().mockResolvedValue([]), }; + challengeApiServiceMock = { + getChallengeDetail: jest.fn().mockResolvedValue({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + type: 'Challenge', + legacy: {}, + phases: [], + }), + getChallenges: jest.fn(), + }; + resourceApiServiceListMock = { + validateSubmitterRegistration: jest.fn(), + getMemberResourcesRoles: jest.fn().mockResolvedValue([]), + }; + memberPrismaMock = { member: { findMany: jest.fn() } }; listService = new SubmissionService( prismaMock as any, prismaErrorServiceMock as any, challengePrismaMock as any, - {} as any, - { - validateSubmitterRegistration: jest.fn(), - getMemberResourcesRoles: jest.fn(), - } as any, + challengeApiServiceMock as any, + resourceApiServiceListMock as any, {} as any, {} as any, - { member: { findMany: jest.fn() } } as any, + memberPrismaMock as any, ); }); @@ -648,5 +670,219 @@ describe('SubmissionService', () => { expect(result.data[0]).not.toHaveProperty('isLatest'); expect(result.data[1]).not.toHaveProperty('isLatest'); }); + + it('omits review data for non-owned submissions before completion', async () => { + const submissions = [ + { + id: 'submission-own', + challengeId: 'challenge-1', + memberId: 'user-1', + submittedDate: new Date('2025-01-02T12:00:00Z'), + createdAt: new Date('2025-01-02T12:00:00Z'), + updatedAt: new Date('2025-01-02T12:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [{ id: 'review-own' }], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + { + id: 'submission-other', + challengeId: 'challenge-1', + memberId: 'user-2', + submittedDate: new Date('2025-01-01T12:00:00Z'), + createdAt: new Date('2025-01-01T12:00:00Z'), + updatedAt: new Date('2025-01-01T12:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [{ id: 'review-other' }], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + ]; + + resourceApiServiceListMock.getMemberResourcesRoles.mockResolvedValue([ + { + roleName: 'Submitter', + roleId: CommonConfig.roles.submitterRoleId, + }, + ]); + + prismaMock.submission.findMany.mockResolvedValue( + submissions.map((entry) => ({ ...entry })), + ); + prismaMock.submission.count.mockResolvedValue(submissions.length); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'submission-own', + }); + + const result = await listService.listSubmission( + { + userId: 'user-1', + isMachine: false, + roles: [UserRole.User], + } as any, + { challengeId: 'challenge-1' } as any, + { page: 1, perPage: 50 } as any, + ); + + const own = result.data.find((entry) => entry.id === 'submission-own'); + const other = result.data.find( + (entry) => entry.id === 'submission-other', + ); + + expect(own?.review).toBeDefined(); + expect(other).not.toHaveProperty('review'); + expect( + resourceApiServiceListMock.getMemberResourcesRoles, + ).toHaveBeenCalledWith('challenge-1', 'user-1'); + expect(challengeApiServiceMock.getChallengeDetail).toHaveBeenCalledWith( + 'challenge-1', + ); + }); + + it('retains review data for other submissions once the challenge completes', async () => { + challengeApiServiceMock.getChallengeDetail.mockResolvedValueOnce({ + id: 'challenge-1', + status: ChallengeStatus.COMPLETED, + type: 'Challenge', + legacy: {}, + phases: [], + }); + resourceApiServiceListMock.getMemberResourcesRoles.mockResolvedValue([ + { + roleName: 'Submitter', + roleId: CommonConfig.roles.submitterRoleId, + }, + ]); + + const submissions = [ + { + id: 'submission-own', + challengeId: 'challenge-1', + memberId: 'user-1', + submittedDate: new Date('2025-01-02T12:00:00Z'), + createdAt: new Date('2025-01-02T12:00:00Z'), + updatedAt: new Date('2025-01-02T12:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.COMPLETED_WITHOUT_WIN, + review: [{ id: 'review-own' }], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + { + id: 'submission-other', + challengeId: 'challenge-1', + memberId: 'user-2', + submittedDate: new Date('2025-01-01T12:00:00Z'), + createdAt: new Date('2025-01-01T12:00:00Z'), + updatedAt: new Date('2025-01-01T12:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.COMPLETED_WITHOUT_WIN, + review: [{ id: 'review-other' }], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + ]; + + prismaMock.submission.findMany.mockResolvedValue( + submissions.map((entry) => ({ ...entry })), + ); + prismaMock.submission.count.mockResolvedValue(submissions.length); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'submission-own', + }); + + const result = await listService.listSubmission( + { + userId: 'user-1', + isMachine: false, + roles: [UserRole.User], + } as any, + { challengeId: 'challenge-1' } as any, + { page: 1, perPage: 50 } as any, + ); + + const other = result.data.find( + (entry) => entry.id === 'submission-other', + ); + + expect(other?.review).toBeDefined(); + }); + + it('retains review data for marathon match submissions', async () => { + challengeApiServiceMock.getChallengeDetail.mockResolvedValueOnce({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + type: 'Marathon Match', + legacy: { subTrack: 'MARATHON_MATCH' }, + phases: [], + }); + resourceApiServiceListMock.getMemberResourcesRoles.mockResolvedValue([ + { + roleName: 'Submitter', + roleId: CommonConfig.roles.submitterRoleId, + }, + ]); + + const submissions = [ + { + id: 'submission-own', + challengeId: 'challenge-1', + memberId: 'user-1', + submittedDate: new Date('2025-01-02T12:00:00Z'), + createdAt: new Date('2025-01-02T12:00:00Z'), + updatedAt: new Date('2025-01-02T12:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [{ id: 'review-own' }], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + { + id: 'submission-other', + challengeId: 'challenge-1', + memberId: 'user-2', + submittedDate: new Date('2025-01-01T12:00:00Z'), + createdAt: new Date('2025-01-01T12:00:00Z'), + updatedAt: new Date('2025-01-01T12:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [{ id: 'review-other' }], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + ]; + + prismaMock.submission.findMany.mockResolvedValue( + submissions.map((entry) => ({ ...entry })), + ); + prismaMock.submission.count.mockResolvedValue(submissions.length); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'submission-own', + }); + + const result = await listService.listSubmission( + { + userId: 'user-1', + isMachine: false, + roles: [UserRole.User], + } as any, + { challengeId: 'challenge-1' } as any, + { page: 1, perPage: 50 } as any, + ); + + const other = result.data.find( + (entry) => entry.id === 'submission-other', + ); + + expect(other?.review).toBeDefined(); + }); }); }); diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index d25a471..d89a7a7 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -19,6 +19,7 @@ import { import { JwtUser, isAdmin } from 'src/shared/modules/global/jwt.service'; import { UserRole } from 'src/shared/enums/userRole.enum'; import { ChallengeStatus } from 'src/shared/enums/challengeStatus.enum'; +import { CommonConfig } from 'src/shared/config/common.config'; import { PrismaService } from 'src/shared/modules/global/prisma.service'; import { ChallengePrismaService } from 'src/shared/modules/global/challenge-prisma.service'; import { MemberPrismaService } from 'src/shared/modules/global/member-prisma.service'; @@ -33,6 +34,7 @@ import { ResourceApiService } from 'src/shared/modules/global/resource.service'; import { ArtifactsCreateResponseDto } from 'src/dto/artifacts.dto'; import { randomUUID } from 'crypto'; import { basename } from 'path'; +import { ResourceInfo } from 'src/shared/models/ResourceInfo.model'; import { S3Client, ListObjectsV2Command, @@ -52,6 +54,13 @@ type SubmissionMinimal = { url: string | null; }; +const REVIEW_ACCESS_ROLE_KEYWORDS = [ + 'reviewer', + 'screener', + 'approver', + 'approval', +]; + @Injectable() export class SubmissionService { private readonly logger = new Logger(SubmissionService.name); @@ -1626,6 +1635,8 @@ export class SubmissionService { } } + await this.applyReviewVisibilityFilters(authUser, submissions); + // Count total entities matching the filter for pagination metadata const totalCount = await this.prisma.submission.count({ where: { @@ -1988,6 +1999,171 @@ export class SubmissionService { return data; } + private async applyReviewVisibilityFilters( + authUser: JwtUser, + submissions: Array<{ + challengeId?: string | null; + memberId?: string | null; + review?: unknown; + }>, + ): Promise { + if (!submissions.length) { + return; + } + + const isPrivilegedRequester = authUser?.isMachine || isAdmin(authUser); + if (isPrivilegedRequester) { + return; + } + + const uid = + authUser?.userId !== undefined && authUser?.userId !== null + ? String(authUser.userId).trim() + : ''; + + if (!uid) { + return; + } + + const challengeIds = Array.from( + new Set( + submissions + .map((submission) => { + if (submission.challengeId == null) { + return null; + } + const id = String(submission.challengeId).trim(); + return id.length ? id : null; + }) + .filter((value): value is string => !!value), + ), + ); + + if (!challengeIds.length) { + return; + } + + const challengeDetails = new Map(); + + await Promise.all( + challengeIds.map(async (challengeId) => { + try { + const detail = + await this.challengeApiService.getChallengeDetail(challengeId); + challengeDetails.set(challengeId, detail); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.debug( + `[applyReviewVisibilityFilters] Failed to load challenge ${challengeId}: ${message}`, + ); + challengeDetails.set(challengeId, null); + } + }), + ); + + const roleSummaryByChallenge = new Map< + string, + { hasCopilot: boolean; hasReviewer: boolean; hasSubmitter: boolean } + >(); + + await Promise.all( + challengeIds.map(async (challengeId) => { + let resources: ResourceInfo[] = []; + try { + resources = await this.resourceApiService.getMemberResourcesRoles( + challengeId, + uid, + ); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.debug( + `[applyReviewVisibilityFilters] Failed to load resource roles for challenge ${challengeId}, member ${uid}: ${message}`, + ); + } + + let hasCopilot = false; + let hasReviewer = false; + let hasSubmitter = false; + + for (const resource of resources ?? []) { + const roleName = (resource.roleName || '').toLowerCase(); + if (roleName.includes('copilot')) { + hasCopilot = true; + } + if ( + REVIEW_ACCESS_ROLE_KEYWORDS.some((keyword) => + roleName.includes(keyword), + ) + ) { + hasReviewer = true; + } + if ( + resource.roleId === CommonConfig.roles.submitterRoleId || + roleName.includes('submitter') + ) { + hasSubmitter = true; + } + } + + roleSummaryByChallenge.set(challengeId, { + hasCopilot, + hasReviewer, + hasSubmitter, + }); + }), + ); + + for (const submission of submissions) { + if (!Object.prototype.hasOwnProperty.call(submission, 'review')) { + continue; + } + + const challengeId = + submission.challengeId != null + ? String(submission.challengeId).trim() + : ''; + if (!challengeId) { + continue; + } + + const isOwnSubmission = + submission.memberId != null && + String(submission.memberId).trim() === uid; + if (isOwnSubmission) { + continue; + } + + const roleSummary = roleSummaryByChallenge.get(challengeId) ?? { + hasCopilot: false, + hasReviewer: false, + hasSubmitter: false, + }; + + if (roleSummary.hasCopilot || roleSummary.hasReviewer) { + continue; + } + + const challenge = challengeDetails.get(challengeId); + + if (this.isCompletedOrCancelledStatus(challenge?.status ?? null)) { + continue; + } + + if (!roleSummary.hasSubmitter) { + delete (submission as any).review; + continue; + } + + if (this.isMarathonMatchChallenge(challenge ?? null)) { + continue; + } + + delete (submission as any).review; + } + } + private async populateLatestSubmissionFlags( submissions: Array<{ id: string; @@ -2193,6 +2369,44 @@ export class SubmissionService { return null; } + private isMarathonMatchChallenge( + challenge: ChallengeData | null | undefined, + ): boolean { + if (!challenge) { + return false; + } + + const typeName = (challenge.type ?? '').trim().toLowerCase(); + if (typeName === 'marathon match') { + return true; + } + + const legacySubTrack = (challenge.legacy?.subTrack ?? '') + .trim() + .toLowerCase(); + if (legacySubTrack.includes('marathon')) { + return true; + } + + const legacyTrack = (challenge.legacy?.track ?? '').trim().toLowerCase(); + return legacyTrack.includes('marathon'); + } + + private isCompletedOrCancelledStatus( + status: ChallengeStatus | null | undefined, + ): boolean { + if (!status) { + return false; + } + if (status === ChallengeStatus.COMPLETED) { + return true; + } + if (status === ChallengeStatus.CANCELLED) { + return true; + } + return String(status).startsWith('CANCELLED_'); + } + private buildResponse(data: any): SubmissionResponseDto { const dto: SubmissionResponseDto = { ...data, From 54fff6407e94af2b724c100f5b6a592b89888c71 Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Thu, 23 Oct 2025 10:17:27 +0300 Subject: [PATCH 28/48] PM-1904 - fix typo and optimize hasAiReview query --- src/api/my-review/myReview.service.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/api/my-review/myReview.service.ts b/src/api/my-review/myReview.service.ts index 2d8ae5c..b2c0718 100644 --- a/src/api/my-review/myReview.service.ts +++ b/src/api/my-review/myReview.service.ts @@ -19,7 +19,7 @@ interface ChallengeSummaryRow { challengeName: string; challengeTypeId: string | null; challengeTypeName: string | null; - hasAsAIReview: boolean; + hasAIReview: boolean; currentPhaseName: string | null; currentPhaseScheduledEnd: Date | null; currentPhaseActualEnd: Date | null; @@ -301,11 +301,12 @@ export class MyReviewService { Prisma.sql` LEFT JOIN LATERAL ( SELECT - CASE WHEN COUNT(*)::bigint > 0 THEN TRUE else FALSE END AS "hasAsAIReview" - FROM challenges."ChallengeReviewer" cr - WHERE cr."challengeId" = c.id - AND cr."isMemberReview" = false - LIMIT 1 + EXISTS ( + SELECT 1 + FROM challenges."ChallengeReviewer" cr + WHERE cr."challengeId" = c.id + AND cr."aiWorkflowId" is not NULL + ) AS "hasAIReview" ) cr ON TRUE `, ); @@ -445,7 +446,7 @@ export class MyReviewService { c."typeId" AS "challengeTypeId", ct.name AS "challengeTypeName", cp.name AS "currentPhaseName", - cr."hasAsAIReview" as "hasAsAIReview", + cr."hasAIReview" as "hasAIReview", cp."scheduledEndDate" AS "currentPhaseScheduledEnd", cp."actualEndDate" AS "currentPhaseActualEnd", rr.name AS "resourceRoleName", @@ -551,7 +552,7 @@ export class MyReviewService { challengeName: row.challengeName, challengeTypeId: row.challengeTypeId, challengeTypeName: row.challengeTypeName, - hasAsAIReview: row.hasAsAIReview, + hasAIReview: row.hasAIReview, challengeEndDate: row.challengeEndDate ? row.challengeEndDate.toISOString() : null, From bfc2fdbe6b726592b04c1ee33309245df986a172 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:40:19 +0000 Subject: [PATCH 29/48] Bump axios from 1.9.0 to 1.12.0 Bumps [axios](https://github.com/axios/axios) from 1.9.0 to 1.12.0. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v1.9.0...v1.12.0) --- updated-dependencies: - dependency-name: axios dependency-version: 1.12.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- package.json | 2 +- pnpm-lock.yaml | 45 ++++++++++++++++++++++++--------------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 1e5cc2d..0900d4a 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@prisma/client": "^6.3.1", "@types/jsonwebtoken": "^9.0.9", "archiver": "^6.0.2", - "axios": "^1.9.0", + "axios": "^1.12.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cors": "^2.8.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9cba1a7..8ccc42b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,7 +16,7 @@ importers: version: 3.888.0(@aws-sdk/client-s3@3.888.0) '@nestjs/axios': specifier: ^4.0.0 - version: 4.0.0(@nestjs/common@11.0.10(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(axios@1.9.0)(rxjs@7.8.2) + version: 4.0.0(@nestjs/common@11.0.10(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(axios@1.12.0)(rxjs@7.8.2) '@nestjs/common': specifier: ^11.0.1 version: 11.0.10(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -45,8 +45,8 @@ importers: specifier: ^6.0.2 version: 6.0.2 axios: - specifier: ^1.9.0 - version: 1.9.0 + specifier: ^1.12.0 + version: 1.12.0 class-transformer: specifier: ^0.5.1 version: 0.5.1 @@ -2198,8 +2198,8 @@ packages: axios@0.22.0: resolution: {integrity: sha512-Z0U3uhqQeg1oNcihswf4ZD57O3NrR1+ZXhxaROaWpDmsDTx7T2HNBV2ulBtie2hwJptu8UvgnJoK+BIqdzh/1w==} - axios@1.9.0: - resolution: {integrity: sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==} + axios@1.12.0: + resolution: {integrity: sha512-oXTDccv8PcfjZmPGlWsPSwtOJCZ/b6W5jAMCNcfwJbCzDckwG0jrYJFaWH1yvivfCXjVzV/SPDEhMB3Q+DSurg==} b4a@1.6.7: resolution: {integrity: sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==} @@ -2995,8 +2995,8 @@ packages: fn.name@1.1.0: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} - follow-redirects@1.15.9: - resolution: {integrity: sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==} + follow-redirects@1.15.11: + resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} engines: {node: '>=4.0'} peerDependencies: debug: '*' @@ -3019,8 +3019,8 @@ packages: resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} engines: {node: '>= 14.17'} - form-data@4.0.2: - resolution: {integrity: sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==} + form-data@4.0.4: + resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==} engines: {node: '>= 6'} formidable@3.5.2: @@ -4536,10 +4536,12 @@ packages: superagent@9.0.2: resolution: {integrity: sha512-xuW7dzkUpcJq7QnhOsnNUgtYp3xRwpt2F7abdRYIpCsAt0hhUqia0EdxyXZQQpNmGtsCzYHryaKSV3q3GJnq7w==} engines: {node: '>=14.18.0'} + deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net supertest@7.0.0: resolution: {integrity: sha512-qlsr7fIC0lSddmA3tzojvzubYxvlGtzumcdHgPwbFWMISQwL22MhM2Y3LNt+6w9Yyx7559VW5ab70dgphm8qQA==} engines: {node: '>=14.18.0'} + deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} @@ -6329,10 +6331,10 @@ snapshots: '@tybys/wasm-util': 0.10.1 optional: true - '@nestjs/axios@4.0.0(@nestjs/common@11.0.10(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(axios@1.9.0)(rxjs@7.8.2)': + '@nestjs/axios@4.0.0(@nestjs/common@11.0.10(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(axios@1.12.0)(rxjs@7.8.2)': dependencies: '@nestjs/common': 11.0.10(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2) - axios: 1.9.0 + axios: 1.12.0 rxjs: 7.8.2 '@nestjs/cli@11.0.4(@swc/cli@0.6.0(@swc/core@1.11.1)(chokidar@4.0.3))(@swc/core@1.11.1)(@types/node@22.13.5)(esbuild@0.25.0)': @@ -7093,7 +7095,7 @@ snapshots: '@types/cookiejar': 2.1.5 '@types/methods': 1.1.4 '@types/node': 22.13.5 - form-data: 4.0.2 + form-data: 4.0.4 '@types/supertest@6.0.2': dependencies: @@ -7457,20 +7459,20 @@ snapshots: axios@0.21.4(debug@4.4.0): dependencies: - follow-redirects: 1.15.9(debug@4.4.0) + follow-redirects: 1.15.11(debug@4.4.0) transitivePeerDependencies: - debug axios@0.22.0: dependencies: - follow-redirects: 1.15.9(debug@4.4.0) + follow-redirects: 1.15.11(debug@4.4.0) transitivePeerDependencies: - debug - axios@1.9.0: + axios@1.12.0: dependencies: - follow-redirects: 1.15.9(debug@4.4.0) - form-data: 4.0.2 + follow-redirects: 1.15.11(debug@4.4.0) + form-data: 4.0.4 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug @@ -8274,7 +8276,7 @@ snapshots: ext-list@2.2.2: dependencies: - mime-db: 1.53.0 + mime-db: 1.54.0 ext-name@5.0.0: dependencies: @@ -8398,7 +8400,7 @@ snapshots: fn.name@1.1.0: {} - follow-redirects@1.15.9(debug@4.4.0): + follow-redirects@1.15.11(debug@4.4.0): optionalDependencies: debug: 4.4.0 @@ -8426,11 +8428,12 @@ snapshots: form-data-encoder@2.1.4: {} - form-data@4.0.2: + form-data@4.0.4: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 es-set-tostringtag: 2.1.0 + hasown: 2.0.2 mime-types: 2.1.35 formidable@3.5.2: @@ -10178,7 +10181,7 @@ snapshots: cookiejar: 2.1.4 debug: 4.4.0 fast-safe-stringify: 2.1.1 - form-data: 4.0.2 + form-data: 4.0.4 formidable: 3.5.2 methods: 1.1.2 mime: 2.6.0 From 06ce94fc7cc15e8928755647b3db42cbd030557a Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 24 Oct 2025 08:31:12 +1100 Subject: [PATCH 30/48] Prod migration updates and incremental migration --- package.json | 5 +- prisma/migrate.ts | 1900 +++++++++++++++++++++++++++++---------------- 2 files changed, 1252 insertions(+), 653 deletions(-) diff --git a/package.json b/package.json index 1e5cc2d..eda5a0b 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", + "db:migrate": "ts-node prisma/migrate.ts", "postinstall": "pnpm exec prisma generate && pnpm exec prisma generate --schema=prisma/challenge-schema.prisma && pnpm exec prisma generate --schema=prisma/resource-schema.prisma && pnpm exec prisma generate --schema=prisma/member-schema.prisma" }, "dependencies": { @@ -81,7 +82,7 @@ "winston": "^3.17.0" }, "prisma": { - "seed": "ts-node prisma/migrate.ts", + "seed": "pnpm run db:migrate", "seed222": "ts-node prisma/seed.ts" }, "jest": { @@ -104,4 +105,4 @@ "coverageDirectory": "../coverage", "testEnvironment": "node" } -} \ No newline at end of file +} diff --git a/prisma/migrate.ts b/prisma/migrate.ts index 5cf270e..bedfd9b 100644 --- a/prisma/migrate.ts +++ b/prisma/migrate.ts @@ -29,10 +29,62 @@ const schema = process.env.POSTGRES_SCHEMA || 'public'; console.log(`Using PostgreSQL schema: ${schema}`); const prisma = new PrismaClient(); -const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'Scorecards'); +const DEFAULT_DATA_DIR = '/mnt/export/review_tables'; +const DATA_DIR = process.env.DATA_DIR || DEFAULT_DATA_DIR; const batchSize = 1000; const logSize = 20000; -const esFileName = 'dev-submissions-api.data.json'; +const DEFAULT_ES_DATA_FILE = path.join( + '/home/ubuntu', + 'submissions-api.data.json', +); +const ES_DATA_FILE = process.env.ES_DATA_FILE || DEFAULT_ES_DATA_FILE; + +const incrementalSinceInput = + process.env.MIGRATE_SINCE || process.env.INCREMENTAL_SINCE; +let incrementalSince: Date | null = null; +if (incrementalSinceInput) { + const parsed = new Date(incrementalSinceInput); + if (Number.isNaN(parsed.getTime())) { + throw new Error( + `Invalid MIGRATE_SINCE/INCREMENTAL_SINCE value "${incrementalSinceInput}". Use an ISO-8601 date.`, + ); + } + incrementalSince = parsed; + console.log( + `Running incremental migration for records updated after ${incrementalSince.toISOString()}`, + ); +} +const isIncrementalRun = incrementalSince !== null; + +const parseDateInput = (value: string | Date | null | undefined) => { + if (!value) { + return null; + } + if (value instanceof Date) { + return Number.isNaN(value.getTime()) ? null : value; + } + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +}; + +const shouldProcessRecord = ( + created?: string | Date | null, + updated?: string | Date | null, +) => { + if (!incrementalSince) { + return true; + } + const createdAt = parseDateInput(created); + const updatedAt = parseDateInput(updated); + if (!createdAt && !updatedAt) { + // If there is no audit information we default to processing. + return true; + } + return ( + (createdAt && createdAt >= incrementalSince) || + (updatedAt && updatedAt >= incrementalSince) + ); +}; const modelMappingKeys = [ 'project_result', @@ -108,6 +160,7 @@ const submissionIdMap = readIdMap('submissionIdMap'); const llmProviderIdMap = readIdMap('llmProviderIdMap'); const llmModelIdMap = readIdMap('llmModelIdMap'); const aiWorkflowIdMap = readIdMap('aiWorkflowIdMap'); +const resourceSubmissionIdMap = readIdMap('resourceSubmissionIdMap'); // read resourceSubmissionSet const rsSetFile = '.tmp/resourceSubmissionSet.json'; @@ -368,7 +421,12 @@ function convertSubmissionES(esData): any { async function migrateElasticSearch() { // migrate elastic search data - const filepath = path.join(DATA_DIR, esFileName); + const filepath = ES_DATA_FILE; + if (!fs.existsSync(filepath)) { + throw new Error( + `ElasticSearch export file not found at ${filepath}. Set ES_DATA_FILE to override the default.`, + ); + } const fileStream = fs.createReadStream(filepath); const rl = readline.createInterface({ input: fileStream, @@ -403,6 +461,11 @@ async function handleElasticSearchSubmission(item) { if (item['legacySubmissionId'] == null) { return; } + const createdAudit = item.created ?? item.submittedDate ?? null; + const updatedAudit = item.updated ?? null; + if (!shouldProcessRecord(createdAudit, updatedAudit)) { + return; + } currentSubmissions.push(item); // if we can batch insert data, + if (currentSubmissions.length >= batchSize) { @@ -446,6 +509,7 @@ async function importSubmissionES() { createdAt: item.created ? new Date(item.created) : new Date(), }, }); + submissionIdMap.set(submission.legacySubmissionId, newId); } } catch { if (newSubmission) { @@ -456,9 +520,9 @@ async function importSubmissionES() { } } -function convertUpload(jsonData) { +function convertUpload(jsonData, existingId?: string) { return { - id: nanoid(14), + id: existingId ?? nanoid(14), legacyId: jsonData['upload_id'], projectId: jsonData['project_id'], resourceId: jsonData['resource_id'], @@ -505,9 +569,24 @@ async function doImportUploadData() { } } -function convertSubmission(jsonData) { +async function upsertUploadData(uploadData) { + const { id, ...updateData } = uploadData; + try { + await prisma.upload.upsert({ + where: { id }, + create: uploadData, + update: updateData, + }); + } catch (err) { + console.error(`Failed to upsert upload data id: ${uploadData.legacyId}`); + console.error(err); + throw err; + } +} + +function convertSubmission(jsonData, existingId?: string) { return { - id: nanoid(14), + id: existingId ?? nanoid(14), legacySubmissionId: jsonData['submission_id'], legacyUploadId: jsonData['upload_id'], uploadId: uploadIdMap.get(jsonData['upload_id']), @@ -560,6 +639,23 @@ async function doImportSubmissionData() { } } +async function upsertSubmissionData(submissionData) { + const { id, ...updateData } = submissionData; + try { + await prisma.submission.upsert({ + where: { id }, + create: submissionData, + update: updateData, + }); + } catch (err) { + console.error( + `Failed to upsert submission ${submissionData.legacySubmissionId}`, + ); + console.error(err); + throw err; + } +} + /** * Read submission data from resource_xxx.json, upload_xxx.json and submission_xxx.json. */ @@ -595,11 +691,26 @@ async function initSubmissionMap() { let dataCount = 0; for (const d of jsonData) { dataCount += 1; - const uploadData = convertUpload(d); - // import upload data if any - if (!uploadIdMap.has(uploadData.legacyId)) { - uploadIdMap.set(uploadData.legacyId, uploadData.id); - await importUploadData(uploadData); + const shouldPersist = shouldProcessRecord( + d['create_date'], + d['modify_date'], + ); + const legacyId = String(d['upload_id']); + const existingId = uploadIdMap.get(legacyId); + const uploadData = convertUpload(d, existingId); + const skipPersistence = !existingId && isIncrementalRun && !shouldPersist; + if (!skipPersistence) { + // import upload data if any + if (!existingId) { + uploadIdMap.set(uploadData.legacyId, uploadData.id); + if (isIncrementalRun) { + await upsertUploadData(uploadData); + } else { + await importUploadData(uploadData); + } + } else if (isIncrementalRun && shouldPersist) { + await upsertUploadData(uploadData); + } } // collect data to resourceUploadMap if ( @@ -629,10 +740,25 @@ async function initSubmissionMap() { let dataCount = 0; for (const d of jsonData) { dataCount += 1; - const dbData = convertSubmission(d); - if (!submissionIdMap.has(dbData.legacySubmissionId)) { - submissionIdMap.set(dbData.legacySubmissionId, dbData.id); - await importSubmissionData(dbData); + const shouldPersist = shouldProcessRecord( + d['create_date'], + d['modify_date'], + ); + const legacyId = String(d['submission_id']); + const existingId = submissionIdMap.get(legacyId); + const dbData = convertSubmission(d, existingId); + const skipPersistence = !existingId && isIncrementalRun && !shouldPersist; + if (!skipPersistence) { + if (!existingId) { + submissionIdMap.set(dbData.legacySubmissionId, dbData.id); + if (isIncrementalRun) { + await upsertSubmissionData(dbData); + } else { + await importSubmissionData(dbData); + } + } else if (isIncrementalRun && shouldPersist) { + await upsertSubmissionData(dbData); + } } // collect data to uploadSubmissionMap if ( @@ -748,85 +874,117 @@ async function processType(type: string, subtype?: string) { switch (type) { case 'project_result': { console.log(`[${type}][${file}] Processing file`); - const processedData = jsonData[key] - .filter((pr) => !projectIdMap.has(pr.project_id + pr.user_id)) - .map((pr) => { - projectIdMap.set( - pr.project_id + pr.user_id, - pr.project_id + pr.user_id, + const convertProjectResult = (pr) => { + let submissionId = ''; + if (submissionMap[pr.project_id]) { + submissionId = submissionMap[pr.project_id][pr.user_id] || ''; + } + return { + challengeId: pr.project_id, + userId: pr.user_id, + paymentId: pr.payment_id, + submissionId, + oldRating: parseInt(pr.old_rating), + newRating: parseInt(pr.new_rating), + initialScore: parseFloat(pr.raw_score || '0.0'), + finalScore: parseFloat(pr.final_score || '0.0'), + placement: parseInt(pr.placed || '0'), + rated: pr.rating_ind === '1', + passedReview: pr.passed_review_ind === '1', + validSubmission: pr.valid_submission_ind === '1', + pointAdjustment: parseFloat(pr.point_adjustment), + ratingOrder: parseInt(pr.rating_order), + createdAt: new Date(pr.create_date), + createdBy: pr.create_user || '', + updatedAt: new Date(pr.modify_date), + updatedBy: pr.modify_user || '', + }; + }; + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter((pr) => !projectIdMap.has(pr.project_id + pr.user_id)) + .map((pr) => { + projectIdMap.set( + pr.project_id + pr.user_id, + pr.project_id + pr.user_id, + ); + return convertProjectResult(pr); + }); + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, ); - let submissionId = ''; - if (submissionMap[pr.project_id]) { - submissionId = submissionMap[pr.project_id][pr.user_id] || ''; + const batch = processedData.slice(i, i + batchSize); + await prisma.challengeResult + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.challengeResult + .create({ + data: item, + }) + .catch((err) => { + projectIdMap.delete( + `${item.challengeId}${item.userId}`, + ); + console.error( + `[${type}][${file}] Error code: ${err.code}, ChallengeId: ${item.challengeId}, UserId: ${item.userId}`, + ); + }); + } + }); + } + } else { + for (const pr of jsonData[key]) { + if (!shouldProcessRecord(pr.create_date, pr.modify_date)) { + continue; } - return { - challengeId: pr.project_id, - userId: pr.user_id, - paymentId: pr.payment_id, - submissionId, - oldRating: parseInt(pr.old_rating), - newRating: parseInt(pr.new_rating), - initialScore: parseFloat(pr.raw_score || '0.0'), - finalScore: parseFloat(pr.final_score || '0.0'), - placement: parseInt(pr.placed || '0'), - rated: pr.rating_ind === '1', - passedReview: pr.passed_review_ind === '1', - validSubmission: pr.valid_submission_ind === '1', - pointAdjustment: parseFloat(pr.point_adjustment), - ratingOrder: parseInt(pr.rating_order), - createdAt: new Date(pr.create_date), - createdBy: pr.create_user || '', - updatedAt: new Date(pr.modify_date), - updatedBy: pr.modify_user || '', - }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.challengeResult - .createMany({ - data: batch, - }) - .catch(async () => { + const mapKey = pr.project_id + pr.user_id; + const data = convertProjectResult(pr); + try { + await prisma.challengeResult.upsert({ + where: { + challengeId_userId: { + challengeId: data.challengeId, + userId: data.userId, + }, + }, + create: data, + update: data, + }); + projectIdMap.set(mapKey, mapKey); + } catch (err) { console.error( - `[${type}][${file}] An error occurred, retrying individually`, + `[${type}][${file}] Failed to upsert challengeResult for ChallengeId: ${data.challengeId}, UserId: ${data.userId}`, ); - for (const item of batch) { - await prisma.challengeResult - .create({ - data: item, - }) - .catch((err) => { - projectIdMap.delete(item.project_id + item.user_id); - console.error( - `[${type}][${file}] Error code: ${err.code}, ChallengeId: ${item.challengeId}, UserId: ${item.userId}`, - ); - }); - } - }); + console.error(err); + } + } } break; } case 'scorecard': { console.log(`[${type}][${file}] Processing file`); - const processedData = jsonData[key].map((sc) => { - const id = nanoid(14); - scorecardIdMap.set(sc.scorecard_id, id); + const convertScorecard = (sc, recordId: string) => { const minScore = parseFloat(sc.min_score); const passingScoreSource = sc.minimum_passing_score ?? sc.passing_score ?? sc.min_score; const parsedPassingScore = parseFloat(passingScoreSource); + const category = projectCategoryMap[sc.project_category_id]; return { - id: id, + id: recordId, legacyId: sc.scorecard_id, status: scorecardStatusMap[sc.scorecard_status_id], type: scorecardTypeMap[sc.scorecard_type_id], - challengeTrack: projectCategoryMap[sc.project_category_id].type, - challengeType: projectCategoryMap[sc.project_category_id].name, + challengeTrack: category.type, + challengeType: category.name, name: sc.name, version: sc.version, minScore: minScore, @@ -839,315 +997,512 @@ async function processType(type: string, subtype?: string) { updatedAt: new Date(sc.modify_date), updatedBy: sc.modify_user, }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.scorecard - .createMany({ - data: batch, - }) - .catch(async () => { + }; + if (!isIncrementalRun) { + const processedData = jsonData[key].map((sc) => { + const id = nanoid(14); + scorecardIdMap.set(sc.scorecard_id, id); + return convertScorecard(sc, id); + }); + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.scorecard + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.scorecard + .create({ + data: item, + }) + .catch((err) => { + scorecardIdMap.delete(item.legacyId); + console.error( + `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const sc of jsonData[key]) { + if (!shouldProcessRecord(sc.create_date, sc.modify_date)) { + continue; + } + const existingId = scorecardIdMap.get(sc.scorecard_id); + const id = existingId ?? nanoid(14); + const data = convertScorecard(sc, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.scorecard.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + scorecardIdMap.set(sc.scorecard_id, id); + } + } catch (err) { console.error( - `[${type}][${file}] An error occurred, retrying individually`, + `[${type}][${file}] Failed to upsert scorecard legacyId ${sc.scorecard_id}`, ); - for (const item of batch) { - await prisma.scorecard - .create({ - data: item, - }) - .catch((err) => { - scorecardIdMap.delete(item.legacyId); - console.error( - `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); - } - }); + console.error(err); + } + } } break; } case 'scorecard_group': { console.log(`[${type}][${file}] Processing file`); - const processedData = jsonData[key] - .filter( - (group) => !scorecardGroupIdMap.has(group.scorecard_group_id), - ) - .map((group) => { - const id = nanoid(14); - scorecardGroupIdMap.set(group.scorecard_group_id, id); - return { - id: id, - legacyId: group.scorecard_group_id, - scorecardId: scorecardIdMap.get(group.scorecard_id), - name: group.name, - weight: parseFloat(group.weight), - sortOrder: parseInt(group.sort), - createdAt: new Date(group.create_date), - createdBy: group.create_user, - updatedAt: new Date(group.modify_date), - updatedBy: group.modify_user, - }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.scorecardGroup - .createMany({ - data: batch, - }) - .catch(async () => { + const convertGroup = (group, recordId: string) => ({ + id: recordId, + legacyId: group.scorecard_group_id, + scorecardId: scorecardIdMap.get(group.scorecard_id), + name: group.name, + weight: parseFloat(group.weight), + sortOrder: parseInt(group.sort), + createdAt: new Date(group.create_date), + createdBy: group.create_user, + updatedAt: new Date(group.modify_date), + updatedBy: group.modify_user, + }); + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter( + (group) => !scorecardGroupIdMap.has(group.scorecard_group_id), + ) + .map((group) => { + const id = nanoid(14); + scorecardGroupIdMap.set(group.scorecard_group_id, id); + return convertGroup(group, id); + }); + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.scorecardGroup + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.scorecardGroup + .create({ + data: item, + }) + .catch((err) => { + scorecardGroupIdMap.delete(item.legacyId); + console.error( + `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const group of jsonData[key]) { + if (!shouldProcessRecord(group.create_date, group.modify_date)) { + continue; + } + const existingId = scorecardGroupIdMap.get( + group.scorecard_group_id, + ); + const id = existingId ?? nanoid(14); + const data = convertGroup(group, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.scorecardGroup.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + scorecardGroupIdMap.set(group.scorecard_group_id, id); + } + } catch (err) { console.error( - `[${type}][${file}] An error occurred, retrying individually`, + `[${type}][${file}] Failed to upsert scorecardGroup legacyId ${group.scorecard_group_id}`, ); - for (const item of batch) { - await prisma.scorecardGroup - .create({ - data: item, - }) - .catch((err) => { - scorecardGroupIdMap.delete(item.legacyId); - console.error( - `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); - } - }); + console.error(err); + } + } } break; } case 'scorecard_section': { console.log(`[${type}][${file}] Processing file`); - const processedData = jsonData[key] - .filter( - (section) => - !scorecardSectionIdMap.has(section.scorecard_section_id), - ) - .map((section) => { - const id = nanoid(14); - scorecardSectionIdMap.set(section.scorecard_section_id, id); - return { - id: id, - legacyId: section.scorecard_section_id, - scorecardGroupId: scorecardGroupIdMap.get( - section.scorecard_group_id, - ), - name: section.name, - weight: parseFloat(section.weight), - sortOrder: parseInt(section.sort), - createdAt: new Date(section.create_date), - createdBy: section.create_user, - updatedAt: new Date(section.modify_date), - updatedBy: section.modify_user, - }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.scorecardSection - .createMany({ - data: batch, - }) - .catch(async () => { + const convertSection = (section, recordId: string) => ({ + id: recordId, + legacyId: section.scorecard_section_id, + scorecardGroupId: scorecardGroupIdMap.get( + section.scorecard_group_id, + ), + name: section.name, + weight: parseFloat(section.weight), + sortOrder: parseInt(section.sort), + createdAt: new Date(section.create_date), + createdBy: section.create_user, + updatedAt: new Date(section.modify_date), + updatedBy: section.modify_user, + }); + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter( + (section) => + !scorecardSectionIdMap.has(section.scorecard_section_id), + ) + .map((section) => { + const id = nanoid(14); + scorecardSectionIdMap.set(section.scorecard_section_id, id); + return convertSection(section, id); + }); + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.scorecardSection + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.scorecardSection + .create({ + data: item, + }) + .catch((err) => { + scorecardSectionIdMap.delete(item.legacyId); + console.error( + `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const section of jsonData[key]) { + if ( + !shouldProcessRecord(section.create_date, section.modify_date) + ) { + continue; + } + const existingId = scorecardSectionIdMap.get( + section.scorecard_section_id, + ); + const id = existingId ?? nanoid(14); + const data = convertSection(section, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.scorecardSection.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + scorecardSectionIdMap.set(section.scorecard_section_id, id); + } + } catch (err) { console.error( - `[${type}][${file}] An error occurred, retrying individually`, + `[${type}][${file}] Failed to upsert scorecardSection legacyId ${section.scorecard_section_id}`, ); - for (const item of batch) { - await prisma.scorecardSection - .create({ - data: item, - }) - .catch((err) => { - scorecardSectionIdMap.delete(item.legacyId); - console.error( - `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); - } - }); + console.error(err); + } + } } break; } case 'scorecard_question': { console.log(`[${type}][${file}] Processing file`); - const processedData = jsonData[key] - .filter( - (question) => - !scorecardQuestionIdMap.has(question.scorecard_question_id), - ) - .map((question) => { - const id = nanoid(14); - scorecardQuestionIdMap.set(question.scorecard_question_id, id); - return { - id: id, - legacyId: question.scorecard_question_id, - scorecardSectionId: scorecardSectionIdMap.get( - question.scorecard_section_id, - ), - type: questionTypeMap[question.scorecard_question_type_id].name, - description: question.description, - guidelines: question.guideline, - weight: parseFloat(question.weight), - requiresUpload: question.upload_document === '1', - sortOrder: parseInt(question.sort), - createdAt: new Date(question.create_date), - createdBy: question.create_user, - updatedAt: new Date(question.modify_date), - updatedBy: question.modify_user, - scaleMin: - questionTypeMap[question.scorecard_question_type_id].min, - scaleMax: - questionTypeMap[question.scorecard_question_type_id].max, - }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.scorecardQuestion - .createMany({ - data: batch, - }) - .catch(async () => { + const convertQuestion = (question, recordId: string) => { + const questionType = + questionTypeMap[question.scorecard_question_type_id]; + return { + id: recordId, + legacyId: question.scorecard_question_id, + scorecardSectionId: scorecardSectionIdMap.get( + question.scorecard_section_id, + ), + type: questionType.name, + description: question.description, + guidelines: question.guideline, + weight: parseFloat(question.weight), + requiresUpload: question.upload_document === '1', + sortOrder: parseInt(question.sort), + createdAt: new Date(question.create_date), + createdBy: question.create_user, + updatedAt: new Date(question.modify_date), + updatedBy: question.modify_user, + scaleMin: questionType.min, + scaleMax: questionType.max, + }; + }; + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter( + (question) => + !scorecardQuestionIdMap.has(question.scorecard_question_id), + ) + .map((question) => { + const id = nanoid(14); + scorecardQuestionIdMap.set(question.scorecard_question_id, id); + return convertQuestion(question, id); + }); + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.scorecardQuestion + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.scorecardQuestion + .create({ + data: item, + }) + .catch((err) => { + scorecardQuestionIdMap.delete(item.legacyId); + console.error( + `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const question of jsonData[key]) { + if ( + !shouldProcessRecord(question.create_date, question.modify_date) + ) { + continue; + } + const existingId = scorecardQuestionIdMap.get( + question.scorecard_question_id, + ); + const id = existingId ?? nanoid(14); + const data = convertQuestion(question, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.scorecardQuestion.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + scorecardQuestionIdMap.set( + question.scorecard_question_id, + id, + ); + } + } catch (err) { console.error( - `[${type}][${file}] An error occurred, retrying individually`, + `[${type}][${file}] Failed to upsert scorecardQuestion legacyId ${question.scorecard_question_id}`, ); - for (const item of batch) { - await prisma.scorecardQuestion - .create({ - data: item, - }) - .catch((err) => { - scorecardQuestionIdMap.delete(item.legacyId); - console.error( - `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); - } - }); + console.error(err); + } + } } break; } case 'review': { console.log(`[${type}][${file}] Processing file`); - const processedData = jsonData[key] - .filter((review) => !reviewIdMap.has(review.review_id)) - .map((review) => { - const id = nanoid(14); - reviewIdMap.set(review.review_id, id); - return { - id, - legacyId: review.review_id, - resourceId: review.resource_id, - phaseId: review.project_phase_id, - submissionId: submissionIdMap.get(review.submission_id) || null, - legacySubmissionId: review.submission_id, - scorecardId: scorecardIdMap.get(review.scorecard_id), - committed: review.committed === '1', - finalScore: review.score ? parseFloat(review.score) : null, - initialScore: review.initial_score - ? parseFloat(review.initial_score) - : null, - createdAt: new Date(review.create_date), - createdBy: review.create_user, - updatedAt: new Date(review.modify_date), - updatedBy: review.modify_user, - }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.review.createMany({ data: batch }).catch(async () => { - console.error( - `[${type}][${file}] An error occurred, retrying individually`, + const convertReview = (review, recordId: string) => ({ + id: recordId, + legacyId: review.review_id, + resourceId: review.resource_id, + phaseId: review.project_phase_id, + submissionId: submissionIdMap.get(review.submission_id) || null, + legacySubmissionId: review.submission_id, + scorecardId: scorecardIdMap.get(review.scorecard_id), + committed: review.committed === '1', + finalScore: review.score ? parseFloat(review.score) : null, + initialScore: review.initial_score + ? parseFloat(review.initial_score) + : null, + createdAt: new Date(review.create_date), + createdBy: review.create_user, + updatedAt: new Date(review.modify_date), + updatedBy: review.modify_user, + }); + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter((review) => !reviewIdMap.has(review.review_id)) + .map((review) => { + const id = nanoid(14); + reviewIdMap.set(review.review_id, id); + return convertReview(review, id); + }); + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, ); - for (const item of batch) { - await prisma.review - .create({ - data: item, - }) - .catch((err) => { - reviewIdMap.delete(item.legacyId); - console.error( - `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); + const batch = processedData.slice(i, i + batchSize); + await prisma.review + .createMany({ data: batch }) + .catch(async () => { + console.error( + `[${type}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.review + .create({ + data: item, + }) + .catch((err) => { + reviewIdMap.delete(item.legacyId); + console.error( + `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const review of jsonData[key]) { + if ( + !shouldProcessRecord(review.create_date, review.modify_date) + ) { + continue; } - }); + const existingId = reviewIdMap.get(review.review_id); + const id = existingId ?? nanoid(14); + const data = convertReview(review, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.review.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + reviewIdMap.set(review.review_id, id); + } + } catch (err) { + console.error( + `[${type}][${file}] Failed to upsert review legacyId ${review.review_id}`, + ); + console.error(err); + } + } } break; } case 'review_item': { console.log(`[${type}][${file}] Processing file`); - const processedData = jsonData[key] - .filter((item) => !reviewItemIdMap.has(item.review_item_id)) - .map((item) => { - const id = nanoid(14); - reviewItemIdMap.set(item.review_item_id, id); - return { - id: id, - legacyId: item.review_item_id, - reviewId: reviewIdMap.get(item.review_id), - scorecardQuestionId: scorecardQuestionIdMap.get( - item.scorecard_question_id, - ), - uploadId: item.upload_id || null, - initialAnswer: item.answer, - finalAnswer: item.answer, - managerComment: item.answer, - createdAt: new Date(item.create_date), - createdBy: item.create_user, - updatedAt: new Date(item.modify_date), - updatedBy: item.modify_user, - }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.reviewItem - .createMany({ - data: batch, - }) - .catch(async () => { + const convertReviewItem = (item, recordId: string) => ({ + id: recordId, + legacyId: item.review_item_id, + reviewId: reviewIdMap.get(item.review_id), + scorecardQuestionId: scorecardQuestionIdMap.get( + item.scorecard_question_id, + ), + uploadId: item.upload_id || null, + initialAnswer: item.answer, + finalAnswer: item.answer, + managerComment: item.answer, + createdAt: new Date(item.create_date), + createdBy: item.create_user, + updatedAt: new Date(item.modify_date), + updatedBy: item.modify_user, + }); + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter((item) => !reviewItemIdMap.has(item.review_item_id)) + .map((item) => { + const id = nanoid(14); + reviewItemIdMap.set(item.review_item_id, id); + return convertReviewItem(item, id); + }); + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.reviewItem + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.reviewItem + .create({ + data: item, + }) + .catch((err) => { + reviewItemIdMap.delete(item.legacyId); + console.error( + `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const item of jsonData[key]) { + if (!shouldProcessRecord(item.create_date, item.modify_date)) { + continue; + } + const existingId = reviewItemIdMap.get(item.review_item_id); + const id = existingId ?? nanoid(14); + const data = convertReviewItem(item, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.reviewItem.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + reviewItemIdMap.set(item.review_item_id, id); + } + } catch (err) { console.error( - `[${type}][${file}] An error occurred, retrying individually`, + `[${type}][${file}] Failed to upsert reviewItem legacyId ${item.review_item_id}`, ); - for (const item of batch) { - await prisma.reviewItem - .create({ - data: item, - }) - .catch((err) => { - reviewItemIdMap.delete(item.legacyId); - console.error( - `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); - } - }); + console.error(err); + } + } } break; } @@ -1155,198 +1510,306 @@ async function processType(type: string, subtype?: string) { switch (subtype) { case 'reviewItemComment': { console.log(`[${type}][${subtype}][${file}] Processing file`); - const processedData = jsonData[key] - .filter( - (c) => - reviewItemCommentTypeMap[c.comment_type_id] in - LegacyCommentType, - ) - .filter( - (c) => - !reviewItemCommentReviewItemCommentIdMap.has( + const isSupportedType = (c) => + reviewItemCommentTypeMap[c.comment_type_id] in + LegacyCommentType; + const convertComment = (c, recordId: string) => ({ + id: recordId, + legacyId: c.review_item_comment_id, + resourceId: c.resource_id, + reviewItemId: reviewItemIdMap.get(c.review_item_id), + content: c.content, + type: LegacyCommentType[ + reviewItemCommentTypeMap[c.comment_type_id] + ], + sortOrder: parseInt(c.sort), + createdAt: new Date(c.create_date), + createdBy: c.create_user, + updatedAt: new Date(c.modify_date), + updatedBy: c.modify_user, + }); + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter(isSupportedType) + .filter( + (c) => + !reviewItemCommentReviewItemCommentIdMap.has( + c.review_item_comment_id, + ), + ) + .map((c) => { + const id = nanoid(14); + reviewItemCommentReviewItemCommentIdMap.set( c.review_item_comment_id, - ), - ) - .map((c) => { - const id = nanoid(14); - reviewItemCommentReviewItemCommentIdMap.set( - c.review_item_comment_id, - id, - ); - return { - id: id, - legacyId: c.review_item_comment_id, - resourceId: c.resource_id, - reviewItemId: reviewItemIdMap.get(c.review_item_id), - content: c.content, - type: LegacyCommentType[ - reviewItemCommentTypeMap[c.comment_type_id] - ], - sortOrder: parseInt(c.sort), - createdAt: new Date(c.create_date), - createdBy: c.create_user, - updatedAt: new Date(c.modify_date), - updatedBy: c.modify_user, - }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + id, + ); + return convertComment(c, id); + }); + const totalBatches = Math.ceil( + processedData.length / batchSize, ); - const batch = processedData.slice(i, i + batchSize); - await prisma.reviewItemComment - .createMany({ - data: batch, - }) - .catch(async () => { - console.error( - `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.reviewItemComment + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.reviewItemComment + .create({ + data: item, + }) + .catch((err) => { + reviewItemCommentReviewItemCommentIdMap.delete( + item.legacyId, + ); + console.error( + `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const c of jsonData[key]) { + if (!isSupportedType(c)) { + continue; + } + if (!shouldProcessRecord(c.create_date, c.modify_date)) { + continue; + } + const existingId = + reviewItemCommentReviewItemCommentIdMap.get( + c.review_item_comment_id, ); - for (const item of batch) { - await prisma.reviewItemComment - .create({ - data: item, - }) - .catch((err) => { - reviewItemCommentReviewItemCommentIdMap.delete( - item.legacyId, - ); - console.error( - `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); + const id = existingId ?? nanoid(14); + const data = convertComment(c, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.reviewItemComment.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + reviewItemCommentReviewItemCommentIdMap.set( + c.review_item_comment_id, + id, + ); } - }); + } catch (err) { + console.error( + `[${type}][${subtype}][${file}] Failed to upsert reviewItemComment legacyId ${c.review_item_comment_id}`, + ); + console.error(err); + } + } } break; } case 'appeal': { console.log(`[${type}][${subtype}][${file}] Processing file`); - const processedData = jsonData[key] - .filter( - (c) => - reviewItemCommentTypeMap[c.comment_type_id] === 'Appeal', - ) - .filter( - (c) => !reviewItemCommentAppealIdMap.has(c.review_item_id), - ) - .map((c) => { - const id = nanoid(14); - reviewItemCommentAppealIdMap.set(c.review_item_id, id); - return { - id: id, - legacyId: c.review_item_comment_id, - resourceId: c.resource_id, - reviewItemCommentId: - reviewItemCommentReviewItemCommentIdMap.get( - c.review_item_id, - ), - content: c.content, - createdAt: new Date(c.create_date), - createdBy: c.create_user, - updatedAt: new Date(c.modify_date), - updatedBy: c.modify_user, - }; - }); + const isAppeal = (c) => + reviewItemCommentTypeMap[c.comment_type_id] === 'Appeal'; + const convertAppeal = (c, recordId: string) => ({ + id: recordId, + legacyId: c.review_item_comment_id, + resourceId: c.resource_id, + reviewItemCommentId: + reviewItemCommentReviewItemCommentIdMap.get(c.review_item_id), + content: c.content, + createdAt: new Date(c.create_date), + createdBy: c.create_user, + updatedAt: new Date(c.modify_date), + updatedBy: c.modify_user, + }); + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter(isAppeal) + .filter( + (c) => !reviewItemCommentAppealIdMap.has(c.review_item_id), + ) + .map((c) => { + const id = nanoid(14); + reviewItemCommentAppealIdMap.set(c.review_item_id, id); + return convertAppeal(c, id); + }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + const totalBatches = Math.ceil( + processedData.length / batchSize, ); - const batch = processedData.slice(i, i + batchSize); - await prisma.appeal - .createMany({ - data: batch, - }) - .catch(async () => { + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.appeal + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.appeal + .create({ + data: item, + }) + .catch((err) => { + reviewItemCommentAppealIdMap.delete(item.legacyId); + console.error( + `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const c of jsonData[key]) { + if (!isAppeal(c)) { + continue; + } + if (!shouldProcessRecord(c.create_date, c.modify_date)) { + continue; + } + const existingId = reviewItemCommentAppealIdMap.get( + c.review_item_id, + ); + const id = existingId ?? nanoid(14); + const data = convertAppeal(c, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.appeal.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + reviewItemCommentAppealIdMap.set(c.review_item_id, id); + } + } catch (err) { console.error( - `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + `[${type}][${subtype}][${file}] Failed to upsert appeal legacyId ${c.review_item_comment_id}`, ); - for (const item of batch) { - await prisma.appeal - .create({ - data: item, - }) - .catch((err) => { - reviewItemCommentAppealIdMap.delete(item.legacyId); - console.error( - `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); - } - }); + console.error(err); + } + } } break; } case 'appealResponse': { console.log(`[${type}][${subtype}][${file}] Processing file`); - const processedData = jsonData[key] - .filter( - (c) => - reviewItemCommentTypeMap[c.comment_type_id] === - 'Appeal Response', - ) - .filter( - (c) => - !reviewItemCommentAppealResponseIdMap.has( + const isAppealResponse = (c) => + reviewItemCommentTypeMap[c.comment_type_id] === + 'Appeal Response'; + const convertAppealResponse = (c, recordId: string) => ({ + id: recordId, + legacyId: c.review_item_comment_id, + appealId: reviewItemCommentAppealIdMap.get(c.review_item_id), + resourceId: c.resource_id, + content: c.content, + success: c.extra_info === 'Succeeded', + createdAt: new Date(c.create_date), + createdBy: c.create_user, + updatedAt: new Date(c.modify_date), + updatedBy: c.modify_user, + }); + if (!isIncrementalRun) { + const processedData = jsonData[key] + .filter(isAppealResponse) + .filter( + (c) => + !reviewItemCommentAppealResponseIdMap.has( + c.review_item_comment_id, + ), + ) + .map((c) => { + const id = nanoid(14); + reviewItemCommentAppealResponseIdMap.set( c.review_item_comment_id, - ), - ) - .map((c) => { - const id = nanoid(14); - reviewItemCommentAppealResponseIdMap.set( + id, + ); + return convertAppealResponse(c, id); + }); + const totalBatches = Math.ceil( + processedData.length / batchSize, + ); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.appealResponse + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.appealResponse + .create({ + data: item, + }) + .catch((err) => { + reviewItemCommentAppealResponseIdMap.delete( + item.legacyId, + ); + console.error( + `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, + ); + }); + } + }); + } + } else { + for (const c of jsonData[key]) { + if (!isAppealResponse(c)) { + continue; + } + if (!shouldProcessRecord(c.create_date, c.modify_date)) { + continue; + } + const existingId = reviewItemCommentAppealResponseIdMap.get( c.review_item_comment_id, - id, ); - return { - id: id, - legacyId: c.review_item_comment_id, - appealId: reviewItemCommentAppealIdMap.get( - c.review_item_id, - ), - resourceId: c.resource_id, - content: c.content, - success: c.extra_info === 'Succeeded', - createdAt: new Date(c.create_date), - createdBy: c.create_user, - updatedAt: new Date(c.modify_date), - updatedBy: c.modify_user, - }; - }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.appealResponse - .createMany({ - data: batch, - }) - .catch(async () => { + const id = existingId ?? nanoid(14); + const data = convertAppealResponse(c, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.appealResponse.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + reviewItemCommentAppealResponseIdMap.set( + c.review_item_comment_id, + id, + ); + } + } catch (err) { console.error( - `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + `[${type}][${subtype}][${file}] Failed to upsert appealResponse legacyId ${c.review_item_comment_id}`, ); - for (const item of batch) { - await prisma.appealResponse - .create({ - data: item, - }) - .catch((err) => { - reviewItemCommentAppealResponseIdMap.delete( - item.legacyId, - ); - console.error( - `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, - ); - }); - } - }); + console.error(err); + } + } } break; } @@ -1355,154 +1818,238 @@ async function processType(type: string, subtype?: string) { } case 'llm_provider': { console.log(`[${type}][${subtype}][${file}] Processing file`); - const idToLegacyIdMap = {}; - const processedData = jsonData[key].map((c) => { - const id = nanoid(14); - llmProviderIdMap.set(c.llm_provider_id, id); - idToLegacyIdMap[id] = c.llm_provider_id; - return { - id: id, - name: c.name, - createdAt: new Date(c.create_date), - createdBy: c.create_user, - }; + const convertProvider = (c, recordId: string) => ({ + id: recordId, + name: c.name, + createdAt: new Date(c.create_date), + createdBy: c.create_user, }); + if (!isIncrementalRun) { + const idToLegacyIdMap = {}; + const processedData = jsonData[key].map((c) => { + const id = nanoid(14); + llmProviderIdMap.set(c.llm_provider_id, id); + idToLegacyIdMap[id] = c.llm_provider_id; + return convertProvider(c, id); + }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.llmProvider - .createMany({ - data: batch, - }) - .catch(async () => { + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.llmProvider + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.llmProvider + .create({ + data: item, + }) + .catch((err) => { + llmProviderIdMap.delete(idToLegacyIdMap[item.id]); + console.error( + `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, + ); + }); + } + }); + } + } else { + for (const c of jsonData[key]) { + if (!shouldProcessRecord(c.create_date, c.modify_date)) { + continue; + } + const existingId = llmProviderIdMap.get(c.llm_provider_id); + const id = existingId ?? nanoid(14); + const data = convertProvider(c, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.llmProvider.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + llmProviderIdMap.set(c.llm_provider_id, id); + } + } catch (err) { console.error( - `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + `[${type}][${subtype}][${file}] Failed to upsert llmProvider legacyId ${c.llm_provider_id}`, ); - for (const item of batch) { - await prisma.llmProvider - .create({ - data: item, - }) - .catch((err) => { - llmProviderIdMap.delete(idToLegacyIdMap[item.id]); - console.error( - `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, - ); - }); - } - }); + console.error(err); + } + } } break; } case 'llm_model': { console.log(`[${type}][${subtype}][${file}] Processing file`); - const idToLegacyIdMap = {}; - const processedData = jsonData[key].map((c) => { - const id = nanoid(14); - llmModelIdMap.set(c.llm_model_id, id); - idToLegacyIdMap[id] = c.llm_model_id; - console.log(llmProviderIdMap.get(c.provider_id), 'c.provider_id'); - return { - id: id, - providerId: llmProviderIdMap.get(c.provider_id), - name: c.name, - description: c.description, - icon: c.icon, - url: c.url, - createdAt: new Date(c.create_date), - createdBy: c.create_user, - }; + const convertModel = (c, recordId: string) => ({ + id: recordId, + providerId: llmProviderIdMap.get(c.provider_id), + name: c.name, + description: c.description, + icon: c.icon, + url: c.url, + createdAt: new Date(c.create_date), + createdBy: c.create_user, }); + if (!isIncrementalRun) { + const idToLegacyIdMap = {}; + const processedData = jsonData[key].map((c) => { + const id = nanoid(14); + llmModelIdMap.set(c.llm_model_id, id); + idToLegacyIdMap[id] = c.llm_model_id; + return convertModel(c, id); + }); - console.log(llmProviderIdMap, processedData, 'processedData'); - - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.llmModel - .createMany({ - data: batch, - }) - .catch(async () => { + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.llmModel + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.llmModel + .create({ + data: item, + }) + .catch((err) => { + llmModelIdMap.delete(idToLegacyIdMap[item.id]); + console.error( + `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, + ); + }); + } + }); + } + } else { + for (const c of jsonData[key]) { + if (!shouldProcessRecord(c.create_date, c.modify_date)) { + continue; + } + const existingId = llmModelIdMap.get(c.llm_model_id); + const id = existingId ?? nanoid(14); + const data = convertModel(c, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.llmModel.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + llmModelIdMap.set(c.llm_model_id, id); + } + } catch (err) { console.error( - `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + `[${type}][${subtype}][${file}] Failed to upsert llmModel legacyId ${c.llm_model_id}`, ); - for (const item of batch) { - await prisma.llmModel - .create({ - data: item, - }) - .catch((err) => { - llmModelIdMap.delete(idToLegacyIdMap[item.id]); - console.error( - `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, - ); - }); - } - }); + console.error(err); + } + } } break; } case 'ai_workflow': { console.log(`[${type}][${subtype}][${file}] Processing file`); - const idToLegacyIdMap = {}; - const processedData = jsonData[key].map((c) => { - const id = nanoid(14); - aiWorkflowIdMap.set(c.ai_workflow_id, id); - idToLegacyIdMap[id] = c.ai_workflow_id; - return { - id: id, - llmId: llmModelIdMap.get(c.llm_id), - name: c.name, - description: c.description, - defUrl: c.def_url, - gitId: c.git_id, - gitOwner: c.git_owner, - scorecardId: scorecardIdMap.get(c.scorecard_id), - createdAt: new Date(c.create_date), - createdBy: c.create_user, - updatedAt: new Date(c.modify_date), - updatedBy: c.modify_user, - }; + const convertWorkflow = (c, recordId: string) => ({ + id: recordId, + llmId: llmModelIdMap.get(c.llm_id), + name: c.name, + description: c.description, + defUrl: c.def_url, + gitId: c.git_id, + gitOwner: c.git_owner, + scorecardId: scorecardIdMap.get(c.scorecard_id), + createdAt: new Date(c.create_date), + createdBy: c.create_user, + updatedAt: new Date(c.modify_date), + updatedBy: c.modify_user, }); + if (!isIncrementalRun) { + const idToLegacyIdMap = {}; + const processedData = jsonData[key].map((c) => { + const id = nanoid(14); + aiWorkflowIdMap.set(c.ai_workflow_id, id); + idToLegacyIdMap[id] = c.ai_workflow_id; + return convertWorkflow(c, id); + }); - const totalBatches = Math.ceil(processedData.length / batchSize); - for (let i = 0; i < processedData.length; i += batchSize) { - const batchIndex = i / batchSize + 1; - console.log( - `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, - ); - const batch = processedData.slice(i, i + batchSize); - await prisma.aiWorkflow - .createMany({ - data: batch, - }) - .catch(async () => { + const totalBatches = Math.ceil(processedData.length / batchSize); + for (let i = 0; i < processedData.length; i += batchSize) { + const batchIndex = i / batchSize + 1; + console.log( + `[${type}][${subtype}][${file}] Processing batch ${batchIndex}/${totalBatches}`, + ); + const batch = processedData.slice(i, i + batchSize); + await prisma.aiWorkflow + .createMany({ + data: batch, + }) + .catch(async () => { + console.error( + `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + ); + for (const item of batch) { + await prisma.aiWorkflow + .create({ + data: item, + }) + .catch((err) => { + aiWorkflowIdMap.delete(idToLegacyIdMap[item.id]); + console.error( + `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, + ); + }); + } + }); + } + } else { + for (const c of jsonData[key]) { + if (!shouldProcessRecord(c.create_date, c.modify_date)) { + continue; + } + const existingId = aiWorkflowIdMap.get(c.ai_workflow_id); + const id = existingId ?? nanoid(14); + const data = convertWorkflow(c, id); + try { + const updateData = { ...data }; + delete updateData.id; + await prisma.aiWorkflow.upsert({ + where: { id }, + create: data, + update: updateData, + }); + if (!existingId) { + aiWorkflowIdMap.set(c.ai_workflow_id, id); + } + } catch (err) { console.error( - `[${type}][${subtype}][${file}] An error occurred, retrying individually`, + `[${type}][${subtype}][${file}] Failed to upsert aiWorkflow legacyId ${c.ai_workflow_id}`, ); - for (const item of batch) { - await prisma.aiWorkflow - .create({ - data: item, - }) - .catch((err) => { - aiWorkflowIdMap.delete(idToLegacyIdMap[item.id]); - console.error( - `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, - ); - }); - } - }); + console.error(err); + } + } } break; } @@ -1528,12 +2075,12 @@ async function processAllTypes() { } } -function convertResourceSubmission(jsonData) { +function convertResourceSubmission(jsonData, existingId?: string) { return { - id: nanoid(14), + id: existingId ?? nanoid(14), resourceId: jsonData['resource_id'], legacySubmissionId: jsonData['submission_id'], - submissionId: submissionIdMap[jsonData['submission_id']] || null, + submissionId: submissionIdMap.get(jsonData['submission_id']) || null, createdAt: new Date(jsonData['create_date']), createdBy: jsonData['create_user'], updatedAt: new Date(jsonData['modify_date']), @@ -1543,6 +2090,10 @@ function convertResourceSubmission(jsonData) { let resourceSubmissions: any[] = []; async function handleResourceSubmission(data) { + if (isIncrementalRun) { + await upsertResourceSubmission(data); + return; + } resourceSubmissions.push(data); if (resourceSubmissions.length > batchSize) { await doImportResourceSubmission(); @@ -1570,6 +2121,23 @@ async function doImportResourceSubmission() { } } +async function upsertResourceSubmission(data) { + const { id, ...updateData } = data; + try { + await prisma.resourceSubmission.upsert({ + where: { id }, + create: data, + update: updateData, + }); + } catch (err) { + console.error( + `Failed to upsert resource_submission ${data.resourceId}_${data.legacySubmissionId}`, + ); + console.error(err); + throw err; + } +} + async function migrateResourceSubmissions() { const filenameRegex = new RegExp(`^resource_submission_\\d+\\.json`); const filenames = fs @@ -1585,10 +2153,29 @@ async function migrateResourceSubmissions() { let dataCount = 0; for (const d of jsonData) { dataCount += 1; - const data = convertResourceSubmission(d); - const key = `${data.resourceId}:${data.legacySubmissionId}`; - if (!resourceSubmissionSet.has(key)) { + const shouldPersist = shouldProcessRecord( + d['create_date'], + d['modify_date'], + ); + const key = `${d['resource_id']}:${d['submission_id']}`; + const existingId = resourceSubmissionIdMap.get(key); + const data = convertResourceSubmission(d, existingId); + if (isIncrementalRun) { + if (!existingId && !shouldPersist) { + continue; + } + if (!resourceSubmissionSet.has(key)) { + resourceSubmissionSet.add(key); + } + if (!existingId) { + resourceSubmissionIdMap.set(key, data.id); + } + if (shouldPersist || !existingId) { + await handleResourceSubmission(data); + } + } else if (!resourceSubmissionSet.has(key)) { resourceSubmissionSet.add(key); + resourceSubmissionIdMap.set(key, data.id); await handleResourceSubmission(data); } if (dataCount % logSize === 0) { @@ -1598,9 +2185,19 @@ async function migrateResourceSubmissions() { totalCount += dataCount; } console.log(`resource_submission total count: ${totalCount}`); + if (!isIncrementalRun) { + await doImportResourceSubmission(); + } } async function migrate() { + if (!fs.existsSync(DATA_DIR)) { + throw new Error( + `DATA_DIR "${DATA_DIR}" does not exist. Set DATA_DIR to a valid export path.`, + ); + } + console.log(`Using data directory: ${DATA_DIR}`); + console.log(`Using ElasticSearch export file: ${ES_DATA_FILE}`); console.log('Starting lookup import...'); processLookupFiles(); console.log('Lookup import completed.'); @@ -1676,6 +2273,7 @@ migrate() { key: 'llmProviderIdMap', value: llmProviderIdMap }, { key: 'llmModelIdMap', value: llmModelIdMap }, { key: 'aiWorkflowIdMap', value: aiWorkflowIdMap }, + { key: 'resourceSubmissionIdMap', value: resourceSubmissionIdMap }, ].forEach((f) => { if (!fs.existsSync('.tmp')) { fs.mkdirSync('.tmp'); From 15125abad138765749069425d34e7cbabd9d2393 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 24 Oct 2025 09:17:24 +1100 Subject: [PATCH 31/48] Fix dirty prod data --- prisma/migrate.ts | 741 ++++++++++++++++++++++++++++++---------------- 1 file changed, 494 insertions(+), 247 deletions(-) diff --git a/prisma/migrate.ts b/prisma/migrate.ts index bedfd9b..f634127 100644 --- a/prisma/migrate.ts +++ b/prisma/migrate.ts @@ -131,16 +131,117 @@ const submissionMap: Record> = {}; // Data lookup maps // Initialize maps from files if they exist, otherwise create new maps -function readIdMap(filename) { - return fs.existsSync(`.tmp/${filename}.json`) - ? new Map( - Object.entries( - JSON.parse(fs.readFileSync(`.tmp/${filename}.json`, 'utf-8')), - ), - ) - : new Map(); +function readIdMap(filename: string): Map { + if (fs.existsSync(`.tmp/${filename}.json`)) { + const entries = Object.entries( + JSON.parse(fs.readFileSync(`.tmp/${filename}.json`, 'utf-8')), + ); + return new Map(entries); + } + return new Map(); } +const describeLegacyId = (value: unknown): string => { + if (value === null) { + return 'null'; + } + if (value === undefined) { + return 'undefined'; + } + if (typeof value === 'symbol') { + return value.toString(); + } + if ( + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' || + typeof value === 'bigint' + ) { + return String(value); + } + if (value instanceof Date) { + return value.toISOString(); + } + if (typeof value === 'object' || typeof value === 'function') { + try { + const json = JSON.stringify(value); + if (typeof json === 'string') { + return json; + } + } catch { + // ignore serialization errors + } + const fallbackDescription: string = Object.prototype.toString.call(value); + return fallbackDescription; + } + const fallbackDescription: string = Object.prototype.toString.call(value); + return fallbackDescription; +}; + +const normalizeLegacyKey = (value: unknown): string => { + if (value === null || value === undefined) { + throw new Error('Missing legacy identifier when normalizing key'); + } + return describeLegacyId(value); +}; + +const tryNormalizeLegacyKey = (value: unknown): string | undefined => { + if (value === null || value === undefined) { + return undefined; + } + return describeLegacyId(value); +}; + +const omitId = (entity: T): Omit => { + const { id: ignoredId, ...rest } = entity; + void ignoredId; + return rest; +}; + +const getMappedId = ( + map: Map, + legacyId: unknown, +): string | undefined => { + const key = tryNormalizeLegacyKey(legacyId); + return key ? map.get(key) : undefined; +}; + +const setMappedId = ( + map: Map, + legacyId: unknown, + value: string, +) => { + map.set(normalizeLegacyKey(legacyId), value); +}; + +const deleteMappedId = (map: Map, legacyId: unknown) => { + const key = tryNormalizeLegacyKey(legacyId); + if (key) { + map.delete(key); + } +}; + +const requireMappedId = ( + map: Map, + legacyId: unknown, + context: string, +): string => { + const id = getMappedId(map, legacyId); + if (!id) { + throw new Error( + `Missing required mapping for ${context} legacy id "${describeLegacyId( + legacyId, + )}"`, + ); + } + return id; +}; + +const hasMappedId = (map: Map, legacyId: unknown): boolean => { + const key = tryNormalizeLegacyKey(legacyId); + return key ? map.has(key) : false; +}; + const projectIdMap = readIdMap('projectIdMap'); const scorecardIdMap = readIdMap('scorecardIdMap'); const scorecardGroupIdMap = readIdMap('scorecardGroupIdMap'); @@ -403,7 +504,7 @@ function convertSubmissionES(esData): any { id: summation.id, legacySubmissionId: String(esData.legacySubmissionId), aggregateScore: summation.aggregateScore, - scorecardId: scorecardIdMap.get(summation.scoreCardId), + scorecardId: getMappedId(scorecardIdMap, summation.scoreCardId), scorecardLegacyId: String(summation.scoreCardId), isPassing: summation.isPassing, reviewedDate: summation.reviewedDate @@ -483,18 +584,22 @@ async function importSubmissionES() { let newSubmission = false; try { - if (submissionIdMap.has(submission.legacySubmissionId)) { + const existingSubmissionId = getMappedId( + submissionIdMap, + submission.legacySubmissionId, + ); + if (existingSubmissionId) { newSubmission = false; await prisma.submission.update({ data: submission, where: { - id: submissionIdMap.get(submission.legacySubmissionId) as string, + id: existingSubmissionId, }, }); } else { newSubmission = true; const newId = nanoid(14); - projectIdMap.set(submission.legacySubmissionId, newId); + setMappedId(projectIdMap, submission.legacySubmissionId, newId); let type = LegacySubmissionType[item.type]; if (!LegacySubmissionType[item.type]) { type = LegacySubmissionType.ContestSubmission; @@ -509,11 +614,11 @@ async function importSubmissionES() { createdAt: item.created ? new Date(item.created) : new Date(), }, }); - submissionIdMap.set(submission.legacySubmissionId, newId); + setMappedId(submissionIdMap, submission.legacySubmissionId, newId); } } catch { if (newSubmission) { - projectIdMap.delete(submission.legacySubmissionId); + deleteMappedId(projectIdMap, submission.legacySubmissionId); } console.error(`Failed to import submission from ES: ${submission.esId}`); } @@ -563,7 +668,7 @@ async function doImportUploadData() { await prisma.upload.create({ data: u }); } catch { console.error(`Cannot import upload data id: ${u.legacyId}`); - uploadIdMap.delete(u.legacyId); + deleteMappedId(uploadIdMap, u.legacyId); } } } @@ -589,7 +694,7 @@ function convertSubmission(jsonData, existingId?: string) { id: existingId ?? nanoid(14), legacySubmissionId: jsonData['submission_id'], legacyUploadId: jsonData['upload_id'], - uploadId: uploadIdMap.get(jsonData['upload_id']), + uploadId: getMappedId(uploadIdMap, jsonData['upload_id']), status: submissionStatusMap[jsonData['submission_status_id']], type: submissionTypeMap[jsonData['submission_type_id']], screeningScore: jsonData['screening_score'], @@ -633,7 +738,7 @@ async function doImportSubmissionData() { await prisma.submission.create({ data: s }); } catch { console.error(`Failed to import submission ${s.legacySubmissionId}`); - submissionIdMap.delete(s.legacySubmissionId); + deleteMappedId(submissionIdMap, s.legacySubmissionId); } } } @@ -696,13 +801,13 @@ async function initSubmissionMap() { d['modify_date'], ); const legacyId = String(d['upload_id']); - const existingId = uploadIdMap.get(legacyId); + const existingId = getMappedId(uploadIdMap, legacyId); const uploadData = convertUpload(d, existingId); const skipPersistence = !existingId && isIncrementalRun && !shouldPersist; if (!skipPersistence) { // import upload data if any if (!existingId) { - uploadIdMap.set(uploadData.legacyId, uploadData.id); + setMappedId(uploadIdMap, uploadData.legacyId, uploadData.id); if (isIncrementalRun) { await upsertUploadData(uploadData); } else { @@ -718,7 +823,10 @@ async function initSubmissionMap() { uploadData.status === UploadStatus.ACTIVE && uploadData.resourceId != null ) { - resourceUploadMap[uploadData.resourceId] = uploadData.legacyId; + const resourceIdKey = tryNormalizeLegacyKey(uploadData.resourceId); + if (resourceIdKey) { + resourceUploadMap[resourceIdKey] = String(uploadData.legacyId); + } } if (dataCount % logSize === 0) { console.log(`Imported upload count: ${dataCount}`); @@ -745,12 +853,12 @@ async function initSubmissionMap() { d['modify_date'], ); const legacyId = String(d['submission_id']); - const existingId = submissionIdMap.get(legacyId); + const existingId = getMappedId(submissionIdMap, legacyId); const dbData = convertSubmission(d, existingId); const skipPersistence = !existingId && isIncrementalRun && !shouldPersist; if (!skipPersistence) { if (!existingId) { - submissionIdMap.set(dbData.legacySubmissionId, dbData.id); + setMappedId(submissionIdMap, dbData.legacySubmissionId, dbData.id); if (isIncrementalRun) { await upsertSubmissionData(dbData); } else { @@ -772,16 +880,20 @@ async function initSubmissionMap() { submissionId: dbData.legacySubmissionId, }; // pick the latest valid submission for each upload - if (uploadSubmissionMap[dbData.legacyUploadId]) { - const existing = uploadSubmissionMap[dbData.legacyUploadId]; + const uploadIdKey = tryNormalizeLegacyKey(dbData.legacyUploadId); + if (!uploadIdKey) { + continue; + } + if (uploadSubmissionMap[uploadIdKey]) { + const existing = uploadSubmissionMap[uploadIdKey]; if ( !existing.score || item.created.getTime() > existing.created.getTime() ) { - uploadSubmissionMap[dbData.legacyUploadId] = item; + uploadSubmissionMap[uploadIdKey] = item; } } else { - uploadSubmissionMap[dbData.legacyUploadId] = item; + uploadSubmissionMap[uploadIdKey] = item; } } if (dataCount % logSize === 0) { @@ -902,12 +1014,13 @@ async function processType(type: string, subtype?: string) { }; if (!isIncrementalRun) { const processedData = jsonData[key] - .filter((pr) => !projectIdMap.has(pr.project_id + pr.user_id)) + .filter((pr) => { + const mapKey = `${pr.project_id}${pr.user_id}`; + return !hasMappedId(projectIdMap, mapKey); + }) .map((pr) => { - projectIdMap.set( - pr.project_id + pr.user_id, - pr.project_id + pr.user_id, - ); + const mapKey = `${pr.project_id}${pr.user_id}`; + setMappedId(projectIdMap, mapKey, mapKey); return convertProjectResult(pr); }); const totalBatches = Math.ceil(processedData.length / batchSize); @@ -931,7 +1044,8 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - projectIdMap.delete( + deleteMappedId( + projectIdMap, `${item.challengeId}${item.userId}`, ); console.error( @@ -946,7 +1060,7 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(pr.create_date, pr.modify_date)) { continue; } - const mapKey = pr.project_id + pr.user_id; + const mapKey = `${pr.project_id}${pr.user_id}`; const data = convertProjectResult(pr); try { await prisma.challengeResult.upsert({ @@ -959,7 +1073,7 @@ async function processType(type: string, subtype?: string) { create: data, update: data, }); - projectIdMap.set(mapKey, mapKey); + setMappedId(projectIdMap, mapKey, mapKey); } catch (err) { console.error( `[${type}][${file}] Failed to upsert challengeResult for ChallengeId: ${data.challengeId}, UserId: ${data.userId}`, @@ -1001,7 +1115,7 @@ async function processType(type: string, subtype?: string) { if (!isIncrementalRun) { const processedData = jsonData[key].map((sc) => { const id = nanoid(14); - scorecardIdMap.set(sc.scorecard_id, id); + setMappedId(scorecardIdMap, sc.scorecard_id, id); return convertScorecard(sc, id); }); const totalBatches = Math.ceil(processedData.length / batchSize); @@ -1025,7 +1139,7 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - scorecardIdMap.delete(item.legacyId); + deleteMappedId(scorecardIdMap, item.legacyId); console.error( `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, ); @@ -1038,19 +1152,18 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(sc.create_date, sc.modify_date)) { continue; } - const existingId = scorecardIdMap.get(sc.scorecard_id); + const existingId = getMappedId(scorecardIdMap, sc.scorecard_id); const id = existingId ?? nanoid(14); const data = convertScorecard(sc, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.scorecard.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - scorecardIdMap.set(sc.scorecard_id, id); + setMappedId(scorecardIdMap, sc.scorecard_id, id); } } catch (err) { console.error( @@ -1064,26 +1177,35 @@ async function processType(type: string, subtype?: string) { } case 'scorecard_group': { console.log(`[${type}][${file}] Processing file`); - const convertGroup = (group, recordId: string) => ({ - id: recordId, - legacyId: group.scorecard_group_id, - scorecardId: scorecardIdMap.get(group.scorecard_id), - name: group.name, - weight: parseFloat(group.weight), - sortOrder: parseInt(group.sort), - createdAt: new Date(group.create_date), - createdBy: group.create_user, - updatedAt: new Date(group.modify_date), - updatedBy: group.modify_user, - }); + const convertGroup = (group, recordId: string) => { + const legacyId = normalizeLegacyKey(group.scorecard_group_id); + const scorecardId = requireMappedId( + scorecardIdMap, + group.scorecard_id, + 'scorecard', + ); + return { + id: recordId, + legacyId, + scorecardId, + name: group.name, + weight: parseFloat(group.weight), + sortOrder: parseInt(group.sort), + createdAt: new Date(group.create_date), + createdBy: group.create_user, + updatedAt: new Date(group.modify_date), + updatedBy: group.modify_user, + }; + }; if (!isIncrementalRun) { const processedData = jsonData[key] .filter( - (group) => !scorecardGroupIdMap.has(group.scorecard_group_id), + (group) => + !hasMappedId(scorecardGroupIdMap, group.scorecard_group_id), ) .map((group) => { const id = nanoid(14); - scorecardGroupIdMap.set(group.scorecard_group_id, id); + setMappedId(scorecardGroupIdMap, group.scorecard_group_id, id); return convertGroup(group, id); }); const totalBatches = Math.ceil(processedData.length / batchSize); @@ -1107,7 +1229,7 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - scorecardGroupIdMap.delete(item.legacyId); + deleteMappedId(scorecardGroupIdMap, item.legacyId); console.error( `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, ); @@ -1120,21 +1242,25 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(group.create_date, group.modify_date)) { continue; } - const existingId = scorecardGroupIdMap.get( + const existingId = getMappedId( + scorecardGroupIdMap, group.scorecard_group_id, ); const id = existingId ?? nanoid(14); const data = convertGroup(group, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.scorecardGroup.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - scorecardGroupIdMap.set(group.scorecard_group_id, id); + setMappedId( + scorecardGroupIdMap, + group.scorecard_group_id, + id, + ); } } catch (err) { console.error( @@ -1148,29 +1274,42 @@ async function processType(type: string, subtype?: string) { } case 'scorecard_section': { console.log(`[${type}][${file}] Processing file`); - const convertSection = (section, recordId: string) => ({ - id: recordId, - legacyId: section.scorecard_section_id, - scorecardGroupId: scorecardGroupIdMap.get( + const convertSection = (section, recordId: string) => { + const legacyId = normalizeLegacyKey(section.scorecard_section_id); + const scorecardGroupId = requireMappedId( + scorecardGroupIdMap, section.scorecard_group_id, - ), - name: section.name, - weight: parseFloat(section.weight), - sortOrder: parseInt(section.sort), - createdAt: new Date(section.create_date), - createdBy: section.create_user, - updatedAt: new Date(section.modify_date), - updatedBy: section.modify_user, - }); + 'scorecard group', + ); + return { + id: recordId, + legacyId, + scorecardGroupId, + name: section.name, + weight: parseFloat(section.weight), + sortOrder: parseInt(section.sort), + createdAt: new Date(section.create_date), + createdBy: section.create_user, + updatedAt: new Date(section.modify_date), + updatedBy: section.modify_user, + }; + }; if (!isIncrementalRun) { const processedData = jsonData[key] .filter( (section) => - !scorecardSectionIdMap.has(section.scorecard_section_id), + !hasMappedId( + scorecardSectionIdMap, + section.scorecard_section_id, + ), ) .map((section) => { const id = nanoid(14); - scorecardSectionIdMap.set(section.scorecard_section_id, id); + setMappedId( + scorecardSectionIdMap, + section.scorecard_section_id, + id, + ); return convertSection(section, id); }); const totalBatches = Math.ceil(processedData.length / batchSize); @@ -1194,7 +1333,7 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - scorecardSectionIdMap.delete(item.legacyId); + deleteMappedId(scorecardSectionIdMap, item.legacyId); console.error( `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, ); @@ -1209,21 +1348,25 @@ async function processType(type: string, subtype?: string) { ) { continue; } - const existingId = scorecardSectionIdMap.get( + const existingId = getMappedId( + scorecardSectionIdMap, section.scorecard_section_id, ); const id = existingId ?? nanoid(14); const data = convertSection(section, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.scorecardSection.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - scorecardSectionIdMap.set(section.scorecard_section_id, id); + setMappedId( + scorecardSectionIdMap, + section.scorecard_section_id, + id, + ); } } catch (err) { console.error( @@ -1240,12 +1383,16 @@ async function processType(type: string, subtype?: string) { const convertQuestion = (question, recordId: string) => { const questionType = questionTypeMap[question.scorecard_question_type_id]; + const legacyId = normalizeLegacyKey(question.scorecard_question_id); + const scorecardSectionId = requireMappedId( + scorecardSectionIdMap, + question.scorecard_section_id, + 'scorecard section', + ); return { id: recordId, - legacyId: question.scorecard_question_id, - scorecardSectionId: scorecardSectionIdMap.get( - question.scorecard_section_id, - ), + legacyId, + scorecardSectionId, type: questionType.name, description: question.description, guidelines: question.guideline, @@ -1264,11 +1411,18 @@ async function processType(type: string, subtype?: string) { const processedData = jsonData[key] .filter( (question) => - !scorecardQuestionIdMap.has(question.scorecard_question_id), + !hasMappedId( + scorecardQuestionIdMap, + question.scorecard_question_id, + ), ) .map((question) => { const id = nanoid(14); - scorecardQuestionIdMap.set(question.scorecard_question_id, id); + setMappedId( + scorecardQuestionIdMap, + question.scorecard_question_id, + id, + ); return convertQuestion(question, id); }); const totalBatches = Math.ceil(processedData.length / batchSize); @@ -1292,7 +1446,7 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - scorecardQuestionIdMap.delete(item.legacyId); + deleteMappedId(scorecardQuestionIdMap, item.legacyId); console.error( `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, ); @@ -1307,21 +1461,22 @@ async function processType(type: string, subtype?: string) { ) { continue; } - const existingId = scorecardQuestionIdMap.get( + const existingId = getMappedId( + scorecardQuestionIdMap, question.scorecard_question_id, ); const id = existingId ?? nanoid(14); const data = convertQuestion(question, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.scorecardQuestion.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - scorecardQuestionIdMap.set( + setMappedId( + scorecardQuestionIdMap, question.scorecard_question_id, id, ); @@ -1338,30 +1493,40 @@ async function processType(type: string, subtype?: string) { } case 'review': { console.log(`[${type}][${file}] Processing file`); - const convertReview = (review, recordId: string) => ({ - id: recordId, - legacyId: review.review_id, - resourceId: review.resource_id, - phaseId: review.project_phase_id, - submissionId: submissionIdMap.get(review.submission_id) || null, - legacySubmissionId: review.submission_id, - scorecardId: scorecardIdMap.get(review.scorecard_id), - committed: review.committed === '1', - finalScore: review.score ? parseFloat(review.score) : null, - initialScore: review.initial_score - ? parseFloat(review.initial_score) - : null, - createdAt: new Date(review.create_date), - createdBy: review.create_user, - updatedAt: new Date(review.modify_date), - updatedBy: review.modify_user, - }); + const convertReview = (review, recordId: string) => { + const legacyId = normalizeLegacyKey(review.review_id); + const submissionId = + getMappedId(submissionIdMap, review.submission_id) ?? null; + const scorecardId = requireMappedId( + scorecardIdMap, + review.scorecard_id, + 'scorecard', + ); + return { + id: recordId, + legacyId, + resourceId: review.resource_id, + phaseId: review.project_phase_id, + submissionId, + legacySubmissionId: review.submission_id, + scorecardId, + committed: review.committed === '1', + finalScore: review.score ? parseFloat(review.score) : null, + initialScore: review.initial_score + ? parseFloat(review.initial_score) + : null, + createdAt: new Date(review.create_date), + createdBy: review.create_user, + updatedAt: new Date(review.modify_date), + updatedBy: review.modify_user, + }; + }; if (!isIncrementalRun) { const processedData = jsonData[key] - .filter((review) => !reviewIdMap.has(review.review_id)) + .filter((review) => !hasMappedId(reviewIdMap, review.review_id)) .map((review) => { const id = nanoid(14); - reviewIdMap.set(review.review_id, id); + setMappedId(reviewIdMap, review.review_id, id); return convertReview(review, id); }); const totalBatches = Math.ceil(processedData.length / batchSize); @@ -1383,7 +1548,7 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - reviewIdMap.delete(item.legacyId); + deleteMappedId(reviewIdMap, item.legacyId); console.error( `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, ); @@ -1398,19 +1563,18 @@ async function processType(type: string, subtype?: string) { ) { continue; } - const existingId = reviewIdMap.get(review.review_id); + const existingId = getMappedId(reviewIdMap, review.review_id); const id = existingId ?? nanoid(14); const data = convertReview(review, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.review.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - reviewIdMap.set(review.review_id, id); + setMappedId(reviewIdMap, review.review_id, id); } } catch (err) { console.error( @@ -1424,28 +1588,41 @@ async function processType(type: string, subtype?: string) { } case 'review_item': { console.log(`[${type}][${file}] Processing file`); - const convertReviewItem = (item, recordId: string) => ({ - id: recordId, - legacyId: item.review_item_id, - reviewId: reviewIdMap.get(item.review_id), - scorecardQuestionId: scorecardQuestionIdMap.get( + const convertReviewItem = (item, recordId: string) => { + const legacyId = normalizeLegacyKey(item.review_item_id); + const reviewId = requireMappedId( + reviewIdMap, + item.review_id, + 'review', + ); + const scorecardQuestionId = requireMappedId( + scorecardQuestionIdMap, item.scorecard_question_id, - ), - uploadId: item.upload_id || null, - initialAnswer: item.answer, - finalAnswer: item.answer, - managerComment: item.answer, - createdAt: new Date(item.create_date), - createdBy: item.create_user, - updatedAt: new Date(item.modify_date), - updatedBy: item.modify_user, - }); + 'scorecard question', + ); + return { + id: recordId, + legacyId, + reviewId, + scorecardQuestionId, + uploadId: item.upload_id || null, + initialAnswer: item.answer, + finalAnswer: item.answer, + managerComment: item.answer, + createdAt: new Date(item.create_date), + createdBy: item.create_user, + updatedAt: new Date(item.modify_date), + updatedBy: item.modify_user, + }; + }; if (!isIncrementalRun) { const processedData = jsonData[key] - .filter((item) => !reviewItemIdMap.has(item.review_item_id)) + .filter( + (item) => !hasMappedId(reviewItemIdMap, item.review_item_id), + ) .map((item) => { const id = nanoid(14); - reviewItemIdMap.set(item.review_item_id, id); + setMappedId(reviewItemIdMap, item.review_item_id, id); return convertReviewItem(item, id); }); const totalBatches = Math.ceil(processedData.length / batchSize); @@ -1469,7 +1646,7 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - reviewItemIdMap.delete(item.legacyId); + deleteMappedId(reviewItemIdMap, item.legacyId); console.error( `[${type}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, ); @@ -1482,19 +1659,21 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(item.create_date, item.modify_date)) { continue; } - const existingId = reviewItemIdMap.get(item.review_item_id); + const existingId = getMappedId( + reviewItemIdMap, + item.review_item_id, + ); const id = existingId ?? nanoid(14); const data = convertReviewItem(item, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.reviewItem.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - reviewItemIdMap.set(item.review_item_id, id); + setMappedId(reviewItemIdMap, item.review_item_id, id); } } catch (err) { console.error( @@ -1513,33 +1692,43 @@ async function processType(type: string, subtype?: string) { const isSupportedType = (c) => reviewItemCommentTypeMap[c.comment_type_id] in LegacyCommentType; - const convertComment = (c, recordId: string) => ({ - id: recordId, - legacyId: c.review_item_comment_id, - resourceId: c.resource_id, - reviewItemId: reviewItemIdMap.get(c.review_item_id), - content: c.content, - type: LegacyCommentType[ - reviewItemCommentTypeMap[c.comment_type_id] - ], - sortOrder: parseInt(c.sort), - createdAt: new Date(c.create_date), - createdBy: c.create_user, - updatedAt: new Date(c.modify_date), - updatedBy: c.modify_user, - }); + const convertComment = (c, recordId: string) => { + const legacyId = normalizeLegacyKey(c.review_item_comment_id); + const reviewItemId = requireMappedId( + reviewItemIdMap, + c.review_item_id, + 'review item', + ); + return { + id: recordId, + legacyId, + resourceId: c.resource_id, + reviewItemId, + content: c.content, + type: LegacyCommentType[ + reviewItemCommentTypeMap[c.comment_type_id] + ], + sortOrder: parseInt(c.sort), + createdAt: new Date(c.create_date), + createdBy: c.create_user, + updatedAt: new Date(c.modify_date), + updatedBy: c.modify_user, + }; + }; if (!isIncrementalRun) { const processedData = jsonData[key] .filter(isSupportedType) .filter( (c) => - !reviewItemCommentReviewItemCommentIdMap.has( + !hasMappedId( + reviewItemCommentReviewItemCommentIdMap, c.review_item_comment_id, ), ) .map((c) => { const id = nanoid(14); - reviewItemCommentReviewItemCommentIdMap.set( + setMappedId( + reviewItemCommentReviewItemCommentIdMap, c.review_item_comment_id, id, ); @@ -1568,7 +1757,8 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - reviewItemCommentReviewItemCommentIdMap.delete( + deleteMappedId( + reviewItemCommentReviewItemCommentIdMap, item.legacyId, ); console.error( @@ -1586,22 +1776,22 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } - const existingId = - reviewItemCommentReviewItemCommentIdMap.get( - c.review_item_comment_id, - ); + const existingId = getMappedId( + reviewItemCommentReviewItemCommentIdMap, + c.review_item_comment_id, + ); const id = existingId ?? nanoid(14); const data = convertComment(c, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.reviewItemComment.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - reviewItemCommentReviewItemCommentIdMap.set( + setMappedId( + reviewItemCommentReviewItemCommentIdMap, c.review_item_comment_id, id, ); @@ -1620,27 +1810,42 @@ async function processType(type: string, subtype?: string) { console.log(`[${type}][${subtype}][${file}] Processing file`); const isAppeal = (c) => reviewItemCommentTypeMap[c.comment_type_id] === 'Appeal'; - const convertAppeal = (c, recordId: string) => ({ - id: recordId, - legacyId: c.review_item_comment_id, - resourceId: c.resource_id, - reviewItemCommentId: - reviewItemCommentReviewItemCommentIdMap.get(c.review_item_id), - content: c.content, - createdAt: new Date(c.create_date), - createdBy: c.create_user, - updatedAt: new Date(c.modify_date), - updatedBy: c.modify_user, - }); + const convertAppeal = (c, recordId: string) => { + const legacyId = normalizeLegacyKey(c.review_item_comment_id); + const reviewItemCommentId = requireMappedId( + reviewItemCommentReviewItemCommentIdMap, + c.review_item_id, + 'review item comment', + ); + return { + id: recordId, + legacyId, + resourceId: c.resource_id, + reviewItemCommentId, + content: c.content, + createdAt: new Date(c.create_date), + createdBy: c.create_user, + updatedAt: new Date(c.modify_date), + updatedBy: c.modify_user, + }; + }; if (!isIncrementalRun) { const processedData = jsonData[key] .filter(isAppeal) .filter( - (c) => !reviewItemCommentAppealIdMap.has(c.review_item_id), + (c) => + !hasMappedId( + reviewItemCommentAppealIdMap, + c.review_item_id, + ), ) .map((c) => { const id = nanoid(14); - reviewItemCommentAppealIdMap.set(c.review_item_id, id); + setMappedId( + reviewItemCommentAppealIdMap, + c.review_item_id, + id, + ); return convertAppeal(c, id); }); @@ -1667,7 +1872,10 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - reviewItemCommentAppealIdMap.delete(item.legacyId); + deleteMappedId( + reviewItemCommentAppealIdMap, + item.legacyId, + ); console.error( `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${item.legacyId}`, ); @@ -1683,21 +1891,25 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } - const existingId = reviewItemCommentAppealIdMap.get( + const existingId = getMappedId( + reviewItemCommentAppealIdMap, c.review_item_id, ); const id = existingId ?? nanoid(14); const data = convertAppeal(c, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.appeal.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - reviewItemCommentAppealIdMap.set(c.review_item_id, id); + setMappedId( + reviewItemCommentAppealIdMap, + c.review_item_id, + id, + ); } } catch (err) { console.error( @@ -1714,30 +1926,40 @@ async function processType(type: string, subtype?: string) { const isAppealResponse = (c) => reviewItemCommentTypeMap[c.comment_type_id] === 'Appeal Response'; - const convertAppealResponse = (c, recordId: string) => ({ - id: recordId, - legacyId: c.review_item_comment_id, - appealId: reviewItemCommentAppealIdMap.get(c.review_item_id), - resourceId: c.resource_id, - content: c.content, - success: c.extra_info === 'Succeeded', - createdAt: new Date(c.create_date), - createdBy: c.create_user, - updatedAt: new Date(c.modify_date), - updatedBy: c.modify_user, - }); + const convertAppealResponse = (c, recordId: string) => { + const legacyId = normalizeLegacyKey(c.review_item_comment_id); + const appealId = requireMappedId( + reviewItemCommentAppealIdMap, + c.review_item_id, + 'appeal', + ); + return { + id: recordId, + legacyId, + appealId, + resourceId: c.resource_id, + content: c.content, + success: c.extra_info === 'Succeeded', + createdAt: new Date(c.create_date), + createdBy: c.create_user, + updatedAt: new Date(c.modify_date), + updatedBy: c.modify_user, + }; + }; if (!isIncrementalRun) { const processedData = jsonData[key] .filter(isAppealResponse) .filter( (c) => - !reviewItemCommentAppealResponseIdMap.has( + !hasMappedId( + reviewItemCommentAppealResponseIdMap, c.review_item_comment_id, ), ) .map((c) => { const id = nanoid(14); - reviewItemCommentAppealResponseIdMap.set( + setMappedId( + reviewItemCommentAppealResponseIdMap, c.review_item_comment_id, id, ); @@ -1766,7 +1988,8 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - reviewItemCommentAppealResponseIdMap.delete( + deleteMappedId( + reviewItemCommentAppealResponseIdMap, item.legacyId, ); console.error( @@ -1784,21 +2007,22 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } - const existingId = reviewItemCommentAppealResponseIdMap.get( + const existingId = getMappedId( + reviewItemCommentAppealResponseIdMap, c.review_item_comment_id, ); const id = existingId ?? nanoid(14); const data = convertAppealResponse(c, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.appealResponse.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - reviewItemCommentAppealResponseIdMap.set( + setMappedId( + reviewItemCommentAppealResponseIdMap, c.review_item_comment_id, id, ); @@ -1828,7 +2052,7 @@ async function processType(type: string, subtype?: string) { const idToLegacyIdMap = {}; const processedData = jsonData[key].map((c) => { const id = nanoid(14); - llmProviderIdMap.set(c.llm_provider_id, id); + setMappedId(llmProviderIdMap, c.llm_provider_id, id); idToLegacyIdMap[id] = c.llm_provider_id; return convertProvider(c, id); }); @@ -1854,7 +2078,10 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - llmProviderIdMap.delete(idToLegacyIdMap[item.id]); + deleteMappedId( + llmProviderIdMap, + idToLegacyIdMap[item.id], + ); console.error( `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, ); @@ -1867,19 +2094,21 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } - const existingId = llmProviderIdMap.get(c.llm_provider_id); + const existingId = getMappedId( + llmProviderIdMap, + c.llm_provider_id, + ); const id = existingId ?? nanoid(14); const data = convertProvider(c, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.llmProvider.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - llmProviderIdMap.set(c.llm_provider_id, id); + setMappedId(llmProviderIdMap, c.llm_provider_id, id); } } catch (err) { console.error( @@ -1893,21 +2122,28 @@ async function processType(type: string, subtype?: string) { } case 'llm_model': { console.log(`[${type}][${subtype}][${file}] Processing file`); - const convertModel = (c, recordId: string) => ({ - id: recordId, - providerId: llmProviderIdMap.get(c.provider_id), - name: c.name, - description: c.description, - icon: c.icon, - url: c.url, - createdAt: new Date(c.create_date), - createdBy: c.create_user, - }); + const convertModel = (c, recordId: string) => { + const providerId = requireMappedId( + llmProviderIdMap, + c.provider_id, + 'llm provider', + ); + return { + id: recordId, + providerId, + name: c.name, + description: c.description, + icon: c.icon, + url: c.url, + createdAt: new Date(c.create_date), + createdBy: c.create_user, + }; + }; if (!isIncrementalRun) { const idToLegacyIdMap = {}; const processedData = jsonData[key].map((c) => { const id = nanoid(14); - llmModelIdMap.set(c.llm_model_id, id); + setMappedId(llmModelIdMap, c.llm_model_id, id); idToLegacyIdMap[id] = c.llm_model_id; return convertModel(c, id); }); @@ -1933,7 +2169,7 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - llmModelIdMap.delete(idToLegacyIdMap[item.id]); + deleteMappedId(llmModelIdMap, idToLegacyIdMap[item.id]); console.error( `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, ); @@ -1946,19 +2182,18 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } - const existingId = llmModelIdMap.get(c.llm_model_id); + const existingId = getMappedId(llmModelIdMap, c.llm_model_id); const id = existingId ?? nanoid(14); const data = convertModel(c, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.llmModel.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - llmModelIdMap.set(c.llm_model_id, id); + setMappedId(llmModelIdMap, c.llm_model_id, id); } } catch (err) { console.error( @@ -1972,25 +2207,33 @@ async function processType(type: string, subtype?: string) { } case 'ai_workflow': { console.log(`[${type}][${subtype}][${file}] Processing file`); - const convertWorkflow = (c, recordId: string) => ({ - id: recordId, - llmId: llmModelIdMap.get(c.llm_id), - name: c.name, - description: c.description, - defUrl: c.def_url, - gitId: c.git_id, - gitOwner: c.git_owner, - scorecardId: scorecardIdMap.get(c.scorecard_id), - createdAt: new Date(c.create_date), - createdBy: c.create_user, - updatedAt: new Date(c.modify_date), - updatedBy: c.modify_user, - }); + const convertWorkflow = (c, recordId: string) => { + const llmId = requireMappedId(llmModelIdMap, c.llm_id, 'llm model'); + const scorecardId = requireMappedId( + scorecardIdMap, + c.scorecard_id, + 'scorecard', + ); + return { + id: recordId, + llmId, + name: c.name, + description: c.description, + defUrl: c.def_url, + gitWorkflowId: c.git_id, + gitOwnerRepo: c.git_owner, + scorecardId, + createdAt: new Date(c.create_date), + createdBy: c.create_user, + updatedAt: new Date(c.modify_date), + updatedBy: c.modify_user, + }; + }; if (!isIncrementalRun) { const idToLegacyIdMap = {}; const processedData = jsonData[key].map((c) => { const id = nanoid(14); - aiWorkflowIdMap.set(c.ai_workflow_id, id); + setMappedId(aiWorkflowIdMap, c.ai_workflow_id, id); idToLegacyIdMap[id] = c.ai_workflow_id; return convertWorkflow(c, id); }); @@ -2016,7 +2259,10 @@ async function processType(type: string, subtype?: string) { data: item, }) .catch((err) => { - aiWorkflowIdMap.delete(idToLegacyIdMap[item.id]); + deleteMappedId( + aiWorkflowIdMap, + idToLegacyIdMap[item.id], + ); console.error( `[${type}][${subtype}][${file}] Error code: ${err.code}, LegacyId: ${idToLegacyIdMap[item.id]}`, ); @@ -2029,19 +2275,18 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } - const existingId = aiWorkflowIdMap.get(c.ai_workflow_id); + const existingId = getMappedId(aiWorkflowIdMap, c.ai_workflow_id); const id = existingId ?? nanoid(14); const data = convertWorkflow(c, id); try { - const updateData = { ...data }; - delete updateData.id; + const updateData = omitId(data); await prisma.aiWorkflow.upsert({ where: { id }, create: data, update: updateData, }); if (!existingId) { - aiWorkflowIdMap.set(c.ai_workflow_id, id); + setMappedId(aiWorkflowIdMap, c.ai_workflow_id, id); } } catch (err) { console.error( @@ -2076,11 +2321,13 @@ async function processAllTypes() { } function convertResourceSubmission(jsonData, existingId?: string) { + const legacySubmissionId = normalizeLegacyKey(jsonData['submission_id']); return { id: existingId ?? nanoid(14), resourceId: jsonData['resource_id'], - legacySubmissionId: jsonData['submission_id'], - submissionId: submissionIdMap.get(jsonData['submission_id']) || null, + legacySubmissionId, + submissionId: + getMappedId(submissionIdMap, jsonData['submission_id']) ?? null, createdAt: new Date(jsonData['create_date']), createdBy: jsonData['create_user'], updatedAt: new Date(jsonData['modify_date']), @@ -2158,7 +2405,7 @@ async function migrateResourceSubmissions() { d['modify_date'], ); const key = `${d['resource_id']}:${d['submission_id']}`; - const existingId = resourceSubmissionIdMap.get(key); + const existingId = getMappedId(resourceSubmissionIdMap, key); const data = convertResourceSubmission(d, existingId); if (isIncrementalRun) { if (!existingId && !shouldPersist) { @@ -2168,14 +2415,14 @@ async function migrateResourceSubmissions() { resourceSubmissionSet.add(key); } if (!existingId) { - resourceSubmissionIdMap.set(key, data.id); + setMappedId(resourceSubmissionIdMap, key, data.id); } if (shouldPersist || !existingId) { await handleResourceSubmission(data); } } else if (!resourceSubmissionSet.has(key)) { resourceSubmissionSet.add(key); - resourceSubmissionIdMap.set(key, data.id); + setMappedId(resourceSubmissionIdMap, key, data.id); await handleResourceSubmission(data); } if (dataCount % logSize === 0) { From 580a97aaa729ae7e2d3d4fd3609a00c4d06c83ac Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 24 Oct 2025 09:26:01 +1100 Subject: [PATCH 32/48] Build fix --- prisma/migrate.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/prisma/migrate.ts b/prisma/migrate.ts index f634127..5fe905b 100644 --- a/prisma/migrate.ts +++ b/prisma/migrate.ts @@ -135,7 +135,16 @@ function readIdMap(filename: string): Map { if (fs.existsSync(`.tmp/${filename}.json`)) { const entries = Object.entries( JSON.parse(fs.readFileSync(`.tmp/${filename}.json`, 'utf-8')), - ); + ).map(([key, value]) => { + if (typeof value !== 'string') { + throw new Error( + `Invalid mapping value for ${filename}: expected string, received "${describeLegacyId( + value, + )}"`, + ); + } + return [key, value] as [string, string]; + }); return new Map(entries); } return new Map(); From bf95ebf647e1fc54be601c974221da2e1f0e6764 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 24 Oct 2025 10:28:56 +1100 Subject: [PATCH 33/48] Fix for dirty prod data --- prisma/migrate.ts | 57 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/prisma/migrate.ts b/prisma/migrate.ts index 5fe905b..2b14148 100644 --- a/prisma/migrate.ts +++ b/prisma/migrate.ts @@ -1597,6 +1597,30 @@ async function processType(type: string, subtype?: string) { } case 'review_item': { console.log(`[${type}][${file}] Processing file`); + const logReviewItemConversionError = (err: unknown, rawItem: any) => { + const legacyId = rawItem?.review_item_id; + const errorMessage = + err instanceof Error ? err.message : String(err); + console.error( + `[${type}][${file}] Failed to convert reviewItem legacyId ${legacyId}: ${errorMessage}`, + ); + if (err instanceof Error && err.stack) { + console.error(err); + } + try { + console.error( + `[${type}][${file}] Skipping record: ${JSON.stringify(rawItem)}`, + ); + } catch (stringifyErr) { + const stringifyMessage = + stringifyErr instanceof Error + ? stringifyErr.message + : String(stringifyErr); + console.error( + `[${type}][${file}] Failed to stringify record for legacyId ${legacyId}: ${stringifyMessage}`, + ); + } + }; const convertReviewItem = (item, recordId: string) => { const legacyId = normalizeLegacyKey(item.review_item_id); const reviewId = requireMappedId( @@ -1625,15 +1649,22 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter( - (item) => !hasMappedId(reviewItemIdMap, item.review_item_id), - ) - .map((item) => { - const id = nanoid(14); - setMappedId(reviewItemIdMap, item.review_item_id, id); - return convertReviewItem(item, id); - }); + type ReviewItemEntity = ReturnType; + const processedData: ReviewItemEntity[] = []; + for (const rawItem of jsonData[key]) { + if (hasMappedId(reviewItemIdMap, rawItem.review_item_id)) { + continue; + } + const id = nanoid(14); + try { + const converted = convertReviewItem(rawItem, id); + setMappedId(reviewItemIdMap, rawItem.review_item_id, id); + processedData.push(converted); + } catch (err) { + logReviewItemConversionError(err, rawItem); + continue; + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { const batchIndex = i / batchSize + 1; @@ -1673,7 +1704,13 @@ async function processType(type: string, subtype?: string) { item.review_item_id, ); const id = existingId ?? nanoid(14); - const data = convertReviewItem(item, id); + let data: ReturnType; + try { + data = convertReviewItem(item, id); + } catch (err) { + logReviewItemConversionError(err, item); + continue; + } try { const updateData = omitId(data); await prisma.reviewItem.upsert({ From 3bc5d544cb2845688e483d6d7cf969a664786444 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 24 Oct 2025 16:38:39 +1100 Subject: [PATCH 34/48] Fix up migrate to handle dirty data with logging and skipping --- prisma/migrate.ts | 609 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 470 insertions(+), 139 deletions(-) diff --git a/prisma/migrate.ts b/prisma/migrate.ts index 2b14148..6f79fb4 100644 --- a/prisma/migrate.ts +++ b/prisma/migrate.ts @@ -86,6 +86,37 @@ const shouldProcessRecord = ( ); }; +const buildLogPrefix = (type: string, file: string, subtype?: string) => + subtype ? `[${type}][${subtype}][${file}]` : `[${type}][${file}]`; + +const logRecordConversionError = ( + type: string, + file: string, + err: unknown, + record: unknown, + message: string, + subtype?: string, +) => { + const prefix = buildLogPrefix(type, file, subtype); + const errorMessage = err instanceof Error ? err.message : String(err); + console.error(`${prefix} ${message}`); + console.error(`${prefix} Error detail: ${errorMessage}`); + if (err instanceof Error && err.stack) { + console.error(err); + } + try { + console.error(`${prefix} Skipping record: ${JSON.stringify(record)}`); + } catch (stringifyErr) { + const stringifyMessage = + stringifyErr instanceof Error + ? stringifyErr.message + : String(stringifyErr); + console.error( + `${prefix} Failed to stringify record while skipping: ${stringifyMessage}`, + ); + } +}; + const modelMappingKeys = [ 'project_result', 'scorecard', @@ -1022,16 +1053,29 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter((pr) => { - const mapKey = `${pr.project_id}${pr.user_id}`; - return !hasMappedId(projectIdMap, mapKey); - }) - .map((pr) => { - const mapKey = `${pr.project_id}${pr.user_id}`; + type ChallengeResultEntity = ReturnType< + typeof convertProjectResult + >; + const processedData: ChallengeResultEntity[] = []; + for (const pr of jsonData[key]) { + const mapKey = `${pr.project_id}${pr.user_id}`; + if (hasMappedId(projectIdMap, mapKey)) { + continue; + } + try { + const converted = convertProjectResult(pr); setMappedId(projectIdMap, mapKey, mapKey); - return convertProjectResult(pr); - }); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + pr, + `Failed to convert projectResult for challengeId ${pr.project_id}, userId ${pr.user_id}`, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { const batchIndex = i / batchSize + 1; @@ -1070,7 +1114,19 @@ async function processType(type: string, subtype?: string) { continue; } const mapKey = `${pr.project_id}${pr.user_id}`; - const data = convertProjectResult(pr); + let data: ReturnType; + try { + data = convertProjectResult(pr); + } catch (err) { + logRecordConversionError( + type, + file, + err, + pr, + `Failed to convert projectResult for challengeId ${pr.project_id}, userId ${pr.user_id}`, + ); + continue; + } try { await prisma.challengeResult.upsert({ where: { @@ -1122,11 +1178,24 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key].map((sc) => { + type ScorecardEntity = ReturnType; + const processedData: ScorecardEntity[] = []; + for (const sc of jsonData[key]) { const id = nanoid(14); - setMappedId(scorecardIdMap, sc.scorecard_id, id); - return convertScorecard(sc, id); - }); + try { + const converted = convertScorecard(sc, id); + setMappedId(scorecardIdMap, sc.scorecard_id, id); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + sc, + `Failed to convert scorecard legacyId ${sc.scorecard_id}`, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { const batchIndex = i / batchSize + 1; @@ -1163,7 +1232,19 @@ async function processType(type: string, subtype?: string) { } const existingId = getMappedId(scorecardIdMap, sc.scorecard_id); const id = existingId ?? nanoid(14); - const data = convertScorecard(sc, id); + let data: ReturnType; + try { + data = convertScorecard(sc, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + sc, + `Failed to convert scorecard legacyId ${sc.scorecard_id}`, + ); + continue; + } try { const updateData = omitId(data); await prisma.scorecard.upsert({ @@ -1207,16 +1288,27 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter( - (group) => - !hasMappedId(scorecardGroupIdMap, group.scorecard_group_id), - ) - .map((group) => { - const id = nanoid(14); + type ScorecardGroupEntity = ReturnType; + const processedData: ScorecardGroupEntity[] = []; + for (const group of jsonData[key]) { + if (hasMappedId(scorecardGroupIdMap, group.scorecard_group_id)) { + continue; + } + const id = nanoid(14); + try { + const converted = convertGroup(group, id); setMappedId(scorecardGroupIdMap, group.scorecard_group_id, id); - return convertGroup(group, id); - }); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + group, + `Failed to convert scorecardGroup legacyId ${group.scorecard_group_id}`, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { const batchIndex = i / batchSize + 1; @@ -1256,7 +1348,19 @@ async function processType(type: string, subtype?: string) { group.scorecard_group_id, ); const id = existingId ?? nanoid(14); - const data = convertGroup(group, id); + let data: ReturnType; + try { + data = convertGroup(group, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + group, + `Failed to convert scorecardGroup legacyId ${group.scorecard_group_id}`, + ); + continue; + } try { const updateData = omitId(data); await prisma.scorecardGroup.upsert({ @@ -1304,23 +1408,33 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter( - (section) => - !hasMappedId( - scorecardSectionIdMap, - section.scorecard_section_id, - ), - ) - .map((section) => { - const id = nanoid(14); + type ScorecardSectionEntity = ReturnType; + const processedData: ScorecardSectionEntity[] = []; + for (const section of jsonData[key]) { + if ( + hasMappedId(scorecardSectionIdMap, section.scorecard_section_id) + ) { + continue; + } + const id = nanoid(14); + try { + const converted = convertSection(section, id); setMappedId( scorecardSectionIdMap, section.scorecard_section_id, id, ); - return convertSection(section, id); - }); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + section, + `Failed to convert scorecardSection legacyId ${section.scorecard_section_id}`, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { const batchIndex = i / batchSize + 1; @@ -1362,7 +1476,19 @@ async function processType(type: string, subtype?: string) { section.scorecard_section_id, ); const id = existingId ?? nanoid(14); - const data = convertSection(section, id); + let data: ReturnType; + try { + data = convertSection(section, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + section, + `Failed to convert scorecardSection legacyId ${section.scorecard_section_id}`, + ); + continue; + } try { const updateData = omitId(data); await prisma.scorecardSection.upsert({ @@ -1417,23 +1543,36 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter( - (question) => - !hasMappedId( - scorecardQuestionIdMap, - question.scorecard_question_id, - ), - ) - .map((question) => { - const id = nanoid(14); + type ScorecardQuestionEntity = ReturnType; + const processedData: ScorecardQuestionEntity[] = []; + for (const question of jsonData[key]) { + if ( + hasMappedId( + scorecardQuestionIdMap, + question.scorecard_question_id, + ) + ) { + continue; + } + const id = nanoid(14); + try { + const converted = convertQuestion(question, id); setMappedId( scorecardQuestionIdMap, question.scorecard_question_id, id, ); - return convertQuestion(question, id); - }); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + question, + `Failed to convert scorecardQuestion legacyId ${question.scorecard_question_id}`, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { const batchIndex = i / batchSize + 1; @@ -1475,7 +1614,19 @@ async function processType(type: string, subtype?: string) { question.scorecard_question_id, ); const id = existingId ?? nanoid(14); - const data = convertQuestion(question, id); + let data: ReturnType; + try { + data = convertQuestion(question, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + question, + `Failed to convert scorecardQuestion legacyId ${question.scorecard_question_id}`, + ); + continue; + } try { const updateData = omitId(data); await prisma.scorecardQuestion.upsert({ @@ -1531,13 +1682,27 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter((review) => !hasMappedId(reviewIdMap, review.review_id)) - .map((review) => { - const id = nanoid(14); + type ReviewEntity = ReturnType; + const processedData: ReviewEntity[] = []; + for (const review of jsonData[key]) { + if (hasMappedId(reviewIdMap, review.review_id)) { + continue; + } + const id = nanoid(14); + try { + const converted = convertReview(review, id); setMappedId(reviewIdMap, review.review_id, id); - return convertReview(review, id); - }); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + review, + `Failed to convert review legacyId ${review.review_id}`, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { const batchIndex = i / batchSize + 1; @@ -1574,7 +1739,19 @@ async function processType(type: string, subtype?: string) { } const existingId = getMappedId(reviewIdMap, review.review_id); const id = existingId ?? nanoid(14); - const data = convertReview(review, id); + let data: ReturnType; + try { + data = convertReview(review, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + review, + `Failed to convert review legacyId ${review.review_id}`, + ); + continue; + } try { const updateData = omitId(data); await prisma.review.upsert({ @@ -1598,28 +1775,13 @@ async function processType(type: string, subtype?: string) { case 'review_item': { console.log(`[${type}][${file}] Processing file`); const logReviewItemConversionError = (err: unknown, rawItem: any) => { - const legacyId = rawItem?.review_item_id; - const errorMessage = - err instanceof Error ? err.message : String(err); - console.error( - `[${type}][${file}] Failed to convert reviewItem legacyId ${legacyId}: ${errorMessage}`, + logRecordConversionError( + type, + file, + err, + rawItem, + `Failed to convert reviewItem legacyId ${rawItem?.review_item_id}`, ); - if (err instanceof Error && err.stack) { - console.error(err); - } - try { - console.error( - `[${type}][${file}] Skipping record: ${JSON.stringify(rawItem)}`, - ); - } catch (stringifyErr) { - const stringifyMessage = - stringifyErr instanceof Error - ? stringifyErr.message - : String(stringifyErr); - console.error( - `[${type}][${file}] Failed to stringify record for legacyId ${legacyId}: ${stringifyMessage}`, - ); - } }; const convertReviewItem = (item, recordId: string) => { const legacyId = normalizeLegacyKey(item.review_item_id); @@ -1762,24 +1924,42 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter(isSupportedType) - .filter( - (c) => - !hasMappedId( - reviewItemCommentReviewItemCommentIdMap, - c.review_item_comment_id, - ), - ) - .map((c) => { - const id = nanoid(14); + type ReviewItemCommentEntity = ReturnType< + typeof convertComment + >; + const processedData: ReviewItemCommentEntity[] = []; + for (const c of jsonData[key]) { + if (!isSupportedType(c)) { + continue; + } + if ( + hasMappedId( + reviewItemCommentReviewItemCommentIdMap, + c.review_item_comment_id, + ) + ) { + continue; + } + const id = nanoid(14); + try { + const converted = convertComment(c, id); setMappedId( reviewItemCommentReviewItemCommentIdMap, c.review_item_comment_id, id, ); - return convertComment(c, id); - }); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert reviewItemComment legacyId ${c.review_item_comment_id}`, + subtype, + ); + } + } const totalBatches = Math.ceil( processedData.length / batchSize, ); @@ -1827,7 +2007,20 @@ async function processType(type: string, subtype?: string) { c.review_item_comment_id, ); const id = existingId ?? nanoid(14); - const data = convertComment(c, id); + let data: ReturnType; + try { + data = convertComment(c, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert reviewItemComment legacyId ${c.review_item_comment_id}`, + subtype, + ); + continue; + } try { const updateData = omitId(data); await prisma.reviewItemComment.upsert({ @@ -1876,24 +2069,37 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter(isAppeal) - .filter( - (c) => - !hasMappedId( - reviewItemCommentAppealIdMap, - c.review_item_id, - ), - ) - .map((c) => { - const id = nanoid(14); + type AppealEntity = ReturnType; + const processedData: AppealEntity[] = []; + for (const c of jsonData[key]) { + if (!isAppeal(c)) { + continue; + } + if ( + hasMappedId(reviewItemCommentAppealIdMap, c.review_item_id) + ) { + continue; + } + const id = nanoid(14); + try { + const converted = convertAppeal(c, id); setMappedId( reviewItemCommentAppealIdMap, c.review_item_id, id, ); - return convertAppeal(c, id); - }); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert appeal legacyId ${c.review_item_comment_id}`, + subtype, + ); + } + } const totalBatches = Math.ceil( processedData.length / batchSize, @@ -1942,7 +2148,20 @@ async function processType(type: string, subtype?: string) { c.review_item_id, ); const id = existingId ?? nanoid(14); - const data = convertAppeal(c, id); + let data: ReturnType; + try { + data = convertAppeal(c, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert appeal legacyId ${c.review_item_comment_id}`, + subtype, + ); + continue; + } try { const updateData = omitId(data); await prisma.appeal.upsert({ @@ -1993,24 +2212,42 @@ async function processType(type: string, subtype?: string) { }; }; if (!isIncrementalRun) { - const processedData = jsonData[key] - .filter(isAppealResponse) - .filter( - (c) => - !hasMappedId( - reviewItemCommentAppealResponseIdMap, - c.review_item_comment_id, - ), - ) - .map((c) => { - const id = nanoid(14); + type AppealResponseEntity = ReturnType< + typeof convertAppealResponse + >; + const processedData: AppealResponseEntity[] = []; + for (const c of jsonData[key]) { + if (!isAppealResponse(c)) { + continue; + } + if ( + hasMappedId( + reviewItemCommentAppealResponseIdMap, + c.review_item_comment_id, + ) + ) { + continue; + } + const id = nanoid(14); + try { + const converted = convertAppealResponse(c, id); setMappedId( reviewItemCommentAppealResponseIdMap, c.review_item_comment_id, id, ); - return convertAppealResponse(c, id); - }); + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert appealResponse legacyId ${c.review_item_comment_id}`, + subtype, + ); + } + } const totalBatches = Math.ceil( processedData.length / batchSize, ); @@ -2058,7 +2295,20 @@ async function processType(type: string, subtype?: string) { c.review_item_comment_id, ); const id = existingId ?? nanoid(14); - const data = convertAppealResponse(c, id); + let data: ReturnType; + try { + data = convertAppealResponse(c, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert appealResponse legacyId ${c.review_item_comment_id}`, + subtype, + ); + continue; + } try { const updateData = omitId(data); await prisma.appealResponse.upsert({ @@ -2096,12 +2346,26 @@ async function processType(type: string, subtype?: string) { }); if (!isIncrementalRun) { const idToLegacyIdMap = {}; - const processedData = jsonData[key].map((c) => { + type LlmProviderEntity = ReturnType; + const processedData: LlmProviderEntity[] = []; + for (const c of jsonData[key]) { const id = nanoid(14); - setMappedId(llmProviderIdMap, c.llm_provider_id, id); - idToLegacyIdMap[id] = c.llm_provider_id; - return convertProvider(c, id); - }); + try { + const converted = convertProvider(c, id); + setMappedId(llmProviderIdMap, c.llm_provider_id, id); + idToLegacyIdMap[id] = c.llm_provider_id; + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert llmProvider legacyId ${c.llm_provider_id}`, + subtype, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -2145,7 +2409,20 @@ async function processType(type: string, subtype?: string) { c.llm_provider_id, ); const id = existingId ?? nanoid(14); - const data = convertProvider(c, id); + let data: ReturnType; + try { + data = convertProvider(c, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert llmProvider legacyId ${c.llm_provider_id}`, + subtype, + ); + continue; + } try { const updateData = omitId(data); await prisma.llmProvider.upsert({ @@ -2187,12 +2464,26 @@ async function processType(type: string, subtype?: string) { }; if (!isIncrementalRun) { const idToLegacyIdMap = {}; - const processedData = jsonData[key].map((c) => { + type LlmModelEntity = ReturnType; + const processedData: LlmModelEntity[] = []; + for (const c of jsonData[key]) { const id = nanoid(14); - setMappedId(llmModelIdMap, c.llm_model_id, id); - idToLegacyIdMap[id] = c.llm_model_id; - return convertModel(c, id); - }); + try { + const converted = convertModel(c, id); + setMappedId(llmModelIdMap, c.llm_model_id, id); + idToLegacyIdMap[id] = c.llm_model_id; + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert llmModel legacyId ${c.llm_model_id}`, + subtype, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -2230,7 +2521,20 @@ async function processType(type: string, subtype?: string) { } const existingId = getMappedId(llmModelIdMap, c.llm_model_id); const id = existingId ?? nanoid(14); - const data = convertModel(c, id); + let data: ReturnType; + try { + data = convertModel(c, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert llmModel legacyId ${c.llm_model_id}`, + subtype, + ); + continue; + } try { const updateData = omitId(data); await prisma.llmModel.upsert({ @@ -2277,12 +2581,26 @@ async function processType(type: string, subtype?: string) { }; if (!isIncrementalRun) { const idToLegacyIdMap = {}; - const processedData = jsonData[key].map((c) => { + type AiWorkflowEntity = ReturnType; + const processedData: AiWorkflowEntity[] = []; + for (const c of jsonData[key]) { const id = nanoid(14); - setMappedId(aiWorkflowIdMap, c.ai_workflow_id, id); - idToLegacyIdMap[id] = c.ai_workflow_id; - return convertWorkflow(c, id); - }); + try { + const converted = convertWorkflow(c, id); + setMappedId(aiWorkflowIdMap, c.ai_workflow_id, id); + idToLegacyIdMap[id] = c.ai_workflow_id; + processedData.push(converted); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert aiWorkflow legacyId ${c.ai_workflow_id}`, + subtype, + ); + } + } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -2323,7 +2641,20 @@ async function processType(type: string, subtype?: string) { } const existingId = getMappedId(aiWorkflowIdMap, c.ai_workflow_id); const id = existingId ?? nanoid(14); - const data = convertWorkflow(c, id); + let data: ReturnType; + try { + data = convertWorkflow(c, id); + } catch (err) { + logRecordConversionError( + type, + file, + err, + c, + `Failed to convert aiWorkflow legacyId ${c.ai_workflow_id}`, + subtype, + ); + continue; + } try { const updateData = omitId(data); await prisma.aiWorkflow.upsert({ From e4ca78e3f2106f63fc759fff97c4de030065efe0 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 24 Oct 2025 17:33:10 +1100 Subject: [PATCH 35/48] Show all reviews to reviewers once a challenge completes --- src/api/review/review.service.spec.ts | 42 +++++++++++++++++++++++ src/api/review/review.service.ts | 48 ++++++++++++++++----------- 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index fbcb904..9f680c4 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -1398,6 +1398,48 @@ describe('ReviewService.getReviews reviewer visibility', () => { }); }); + it('allows a reviewer to see all reviews when the challenge is completed', async () => { + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + buildResource('Reviewer'), + ]); + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Completed Challenge', + status: ChallengeStatus.COMPLETED, + phases: [ + { id: 'phase-review', name: 'Review', isOpen: false }, + { id: 'phase-screening', name: 'Screening', isOpen: false }, + ], + }); + + await service.getReviews(baseAuthUser, undefined, 'challenge-1'); + + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; + const whereClauses = flattenWhereClauses(callArgs.where); + expect(findClauseWithKey(whereClauses, 'resourceId')).toBeUndefined(); + }); + + it('allows a reviewer to see all reviews when the challenge is cancelled', async () => { + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + buildResource('Reviewer'), + ]); + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + name: 'Cancelled Challenge', + status: ChallengeStatus.CANCELLED_FAILED_REVIEW, + phases: [ + { id: 'phase-review', name: 'Review', isOpen: false }, + { id: 'phase-screening', name: 'Screening', isOpen: false }, + ], + }); + + await service.getReviews(baseAuthUser, undefined, 'challenge-1'); + + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; + const whereClauses = flattenWhereClauses(callArgs.where); + expect(findClauseWithKey(whereClauses, 'resourceId')).toBeUndefined(); + }); + it('filters reviews by the resource id when the requester is an approver', async () => { resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ buildResource('Approver'), diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 522d7cc..9e27d70 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -2765,29 +2765,37 @@ export class ReviewService { // Copilots retain full visibility for the challenge } else if (reviewerResourceIdSet.size) { const reviewerResourceIds = Array.from(reviewerResourceIdSet); - if (hasReviewerRoleForChallenge && challengeId) { - let screeningPhaseIds: string[] = []; - if (!challengeDetail) { - try { - challengeDetail = - await this.challengeApiService.getChallengeDetail( - challengeId, - ); - } catch (error) { - const message = - error instanceof Error ? error.message : String(error); - this.logger.debug( - `[getReviews] Unable to fetch challenge ${challengeId} for reviewer screening access: ${message}`, + let challengeCompletedOrCancelled = false; + if (challengeId && !challengeDetail) { + try { + challengeDetail = + await this.challengeApiService.getChallengeDetail( + challengeId, ); - } + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.debug( + `[getReviews] Unable to fetch challenge ${challengeId} for reviewer screening access: ${message}`, + ); } + } - if (challengeDetail) { - screeningPhaseIds = this.getPhaseIdsForNames(challengeDetail, [ - 'screening', - 'checkpoint screening', - ]); - } + if (challengeDetail) { + challengeCompletedOrCancelled = this.isCompletedOrCancelledStatus( + challengeDetail.status, + ); + } + + if (challengeCompletedOrCancelled) { + // Completed or cancelled challenges should expose all reviews to reviewers. + } else if (hasReviewerRoleForChallenge && challengeId) { + const screeningPhaseIds = challengeDetail + ? this.getPhaseIdsForNames(challengeDetail, [ + 'screening', + 'checkpoint screening', + ]) + : []; if (screeningPhaseIds.length) { reviewWhereClause.OR = [ From 1dabf7ccee0ba152ffd7a396daeb490567ebf3c8 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 24 Oct 2025 19:01:08 +1100 Subject: [PATCH 36/48] Review detail visibility for reviewers --- src/api/review/review.service.spec.ts | 142 ++++++++++++ src/api/review/review.service.ts | 15 +- src/api/submission/submission.service.spec.ts | 153 ++++++++++++- src/api/submission/submission.service.ts | 211 +++++++++++++++++- 4 files changed, 514 insertions(+), 7 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index 9f680c4..e861e1c 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -1501,6 +1501,148 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(callArgs.where.resourceId).toBeUndefined(); }); + it('hides scores and review items for other reviewers on active challenges while keeping reviewer metadata', async () => { + const now = new Date(); + const reviewerUser: JwtUser = { + userId: '101', + roles: [], + isMachine: false, + }; + + const reviewerResource = { + ...buildResource('Reviewer', reviewerUser.userId), + id: 'resource-self', + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + reviewerResource, + ]); + + const reviewRecords = [ + { + id: 'review-self', + resourceId: 'resource-self', + submissionId: baseSubmission.id, + phaseId: 'phase-review', + scorecardId: 'scorecard-1', + typeId: 'type-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + reviewItems: [ + { + id: 'item-self', + scorecardQuestionId: 'question-1', + initialAnswer: 'Yes', + finalAnswer: 'Yes', + managerComment: null, + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 95, + initialScore: 90, + submission: { + id: baseSubmission.id, + memberId: '301', + challengeId: 'challenge-1', + }, + }, + { + id: 'review-other', + resourceId: 'resource-other', + submissionId: baseSubmission.id, + phaseId: 'phase-review', + scorecardId: 'scorecard-1', + typeId: 'type-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + reviewItems: [ + { + id: 'item-other', + scorecardQuestionId: 'question-2', + initialAnswer: 'No', + finalAnswer: 'No', + managerComment: null, + createdAt: now, + createdBy: 'other-reviewer', + updatedAt: now, + updatedBy: 'other-reviewer', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 75, + initialScore: 70, + submission: { + id: baseSubmission.id, + memberId: '301', + challengeId: 'challenge-1', + }, + }, + ]; + + prismaMock.review.findMany.mockResolvedValue(reviewRecords); + prismaMock.review.count.mockResolvedValue(reviewRecords.length); + + resourcePrismaMock.resource.findMany.mockResolvedValue([ + { id: 'resource-self', memberId: '101' }, + { id: 'resource-other', memberId: '202' }, + ]); + + memberPrismaMock.member.findMany.mockResolvedValue([ + { + userId: BigInt(101), + handle: 'selfHandle', + maxRating: { rating: 2400 }, + }, + { + userId: BigInt(202), + handle: 'otherHandle', + maxRating: { rating: 1800 }, + }, + ]); + + const response = await service.getReviews( + reviewerUser, + undefined, + 'challenge-1', + ); + + expect(response.data).toHaveLength(2); + const selfReview = response.data.find( + (review) => review.id === 'review-self', + ); + const otherReview = response.data.find( + (review) => review.id === 'review-other', + ); + + expect(selfReview?.finalScore).toBe(95); + expect(selfReview?.initialScore).toBe(90); + expect(selfReview?.reviewItems).toHaveLength(1); + expect(selfReview?.reviewerHandle).toBe('selfHandle'); + expect(selfReview?.reviewerMaxRating).toBe(2400); + + expect(otherReview?.finalScore).toBeNull(); + expect(otherReview?.initialScore).toBeNull(); + expect(otherReview?.reviewItems).toEqual([]); + expect(otherReview?.reviewerHandle).toBe('otherHandle'); + expect(otherReview?.reviewerMaxRating).toBe(1800); + }); + it('returns empty results for submitters when reviews are not yet accessible', async () => { const submitterUser: JwtUser = { userId: 'submitter-1', diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 9e27d70..34cda02 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -3343,8 +3343,18 @@ export class ReviewService { !isOwnSubmission && !submitterVisibilityState.allowAny; + const challengeStatusForReview = + challengeForReview?.status ?? challengeDetail?.status ?? null; + const shouldMaskOtherReviewerDetails = + !isPrivilegedRequester && + reviewerResourceIdSet.size > 0 && + !isReviewerForReview && + !this.isCompletedOrCancelledStatus(challengeStatusForReview); + const sanitizeScores = - shouldMaskReviewDetails || shouldLimitNonOwnerVisibility; + shouldMaskReviewDetails || + shouldLimitNonOwnerVisibility || + shouldMaskOtherReviewerDetails; const sanitizedReview: typeof review & { reviewItems?: typeof review.reviewItems; } = { @@ -3364,7 +3374,8 @@ export class ReviewService { const shouldStripReviewItems = shouldMaskReviewDetails || shouldTrimIterativeReviewForOtherSubmitters || - shouldLimitNonOwnerVisibility; + shouldLimitNonOwnerVisibility || + shouldMaskOtherReviewerDetails; if (!isThin) { sanitizedReview.reviewItems = shouldStripReviewItems diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index 8592c15..04ed2c7 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -14,6 +14,7 @@ jest.mock('nanoid', () => ({ describe('SubmissionService', () => { let service: SubmissionService; let resourceApiService: { getMemberResourcesRoles: jest.Mock }; + let resourcePrisma: { resource: { findMany: jest.Mock } }; let s3Send: jest.Mock; const submission = { id: 'submission-123', @@ -36,12 +37,18 @@ describe('SubmissionService', () => { resourceApiService = { getMemberResourcesRoles: jest.fn().mockResolvedValue([]), }; + resourcePrisma = { + resource: { + findMany: jest.fn().mockResolvedValue([]), + }, + }; service = new SubmissionService( {} as any, {} as any, {} as any, {} as any, resourceApiService as any, + resourcePrisma as any, {} as any, {} as any, {} as any, @@ -297,12 +304,18 @@ describe('SubmissionService', () => { resourceApiService = { getMemberResourcesRoles: jest.fn(), }; + resourcePrisma = { + resource: { + findMany: jest.fn().mockResolvedValue([]), + }, + }; service = new SubmissionService( prismaMock as any, {} as any, {} as any, challengeApiServiceMock as any, resourceApiService as any, + resourcePrisma as any, {} as any, {} as any, {} as any, @@ -506,6 +519,7 @@ describe('SubmissionService', () => { validateSubmitterRegistration: jest.Mock; getMemberResourcesRoles: jest.Mock; }; + let resourcePrismaListMock: { resource: { findMany: jest.Mock } }; let memberPrismaMock: { member: { findMany: jest.Mock } }; let listService: SubmissionService; @@ -537,13 +551,21 @@ describe('SubmissionService', () => { validateSubmitterRegistration: jest.fn(), getMemberResourcesRoles: jest.fn().mockResolvedValue([]), }; - memberPrismaMock = { member: { findMany: jest.fn() } }; + resourcePrismaListMock = { + resource: { + findMany: jest.fn().mockResolvedValue([]), + }, + }; + memberPrismaMock = { + member: { findMany: jest.fn().mockResolvedValue([]) }, + }; listService = new SubmissionService( prismaMock as any, prismaErrorServiceMock as any, challengePrismaMock as any, challengeApiServiceMock as any, resourceApiServiceListMock as any, + resourcePrismaListMock as any, {} as any, {} as any, memberPrismaMock as any, @@ -884,5 +906,134 @@ describe('SubmissionService', () => { expect(other?.review).toBeDefined(); }); + + it('masks other reviewers scores while preserving reviewer metadata on active challenges', async () => { + const now = new Date('2025-01-05T10:00:00Z'); + resourceApiServiceListMock.getMemberResourcesRoles.mockResolvedValue([ + { + roleName: 'Reviewer', + id: 'resource-self', + memberId: '101', + }, + ]); + + const submissions = [ + { + id: 'submission-1', + challengeId: 'challenge-1', + memberId: 'submitter-1', + submittedDate: now, + createdAt: now, + updatedAt: now, + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [ + { + id: 'review-self', + resourceId: 'resource-self', + submissionId: 'submission-1', + finalScore: 95, + initialScore: 92, + reviewItems: [ + { + id: 'item-self', + scorecardQuestionId: 'q1', + initialAnswer: 'YES', + finalAnswer: 'YES', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + }, + { + id: 'review-other', + resourceId: 'resource-other', + submissionId: 'submission-1', + finalScore: 80, + initialScore: 78, + reviewItems: [ + { + id: 'item-other', + scorecardQuestionId: 'q2', + initialAnswer: 'NO', + finalAnswer: 'NO', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'other-reviewer', + updatedAt: now, + updatedBy: 'other-reviewer', + }, + ], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + ]; + + prismaMock.submission.findMany.mockResolvedValue( + submissions.map((entry) => ({ ...entry })), + ); + prismaMock.submission.count.mockResolvedValue(submissions.length); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'submission-1', + }); + + resourcePrismaListMock.resource.findMany.mockResolvedValue([ + { id: 'resource-self', memberId: '101' }, + { id: 'resource-other', memberId: '202' }, + ]); + + memberPrismaMock.member.findMany.mockResolvedValue([ + { + userId: BigInt(101), + handle: 'selfHandle', + maxRating: { rating: 2500 }, + }, + { + userId: BigInt(202), + handle: 'otherHandle', + maxRating: { rating: 1800 }, + }, + ]); + + const result = await listService.listSubmission( + { + userId: '101', + isMachine: false, + roles: [UserRole.Reviewer], + } as any, + { challengeId: 'challenge-1' } as any, + { page: 1, perPage: 50 } as any, + ); + + const submissionResult = result.data.find( + (entry) => entry.id === 'submission-1', + ); + + expect(submissionResult).toBeDefined(); + const selfReview = submissionResult?.review?.find( + (review) => review.id === 'review-self', + ); + const otherReview = submissionResult?.review?.find( + (review) => review.id === 'review-other', + ); + + expect(selfReview?.initialScore).toBe(92); + expect(selfReview?.finalScore).toBe(95); + expect(selfReview?.reviewItems).toHaveLength(1); + expect(selfReview?.reviewerHandle).toBe('selfHandle'); + expect(selfReview?.reviewerMaxRating).toBe(2500); + + expect(otherReview?.initialScore).toBeNull(); + expect(otherReview?.finalScore).toBeNull(); + expect(otherReview?.reviewItems).toEqual([]); + expect(otherReview?.reviewerHandle).toBe('otherHandle'); + expect(otherReview?.reviewerMaxRating).toBe(1800); + }); }); }); diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index d89a7a7..b85312e 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -31,6 +31,7 @@ import { } from 'src/shared/modules/global/challenge.service'; import { ChallengeCatalogService } from 'src/shared/modules/global/challenge-catalog.service'; import { ResourceApiService } from 'src/shared/modules/global/resource.service'; +import { ResourcePrismaService } from 'src/shared/modules/global/resource-prisma.service'; import { ArtifactsCreateResponseDto } from 'src/dto/artifacts.dto'; import { randomUUID } from 'crypto'; import { basename } from 'path'; @@ -61,6 +62,18 @@ const REVIEW_ACCESS_ROLE_KEYWORDS = [ 'approval', ]; +const REVIEW_ITEM_COMMENTS_INCLUDE = { + reviewItemComments: { + include: { + appeal: { + include: { + appealResponse: true, + }, + }, + }, + }, +} as const; + @Injectable() export class SubmissionService { private readonly logger = new Logger(SubmissionService.name); @@ -71,6 +84,7 @@ export class SubmissionService { private readonly challengePrisma: ChallengePrismaService, private readonly challengeApiService: ChallengeApiService, private readonly resourceApiService: ResourceApiService, + private readonly resourcePrisma: ResourcePrismaService, private readonly eventBusService: EventBusService, private readonly challengeCatalogService: ChallengeCatalogService, private readonly memberPrisma: MemberPrismaService, @@ -1582,7 +1596,13 @@ export class SubmissionService { ...submissionWhereClause, }, include: { - review: {}, + review: { + include: { + reviewItems: { + include: REVIEW_ITEM_COMMENTS_INCLUDE, + }, + }, + }, reviewSummation: {}, }, skip, @@ -1636,6 +1656,7 @@ export class SubmissionService { } await this.applyReviewVisibilityFilters(authUser, submissions); + await this.enrichReviewerMetadata(submissions); // Count total entities matching the filter for pagination metadata const totalCount = await this.prisma.submission.count({ @@ -2064,7 +2085,12 @@ export class SubmissionService { const roleSummaryByChallenge = new Map< string, - { hasCopilot: boolean; hasReviewer: boolean; hasSubmitter: boolean } + { + hasCopilot: boolean; + hasReviewer: boolean; + hasSubmitter: boolean; + reviewerResourceIds: string[]; + } >(); await Promise.all( @@ -2086,6 +2112,7 @@ export class SubmissionService { let hasCopilot = false; let hasReviewer = false; let hasSubmitter = false; + const reviewerResourceIds: string[] = []; for (const resource of resources ?? []) { const roleName = (resource.roleName || '').toLowerCase(); @@ -2098,6 +2125,10 @@ export class SubmissionService { ) ) { hasReviewer = true; + const resourceId = String(resource.id ?? '').trim(); + if (resourceId && !reviewerResourceIds.includes(resourceId)) { + reviewerResourceIds.push(resourceId); + } } if ( resource.roleId === CommonConfig.roles.submitterRoleId || @@ -2111,6 +2142,7 @@ export class SubmissionService { hasCopilot, hasReviewer, hasSubmitter, + reviewerResourceIds, }); }), ); @@ -2139,13 +2171,50 @@ export class SubmissionService { hasCopilot: false, hasReviewer: false, hasSubmitter: false, + reviewerResourceIds: [], }; - if (roleSummary.hasCopilot || roleSummary.hasReviewer) { + const challenge = challengeDetails.get(challengeId); + + if (roleSummary.hasCopilot) { continue; } - const challenge = challengeDetails.get(challengeId); + if (roleSummary.hasReviewer) { + const reviews = Array.isArray((submission as any).review) + ? ((submission as any).review as Array>) + : []; + + if (reviews.length) { + const challengeCompletedOrCancelled = + this.isCompletedOrCancelledStatus(challenge?.status ?? null); + + if (!challengeCompletedOrCancelled) { + for (const review of reviews) { + if (!review || typeof review !== 'object') { + continue; + } + + const resourceId = String(review.resourceId ?? '').trim(); + const ownsReview = + resourceId.length > 0 && + roleSummary.reviewerResourceIds.includes(resourceId); + + if (!ownsReview) { + review.initialScore = null; + review.finalScore = null; + if (Array.isArray(review.reviewItems)) { + review.reviewItems = []; + } else { + review.reviewItems = []; + } + } + } + } + } + + continue; + } if (this.isCompletedOrCancelledStatus(challenge?.status ?? null)) { continue; @@ -2164,6 +2233,140 @@ export class SubmissionService { } } + private async enrichReviewerMetadata( + submissions: Array<{ review?: unknown }>, + ): Promise { + const reviews: Array> = []; + + for (const submission of submissions) { + const reviewList = Array.isArray((submission as any).review) + ? ((submission as any).review as Array>) + : []; + + for (const review of reviewList) { + if (review && typeof review === 'object') { + reviews.push(review); + } + } + } + + if (!reviews.length) { + return; + } + + const resourceIds = Array.from( + new Set( + reviews + .map((review) => String(review.resourceId ?? '').trim()) + .filter((id) => id.length > 0), + ), + ); + + if (!resourceIds.length) { + for (const review of reviews) { + if (!Object.prototype.hasOwnProperty.call(review, 'reviewerHandle')) { + review.reviewerHandle = null; + } + if ( + !Object.prototype.hasOwnProperty.call(review, 'reviewerMaxRating') + ) { + review.reviewerMaxRating = null; + } + } + return; + } + + try { + const resources = await this.resourcePrisma.resource.findMany({ + where: { id: { in: resourceIds } }, + select: { id: true, memberId: true }, + }); + + const memberIds = Array.from( + new Set( + resources + .map((resource) => String(resource.memberId ?? '').trim()) + .filter((id) => id.length > 0), + ), + ); + + const memberIdsAsBigInt: bigint[] = []; + for (const id of memberIds) { + try { + memberIdsAsBigInt.push(BigInt(id)); + } catch (error) { + this.logger.debug( + `[enrichReviewerMetadata] Skipping reviewer memberId ${id}: unable to convert to BigInt. ${error}`, + ); + } + } + + const memberInfoById = new Map< + string, + { handle: string | null; maxRating: number | null } + >(); + + if (memberIdsAsBigInt.length) { + const members = await this.memberPrisma.member.findMany({ + where: { userId: { in: memberIdsAsBigInt } }, + select: { + userId: true, + handle: true, + maxRating: { select: { rating: true } }, + }, + }); + + members.forEach((member) => { + memberInfoById.set(member.userId.toString(), { + handle: member.handle ?? null, + maxRating: member.maxRating?.rating ?? null, + }); + }); + } + + const profileByResourceId = new Map< + string, + { handle: string | null; maxRating: number | null } + >(); + + resources.forEach((resource) => { + const resourceId = String(resource.id ?? '').trim(); + if (!resourceId) { + return; + } + const memberId = String(resource.memberId ?? '').trim(); + const profile = memberInfoById.get(memberId) ?? { + handle: null, + maxRating: null, + }; + profileByResourceId.set(resourceId, profile); + }); + + for (const review of reviews) { + const resourceId = String(review.resourceId ?? '').trim(); + const profile = resourceId ? profileByResourceId.get(resourceId) : null; + review.reviewerHandle = profile?.handle ?? null; + review.reviewerMaxRating = profile?.maxRating ?? null; + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.warn( + `[enrichReviewerMetadata] Failed to enrich reviewer metadata: ${message}`, + ); + } finally { + for (const review of reviews) { + if (!Object.prototype.hasOwnProperty.call(review, 'reviewerHandle')) { + review.reviewerHandle = null; + } + if ( + !Object.prototype.hasOwnProperty.call(review, 'reviewerMaxRating') + ) { + review.reviewerMaxRating = null; + } + } + } + } + private async populateLatestSubmissionFlags( submissions: Array<{ id: string; From 9d445c88110d192d9dcedd3bb81bb0db5eb77c92 Mon Sep 17 00:00:00 2001 From: Kiril Kartunov Date: Fri, 24 Oct 2025 11:27:30 +0300 Subject: [PATCH 37/48] adds Trivy action --- .github/workflows/trivy.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/trivy.yaml diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 0000000..85b3cad --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,30 @@ +name: Trivy Scanner +on: + push: + branches: + - main + - dev + pull_request: +jobs: + trivy-scan: + name: Use Trivy + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Trivy scanner in repo mode + uses: aquasecurity/trivy-action@0.33.1 + with: + scan-type: 'fs' + ignore-unfixed: true + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH,UNKNOWN' + scanners: vuln,secret,misconfig,license + github-pat: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' \ No newline at end of file From 929215316ce14ca978eb884e02f83a74f3e806a7 Mon Sep 17 00:00:00 2001 From: Hentry Martin Date: Sat, 25 Oct 2025 00:45:02 +0200 Subject: [PATCH 38/48] fix: added timeout for prisma --- .env.sample | 2 ++ src/shared/modules/global/challenge-prisma.service.ts | 5 +++++ src/shared/modules/global/member-prisma.service.ts | 5 +++++ src/shared/modules/global/prisma.service.ts | 5 +++++ src/shared/modules/global/resource-prisma.service.ts | 5 +++++ 5 files changed, 22 insertions(+) diff --git a/.env.sample b/.env.sample index ef398d3..4698fae 100644 --- a/.env.sample +++ b/.env.sample @@ -73,3 +73,5 @@ SENDGRID_ACCEPT_REVIEW_APPLICATION="d-2de72880bd69499e9c16369398d34bb9" SENDGRID_REJECT_REVIEW_APPLICATION="d-82ed74e778e84d8c9bc02eeda0f44b5e" # For pulling payment details (used by platform-ui) FINANCE_DB_URL= +#Prisma timeout +REVIEW_SERVICE_PRISMA_TIMEOUT=10000 \ No newline at end of file diff --git a/src/shared/modules/global/challenge-prisma.service.ts b/src/shared/modules/global/challenge-prisma.service.ts index 8f32b4f..378344f 100644 --- a/src/shared/modules/global/challenge-prisma.service.ts +++ b/src/shared/modules/global/challenge-prisma.service.ts @@ -11,6 +11,11 @@ export class ChallengePrismaService constructor() { super({ + transactionOptions: { + timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT + ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) + : 10000, + }, log: [ { level: 'query', emit: 'event' }, { level: 'info', emit: 'event' }, diff --git a/src/shared/modules/global/member-prisma.service.ts b/src/shared/modules/global/member-prisma.service.ts index cc1909b..3c71cdc 100644 --- a/src/shared/modules/global/member-prisma.service.ts +++ b/src/shared/modules/global/member-prisma.service.ts @@ -11,6 +11,11 @@ export class MemberPrismaService constructor() { super({ + transactionOptions: { + timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT + ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) + : 10000, + }, log: [ { level: 'query', emit: 'event' }, { level: 'info', emit: 'event' }, diff --git a/src/shared/modules/global/prisma.service.ts b/src/shared/modules/global/prisma.service.ts index 404bb7c..c1e1d00 100644 --- a/src/shared/modules/global/prisma.service.ts +++ b/src/shared/modules/global/prisma.service.ts @@ -197,6 +197,11 @@ export class PrismaService const schema = process.env.POSTGRES_SCHEMA || 'public'; super({ + transactionOptions: { + timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT + ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) + : 10000, + }, log: [ { level: 'query', emit: 'event' }, { level: 'info', emit: 'event' }, diff --git a/src/shared/modules/global/resource-prisma.service.ts b/src/shared/modules/global/resource-prisma.service.ts index 6d8a275..2996647 100644 --- a/src/shared/modules/global/resource-prisma.service.ts +++ b/src/shared/modules/global/resource-prisma.service.ts @@ -11,6 +11,11 @@ export class ResourcePrismaService constructor() { super({ + transactionOptions: { + timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT + ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) + : 10000, + }, log: [ { level: 'query', emit: 'event' }, { level: 'info', emit: 'event' }, From f8f667dd028b5ae13a248d42607c52adb22a5f32 Mon Sep 17 00:00:00 2001 From: Hentry Martin Date: Sat, 25 Oct 2025 00:45:15 +0200 Subject: [PATCH 39/48] fix: added timeout for prisma --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b7a392..7a974fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,6 +77,7 @@ workflows: - feat/ai-workflows - pm-1955_2 - re-try-failed-jobs + - pm-2539 - 'build-prod': From 77db0d116907f222793b0e0c4edd0a5444c8c2b1 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Mon, 27 Oct 2025 10:31:57 +1100 Subject: [PATCH 40/48] Additional visibility tweaks and fixes for proper phase handling --- prisma/schema.prisma | 2 + src/api/review/review.service.ts | 491 ++++++-- src/api/submission/submission.service.spec.ts | 65 +- src/api/submission/submission.service.ts | 1075 ++++++++++++++++- src/dto/review.dto.ts | 10 + 5 files changed, 1492 insertions(+), 151 deletions(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d628d49..eb41225 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -33,6 +33,7 @@ model scorecard { scorecardGroups scorecardGroup[] reviews review[] + reviewSummations reviewSummation[] aiWorkflow aiWorkflow[] // Indexes for faster searches @@ -373,6 +374,7 @@ model reviewSummation { metadata Json? submission submission @relation(fields: [submissionId], references: [id], onDelete: Cascade) + scorecard scorecard? @relation(fields: [scorecardId], references: [id], onDelete: Cascade) @@index([submissionId]) // Index for joining with submission table @@index([scorecardId]) // Index for joining with scorecard table diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 34cda02..d185096 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -19,7 +19,7 @@ import { mapReviewItemRequestToDto, mapReviewRequestToDto, } from 'src/dto/review.dto'; -import { Prisma } from '@prisma/client'; +import { Prisma, ScorecardType, SubmissionType } from '@prisma/client'; import { PaginatedResponse, PaginationDto } from 'src/dto/pagination.dto'; import { PrismaService } from 'src/shared/modules/global/prisma.service'; import { PrismaErrorService } from 'src/shared/modules/global/prisma-error.service'; @@ -2572,13 +2572,12 @@ export class ReviewService { const reviewerResourceIdSet = new Set(); const submitterSubmissionIdSet = new Set(); let hasCopilotRoleForChallenge = false; - let hasReviewerRoleForChallenge = false; let hasSubmitterRoleForChallenge = false; let submitterVisibilityState = { allowAny: false, allowOwn: false, }; - let allowLimitedVisibilityForOtherSubmissions = false; + const allowLimitedVisibilityForOtherSubmissions = false; // Utility to merge an allowed set of submission IDs into where clause const restrictToSubmissionIds = (allowedIds: string[]) => { @@ -2740,9 +2739,6 @@ export class ReviewService { ) { reviewerResourceIdSet.add(r.id); } - if (roleName.includes('reviewer')) { - hasReviewerRoleForChallenge = true; - } }); hasCopilotRoleForChallenge = normalized.some((r) => (r.roleName || '').toLowerCase().includes('copilot'), @@ -2765,6 +2761,9 @@ export class ReviewService { // Copilots retain full visibility for the challenge } else if (reviewerResourceIdSet.size) { const reviewerResourceIds = Array.from(reviewerResourceIdSet); + const reviewerResources = normalized.filter((resource) => + reviewerResourceIdSet.has(resource.id), + ); let challengeCompletedOrCancelled = false; if (challengeId && !challengeDetail) { try { @@ -2789,19 +2788,35 @@ export class ReviewService { if (challengeCompletedOrCancelled) { // Completed or cancelled challenges should expose all reviews to reviewers. - } else if (hasReviewerRoleForChallenge && challengeId) { - const screeningPhaseIds = challengeDetail - ? this.getPhaseIdsForNames(challengeDetail, [ - 'screening', - 'checkpoint screening', - ]) - : []; - - if (screeningPhaseIds.length) { - reviewWhereClause.OR = [ - { resourceId: { in: reviewerResourceIds } }, - { phaseId: { in: screeningPhaseIds } }, - ]; + } else if (challengeId) { + const reviewerRoleFilter = this.buildReviewerRoleFilters( + reviewerResources, + challengeDetail, + challengeCompletedOrCancelled, + ); + + if (reviewerRoleFilter) { + const roleOr = reviewerRoleFilter.OR; + if (roleOr) { + const normalizedOr = Array.isArray(roleOr) + ? roleOr + : [roleOr]; + const existingOr = reviewWhereClause.OR; + if (Array.isArray(existingOr)) { + reviewWhereClause.OR = [...existingOr, ...normalizedOr]; + } else if (existingOr) { + reviewWhereClause.OR = [existingOr, ...normalizedOr]; + } else { + reviewWhereClause.OR = normalizedOr; + } + } + + Object.entries(reviewerRoleFilter).forEach(([key, value]) => { + if (key === 'OR' || value === undefined) { + return; + } + reviewWhereClause[key] = value; + }); } else { restrictToResourceIds(reviewerResourceIds); } @@ -2839,9 +2854,16 @@ export class ReviewService { (p.name || '').toLowerCase() === 'submission' && p.isOpen === false, ); - const screeningPhaseCompleted = this.hasChallengePhaseCompleted( + const submitterReviewPhaseNames = [ + 'checkpoint screening', + 'checkpoint review', + 'screening', + 'review', + 'iterative review', + ]; + const reviewPhasesCompleted = this.hasChallengePhaseCompleted( challenge, - ['screening'], + submitterReviewPhaseNames, ); const challengeCompletedOrCancelled = this.isCompletedOrCancelledStatus(challenge.status); @@ -2856,24 +2878,29 @@ export class ReviewService { } if (challengeCompletedOrCancelled) { - // Allowed to see all reviews on this challenge - // reviewWhereClause already limited to submissions on this challenge + const hasPassingSubmission = + await this.hasPassingSubmissionForReviewScorecard( + challengeId, + uid, + ); + + if (!hasPassingSubmission) { + restrictToSubmissionIds(mySubmissionIds); + } } else if (isMarathonMatch) { - if (appealsOpen || appealsResponseOpen) { - // Marathon matches retain limited visibility for other submissions during appeals phases. - allowLimitedVisibilityForOtherSubmissions = true; - } else if ( - challenge.status === ChallengeStatus.ACTIVE && - submissionPhaseClosed && - hasSubmitterRoleForChallenge + if ( + appealsOpen || + appealsResponseOpen || + (challenge.status === ChallengeStatus.ACTIVE && + submissionPhaseClosed && + hasSubmitterRoleForChallenge) ) { - // Allow limited visibility once marathon match submission phase closes. - allowLimitedVisibilityForOtherSubmissions = true; + restrictToSubmissionIds(mySubmissionIds); } else if ( hasSubmitterRoleForChallenge && - screeningPhaseCompleted + reviewPhasesCompleted ) { - // Marathon submitters can still inspect their own reviews once screening completes. + // Marathon submitters can still inspect their own reviews once an allowed review phase completes. restrictToSubmissionIds(mySubmissionIds); } else { this.logger.debug( @@ -2886,7 +2913,7 @@ export class ReviewService { (appealsOpen || appealsResponseOpen || submissionPhaseClosed || - screeningPhaseCompleted) + reviewPhasesCompleted) ) { // Non-marathon submitters can only inspect their own submissions until completion/cancellation. restrictToSubmissionIds(mySubmissionIds); @@ -2926,12 +2953,18 @@ export class ReviewService { const challenges = await this.challengeApiService.getChallenges(myChallengeIds); const completedIds = challenges - .filter((c) => - [ - ChallengeStatus.COMPLETED, - ChallengeStatus.CANCELLED_FAILED_REVIEW, - ].includes(c.status), - ) + .filter((challenge) => { + const status = challenge.status; + if (!status) { + return false; + } + const statusString = String(status); + return ( + status === ChallengeStatus.COMPLETED || + status === ChallengeStatus.CANCELLED || + statusString.startsWith('CANCELLED_') + ); + }) .map((c) => c.id); const appealsAllowedIds = challenges .filter((c) => { @@ -2945,14 +2978,40 @@ export class ReviewService { .map((c) => c.id); const allowed = new Set(); + const mySubsByChallenge = new Map(); - // For completed challenges, allow all submissions in those challenges - if (completedIds.length) { - const subs = await this.prisma.submission.findMany({ - where: { challengeId: { in: completedIds } }, - select: { id: true }, - }); - subs.forEach((s) => allowed.add(s.id)); + for (const sub of mySubs) { + const subChallengeId = sub.challengeId; + if (!subChallengeId) { + continue; + } + const existing = mySubsByChallenge.get(subChallengeId); + if (existing) { + existing.push(sub.id); + } else { + mySubsByChallenge.set(subChallengeId, [sub.id]); + } + } + + // For completed challenges, allow all submissions only when the member has a passing submission + for (const completedId of completedIds) { + const hasPassingSubmission = + await this.hasPassingSubmissionForReviewScorecard( + completedId, + uid, + ); + + if (hasPassingSubmission) { + const subs = await this.prisma.submission.findMany({ + where: { challengeId: completedId }, + select: { id: true }, + }); + subs.forEach((s) => allowed.add(s.id)); + } else { + const ownSubmissions = + mySubsByChallenge.get(completedId) ?? []; + ownSubmissions.forEach((id) => allowed.add(id)); + } } // For appeals or appeals response, allow only the user's own submissions @@ -3283,19 +3342,27 @@ export class ReviewService { challengeForReview, ['screening'], ); - const checkpointReviewPhaseCompleted = - this.hasChallengePhaseClosedWithActualDates(challengeForReview, [ - 'checkpoint review', - ]); + const checkpointReviewPhaseCompleted = this.hasChallengePhaseCompleted( + challengeForReview, + ['checkpoint review'], + ); + const reviewPhaseCompleted = this.hasChallengePhaseCompleted( + challengeForReview, + ['review'], + ); + const iterativeReviewPhaseCompleted = this.hasChallengePhaseCompleted( + challengeForReview, + ['iterative review'], + ); const phaseNamesForCompletionCheck: string[] = []; if (phaseName && phaseName.trim().length > 0) { phaseNamesForCompletionCheck.push(phaseName); } else if (normalizedPhaseName.length > 0) { phaseNamesForCompletionCheck.push(normalizedPhaseName); } - const phaseClosedWithActualDates = + const phaseCompletedForResolvedNames = phaseNamesForCompletionCheck.length > 0 && - this.hasChallengePhaseClosedWithActualDates( + this.hasChallengePhaseCompleted( challengeForReview, phaseNamesForCompletionCheck, ); @@ -3315,16 +3382,34 @@ export class ReviewService { isOwnSubmission && checkpointReviewPhaseCompleted && normalizedPhaseName === 'checkpoint review'; + const allowOwnReviewVisibility = + !isPrivilegedRequester && + hasSubmitterRoleForChallenge && + !hasCopilotRoleForChallenge && + !isReviewerForReview && + isOwnSubmission && + reviewPhaseCompleted && + normalizedPhaseName === 'review'; + const allowOwnIterativeReviewVisibility = + !isPrivilegedRequester && + hasSubmitterRoleForChallenge && + !hasCopilotRoleForChallenge && + !isReviewerForReview && + isOwnSubmission && + iterativeReviewPhaseCompleted && + normalizedPhaseName === 'iterative review'; const allowOwnClosedPhaseVisibility = !isPrivilegedRequester && hasSubmitterRoleForChallenge && !hasCopilotRoleForChallenge && !isReviewerForReview && isOwnSubmission && - phaseClosedWithActualDates; + phaseCompletedForResolvedNames; const shouldMaskReviewDetails = !allowOwnScreeningVisibility && !allowOwnCheckpointReviewVisibility && + !allowOwnReviewVisibility && + !allowOwnIterativeReviewVisibility && !allowOwnClosedPhaseVisibility && !isPrivilegedRequester && hasSubmitterRoleForChallenge && @@ -3667,13 +3752,11 @@ export class ReviewService { const canSeeOwnCheckpointReview = isOwnSubmission && normalizedPhaseNameForAccess === 'checkpoint review' && - this.hasChallengePhaseClosedWithActualDates(challenge, [ - 'checkpoint review', - ]); + this.hasChallengePhaseCompleted(challenge, ['checkpoint review']); const canSeeOwnClosedPhaseReview = isOwnSubmission && phaseNamesForAccessCheck.length > 0 && - this.hasChallengePhaseClosedWithActualDates( + this.hasChallengePhaseCompleted( challenge, phaseNamesForAccessCheck, ); @@ -3925,55 +4008,6 @@ export class ReviewService { }); } - private hasChallengePhaseClosedWithActualDates( - challenge: ChallengeData | null | undefined, - phaseNames: string[], - ): boolean { - if (!challenge?.phases?.length) { - return false; - } - - const normalizedTargets = new Set( - (phaseNames ?? []) - .map((name) => this.normalizePhaseName(name)) - .filter((name) => name.length > 0), - ); - - if (!normalizedTargets.size) { - return false; - } - - return (challenge.phases ?? []).some((phase) => { - if (!phase) { - return false; - } - - const normalizedName = this.normalizePhaseName((phase as any).name); - if (!normalizedTargets.has(normalizedName)) { - return false; - } - - if ((phase as any).isOpen === true) { - return false; - } - - const actualStart = - (phase as any).actualStartTime ?? - (phase as any).actualStartDate ?? - (phase as any).actualStart ?? - null; - const actualEnd = - (phase as any).actualEndTime ?? - (phase as any).actualEndDate ?? - (phase as any).actualEnd ?? - null; - - return ( - this.hasTimestampValue(actualStart) && this.hasTimestampValue(actualEnd) - ); - }); - } - private getPhaseIdsForNames( challenge: ChallengeData | null | undefined, phaseNames: string[], @@ -4107,6 +4141,44 @@ export class ReviewService { return legacyTrack.includes('marathon'); } + private async hasPassingSubmissionForReviewScorecard( + challengeId: string, + memberId: string, + ): Promise { + const normalizedChallengeId = String(challengeId ?? '').trim(); + const normalizedMemberId = String(memberId ?? '').trim(); + + if (!normalizedChallengeId || !normalizedMemberId) { + return false; + } + + try { + const passingSummation = await this.prisma.reviewSummation.findFirst({ + where: { + isPassing: true, + scorecard: { + type: { + in: [ScorecardType.REVIEW, ScorecardType.ITERATIVE_REVIEW], + }, + }, + submission: { + challengeId: normalizedChallengeId, + memberId: normalizedMemberId, + }, + }, + select: { id: true }, + }); + + return Boolean(passingSummation); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.warn( + `[hasPassingSubmissionForReviewScorecard] Failed to check passing submission for challenge ${normalizedChallengeId}, member ${normalizedMemberId}: ${message}`, + ); + return false; + } + } + private isCompletedOrCancelledStatus( status?: ChallengeStatus | null, ): boolean { @@ -4122,6 +4194,209 @@ export class ReviewService { return String(status).startsWith('CANCELLED_'); } + private identifyReviewerRoleType( + roleName: string, + ): + | 'screener' + | 'checkpoint-screener' + | 'checkpoint-reviewer' + | 'reviewer' + | 'approver' + | 'iterative-reviewer' + | 'unknown' { + const normalized = (roleName ?? '').toLowerCase().trim(); + + if (!normalized.length) { + return 'unknown'; + } + + if (normalized.includes('checkpoint') && normalized.includes('screener')) { + return 'checkpoint-screener'; + } + + if (normalized.includes('checkpoint') && normalized.includes('reviewer')) { + return 'checkpoint-reviewer'; + } + + if (normalized.includes('screener') && !normalized.includes('checkpoint')) { + return 'screener'; + } + + if (normalized.includes('approver') || normalized.includes('approval')) { + return 'approver'; + } + + if (normalized.includes('iterative') && normalized.includes('reviewer')) { + return 'iterative-reviewer'; + } + + if ( + normalized.includes('reviewer') && + !normalized.includes('checkpoint') && + !normalized.includes('iterative') + ) { + return 'reviewer'; + } + + return 'unknown'; + } + + private buildReviewerRoleFilters( + resources: ResourceInfo[], + challengeDetail: ChallengeData | null, + challengeCompletedOrCancelled: boolean, + ): Prisma.reviewWhereInput | null { + if (challengeCompletedOrCancelled) { + return null; + } + + const allReviewerResourceIds = (resources ?? []) + .map((resource) => String(resource?.id ?? '').trim()) + .filter((id) => id.length > 0); + + if (!challengeDetail) { + return allReviewerResourceIds.length + ? { resourceId: { in: allReviewerResourceIds } } + : null; + } + + const groupedByRole = new Map< + ReturnType, + ResourceInfo[] + >(); + + for (const resource of resources ?? []) { + if (!resource) { + continue; + } + const roleType = this.identifyReviewerRoleType(resource.roleName ?? ''); + if (roleType === 'unknown') { + continue; + } + + const existing = groupedByRole.get(roleType); + if (existing) { + existing.push(resource); + } else { + groupedByRole.set(roleType, [resource]); + } + } + + const filters: Prisma.reviewWhereInput[] = []; + + if (groupedByRole.has('screener')) { + const screeningPhaseIds = this.getPhaseIdsForNames(challengeDetail, [ + 'screening', + ]); + if (screeningPhaseIds.length) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CONTEST_SUBMISSION } }, + { phaseId: { in: screeningPhaseIds } }, + ], + }); + } + } + + if (groupedByRole.has('checkpoint-screener')) { + const checkpointScreeningPhaseIds = this.getPhaseIdsForNames( + challengeDetail, + ['checkpoint screening'], + ); + if (checkpointScreeningPhaseIds.length) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CHECKPOINT_SUBMISSION } }, + { phaseId: { in: checkpointScreeningPhaseIds } }, + ], + }); + } + } + + if (groupedByRole.has('checkpoint-reviewer')) { + const checkpointReviewPhaseIds = this.getPhaseIdsForNames( + challengeDetail, + ['checkpoint review'], + ); + if (checkpointReviewPhaseIds.length) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CHECKPOINT_SUBMISSION } }, + { phaseId: { in: checkpointReviewPhaseIds } }, + ], + }); + } + } + + if (groupedByRole.has('reviewer')) { + const reviewPhaseIds = this.getPhaseIdsForNames(challengeDetail, [ + 'review', + ]); + const reviewerResourceIds = (groupedByRole.get('reviewer') ?? []) + .map((resource) => String(resource?.id ?? '').trim()) + .filter((id) => id.length > 0); + if (reviewPhaseIds.length && reviewerResourceIds.length) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CONTEST_SUBMISSION } }, + { phaseId: { in: reviewPhaseIds } }, + { resourceId: { in: reviewerResourceIds } }, + ], + }); + } + } + + if (groupedByRole.has('iterative-reviewer')) { + const iterativeReviewPhaseIds = this.getPhaseIdsForNames( + challengeDetail, + ['iterative review'], + ); + const iterativeReviewerResourceIds = ( + groupedByRole.get('iterative-reviewer') ?? [] + ) + .map((resource) => String(resource?.id ?? '').trim()) + .filter((id) => id.length > 0); + if ( + iterativeReviewPhaseIds.length && + iterativeReviewerResourceIds.length + ) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CONTEST_SUBMISSION } }, + { phaseId: { in: iterativeReviewPhaseIds } }, + { resourceId: { in: iterativeReviewerResourceIds } }, + ], + }); + } + } + + if (groupedByRole.has('approver')) { + const approvalPhaseIds = this.getPhaseIdsForNames(challengeDetail, [ + 'approval', + ]); + const approverResourceIds = (groupedByRole.get('approver') ?? []) + .map((resource) => String(resource?.id ?? '').trim()) + .filter((id) => id.length > 0); + if (approvalPhaseIds.length && approverResourceIds.length) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CONTEST_SUBMISSION } }, + { phaseId: { in: approvalPhaseIds } }, + { resourceId: { in: approverResourceIds } }, + ], + }); + } + } + + if (filters.length) { + return { OR: filters }; + } + + return allReviewerResourceIds.length + ? { resourceId: { in: allReviewerResourceIds } } + : null; + } + private shouldEnforceLatestSubmissionForReview( reviewTypeName: string | null | undefined, challenge: ChallengeData | null | undefined, diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index 04ed2c7..09149fc 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -506,6 +506,9 @@ describe('SubmissionService', () => { count: jest.Mock; findFirst: jest.Mock; }; + reviewType: { + findMany: jest.Mock; + }; }; let prismaErrorServiceMock: { handleError: jest.Mock }; let challengePrismaMock: { @@ -530,6 +533,9 @@ describe('SubmissionService', () => { count: jest.fn(), findFirst: jest.fn(), }, + reviewType: { + findMany: jest.fn().mockResolvedValue([]), + }, }; prismaErrorServiceMock = { handleError: jest.fn(), @@ -543,7 +549,13 @@ describe('SubmissionService', () => { status: ChallengeStatus.ACTIVE, type: 'Challenge', legacy: {}, - phases: [], + phases: [ + { + id: 'phase-123', + phaseId: 'legacy-phase-123', + name: 'Review Phase', + }, + ], }), getChallenges: jest.fn(), }; @@ -637,6 +649,57 @@ describe('SubmissionService', () => { ]); }); + it('enriches reviews with review type names when typeId is present', async () => { + const submissions = [ + { + id: 'submission-1', + challengeId: 'challenge-1', + memberId: 'member-1', + submittedDate: new Date('2024-01-01T10:00:00Z'), + createdAt: new Date('2024-01-01T10:00:00Z'), + updatedAt: new Date('2024-01-01T10:00:00Z'), + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [ + { + id: 'review-1', + typeId: 'type-123', + resourceId: 'resource-1', + phaseId: 'phase-123', + reviewItems: [], + }, + ], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + ]; + + prismaMock.submission.findMany.mockResolvedValue( + submissions.map((entry) => ({ ...entry })), + ); + prismaMock.submission.count.mockResolvedValue(submissions.length); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'submission-1', + }); + prismaMock.reviewType.findMany.mockResolvedValue([ + { id: 'type-123', name: 'Iterative Review' }, + ]); + + const result = await listService.listSubmission( + { isMachine: false, roles: [UserRole.Admin] } as any, + { challengeId: 'challenge-1' } as any, + { page: 1, perPage: 20 } as any, + ); + + expect(result.data[0].review?.[0]?.reviewType).toBe('Iterative Review'); + expect(result.data[0].review?.[0]?.phaseName).toBe('Review Phase'); + expect(prismaMock.reviewType.findMany).toHaveBeenCalledWith({ + where: { id: { in: ['type-123'] } }, + select: { id: true, name: true }, + }); + }); + it('omits isLatest when submission metadata indicates unlimited submissions', async () => { challengePrismaMock.$queryRaw.mockResolvedValue([ { diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index b85312e..afcbdb0 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -6,7 +6,11 @@ import { BadRequestException, ForbiddenException, } from '@nestjs/common'; -import { SubmissionStatus, SubmissionType } from '@prisma/client'; +import { + SubmissionStatus, + SubmissionType, + ScorecardType, +} from '@prisma/client'; import { PaginationDto } from 'src/dto/pagination.dto'; import { ReviewResponseDto } from 'src/dto/review.dto'; import { SortDto } from 'src/dto/sort.dto'; @@ -55,6 +59,26 @@ type SubmissionMinimal = { url: string | null; }; +type ChallengeRoleSummary = { + hasCopilot: boolean; + hasReviewer: boolean; + hasSubmitter: boolean; + reviewerResourceIds: string[]; +}; + +type ReviewVisibilityContext = { + roleSummaryByChallenge: Map; + challengeDetailsById: Map; + requesterUserId: string; +}; + +const EMPTY_ROLE_SUMMARY: ChallengeRoleSummary = { + hasCopilot: false, + hasReviewer: false, + hasSubmitter: false, + reviewerResourceIds: [], +}; + const REVIEW_ACCESS_ROLE_KEYWORDS = [ 'reviewer', 'screener', @@ -508,8 +532,6 @@ export class SubmissionService { let isReviewer = false; let isCopilot = false; let isSubmitter = false; - const isCheckpointSubmission = - submission.type === SubmissionType.CHECKPOINT_SUBMISSION; if (!isOwner && submission.challengeId && uid) { try { const resources = @@ -518,18 +540,27 @@ export class SubmissionService { uid, ); for (const r of resources) { - const rn = (r.roleName || '').toLowerCase(); - if (rn.includes('reviewer')) { - isReviewer = true; - } - if (rn.includes('screener')) { - const roleIsCheckpoint = rn.includes('checkpoint'); - if ( - (isCheckpointSubmission && roleIsCheckpoint) || - (!isCheckpointSubmission && !roleIsCheckpoint) - ) { - isReviewer = true; - } + const roleName = r.roleName || ''; + const rn = roleName.toLowerCase(); + const roleType = this.identifyReviewerRoleType(roleName); + + switch (roleType) { + case 'screener': + case 'reviewer': + case 'iterative-reviewer': + case 'approver': + if (submission.type === SubmissionType.CONTEST_SUBMISSION) { + isReviewer = true; + } + break; + case 'checkpoint-screener': + case 'checkpoint-reviewer': + if (submission.type === SubmissionType.CHECKPOINT_SUBMISSION) { + isReviewer = true; + } + break; + default: + break; } if (rn.includes('copilot')) { isCopilot = true; @@ -1590,11 +1621,64 @@ export class SubmissionService { submissionWhereClause.submissionPhaseId = queryDto.submissionPhaseId; } + const isPrivilegedRequester = authUser?.isMachine || isAdmin(authUser); + const requesterUserId = + authUser?.userId !== undefined && authUser?.userId !== null + ? String(authUser.userId) + : ''; + + let restrictedChallengeIds = new Set(); + if (!isPrivilegedRequester && requesterUserId) { + try { + restrictedChallengeIds = + await this.getActiveSubmitterRestrictedChallengeIds( + requesterUserId, + queryDto.challengeId, + ); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.debug( + `[listSubmission] Unable to resolve submitter visibility restrictions for member ${requesterUserId}: ${message}`, + ); + } + } + + const whereClause: Prisma.submissionWhereInput = { + ...submissionWhereClause, + }; + + if ( + !isPrivilegedRequester && + requesterUserId && + restrictedChallengeIds.size + ) { + const restrictedList = Array.from(restrictedChallengeIds); + const restrictionCriteria: Prisma.submissionWhereInput = { + OR: [ + { + AND: [ + { challengeId: { in: restrictedList } }, + { memberId: requesterUserId }, + ], + }, + { challengeId: { notIn: restrictedList } }, + { challengeId: null }, + ], + }; + + if (Array.isArray(whereClause.AND)) { + whereClause.AND = [...whereClause.AND, restrictionCriteria]; + } else if (whereClause.AND) { + whereClause.AND = [whereClause.AND, restrictionCriteria]; + } else { + whereClause.AND = [restrictionCriteria]; + } + } + // find entities by filters - const submissions = await this.prisma.submission.findMany({ - where: { - ...submissionWhereClause, - }, + let submissions = await this.prisma.submission.findMany({ + where: whereClause, include: { review: { include: { @@ -1655,17 +1739,39 @@ export class SubmissionService { } } - await this.applyReviewVisibilityFilters(authUser, submissions); + const reviewVisibilityContext = await this.applyReviewVisibilityFilters( + authUser, + submissions, + ); + const filtered = this.filterSubmissionsForActiveSubmitters( + authUser, + submissions, + reviewVisibilityContext, + ); + submissions = filtered.submissions; + await this.populateReviewPhaseNames(submissions); + await this.populateReviewTypeNames(submissions); await this.enrichReviewerMetadata(submissions); // Count total entities matching the filter for pagination metadata - const totalCount = await this.prisma.submission.count({ - where: { - ...submissionWhereClause, - }, + let totalCount = await this.prisma.submission.count({ + where: whereClause, }); + if (filtered.filteredOut) { + totalCount = submissions.length; + } await this.populateLatestSubmissionFlags(submissions); + this.stripSubmitterSubmissionDetails( + authUser, + submissions, + reviewVisibilityContext, + ); + this.stripSubmitterMemberIds( + authUser, + submissions, + reviewVisibilityContext, + ); await this.stripIsLatestForUnlimitedChallenges(submissions); this.logger.log( @@ -2017,6 +2123,8 @@ export class SubmissionService { details: { submissionId: id }, }); } + await this.populateReviewPhaseNames([data]); + await this.populateReviewTypeNames([data]); return data; } @@ -2027,14 +2135,27 @@ export class SubmissionService { memberId?: string | null; review?: unknown; }>, - ): Promise { + ): Promise { + const emptyContext: ReviewVisibilityContext = { + roleSummaryByChallenge: new Map(), + challengeDetailsById: new Map(), + requesterUserId: '', + }; + if (!submissions.length) { - return; + return emptyContext; } const isPrivilegedRequester = authUser?.isMachine || isAdmin(authUser); if (isPrivilegedRequester) { - return; + const requesterUserId = + authUser?.userId !== undefined && authUser?.userId !== null + ? String(authUser.userId).trim() + : ''; + return { + ...emptyContext, + requesterUserId, + }; } const uid = @@ -2043,7 +2164,7 @@ export class SubmissionService { : ''; if (!uid) { - return; + return emptyContext; } const challengeIds = Array.from( @@ -2061,10 +2182,14 @@ export class SubmissionService { ); if (!challengeIds.length) { - return; + return { + ...emptyContext, + requesterUserId: uid, + }; } const challengeDetails = new Map(); + const passingSubmissionCache = new Map(); await Promise.all( challengeIds.map(async (challengeId) => { @@ -2083,15 +2208,7 @@ export class SubmissionService { }), ); - const roleSummaryByChallenge = new Map< - string, - { - hasCopilot: boolean; - hasReviewer: boolean; - hasSubmitter: boolean; - reviewerResourceIds: string[]; - } - >(); + const roleSummaryByChallenge = new Map(); await Promise.all( challengeIds.map(async (challengeId) => { @@ -2163,9 +2280,6 @@ export class SubmissionService { const isOwnSubmission = submission.memberId != null && String(submission.memberId).trim() === uid; - if (isOwnSubmission) { - continue; - } const roleSummary = roleSummaryByChallenge.get(challengeId) ?? { hasCopilot: false, @@ -2180,6 +2294,101 @@ export class SubmissionService { continue; } + if (isOwnSubmission) { + const reviews = Array.isArray((submission as any).review) + ? ((submission as any).review as Array>) + : []; + + if (!reviews.length) { + continue; + } + + const allowedPhaseNames = [ + 'checkpoint screening', + 'checkpoint review', + 'screening', + 'review', + 'iterative review', + ]; + const normalizedAllowedPhases = new Set( + allowedPhaseNames.map((name) => this.normalizePhaseName(name)), + ); + const phaseCompletionCache = new Map(); + const getPhaseCompletion = ( + phaseName: string | null | undefined, + ): boolean => { + const normalized = this.normalizePhaseName(phaseName); + if (!normalized.length) { + return false; + } + if (phaseCompletionCache.has(normalized)) { + return phaseCompletionCache.get(normalized) ?? false; + } + if (!challenge) { + phaseCompletionCache.set(normalized, false); + return false; + } + const candidates: string[] = []; + if (phaseName && String(phaseName).trim().length > 0) { + candidates.push(String(phaseName)); + } + if (!candidates.includes(normalized)) { + candidates.push(normalized); + } + const completed = this.hasChallengePhaseCompleted( + challenge, + candidates, + ); + phaseCompletionCache.set(normalized, completed); + return completed; + }; + + const challengeForPhaseResolution = challenge ?? null; + const filteredReviews: Array> = []; + + for (const review of reviews) { + if (!review || typeof review !== 'object') { + continue; + } + + const phaseId = + review.phaseId !== undefined && review.phaseId !== null + ? String(review.phaseId).trim() + : ''; + const resolvedPhaseName = this.getPhaseNameFromId( + challengeForPhaseResolution, + phaseId, + ); + const normalizedPhaseName = + this.normalizePhaseName(resolvedPhaseName); + + const phaseAllowed = normalizedAllowedPhases.has(normalizedPhaseName); + const phaseCompleted = + phaseAllowed && getPhaseCompletion(resolvedPhaseName); + + if (phaseAllowed && phaseCompleted) { + filteredReviews.push(review); + continue; + } + + review.initialScore = null; + review.finalScore = null; + if (Array.isArray(review.reviewItems)) { + review.reviewItems = []; + } else { + review.reviewItems = []; + } + } + + if (!filteredReviews.length) { + delete (submission as any).review; + } else if (filteredReviews.length !== reviews.length) { + (submission as any).review = filteredReviews; + } + + continue; + } + if (roleSummary.hasReviewer) { const reviews = Array.isArray((submission as any).review) ? ((submission as any).review as Array>) @@ -2217,6 +2426,18 @@ export class SubmissionService { } if (this.isCompletedOrCancelledStatus(challenge?.status ?? null)) { + let hasPassingSubmission = passingSubmissionCache.get(challengeId); + if (hasPassingSubmission === undefined) { + hasPassingSubmission = + await this.hasPassingSubmissionForReviewScorecard(challengeId, uid); + passingSubmissionCache.set(challengeId, hasPassingSubmission); + } + + if (hasPassingSubmission) { + continue; + } + + delete (submission as any).review; continue; } @@ -2229,8 +2450,26 @@ export class SubmissionService { continue; } - delete (submission as any).review; + const reviews = Array.isArray((submission as any).review) + ? ((submission as any).review as Array>) + : []; + + if (!reviews.length) { + delete (submission as any).review; + continue; + } + + const challengeStatus = challenge?.status ?? null; + if (!challenge || challengeStatus === ChallengeStatus.ACTIVE) { + delete (submission as any).review; + continue; + } } + return { + roleSummaryByChallenge, + challengeDetailsById: challengeDetails, + requesterUserId: uid, + }; } private async enrichReviewerMetadata( @@ -2367,6 +2606,192 @@ export class SubmissionService { } } + private async populateReviewPhaseNames( + submissions: Array<{ challengeId?: string | null; review?: unknown }>, + ): Promise { + const reviewEntries: Array<{ + review: Record; + challengeId: string; + }> = []; + + for (const submission of submissions) { + const challengeId = + submission.challengeId !== undefined && submission.challengeId !== null + ? String(submission.challengeId).trim() + : ''; + if (!challengeId) { + continue; + } + + const reviewList = Array.isArray((submission as any).review) + ? ((submission as any).review as Array>) + : []; + + for (const review of reviewList) { + if (review && typeof review === 'object') { + reviewEntries.push({ review, challengeId }); + } + } + } + + if (!reviewEntries.length) { + return; + } + + const phaseMapByChallenge = new Map>(); + const uniqueChallengeIds = Array.from( + new Set(reviewEntries.map((entry) => entry.challengeId)), + ); + + await Promise.all( + uniqueChallengeIds.map(async (challengeId) => { + try { + const challenge = + await this.challengeApiService.getChallengeDetail(challengeId); + const phases = Array.isArray(challenge?.phases) + ? (challenge?.phases as Array>) + : []; + const phaseMap = new Map(); + + for (const phase of phases) { + if (!phase || typeof phase !== 'object') { + continue; + } + + const rawName = + typeof (phase as any).name === 'string' + ? ((phase as any).name as string) + : null; + const normalizedName = + rawName && rawName.trim().length ? rawName.trim() : rawName; + const identifiers = [ + String((phase as any)?.id ?? '').trim(), + String((phase as any)?.phaseId ?? '').trim(), + ].filter( + (value, index, arr) => + value.length > 0 && arr.indexOf(value) === index, + ); + + for (const identifier of identifiers) { + phaseMap.set(identifier, normalizedName ?? rawName ?? null); + } + } + + phaseMapByChallenge.set(challengeId, phaseMap); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + this.logger.warn( + `[populateReviewPhaseNames] Failed to load phases for challenge ${challengeId}: ${message}`, + ); + phaseMapByChallenge.set(challengeId, new Map()); + } + }), + ); + + for (const { review } of reviewEntries) { + if (!Object.prototype.hasOwnProperty.call(review, 'phaseName')) { + review.phaseName = null; + } + } + + for (const { review, challengeId } of reviewEntries) { + const phaseId = String(review.phaseId ?? '').trim(); + if (!phaseId) { + review.phaseName = null; + continue; + } + + const phaseMap = phaseMapByChallenge.get(challengeId); + if (!phaseMap?.size) { + review.phaseName = null; + continue; + } + + review.phaseName = phaseMap.get(phaseId) ?? null; + } + } + + private async populateReviewTypeNames( + submissions: Array<{ review?: unknown }>, + ): Promise { + const reviews: Array> = []; + + for (const submission of submissions) { + const reviewList = Array.isArray((submission as any).review) + ? ((submission as any).review as Array>) + : []; + + for (const review of reviewList) { + if (review && typeof review === 'object') { + reviews.push(review); + } + } + } + + if (!reviews.length) { + return; + } + + const typeIds = Array.from( + new Set( + reviews + .map((review) => String(review.typeId ?? '').trim()) + .filter((id) => id.length > 0), + ), + ); + + if (!typeIds.length) { + for (const review of reviews) { + if (!Object.prototype.hasOwnProperty.call(review, 'reviewType')) { + review.reviewType = null; + } + } + return; + } + + try { + const reviewTypes = await this.prisma.reviewType.findMany({ + where: { id: { in: typeIds } }, + select: { id: true, name: true }, + }); + + const typeNameById = new Map(); + for (const entry of reviewTypes) { + const identifier = String(entry.id ?? '').trim(); + if (!identifier) { + continue; + } + let label: string | null = null; + if (typeof entry.name === 'string') { + const trimmed = entry.name.trim(); + label = trimmed.length ? trimmed : entry.name; + } + typeNameById.set(identifier, label); + } + + for (const review of reviews) { + const typeId = String(review.typeId ?? '').trim(); + if (!typeId) { + review.reviewType = null; + continue; + } + review.reviewType = typeNameById.get(typeId) ?? null; + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.warn( + `[populateReviewTypeNames] Failed to enrich review types: ${message}`, + ); + } finally { + for (const review of reviews) { + if (!Object.prototype.hasOwnProperty.call(review, 'reviewType')) { + review.reviewType = null; + } + } + } + } + private async populateLatestSubmissionFlags( submissions: Array<{ id: string; @@ -2439,6 +2864,368 @@ export class SubmissionService { } } + private async getActiveSubmitterRestrictedChallengeIds( + userId: string, + challengeId?: string, + ): Promise> { + const restricted = new Set(); + if (!userId) { + return restricted; + } + + const summaryByChallenge = new Map< + string, + { hasSubmitter: boolean; hasCopilot: boolean; hasReviewer: boolean } + >(); + + const accumulateRole = ( + challengeKey: string, + roleId?: string | null, + roleName?: string | null, + ) => { + if (!challengeKey) { + return; + } + const normalizedRoleName = (roleName ?? '').toLowerCase(); + const summary = summaryByChallenge.get(challengeKey) ?? { + hasSubmitter: false, + hasCopilot: false, + hasReviewer: false, + }; + if ( + (roleId && roleId === CommonConfig.roles.submitterRoleId) || + normalizedRoleName.includes('submitter') + ) { + summary.hasSubmitter = true; + } + if (normalizedRoleName.includes('copilot')) { + summary.hasCopilot = true; + } + if ( + REVIEW_ACCESS_ROLE_KEYWORDS.some((keyword) => + normalizedRoleName.includes(keyword), + ) + ) { + summary.hasReviewer = true; + } + summaryByChallenge.set(challengeKey, summary); + }; + + let resourcesLoaded = false; + try { + const resources = await this.resourceApiService.getMemberResourcesRoles( + challengeId, + userId, + ); + for (const resource of resources ?? []) { + const challengeKey = String(resource.challengeId ?? '').trim(); + accumulateRole(challengeKey, resource.roleId, resource.roleName ?? ''); + } + resourcesLoaded = true; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.debug( + `[getActiveSubmitterRestrictedChallengeIds] Failed to load resource roles via API for member ${userId}: ${message}`, + ); + } + + if (!resourcesLoaded) { + if (challengeId) { + accumulateRole(challengeId, CommonConfig.roles.submitterRoleId, null); + } else { + try { + const fallbackResources = await this.resourcePrisma.resource.findMany( + { + where: { memberId: userId }, + select: { challengeId: true, roleId: true }, + }, + ); + for (const resource of fallbackResources) { + const challengeKey = String(resource.challengeId ?? '').trim(); + accumulateRole(challengeKey, resource.roleId, null); + } + } catch (fallbackError) { + const fallbackMessage = + fallbackError instanceof Error + ? fallbackError.message + : String(fallbackError); + this.logger.debug( + `[getActiveSubmitterRestrictedChallengeIds] Fallback resource lookup failed for member ${userId}: ${fallbackMessage}`, + ); + } + } + } + + const candidateIds = Array.from(summaryByChallenge.entries()) + .filter(([, summary]) => summary.hasSubmitter) + .filter(([, summary]) => !summary.hasCopilot && !summary.hasReviewer) + .map(([challengeKey]) => challengeKey) + .filter((id) => id); + + if (!candidateIds.length) { + return restricted; + } + + try { + let details: ChallengeData[] = []; + if (candidateIds.length === 1) { + const detail = await this.challengeApiService.getChallengeDetail( + candidateIds[0], + ); + details = detail ? [detail] : []; + } else { + details = await this.challengeApiService.getChallenges(candidateIds); + } + const detailById = new Map(); + for (const detail of details ?? []) { + if (detail?.id) { + detailById.set(detail.id, detail); + } + } + for (const id of candidateIds) { + const detail = detailById.get(id); + if (!detail) { + restricted.add(id); + continue; + } + if (!this.isCompletedOrCancelledStatus(detail.status)) { + restricted.add(id); + } + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.debug( + `[getActiveSubmitterRestrictedChallengeIds] Unable to resolve challenge statuses for submitter visibility: ${message}`, + ); + candidateIds.forEach((id) => restricted.add(id)); + } + + return restricted; + } + + private filterSubmissionsForActiveSubmitters< + T extends { + challengeId?: string | null; + memberId?: string | null; + } & Record, + >( + authUser: JwtUser, + submissions: T[], + visibilityContext: ReviewVisibilityContext, + ): { + submissions: T[]; + filteredOut: boolean; + } { + if (!submissions.length) { + return { submissions, filteredOut: false }; + } + if (authUser?.isMachine || isAdmin(authUser)) { + return { submissions, filteredOut: false }; + } + + const uid = visibilityContext.requesterUserId; + if (!uid) { + return { submissions, filteredOut: false }; + } + + const filtered = submissions.filter((submission) => { + const challengeId = + submission.challengeId !== undefined && submission.challengeId !== null + ? String(submission.challengeId).trim() + : ''; + if (!challengeId) { + return true; + } + + const memberIdValue = + submission.memberId !== undefined && submission.memberId !== null + ? String(submission.memberId).trim() + : null; + if (memberIdValue && memberIdValue === uid) { + return true; + } + + const roleSummary = + visibilityContext.roleSummaryByChallenge.get(challengeId); + if (!roleSummary) { + return false; + } + if (roleSummary.hasCopilot || roleSummary.hasReviewer) { + return true; + } + if (!roleSummary.hasSubmitter) { + return true; + } + + const challengeDetail = + visibilityContext.challengeDetailsById.get(challengeId); + const challengeStatus = challengeDetail?.status ?? null; + const isActiveChallenge = + !this.isCompletedOrCancelledStatus(challengeStatus); + if (challengeDetail == null) { + return false; + } + return !isActiveChallenge; + }); + + return { + submissions: filtered, + filteredOut: filtered.length !== submissions.length, + }; + } + + private stripSubmitterMemberIds( + authUser: JwtUser, + submissions: Array< + { challengeId?: string | null; memberId?: string | null } & Record< + string, + unknown + > + >, + visibilityContext: ReviewVisibilityContext, + ): void { + if (!submissions.length) { + return; + } + if (authUser?.isMachine || isAdmin(authUser)) { + return; + } + + const uid = visibilityContext.requesterUserId; + if (!uid) { + return; + } + + for (const submission of submissions) { + const challengeId = + submission.challengeId !== undefined && submission.challengeId !== null + ? String(submission.challengeId).trim() + : ''; + if (!challengeId) { + continue; + } + + const memberIdValue = + submission.memberId !== undefined && submission.memberId !== null + ? String(submission.memberId).trim() + : null; + if (!memberIdValue || memberIdValue === uid) { + continue; + } + + const roleSummary = + visibilityContext.roleSummaryByChallenge.get(challengeId) ?? + EMPTY_ROLE_SUMMARY; + + const shouldStrip = + roleSummary.hasSubmitter && + !roleSummary.hasCopilot && + !roleSummary.hasReviewer; + if (!shouldStrip) { + continue; + } + + (submission as any).memberId = null; + if (Object.prototype.hasOwnProperty.call(submission, 'submitterHandle')) { + delete (submission as any).submitterHandle; + } + if ( + Object.prototype.hasOwnProperty.call(submission, 'submitterMaxRating') + ) { + delete (submission as any).submitterMaxRating; + } + } + } + + private stripSubmitterSubmissionDetails( + authUser: JwtUser, + submissions: Array< + { + challengeId?: string | null; + memberId?: string | null; + review?: unknown; + reviewSummation?: unknown; + url?: string | null; + } & Record + >, + visibilityContext: ReviewVisibilityContext, + ): void { + if (!submissions.length) { + return; + } + if (authUser?.isMachine || isAdmin(authUser)) { + return; + } + + const uid = visibilityContext.requesterUserId; + if (!uid) { + return; + } + + for (const submission of submissions) { + const challengeId = + submission.challengeId !== undefined && submission.challengeId !== null + ? String(submission.challengeId).trim() + : ''; + if (!challengeId) { + continue; + } + + const memberIdValue = + submission.memberId !== undefined && submission.memberId !== null + ? String(submission.memberId).trim() + : null; + if (!memberIdValue || memberIdValue === uid) { + continue; + } + + const roleSummary = + visibilityContext.roleSummaryByChallenge.get(challengeId) ?? + EMPTY_ROLE_SUMMARY; + if ( + !roleSummary.hasSubmitter || + roleSummary.hasCopilot || + roleSummary.hasReviewer + ) { + continue; + } + + const challenge = visibilityContext.challengeDetailsById.get(challengeId); + const isActiveChallenge = + !challenge || challenge.status === ChallengeStatus.ACTIVE; + if (!isActiveChallenge) { + continue; + } + + if (Array.isArray((submission as any).review)) { + for (const review of (submission as any).review as Array< + Record + >) { + if (!review || typeof review !== 'object') { + continue; + } + if (Object.prototype.hasOwnProperty.call(review, 'reviewItems')) { + delete review.reviewItems; + } + if (Object.prototype.hasOwnProperty.call(review, 'initialScore')) { + review.initialScore = null; + } + if (Object.prototype.hasOwnProperty.call(review, 'finalScore')) { + review.finalScore = null; + } + } + } + + if (Object.prototype.hasOwnProperty.call(submission, 'reviewSummation')) { + delete (submission as any).reviewSummation; + } + + if (Object.prototype.hasOwnProperty.call(submission, 'url')) { + (submission as any).url = null; + } + } + } + private async stripIsLatestForUnlimitedChallenges( submissions: Array< { challengeId?: string | null } & Record @@ -2610,6 +3397,210 @@ export class SubmissionService { return String(status).startsWith('CANCELLED_'); } + private normalizePhaseName(phaseName: string | null | undefined): string { + return String(phaseName ?? '') + .trim() + .toLowerCase(); + } + + private hasTimestampValue(value: unknown): boolean { + if (value == null) { + return false; + } + if (value instanceof Date) { + return true; + } + switch (typeof value) { + case 'string': + return value.trim().length > 0; + case 'number': + return Number.isFinite(value); + case 'bigint': + case 'boolean': + return true; + case 'symbol': + case 'function': + return false; + case 'object': { + const valueWithToISOString = value as { + toISOString?: (() => string) | undefined; + valueOf?: (() => unknown) | undefined; + }; + if (typeof valueWithToISOString.toISOString === 'function') { + try { + return valueWithToISOString.toISOString().trim().length > 0; + } catch { + return false; + } + } + if (typeof valueWithToISOString.valueOf === 'function') { + const primitiveValue = valueWithToISOString.valueOf(); + if (primitiveValue !== value) { + return this.hasTimestampValue(primitiveValue); + } + } + return false; + } + default: + return false; + } + } + + private hasChallengePhaseCompleted( + challenge: ChallengeData | null | undefined, + phaseNames: string[], + ): boolean { + if (!challenge?.phases?.length) { + return false; + } + + const normalizedTargets = new Set( + (phaseNames ?? []) + .map((name) => this.normalizePhaseName(name)) + .filter((name) => name.length > 0), + ); + + if (!normalizedTargets.size) { + return false; + } + + return (challenge.phases ?? []).some((phase) => { + if (!phase) { + return false; + } + + const normalizedName = this.normalizePhaseName((phase as any).name); + if (!normalizedTargets.has(normalizedName)) { + return false; + } + + if ((phase as any).isOpen === true) { + return false; + } + + const actualEnd = + (phase as any).actualEndTime ?? + (phase as any).actualEndDate ?? + (phase as any).actualEnd ?? + null; + + return this.hasTimestampValue(actualEnd); + }); + } + + private getPhaseNameFromId( + challenge: ChallengeData | null | undefined, + phaseId: string | null | undefined, + ): string | null { + if (!challenge?.phases?.length || phaseId == null) { + return null; + } + + const normalizedPhaseId = String(phaseId).trim(); + if (!normalizedPhaseId.length) { + return null; + } + + const match = (challenge.phases ?? []).find((phase) => { + if (!phase) { + return false; + } + + const candidateIds = [ + String((phase as any).id ?? '').trim(), + String((phase as any).phaseId ?? '').trim(), + ].filter((candidate) => candidate.length > 0); + + return candidateIds.includes(normalizedPhaseId); + }); + + const matchName = + match != null ? (match as { name?: unknown }).name : undefined; + return typeof matchName === 'string' ? matchName : null; + } + + private identifyReviewerRoleType( + roleName: string, + ): + | 'screener' + | 'checkpoint-screener' + | 'checkpoint-reviewer' + | 'reviewer' + | 'approver' + | 'iterative-reviewer' + | 'unknown' { + const normalized = String(roleName ?? '') + .trim() + .toLowerCase(); + if (!normalized) { + return 'unknown'; + } + + if (normalized.includes('checkpoint') && normalized.includes('screener')) { + return 'checkpoint-screener'; + } + + if (normalized.includes('checkpoint') && normalized.includes('reviewer')) { + return 'checkpoint-reviewer'; + } + + if (normalized.includes('screener')) { + return 'screener'; + } + + if (normalized.includes('approver') || normalized.includes('approval')) { + return 'approver'; + } + + if (normalized.includes('iterative') && normalized.includes('reviewer')) { + return 'iterative-reviewer'; + } + + if (normalized.includes('reviewer')) { + return 'reviewer'; + } + + return 'unknown'; + } + + private async hasPassingSubmissionForReviewScorecard( + challengeId: string, + memberId: string, + ): Promise { + const normalizedChallengeId = String(challengeId ?? '').trim(); + const normalizedMemberId = String(memberId ?? '').trim(); + + if (!normalizedChallengeId || !normalizedMemberId) { + return false; + } + + try { + const passingSummation = await this.prisma.reviewSummation.findFirst({ + where: { + isPassing: true, + scorecard: { + type: { + in: [ScorecardType.REVIEW, ScorecardType.ITERATIVE_REVIEW], + }, + }, + submission: { + challengeId: normalizedChallengeId, + memberId: normalizedMemberId, + }, + }, + select: { id: true }, + }); + + return Boolean(passingSummation); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.logger.warn( + `[hasPassingSubmissionForReviewScorecard] Failed to check passing submission for challenge ${normalizedChallengeId}, member ${normalizedMemberId}: ${message}`, + ); + return false; + } + } + private buildResponse(data: any): SubmissionResponseDto { const dto: SubmissionResponseDto = { ...data, diff --git a/src/dto/review.dto.ts b/src/dto/review.dto.ts index 862c153..bae19e1 100644 --- a/src/dto/review.dto.ts +++ b/src/dto/review.dto.ts @@ -396,6 +396,16 @@ export class ReviewResponseDto extends ReviewCommonDto { @IsString() phaseName?: string | null; + @ApiProperty({ + description: 'Human-readable name of the review type', + example: 'Iterative Review', + required: false, + nullable: true, + }) + @IsOptional() + @IsString() + reviewType?: string | null; + @ApiProperty({ description: 'Final score of the review', example: 85.5 }) finalScore: number | null; From 405a0a1668c58afda7c40b2e93841d3d7ab1c478 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Mon, 27 Oct 2025 15:26:35 +1100 Subject: [PATCH 41/48] Tweaks for data visibility at different stages --- src/api/submission/submission.service.spec.ts | 2 + src/api/submission/submission.service.ts | 18 +++-- .../modules/global/resource.service.spec.ts | 72 +++++++++++++++++++ src/shared/modules/global/resource.service.ts | 49 ++++++++++--- 4 files changed, 126 insertions(+), 15 deletions(-) create mode 100644 src/shared/modules/global/resource.service.spec.ts diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index 09149fc..abc4aab 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -326,6 +326,7 @@ describe('SubmissionService', () => { id: 'sub-123', memberId: 'owner-user', challengeId: 'challenge-xyz', + type: SubmissionType.CONTEST_SUBMISSION, url: 'https://s3.amazonaws.com/dummy/submission.zip', }); jest @@ -897,6 +898,7 @@ describe('SubmissionService', () => { ); expect(other?.review).toBeDefined(); + expect(other?.memberId).toBe('user-2'); }); it('retains review data for marathon match submissions', async () => { diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index afcbdb0..5d98a1f 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -2381,7 +2381,7 @@ export class SubmissionService { } if (!filteredReviews.length) { - delete (submission as any).review; + (submission as any).review = reviews; } else if (filteredReviews.length !== reviews.length) { (submission as any).review = filteredReviews; } @@ -2426,6 +2426,9 @@ export class SubmissionService { } if (this.isCompletedOrCancelledStatus(challenge?.status ?? null)) { + if (challenge?.status === ChallengeStatus.COMPLETED) { + continue; + } let hasPassingSubmission = passingSubmissionCache.get(challengeId); if (hasPassingSubmission === undefined) { hasPassingSubmission = @@ -3059,13 +3062,10 @@ export class SubmissionService { const challengeDetail = visibilityContext.challengeDetailsById.get(challengeId); - const challengeStatus = challengeDetail?.status ?? null; - const isActiveChallenge = - !this.isCompletedOrCancelledStatus(challengeStatus); if (challengeDetail == null) { return false; } - return !isActiveChallenge; + return true; }); return { @@ -3116,6 +3116,14 @@ export class SubmissionService { const roleSummary = visibilityContext.roleSummaryByChallenge.get(challengeId) ?? EMPTY_ROLE_SUMMARY; + const challengeDetail = + visibilityContext.challengeDetailsById.get(challengeId) ?? null; + const isCompletedChallenge = this.isCompletedOrCancelledStatus( + challengeDetail?.status ?? null, + ); + if (isCompletedChallenge) { + continue; + } const shouldStrip = roleSummary.hasSubmitter && diff --git a/src/shared/modules/global/resource.service.spec.ts b/src/shared/modules/global/resource.service.spec.ts new file mode 100644 index 0000000..eb01e08 --- /dev/null +++ b/src/shared/modules/global/resource.service.spec.ts @@ -0,0 +1,72 @@ +import { ResourceApiService } from './resource.service'; +import { HttpService } from '@nestjs/axios'; +import { M2MService } from './m2m.service'; +import { of } from 'rxjs'; +import { AxiosResponse } from 'axios'; +import { ResourceInfo } from 'src/shared/models/ResourceInfo.model'; + +describe('ResourceApiService', () => { + let httpService: { get: jest.Mock }; + let m2mService: { getM2MToken: jest.Mock }; + let service: ResourceApiService; + + const createResponse = ( + data: ResourceInfo[], + headers: Record, + ): AxiosResponse => + ({ + data, + headers, + status: 200, + statusText: 'OK', + config: {}, + }) as AxiosResponse; + + const buildResource = (id: string): ResourceInfo => ({ + id, + challengeId: 'challenge', + memberId: '123', + memberHandle: `handle-${id}`, + roleId: 'role', + createdBy: 'tc', + created: new Date().toISOString(), + }); + + beforeEach(() => { + httpService = { get: jest.fn() }; + m2mService = { getM2MToken: jest.fn().mockResolvedValue('token') }; + service = new ResourceApiService( + m2mService as unknown as M2MService, + httpService as unknown as HttpService, + ); + }); + + it('aggregates resources across multiple pages', async () => { + const firstPage = [buildResource('r1')]; + const secondPage = [buildResource('r2')]; + + httpService.get + .mockReturnValueOnce( + of(createResponse(firstPage, { 'x-total-pages': '2' })), + ) + .mockReturnValueOnce(of(createResponse(secondPage, {}))); + + const result = await service.getResources({ memberId: '123' }); + + expect(result).toEqual([...firstPage, ...secondPage]); + expect(httpService.get).toHaveBeenCalledTimes(2); + expect(httpService.get.mock.calls[0][0]).toContain('page=1'); + expect(httpService.get.mock.calls[1][0]).toContain('page=2'); + expect(httpService.get.mock.calls[0][0]).toContain('perPage=1000'); + }); + + it('stops pagination when the final page contains fewer results than the perPage size', async () => { + const singlePage = [buildResource('r1')]; + httpService.get.mockReturnValueOnce(of(createResponse(singlePage, {}))); + + const result = await service.getResources({ memberId: '123' }); + + expect(result).toEqual(singlePage); + expect(httpService.get).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/shared/modules/global/resource.service.ts b/src/shared/modules/global/resource.service.ts index 59121bc..6775129 100644 --- a/src/shared/modules/global/resource.service.ts +++ b/src/shared/modules/global/resource.service.ts @@ -83,21 +83,50 @@ export class ResourceApiService { memberId?: string; }): Promise { try { - // Send request to resource api const params = new URLSearchParams(); if (query.challengeId) params.append('challengeId', query.challengeId); if (query.memberId) params.append('memberId', query.memberId); - const url = `${CommonConfig.apis.resourceApiUrl}resources?${params.toString()}`; + const perPage = 1000; + params.set('perPage', String(perPage)); + const token = await this.m2mService.getM2MToken(); - const response = await firstValueFrom( - this.httpService.get(url, { - headers: { - Authorization: 'Bearer ' + token, - }, - }), - ); - return response.data; + const resources: ResourceInfo[] = []; + + let page = 1; + while (true) { + params.set('page', String(page)); + + const url = `${CommonConfig.apis.resourceApiUrl}resources?${params.toString()}`; + const response = await firstValueFrom( + this.httpService.get(url, { + headers: { + Authorization: 'Bearer ' + token, + }, + }), + ); + + const batch = Array.isArray(response.data) ? response.data : []; + resources.push(...batch); + + const totalPagesHeader = + response.headers?.['x-total-pages'] ?? + (response.headers?.['X-Total-Pages'] as string | undefined); + const totalPages = Number(totalPagesHeader); + + const hasMorePagesByHeader = + Number.isFinite(totalPages) && totalPages > 0 && page < totalPages; + const hasMorePagesByCount = + !Number.isFinite(totalPages) && batch.length === perPage; + + if (!hasMorePagesByHeader && !hasMorePagesByCount) { + break; + } + + page += 1; + } + + return resources; } catch (e) { if (e instanceof AxiosError) { this.logger.error(`Http Error: ${e.message}`, e.response?.data); From bc1fa9f0c013574d32146dc4b85f20c84cecd03f Mon Sep 17 00:00:00 2001 From: Kiril Kartunov Date: Mon, 27 Oct 2025 09:51:27 +0200 Subject: [PATCH 42/48] add permissions to Trivy action --- .github/workflows/trivy.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 85b3cad..7b9fa48 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -1,4 +1,8 @@ name: Trivy Scanner + +permissions: + contents: read + security-events: write on: push: branches: @@ -16,15 +20,15 @@ jobs: - name: Run Trivy scanner in repo mode uses: aquasecurity/trivy-action@0.33.1 with: - scan-type: 'fs' + scan-type: "fs" ignore-unfixed: true - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH,UNKNOWN' + format: "sarif" + output: "trivy-results.sarif" + severity: "CRITICAL,HIGH,UNKNOWN" scanners: vuln,secret,misconfig,license github-pat: ${{ secrets.GITHUB_TOKEN }} - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v3 with: - sarif_file: 'trivy-results.sarif' \ No newline at end of file + sarif_file: "trivy-results.sarif" From 4317c093dbfcd8a478c1f15d714bb3c64a1847b6 Mon Sep 17 00:00:00 2001 From: Hentry Martin Date: Mon, 27 Oct 2025 16:43:55 +0100 Subject: [PATCH 43/48] fix: extracted to a util method --- src/shared/modules/global/challenge-prisma.service.ts | 7 ++----- src/shared/modules/global/member-prisma.service.ts | 7 ++----- src/shared/modules/global/prisma.service.ts | 7 ++----- src/shared/modules/global/resource-prisma.service.ts | 7 ++----- src/shared/modules/global/utils.service.ts | 10 ++++++++++ 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/shared/modules/global/challenge-prisma.service.ts b/src/shared/modules/global/challenge-prisma.service.ts index 378344f..ba36c20 100644 --- a/src/shared/modules/global/challenge-prisma.service.ts +++ b/src/shared/modules/global/challenge-prisma.service.ts @@ -1,6 +1,7 @@ import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { PrismaClient, Prisma } from '@prisma/client'; import { LoggerService } from './logger.service'; +import { Utils } from './utils.service'; @Injectable() export class ChallengePrismaService @@ -11,11 +12,7 @@ export class ChallengePrismaService constructor() { super({ - transactionOptions: { - timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT - ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) - : 10000, - }, + ...Utils.getPrismaTimeout(), log: [ { level: 'query', emit: 'event' }, { level: 'info', emit: 'event' }, diff --git a/src/shared/modules/global/member-prisma.service.ts b/src/shared/modules/global/member-prisma.service.ts index 3c71cdc..5c2dbb1 100644 --- a/src/shared/modules/global/member-prisma.service.ts +++ b/src/shared/modules/global/member-prisma.service.ts @@ -1,6 +1,7 @@ import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { PrismaClient, Prisma } from '@prisma/client-member'; import { LoggerService } from './logger.service'; +import { Utils } from './utils.service'; @Injectable() export class MemberPrismaService @@ -11,11 +12,7 @@ export class MemberPrismaService constructor() { super({ - transactionOptions: { - timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT - ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) - : 10000, - }, + ...Utils.getPrismaTimeout(), log: [ { level: 'query', emit: 'event' }, { level: 'info', emit: 'event' }, diff --git a/src/shared/modules/global/prisma.service.ts b/src/shared/modules/global/prisma.service.ts index c1e1d00..eca8356 100644 --- a/src/shared/modules/global/prisma.service.ts +++ b/src/shared/modules/global/prisma.service.ts @@ -3,6 +3,7 @@ import { PrismaClient, Prisma } from '@prisma/client'; import { LoggerService } from './logger.service'; import { PrismaErrorService } from './prisma-error.service'; import { getStore } from 'src/shared/request/requestStore'; +import { Utils } from './utils.service'; enum auditField { createdBy = 'createdBy', @@ -197,11 +198,7 @@ export class PrismaService const schema = process.env.POSTGRES_SCHEMA || 'public'; super({ - transactionOptions: { - timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT - ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) - : 10000, - }, + ...Utils.getPrismaTimeout(), log: [ { level: 'query', emit: 'event' }, { level: 'info', emit: 'event' }, diff --git a/src/shared/modules/global/resource-prisma.service.ts b/src/shared/modules/global/resource-prisma.service.ts index 2996647..fff2152 100644 --- a/src/shared/modules/global/resource-prisma.service.ts +++ b/src/shared/modules/global/resource-prisma.service.ts @@ -1,6 +1,7 @@ import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { PrismaClient, Prisma } from '@prisma/client-resource'; import { LoggerService } from './logger.service'; +import { Utils } from './utils.service'; @Injectable() export class ResourcePrismaService @@ -11,11 +12,7 @@ export class ResourcePrismaService constructor() { super({ - transactionOptions: { - timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT - ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) - : 10000, - }, + ...Utils.getPrismaTimeout(), log: [ { level: 'query', emit: 'event' }, { level: 'info', emit: 'event' }, diff --git a/src/shared/modules/global/utils.service.ts b/src/shared/modules/global/utils.service.ts index 63cf5d4..0a3f00e 100644 --- a/src/shared/modules/global/utils.service.ts +++ b/src/shared/modules/global/utils.service.ts @@ -4,4 +4,14 @@ export class Utils { static bigIntToNumber(t) { return t ? Number(t) : null; } + + static getPrismaTimeout() { + return { + transactionOptions: { + timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT + ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) + : 10000, + } + } + } } From e0bbc81226962e9707828164e9c50064ae3f6f75 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Tue, 28 Oct 2025 15:37:46 +1100 Subject: [PATCH 44/48] Allow approval visibility to submitters --- src/api/review/review.service.spec.ts | 107 ++++++++++++++++++----- src/api/review/review.service.ts | 58 +++++++++--- src/api/submission/submission.service.ts | 1 + 3 files changed, 128 insertions(+), 38 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index e861e1c..13d9498 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -1291,6 +1291,29 @@ describe('ReviewService.getReviews reviewer visibility', () => { return undefined; }; + const collectSubmissionFilters = ( + clause: PrismaWhereClause | PrismaWhereClause[] | undefined | null, + ): Array<{ in?: string[] }> => { + if (!clause) { + return []; + } + if (Array.isArray(clause)) { + return clause.flatMap((entry) => collectSubmissionFilters(entry)); + } + const collected: Array<{ in?: string[] }> = []; + const submissionId = (clause as any).submissionId; + if (submissionId && typeof submissionId === 'object') { + collected.push(submissionId); + } + if (clause.AND) { + collected.push(...collectSubmissionFilters(clause.AND)); + } + if (clause.OR) { + collected.push(...collectSubmissionFilters(clause.OR)); + } + return collected; + }; + const buildResource = (roleName: string, memberId = baseAuthUser.userId) => ({ id: 'resource-1', challengeId: 'challenge-1', @@ -1383,15 +1406,25 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(prismaMock.review.findMany).toHaveBeenCalledTimes(1); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; const whereClauses = flattenWhereClauses(callArgs.where); - const resourceOrClause = whereClauses.find( + const resourceOrClause = whereClauses.find((clause) => + Array.isArray(clause.OR), + ); + expect(resourceOrClause).toBeDefined(); + const flattenedRoleClauses = flattenWhereClauses(resourceOrClause?.OR); + const screeningVisibility = flattenedRoleClauses.find( (clause) => - clause.OR?.some((entry) => typeof entry.resourceId !== 'undefined') ?? - false, + typeof (clause as any).phaseId !== 'undefined' && + Array.isArray((clause as any).phaseId?.in) && + (clause as any).phaseId.in.includes('phase-screening'), ); - expect(resourceOrClause?.OR).toEqual([ - { resourceId: { in: ['resource-1'] } }, - { phaseId: { in: ['phase-screening'] } }, - ]); + expect(screeningVisibility).toBeDefined(); + const reviewerVisibility = flattenedRoleClauses.find( + (clause) => + typeof (clause as any).resourceId !== 'undefined' && + Array.isArray((clause as any).resourceId?.in) && + (clause as any).resourceId.in.includes('resource-1'), + ); + expect(reviewerVisibility).toBeDefined(); const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); expect(submissionClause?.submissionId).toEqual({ in: [baseSubmission.id], @@ -1479,15 +1512,25 @@ describe('ReviewService.getReviews reviewer visibility', () => { const callArgs = prismaMock.review.findMany.mock.calls[0][0]; const whereClauses = flattenWhereClauses(callArgs.where); - const resourceOrClause = whereClauses.find( + const resourceOrClause = whereClauses.find((clause) => + Array.isArray(clause.OR), + ); + expect(resourceOrClause).toBeDefined(); + const flattenedRoleClauses = flattenWhereClauses(resourceOrClause?.OR); + const screeningVisibility = flattenedRoleClauses.find( (clause) => - clause.OR?.some((entry) => typeof entry.resourceId !== 'undefined') ?? - false, + typeof (clause as any).phaseId !== 'undefined' && + Array.isArray((clause as any).phaseId?.in) && + (clause as any).phaseId.in.includes('phase-screening'), ); - expect(resourceOrClause?.OR).toEqual([ - { resourceId: { in: ['resource-1'] } }, - { phaseId: { in: ['phase-screening'] } }, - ]); + expect(screeningVisibility).toBeDefined(); + const checkpointReviewerVisibility = flattenedRoleClauses.find( + (clause) => + typeof (clause as any).resourceId !== 'undefined' && + Array.isArray((clause as any).resourceId?.in) && + (clause as any).resourceId.in.includes('resource-1'), + ); + expect(checkpointReviewerVisibility).toBeDefined(); }); it('does not restrict resource visibility for copilots', async () => { @@ -1765,11 +1808,19 @@ describe('ReviewService.getReviews reviewer visibility', () => { await service.getReviews(baseAuthUser, undefined, 'challenge-1'); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - const whereClauses = flattenWhereClauses(callArgs.where); - const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); - expect(submissionClause?.submissionId).toEqual({ - in: [baseSubmission.id, 'submission-other'], - }); + const submissionFilters = collectSubmissionFilters(callArgs.where); + const hasOwnRestriction = submissionFilters.some( + (filter) => + Array.isArray(filter.in) && + filter.in.length === 1 && + filter.in[0] === baseSubmission.id, + ); + expect(hasOwnRestriction).toBe(true); + const hasChallengeVisibility = submissionFilters.some( + (filter) => + Array.isArray(filter.in) && filter.in.includes('submission-other'), + ); + expect(hasChallengeVisibility).toBe(true); }); it('allows submitters to access reviews when the challenge failed review cancellation', async () => { @@ -1803,11 +1854,19 @@ describe('ReviewService.getReviews reviewer visibility', () => { ); const callArgs = prismaMock.review.findMany.mock.calls[0][0]; - const whereClauses = flattenWhereClauses(callArgs.where); - const submissionClause = findClauseWithKey(whereClauses, 'submissionId'); - expect(submissionClause?.submissionId).toEqual({ - in: [baseSubmission.id, 'submission-other'], - }); + const submissionFilters = collectSubmissionFilters(callArgs.where); + const hasOwnRestriction = submissionFilters.some( + (filter) => + Array.isArray(filter.in) && + filter.in.length === 1 && + filter.in[0] === baseSubmission.id, + ); + expect(hasOwnRestriction).toBe(true); + const hasChallengeVisibility = submissionFilters.some( + (filter) => + Array.isArray(filter.in) && filter.in.includes('submission-other'), + ); + expect(hasChallengeVisibility).toBe(true); expect(result.data).toEqual([]); }); diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index d185096..1de246c 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -4158,7 +4158,11 @@ export class ReviewService { isPassing: true, scorecard: { type: { - in: [ScorecardType.REVIEW, ScorecardType.ITERATIVE_REVIEW], + in: [ + ScorecardType.REVIEW, + ScorecardType.ITERATIVE_REVIEW, + ScorecardType.APPROVAL, + ], }, }, submission: { @@ -4284,18 +4288,21 @@ export class ReviewService { const filters: Prisma.reviewWhereInput[] = []; - if (groupedByRole.has('screener')) { - const screeningPhaseIds = this.getPhaseIdsForNames(challengeDetail, [ - 'screening', - ]); - if (screeningPhaseIds.length) { - filters.push({ - AND: [ - { submission: { type: SubmissionType.CONTEST_SUBMISSION } }, - { phaseId: { in: screeningPhaseIds } }, - ], - }); - } + const screeningPhaseIds = this.getPhaseIdsForNames(challengeDetail, [ + 'screening', + ]); + const hasScreeningVisibility = + screeningPhaseIds.length > 0 && + (groupedByRole.has('screener') || + groupedByRole.has('reviewer') || + groupedByRole.has('checkpoint-reviewer')); + if (hasScreeningVisibility) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CONTEST_SUBMISSION } }, + { phaseId: { in: screeningPhaseIds } }, + ], + }); } if (groupedByRole.has('checkpoint-screener')) { @@ -4318,11 +4325,27 @@ export class ReviewService { challengeDetail, ['checkpoint review'], ); - if (checkpointReviewPhaseIds.length) { + const checkpointReviewerResourceIds = ( + groupedByRole.get('checkpoint-reviewer') ?? [] + ) + .map((resource) => String(resource?.id ?? '').trim()) + .filter((id) => id.length > 0); + if ( + checkpointReviewPhaseIds.length && + checkpointReviewerResourceIds.length + ) { filters.push({ AND: [ { submission: { type: SubmissionType.CHECKPOINT_SUBMISSION } }, { phaseId: { in: checkpointReviewPhaseIds } }, + { resourceId: { in: checkpointReviewerResourceIds } }, + ], + }); + } else if (checkpointReviewerResourceIds.length) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CHECKPOINT_SUBMISSION } }, + { resourceId: { in: checkpointReviewerResourceIds } }, ], }); } @@ -4343,6 +4366,13 @@ export class ReviewService { { resourceId: { in: reviewerResourceIds } }, ], }); + } else if (reviewerResourceIds.length) { + filters.push({ + AND: [ + { submission: { type: SubmissionType.CONTEST_SUBMISSION } }, + { resourceId: { in: reviewerResourceIds } }, + ], + }); } } diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index 5d98a1f..59cdfc4 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -2309,6 +2309,7 @@ export class SubmissionService { 'screening', 'review', 'iterative review', + 'approval', ]; const normalizedAllowedPhases = new Set( allowedPhaseNames.map((name) => this.normalizePhaseName(name)), From ef42e7eb5fba5d51afd4dcfe926e22db30358c9b Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Tue, 28 Oct 2025 18:26:22 +1100 Subject: [PATCH 45/48] Additional fixes for returning screenings if a user is a reviewer --- src/api/review/review.service.spec.ts | 133 +++++++++++++++++ src/api/review/review.service.ts | 11 +- src/api/submission/submission.service.spec.ts | 141 ++++++++++++++++++ src/api/submission/submission.service.ts | 28 +++- 4 files changed, 306 insertions(+), 7 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index 13d9498..ca6c288 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -1686,6 +1686,139 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(otherReview?.reviewerMaxRating).toBe(1800); }); + it('exposes screening review items and scores to other reviewers', async () => { + const now = new Date(); + const reviewerUser: JwtUser = { + userId: '101', + roles: [], + isMachine: false, + }; + + const reviewerResource = { + ...buildResource('Reviewer', reviewerUser.userId), + id: 'resource-self', + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + reviewerResource, + ]); + + const reviewRecords = [ + { + id: 'review-self', + resourceId: 'resource-self', + submissionId: baseSubmission.id, + phaseId: 'phase-review', + scorecardId: 'scorecard-1', + typeId: 'type-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + reviewItems: [ + { + id: 'item-self', + scorecardQuestionId: 'question-1', + initialAnswer: 'Yes', + finalAnswer: 'Yes', + managerComment: null, + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 98, + initialScore: 96, + submission: { + id: baseSubmission.id, + memberId: '301', + challengeId: 'challenge-1', + }, + }, + { + id: 'review-screening', + resourceId: 'resource-other', + submissionId: baseSubmission.id, + phaseId: 'phase-screening', + scorecardId: 'scorecard-2', + typeId: 'type-2', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + reviewItems: [ + { + id: 'item-screening', + scorecardQuestionId: 'question-3', + initialAnswer: 'Yes', + finalAnswer: 'Yes', + managerComment: null, + createdAt: now, + createdBy: 'other-reviewer', + updatedAt: now, + updatedBy: 'other-reviewer', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'creator', + updatedAt: now, + updatedBy: 'updater', + finalScore: 82, + initialScore: 80, + submission: { + id: baseSubmission.id, + memberId: '301', + challengeId: 'challenge-1', + }, + }, + ]; + + prismaMock.review.findMany.mockResolvedValue(reviewRecords); + prismaMock.review.count.mockResolvedValue(reviewRecords.length); + + resourcePrismaMock.resource.findMany.mockResolvedValue([ + { id: 'resource-self', memberId: '101' }, + { id: 'resource-other', memberId: '202' }, + ]); + + memberPrismaMock.member.findMany.mockResolvedValue([ + { + userId: BigInt(101), + handle: 'selfHandle', + maxRating: { rating: 2400 }, + }, + { + userId: BigInt(202), + handle: 'screeningHandle', + maxRating: { rating: 2100 }, + }, + ]); + + const response = await service.getReviews( + reviewerUser, + undefined, + 'challenge-1', + ); + + expect(response.data).toHaveLength(2); + const screeningReview = response.data.find( + (review) => review.id === 'review-screening', + ); + + expect(screeningReview?.reviewItems).toHaveLength(1); + expect(screeningReview?.finalScore).toBe(82); + expect(screeningReview?.initialScore).toBe(80); + expect(screeningReview?.reviewerHandle).toBe('screeningHandle'); + expect(screeningReview?.reviewerMaxRating).toBe(2100); + }); + it('returns empty results for submitters when reviews are not yet accessible', async () => { const submitterUser: JwtUser = { userId: 'submitter-1', diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index 1de246c..a115c38 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -3434,7 +3434,9 @@ export class ReviewService { !isPrivilegedRequester && reviewerResourceIdSet.size > 0 && !isReviewerForReview && - !this.isCompletedOrCancelledStatus(challengeStatusForReview); + !this.isCompletedOrCancelledStatus(challengeStatusForReview) && + normalizedPhaseName !== 'screening' && + normalizedPhaseName !== 'checkpoint screening'; const sanitizeScores = shouldMaskReviewDetails || @@ -3456,11 +3458,16 @@ export class ReviewService { !isOwnSubmission && normalizedPhaseName === 'iterative review' && this.isFirst2FinishChallenge(challengeForReview); + const shouldStripOtherReviewerContent = + shouldMaskOtherReviewerDetails && + !['screening', 'checkpoint screening'].includes( + normalizedPhaseName ?? '', + ); const shouldStripReviewItems = shouldMaskReviewDetails || shouldTrimIterativeReviewForOtherSubmitters || shouldLimitNonOwnerVisibility || - shouldMaskOtherReviewerDetails; + shouldStripOtherReviewerContent; if (!isThin) { sanitizedReview.reviewItems = shouldStripReviewItems diff --git a/src/api/submission/submission.service.spec.ts b/src/api/submission/submission.service.spec.ts index abc4aab..8cdb3e7 100644 --- a/src/api/submission/submission.service.spec.ts +++ b/src/api/submission/submission.service.spec.ts @@ -997,6 +997,7 @@ describe('SubmissionService', () => { id: 'review-self', resourceId: 'resource-self', submissionId: 'submission-1', + phaseId: 'phase-123', finalScore: 95, initialScore: 92, reviewItems: [ @@ -1017,6 +1018,7 @@ describe('SubmissionService', () => { id: 'review-other', resourceId: 'resource-other', submissionId: 'submission-1', + phaseId: 'phase-123', finalScore: 80, initialScore: 78, reviewItems: [ @@ -1100,5 +1102,144 @@ describe('SubmissionService', () => { expect(otherReview?.reviewerHandle).toBe('otherHandle'); expect(otherReview?.reviewerMaxRating).toBe(1800); }); + + it('preserves screening review items and scores for other reviewers', async () => { + const now = new Date('2025-01-06T10:00:00Z'); + challengeApiServiceMock.getChallengeDetail.mockResolvedValue({ + id: 'challenge-1', + status: ChallengeStatus.ACTIVE, + type: 'Challenge', + legacy: {}, + phases: [ + { + id: 'phase-review', + phaseId: 'legacy-phase-review', + name: 'Review', + }, + { + id: 'phase-screening', + phaseId: 'legacy-phase-screening', + name: 'Screening', + }, + ], + }); + resourceApiServiceListMock.getMemberResourcesRoles.mockResolvedValue([ + { + roleName: 'Reviewer', + id: 'resource-self', + memberId: '101', + }, + ]); + + const submissions = [ + { + id: 'submission-2', + challengeId: 'challenge-1', + memberId: 'submitter-1', + submittedDate: now, + createdAt: now, + updatedAt: now, + type: SubmissionType.CONTEST_SUBMISSION, + status: SubmissionStatus.ACTIVE, + review: [ + { + id: 'review-self', + resourceId: 'resource-self', + submissionId: 'submission-2', + phaseId: 'phase-review', + finalScore: 90, + initialScore: 88, + reviewItems: [ + { + id: 'item-self', + scorecardQuestionId: 'q1', + initialAnswer: 'YES', + finalAnswer: 'YES', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'reviewer', + updatedAt: now, + updatedBy: 'reviewer', + }, + { + id: 'review-screening', + resourceId: 'resource-other', + submissionId: 'submission-2', + phaseId: 'phase-screening', + finalScore: 75, + initialScore: 70, + reviewItems: [ + { + id: 'item-screening', + scorecardQuestionId: 'q2', + initialAnswer: 'NO', + finalAnswer: 'NO', + reviewItemComments: [], + }, + ], + createdAt: now, + createdBy: 'screening-reviewer', + updatedAt: now, + updatedBy: 'screening-reviewer', + }, + ], + reviewSummation: [], + legacyChallengeId: null, + prizeId: null, + }, + ]; + + prismaMock.submission.findMany.mockResolvedValue( + submissions.map((entry) => ({ ...entry })), + ); + prismaMock.submission.count.mockResolvedValue(submissions.length); + prismaMock.submission.findFirst.mockResolvedValue({ + id: 'submission-2', + }); + + resourcePrismaListMock.resource.findMany.mockResolvedValue([ + { id: 'resource-self', memberId: '101' }, + { id: 'resource-other', memberId: '202' }, + ]); + + memberPrismaMock.member.findMany.mockResolvedValue([ + { + userId: BigInt(101), + handle: 'selfHandle', + maxRating: { rating: 2500 }, + }, + { + userId: BigInt(202), + handle: 'screeningHandle', + maxRating: { rating: 2000 }, + }, + ]); + + const result = await listService.listSubmission( + { + userId: '101', + isMachine: false, + roles: [UserRole.Reviewer], + } as any, + { challengeId: 'challenge-1' } as any, + { page: 1, perPage: 50 } as any, + ); + + const submissionResult = result.data.find( + (entry) => entry.id === 'submission-2', + ); + const screeningReview = submissionResult?.review?.find( + (review) => review.id === 'review-screening', + ); + + expect(screeningReview).toBeDefined(); + expect(screeningReview?.initialScore).toBe(70); + expect(screeningReview?.finalScore).toBe(75); + expect(screeningReview?.reviewItems).toHaveLength(1); + expect(screeningReview?.reviewerHandle).toBe('screeningHandle'); + expect(screeningReview?.reviewerMaxRating).toBe(2000); + }); }); }); diff --git a/src/api/submission/submission.service.ts b/src/api/submission/submission.service.ts index 59cdfc4..1fb4db3 100644 --- a/src/api/submission/submission.service.ts +++ b/src/api/submission/submission.service.ts @@ -2409,14 +2409,32 @@ export class SubmissionService { const ownsReview = resourceId.length > 0 && roleSummary.reviewerResourceIds.includes(resourceId); + const resolvedPhaseName = this.getPhaseNameFromId( + challenge, + (review as any).phaseId ?? null, + ); + const normalizedPhaseName = + this.normalizePhaseName(resolvedPhaseName); + const isScreeningPhase = + normalizedPhaseName === 'screening' || + normalizedPhaseName === 'checkpoint screening'; if (!ownsReview) { - review.initialScore = null; - review.finalScore = null; - if (Array.isArray(review.reviewItems)) { - review.reviewItems = []; + if (!isScreeningPhase) { + review.initialScore = null; + review.finalScore = null; + review.reviewItems = Array.isArray(review.reviewItems) + ? [] + : []; } else { - review.reviewItems = []; + review.initialScore = + typeof review.initialScore === 'number' + ? review.initialScore + : (review.initialScore ?? null); + review.finalScore = + typeof review.finalScore === 'number' + ? review.finalScore + : (review.finalScore ?? null); } } } From 1b8bbfd48a1d9855d3558e4a898fbb0eddd43842 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Tue, 28 Oct 2025 21:53:28 +1100 Subject: [PATCH 46/48] Tweaks for showing specific details to mixed roles --- src/api/review/review.service.spec.ts | 140 ++++++++++++++++++++++++++ src/api/review/review.service.ts | 7 +- 2 files changed, 145 insertions(+), 2 deletions(-) diff --git a/src/api/review/review.service.spec.ts b/src/api/review/review.service.spec.ts index ca6c288..03f7232 100644 --- a/src/api/review/review.service.spec.ts +++ b/src/api/review/review.service.spec.ts @@ -806,6 +806,35 @@ describe('ReviewService.getReview authorization checks', () => { }); }); + it('allows copilots who also have reviewer resources to access other reviews before completion', async () => { + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValueOnce([ + baseReviewerResource, + { + ...baseReviewerResource, + id: 'resource-copilot', + roleName: 'Copilot', + }, + ]); + + prismaMock.review.findUniqueOrThrow.mockResolvedValueOnce({ + ...defaultReviewData(), + resourceId: 'resource-other', + phaseId: 'phase-review', + }); + + const copilotReviewerUser: JwtUser = { + ...baseAuthUser, + roles: [UserRole.Reviewer, UserRole.Copilot], + }; + + await expect( + service.getReview(copilotReviewerUser, 'review-1'), + ).resolves.toMatchObject({ + id: 'review-1', + resourceId: 'resource-other', + }); + }); + it('allows reviewers to access screening reviews that are not their own before completion', async () => { prismaMock.review.findUniqueOrThrow.mockResolvedValue({ ...defaultReviewData(), @@ -1544,6 +1573,117 @@ describe('ReviewService.getReviews reviewer visibility', () => { expect(callArgs.where.resourceId).toBeUndefined(); }); + it('uses the most permissive access when the requester is both reviewer and copilot', async () => { + const now = new Date(); + const reviewerResource = { + ...buildResource('Reviewer'), + id: 'resource-reviewer', + }; + const copilotResource = { + ...buildResource('Copilot'), + id: 'resource-copilot', + roleId: 'role-copilot', + }; + + resourceApiServiceMock.getMemberResourcesRoles.mockResolvedValue([ + reviewerResource, + copilotResource, + ]); + + const reviewRecords = [ + { + id: 'review-1', + resourceId: 'resource-other', + submissionId: 'submission-1', + phaseId: 'phase-review', + scorecardId: 'scorecard-1', + typeId: 'type-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + initialScore: 80, + finalScore: 85, + createdAt: now, + createdBy: 'system', + updatedAt: now, + updatedBy: 'system', + reviewItems: [ + { + id: 'item-1', + reviewId: 'review-1', + scorecardQuestionId: 'question-1', + initialAnswer: 'Yes', + finalAnswer: 'No', + managerComment: null, + reviewItemComments: [], + }, + ], + submission: { + id: 'submission-1', + memberId: 'submitter-1', + challengeId: 'challenge-1', + }, + }, + { + id: 'review-2', + resourceId: 'resource-reviewer', + submissionId: 'submission-2', + phaseId: 'phase-review', + scorecardId: 'scorecard-1', + typeId: 'type-1', + status: ReviewStatus.COMPLETED, + reviewDate: now, + committed: true, + metadata: null, + initialScore: 90, + finalScore: 95, + createdAt: now, + createdBy: 'system', + updatedAt: now, + updatedBy: 'system', + reviewItems: [ + { + id: 'item-2', + reviewId: 'review-2', + scorecardQuestionId: 'question-2', + initialAnswer: 'Yes', + finalAnswer: 'Yes', + managerComment: 'Great work', + reviewItemComments: [], + }, + ], + submission: { + id: 'submission-2', + memberId: 'submitter-2', + challengeId: 'challenge-1', + }, + }, + ]; + + prismaMock.review.findMany.mockResolvedValue(reviewRecords); + prismaMock.review.count.mockResolvedValue(reviewRecords.length); + + const response = await service.getReviews( + { + ...baseAuthUser, + roles: [UserRole.Reviewer, UserRole.Copilot], + }, + undefined, + 'challenge-1', + ); + + const callArgs = prismaMock.review.findMany.mock.calls[0][0]; + expect(callArgs.where.resourceId).toBeUndefined(); + + const otherReview = response.data.find( + (review) => review.id === 'review-1', + ); + expect(otherReview).toBeDefined(); + expect(otherReview?.finalScore).toBe(85); + expect(otherReview?.reviewItems?.length).toBe(1); + }); + it('hides scores and review items for other reviewers on active challenges while keeping reviewer metadata', async () => { const now = new Date(); const reviewerUser: JwtUser = { diff --git a/src/api/review/review.service.ts b/src/api/review/review.service.ts index a115c38..e250f9e 100644 --- a/src/api/review/review.service.ts +++ b/src/api/review/review.service.ts @@ -3432,6 +3432,7 @@ export class ReviewService { challengeForReview?.status ?? challengeDetail?.status ?? null; const shouldMaskOtherReviewerDetails = !isPrivilegedRequester && + !hasCopilotRoleForChallenge && reviewerResourceIdSet.size > 0 && !isReviewerForReview && !this.isCompletedOrCancelledStatus(challengeStatusForReview) && @@ -3690,7 +3691,9 @@ export class ReviewService { ); isReviewerForReview = reviewerResourceIds.has(reviewResourceId); - if (reviewerResources.length > 0) { + if (hasCopilotRole) { + // Copilots on the challenge can view any review regardless of reviewer ownership. + } else if (reviewerResources.length > 0) { const challengeInFinalState = [ ChallengeStatus.COMPLETED, ChallengeStatus.CANCELLED_FAILED_REVIEW, @@ -3720,7 +3723,7 @@ export class ReviewService { details: { challengeId, reviewId, requester: uid }, }); } - } else if (!hasCopilotRole) { + } else { // Confirm the user has actually submitted to this challenge (has a submission record) const mySubs = await this.prisma.submission.findMany({ where: { challengeId, memberId: uid }, From afb492d07a882ea40f5d515aa0545d0b5f54562c Mon Sep 17 00:00:00 2001 From: Hentry Martin Date: Tue, 28 Oct 2025 18:43:33 +0100 Subject: [PATCH 47/48] fix: lint --- src/shared/modules/global/utils.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shared/modules/global/utils.service.ts b/src/shared/modules/global/utils.service.ts index 0a3f00e..60fa295 100644 --- a/src/shared/modules/global/utils.service.ts +++ b/src/shared/modules/global/utils.service.ts @@ -11,7 +11,7 @@ export class Utils { timeout: process.env.REVIEW_SERVICE_PRISMA_TIMEOUT ? parseInt(process.env.REVIEW_SERVICE_PRISMA_TIMEOUT, 10) : 10000, - } - } + }, + }; } } From 6167afbefbb0d1c7302df3473bffcd429dc8ffb6 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 30 Oct 2025 11:25:18 +1100 Subject: [PATCH 48/48] Incremental update tweaks --- prisma/migrate.ts | 459 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 458 insertions(+), 1 deletion(-) diff --git a/prisma/migrate.ts b/prisma/migrate.ts index 6f79fb4..c678cad 100644 --- a/prisma/migrate.ts +++ b/prisma/migrate.ts @@ -56,6 +56,28 @@ if (incrementalSinceInput) { } const isIncrementalRun = incrementalSince !== null; +const cliArgs = new Set(process.argv.slice(2)); +const shouldRepairMaps = cliArgs.has('--repair-maps'); +if (shouldRepairMaps) { + console.log( + '[map] --repair-maps enabled; duplicate or invalid mapping entries will be cleaned.', + ); +} + +const errorSummary = new Map< + string, + { count: number; files: Set; examples: string[] } +>(); +const errorPositionTracker = new Map(); +const sourceDeltaCounters = new Map(); + +const incrementSourceDelta = (key: string) => { + if (!isIncrementalRun) { + return; + } + sourceDeltaCounters.set(key, (sourceDeltaCounters.get(key) ?? 0) + 1); +}; + const parseDateInput = (value: string | Date | null | undefined) => { if (!value) { return null; @@ -89,6 +111,37 @@ const shouldProcessRecord = ( const buildLogPrefix = (type: string, file: string, subtype?: string) => subtype ? `[${type}][${subtype}][${file}]` : `[${type}][${file}]`; +const extractLegacyIdentifier = (record: any): string | null => { + if (!record || typeof record !== 'object') { + return null; + } + const candidates = [ + 'legacyId', + 'legacy_id', + 'legacySubmissionId', + 'submission_id', + 'scorecard_id', + 'scorecard_group_id', + 'scorecard_section_id', + 'scorecard_question_id', + 'review_id', + 'review_item_id', + 'review_item_comment_id', + 'project_id', + 'upload_id', + 'resource_id', + 'ai_workflow_id', + 'llm_provider_id', + 'llm_model_id', + ]; + for (const key of candidates) { + if (Object.prototype.hasOwnProperty.call(record, key)) { + return describeLegacyId(record[key]); + } + } + return null; +}; + const logRecordConversionError = ( type: string, file: string, @@ -96,10 +149,24 @@ const logRecordConversionError = ( record: unknown, message: string, subtype?: string, + context?: { index?: number }, ) => { const prefix = buildLogPrefix(type, file, subtype); const errorMessage = err instanceof Error ? err.message : String(err); - console.error(`${prefix} ${message}`); + let recordPosition: number | null = null; + if (context?.index != null) { + recordPosition = context.index + 1; + } else { + const trackerKey = `${type}|${file}|${subtype ?? 'default'}`; + const nextPosition = (errorPositionTracker.get(trackerKey) ?? 0) + 1; + errorPositionTracker.set(trackerKey, nextPosition); + recordPosition = nextPosition; + } + const legacyIdentifier = extractLegacyIdentifier(record); + const positionSuffix = recordPosition ? ` record#${recordPosition}` : ''; + const legacySuffix = legacyIdentifier ? ` legacy=${legacyIdentifier}` : ''; + + console.error(`${prefix} ${message}${positionSuffix}${legacySuffix}`); console.error(`${prefix} Error detail: ${errorMessage}`); if (err instanceof Error && err.stack) { console.error(err); @@ -115,6 +182,20 @@ const logRecordConversionError = ( `${prefix} Failed to stringify record while skipping: ${stringifyMessage}`, ); } + + const aggregationKey = `${type}${subtype ? `:${subtype}` : ''} | ${message}`; + const summary = errorSummary.get(aggregationKey) ?? { + count: 0, + files: new Set(), + examples: [] as string[], + }; + summary.count += 1; + summary.files.add(file); + if (summary.examples.length < 5) { + const exampleLabel = `${file}${recordPosition ? `#${recordPosition}` : ''}${legacyIdentifier ? `:${legacyIdentifier}` : ''}`; + summary.examples.push(exampleLabel); + } + errorSummary.set(aggregationKey, summary); }; const modelMappingKeys = [ @@ -181,6 +262,80 @@ function readIdMap(filename: string): Map { return new Map(); } +function validateIdMap(map: Map, label: string) { + const seenValues = new Map(); + let duplicateCount = 0; + let cleanedCount = 0; + + for (const [key, value] of map.entries()) { + if (!value || typeof value !== 'string') { + console.warn( + `[map] ${label}: removing invalid mapping for key=${key} value=${describeLegacyId(value)}`, + ); + map.delete(key); + cleanedCount += 1; + continue; + } + + if (seenValues.has(value)) { + duplicateCount += 1; + const firstKey = seenValues.get(value); + console.warn( + `[map] ${label}: duplicate target id ${value} for keys ${firstKey} and ${key}`, + ); + if (shouldRepairMaps) { + map.delete(key); + cleanedCount += 1; + continue; + } + } else { + seenValues.set(value, key); + } + } + + if (duplicateCount > 0 && !shouldRepairMaps) { + console.warn( + `[map] ${label}: detected ${duplicateCount} duplicate value(s). Re-run with --repair-maps to clean them automatically.`, + ); + } + + if (cleanedCount > 0 && shouldRepairMaps) { + console.log( + `[map] ${label}: repaired ${cleanedCount} mapping entry/entries.`, + ); + } +} + +function validateAllIdMaps() { + [ + { label: 'projectIdMap', map: projectIdMap }, + { label: 'scorecardIdMap', map: scorecardIdMap }, + { label: 'scorecardGroupIdMap', map: scorecardGroupIdMap }, + { label: 'scorecardSectionIdMap', map: scorecardSectionIdMap }, + { label: 'scorecardQuestionIdMap', map: scorecardQuestionIdMap }, + { label: 'reviewIdMap', map: reviewIdMap }, + { label: 'reviewItemIdMap', map: reviewItemIdMap }, + { + label: 'reviewItemCommentReviewItemCommentIdMap', + map: reviewItemCommentReviewItemCommentIdMap, + }, + { + label: 'reviewItemCommentAppealIdMap', + map: reviewItemCommentAppealIdMap, + }, + { + label: 'reviewItemCommentAppealResponseIdMap', + map: reviewItemCommentAppealResponseIdMap, + }, + { label: 'uploadIdMap', map: uploadIdMap }, + { label: 'submissionIdMap', map: submissionIdMap }, + { label: 'llmProviderIdMap', map: llmProviderIdMap }, + { label: 'llmModelIdMap', map: llmModelIdMap }, + { label: 'aiWorkflowIdMap', map: aiWorkflowIdMap }, + { label: 'resourceSubmissionIdMap', map: resourceSubmissionIdMap }, + ].forEach(({ label, map }) => validateIdMap(map, label)); +} + const describeLegacyId = (value: unknown): string => { if (value === null) { return 'null'; @@ -303,6 +458,8 @@ const llmModelIdMap = readIdMap('llmModelIdMap'); const aiWorkflowIdMap = readIdMap('aiWorkflowIdMap'); const resourceSubmissionIdMap = readIdMap('resourceSubmissionIdMap'); +validateAllIdMaps(); + // read resourceSubmissionSet const rsSetFile = '.tmp/resourceSubmissionSet.json'; if (fs.existsSync(rsSetFile)) { @@ -607,6 +764,7 @@ async function handleElasticSearchSubmission(item) { if (!shouldProcessRecord(createdAudit, updatedAudit)) { return; } + incrementSourceDelta('submission'); currentSubmissions.push(item); // if we can batch insert data, + if (currentSubmissions.length >= batchSize) { @@ -1057,9 +1215,11 @@ async function processType(type: string, subtype?: string) { typeof convertProjectResult >; const processedData: ChallengeResultEntity[] = []; + let recordIndex = 0; for (const pr of jsonData[key]) { const mapKey = `${pr.project_id}${pr.user_id}`; if (hasMappedId(projectIdMap, mapKey)) { + recordIndex += 1; continue; } try { @@ -1073,8 +1233,11 @@ async function processType(type: string, subtype?: string) { err, pr, `Failed to convert projectResult for challengeId ${pr.project_id}, userId ${pr.user_id}`, + undefined, + { index: recordIndex }, ); } + recordIndex += 1; } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -1109,10 +1272,13 @@ async function processType(type: string, subtype?: string) { }); } } else { + let recordIndex = 0; for (const pr of jsonData[key]) { if (!shouldProcessRecord(pr.create_date, pr.modify_date)) { + recordIndex += 1; continue; } + incrementSourceDelta(type); const mapKey = `${pr.project_id}${pr.user_id}`; let data: ReturnType; try { @@ -1124,7 +1290,10 @@ async function processType(type: string, subtype?: string) { err, pr, `Failed to convert projectResult for challengeId ${pr.project_id}, userId ${pr.user_id}`, + undefined, + { index: recordIndex }, ); + recordIndex += 1; continue; } try { @@ -1145,6 +1314,7 @@ async function processType(type: string, subtype?: string) { ); console.error(err); } + recordIndex += 1; } } break; @@ -1180,6 +1350,7 @@ async function processType(type: string, subtype?: string) { if (!isIncrementalRun) { type ScorecardEntity = ReturnType; const processedData: ScorecardEntity[] = []; + let recordIndex = 0; for (const sc of jsonData[key]) { const id = nanoid(14); try { @@ -1193,8 +1364,11 @@ async function processType(type: string, subtype?: string) { err, sc, `Failed to convert scorecard legacyId ${sc.scorecard_id}`, + undefined, + { index: recordIndex }, ); } + recordIndex += 1; } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -1226,10 +1400,13 @@ async function processType(type: string, subtype?: string) { }); } } else { + let recordIndex = 0; for (const sc of jsonData[key]) { if (!shouldProcessRecord(sc.create_date, sc.modify_date)) { + recordIndex += 1; continue; } + incrementSourceDelta(type); const existingId = getMappedId(scorecardIdMap, sc.scorecard_id); const id = existingId ?? nanoid(14); let data: ReturnType; @@ -1242,7 +1419,10 @@ async function processType(type: string, subtype?: string) { err, sc, `Failed to convert scorecard legacyId ${sc.scorecard_id}`, + undefined, + { index: recordIndex }, ); + recordIndex += 1; continue; } try { @@ -1261,6 +1441,7 @@ async function processType(type: string, subtype?: string) { ); console.error(err); } + recordIndex += 1; } } break; @@ -1290,8 +1471,10 @@ async function processType(type: string, subtype?: string) { if (!isIncrementalRun) { type ScorecardGroupEntity = ReturnType; const processedData: ScorecardGroupEntity[] = []; + let recordIndex = 0; for (const group of jsonData[key]) { if (hasMappedId(scorecardGroupIdMap, group.scorecard_group_id)) { + recordIndex += 1; continue; } const id = nanoid(14); @@ -1306,8 +1489,11 @@ async function processType(type: string, subtype?: string) { err, group, `Failed to convert scorecardGroup legacyId ${group.scorecard_group_id}`, + undefined, + { index: recordIndex }, ); } + recordIndex += 1; } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -1339,10 +1525,13 @@ async function processType(type: string, subtype?: string) { }); } } else { + let recordIndex = 0; for (const group of jsonData[key]) { if (!shouldProcessRecord(group.create_date, group.modify_date)) { + recordIndex += 1; continue; } + incrementSourceDelta(type); const existingId = getMappedId( scorecardGroupIdMap, group.scorecard_group_id, @@ -1358,7 +1547,10 @@ async function processType(type: string, subtype?: string) { err, group, `Failed to convert scorecardGroup legacyId ${group.scorecard_group_id}`, + undefined, + { index: recordIndex }, ); + recordIndex += 1; continue; } try { @@ -1381,6 +1573,7 @@ async function processType(type: string, subtype?: string) { ); console.error(err); } + recordIndex += 1; } } break; @@ -1410,10 +1603,12 @@ async function processType(type: string, subtype?: string) { if (!isIncrementalRun) { type ScorecardSectionEntity = ReturnType; const processedData: ScorecardSectionEntity[] = []; + let recordIndex = 0; for (const section of jsonData[key]) { if ( hasMappedId(scorecardSectionIdMap, section.scorecard_section_id) ) { + recordIndex += 1; continue; } const id = nanoid(14); @@ -1432,8 +1627,11 @@ async function processType(type: string, subtype?: string) { err, section, `Failed to convert scorecardSection legacyId ${section.scorecard_section_id}`, + undefined, + { index: recordIndex }, ); } + recordIndex += 1; } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -1465,12 +1663,15 @@ async function processType(type: string, subtype?: string) { }); } } else { + let recordIndex = 0; for (const section of jsonData[key]) { if ( !shouldProcessRecord(section.create_date, section.modify_date) ) { + recordIndex += 1; continue; } + incrementSourceDelta(type); const existingId = getMappedId( scorecardSectionIdMap, section.scorecard_section_id, @@ -1486,7 +1687,10 @@ async function processType(type: string, subtype?: string) { err, section, `Failed to convert scorecardSection legacyId ${section.scorecard_section_id}`, + undefined, + { index: recordIndex }, ); + recordIndex += 1; continue; } try { @@ -1509,6 +1713,7 @@ async function processType(type: string, subtype?: string) { ); console.error(err); } + recordIndex += 1; } } break; @@ -1545,6 +1750,7 @@ async function processType(type: string, subtype?: string) { if (!isIncrementalRun) { type ScorecardQuestionEntity = ReturnType; const processedData: ScorecardQuestionEntity[] = []; + let recordIndex = 0; for (const question of jsonData[key]) { if ( hasMappedId( @@ -1552,6 +1758,7 @@ async function processType(type: string, subtype?: string) { question.scorecard_question_id, ) ) { + recordIndex += 1; continue; } const id = nanoid(14); @@ -1570,8 +1777,11 @@ async function processType(type: string, subtype?: string) { err, question, `Failed to convert scorecardQuestion legacyId ${question.scorecard_question_id}`, + undefined, + { index: recordIndex }, ); } + recordIndex += 1; } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -1603,12 +1813,15 @@ async function processType(type: string, subtype?: string) { }); } } else { + let recordIndex = 0; for (const question of jsonData[key]) { if ( !shouldProcessRecord(question.create_date, question.modify_date) ) { + recordIndex += 1; continue; } + incrementSourceDelta(type); const existingId = getMappedId( scorecardQuestionIdMap, question.scorecard_question_id, @@ -1624,7 +1837,10 @@ async function processType(type: string, subtype?: string) { err, question, `Failed to convert scorecardQuestion legacyId ${question.scorecard_question_id}`, + undefined, + { index: recordIndex }, ); + recordIndex += 1; continue; } try { @@ -1647,6 +1863,7 @@ async function processType(type: string, subtype?: string) { ); console.error(err); } + recordIndex += 1; } } break; @@ -1684,8 +1901,10 @@ async function processType(type: string, subtype?: string) { if (!isIncrementalRun) { type ReviewEntity = ReturnType; const processedData: ReviewEntity[] = []; + let recordIndex = 0; for (const review of jsonData[key]) { if (hasMappedId(reviewIdMap, review.review_id)) { + recordIndex += 1; continue; } const id = nanoid(14); @@ -1700,8 +1919,11 @@ async function processType(type: string, subtype?: string) { err, review, `Failed to convert review legacyId ${review.review_id}`, + undefined, + { index: recordIndex }, ); } + recordIndex += 1; } const totalBatches = Math.ceil(processedData.length / batchSize); for (let i = 0; i < processedData.length; i += batchSize) { @@ -1731,12 +1953,15 @@ async function processType(type: string, subtype?: string) { }); } } else { + let recordIndex = 0; for (const review of jsonData[key]) { if ( !shouldProcessRecord(review.create_date, review.modify_date) ) { + recordIndex += 1; continue; } + incrementSourceDelta(type); const existingId = getMappedId(reviewIdMap, review.review_id); const id = existingId ?? nanoid(14); let data: ReturnType; @@ -1749,7 +1974,10 @@ async function processType(type: string, subtype?: string) { err, review, `Failed to convert review legacyId ${review.review_id}`, + undefined, + { index: recordIndex }, ); + recordIndex += 1; continue; } try { @@ -1768,6 +1996,7 @@ async function processType(type: string, subtype?: string) { ); console.error(err); } + recordIndex += 1; } } break; @@ -1861,6 +2090,7 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(item.create_date, item.modify_date)) { continue; } + incrementSourceDelta(type); const existingId = getMappedId( reviewItemIdMap, item.review_item_id, @@ -1897,6 +2127,7 @@ async function processType(type: string, subtype?: string) { switch (subtype) { case 'reviewItemComment': { console.log(`[${type}][${subtype}][${file}] Processing file`); + const deltaKey = `${type}:${subtype}`; const isSupportedType = (c) => reviewItemCommentTypeMap[c.comment_type_id] in LegacyCommentType; @@ -2002,6 +2233,7 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } + incrementSourceDelta(deltaKey); const existingId = getMappedId( reviewItemCommentReviewItemCommentIdMap, c.review_item_comment_id, @@ -2047,6 +2279,7 @@ async function processType(type: string, subtype?: string) { } case 'appeal': { console.log(`[${type}][${subtype}][${file}] Processing file`); + const deltaKey = `${type}:${subtype}`; const isAppeal = (c) => reviewItemCommentTypeMap[c.comment_type_id] === 'Appeal'; const convertAppeal = (c, recordId: string) => { @@ -2143,6 +2376,7 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } + incrementSourceDelta(deltaKey); const existingId = getMappedId( reviewItemCommentAppealIdMap, c.review_item_id, @@ -2188,6 +2422,7 @@ async function processType(type: string, subtype?: string) { } case 'appealResponse': { console.log(`[${type}][${subtype}][${file}] Processing file`); + const deltaKey = `${type}:${subtype}`; const isAppealResponse = (c) => reviewItemCommentTypeMap[c.comment_type_id] === 'Appeal Response'; @@ -2290,6 +2525,7 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } + incrementSourceDelta(deltaKey); const existingId = getMappedId( reviewItemCommentAppealResponseIdMap, c.review_item_comment_id, @@ -2404,6 +2640,7 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } + incrementSourceDelta(type); const existingId = getMappedId( llmProviderIdMap, c.llm_provider_id, @@ -2519,6 +2756,7 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } + incrementSourceDelta(type); const existingId = getMappedId(llmModelIdMap, c.llm_model_id); const id = existingId ?? nanoid(14); let data: ReturnType; @@ -2639,6 +2877,7 @@ async function processType(type: string, subtype?: string) { if (!shouldProcessRecord(c.create_date, c.modify_date)) { continue; } + incrementSourceDelta(type); const existingId = getMappedId(aiWorkflowIdMap, c.ai_workflow_id); const id = existingId ?? nanoid(14); let data: ReturnType; @@ -2745,6 +2984,220 @@ async function doImportResourceSubmission() { } } +function buildIncrementalWhereClause( + dateFields: string[], + baseFilter?: Record, +): Record | undefined { + if (!incrementalSince) { + return baseFilter; + } + + const dateFilter: Record = { + OR: dateFields.map((field) => ({ [field]: { gte: incrementalSince } })), + }; + if (!baseFilter) { + return dateFilter; + } + return { AND: [baseFilter, dateFilter] }; +} + +async function verifyIncrementalReferentialIntegrity() { + if (!isIncrementalRun || !incrementalSince) { + return; + } + + console.log('[incremental] Checking referential integrity...'); + const checks: Array<{ label: string; query: () => Promise }> = [ + { + label: 'reviewItem.review', + query: () => + prisma.reviewItem.count({ + where: buildIncrementalWhereClause(['updatedAt', 'createdAt'], { + review: { is: null }, + }), + }), + }, + { + label: 'reviewItemComment.reviewItem', + query: () => + prisma.reviewItemComment.count({ + where: buildIncrementalWhereClause(['updatedAt', 'createdAt'], { + reviewItem: { is: null }, + }), + }), + }, + { + label: 'appeal.reviewItemComment', + query: () => + prisma.appeal.count({ + where: buildIncrementalWhereClause(['updatedAt', 'createdAt'], { + reviewItemComment: { is: null }, + }), + }), + }, + ]; + + for (const check of checks) { + try { + const orphanCount = await check.query(); + if (orphanCount > 0) { + console.warn( + `[incremental] Referential issue: ${check.label} has ${orphanCount} orphan record(s) in the incremental window.`, + ); + } else { + console.log( + `[incremental] Referential check passed for ${check.label}.`, + ); + } + } catch (err: any) { + console.warn( + `[incremental] Referential check failed for ${check.label}: ${err.message}`, + ); + } + } +} + +async function verifyIncrementalMigration() { + if (!isIncrementalRun || !incrementalSince) { + return; + } + + console.log('[incremental] Verifying incremental migration results...'); + const configs = new Map< + string, + { delegate: any; dateFields: string[]; where?: any } + >([ + [ + 'submission', + { delegate: prisma.submission, dateFields: ['updatedAt', 'createdAt'] }, + ], + [ + 'project_result', + { + delegate: prisma.challengeResult, + dateFields: ['updatedAt', 'createdAt'], + }, + ], + [ + 'scorecard', + { delegate: prisma.scorecard, dateFields: ['updatedAt', 'createdAt'] }, + ], + [ + 'scorecard_group', + { + delegate: prisma.scorecardGroup, + dateFields: ['updatedAt', 'createdAt'], + }, + ], + [ + 'scorecard_section', + { + delegate: prisma.scorecardSection, + dateFields: ['updatedAt', 'createdAt'], + }, + ], + [ + 'scorecard_question', + { + delegate: prisma.scorecardQuestion, + dateFields: ['updatedAt', 'createdAt'], + }, + ], + [ + 'review', + { delegate: prisma.review, dateFields: ['updatedAt', 'createdAt'] }, + ], + [ + 'review_item', + { delegate: prisma.reviewItem, dateFields: ['updatedAt', 'createdAt'] }, + ], + [ + 'review_item_comment:reviewItemComment', + { + delegate: prisma.reviewItemComment, + dateFields: ['updatedAt', 'createdAt'], + }, + ], + [ + 'review_item_comment:appeal', + { delegate: prisma.appeal, dateFields: ['updatedAt', 'createdAt'] }, + ], + [ + 'review_item_comment:appealResponse', + { + delegate: prisma.appealResponse, + dateFields: ['updatedAt', 'createdAt'], + }, + ], + [ + 'llm_provider', + { delegate: prisma.llmProvider, dateFields: ['updatedAt', 'createdAt'] }, + ], + [ + 'llm_model', + { delegate: prisma.llmModel, dateFields: ['updatedAt', 'createdAt'] }, + ], + [ + 'ai_workflow', + { delegate: prisma.aiWorkflow, dateFields: ['updatedAt', 'createdAt'] }, + ], + [ + 'resource_submission', + { + delegate: prisma.resourceSubmission, + dateFields: ['updatedAt', 'createdAt'], + }, + ], + ]); + + for (const [key, sourceCount] of sourceDeltaCounters.entries()) { + const config = configs.get(key) ?? configs.get(key.split(':')[0]); + if (!config) { + continue; + } + + let targetCount = 0; + try { + const whereClause = buildIncrementalWhereClause( + config.dateFields, + config.where, + ); + targetCount = await config.delegate.count({ where: whereClause }); + } catch (err: any) { + console.warn( + `[incremental] ${key}: failed to read target counts (${err.message}).`, + ); + continue; + } + + if (sourceCount !== targetCount) { + console.warn( + `[incremental] ${key}: source=${sourceCount}, target=${targetCount}`, + ); + } else { + console.log(`[incremental] ${key}: counts OK (${targetCount}).`); + } + } + + await verifyIncrementalReferentialIntegrity(); +} + +function reportErrorSummary() { + if (errorSummary.size === 0) { + console.log('No conversion errors recorded.'); + return; + } + + console.log('Error summary by type:'); + errorSummary.forEach((info, key) => { + const files = Array.from(info.files).join(', '); + const examples = info.examples.join(', '); + console.log( + `- ${key}: count=${info.count}, files=[${files}]${examples ? `, examples=${examples}` : ''}`, + ); + }); +} + async function upsertResourceSubmission(data) { const { id, ...updateData } = data; try { @@ -2795,6 +3248,7 @@ async function migrateResourceSubmissions() { setMappedId(resourceSubmissionIdMap, key, data.id); } if (shouldPersist || !existingId) { + incrementSourceDelta('resource_submission'); await handleResourceSubmission(data); } } else if (!resourceSubmissionSet.has(key)) { @@ -2844,6 +3298,9 @@ async function migrate() { console.log('Starting importing resource-submissions...'); await migrateResourceSubmissions(); console.log('Resource-submissions import completed.'); + + await verifyIncrementalMigration(); + reportErrorSummary(); } migrate()