diff --git a/backend/src/common/auth/keycloak-auth.service.spec.ts b/backend/src/common/auth/keycloak-auth.service.spec.ts index c589a832..8b849771 100644 --- a/backend/src/common/auth/keycloak-auth.service.spec.ts +++ b/backend/src/common/auth/keycloak-auth.service.spec.ts @@ -1,7 +1,7 @@ -import { Test, TestingModule } from '@nestjs/testing' import { HttpService } from '@nestjs/axios' import { ConfigService } from '@nestjs/config' -import { of } from 'rxjs' +import { Test, TestingModule } from '@nestjs/testing' +import { of, throwError } from 'rxjs' import { KeycloakAuthService } from './keycloak-auth.service' describe('KeycloakAuthService', () => { @@ -35,7 +35,7 @@ describe('KeycloakAuthService', () => { }) it('should request token from Keycloak with client_credentials', async () => { - httpService.post.mockReturnValue(of({ data: { access_token: 'test-token' } })) + httpService.post.mockReturnValue(of({ data: { access_token: 'test-token', expires_in: 300 } })) const token = await service.getBearerToken() @@ -48,4 +48,91 @@ describe('KeycloakAuthService', () => { }), ) }) + + it('should reuse cached token on subsequent calls', async () => { + httpService.post.mockReturnValue(of({ data: { access_token: 'test-token', expires_in: 300 } })) + + const token1 = await service.getBearerToken() + const token2 = await service.getBearerToken() + + expect(token1).toBe('test-token') + expect(token2).toBe('test-token') + // Should only make one HTTP call since token is cached + expect(httpService.post).toHaveBeenCalledTimes(1) + }) + + it('should refresh token when access token is expired', async () => { + // First call returns token with refresh_token + httpService.post.mockReturnValueOnce( + of({ + data: { + access_token: 'initial-token', + refresh_token: 'refresh-token', + expires_in: 1, // Very short expiry + }, + }), + ) + + const token1 = await service.getBearerToken() + expect(token1).toBe('initial-token') + + // Wait for token to expire (considering the 60s buffer, it should already be considered expired) + // Second call should use refresh_token + httpService.post.mockReturnValueOnce( + of({ + data: { + access_token: 'refreshed-token', + refresh_token: 'new-refresh-token', + expires_in: 300, + }, + }), + ) + + const token2 = await service.getBearerToken() + expect(token2).toBe('refreshed-token') + expect(httpService.post).toHaveBeenCalledTimes(2) + }) + + it('should fall back to client_credentials when refresh fails', async () => { + // First call returns token with refresh_token + httpService.post.mockReturnValueOnce( + of({ + data: { + access_token: 'initial-token', + refresh_token: 'refresh-token', + expires_in: 1, + }, + }), + ) + + await service.getBearerToken() + + // Refresh fails + httpService.post.mockReturnValueOnce(throwError(() => new Error('Refresh failed'))) + // Fallback to client_credentials succeeds + httpService.post.mockReturnValueOnce( + of({ + data: { + access_token: 'new-token', + expires_in: 300, + }, + }), + ) + + const token = await service.getBearerToken() + expect(token).toBe('new-token') + }) + + it('should clear token cache when clearTokenCache is called', async () => { + httpService.post.mockReturnValue(of({ data: { access_token: 'test-token', expires_in: 300 } })) + + await service.getBearerToken() + expect(httpService.post).toHaveBeenCalledTimes(1) + + service.clearTokenCache() + + await service.getBearerToken() + // Should make a new HTTP call after cache is cleared + expect(httpService.post).toHaveBeenCalledTimes(2) + }) }) diff --git a/backend/src/common/auth/keycloak-auth.service.ts b/backend/src/common/auth/keycloak-auth.service.ts index 9feda150..f9ce948a 100644 --- a/backend/src/common/auth/keycloak-auth.service.ts +++ b/backend/src/common/auth/keycloak-auth.service.ts @@ -3,9 +3,18 @@ import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import { firstValueFrom } from 'rxjs' +interface TokenCache { + accessToken: string + refreshToken: string | null + expiresAt: number // Unix timestamp in milliseconds +} + @Injectable() export class KeycloakAuthService { private readonly logger = new Logger(KeycloakAuthService.name) + private tokenCache: TokenCache | null = null + // Refresh token 60 seconds before expiry to avoid race conditions + private readonly TOKEN_EXPIRY_BUFFER_MS = 60 * 1000 constructor( private readonly httpService: HttpService, @@ -13,6 +22,34 @@ export class KeycloakAuthService { ) {} async getBearerToken(): Promise { + // Check if we have a valid cached token + if (this.tokenCache && this.isTokenValid()) { + this.logger.debug('Using cached access token') + return this.tokenCache.accessToken + } + + // Try to refresh if we have a refresh token + if (this.tokenCache?.refreshToken) { + try { + this.logger.log('Access token expired, attempting to refresh...') + return await this.refreshAccessToken() + } catch (error) { + this.logger.warn('Failed to refresh token, falling back to client_credentials:', error) + // Fall through to get a new token + } + } + + // Get a new token using client_credentials + return await this.fetchNewToken() + } + + private isTokenValid(): boolean { + if (!this.tokenCache) return false + const now = Date.now() + return this.tokenCache.expiresAt - this.TOKEN_EXPIRY_BUFFER_MS > now + } + + private async fetchNewToken(): Promise { const keycloakTokenUrl = this.configService.get('admin.keycloakTokenUrl')! const keycloakClientId = this.configService.get('admin.keycloakClientId')! const keycloakClientSecret = this.configService.get('admin.keycloakClientSecret')! @@ -22,7 +59,41 @@ export class KeycloakAuthService { params.append('client_id', keycloakClientId) params.append('client_secret', keycloakClientSecret) - this.logger.log('Requesting Keycloak token from:', keycloakTokenUrl) + try { + this.logger.log('Requesting new Keycloak token from:', keycloakTokenUrl) + const response = await firstValueFrom( + this.httpService.post(keycloakTokenUrl, params, { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + }), + ) + + this.cacheTokenResponse(response.data) + this.logger.log('New Keycloak token obtained successfully') + + return response.data.access_token + } catch (error) { + this.logger.error('Failed to obtain Keycloak bearer token:', error) + throw new HttpException( + 'Failed to authenticate with ICM service', + HttpStatus.INTERNAL_SERVER_ERROR, + ) + } + } + + private async refreshAccessToken(): Promise { + const keycloakTokenUrl = this.configService.get('admin.keycloakTokenUrl')! + const keycloakClientId = this.configService.get('admin.keycloakClientId')! + const keycloakClientSecret = this.configService.get('admin.keycloakClientSecret')! + + const params = new URLSearchParams() + params.append('grant_type', 'refresh_token') + params.append('refresh_token', this.tokenCache!.refreshToken!) + params.append('client_id', keycloakClientId) + params.append('client_secret', keycloakClientSecret) + + this.logger.log('Refreshing Keycloak token...') const response = await firstValueFrom( this.httpService.post(keycloakTokenUrl, params, { headers: { @@ -30,16 +101,38 @@ export class KeycloakAuthService { }, }), ) - // TODO: this.logger.log('ICM token token response received ', response.data.access_token) - this.logger.log('ICM token token response received ') + + this.cacheTokenResponse(response.data) + this.logger.log('Keycloak token refreshed successfully') return response.data.access_token } - catch(error) { - console.error('Failed to obtain ICM bearer token:', error) - throw new HttpException( - 'Failed to authenticate with ICM service', - HttpStatus.INTERNAL_SERVER_ERROR, + + private cacheTokenResponse(data: { + access_token: string + refresh_token?: string + expires_in: number + }): void { + const now = Date.now() + // expires_in is in seconds, convert to milliseconds + const expiresAt = now + data.expires_in * 1000 + + this.tokenCache = { + accessToken: data.access_token, + refreshToken: data.refresh_token || this.tokenCache?.refreshToken || null, + expiresAt, + } + + this.logger.debug( + `Token cached, expires at ${new Date(expiresAt).toISOString()} (in ${data.expires_in} seconds)`, ) } + + /** + * Force clear the token cache. Useful for testing or when token is revoked. + */ + clearTokenCache(): void { + this.tokenCache = null + this.logger.log('Token cache cleared') + } } diff --git a/backend/src/common/utils.ts b/backend/src/common/utils.ts index 6388d763..80ce20c3 100644 --- a/backend/src/common/utils.ts +++ b/backend/src/common/utils.ts @@ -137,3 +137,41 @@ export function getAgeCutoffDate(referenceDate: Date = pacificToday()): Date { export function isEligibleAge(dateOfBirth: Date, referenceDate: Date = pacificToday()): boolean { return dateOfBirth >= getAgeCutoffDate(referenceDate) } + +export interface S3ConnectionParams { + endPoint: string + port: number | undefined + useSSL: boolean + accessKey: string + secretKey: string +} + +// Parses s3URI manually because `new URL()` breaks when credentials contain `/` or `@` +export function parseS3Uri(uri: string): S3ConnectionParams { + const schemeMatch = uri.match(/^(https?):\/\/(.+)$/) + if (!schemeMatch) throw new Error('Invalid s3URI: missing http(s):// scheme') + + const [, scheme, rest] = schemeMatch + + const lastAt = rest.lastIndexOf('@') + if (lastAt === -1) throw new Error('Invalid s3URI: expected user:pass@host') + + const credentials = rest.substring(0, lastAt) + const hostPart = rest.substring(lastAt + 1).split('/')[0] + + const firstColon = credentials.indexOf(':') + if (firstColon === -1) throw new Error('Invalid s3URI: expected user:pass@host') + + const accessKey = credentials.substring(0, firstColon) + const secretKey = credentials.substring(firstColon + 1) + + const [host, portStr] = hostPart.split(':') + + return { + endPoint: host, + port: portStr ? parseInt(portStr, 10) : undefined, + useSSL: scheme === 'https', + accessKey, + secretKey, + } +} diff --git a/backend/src/config/cra.config.ts b/backend/src/config/cra.config.ts index ed399da8..b27c7aa0 100644 --- a/backend/src/config/cra.config.ts +++ b/backend/src/config/cra.config.ts @@ -15,6 +15,8 @@ export const craConfig = registerAs('cra', () => { return { enabled: process.env.CRA_INTEGRATION_ENABLED === 'true', + transferMode: process.env.CRA_TRANSFER_MODE || 's3', + s3Prefix: process.env.CRA_S3_PREFIX || '', environmentCode: isProduction ? 'PCSAIN' : 'ACSAIN', fileTypeCode: isProduction ? 'PAPL' : 'AAPL', fileNamePrefix: isProduction ? 'HT' : 'II', diff --git a/backend/src/cra/cra.constant.ts b/backend/src/cra/cra.constant.ts index 18732aab..ee67b427 100644 --- a/backend/src/cra/cra.constant.ts +++ b/backend/src/cra/cra.constant.ts @@ -1,5 +1,5 @@ export const CRA_DATA_HANDLING_CONSTANT = { - DESTINATION_ID: 'cra-ftp', + DESTINATION_ID: 'cra', RESPONSE_FILE_TYPE: { RSP: 'RSP', WKL: 'WKL', diff --git a/backend/src/cra/cra.module.spec.ts b/backend/src/cra/cra.module.spec.ts index db61e405..3bc91033 100644 --- a/backend/src/cra/cra.module.spec.ts +++ b/backend/src/cra/cra.module.spec.ts @@ -3,6 +3,7 @@ import { Test } from '@nestjs/testing' import { JobType } from 'src/jobs/enums/job-type.enum' import { JobRegistry } from 'src/jobs/job-registry.service' import { JobsModule } from 'src/jobs/jobs.module' +import { SyncIcmHandler } from 'src/sync/handlers/sync-icm.handler' import { CraModule } from './cra.module' import { PollCraResponseHandler } from './handlers/poll-cra-response.handler' import { SendCraFileHandler } from './handlers/send-cra-file.handler' @@ -29,17 +30,20 @@ describe('CraModule', () => { expect(craModule).toBeDefined() }) - it('should register both CRA handlers', () => { + it('should register all CRA handlers including SYNC_ICM', () => { expect(registry.hasHandler(JobType.SEND_CRA_FILE)).toBe(true) expect(registry.hasHandler(JobType.POLL_CRA_RESPONSE)).toBe(true) + expect(registry.hasHandler(JobType.SYNC_ICM)).toBe(true) }) it('should register handlers with correct types', () => { const sendHandler = registry.getHandler(JobType.SEND_CRA_FILE) const pollHandler = registry.getHandler(JobType.POLL_CRA_RESPONSE) + const syncHandler = registry.getHandler(JobType.SYNC_ICM) expect(sendHandler).toBeInstanceOf(SendCraFileHandler) expect(pollHandler).toBeInstanceOf(PollCraResponseHandler) + expect(syncHandler).toBeInstanceOf(SyncIcmHandler) }) it('should export all handler providers', () => { diff --git a/backend/src/cra/cra.module.ts b/backend/src/cra/cra.module.ts index 97d0ade4..10b82038 100644 --- a/backend/src/cra/cra.module.ts +++ b/backend/src/cra/cra.module.ts @@ -1,32 +1,34 @@ import 'dotenv/config' -import { HttpModule } from '@nestjs/axios' -import { Module, OnModuleInit } from '@nestjs/common' -import { ConfigModule } from '@nestjs/config' +import { HttpModule, HttpService } from '@nestjs/axios' +import { Logger, Module, OnModuleInit } from '@nestjs/common' +import { ConfigModule, ConfigService } from '@nestjs/config' +import path from 'path' import { BatchesModule } from 'src/api/batches/batches.module' import { ContactsModule } from 'src/api/contacts/contacts.module' import { PrismaModule } from 'src/common/database/prisma.module' import { JobRegistry } from 'src/jobs/job-registry.service' import { JobsModule } from 'src/jobs/jobs.module' import { IcmSyncBackModule } from 'src/sync/icm/icm-sync-back.module' +import { SyncIcmHandler } from 'src/sync/handlers/sync-icm.handler' import { appConfig } from '../config/app.config' import { craConfig } from '../config/cra.config' +import { syncConfig } from '../config/sync.config' import { PollCraResponseHandler } from './handlers/poll-cra-response.handler' import { SendCraFileHandler } from './handlers/send-cra-file.handler' -import { OutboundFileService } from './outbound/outbound-file.service' -import { OutboundTransferService } from './outbound/outbound-transfer.service' import { InboundFileService } from './inbound/inbound-file.service' import { InboundResponseService } from './inbound/inbound-response.service' import { OutboundDataService } from './outbound/outbound-data.service' +import { OutboundFileService } from './outbound/outbound-file.service' +import { CraTransferService } from './transfer/cra-transfer.service' +import { HttpCraTransferService } from './transfer/http-cra-transfer.service' +import { MockCraTransferService } from './transfer/mock-cra-transfer.service' +import { S3CraTransferService } from './transfer/s3-cra-transfer.service' -/* - * Generates and sends files to CRA - * Polls and process response files from CRA - */ @Module({ imports: [ ConfigModule.forRoot({ isGlobal: true, - load: [craConfig, appConfig], + load: [craConfig, appConfig, syncConfig], }), JobsModule, PrismaModule, @@ -40,11 +42,31 @@ import { OutboundDataService } from './outbound/outbound-data.service' providers: [ SendCraFileHandler, PollCraResponseHandler, + SyncIcmHandler, OutboundFileService, - OutboundTransferService, InboundFileService, InboundResponseService, OutboundDataService, + { + provide: CraTransferService, + useFactory: (configService: ConfigService, httpService: HttpService) => { + const logger = new Logger('CraModule') + const useMock = configService.get('sync.useMockData') + if (useMock) { + const storagePath = configService.get('app.fileStoragePath')! + return new MockCraTransferService(path.join(storagePath, 'cra-mock')) + } + const transferMode = configService.get('cra.transferMode') + if (transferMode === 's3') { + return new S3CraTransferService(configService) + } + if (transferMode !== 'http') { + logger.warn(`Unknown CRA_TRANSFER_MODE "${transferMode}", falling back to http`) + } + return new HttpCraTransferService(httpService, configService) + }, + inject: [ConfigService, HttpService], + }, ], exports: [SendCraFileHandler, PollCraResponseHandler], }) @@ -53,11 +75,12 @@ export class CraModule implements OnModuleInit { private readonly registry: JobRegistry, private readonly sendCraFileHandler: SendCraFileHandler, private readonly pollCraResponseHandler: PollCraResponseHandler, + private readonly syncIcmHandler: SyncIcmHandler, ) {} onModuleInit() { - // Register CRA-related job handlers this.registry.register(this.sendCraFileHandler.jobType, this.sendCraFileHandler) this.registry.register(this.pollCraResponseHandler.jobType, this.pollCraResponseHandler) + this.registry.register(this.syncIcmHandler.jobType, this.syncIcmHandler) } } diff --git a/backend/src/cra/handlers/poll-cra-response.handler.spec.ts b/backend/src/cra/handlers/poll-cra-response.handler.spec.ts index c290d1e8..1a5f67ed 100644 --- a/backend/src/cra/handlers/poll-cra-response.handler.spec.ts +++ b/backend/src/cra/handlers/poll-cra-response.handler.spec.ts @@ -11,6 +11,15 @@ import { CRA_DATA_HANDLING_CONSTANT } from '../cra.constant' import { DETAIL_OUTCOME } from '../inbound/inbound.interface' import { PollCraResponseHandler } from './poll-cra-response.handler' +vi.mock('fs', () => ({ + existsSync: vi.fn().mockReturnValue(true), + mkdirSync: vi.fn(), +})) + +vi.mock('fs/promises', () => ({ + writeFile: vi.fn().mockResolvedValue(undefined), +})) + const DESTINATION_ID = CRA_DATA_HANDLING_CONSTANT.DESTINATION_ID const { TRAN_STAT_CODE, FILE_STAT_CODE } = CRA_DATA_HANDLING_CONSTANT @@ -21,7 +30,6 @@ const mockContext: JobContext = { retryCount: 0, } -// Mock factory for CRA response details const makeDetail = (overrides = {}) => ({ referenceNum: '100', tranStatCd: TRAN_STAT_CODE.TRAN_ACCEPTED, @@ -36,13 +44,12 @@ const makeDetail = (overrides = {}) => ({ ...overrides, }) -// Valid response file name: {userId}.{envFlag}RSP{seq}.txt -// In test env NODE_ENV is 'test' (not 'production'), so expected env flag is 'V' const VALID_FILE_NAME = 'craUserId.VRSP0001.txt' describe('PollCraResponseHandler', () => { let handler: PollCraResponseHandler + let mockCraTransferService: any let mockInboundFileService: any let mockInboundResponseService: any let mockPrisma: any @@ -52,9 +59,14 @@ describe('PollCraResponseHandler', () => { let mockIcmSyncBackService: any beforeEach(() => { + mockCraTransferService = { + listInboundFiles: vi.fn().mockResolvedValue([]), + downloadInboundFile: vi.fn(), + } + mockInboundFileService = { - downloadNewResponseFiles: vi.fn().mockResolvedValue([]), - getLocalFilePath: vi.fn().mockReturnValue('/tmp/cra-ftp/inbound/default.txt'), + getLocalFilePath: vi.fn().mockReturnValue('/tmp/cra/inbound/default.txt'), + isValidResponseFile: vi.fn().mockReturnValue(true), } mockInboundResponseService = { @@ -78,7 +90,6 @@ describe('PollCraResponseHandler', () => { return { outcome: DETAIL_OUTCOME.RECYCLED, systemComments, din: null } } - // REJECTED, PROBLEM_DETECTED, NOT_SET, or any unknown code return { outcome: DETAIL_OUTCOME.REJECTED, systemComments, din: null } }), } @@ -120,6 +131,7 @@ describe('PollCraResponseHandler', () => { } handler = new PollCraResponseHandler( + mockCraTransferService, mockInboundFileService, mockInboundResponseService, mockPrisma, @@ -134,17 +146,13 @@ describe('PollCraResponseHandler', () => { expect(handler.jobType).toBe(JobType.POLL_CRA_RESPONSE) }) - // Helpers - - // Simulates a previously downloaded file. function setupUnprocessedFile(fileName: string, id = 1) { mockPrisma.transferFile.findMany.mockResolvedValue([ { id, fileName, isDetailsProcessed: false, isValid: true }, ]) - mockInboundFileService.getLocalFilePath.mockReturnValue(`/tmp/cra-ftp/inbound/${fileName}`) + mockInboundFileService.getLocalFilePath.mockReturnValue(`/tmp/cra/inbound/${fileName}`) } - // Returns the given details object function setupParseFile(details: any[]) { mockInboundResponseService.parseFile.mockReturnValue({ header: { recordCount: details.length + 2 }, @@ -152,7 +160,6 @@ describe('PollCraResponseHandler', () => { }) } - // Returns a single batch detail lookup. Chain for multiple details. function setupBatchDetail(detailId: number, contactId: number, batchId: number) { mockPrisma.contactBatchDetail.findUnique.mockResolvedValueOnce({ id: detailId, @@ -165,8 +172,6 @@ describe('PollCraResponseHandler', () => { describe('No new files', () => { it('should return success with files_processed: 0 when no unprocessed files', async () => { - // Default: transferFile.findMany returns [] (no unprocessed files) - const result = await handler.execute(mockContext) expect(result.success).toBe(true) @@ -174,18 +179,15 @@ describe('PollCraResponseHandler', () => { expect(mockInboundResponseService.parseFile).not.toHaveBeenCalled() }) - it('should still call downloadNewResponseFiles to download new files', async () => { + it('should still call listInboundFiles to check for new files', async () => { await handler.execute(mockContext) - expect(mockInboundFileService.downloadNewResponseFiles).toHaveBeenCalledWith(DESTINATION_ID) + expect(mockCraTransferService.listInboundFiles).toHaveBeenCalled() }) }) describe('Invalid file format', () => { it('should return files_processed: 0 when no valid response file found', async () => { - // downloadNewResponseFiles handles validation internally - // No unprocessed files in DB (default mock returns []) - const result = await handler.execute(mockContext) expect(result.success).toBe(true) @@ -743,7 +745,6 @@ describe('PollCraResponseHandler', () => { const result1 = await handler.execute(mockContext) expect(result1.metadata.records_accepted).toBe(1) - // Second execution: rejected detail (counters should reset) const detail2 = makeDetail({ referenceNum: '200', tranStatCd: TRAN_STAT_CODE.TRAN_REJECTED, @@ -758,7 +759,6 @@ describe('PollCraResponseHandler', () => { const result2 = await handler.execute(mockContext) - // Counters should reflect only the second execution, not accumulate expect(result2.metadata.records_accepted).toBe(0) expect(result2.metadata.records_rejected).toBe(1) expect(result2.metadata.records_recycled).toBe(0) @@ -782,7 +782,7 @@ describe('PollCraResponseHandler', () => { }) describe('File download and processing', () => { - it('should call downloadNewResponseFiles with destination', async () => { + it('should call listInboundFiles to check for new files', async () => { const detail = makeDetail({ referenceNum: '100', tranStatCd: TRAN_STAT_CODE.TRAN_ACCEPTED }) setupUnprocessedFile(VALID_FILE_NAME) setupParseFile([detail]) @@ -794,7 +794,7 @@ describe('PollCraResponseHandler', () => { await handler.execute(mockContext) - expect(mockInboundFileService.downloadNewResponseFiles).toHaveBeenCalledWith(DESTINATION_ID) + expect(mockCraTransferService.listInboundFiles).toHaveBeenCalled() }) it('should call getLocalFilePath with destination and fileName', async () => { diff --git a/backend/src/cra/handlers/poll-cra-response.handler.ts b/backend/src/cra/handlers/poll-cra-response.handler.ts index 198a597b..31f4f004 100644 --- a/backend/src/cra/handlers/poll-cra-response.handler.ts +++ b/backend/src/cra/handlers/poll-cra-response.handler.ts @@ -1,4 +1,7 @@ import { Injectable } from '@nestjs/common' +import { existsSync, mkdirSync } from 'fs' +import { writeFile } from 'fs/promises' +import path from 'path' import { BatchesService } from 'src/api/batches/batches.service' import { ContactsService } from 'src/api/contacts/contacts.service' import { PrismaService } from 'src/common/database/prisma.service' @@ -14,24 +17,21 @@ import { CRA_DATA_HANDLING_CONSTANT } from '../cra.constant' import { InboundFileService } from '../inbound/inbound-file.service' import { InboundResponseService } from '../inbound/inbound-response.service' import { DETAIL_OUTCOME, type CraResDetail } from '../inbound/inbound.interface' +import { CraTransferService } from '../transfer/cra-transfer.service' const { DESTINATION_ID, FILE_DIRECTION, UPDATED_BY } = CRA_DATA_HANDLING_CONSTANT -/* - * Checks for response files from CRA and processes them - * Triggered by CronJob POLL_CRA_RESPONSE - */ @Injectable() export class PollCraResponseHandler extends BaseJob { readonly jobType = JobType.POLL_CRA_RESPONSE - // Per-run state shared across private methods private processedBatchIds!: Set private recordsAccepted!: number private recordsRejected!: number private recordsRecycled!: number constructor( + private readonly craTransferService: CraTransferService, private readonly inboundFileService: InboundFileService, private readonly inboundResponseService: InboundResponseService, private readonly prisma: PrismaService, @@ -49,7 +49,7 @@ export class PollCraResponseHandler extends BaseJob { this.recordsRejected = 0 this.recordsRecycled = 0 - await this.inboundFileService.downloadNewResponseFiles(DESTINATION_ID) + await this.downloadAndRegisterNewFiles() const unprocessedResponseFiles = await this.prisma.transferFile.findMany({ where: { direction: FILE_DIRECTION.INBOUND, isDetailsProcessed: false, isValid: true }, @@ -92,6 +92,45 @@ export class PollCraResponseHandler extends BaseJob { } } + private async downloadAndRegisterNewFiles(): Promise { + const existingFiles = await this.prisma.transferFile.findMany({ + where: { direction: FILE_DIRECTION.INBOUND }, + select: { fileName: true }, + }) + const existingNames = new Set(existingFiles.map((file) => file.fileName)) + + const remoteFiles = await this.craTransferService.listInboundFiles() + const newFiles = remoteFiles.filter((file) => !existingNames.has(file.fileName)) + + for (const file of newFiles) { + const fileBuffer = await this.craTransferService.downloadInboundFile(file.fileName) + const localFilePath = this.inboundFileService.getLocalFilePath(DESTINATION_ID, file.fileName) + + const dir = path.dirname(localFilePath) + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }) + } + await writeFile(localFilePath, fileBuffer) + + const valid = this.inboundFileService.isValidResponseFile(file.fileName) + if (!valid) { + this.logger.warn(`Invalid response file format: ${file.fileName}`) + } + + await this.prisma.transferFile.create({ + data: { + destinationId: DESTINATION_ID, + direction: FILE_DIRECTION.INBOUND, + fileName: file.fileName, + fileSize: String(fileBuffer.length), + downloadedAt: new Date(), + isValid: valid, + isDetailsProcessed: !valid, + }, + }) + } + } + private async processResponseFile(responseFile: { id: number fileName: string @@ -214,7 +253,6 @@ export class PollCraResponseHandler extends BaseJob { ) this.recordsRejected++ } else { - // recycled: no state change, just update system comments this.logger.log(`Detail ${batchDetail.id} recycled, no status change`) await this.prisma.contactBatchDetail.update({ where: { id: batchDetail.id }, diff --git a/backend/src/cra/handlers/send-cra-file.handler.spec.ts b/backend/src/cra/handlers/send-cra-file.handler.spec.ts index e92d7a26..260bb938 100644 --- a/backend/src/cra/handlers/send-cra-file.handler.spec.ts +++ b/backend/src/cra/handlers/send-cra-file.handler.spec.ts @@ -6,6 +6,10 @@ import { JobContext } from 'src/jobs/interfaces/job.interface' import { BATCH_EVENT, CSA_EVENT } from 'src/common/state-machine/constants' import { CRA_DATA_HANDLING_CONSTANT } from '../cra.constant' +vi.mock('fs/promises', () => ({ + readFile: vi.fn().mockResolvedValue(Buffer.from('mock-file-content')), +})) + const DESTINATION_ID = CRA_DATA_HANDLING_CONSTANT.DESTINATION_ID const mockContext: JobContext = { @@ -70,7 +74,7 @@ describe('SendCraFileHandler', () => { let mockContactsService: any let mockOutboundDataService: any let mockOutboundFileService: any - let mockOutboundTransferService: any + let mockCraTransferService: any let mockJobRunner: any let mockIcmSyncBackService: any @@ -109,14 +113,14 @@ describe('SendCraFileHandler', () => { mockOutboundFileService = { createFile: vi.fn().mockReturnValue({ - filePath: '/tmp/cra-ftp/testfile.txt', + filePath: '/tmp/cra/testfile.txt', fileName: 'testfile.txt', recordCount: 3, }), } - mockOutboundTransferService = { - sendFileToTransferService: vi.fn().mockResolvedValue({ statusCode: 226 }), + mockCraTransferService = { + sendFile: vi.fn().mockResolvedValue({ success: true, fileName: 'testfile.txt' }), } mockJobRunner = { @@ -143,7 +147,7 @@ describe('SendCraFileHandler', () => { mockContactsService, mockOutboundDataService, mockOutboundFileService, - mockOutboundTransferService, + mockCraTransferService, mockJobRunner, mockIcmSyncBackService, ) @@ -285,10 +289,9 @@ describe('SendCraFileHandler', () => { 1, ) - expect(mockOutboundTransferService.sendFileToTransferService).toHaveBeenCalledWith( - '/tmp/cra-ftp/testfile.txt', + expect(mockCraTransferService.sendFile).toHaveBeenCalledWith( 'testfile.txt', - DESTINATION_ID, + expect.any(Buffer), ) }) @@ -315,7 +318,7 @@ describe('SendCraFileHandler', () => { expect(result.message).toContain('Batch 10') expect(result.metadata).toEqual({ batch_id: 10, - file_path: '/tmp/cra-ftp/testfile.txt', + file_path: '/tmp/cra/testfile.txt', record_count: 3, contacts_count: 2, }) @@ -452,9 +455,7 @@ describe('SendCraFileHandler', () => { mockPrisma.batch.findFirst.mockResolvedValue(batch) mockPrisma.contactBatchDetail.findMany.mockResolvedValue([detail]) - mockOutboundTransferService.sendFileToTransferService.mockRejectedValue( - new Error('Connection refused'), - ) + mockCraTransferService.sendFile.mockRejectedValue(new Error('Connection refused')) await handler.onStart(mockContext) await expect(handler.execute(mockContext)).rejects.toThrow('Connection refused') @@ -467,9 +468,7 @@ describe('SendCraFileHandler', () => { mockPrisma.batch.findFirst.mockResolvedValue(batch) mockPrisma.contactBatchDetail.findMany.mockResolvedValue([detail]) - mockOutboundTransferService.sendFileToTransferService.mockRejectedValue( - new Error('Transfer failed'), - ) + mockCraTransferService.sendFile.mockRejectedValue(new Error('Transfer failed')) await handler.onStart(mockContext) await expect(handler.execute(mockContext)).rejects.toThrow() @@ -484,9 +483,7 @@ describe('SendCraFileHandler', () => { mockPrisma.batch.findFirst.mockResolvedValue(batch) mockPrisma.contactBatchDetail.findMany.mockResolvedValue([detail]) - mockOutboundTransferService.sendFileToTransferService.mockRejectedValue( - new Error('Transfer failed'), - ) + mockCraTransferService.sendFile.mockRejectedValue(new Error('Transfer failed')) await handler.onStart(mockContext) await expect(handler.execute(mockContext)).rejects.toThrow() diff --git a/backend/src/cra/handlers/send-cra-file.handler.ts b/backend/src/cra/handlers/send-cra-file.handler.ts index 5b923ea1..177ea6e4 100644 --- a/backend/src/cra/handlers/send-cra-file.handler.ts +++ b/backend/src/cra/handlers/send-cra-file.handler.ts @@ -1,5 +1,6 @@ import { Injectable } from '@nestjs/common' import { ConfigService } from '@nestjs/config' +import { readFile } from 'fs/promises' import { appendSystemComment, pacificToday } from 'src/common/utils' import type { Batch, Contact, ContactBatchDetail } from '@prisma/client' import { BatchesService } from 'src/api/batches/batches.service' @@ -21,7 +22,7 @@ import { IcmSyncBackService } from 'src/sync/icm/icm-sync-back.service' import { CRA_DATA_HANDLING_CONSTANT } from '../cra.constant' import { OutboundDataService } from '../outbound/outbound-data.service' import { OutboundFileService } from '../outbound/outbound-file.service' -import { OutboundTransferService } from '../outbound/outbound-transfer.service' +import { CraTransferService } from '../transfer/cra-transfer.service' const { DESTINATION_ID, FILE_DIRECTION, UPDATED_BY } = CRA_DATA_HANDLING_CONSTANT @@ -40,7 +41,7 @@ export class SendCraFileHandler extends BaseJob { private readonly contactsService: ContactsService, private readonly outboundDataService: OutboundDataService, private readonly outboundFileService: OutboundFileService, - private readonly outboundTransferService: OutboundTransferService, + private readonly craTransferService: CraTransferService, private readonly jobRunner: JobRunner, private readonly icmSyncBackService: IcmSyncBackService, ) { @@ -105,7 +106,8 @@ export class SendCraFileHandler extends BaseJob { nextSequence, ) - await this.outboundTransferService.sendFileToTransferService(filePath, fileName, DESTINATION_ID) + const fileBuffer = await readFile(filePath) + await this.craTransferService.sendFile(fileName, fileBuffer) await this.prisma.transferFile.create({ data: { diff --git a/backend/src/cra/inbound/inbound-file.service.ts b/backend/src/cra/inbound/inbound-file.service.ts index 022bd5b1..5236bfda 100644 --- a/backend/src/cra/inbound/inbound-file.service.ts +++ b/backend/src/cra/inbound/inbound-file.service.ts @@ -1,127 +1,28 @@ -import { Injectable, Logger } from '@nestjs/common' -import { HttpService } from '@nestjs/axios' +import { Injectable } from '@nestjs/common' import { ConfigService } from '@nestjs/config' -import { existsSync, mkdirSync, statSync } from 'fs' -import { writeFile } from 'fs/promises' import path from 'path' -import { firstValueFrom } from 'rxjs' -import { PrismaService } from 'src/common/database/prisma.service' import { CRA_DATA_HANDLING_CONSTANT } from '../cra.constant' -const { LOCAL_DIR, RESPONSE_FILE_TYPE, FILE_DIRECTION } = CRA_DATA_HANDLING_CONSTANT - -export interface DownloadedFile { - fileName: string - localFilePath: string -} +const { LOCAL_DIR, RESPONSE_FILE_TYPE } = CRA_DATA_HANDLING_CONSTANT @Injectable() export class InboundFileService { - private readonly logger = new Logger(InboundFileService.name) private readonly fileStoragePath: string - private readonly fileTransferServiceUrl: string private readonly responseEnvFlag: string - private readonly craEnabled: boolean - - constructor( - private readonly httpService: HttpService, - private readonly configService: ConfigService, - private readonly prisma: PrismaService, - ) { + constructor(private readonly configService: ConfigService) { this.fileStoragePath = this.configService.get('app.fileStoragePath')! - this.fileTransferServiceUrl = this.configService.get('app.fileTransferServiceUrl')! this.responseEnvFlag = this.configService.get('cra.responseEnvFlag')! - this.craEnabled = this.configService.get('cra.enabled')! - } - - async downloadNewResponseFiles(destinationId: string): Promise { - const existingFiles = await this.prisma.transferFile.findMany({ - where: { direction: FILE_DIRECTION.INBOUND }, - select: { fileName: true }, - }) - const mockResponseFile = [ - { fileName: 'TST0016.VRSP0001', size: 4000, lastModifiedAt: new Date() }, - ] - const remoteFiles = this.craEnabled - ? await this.listRemoteFiles(destinationId) - : mockResponseFile - const newFiles = remoteFiles.filter( - (remote) => !existingFiles.some((db) => db.fileName === remote.fileName), - ) - - const downloaded: DownloadedFile[] = [] - for (const file of newFiles) { - const mockFilePath = this.getLocalFilePath(destinationId, file.fileName) - const localFilePath = this.craEnabled - ? await this.downloadFile(destinationId, file.fileName) - : mockFilePath - const valid = this.isValidResponseFile(file.fileName) - - if (!valid) { - this.logger.warn(`Invalid response file format: ${file.fileName}`) - } - - await this.prisma.transferFile.create({ - data: { - destinationId, - direction: FILE_DIRECTION.INBOUND, - fileName: file.fileName, - fileSize: String(statSync(localFilePath).size), - downloadedAt: new Date(), - isValid: valid, - isDetailsProcessed: !valid, - }, - }) - - if (valid) { - downloaded.push({ fileName: file.fileName, localFilePath }) - } - } - - return downloaded } getLocalFilePath(destinationId: string, fileName: string): string { return path.join(this.fileStoragePath, destinationId, LOCAL_DIR.INBOUND, fileName) } - private isValidResponseFile(fileName: string): boolean { + isValidResponseFile(fileName: string): boolean { const fileMiddle = fileName.split('.')[1] ?? '' const fileEnvFlag = fileMiddle.slice(0, 1) const fileTypeFlag = fileMiddle.slice(1, 4) return fileTypeFlag === RESPONSE_FILE_TYPE.RSP && fileEnvFlag === this.responseEnvFlag } - - private async listRemoteFiles(destinationId: string): Promise<{ fileName: string }[]> { - const response = await firstValueFrom( - this.httpService.get( - `${this.fileTransferServiceUrl}/api/destinations/${destinationId}/transfers/inbound`, - { headers: { 'Content-Type': 'application/json' } }, - ), - ) - return response?.data?.files ?? [] - } - - private async downloadFile(destinationId: string, fileName: string): Promise { - const response = await firstValueFrom( - this.httpService.get( - `${this.fileTransferServiceUrl}/api/destinations/${destinationId}/transfers/${fileName}`, - { headers: { 'Content-Type': 'text/plain' }, responseType: 'arraybuffer' }, - ), - ) - - const localFilePath = this.getLocalFilePath(destinationId, fileName) - const inboundDir = path.dirname(localFilePath) - if (!existsSync(inboundDir)) { - mkdirSync(inboundDir, { recursive: true }) - } - - await writeFile( - localFilePath, - Buffer.isBuffer(response.data) ? response.data : Buffer.from(response.data), - ) - - return localFilePath - } } diff --git a/backend/src/cra/outbound/outbound-file.service.spec.ts b/backend/src/cra/outbound/outbound-file.service.spec.ts index 51185df1..3c5e698c 100644 --- a/backend/src/cra/outbound/outbound-file.service.spec.ts +++ b/backend/src/cra/outbound/outbound-file.service.spec.ts @@ -7,7 +7,6 @@ import { CRA_DATA_HANDLING_CONSTANT } from '../cra.constant' import { OutboundDataService } from './outbound-data.service' import { OutboundFileService } from './outbound-file.service' import { FILE_MOCK_DATA } from './outbound-mock-data' -import { OutboundTransferService } from './outbound-transfer.service' const { header, details, trailer } = FILE_MOCK_DATA const { REQUEST_FILE } = CRA_DATA_HANDLING_CONSTANT @@ -51,7 +50,6 @@ const currentDate = (): string => formatDatePacificCompact(new Date()) describe('OutboundFileService', () => { let service: OutboundFileService - let fileTransferClientService: OutboundTransferService beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -61,29 +59,18 @@ describe('OutboundFileService', () => { provide: ConfigService, useValue: mockConfigService, }, - { - provide: OutboundTransferService, - useValue: { - sendFileToTransferService: vi.fn(), - }, - }, ], }).compile() service = module.get(OutboundFileService) - fileTransferClientService = module.get(OutboundTransferService) }) afterEach(() => { vi.clearAllMocks() }) - it('should create file and send it to transfer service successfully', async () => { + it('should create file successfully', () => { ;(existsSync as unknown as Mock).mockReturnValue(true) - ;(fileTransferClientService.sendFileToTransferService as Mock).mockResolvedValue({ - statusCode: 226, - message: 'Success', - }) service.createFile(header, details, trailer, 'test-destination', 1) expect(writeFileSync).toHaveBeenCalled() diff --git a/backend/src/cra/outbound/outbound-transfer.service.ts b/backend/src/cra/outbound/outbound-transfer.service.ts deleted file mode 100644 index dac5be88..00000000 --- a/backend/src/cra/outbound/outbound-transfer.service.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { HttpService } from '@nestjs/axios' -import { Injectable, Logger } from '@nestjs/common' -import { ConfigService } from '@nestjs/config' -import FormData from 'form-data' -import fs from 'fs' -import { firstValueFrom } from 'rxjs' - -@Injectable() -export class OutboundTransferService { - private readonly logger = new Logger(OutboundTransferService.name) - private readonly fileTransferServiceUrl: string - private readonly craEnabled: boolean - - constructor( - private readonly httpService: HttpService, - private readonly configService: ConfigService, - ) { - this.fileTransferServiceUrl = this.configService.get('app.fileTransferServiceUrl')! - this.craEnabled = this.configService.get('cra.enabled')! - } - - async sendFileToTransferService( - filePath: string, - fileName: string, - destinationId: string, - ): Promise { - if (!this.craEnabled) { - this.logger.log(`[CRA Disabled] File transfer skipped — file saved at ${filePath}`) - return { success: true, fileName } - } - - const formData = new FormData() - formData.append('file', fs.createReadStream(filePath), fileName) - formData.append('fileName', fileName) - - const url = `${this.fileTransferServiceUrl}/api/destinations/${destinationId}/transfers` - const response = await firstValueFrom( - this.httpService.post(url, formData, { - headers: { ...formData.getHeaders() }, - }), - ) - return response.data - } -} diff --git a/backend/src/cra/transfer/cra-transfer.service.ts b/backend/src/cra/transfer/cra-transfer.service.ts new file mode 100644 index 00000000..ccd527af --- /dev/null +++ b/backend/src/cra/transfer/cra-transfer.service.ts @@ -0,0 +1,20 @@ +import { Logger } from '@nestjs/common' + +export interface TransferResult { + success: boolean + fileName: string +} + +export interface InboundFileInfo { + fileName: string + size?: number + lastModifiedAt?: Date +} + +export abstract class CraTransferService { + protected readonly logger = new Logger(this.constructor.name) + + abstract sendFile(fileName: string, fileBuffer: Buffer): Promise + abstract listInboundFiles(): Promise + abstract downloadInboundFile(fileName: string): Promise +} diff --git a/backend/src/cra/transfer/http-cra-transfer.service.spec.ts b/backend/src/cra/transfer/http-cra-transfer.service.spec.ts new file mode 100644 index 00000000..8bdce642 --- /dev/null +++ b/backend/src/cra/transfer/http-cra-transfer.service.spec.ts @@ -0,0 +1,117 @@ +import { HttpService } from '@nestjs/axios' +import { ConfigService } from '@nestjs/config' +import { of } from 'rxjs' +import { HttpCraTransferService } from './http-cra-transfer.service' + +describe('HttpCraTransferService', () => { + let service: HttpCraTransferService + let httpService: { get: ReturnType; post: ReturnType } + let configService: { get: ReturnType } + + const baseUrl = 'http://file-transfer:3000' + + function createService(craEnabled = true) { + configService = { + get: vi.fn((key: string) => { + if (key === 'app.fileTransferServiceUrl') return baseUrl + if (key === 'cra.enabled') return craEnabled + return undefined + }), + } + httpService = { get: vi.fn(), post: vi.fn() } + return new HttpCraTransferService( + httpService as unknown as HttpService, + configService as unknown as ConfigService, + ) + } + + beforeEach(() => { + vi.clearAllMocks() + service = createService(true) + }) + + describe('sendFile', () => { + it('should POST file via FormData when CRA is enabled', async () => { + httpService.post.mockReturnValue(of({ data: { success: true, fileName: 'test.dat' } })) + + const result = await service.sendFile('test.dat', Buffer.from('file content')) + + expect(result).toEqual({ success: true, fileName: 'test.dat' }) + expect(httpService.post).toHaveBeenCalledTimes(1) + + const [url, body, options] = httpService.post.mock.calls[0] + expect(url).toBe(`${baseUrl}/api/destinations/cra/transfers`) + expect(options.headers).toBeDefined() + expect(body).toBeDefined() + }) + + it('should skip HTTP call and return success when CRA is disabled', async () => { + service = createService(false) + + const result = await service.sendFile('test.dat', Buffer.from('file content')) + + expect(result).toEqual({ success: true, fileName: 'test.dat' }) + expect(httpService.post).not.toHaveBeenCalled() + }) + }) + + describe('listInboundFiles', () => { + it('should return files from the file transfer service', async () => { + const remoteFiles = [ + { fileName: 'response1.dat', size: 1024, lastModifiedAt: '2026-01-15T00:00:00Z' }, + { fileName: 'response2.dat', size: 2048 }, + ] + httpService.get.mockReturnValue(of({ data: { files: remoteFiles } })) + + const result = await service.listInboundFiles() + + expect(result).toEqual(remoteFiles) + expect(httpService.get).toHaveBeenCalledWith( + `${baseUrl}/api/destinations/cra/transfers/inbound`, + { headers: { 'Content-Type': 'application/json' } }, + ) + }) + + it('should return empty array when response has no files', async () => { + httpService.get.mockReturnValue(of({ data: {} })) + + const result = await service.listInboundFiles() + + expect(result).toEqual([]) + }) + + it('should return empty array when CRA is disabled', async () => { + service = createService(false) + + const result = await service.listInboundFiles() + + expect(result).toEqual([]) + expect(httpService.get).not.toHaveBeenCalled() + }) + }) + + describe('downloadInboundFile', () => { + it('should download file as buffer', async () => { + const fileContent = Buffer.from('response data') + httpService.get.mockReturnValue(of({ data: fileContent })) + + const result = await service.downloadInboundFile('response1.dat') + + expect(result).toEqual(fileContent) + expect(httpService.get).toHaveBeenCalledWith( + `${baseUrl}/api/destinations/cra/transfers/response1.dat`, + { headers: { 'Content-Type': 'text/plain' }, responseType: 'arraybuffer' }, + ) + }) + + it('should convert non-Buffer response data to Buffer', async () => { + const arrayData = new Uint8Array([72, 101, 108, 108, 111]) + httpService.get.mockReturnValue(of({ data: arrayData })) + + const result = await service.downloadInboundFile('response1.dat') + + expect(Buffer.isBuffer(result)).toBe(true) + expect(result.toString()).toBe('Hello') + }) + }) +}) diff --git a/backend/src/cra/transfer/http-cra-transfer.service.ts b/backend/src/cra/transfer/http-cra-transfer.service.ts new file mode 100644 index 00000000..274d48b5 --- /dev/null +++ b/backend/src/cra/transfer/http-cra-transfer.service.ts @@ -0,0 +1,67 @@ +import { HttpService } from '@nestjs/axios' +import { Injectable } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' +import FormData from 'form-data' +import { Readable } from 'stream' +import { firstValueFrom } from 'rxjs' +import { CRA_DATA_HANDLING_CONSTANT } from '../cra.constant' +import { CraTransferService, InboundFileInfo, TransferResult } from './cra-transfer.service' + +const { DESTINATION_ID } = CRA_DATA_HANDLING_CONSTANT + +@Injectable() +export class HttpCraTransferService extends CraTransferService { + private readonly fileTransferServiceUrl: string + private readonly craEnabled: boolean + + constructor( + private readonly httpService: HttpService, + private readonly configService: ConfigService, + ) { + super() + this.fileTransferServiceUrl = this.configService.get('app.fileTransferServiceUrl')! + this.craEnabled = this.configService.get('cra.enabled')! + } + + async sendFile(fileName: string, fileBuffer: Buffer): Promise { + if (!this.craEnabled) { + this.logger.log(`[CRA Disabled] File transfer skipped for ${fileName}`) + return { success: true, fileName } + } + + const formData = new FormData() + formData.append('file', Readable.from(fileBuffer), fileName) + formData.append('fileName', fileName) + + const url = `${this.fileTransferServiceUrl}/api/destinations/${DESTINATION_ID}/transfers` + const response = await firstValueFrom( + this.httpService.post(url, formData, { + headers: { ...formData.getHeaders() }, + }), + ) + return response.data + } + + async listInboundFiles(): Promise { + if (!this.craEnabled) return [] + + const url = `${this.fileTransferServiceUrl}/api/destinations/${DESTINATION_ID}/transfers/inbound` + const response = await firstValueFrom( + this.httpService.get(url, { + headers: { 'Content-Type': 'application/json' }, + }), + ) + return response?.data?.files ?? [] + } + + async downloadInboundFile(fileName: string): Promise { + const url = `${this.fileTransferServiceUrl}/api/destinations/${DESTINATION_ID}/transfers/${fileName}` + const response = await firstValueFrom( + this.httpService.get(url, { + headers: { 'Content-Type': 'text/plain' }, + responseType: 'arraybuffer', + }), + ) + return Buffer.isBuffer(response.data) ? response.data : Buffer.from(response.data) + } +} diff --git a/backend/src/cra/transfer/mock-cra-transfer.service.spec.ts b/backend/src/cra/transfer/mock-cra-transfer.service.spec.ts new file mode 100644 index 00000000..c814a881 --- /dev/null +++ b/backend/src/cra/transfer/mock-cra-transfer.service.spec.ts @@ -0,0 +1,99 @@ +import * as fs from 'fs' +import * as fsp from 'fs/promises' +import { MockCraTransferService } from './mock-cra-transfer.service' + +vi.mock('fs') +vi.mock('fs/promises') + +describe('MockCraTransferService', () => { + const basePath = '/tmp/mock-cra' + let service: MockCraTransferService + + beforeEach(() => { + vi.clearAllMocks() + service = new MockCraTransferService(basePath) + }) + + describe('sendFile', () => { + it('should write file to outbound directory', async () => { + vi.mocked(fsp.mkdir).mockResolvedValue(undefined) + vi.mocked(fsp.writeFile).mockResolvedValue(undefined) + + const result = await service.sendFile('test.dat', Buffer.from('file content')) + + expect(fsp.mkdir).toHaveBeenCalledWith('/tmp/mock-cra/outbound', { recursive: true }) + expect(fsp.writeFile).toHaveBeenCalledWith( + '/tmp/mock-cra/outbound/test.dat', + Buffer.from('file content'), + ) + expect(result).toEqual({ success: true, fileName: 'test.dat' }) + }) + + it('should throw when write fails', async () => { + vi.mocked(fsp.mkdir).mockResolvedValue(undefined) + vi.mocked(fsp.writeFile).mockRejectedValue(new Error('disk full')) + + await expect(service.sendFile('test.dat', Buffer.from('data'))).rejects.toThrow('disk full') + }) + }) + + describe('listInboundFiles', () => { + it('should return file info for files in inbound directory', async () => { + vi.mocked(fs.existsSync).mockReturnValue(true) + vi.mocked(fsp.readdir).mockResolvedValue([ + { name: 'response1.dat', isDirectory: () => false }, + { name: 'response2.dat', isDirectory: () => false }, + ] as any) + vi.mocked(fsp.stat).mockResolvedValue({ size: 1024, mtime: new Date('2026-01-15') } as any) + + const result = await service.listInboundFiles() + + expect(result).toHaveLength(2) + expect(result[0]).toEqual({ + fileName: 'response1.dat', + size: 1024, + lastModifiedAt: new Date('2026-01-15'), + }) + }) + + it('should filter out directories', async () => { + vi.mocked(fs.existsSync).mockReturnValue(true) + vi.mocked(fsp.readdir).mockResolvedValue([ + { name: 'response1.dat', isDirectory: () => false }, + { name: 'PROCESSED', isDirectory: () => true }, + ] as any) + vi.mocked(fsp.stat).mockResolvedValue({ size: 512, mtime: new Date('2026-01-15') } as any) + + const result = await service.listInboundFiles() + + expect(result).toHaveLength(1) + expect(result[0].fileName).toBe('response1.dat') + }) + + it('should return empty array when inbound directory does not exist', async () => { + vi.mocked(fs.existsSync).mockReturnValue(false) + + const result = await service.listInboundFiles() + + expect(result).toEqual([]) + }) + }) + + describe('downloadInboundFile', () => { + it('should read file from inbound directory', async () => { + const fileContent = Buffer.from('response data') + vi.mocked(fsp.readFile).mockResolvedValue(fileContent) + + const result = await service.downloadInboundFile('response1.dat') + + expect(fsp.readFile).toHaveBeenCalledWith('/tmp/mock-cra/inbound/response1.dat') + expect(result).toEqual(fileContent) + }) + + it('should throw when file does not exist', async () => { + vi.mocked(fsp.readFile).mockRejectedValue(new Error('ENOENT: no such file')) + + await expect(service.downloadInboundFile('missing.dat')).rejects.toThrow('ENOENT') + }) + }) +}) diff --git a/backend/src/cra/transfer/mock-cra-transfer.service.ts b/backend/src/cra/transfer/mock-cra-transfer.service.ts new file mode 100644 index 00000000..272d98ff --- /dev/null +++ b/backend/src/cra/transfer/mock-cra-transfer.service.ts @@ -0,0 +1,49 @@ +import { Injectable } from '@nestjs/common' +import * as fs from 'fs' +import * as fsp from 'fs/promises' +import * as path from 'path' +import { CraTransferService, InboundFileInfo, TransferResult } from './cra-transfer.service' + +@Injectable() +export class MockCraTransferService extends CraTransferService { + constructor(private readonly mockBasePath: string) { + super() + } + + async sendFile(fileName: string, fileBuffer: Buffer): Promise { + const outboundDir = path.join(this.mockBasePath, 'outbound') + await fsp.mkdir(outboundDir, { recursive: true }) + await fsp.writeFile(path.join(outboundDir, fileName), fileBuffer) + this.logger.log(`Wrote outbound file ${fileName}`) + return { success: true, fileName } + } + + async listInboundFiles(): Promise { + const inboundDir = path.join(this.mockBasePath, 'inbound') + if (!fs.existsSync(inboundDir)) { + return [] + } + + const entries = await fsp.readdir(inboundDir, { withFileTypes: true }) + const files = entries.filter((entry) => !entry.isDirectory()) + + const results: InboundFileInfo[] = [] + for (const file of files) { + const filePath = path.join(inboundDir, file.name) + const stats = await fsp.stat(filePath) + results.push({ + fileName: file.name, + size: stats.size, + lastModifiedAt: stats.mtime, + }) + } + + this.logger.log(`Found ${results.length} inbound file(s)`) + return results + } + + async downloadInboundFile(fileName: string): Promise { + const filePath = path.join(this.mockBasePath, 'inbound', fileName) + return fsp.readFile(filePath) + } +} diff --git a/backend/src/cra/transfer/s3-cra-transfer.service.spec.ts b/backend/src/cra/transfer/s3-cra-transfer.service.spec.ts new file mode 100644 index 00000000..b519b6e1 --- /dev/null +++ b/backend/src/cra/transfer/s3-cra-transfer.service.spec.ts @@ -0,0 +1,153 @@ +import { ConfigService } from '@nestjs/config' +import { Test, TestingModule } from '@nestjs/testing' +import { Readable } from 'stream' +import { S3CraTransferService } from './s3-cra-transfer.service' + +const mockPutObject = vi.fn() +const mockGetObject = vi.fn() +const mockListObjectsV2 = vi.fn() + +vi.mock('minio', () => { + return { + Client: class MockClient { + putObject = mockPutObject + getObject = mockGetObject + listObjectsV2 = mockListObjectsV2 + }, + } +}) + +describe('S3CraTransferService', () => { + const PREFIX = 'NONPROD/CRA/DEV' + let service: S3CraTransferService + + beforeEach(async () => { + vi.clearAllMocks() + + const configService = { + get: vi.fn((key: string) => { + const values: Record = { + 'sync.s3Uri': 'http://minioadmin:minioadmin@localhost:9000', + 'sync.s3Bucket': 'test-bucket', + 'cra.s3Prefix': PREFIX, + } + return values[key] + }), + } + + const module: TestingModule = await Test.createTestingModule({ + providers: [S3CraTransferService, { provide: ConfigService, useValue: configService }], + }).compile() + + service = module.get(S3CraTransferService) + }) + + describe('sendFile', () => { + it('should upload file to OUTBOUND prefix and return success', async () => { + mockPutObject.mockResolvedValue({ etag: 'abc123' }) + + const result = await service.sendFile('ACSAIN.20260311.dat', Buffer.from('file data')) + + expect(mockPutObject).toHaveBeenCalledWith( + 'test-bucket', + `${PREFIX}/OUTBOUND/ACSAIN.20260311.dat`, + Buffer.from('file data'), + ) + expect(result).toEqual({ success: true, fileName: 'ACSAIN.20260311.dat' }) + }) + + it('should throw when putObject fails', async () => { + mockPutObject.mockRejectedValue(new Error('upload failed')) + + await expect(service.sendFile('ACSAIN.20260311.dat', Buffer.from('data'))).rejects.toThrow( + 'upload failed', + ) + }) + }) + + describe('listInboundFiles', () => { + function createMockListStream( + objects: Array<{ name: string; size?: number; lastModified?: Date }>, + ) { + const stream = new Readable({ objectMode: true, read() {} }) + for (const obj of objects) { + stream.push(obj) + } + stream.push(null) + return stream + } + + it('should list inbound files with prefix stripped', async () => { + const mockDate = new Date('2026-03-10') + mockListObjectsV2.mockReturnValue( + createMockListStream([ + { name: `${PREFIX}/INBOUND/response1.dat`, size: 1024, lastModified: mockDate }, + { name: `${PREFIX}/INBOUND/response2.dat`, size: 2048, lastModified: mockDate }, + ]), + ) + + const result = await service.listInboundFiles() + + expect(mockListObjectsV2).toHaveBeenCalledWith('test-bucket', `${PREFIX}/INBOUND/`, true) + expect(result).toEqual([ + { fileName: 'response1.dat', size: 1024, lastModifiedAt: mockDate }, + { fileName: 'response2.dat', size: 2048, lastModifiedAt: mockDate }, + ]) + }) + + it('should return empty array when no objects exist', async () => { + mockListObjectsV2.mockReturnValue(createMockListStream([])) + + const result = await service.listInboundFiles() + + expect(result).toEqual([]) + }) + + it('should reject when the stream emits an error', async () => { + const stream = new Readable({ + objectMode: true, + read() { + process.nextTick(() => this.destroy(new Error('network error'))) + }, + }) + mockListObjectsV2.mockReturnValue(stream) + + await expect(service.listInboundFiles()).rejects.toThrow('network error') + }) + }) + + describe('downloadInboundFile', () => { + it('should download and return file contents as a Buffer', async () => { + const stream = new Readable({ + read() { + this.push(Buffer.from('chunk1')) + this.push(Buffer.from('chunk2')) + this.push(null) + }, + }) + mockGetObject.mockResolvedValue(stream) + + const result = await service.downloadInboundFile('response1.dat') + + expect(mockGetObject).toHaveBeenCalledWith('test-bucket', `${PREFIX}/INBOUND/response1.dat`) + expect(result).toEqual(Buffer.concat([Buffer.from('chunk1'), Buffer.from('chunk2')])) + }) + + it('should reject when getObject throws', async () => { + mockGetObject.mockRejectedValue(new Error('Not Found')) + + await expect(service.downloadInboundFile('missing.dat')).rejects.toThrow('Not Found') + }) + + it('should reject when stream emits an error', async () => { + const stream = new Readable({ + read() { + this.destroy(new Error('stream error')) + }, + }) + mockGetObject.mockResolvedValue(stream) + + await expect(service.downloadInboundFile('bad.dat')).rejects.toThrow('stream error') + }) + }) +}) diff --git a/backend/src/cra/transfer/s3-cra-transfer.service.ts b/backend/src/cra/transfer/s3-cra-transfer.service.ts new file mode 100644 index 00000000..93e42ab8 --- /dev/null +++ b/backend/src/cra/transfer/s3-cra-transfer.service.ts @@ -0,0 +1,80 @@ +import { Injectable } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' +import * as Minio from 'minio' +import { parseS3Uri } from 'src/common/utils' +import { CraTransferService, InboundFileInfo, TransferResult } from './cra-transfer.service' + +@Injectable() +export class S3CraTransferService extends CraTransferService { + private client: Minio.Client | null = null + + constructor(private readonly configService: ConfigService) { + super() + } + + private getClient(): Minio.Client { + if (!this.client) { + const uri = this.configService.get('sync.s3Uri')! + const { endPoint, port, useSSL, accessKey, secretKey } = parseS3Uri(uri) + + this.client = new Minio.Client({ + endPoint, + port, + useSSL, + accessKey, + secretKey, + }) + } + return this.client + } + + private getBucket(): string { + return this.configService.get('sync.s3Bucket')! + } + + private getPrefix(): string { + return this.configService.get('cra.s3Prefix')!.replace(/\/+$/, '') + } + + async sendFile(fileName: string, fileBuffer: Buffer): Promise { + const key = `${this.getPrefix()}/OUTBOUND/${fileName}` + await this.getClient().putObject(this.getBucket(), key, fileBuffer) + this.logger.log(`Uploaded ${key}`) + return { success: true, fileName } + } + + async listInboundFiles(): Promise { + const prefix = `${this.getPrefix()}/INBOUND/` + const stream = this.getClient().listObjectsV2(this.getBucket(), prefix, true) + + return new Promise((resolve, reject) => { + const files: InboundFileInfo[] = [] + + stream.on('data', (obj) => { + if (!obj.name) return + + const fileName = obj.name.substring(prefix.length) + files.push({ + fileName, + size: obj.size, + lastModifiedAt: obj.lastModified, + }) + }) + + stream.on('end', () => resolve(files)) + stream.on('error', (err) => reject(err)) + }) + } + + async downloadInboundFile(fileName: string): Promise { + const key = `${this.getPrefix()}/INBOUND/${fileName}` + const stream = await this.getClient().getObject(this.getBucket(), key) + + return new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + stream.on('data', (chunk) => chunks.push(Buffer.from(chunk))) + stream.on('end', () => resolve(Buffer.concat(chunks))) + stream.on('error', (err) => reject(err)) + }) + } +} diff --git a/backend/src/jobs/entrypoints/retry-failed.ts b/backend/src/jobs/entrypoints/retry-failed.ts index 616eedb6..5962269e 100644 --- a/backend/src/jobs/entrypoints/retry-failed.ts +++ b/backend/src/jobs/entrypoints/retry-failed.ts @@ -4,7 +4,7 @@ import { NestFactory } from '@nestjs/core' import { JobTrigger } from '../enums/job-trigger.enum' import { JobType } from '../enums/job-type.enum' import { JobRunner } from '../job-runner.service' -import { JobsModule } from '../jobs.module' +import { RetryModule } from '../retry.module' // Marks stuck jobs as failed and retries all failed jobs async function bootstrap() { @@ -14,7 +14,7 @@ async function bootstrap() { logger.log('Bootstrapping retry failed jobs...') // Create NestJS application context (no HTTP server) - const app = await NestFactory.createApplicationContext(JobsModule, { + const app = await NestFactory.createApplicationContext(RetryModule, { logger: ['log', 'error', 'warn'], }) diff --git a/backend/src/jobs/job-runner.service.spec.ts b/backend/src/jobs/job-runner.service.spec.ts index 707c21b7..1bca9c32 100644 --- a/backend/src/jobs/job-runner.service.spec.ts +++ b/backend/src/jobs/job-runner.service.spec.ts @@ -146,6 +146,20 @@ describe('JobRunner', () => { expect(mockHandler.execute).not.toHaveBeenCalled() }) + it('should handle onSuccess hook error gracefully and still return success', async () => { + const successResult: JobResult = { success: true, message: 'Done' } + vi.mocked(mockHandler.execute).mockResolvedValue(successResult) + vi.mocked(mockHandler.onSuccess).mockRejectedValue(new Error('onSuccess boom')) + + const result = await runner.executeJob(1) + + expect(result.success).toBe(true) + expect(jobsService.markSuccess).toHaveBeenCalledWith(1, undefined) + // Should not retry or mark as failed + expect(mockHandler.execute).toHaveBeenCalledTimes(1) + expect(jobsService.markFailed).not.toHaveBeenCalled() + }) + it('should handle onFailure hook error without masking original error', async () => { vi.mocked(mockHandler.execute).mockRejectedValue(new Error('Original error')) vi.mocked(mockHandler.onFailure).mockRejectedValue(new Error('onFailure boom')) diff --git a/backend/src/jobs/job-runner.service.ts b/backend/src/jobs/job-runner.service.ts index 84bc388f..f216cfb6 100644 --- a/backend/src/jobs/job-runner.service.ts +++ b/backend/src/jobs/job-runner.service.ts @@ -63,7 +63,12 @@ export class JobRunner { if (result.success) { await this.jobsService.markSuccess(jobId, result.metadata) - await handler.onSuccess?.(context, result) + try { + await handler.onSuccess?.(context, result) + } catch (hookError) { + const err = hookError instanceof Error ? hookError : new Error(String(hookError)) + this.logger.error(`Job ${jobId} onSuccess hook threw: ${err.message}`, err.stack) + } return result } else { lastError = new Error(result.message || 'Job execution returned unsuccessful result') @@ -83,7 +88,8 @@ export class JobRunner { try { await handler.onFailure?.(context, lastError!) } catch (hookError) { - this.logger.error(`Job ${jobId} onFailure hook threw: ${hookError}`) + const err = hookError instanceof Error ? hookError : new Error(String(hookError)) + this.logger.error(`Job ${jobId} onFailure hook threw: ${err.message}`, err.stack) } return { diff --git a/backend/src/jobs/retry.module.ts b/backend/src/jobs/retry.module.ts new file mode 100644 index 00000000..8d089870 --- /dev/null +++ b/backend/src/jobs/retry.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common' +import { CraModule } from 'src/cra/cra.module' +import { SyncModule } from 'src/sync/sync.module' +import { JobsModule } from './jobs.module' + +/** + * Imports all job-providing modules so RETRY_FAILED can retry any job type. + */ +@Module({ + imports: [JobsModule, SyncModule, CraModule], +}) +export class RetryModule {} diff --git a/backend/src/sync/eligibility/eligibility.queries.ts b/backend/src/sync/eligibility/eligibility.queries.ts index 5967ce26..47d4804f 100644 --- a/backend/src/sync/eligibility/eligibility.queries.ts +++ b/backend/src/sync/eligibility/eligibility.queries.ts @@ -62,7 +62,7 @@ const CHANGED_CONTACTS_CTE = ` SELECT DISTINCT cases.X_CONTACT_NUM FROM stg_icm_cases cases INNER JOIN stg_mis_placements mis_plc ON mis_plc.person_id_mis = cases.PERSON_ID_MIS - WHERE mis_plc.last_updated_date::DATE >= $1 + WHERE mis_plc.last_updated_date::DATE >= ($1 AT TIME ZONE 'America/Vancouver')::DATE UNION @@ -71,7 +71,7 @@ const CHANGED_CONTACTS_CTE = ` FROM stg_icm_cases cases INNER JOIN stg_mis_placements mis_plc ON mis_plc.person_id_mis = cases.PERSON_ID_MIS INNER JOIN stg_mis_contracts mis_con ON mis_con.service_provider_id = mis_plc.service_provider_id - WHERE mis_con.last_updated_date::DATE >= $1 + WHERE mis_con.last_updated_date::DATE >= ($1 AT TIME ZONE 'America/Vancouver')::DATE UNION @@ -81,7 +81,7 @@ const CHANGED_CONTACTS_CTE = ` INNER JOIN stg_mis_placements mis_plc ON mis_plc.person_id_mis = cases.PERSON_ID_MIS INNER JOIN stg_mis_contracts mis_con ON mis_con.service_provider_id = mis_plc.service_provider_id INNER JOIN stg_mis_payments mis_pay ON mis_pay.contract_number = mis_con.contract_number - WHERE mis_pay.last_updated_date::DATE >= $1 + WHERE mis_pay.last_updated_date::DATE >= ($1 AT TIME ZONE 'America/Vancouver')::DATE ),` /** diff --git a/backend/src/sync/eligibility/eligibility.service.spec.ts b/backend/src/sync/eligibility/eligibility.service.spec.ts index 28187291..ed1936dc 100644 --- a/backend/src/sync/eligibility/eligibility.service.spec.ts +++ b/backend/src/sync/eligibility/eligibility.service.spec.ts @@ -177,8 +177,9 @@ describe('EligibilityService', () => { it('should skip contacts with null required fields and log warning', async () => { const logSpy = vi.spyOn(service['logger'], 'warn') - mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([makeOver18Contact({ personIdIcm: null })]) - + mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([ + makeOver18Contact({ personIdIcm: null, existingContactId: 99 }), + ]) const result = await service.run() expect(result.statusChanges).toBe(1) @@ -189,8 +190,8 @@ describe('EligibilityService', () => { it('should skip invalid contacts and upsert valid ones in same batch', async () => { mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([ - makeOver18Contact({ personIdIcm: 'ICM-VALID' }), - makeOver18Contact({ personIdIcm: null }), + makeOver18Contact({ personIdIcm: 'ICM-VALID', existingContactId: 99 }), + makeOver18Contact({ personIdIcm: null, existingContactId: 99 }), ]) const result = await service.run() @@ -203,8 +204,9 @@ describe('EligibilityService', () => { it('should report caseRowId and null fields in the warning', async () => { const logSpy = vi.spyOn(service['logger'], 'warn') - mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([makeOver18Contact({ personIdIcm: null })]) - + mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([ + makeOver18Contact({ personIdIcm: null, existingContactId: 99 }), + ]) await service.run() expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('caseRowId=CASE-1')) @@ -226,7 +228,7 @@ describe('EligibilityService', () => { it('should preserve protected status and run eligibility for others in same batch', async () => { mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([ makeOver18Contact({ personIdIcm: 'ICM-HOLD', csaStatus: 'on_hold', existingContactId: 99 }), - makeOver18Contact({ personIdIcm: 'ICM-ELIG', csaStatus: 'eligible' }), + makeOver18Contact({ personIdIcm: 'ICM-ELIG', csaStatus: 'eligible', existingContactId: 88 }), ]) const result = await service.run() diff --git a/backend/src/sync/eligibility/eligibility.service.ts b/backend/src/sync/eligibility/eligibility.service.ts index 4961bbf3..02bc856f 100644 --- a/backend/src/sync/eligibility/eligibility.service.ts +++ b/backend/src/sync/eligibility/eligibility.service.ts @@ -4,7 +4,7 @@ import { TRANSACTION_TYPES } from 'src/api/contacts/constants' import { PrismaService } from 'src/common/database/prisma.service' import { BATCH_STATUS } from 'src/common/state-machine/constants/batch-status.constants' import { CSA_STATUS } from 'src/common/state-machine/constants/csa-status.constants' -import { getAgeCutoffDate, normalize, pacificToday } from 'src/common/utils' +import { getAgeCutoffDate, isEligibleAge, normalize, pacificToday } from 'src/common/utils' import { JobType } from 'src/jobs/enums/job-type.enum' import { JobsService } from 'src/jobs/jobs.service' import { CANCEL_REASON } from './cancellation/cancellation-reason.constants' @@ -376,6 +376,12 @@ export class EligibilityService { continue } + // New contacts over 18 should not be inserted into the master table + if (!profile.existingContactId && !isEligibleAge(profile.dateOfBirth, referenceDate)) { + stats.skipped++ + continue + } + // Protected statuses: preserve existing csa_status, still upsert data if ( profile.csaStatus && diff --git a/backend/src/sync/mis/file-storage/s3.service.spec.ts b/backend/src/sync/mis/file-storage/s3.service.spec.ts index d4dcf9d0..5d019b6d 100644 --- a/backend/src/sync/mis/file-storage/s3.service.spec.ts +++ b/backend/src/sync/mis/file-storage/s3.service.spec.ts @@ -1,5 +1,6 @@ import { Test, TestingModule } from '@nestjs/testing' import { ConfigService } from '@nestjs/config' +import { parseS3Uri } from 'src/common/utils' import { S3Service } from './s3.service' describe('S3Service', () => { @@ -56,7 +57,7 @@ describe('S3Service', () => { describe('parseS3Uri', () => { it('should parse a simple URI', () => { - const result = (service as any).parseS3Uri('http://minioadmin:minioadmin@localhost:9000') + const result = parseS3Uri('http://minioadmin:minioadmin@localhost:9000') expect(result).toEqual({ endPoint: 'localhost', @@ -68,7 +69,7 @@ describe('S3Service', () => { }) it('should handle password with / and special chars', () => { - const result = (service as any).parseS3Uri('https://mykey:pass/word@host.objectstore.ca') + const result = parseS3Uri('https://mykey:pass/word@host.objectstore.ca') expect(result).toEqual({ endPoint: 'host.objectstore.ca', @@ -80,7 +81,7 @@ describe('S3Service', () => { }) it('should handle password with multiple special chars (: / @)', () => { - const result = (service as any).parseS3Uri('https://key123:p@ss:w/rd@storage.example.ca') + const result = parseS3Uri('https://key123:p@ss:w/rd@storage.example.ca') expect(result).toEqual({ endPoint: 'storage.example.ca', @@ -92,9 +93,7 @@ describe('S3Service', () => { }) it('should throw on missing scheme', () => { - expect(() => (service as any).parseS3Uri('localhost:9000')).toThrow( - 'missing http(s):// scheme', - ) + expect(() => parseS3Uri('localhost:9000')).toThrow('missing http(s):// scheme') }) }) diff --git a/backend/src/sync/mis/file-storage/s3.service.ts b/backend/src/sync/mis/file-storage/s3.service.ts index 1d553fb1..498a991d 100644 --- a/backend/src/sync/mis/file-storage/s3.service.ts +++ b/backend/src/sync/mis/file-storage/s3.service.ts @@ -2,6 +2,7 @@ import { Injectable, Logger } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import * as Minio from 'minio' import { Readable } from 'stream' +import { parseS3Uri } from 'src/common/utils' import { FileStorageService } from './file-storage.service' @Injectable() @@ -16,7 +17,7 @@ export class S3Service extends FileStorageService { private getClient(): Minio.Client { if (!this.client) { const uri = this.configService.get('sync.s3Uri')! - const { endPoint, port, useSSL, accessKey, secretKey } = this.parseS3Uri(uri) + const { endPoint, port, useSSL, accessKey, secretKey } = parseS3Uri(uri) this.client = new Minio.Client({ endPoint, @@ -29,42 +30,6 @@ export class S3Service extends FileStorageService { return this.client } - // Parse s3URI manually - // `new URL()` breaks when credentials contain `/` or `@` - private parseS3Uri(uri: string) { - const schemeMatch = uri.match(/^(https?):\/\/(.+)$/) - if (!schemeMatch) throw new Error('Invalid s3URI: missing http(s):// scheme') - - const [, scheme, rest] = schemeMatch - - // Split on last `@` - // password may contain @ - const lastAt = rest.lastIndexOf('@') - if (lastAt === -1) throw new Error('Invalid s3URI: expected user:pass@host') - - const credentials = rest.substring(0, lastAt) - const hostPart = rest.substring(lastAt + 1).split('/')[0] // strip trailing path - - // Split credentials on first `:` - // password may contain : - const firstColon = credentials.indexOf(':') - if (firstColon === -1) throw new Error('Invalid s3URI: expected user:pass@host') - - const accessKey = credentials.substring(0, firstColon) - const secretKey = credentials.substring(firstColon + 1) - - // Parse host:port - const [host, portStr] = hostPart.split(':') - - return { - endPoint: host, - port: portStr ? parseInt(portStr, 10) : undefined, - useSSL: scheme === 'https', - accessKey, - secretKey, - } - } - private getBucket(): string { return this.configService.get('sync.s3Bucket')! } diff --git a/backend/src/sync/mis/mis.service.spec.ts b/backend/src/sync/mis/mis.service.spec.ts index 53bf766d..94c72f4f 100644 --- a/backend/src/sync/mis/mis.service.spec.ts +++ b/backend/src/sync/mis/mis.service.spec.ts @@ -82,7 +82,7 @@ describe('MisService', () => { }) describe('ingestAll', () => { - it('should throw when any MIS file is missing', async () => { + it('should throw when some but not all MIS files are missing', async () => { mockFileStorage.exists .mockResolvedValueOnce(true) .mockResolvedValueOnce(false) @@ -94,12 +94,13 @@ describe('MisService', () => { expect(mockFileStorage.download).not.toHaveBeenCalled() }) - it('should throw listing all missing files', async () => { + it('should return empty results when no files are present', async () => { mockFileStorage.exists.mockResolvedValue(false) - await expect(service.ingestAll()).rejects.toThrow( - 'MIS ingestion aborted: missing files [rap_payments.csv, rap_contracts.csv, rap_placements.csv]', - ) + const results = await service.ingestAll() + + expect(results).toEqual([]) + expect(mockFileStorage.download).not.toHaveBeenCalled() }) it('should ingest all 3 MIS files when they exist', async () => { diff --git a/backend/src/sync/mis/mis.service.ts b/backend/src/sync/mis/mis.service.ts index 50033ea5..fe20d4b9 100644 --- a/backend/src/sync/mis/mis.service.ts +++ b/backend/src/sync/mis/mis.service.ts @@ -1,8 +1,8 @@ import { Injectable, Logger } from '@nestjs/common' -import { pacificTodayISO } from 'src/common/utils' import { ConfigService } from '@nestjs/config' import { from as copyFrom } from 'pg-copy-streams' import { PrismaService } from 'src/common/database/prisma.service' +import { pacificTodayISO } from 'src/common/utils' import { Readable } from 'stream' import { pipeline } from 'stream/promises' import { FileStorageService } from './file-storage/file-storage.service' @@ -30,7 +30,7 @@ export class MisService { } const prefix = this.configService.get('sync.misS3Prefix') || '' - // Phase 1: All files must exist + // 1. Check file availability — all or none must be present const missingFiles: string[] = [] for (const config of MIS_FILE_CONFIGS) { const key = `${prefix}${config.s3Key}` @@ -38,11 +38,17 @@ export class MisService { missingFiles.push(config.s3Key) } } + + if (missingFiles.length === MIS_FILE_CONFIGS.length) { + this.logger.log('No MIS files found — nothing to ingest') + return [] + } + if (missingFiles.length > 0) { throw new Error(`MIS ingestion aborted: missing files [${missingFiles.join(', ')}]`) } - // Phase 2: Ingest all files + // 2. Ingest all files const results: MisResult[] = [] for (const config of MIS_FILE_CONFIGS) { const key = `${prefix}${config.s3Key}` @@ -53,7 +59,7 @@ export class MisService { results.push({ name: config.name, rows }) } - // Phase 3: Move all to PROCESSED (non-fatal) + // 3. Move all to PROCESSED for (const config of MIS_FILE_CONFIGS) { await this.moveToProcessed(`${prefix}${config.s3Key}`) } diff --git a/backend/src/sync/mock-data/mis/rap_contracts.csv b/backend/src/sync/mock-data/mis/rap_contracts.csv index f94d38bd..ddf39203 100644 --- a/backend/src/sync/mock-data/mis/rap_contracts.csv +++ b/backend/src/sync/mock-data/mis/rap_contracts.csv @@ -1,4 +1,4 @@ ID,LAST_UPDATED_DATE,SERVICE_PROVIDER_ID,SERVICE_PROVIDER_NAME,CONTRACT_NUMBER,STATUS,CONTRACT_START_DATE,CONTRACT_END_DATE,CONTRACT_TYPE,TERMINATION_DATE -CT000002,2026-02-09,SP-000002,Bashirian Inc,CN-000002,Active,2025-03-13,2026-05-25,FCH, -CT000003,2026-02-09,SP-000003,"Mosciski, Bradtke and Boehm",CN-000003,Active,2025-07-06,2026-03-08,FCH, -CT000004,2026-02-09,SP-000004,Koelpin - Wuckert,CN-000004,Active,2025-02-04,2026-06-09,FCH, +CT000002,2026-02-09 14:30:00,SP-000002,Bashirian Inc,CN-000002,Active,2025-03-13,2026-05-25,FCH, +CT000003,2026-02-09 09:15:00,SP-000003,"Mosciski, Bradtke and Boehm",CN-000003,Active,2025-07-06,2026-03-08,FCH, +CT000004,2026-02-09 22:45:00,SP-000004,Koelpin - Wuckert,CN-000004,Active,2025-02-04,2026-06-09,FCH, diff --git a/backend/src/sync/mock-data/mis/rap_payments.csv b/backend/src/sync/mock-data/mis/rap_payments.csv index 27979eae..c0529293 100644 --- a/backend/src/sync/mock-data/mis/rap_payments.csv +++ b/backend/src/sync/mock-data/mis/rap_payments.csv @@ -1,4 +1,4 @@ ID,LAST_UPDATED_DATE,PAYMENT_NUMBER,PAYMENT_TYPE,PAYMENT_STATUS,PAYMENT_AMOUNT,PAYMENT_EFFECTIVE_START_DATE,PAYMENT_EFFECTIVE_END_DATE,CONTRACT_NUMBER,PAYMENT_UPDATED,PERSON_ID_MIS -PM000002,2026-02-09,691587,MAINTENANCE PAYMENT,Pending,2117.71,2026-01-09,2026-02-22,CN-000002,Paid,1LI1AW85HZ -PM000003,2026-02-09,090399,MAINTENANCE PAYMENT,Processed,2518.00,2026-01-09,2026-02-09,CN-000003,Paid,L11TC4IR7M -PM000004,2026-02-09,107672,MONTHLY FAMILY CARE RATE,Closed,2077.64,2026-01-18,2026-02-11,CN-000004,Paid,TP23ZDDTYZ +PM000002,2026-02-09 14:30:00,691587,MAINTENANCE PAYMENT,Pending,2117.71,2026-01-09,2026-02-22,CN-000002,Paid,1LI1AW85HZ +PM000003,2026-02-09 09:15:00,090399,MAINTENANCE PAYMENT,Processed,2518.00,2026-01-09,2026-02-09,CN-000003,Paid,L11TC4IR7M +PM000004,2026-02-09 22:45:00,107672,MONTHLY FAMILY CARE RATE,Closed,2077.64,2026-01-18,2026-02-11,CN-000004,Paid,TP23ZDDTYZ diff --git a/backend/src/sync/mock-data/mis/rap_placements.csv b/backend/src/sync/mock-data/mis/rap_placements.csv index fa44a9c8..d9b062ef 100644 --- a/backend/src/sync/mock-data/mis/rap_placements.csv +++ b/backend/src/sync/mock-data/mis/rap_placements.csv @@ -1,4 +1,4 @@ ID,LAST_UPDATED_DATE,PLACEMENT_LOCATION_NO,TYPE,SUB_TYPE,STATUS,START_DATE,END_DATE,PLACE_OF_SERVICE_NAME,SERVICE_PROVIDER_NAME,SERVICE_PROVIDER_ID,CONTRACT_NUMBER,LEGACY_FILE_NUMBER,PERSON_ID_MIS -PLC000002,2026-02-09,LOC8895,Placement,Foster Care,Interrupted,2025-11-28,,Prosacco and Sons,"Hauck, Hirthe and Botsford",SP-000002,CN-000002,CS88760186,1LI1AW85HZ -PLC000003,2026-02-09,LOC6777,Placement,Foster Care,Active,2025-08-09,,Lowe - Kiehn,"Heller, Carroll and Conroy",SP-000003,CN-000003,CS65861060,L11TC4IR7M -PLC000004,2026-02-09,LOC4935,Placement,Foster Care,Interrupted,2025-07-04,,Stoltenberg and Sons,Walter - Kulas,SP-000004,CN-000004,CS09965386,TP23ZDDTYZ +PLC000002,2026-02-09 14:30:00,LOC8895,Placement,Foster Care,Interrupted,2025-11-28,,Prosacco and Sons,"Hauck, Hirthe and Botsford",SP-000002,CN-000002,CS88760186,1LI1AW85HZ +PLC000003,2026-02-09 09:15:00,LOC6777,Placement,Foster Care,Active,2025-08-09,,Lowe - Kiehn,"Heller, Carroll and Conroy",SP-000003,CN-000003,CS65861060,L11TC4IR7M +PLC000004,2026-02-09 22:45:00,LOC4935,Placement,Foster Care,Interrupted,2025-07-04,,Stoltenberg and Sons,Walter - Kulas,SP-000004,CN-000004,CS09965386,TP23ZDDTYZ diff --git a/backend/storage/cra-ftp/inbound/TST0016.VRSP0001 b/backend/storage/cra-mock/inbound/TST0016.VRSP0001 similarity index 100% rename from backend/storage/cra-ftp/inbound/TST0016.VRSP0001 rename to backend/storage/cra-mock/inbound/TST0016.VRSP0001 diff --git a/docker-compose.yml b/docker-compose.yml index f1f2597f..ebe4b67b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,3 +114,34 @@ services: depends_on: backend-prod: condition: service_healthy + + minio: + image: minio/minio + container_name: minio + profiles: ["storage"] + command: server /data --console-address ":9001" + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + + minio-init: + image: minio/mc + container_name: minio-init + profiles: ["storage"] + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 minioadmin minioadmin; + mc mb --ignore-existing local/csa-bucket; + exit 0; + "