diff --git a/backend/.env.example b/backend/.env.example index f728b7e1..7cd08f17 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -68,6 +68,9 @@ UEX_ITEMS_SYNC_ENABLED=true UEX_LOCATIONS_SYNC_ENABLED=true UEX_API_BASE_URL=https://uexcorp.space/api/2.0 UEX_TIMEOUT_MS=60000 +# Milliseconds to wait between UEX API requests (default: 500). +# Increase if UEX rate-limits this application. +UEX_REQUEST_DELAY_MS=500 UEX_BATCH_SIZE=100 UEX_CONCURRENT_CATEGORIES=3 UEX_RETRY_ATTEMPTS=3 diff --git a/backend/src/common/services/advisory-lock.service.spec.ts b/backend/src/common/services/advisory-lock.service.spec.ts new file mode 100644 index 00000000..a728f581 --- /dev/null +++ b/backend/src/common/services/advisory-lock.service.spec.ts @@ -0,0 +1,112 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ConflictException } from '@nestjs/common'; +import { getDataSourceToken } from '@nestjs/typeorm'; +import { AdvisoryLockService } from './advisory-lock.service'; + +describe('AdvisoryLockService', () => { + let service: AdvisoryLockService; + let mockQr: { + connect: jest.Mock; + query: jest.Mock; + release: jest.Mock; + }; + let mockDataSource: { createQueryRunner: jest.Mock }; + + beforeEach(async () => { + mockQr = { + connect: jest.fn().mockResolvedValue(undefined), + query: jest.fn(), + release: jest.fn().mockResolvedValue(undefined), + }; + + mockDataSource = { + createQueryRunner: jest.fn().mockReturnValue(mockQr), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + AdvisoryLockService, + { + provide: getDataSourceToken(), + useValue: mockDataSource, + }, + ], + }).compile(); + + service = module.get(AdvisoryLockService); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('withLock', () => { + it('acquires lock, runs fn, releases lock and QueryRunner', async () => { + mockQr.query + .mockResolvedValueOnce([{ acquired: true }]) // pg_try_advisory_lock + .mockResolvedValueOnce([{}]); // pg_advisory_unlock + + const fn = jest.fn().mockResolvedValue('result'); + + const result = await service.withLock('test-lock', fn); + + expect(result).toBe('result'); + expect(mockDataSource.createQueryRunner).toHaveBeenCalledTimes(1); + + // Both queries go through the same QueryRunner + expect(mockQr.connect).toHaveBeenCalledTimes(1); + expect(mockQr.query).toHaveBeenNthCalledWith( + 1, + `SELECT pg_try_advisory_lock(hashtext($1)) AS acquired`, + ['test-lock'], + ); + expect(fn).toHaveBeenCalledTimes(1); + expect(mockQr.query).toHaveBeenNthCalledWith( + 2, + `SELECT pg_advisory_unlock(hashtext($1))`, + ['test-lock'], + ); + expect(mockQr.release).toHaveBeenCalledTimes(1); + }); + + it('throws ConflictException when lock is not acquired; fn not called; runner still released', async () => { + mockQr.query.mockResolvedValueOnce([{ acquired: false }]); + + const fn = jest.fn(); + + await expect(service.withLock('test-lock', fn)).rejects.toThrow( + ConflictException, + ); + + expect(fn).not.toHaveBeenCalled(); + // unlock query should NOT have been called + expect(mockQr.query).not.toHaveBeenCalledWith( + `SELECT pg_advisory_unlock(hashtext($1))`, + expect.anything(), + ); + // runner must still be released + expect(mockQr.release).toHaveBeenCalled(); + }); + + it('releases lock and runner even when fn throws', async () => { + mockQr.query + .mockResolvedValueOnce([{ acquired: true }]) // pg_try_advisory_lock + .mockResolvedValueOnce([{}]); // pg_advisory_unlock + + const error = new Error('fn blew up'); + const fn = jest.fn().mockRejectedValue(error); + + await expect(service.withLock('test-lock', fn)).rejects.toThrow( + 'fn blew up', + ); + + // unlock still called + expect(mockQr.query).toHaveBeenCalledWith( + `SELECT pg_advisory_unlock(hashtext($1))`, + ['test-lock'], + ); + // runner still released + expect(mockQr.release).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/backend/src/common/services/advisory-lock.service.ts b/backend/src/common/services/advisory-lock.service.ts new file mode 100644 index 00000000..bc6dfde8 --- /dev/null +++ b/backend/src/common/services/advisory-lock.service.ts @@ -0,0 +1,31 @@ +import { Injectable, ConflictException } from '@nestjs/common'; +import { InjectDataSource } from '@nestjs/typeorm'; +import { DataSource } from 'typeorm'; + +@Injectable() +export class AdvisoryLockService { + constructor( + @InjectDataSource() + private readonly dataSource: DataSource, + ) {} + + async withLock(lockKey: string, fn: () => Promise): Promise { + const qr = this.dataSource.createQueryRunner(); + await qr.connect(); + try { + const [{ acquired }] = await qr.query( + `SELECT pg_try_advisory_lock(hashtext($1)) AS acquired`, + [lockKey], + ); + if (!acquired) + throw new ConflictException(`Lock '${lockKey}' already held`); + try { + return await fn(); + } finally { + await qr.query(`SELECT pg_advisory_unlock(hashtext($1))`, [lockKey]); + } + } finally { + await qr.release(); + } + } +} diff --git a/backend/src/common/services/index.ts b/backend/src/common/services/index.ts new file mode 100644 index 00000000..257b6cc9 --- /dev/null +++ b/backend/src/common/services/index.ts @@ -0,0 +1 @@ +export { AdvisoryLockService } from './advisory-lock.service'; diff --git a/backend/src/modules/catalog-etl/catalog-etl.module.ts b/backend/src/modules/catalog-etl/catalog-etl.module.ts index 755f4e47..206efadf 100644 --- a/backend/src/modules/catalog-etl/catalog-etl.module.ts +++ b/backend/src/modules/catalog-etl/catalog-etl.module.ts @@ -4,11 +4,12 @@ import { CatalogEtlService } from './catalog-etl.service'; import { CatalogEtlController } from './catalog-etl.controller'; import { EtlRun } from './entities/etl-run.entity'; import { EtlWarning } from './entities/etl-warning.entity'; +import { AdvisoryLockService } from '../../common/services'; @Module({ imports: [TypeOrmModule.forFeature([EtlRun, EtlWarning])], controllers: [CatalogEtlController], - providers: [CatalogEtlService], + providers: [CatalogEtlService, AdvisoryLockService], exports: [CatalogEtlService], }) export class CatalogEtlModule {} diff --git a/backend/src/modules/catalog-etl/catalog-etl.service.spec.ts b/backend/src/modules/catalog-etl/catalog-etl.service.spec.ts index ce9c759b..2af16f07 100644 --- a/backend/src/modules/catalog-etl/catalog-etl.service.spec.ts +++ b/backend/src/modules/catalog-etl/catalog-etl.service.spec.ts @@ -2,11 +2,11 @@ import { Test, TestingModule } from '@nestjs/testing'; import { getRepositoryToken } from '@nestjs/typeorm'; import { ConflictException } from '@nestjs/common'; import { getLoggerToken } from 'nestjs-pino'; -import { DataSource } from 'typeorm'; import { CatalogEtlService } from './catalog-etl.service'; import { EtlRun } from './entities/etl-run.entity'; import { EtlWarning } from './entities/etl-warning.entity'; import { EtlStep } from './interfaces/etl-step.interface'; +import { AdvisoryLockService } from '../../common/services'; function buildMockRun(overrides: Partial = {}): EtlRun { const run = new EtlRun(); @@ -25,7 +25,7 @@ describe('CatalogEtlService', () => { let service: CatalogEtlService; let mockEtlRunRepository: Record; let mockEtlWarningRepository: Record; - let mockDataSource: { query: jest.Mock }; + let mockAdvisoryLockService: { withLock: jest.Mock }; beforeEach(async () => { mockEtlRunRepository = { @@ -41,8 +41,10 @@ describe('CatalogEtlService', () => { find: jest.fn(), }; - mockDataSource = { - query: jest.fn(), + mockAdvisoryLockService = { + withLock: jest + .fn() + .mockImplementation((_key: string, fn: () => Promise) => fn()), }; const module: TestingModule = await Test.createTestingModule({ @@ -66,8 +68,8 @@ describe('CatalogEtlService', () => { useValue: mockEtlWarningRepository, }, { - provide: DataSource, - useValue: mockDataSource, + provide: AdvisoryLockService, + useValue: mockAdvisoryLockService, }, ], }).compile(); @@ -101,11 +103,6 @@ describe('CatalogEtlService', () => { step2, ]; - // Lock acquired - mockDataSource.query - .mockResolvedValueOnce([{ acquired: true }]) // pg_try_advisory_lock - .mockResolvedValueOnce([{}]); // pg_advisory_unlock - const initialRun = buildMockRun({ stepsTotal: 2 }); mockEtlRunRepository.create.mockReturnValue(initialRun); mockEtlRunRepository.save @@ -119,6 +116,10 @@ describe('CatalogEtlService', () => { const result = await service.runEtl(); + expect(mockAdvisoryLockService.withLock).toHaveBeenCalledWith( + 'catalog_etl', + expect.any(Function), + ); expect(result.status).toBe('completed'); expect(result.stepsSucceeded).toBe(2); expect(result.stepsFailed).toBe(0); @@ -141,10 +142,6 @@ describe('CatalogEtlService', () => { failingStep, ]; - mockDataSource.query - .mockResolvedValueOnce([{ acquired: true }]) - .mockResolvedValueOnce([{}]); - const initialRun = buildMockRun({ stepsTotal: 2 }); mockEtlRunRepository.create.mockReturnValue(initialRun); mockEtlRunRepository.save @@ -186,10 +183,6 @@ describe('CatalogEtlService', () => { failingStep2, ]; - mockDataSource.query - .mockResolvedValueOnce([{ acquired: true }]) - .mockResolvedValueOnce([{}]); - const initialRun = buildMockRun({ stepsTotal: 2 }); mockEtlRunRepository.create.mockReturnValue(initialRun); mockEtlRunRepository.save @@ -211,10 +204,6 @@ describe('CatalogEtlService', () => { it('no steps registered → status no_steps', async () => { // ETL_STEPS remains [] (default) - mockDataSource.query - .mockResolvedValueOnce([{ acquired: true }]) - .mockResolvedValueOnce([{}]); - const initialRun = buildMockRun({ stepsTotal: 0 }); mockEtlRunRepository.create.mockReturnValue(initialRun); mockEtlRunRepository.save @@ -232,12 +221,13 @@ describe('CatalogEtlService', () => { }); it('concurrent lock rejection → throws 409 ConflictException', async () => { - mockDataSource.query.mockResolvedValue([{ acquired: false }]); + mockAdvisoryLockService.withLock.mockRejectedValueOnce( + new ConflictException("Lock 'catalog_etl' already held"), + ); const runPromise = service.runEtl(); await expect(runPromise).rejects.toThrow(ConflictException); - await expect(runPromise).rejects.toThrow('ETL run already in progress'); // Repository should never be touched when lock not acquired expect(mockEtlRunRepository.create).not.toHaveBeenCalled(); diff --git a/backend/src/modules/catalog-etl/catalog-etl.service.ts b/backend/src/modules/catalog-etl/catalog-etl.service.ts index 3669616f..3117a69e 100644 --- a/backend/src/modules/catalog-etl/catalog-etl.service.ts +++ b/backend/src/modules/catalog-etl/catalog-etl.service.ts @@ -1,10 +1,11 @@ -import { Injectable, ConflictException } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository, DataSource } from 'typeorm'; +import { Repository } from 'typeorm'; import { InjectPinoLogger, PinoLogger } from 'nestjs-pino'; import { EtlRun } from './entities/etl-run.entity'; import { EtlWarning } from './entities/etl-warning.entity'; import { EtlStep } from './interfaces/etl-step.interface'; +import { AdvisoryLockService } from '../../common/services'; @Injectable() export class CatalogEtlService { @@ -15,34 +16,24 @@ export class CatalogEtlService { private readonly etlRunRepository: Repository, @InjectRepository(EtlWarning) private readonly etlWarningRepository: Repository, - private readonly dataSource: DataSource, + private readonly advisoryLockService: AdvisoryLockService, @InjectPinoLogger(CatalogEtlService.name) private readonly logger: PinoLogger, ) {} async runEtl(): Promise { - // Acquire advisory lock to prevent concurrent ETL runs - const lockResult = await this.dataSource.query<{ acquired: boolean }[]>( - `SELECT pg_try_advisory_lock(hashtext('catalog_etl')) AS acquired`, - ); + return this.advisoryLockService.withLock('catalog_etl', async () => { + // Create the run record + const runState = this.etlRunRepository.create({ + status: 'running', + stepsTotal: this.ETL_STEPS.length, + stepsSucceeded: 0, + stepsFailed: 0, + }); + await this.etlRunRepository.save(runState); - const acquired = lockResult[0]?.acquired; - if (!acquired) { - throw new ConflictException('ETL run already in progress'); - } + this.logger.info({ runId: runState.runId }, 'ETL run started'); - // Create the run record - const runState = this.etlRunRepository.create({ - status: 'running', - stepsTotal: this.ETL_STEPS.length, - stepsSucceeded: 0, - stepsFailed: 0, - }); - await this.etlRunRepository.save(runState); - - this.logger.info({ runId: runState.runId }, 'ETL run started'); - - try { for (const step of this.ETL_STEPS) { try { await step.execute({ runId: runState.runId }); @@ -69,33 +60,28 @@ export class CatalogEtlService { await this.etlWarningRepository.save(warning); } } - } finally { - // Release advisory lock regardless of outcome - await this.dataSource.query( - `SELECT pg_advisory_unlock(hashtext('catalog_etl'))`, - ); - } - // Determine final status - if (this.ETL_STEPS.length === 0) { - runState.status = 'no_steps'; - } else if (runState.stepsFailed === 0) { - runState.status = 'completed'; - } else if (runState.stepsSucceeded === 0) { - runState.status = 'failed'; - } else { - runState.status = 'partial'; - } + // Determine final status + if (this.ETL_STEPS.length === 0) { + runState.status = 'no_steps'; + } else if (runState.stepsFailed === 0) { + runState.status = 'completed'; + } else if (runState.stepsSucceeded === 0) { + runState.status = 'failed'; + } else { + runState.status = 'partial'; + } - runState.completedAt = new Date(); - const savedRun = await this.etlRunRepository.save(runState); + runState.completedAt = new Date(); + const savedRun = await this.etlRunRepository.save(runState); - this.logger.info( - { runId: savedRun.runId, status: savedRun.status }, - 'ETL run completed', - ); + this.logger.info( + { runId: savedRun.runId, status: savedRun.status }, + 'ETL run completed', + ); - return savedRun; + return savedRun; + }); } async getRuns(page: number, limit: number): Promise<[EtlRun[], number]> { diff --git a/backend/src/modules/uex-sync/clients/uex-api.client.spec.ts b/backend/src/modules/uex-sync/clients/uex-api.client.spec.ts new file mode 100644 index 00000000..c4f31d83 --- /dev/null +++ b/backend/src/modules/uex-sync/clients/uex-api.client.spec.ts @@ -0,0 +1,285 @@ +import { UexApiClient, UexApiConfig } from './uex-api.client'; +import { + RateLimitException, + UEXServerException, + UEXClientException, +} from '../exceptions/uex-exceptions'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeConfig(overrides: Partial = {}): UexApiConfig { + return { + baseUrl: 'https://uexcorp.space/api/2.0', + requestDelayMs: 100, + timeoutMs: 5000, + ...overrides, + }; +} + +function makeLogger() { + return { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('UexApiClient', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + jest.clearAllMocks(); + }); + + describe('get()', () => { + it('sleeps for requestDelayMs then returns response.data.data', async () => { + const client = new UexApiClient( + makeLogger() as never, + makeConfig({ requestDelayMs: 200 }), + ); + + // Patch the internal axios instance + const fakeData = [{ id: 1, name: 'test' }]; + const axiosGet = jest + .spyOn( + (client as unknown as { axiosInstance: { get: jest.Mock } }) + .axiosInstance, + 'get', + ) + .mockResolvedValueOnce({ + data: { data: fakeData }, + headers: {}, + status: 200, + }); + + // Start the call but don't await yet — it will be blocked on sleep(200) + const promise = client.get('/categories'); + + // Advance timers past the sleep + jest.advanceTimersByTime(200); + + const result = await promise; + + expect(result).toEqual(fakeData); + expect(axiosGet).toHaveBeenCalledWith('/categories', { + params: undefined, + }); + }); + + it('throws RateLimitException on HTTP 200 with status=error and requests_limit_reached', async () => { + const client = new UexApiClient(makeLogger() as never, makeConfig()); + + jest + .spyOn( + (client as unknown as { axiosInstance: { get: jest.Mock } }) + .axiosInstance, + 'get', + ) + .mockResolvedValueOnce({ + data: { status: 'error', message: 'requests_limit_reached' }, + headers: {}, + status: 200, + }); + + const promise = client.get('/categories'); + jest.advanceTimersByTime(200); + + await expect(promise).rejects.toBeInstanceOf(RateLimitException); + }); + + it('doubles delayMs when x-ratelimit-remaining ≤ 5', async () => { + const config = makeConfig({ requestDelayMs: 100 }); + const client = new UexApiClient(makeLogger() as never, config); + + const axiosInstance = ( + client as unknown as { + axiosInstance: { + get: jest.Mock; + interceptors: { + response: { + handlers: Array<{ fulfilled: (r: unknown) => unknown }>; + }; + }; + }; + } + ).axiosInstance; + + // Trigger the response interceptor manually to simulate a low-remaining header + const responseInterceptor = + axiosInstance.interceptors.response.handlers[0].fulfilled; + + const fakeResponse = { + data: { data: [] }, + headers: { 'x-ratelimit-remaining': '3' }, + status: 200, + }; + + const returned = responseInterceptor(fakeResponse); + expect(returned).toBe(fakeResponse); + + // delayMs should be doubled + expect((client as unknown as { delayMs: number }).delayMs).toBe(200); + }); + + it('does NOT double delayMs when x-ratelimit-remaining > 5', async () => { + const config = makeConfig({ requestDelayMs: 100 }); + const client = new UexApiClient(makeLogger() as never, config); + + const axiosInstance = ( + client as unknown as { + axiosInstance: { + interceptors: { + response: { + handlers: Array<{ fulfilled: (r: unknown) => unknown }>; + }; + }; + }; + } + ).axiosInstance; + const responseInterceptor = + axiosInstance.interceptors.response.handlers[0].fulfilled; + + responseInterceptor({ + data: { data: [] }, + headers: { 'x-ratelimit-remaining': '10' }, + status: 200, + }); + + expect((client as unknown as { delayMs: number }).delayMs).toBe(100); + }); + }); + + describe('response interceptor error handling', () => { + it('throws RateLimitException immediately when retryCount >= 3 (max retries exhausted)', async () => { + const client = new UexApiClient(makeLogger() as never, makeConfig()); + + const axiosInstance = ( + client as unknown as { + axiosInstance: { + interceptors: { + response: { + handlers: Array<{ rejected: (e: unknown) => unknown }>; + }; + }; + }; + } + ).axiosInstance; + + const errorInterceptor = + axiosInstance.interceptors.response.handlers[0].rejected; + + // When __retryCount is already 3 (>= 3), no further retries — throws immediately + const error = { + response: { status: 429, headers: { 'retry-after': '1' } }, + config: { __retryCount: 3 }, + }; + + await expect(errorInterceptor(error)).rejects.toBeInstanceOf( + RateLimitException, + ); + }); + + it('schedules a retry when 429 and retryCount < 3', async () => { + const client = new UexApiClient(makeLogger() as never, makeConfig()); + + const axiosInstance = ( + client as unknown as { + axiosInstance: { + request: jest.Mock; + interceptors: { + response: { + handlers: Array<{ rejected: (e: unknown) => unknown }>; + }; + }; + }; + } + ).axiosInstance; + + const errorInterceptor = + axiosInstance.interceptors.response.handlers[0].rejected; + + // Mock request to return a resolved value (simulating a successful retry) + const fakeResult = { data: { data: [] }, headers: {}, status: 200 }; + axiosInstance.request = jest.fn().mockResolvedValue(fakeResult); + + const error = { + response: { status: 429, headers: { 'retry-after': '0' } }, + config: { __retryCount: 0 }, + }; + + const promise = errorInterceptor(error); + // Advance fake timers to let sleep(0) resolve + jest.runAllTimers(); + + const result = await promise; + expect(result).toBe(fakeResult); + expect(axiosInstance.request).toHaveBeenCalledTimes(1); + // __retryCount should have been incremented to 1 + expect( + (axiosInstance.request as jest.Mock).mock.calls[0][0].__retryCount, + ).toBe(1); + }); + + it('throws UEXServerException on 5xx response', async () => { + const client = new UexApiClient(makeLogger() as never, makeConfig()); + + const axiosInstance = ( + client as unknown as { + axiosInstance: { + interceptors: { + response: { + handlers: Array<{ rejected: (e: unknown) => unknown }>; + }; + }; + }; + } + ).axiosInstance; + + const errorInterceptor = + axiosInstance.interceptors.response.handlers[0].rejected; + + await expect( + errorInterceptor({ + response: { status: 503, headers: {} }, + config: {}, + }), + ).rejects.toBeInstanceOf(UEXServerException); + }); + + it('throws UEXClientException on other errors', async () => { + const client = new UexApiClient(makeLogger() as never, makeConfig()); + + const axiosInstance = ( + client as unknown as { + axiosInstance: { + interceptors: { + response: { + handlers: Array<{ rejected: (e: unknown) => unknown }>; + }; + }; + }; + } + ).axiosInstance; + + const errorInterceptor = + axiosInstance.interceptors.response.handlers[0].rejected; + + await expect( + errorInterceptor({ + response: { status: 400, headers: {} }, + config: {}, + }), + ).rejects.toBeInstanceOf(UEXClientException); + }); + }); +}); diff --git a/backend/src/modules/uex-sync/clients/uex-api.client.ts b/backend/src/modules/uex-sync/clients/uex-api.client.ts new file mode 100644 index 00000000..c7944678 --- /dev/null +++ b/backend/src/modules/uex-sync/clients/uex-api.client.ts @@ -0,0 +1,116 @@ +// UexApiClient is the shared HTTP wrapper for all ETL step implementations +// (#190–#199). The legacy per-endpoint clients (UEXCategoriesClient etc.) +// predate this and are not migrated here; new ETL steps inject UexApiClient +// directly and inherit rate-limiting, backoff, and retry automatically. + +import { Injectable } from '@nestjs/common'; +import { InjectPinoLogger, PinoLogger } from 'nestjs-pino'; +import axios, { + AxiosInstance, + AxiosResponse, + InternalAxiosRequestConfig, +} from 'axios'; +import { + RateLimitException, + UEXServerException, + UEXClientException, +} from '../exceptions/uex-exceptions'; + +export interface UexApiConfig { + baseUrl: string; + requestDelayMs: number; + timeoutMs: number; +} + +const sleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +@Injectable() +export class UexApiClient { + private readonly axiosInstance: AxiosInstance; + private delayMs: number; + + constructor( + @InjectPinoLogger(UexApiClient.name) private readonly logger: PinoLogger, + private readonly config: UexApiConfig, + ) { + this.delayMs = config.requestDelayMs; + + this.axiosInstance = axios.create({ + baseURL: config.baseUrl, + timeout: config.timeoutMs, + }); + + this.axiosInstance.interceptors.response.use( + (response: AxiosResponse) => { + const remaining = response.headers['x-ratelimit-remaining']; + if (remaining !== undefined && Number(remaining) <= 5) { + this.delayMs *= 2; + this.logger.warn( + { remaining, newDelayMs: this.delayMs }, + 'Rate limit headroom low — doubling request delay', + ); + } + return response; + }, + async (error: { + response?: { status: number; headers: Record }; + config: InternalAxiosRequestConfig & { __retryCount?: number }; + }) => { + const status = error.response?.status; + + if (status === 429) { + const retryAfterHeader = error.response?.headers['retry-after']; + const retryAfterSeconds = retryAfterHeader + ? Number(retryAfterHeader) + : 5; + + const retryCount = error.config.__retryCount ?? 0; + + if (retryCount < 3) { + const backoffMs = + retryAfterSeconds * 1000 * Math.pow(2, retryCount); + this.logger.warn( + { retryCount, backoffMs }, + 'UEX 429 received — backing off before retry', + ); + await sleep(backoffMs); + error.config.__retryCount = retryCount + 1; + return this.axiosInstance.request(error.config); + } + + throw new RateLimitException( + 'UEX API rate limit exceeded after retries', + ); + } + + if (status !== undefined && status >= 500) { + throw new UEXServerException(`UEX server error: ${status}`); + } + + throw new UEXClientException( + `UEX client error: ${status ?? 'unknown'}`, + ); + }, + ); + } + + async get(path: string, params?: Record): Promise { + await sleep(this.delayMs); + const response = await this.axiosInstance.get<{ + status: string; + data: T; + message?: string; + }>(path, { params }); + + // UEX returns HTTP 200 with status='error' for soft rate-limit hits + if ( + response.data.status === 'error' && + response.data.message?.includes('requests_limit_reached') + ) { + throw new RateLimitException('UEX API rate limit exceeded'); + } + + return response.data.data; + } +} diff --git a/backend/src/modules/uex-sync/uex-sync.module.ts b/backend/src/modules/uex-sync/uex-sync.module.ts index 56fbb206..b13df00c 100644 --- a/backend/src/modules/uex-sync/uex-sync.module.ts +++ b/backend/src/modules/uex-sync/uex-sync.module.ts @@ -1,6 +1,8 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { HttpModule } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; +import { getLoggerToken, PinoLogger } from 'nestjs-pino'; import { UexSyncService } from './uex-sync.service'; import { UexSyncController } from './uex-sync.controller'; import { UexSyncState } from './uex-sync-state.entity'; @@ -20,6 +22,7 @@ import { UEXCategoriesClient } from './clients/uex-categories.client'; import { UEXCommoditiesClient } from './clients/uex-commodities.client'; import { UEXItemsClient } from './clients/uex-items.client'; import { UEXCompaniesClient } from './clients/uex-companies.client'; +import { UexApiClient } from './clients/uex-api.client'; import { CategoriesSyncService } from './services/categories-sync.service'; import { CommoditiesSyncService } from './services/commodities-sync.service'; import { ItemsSyncService } from './services/items-sync.service'; @@ -54,6 +57,22 @@ import { UsersModule } from '../users/users.module'; UEXCommoditiesClient, UEXItemsClient, UEXCompaniesClient, + { + provide: UexApiClient, + useFactory: (configService: ConfigService, logger: PinoLogger) => + new UexApiClient(logger, { + baseUrl: configService.get( + 'UEX_API_BASE_URL', + 'https://uexcorp.space/api/2.0', + ), + requestDelayMs: configService.get( + 'UEX_REQUEST_DELAY_MS', + 500, + ), + timeoutMs: configService.get('UEX_TIMEOUT_MS', 30000), + }), + inject: [ConfigService, getLoggerToken(UexApiClient.name)], + }, CategoriesSyncService, CommoditiesSyncService, ItemsSyncService, @@ -65,6 +84,7 @@ import { UsersModule } from '../users/users.module'; CategoriesSyncService, CommoditiesSyncService, ItemsSyncService, + UexApiClient, ], }) export class UexSyncModule {}