From 585611112bae39fbd5be651348d9bf791102971e Mon Sep 17 00:00:00 2001 From: gitaddremote Date: Mon, 25 May 2026 16:49:31 -0400 Subject: [PATCH 1/3] feat: add CategoriesSyncStep for ISSUE-196 - Fetches GET /categories from UEX and upserts into station_category - Synthetic section rows created per unique (type, section) pair using ON CONFLICT (type, name) WHERE is_section = TRUE; RETURNING id used to resolve parent_id FK for leaf rows in the same execute() call - Section rows always upserted before leaf rows to satisfy self-referencing FK - Leaf rows conflict on uex_id WHERE uex_id IS NOT NULL; parent_id, type, section, is_game_related, is_mining kept current on re-runs - Category attributes upserted into station_category_attribute per uex_id - Warns on missing name (category or attribute), unknown type (stored as null) - Registered in CatalogEtlModule and ETL_STEPS pipeline at tier-8 (after jump-points-sync, before items/vehicles/commodities) - 22 unit tests: section creation, parent FK resolution, attribute upsert, idempotency, warnings, empty list handling --- .../modules/catalog-etl/catalog-etl.module.ts | 2 + .../catalog-etl/catalog-etl.service.spec.ts | 5 + .../catalog-etl/catalog-etl.service.ts | 3 + .../steps/categories-sync.step.spec.ts | 450 ++++++++++++++++++ .../catalog-etl/steps/categories-sync.step.ts | 223 +++++++++ 5 files changed, 683 insertions(+) create mode 100644 backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts create mode 100644 backend/src/modules/catalog-etl/steps/categories-sync.step.ts diff --git a/backend/src/modules/catalog-etl/catalog-etl.module.ts b/backend/src/modules/catalog-etl/catalog-etl.module.ts index 6e006fd..5b209ec 100644 --- a/backend/src/modules/catalog-etl/catalog-etl.module.ts +++ b/backend/src/modules/catalog-etl/catalog-etl.module.ts @@ -17,6 +17,7 @@ import { SpaceStationsSyncStep } from './steps/space-stations-sync.step'; import { OutpostsSyncStep } from './steps/outposts-sync.step'; import { PoisSyncStep } from './steps/pois-sync.step'; import { JumpPointsSyncStep } from './steps/jump-points-sync.step'; +import { CategoriesSyncStep } from './steps/categories-sync.step'; import { UexSyncModule } from '../uex-sync/uex-sync.module'; @Module({ @@ -37,6 +38,7 @@ import { UexSyncModule } from '../uex-sync/uex-sync.module'; OutpostsSyncStep, PoisSyncStep, JumpPointsSyncStep, + CategoriesSyncStep, ], exports: [CatalogEtlService], }) diff --git a/backend/src/modules/catalog-etl/catalog-etl.service.spec.ts b/backend/src/modules/catalog-etl/catalog-etl.service.spec.ts index b7dd013..06f2fcd 100644 --- a/backend/src/modules/catalog-etl/catalog-etl.service.spec.ts +++ b/backend/src/modules/catalog-etl/catalog-etl.service.spec.ts @@ -19,6 +19,7 @@ import { SpaceStationsSyncStep } from './steps/space-stations-sync.step'; import { OutpostsSyncStep } from './steps/outposts-sync.step'; import { PoisSyncStep } from './steps/pois-sync.step'; import { JumpPointsSyncStep } from './steps/jump-points-sync.step'; +import { CategoriesSyncStep } from './steps/categories-sync.step'; function buildMockRun(overrides: Partial = {}): EtlRun { const run = new EtlRun(); @@ -131,6 +132,10 @@ describe('CatalogEtlService', () => { provide: JumpPointsSyncStep, useValue: { name: 'jump-points', execute: jest.fn() }, }, + { + provide: CategoriesSyncStep, + useValue: { name: 'categories-sync', execute: jest.fn() }, + }, ], }).compile(); diff --git a/backend/src/modules/catalog-etl/catalog-etl.service.ts b/backend/src/modules/catalog-etl/catalog-etl.service.ts index 94edbc8..1aa8ad0 100644 --- a/backend/src/modules/catalog-etl/catalog-etl.service.ts +++ b/backend/src/modules/catalog-etl/catalog-etl.service.ts @@ -18,6 +18,7 @@ import { SpaceStationsSyncStep } from './steps/space-stations-sync.step'; import { OutpostsSyncStep } from './steps/outposts-sync.step'; import { PoisSyncStep } from './steps/pois-sync.step'; import { JumpPointsSyncStep } from './steps/jump-points-sync.step'; +import { CategoriesSyncStep } from './steps/categories-sync.step'; @Injectable() export class CatalogEtlService { @@ -43,6 +44,7 @@ export class CatalogEtlService { private readonly outpostsSyncStep: OutpostsSyncStep, private readonly poisSyncStep: PoisSyncStep, private readonly jumpPointsSyncStep: JumpPointsSyncStep, + private readonly categoriesSyncStep: CategoriesSyncStep, ) { this.ETL_STEPS = [ factionsSyncStep, @@ -57,6 +59,7 @@ export class CatalogEtlService { outpostsSyncStep, poisSyncStep, jumpPointsSyncStep, + categoriesSyncStep, ]; } diff --git a/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts b/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts new file mode 100644 index 0000000..92eed3b --- /dev/null +++ b/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts @@ -0,0 +1,450 @@ +import { CategoriesSyncStep } from './categories-sync.step'; + +const CTX = { runId: 'test-run-id' }; + +function makeLogger() { + return { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }; +} + +function makeCategory(overrides: Record = {}) { + return { + id: 1, + name: 'Rifles', + code: 'rifles', + type: 'item', + section: 'Weapons', + is_game_related: 1, + is_mining: 0, + date_added: 1700000000, + date_modified: 1710000000, + attributes: [], + ...overrides, + }; +} + +function makeAttribute(overrides: Record = {}) { + return { + id: 10, + name: 'damage', + description: 'Damage per shot', + is_lower_better: 0, + date_added: 1700000000, + date_modified: 1710000000, + ...overrides, + }; +} + +// dsQuery mock: RETURNING id for section upserts returns { id: 999 } +function buildDsQuery(sectionId = 999): jest.Mock { + return jest.fn().mockImplementation((sql: string) => { + if (sql.includes('RETURNING id')) { + return Promise.resolve([{ id: sectionId }]); + } + return Promise.resolve([]); + }); +} + +function buildStep( + uexGet: jest.Mock, + dsQuery: jest.Mock, + repoCreate: jest.Mock, + repoSave: jest.Mock, +) { + return new CategoriesSyncStep( + { get: uexGet } as never, + { query: dsQuery } as never, + { create: repoCreate, save: repoSave } as never, + makeLogger() as never, + ); +} + +describe('CategoriesSyncStep', () => { + let uexGet: jest.Mock; + let repoCreate: jest.Mock; + let repoSave: jest.Mock; + + beforeEach(() => { + uexGet = jest.fn(); + repoCreate = jest.fn().mockImplementation((dto) => dto); + repoSave = jest.fn().mockResolvedValue({}); + }); + + afterEach(() => jest.clearAllMocks()); + + describe('section row creation', () => { + it('upserts one section row per unique (type, section) pair', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ id: 1, section: 'Weapons', type: 'item' }), + makeCategory({ + id: 2, + name: 'Pistols', + section: 'Weapons', + type: 'item', + }), + makeCategory({ + id: 3, + name: 'Ships', + section: 'Vehicles', + type: 'item', + }), + ]); + + await step.execute(CTX); + + const sectionInserts = dsQuery.mock.calls.filter( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + sql.includes('is_section') && + sql.includes('RETURNING id'), + ); + expect(sectionInserts).toHaveLength(2); + }); + + it('section INSERT uses is_section=TRUE and parent_id=NULL', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory()]); + + await step.execute(CTX); + + const sectionInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + sql.includes('RETURNING id'), + ); + expect(sectionInsert).toBeDefined(); + // type=$1, section=$2, name=$3 + expect(sectionInsert[1][0]).toBe('item'); // type + expect(sectionInsert[1][1]).toBe('Weapons'); // section + expect(sectionInsert[1][2]).toBe('Weapons'); // name = section label + }); + + it('section rows conflict on (type, name) WHERE is_section = TRUE', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory()]); + + await step.execute(CTX); + + const sectionInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + sql.includes('RETURNING id'), + ); + expect(sectionInsert[0]).toContain( + 'ON CONFLICT (type, name) WHERE is_section = TRUE', + ); + }); + }); + + describe('leaf row parent FK', () => { + it('sets parent_id on leaf row to the id returned by section upsert', async () => { + const dsQuery = buildDsQuery(42); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory({ id: 1, section: 'Weapons' })]); + + await step.execute(CTX); + + const leafInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInsert).toBeDefined(); + expect(leafInsert[1][1]).toBe(42); // parent_id = $2 + }); + + it('sets parent_id to null when category has no section', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory({ section: null })]); + + await step.execute(CTX); + + const leafInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInsert[1][1]).toBeNull(); // parent_id = $2 + }); + }); + + describe('leaf row upsert', () => { + it('upserts leaf row with uex_id as conflict target', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory({ id: 5 })]); + + await step.execute(CTX); + + const leafInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInsert).toBeDefined(); + expect(leafInsert[0]).toContain( + 'ON CONFLICT (uex_id) WHERE uex_id IS NOT NULL', + ); + expect(leafInsert[1][0]).toBe(5); // uex_id = $1 + }); + + it('maps is_game_related and is_mining as booleans', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ is_game_related: 1, is_mining: 0 }), + ]); + + await step.execute(CTX); + + const leafInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInsert[1][5]).toBe(true); // is_game_related = $6 + expect(leafInsert[1][6]).toBe(false); // is_mining = $7 + }); + + it('stores null type when UEX type is unknown', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory({ type: 'unknown_type' })]); + + await step.execute(CTX); + + const leafInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInsert[1][2]).toBeNull(); // type = $3 + }); + + it('emits warning for unknown type but still upserts the row', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory({ type: 'unknown_type' })]); + + await step.execute(CTX); + + expect(repoSave).toHaveBeenCalledWith( + expect.objectContaining({ + severity: 'warn', + message: expect.stringContaining("unknown type 'unknown_type'"), + }), + ); + const leafInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInsert).toBeDefined(); + }); + }); + + describe('attribute upsert', () => { + it('upserts attributes keyed by uex_id with category_uex_id set', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ + id: 3, + attributes: [ + makeAttribute({ id: 10 }), + makeAttribute({ id: 11, name: 'range' }), + ], + }), + ]); + + await step.execute(CTX); + + const attrInserts = dsQuery.mock.calls.filter(([sql]: [string]) => + sql.includes('INSERT INTO station_category_attribute'), + ); + expect(attrInserts).toHaveLength(2); + expect(attrInserts[0][1][0]).toBe(10); // uex_id = $1 + expect(attrInserts[0][1][1]).toBe(3); // category_uex_id = $2 + expect(attrInserts[1][1][0]).toBe(11); + }); + + it('upserts attributes ON CONFLICT (uex_id)', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ attributes: [makeAttribute()] }), + ]); + + await step.execute(CTX); + + const attrInsert = dsQuery.mock.calls.find(([sql]: [string]) => + sql.includes('INSERT INTO station_category_attribute'), + ); + expect(attrInsert[0]).toContain('ON CONFLICT (uex_id)'); + }); + + it('maps is_lower_better as boolean', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ attributes: [makeAttribute({ is_lower_better: 1 })] }), + ]); + + await step.execute(CTX); + + const attrInsert = dsQuery.mock.calls.find(([sql]: [string]) => + sql.includes('INSERT INTO station_category_attribute'), + ); + expect(attrInsert[1][4]).toBe(true); // is_lower_better = $5 + }); + + it('stores null for is_lower_better when not provided', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ + attributes: [makeAttribute({ is_lower_better: null })], + }), + ]); + + await step.execute(CTX); + + const attrInsert = dsQuery.mock.calls.find(([sql]: [string]) => + sql.includes('INSERT INTO station_category_attribute'), + ); + expect(attrInsert[1][4]).toBeNull(); + }); + + it('skips attribute with missing name and emits warning', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ attributes: [makeAttribute({ name: '' })] }), + ]); + + await step.execute(CTX); + + expect(repoSave).toHaveBeenCalledWith( + expect.objectContaining({ + severity: 'warn', + message: expect.stringContaining('attribute'), + }), + ); + const attrInsert = dsQuery.mock.calls.find(([sql]: [string]) => + sql.includes('INSERT INTO station_category_attribute'), + ); + expect(attrInsert).toBeUndefined(); + }); + }); + + describe('idempotency', () => { + it('produces same section upsert SQL on second run (deterministic conflict target)', async () => { + const dsQuery1 = buildDsQuery(1); + const dsQuery2 = buildDsQuery(1); + const step1 = buildStep(uexGet, dsQuery1, repoCreate, repoSave); + const step2 = buildStep( + jest.fn().mockResolvedValue([makeCategory()]), + dsQuery2, + repoCreate, + repoSave, + ); + uexGet.mockResolvedValue([makeCategory()]); + + await step1.execute(CTX); + const firstRunSectionSql = dsQuery1.mock.calls + .filter(([sql]: [string]) => sql.includes('RETURNING id')) + .map(([sql, params]: [string, unknown[]]) => ({ sql, params })); + + await step2.execute(CTX); + const secondRunSectionSql = dsQuery2.mock.calls + .filter(([sql]: [string]) => sql.includes('RETURNING id')) + .map(([sql, params]: [string, unknown[]]) => ({ sql, params })); + + expect(firstRunSectionSql).toEqual(secondRunSectionSql); + }); + + it('does not duplicate section rows for same section across multiple categories', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ id: 1, name: 'Rifles', section: 'Weapons' }), + makeCategory({ id: 2, name: 'Pistols', section: 'Weapons' }), + makeCategory({ id: 3, name: 'SMGs', section: 'Weapons' }), + ]); + + await step.execute(CTX); + + const sectionInserts = dsQuery.mock.calls.filter( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + sql.includes('RETURNING id'), + ); + expect(sectionInserts).toHaveLength(1); + }); + }); + + describe('warnings', () => { + it('skips category with missing name and emits warning', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory({ name: '' })]); + + await step.execute(CTX); + + expect(repoSave).toHaveBeenCalledWith( + expect.objectContaining({ + severity: 'warn', + message: 'Category missing name', + }), + ); + const leafInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInsert).toBeUndefined(); + }); + + it('processes valid categories even when some are skipped', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([ + makeCategory({ name: '' }), + makeCategory({ id: 2, name: 'Rifles' }), + ]); + + await step.execute(CTX); + + const leafInserts = dsQuery.mock.calls.filter( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInserts).toHaveLength(1); + expect(repoSave).toHaveBeenCalledTimes(1); + }); + + it('handles empty categories list without error', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([]); + + await step.execute(CTX); + + const inserts = dsQuery.mock.calls.filter(([sql]: [string]) => + sql.includes('INSERT INTO station_category'), + ); + expect(inserts).toHaveLength(0); + expect(repoSave).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/backend/src/modules/catalog-etl/steps/categories-sync.step.ts b/backend/src/modules/catalog-etl/steps/categories-sync.step.ts new file mode 100644 index 0000000..b5d8e52 --- /dev/null +++ b/backend/src/modules/catalog-etl/steps/categories-sync.step.ts @@ -0,0 +1,223 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, DataSource } from 'typeorm'; +import { InjectPinoLogger, PinoLogger } from 'nestjs-pino'; +import { EtlStep, EtlStepContext } from '../interfaces/etl-step.interface'; +import { EtlWarning } from '../entities/etl-warning.entity'; +import { UexApiClient } from '../../uex-sync/clients/uex-api.client'; + +interface UexCategoryAttribute { + id: number; + name: string; + description: string | null; + is_lower_better: number | null; + date_added: number | null; + date_modified: number | null; +} + +interface UexCategory { + id: number; + name: string; + code: string | null; + type: string | null; + section: string | null; + is_game_related: number; + is_mining: number; + date_added: number | null; + date_modified: number | null; + attributes?: UexCategoryAttribute[]; +} + +function toDate(unixTs: number | null | undefined): Date | null { + if (!unixTs) return null; + return new Date(unixTs * 1000); +} + +const VALID_TYPES = new Set(['item', 'service', 'contract']); + +function mapType(raw: string | null | undefined): string | null { + if (!raw) return null; + const lower = raw.toLowerCase(); + return VALID_TYPES.has(lower) ? lower : null; +} + +@Injectable() +export class CategoriesSyncStep implements EtlStep { + readonly name = 'categories-sync'; + + constructor( + private readonly uexApiClient: UexApiClient, + private readonly dataSource: DataSource, + @InjectRepository(EtlWarning) + private readonly warningsRepo: Repository, + @InjectPinoLogger(CategoriesSyncStep.name) + private readonly logger: PinoLogger, + ) {} + + async execute(ctx: EtlStepContext): Promise { + const categories = + await this.uexApiClient.get('/categories'); + this.logger.info( + { runId: ctx.runId, count: categories.length }, + 'Fetched categories from UEX', + ); + + // Step 1 — collect unique (type, section) pairs and upsert synthetic section rows. + // Section rows are uniquely identified by (type, name) WHERE is_section = TRUE. + // We RETURNING id so we can resolve parent_id for leaf rows. + const sectionKey = (type: string | null, section: string) => + `${type ?? ''}::${section}`; + + const sectionRows = new Map< + string, + { type: string | null; section: string } + >(); + for (const cat of categories) { + if (cat.section) { + const key = sectionKey(mapType(cat.type), cat.section); + if (!sectionRows.has(key)) { + sectionRows.set(key, { + type: mapType(cat.type), + section: cat.section, + }); + } + } + } + + // Upsert section rows and build a key → id map for parent FK resolution + const sectionIdByKey = new Map(); + for (const [key, { type, section }] of sectionRows) { + const rows = await this.dataSource.query<{ id: number }[]>( + `INSERT INTO station_category + (uex_id, parent_id, type, section, name, is_section, + is_game_related, is_mining, synced_at) + VALUES (NULL, NULL, $1, $2, $3, TRUE, FALSE, FALSE, NOW()) + ON CONFLICT (type, name) WHERE is_section = TRUE DO UPDATE SET + section=EXCLUDED.section, + synced_at=NOW() + RETURNING id`, + [type, section, section], + ); + sectionIdByKey.set(key, rows[0].id); + } + + // Step 2 — upsert leaf category rows with parent_id set to the section row + let upserted = 0; + let skipped = 0; + + for (const record of categories) { + if (!record.name) { + await this.warningsRepo.save( + this.warningsRepo.create({ + runId: ctx.runId, + stepName: this.name, + severity: 'warn', + message: 'Category missing name', + rawPayload: { id: record.id }, + }), + ); + skipped++; + continue; + } + + const type = mapType(record.type); + if (record.type && type === null) { + await this.warningsRepo.save( + this.warningsRepo.create({ + runId: ctx.runId, + stepName: this.name, + severity: 'warn', + message: `Category ${record.id} has unknown type '${record.type}' — stored as null`, + rawPayload: { id: record.id, raw_type: record.type }, + }), + ); + } + + const parentId = record.section + ? (sectionIdByKey.get(sectionKey(type, record.section)) ?? null) + : null; + + await this.dataSource.query( + `INSERT INTO station_category + (uex_id, parent_id, type, section, name, is_section, + is_game_related, is_mining, + uex_date_added, uex_date_modified, synced_at) + VALUES ($1,$2,$3,$4,$5,FALSE,$6,$7,$8,$9,NOW()) + ON CONFLICT (uex_id) WHERE uex_id IS NOT NULL DO UPDATE SET + parent_id=EXCLUDED.parent_id, + type=EXCLUDED.type, + section=EXCLUDED.section, + name=EXCLUDED.name, + is_game_related=EXCLUDED.is_game_related, + is_mining=EXCLUDED.is_mining, + uex_date_added=EXCLUDED.uex_date_added, + uex_date_modified=EXCLUDED.uex_date_modified, + synced_at=NOW()`, + [ + record.id, + parentId, + type, + record.section ?? null, + record.name, + Boolean(record.is_game_related), + Boolean(record.is_mining), + toDate(record.date_added), + toDate(record.date_modified), + ], + ); + upserted++; + + // Step 3 — upsert attributes for this category + for (const attr of record.attributes ?? []) { + if (!attr.name) { + await this.warningsRepo.save( + this.warningsRepo.create({ + runId: ctx.runId, + stepName: this.name, + severity: 'warn', + message: `Category ${record.id} attribute ${attr.id} missing name — skipped`, + rawPayload: { category_id: record.id, attribute_id: attr.id }, + }), + ); + continue; + } + + await this.dataSource.query( + `INSERT INTO station_category_attribute + (uex_id, category_uex_id, name, description, is_lower_better, + uex_date_added, uex_date_modified, synced_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,NOW()) + ON CONFLICT (uex_id) DO UPDATE SET + category_uex_id=EXCLUDED.category_uex_id, + name=EXCLUDED.name, + description=EXCLUDED.description, + is_lower_better=EXCLUDED.is_lower_better, + uex_date_added=EXCLUDED.uex_date_added, + uex_date_modified=EXCLUDED.uex_date_modified, + synced_at=NOW()`, + [ + attr.id, + record.id, + attr.name, + attr.description ?? null, + attr.is_lower_better !== null && attr.is_lower_better !== undefined + ? Boolean(attr.is_lower_better) + : null, + toDate(attr.date_added), + toDate(attr.date_modified), + ], + ); + } + } + + this.logger.info( + { + runId: ctx.runId, + sections: sectionRows.size, + upserted, + skipped, + }, + 'categories-sync step complete', + ); + } +} From 5a3fddfca8f529af257670e6c5de5af556ce0469 Mon Sep 17 00:00:00 2001 From: gitaddremote Date: Mon, 25 May 2026 17:34:57 -0400 Subject: [PATCH 2/3] fix: normalize section to null and use COALESCE index for section upserts Two correctness bugs in the categories-sync step: - NULL type in ON CONFLICT: Postgres unique indexes treat NULLs as distinct, so ON CONFLICT (type, name) WHERE is_section = TRUE would insert a new section row on every run when type is NULL. Changed the index and ON CONFLICT clause to use COALESCE(type, '') so NULL types are treated as a single deterministic sentinel value. - Empty-string section: cat.section truthiness check correctly skips '' for section-row collection, but record.section ?? null passed '' into the leaf INSERT. Normalize once via section?.trim() || null and use that value for section-row collection, parent lookup, and leaf insert. Updated spec: renamed conflict-target test to assert the COALESCE form; added tests for empty-string and whitespace-only section normalization. --- .../1748000000000-BigBangBaselineMigration.ts | 2 +- .../steps/categories-sync.step.spec.ts | 42 ++++++++++++++++++- .../catalog-etl/steps/categories-sync.step.ts | 19 ++++----- 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/backend/src/migrations/1748000000000-BigBangBaselineMigration.ts b/backend/src/migrations/1748000000000-BigBangBaselineMigration.ts index d24d34d..ea24874 100644 --- a/backend/src/migrations/1748000000000-BigBangBaselineMigration.ts +++ b/backend/src/migrations/1748000000000-BigBangBaselineMigration.ts @@ -1421,7 +1421,7 @@ export class BigBangBaselineMigration1748000000000 `CREATE UNIQUE INDEX "uq_categories_uex_id" ON "station_category" ("uex_id") WHERE "uex_id" IS NOT NULL`, ); await queryRunner.query( - `CREATE UNIQUE INDEX "uq_categories_section_type" ON "station_category" ("type", "name") WHERE "is_section" = TRUE`, + `CREATE UNIQUE INDEX "uq_categories_section_type" ON "station_category" (COALESCE("type", ''), "name") WHERE "is_section" = TRUE`, ); await queryRunner.query( `CREATE INDEX "idx_categories_parent" ON "station_category" ("parent_id")`, diff --git a/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts b/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts index 92eed3b..f361a6b 100644 --- a/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts +++ b/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts @@ -126,7 +126,7 @@ describe('CategoriesSyncStep', () => { expect(sectionInsert[1][2]).toBe('Weapons'); // name = section label }); - it('section rows conflict on (type, name) WHERE is_section = TRUE', async () => { + it("section rows conflict on COALESCE(type, '') to handle NULL type idempotently", async () => { const dsQuery = buildDsQuery(); const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); uexGet.mockResolvedValue([makeCategory()]); @@ -139,9 +139,47 @@ describe('CategoriesSyncStep', () => { sql.includes('RETURNING id'), ); expect(sectionInsert[0]).toContain( - 'ON CONFLICT (type, name) WHERE is_section = TRUE', + "ON CONFLICT (COALESCE(type, ''), name) WHERE is_section = TRUE", ); }); + + it('normalizes empty-string section to null — no section row created, leaf parent_id is null', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory({ section: '' })]); + + await step.execute(CTX); + + const sectionInserts = dsQuery.mock.calls.filter( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + sql.includes('RETURNING id'), + ); + expect(sectionInserts).toHaveLength(0); + + const leafInsert = dsQuery.mock.calls.find( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + !sql.includes('RETURNING id'), + ); + expect(leafInsert[1][1]).toBeNull(); // parent_id = $2 + expect(leafInsert[1][3]).toBeNull(); // section = $4 + }); + + it('normalizes whitespace-only section to null', async () => { + const dsQuery = buildDsQuery(); + const step = buildStep(uexGet, dsQuery, repoCreate, repoSave); + uexGet.mockResolvedValue([makeCategory({ section: ' ' })]); + + await step.execute(CTX); + + const sectionInserts = dsQuery.mock.calls.filter( + ([sql]: [string]) => + sql.includes('INSERT INTO station_category') && + sql.includes('RETURNING id'), + ); + expect(sectionInserts).toHaveLength(0); + }); }); describe('leaf row parent FK', () => { diff --git a/backend/src/modules/catalog-etl/steps/categories-sync.step.ts b/backend/src/modules/catalog-etl/steps/categories-sync.step.ts index b5d8e52..622d8a0 100644 --- a/backend/src/modules/catalog-etl/steps/categories-sync.step.ts +++ b/backend/src/modules/catalog-etl/steps/categories-sync.step.ts @@ -73,13 +73,11 @@ export class CategoriesSyncStep implements EtlStep { { type: string | null; section: string } >(); for (const cat of categories) { - if (cat.section) { - const key = sectionKey(mapType(cat.type), cat.section); + const section = cat.section?.trim() || null; + if (section) { + const key = sectionKey(mapType(cat.type), section); if (!sectionRows.has(key)) { - sectionRows.set(key, { - type: mapType(cat.type), - section: cat.section, - }); + sectionRows.set(key, { type: mapType(cat.type), section }); } } } @@ -92,7 +90,7 @@ export class CategoriesSyncStep implements EtlStep { (uex_id, parent_id, type, section, name, is_section, is_game_related, is_mining, synced_at) VALUES (NULL, NULL, $1, $2, $3, TRUE, FALSE, FALSE, NOW()) - ON CONFLICT (type, name) WHERE is_section = TRUE DO UPDATE SET + ON CONFLICT (COALESCE(type, ''), name) WHERE is_section = TRUE DO UPDATE SET section=EXCLUDED.section, synced_at=NOW() RETURNING id`, @@ -133,8 +131,9 @@ export class CategoriesSyncStep implements EtlStep { ); } - const parentId = record.section - ? (sectionIdByKey.get(sectionKey(type, record.section)) ?? null) + const section = record.section?.trim() || null; + const parentId = section + ? (sectionIdByKey.get(sectionKey(type, section)) ?? null) : null; await this.dataSource.query( @@ -157,7 +156,7 @@ export class CategoriesSyncStep implements EtlStep { record.id, parentId, type, - record.section ?? null, + section, record.name, Boolean(record.is_game_related), Boolean(record.is_mining), From a57be308eae0b93119aed324174e54f3d80ce589 Mon Sep 17 00:00:00 2001 From: gitaddremote Date: Mon, 25 May 2026 23:01:48 -0400 Subject: [PATCH 3/3] fix: use expression ON CONFLICT target and forward migration for categories section index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wrap COALESCE expression in extra parens in ON CONFLICT clause: (COALESCE(type, ''), name) → ((COALESCE(type, '')), name) so Postgres matches it against the expression index rather than treating it as a column list (which would fail with no matching unique constraint) - Update spec assertion to match corrected ON CONFLICT form - Add migration 1779700000000 to drop/recreate uq_categories_section_type on already-applied databases: drops the old plain-column index ("type", "name") and creates the expression index (COALESCE("type", ''), "name") so existing environments stay in sync without re-running the baseline --- ...FixCategoriesSectionTypeExpressionIndex.ts | 25 +++++++++++++++++++ .../steps/categories-sync.step.spec.ts | 2 +- .../catalog-etl/steps/categories-sync.step.ts | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 backend/src/migrations/1779700000000-FixCategoriesSectionTypeExpressionIndex.ts diff --git a/backend/src/migrations/1779700000000-FixCategoriesSectionTypeExpressionIndex.ts b/backend/src/migrations/1779700000000-FixCategoriesSectionTypeExpressionIndex.ts new file mode 100644 index 0000000..18c430b --- /dev/null +++ b/backend/src/migrations/1779700000000-FixCategoriesSectionTypeExpressionIndex.ts @@ -0,0 +1,25 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class FixCategoriesSectionTypeExpressionIndex1779700000000 + implements MigrationInterface +{ + name = 'FixCategoriesSectionTypeExpressionIndex1779700000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX IF EXISTS "uq_categories_section_type"`, + ); + await queryRunner.query( + `CREATE UNIQUE INDEX "uq_categories_section_type" ON "station_category" (COALESCE("type", ''), "name") WHERE "is_section" = TRUE`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX IF EXISTS "uq_categories_section_type"`, + ); + await queryRunner.query( + `CREATE UNIQUE INDEX "uq_categories_section_type" ON "station_category" ("type", "name") WHERE "is_section" = TRUE`, + ); + } +} diff --git a/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts b/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts index f361a6b..dc26345 100644 --- a/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts +++ b/backend/src/modules/catalog-etl/steps/categories-sync.step.spec.ts @@ -139,7 +139,7 @@ describe('CategoriesSyncStep', () => { sql.includes('RETURNING id'), ); expect(sectionInsert[0]).toContain( - "ON CONFLICT (COALESCE(type, ''), name) WHERE is_section = TRUE", + "ON CONFLICT ((COALESCE(type, '')), name) WHERE is_section = TRUE", ); }); diff --git a/backend/src/modules/catalog-etl/steps/categories-sync.step.ts b/backend/src/modules/catalog-etl/steps/categories-sync.step.ts index 622d8a0..4f41717 100644 --- a/backend/src/modules/catalog-etl/steps/categories-sync.step.ts +++ b/backend/src/modules/catalog-etl/steps/categories-sync.step.ts @@ -90,7 +90,7 @@ export class CategoriesSyncStep implements EtlStep { (uex_id, parent_id, type, section, name, is_section, is_game_related, is_mining, synced_at) VALUES (NULL, NULL, $1, $2, $3, TRUE, FALSE, FALSE, NOW()) - ON CONFLICT (COALESCE(type, ''), name) WHERE is_section = TRUE DO UPDATE SET + ON CONFLICT ((COALESCE(type, '')), name) WHERE is_section = TRUE DO UPDATE SET section=EXCLUDED.section, synced_at=NOW() RETURNING id`,