From fb5de7918b70bb67ac3a167e3764232544444c58 Mon Sep 17 00:00:00 2001 From: burgerni10 Date: Wed, 5 Jun 2024 16:27:20 +0200 Subject: [PATCH] fix(scan-mode): allow creation of connectors with scan mode name --- .../src/repository/south-item.repository.ts | 6 +- backend/src/service/reload.service.ts | 8 +- .../controllers/history-query.controller.ts | 7 +- .../north-connector.controller.spec.ts | 68 ++++++++++++++ .../controllers/north-connector.controller.ts | 16 ++++ .../south-connector.controller.spec.ts | 91 ++++++++++++++++++- .../controllers/south-connector.controller.ts | 33 ++++++- backend/src/web-server/routes/index.ts | 1 + .../history-query-items.component.ts | 9 +- .../south-items/south-items.component.ts | 9 +- shared/model/north-connector.model.ts | 15 ++- shared/model/south-connector.model.ts | 5 +- 12 files changed, 249 insertions(+), 19 deletions(-) diff --git a/backend/src/repository/south-item.repository.ts b/backend/src/repository/south-item.repository.ts index c7543285aa..d005573a55 100644 --- a/backend/src/repository/south-item.repository.ts +++ b/backend/src/repository/south-item.repository.ts @@ -167,7 +167,11 @@ export default class SouthItemRepository { this.database.prepare(query).run(southId); } - createAndUpdateSouthItems(southId: string, itemsToAdd: Array, itemsToUpdate: Array): void { + createAndUpdateSouthItems( + southId: string, + itemsToAdd: Array, + itemsToUpdate: Array + ): void { const insert = this.database.prepare( `INSERT INTO ${SOUTH_ITEMS_TABLE} (id, name, enabled, connector_id, scan_mode_id, settings) VALUES (?, ?, ?, ?, ?, ?);` ); diff --git a/backend/src/service/reload.service.ts b/backend/src/service/reload.service.ts index f5f54ab042..d3334186df 100644 --- a/backend/src/service/reload.service.ts +++ b/backend/src/service/reload.service.ts @@ -190,8 +190,8 @@ export default class ReloadService { async onCreateOrUpdateSouthItems( southConnector: SouthConnectorDTO, - itemsToAdd: Array, - itemsToUpdate: Array, + itemsToAdd: Array, + itemsToUpdate: Array, restart = true ): Promise { await this.oibusEngine.stopSouth(southConnector.id); @@ -425,10 +425,10 @@ export default class ReloadService { /** * Handle the change of a south item's scan mode */ - private onSouthItemScanModeChange(southId: string, previousItem: SouthConnectorItemDTO, newItem: SouthConnectorItemDTO) { + private onSouthItemScanModeChange(southId: string, previousItem: SouthConnectorItemDTO, newItem: SouthConnectorItemCommandDTO) { const settings = this.repositoryService.southConnectorRepository.getSouthConnector(southId)!; const oldScanModeId = previousItem.scanModeId; - const newScanModeId = newItem.scanModeId; + const newScanModeId = newItem.scanModeId!; if (oldScanModeId === newScanModeId) { return; diff --git a/backend/src/web-server/controllers/history-query.controller.ts b/backend/src/web-server/controllers/history-query.controller.ts index ead12292e7..f1dde77ebb 100644 --- a/backend/src/web-server/controllers/history-query.controller.ts +++ b/backend/src/web-server/controllers/history-query.controller.ts @@ -14,7 +14,7 @@ import csv from 'papaparse'; import fs from 'node:fs/promises'; import AbstractController from './abstract.controller'; import Joi from 'joi'; -import { NorthConnectorCommandDTO, NorthConnectorDTO } from '../../../../shared/model/north-connector.model'; +import { NorthCacheSettingsDTO, NorthConnectorCommandDTO, NorthConnectorDTO } from '../../../../shared/model/north-connector.model'; interface HistoryQueryWithItemsCommandDTO { historyQuery: HistoryQueryCommandDTO; @@ -555,9 +555,14 @@ export default class HistoryQueryController extends AbstractController { } await this.validator.validateSettings(manifest.settings, ctx.request.body!.settings); + const northCaching = { ...ctx.request.body!.caching }; + delete northCaching.scanModeName; + northCaching.scanModeId = ''; + const command: NorthConnectorDTO = { id: 'test', ...ctx.request.body!, + caching: northCaching as NorthCacheSettingsDTO, name: `${ctx.request.body!.type}:test-connection` }; command.settings = await ctx.app.encryptionService.encryptConnectorSecrets(command.settings, northSettings, manifest.settings); diff --git a/backend/src/web-server/controllers/north-connector.controller.spec.ts b/backend/src/web-server/controllers/north-connector.controller.spec.ts index 6266f8a5d7..6ed0cfb519 100644 --- a/backend/src/web-server/controllers/north-connector.controller.spec.ts +++ b/backend/src/web-server/controllers/north-connector.controller.spec.ts @@ -2,6 +2,7 @@ import NorthConnectorController from './north-connector.controller'; import KoaContextMock from '../../tests/__mocks__/koa-context.mock'; import JoiValidator from './validators/joi.validator'; import { northTestManifest } from '../../tests/__mocks__/north-service.mock'; +import { ScanModeDTO } from '../../../../shared/model/scan-mode.model'; jest.mock('./validators/joi.validator'); @@ -163,6 +164,73 @@ describe('North connector controller', () => { expect(ctx.created).toHaveBeenCalledWith(northConnector); }); + it('createNorthConnector() should throw an error if scan mode not specifiec', async () => { + ctx.request.body = { + north: JSON.parse(JSON.stringify(northConnectorCommand)), + subscriptions: [ + { type: 'south', subscription: { id: 'id1' } }, + { type: 'external-source', externalSubscription: { id: 'id2' } } + ] + }; + delete ctx.request.body.north.caching.scanModeId; + ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(northConnectorCommand.settings); + ctx.app.reloadService.onCreateNorth.mockReturnValue(northConnector); + + await northConnectorController.createNorthConnector(ctx); + + expect(ctx.badRequest).toHaveBeenCalledWith('Scan mode not specified'); + }); + + it('createNorthConnector() should throw an error if scan mode not found', async () => { + ctx.request.body = { + north: JSON.parse(JSON.stringify(northConnectorCommand)), + subscriptions: [ + { type: 'south', subscription: { id: 'id1' } }, + { type: 'external-source', externalSubscription: { id: 'id2' } } + ] + }; + delete ctx.request.body.north.caching.scanModeId; + ctx.request.body.north.caching.scanModeName = 'invalid'; + const scanMode: ScanModeDTO = { + id: '1', + name: 'scan mode', + description: 'description', + cron: '* * * * *' + }; + ctx.app.repositoryService.scanModeRepository.getScanModes.mockReturnValue([scanMode]); + ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(northConnectorCommand.settings); + ctx.app.reloadService.onCreateNorth.mockReturnValue(northConnector); + + await northConnectorController.createNorthConnector(ctx); + + expect(ctx.badRequest).toHaveBeenCalledWith('Scan mode invalid not found'); + }); + + it('createNorthConnector() should create North connector with found scan mode', async () => { + ctx.request.body = { + north: JSON.parse(JSON.stringify(northConnectorCommand)), + subscriptions: [ + { type: 'south', subscription: { id: 'id1' } }, + { type: 'external-source', externalSubscription: { id: 'id2' } } + ] + }; + delete ctx.request.body.north.caching.scanModeId; + ctx.request.body.north.caching.scanModeName = 'scan mode'; + const scanMode: ScanModeDTO = { + id: '1', + name: 'scan mode', + description: 'description', + cron: '* * * * *' + }; + ctx.app.repositoryService.scanModeRepository.getScanModes.mockReturnValue([scanMode]); + ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(northConnectorCommand.settings); + ctx.app.reloadService.onCreateNorth.mockReturnValue(northConnector); + + await northConnectorController.createNorthConnector(ctx); + + expect(ctx.created).toHaveBeenCalledWith(northConnector); + }); + it('createNorthConnector() should create North connector and not start it', async () => { ctx.request.body = { north: { ...northConnectorCommand, enabled: false }, diff --git a/backend/src/web-server/controllers/north-connector.controller.ts b/backend/src/web-server/controllers/north-connector.controller.ts index 57ccbdd829..33dc1f7a84 100644 --- a/backend/src/web-server/controllers/north-connector.controller.ts +++ b/backend/src/web-server/controllers/north-connector.controller.ts @@ -1,5 +1,6 @@ import { KoaContext } from '../koa'; import { + NorthCacheSettingsDTO, NorthConnectorCommandDTO, NorthConnectorDTO, NorthConnectorWithItemsCommandDTO, @@ -73,6 +74,17 @@ export default class NorthConnectorController { await this.validator.validateSettings(manifest.settings, command.settings); + if (!command.caching.scanModeId && !command.caching.scanModeName) { + throw new Error(`Scan mode not specified`); + } else if (!command.caching.scanModeId && command.caching.scanModeName) { + const scanModes = ctx.app.repositoryService.scanModeRepository.getScanModes(); + const scanMode = scanModes.find(element => element.name === command.caching.scanModeName); + if (!scanMode) { + throw new Error(`Scan mode ${command.caching.scanModeName} not found`); + } + command.caching.scanModeId = scanMode.id; + } + let duplicatedConnector: NorthConnectorDTO | null = null; if (ctx.query.duplicateId) { duplicatedConnector = ctx.app.repositoryService.northConnectorRepository.getNorthConnector(ctx.query.duplicateId); @@ -544,9 +556,13 @@ export default class NorthConnectorController { } await this.validator.validateSettings(manifest.settings, ctx.request.body!.settings); + const northCaching = { ...ctx.request.body!.caching }; + delete northCaching.scanModeName; + northCaching.scanModeId = ''; const command: NorthConnectorDTO = { id: northConnector?.id || 'test', ...ctx.request.body!, + caching: northCaching as NorthCacheSettingsDTO, name: northConnector?.name || `${ctx.request.body!.type}:test-connection` }; command.settings = await ctx.app.encryptionService.encryptConnectorSecrets( diff --git a/backend/src/web-server/controllers/south-connector.controller.spec.ts b/backend/src/web-server/controllers/south-connector.controller.spec.ts index acbef74dfe..eb5b57ca9b 100644 --- a/backend/src/web-server/controllers/south-connector.controller.spec.ts +++ b/backend/src/web-server/controllers/south-connector.controller.spec.ts @@ -10,6 +10,7 @@ import { SouthConnectorItemDTO } from '../../../../shared/model/south-connector.model'; import { southTestManifest } from '../../tests/__mocks__/south-service.mock'; +import { ScanModeDTO } from '../../../../shared/model/scan-mode.model'; jest.mock('./validators/joi.validator'); jest.mock('papaparse'); @@ -68,7 +69,12 @@ const itemCommand: SouthConnectorItemCommandDTO = { const item: SouthConnectorItemDTO = { id: 'id', connectorId: 'connectorId', - ...itemCommand + enabled: true, + name: 'name', + settings: { + regex: '.*' + }, + scanModeId: 'scanModeId' }; const page = { content: [item], @@ -226,10 +232,91 @@ describe('South connector controller', () => { ctx.query.duplicateId = null; }); + it('createSouthConnector() should throw error when scan mode is not specified in item', async () => { + ctx.request.body = { + south: southConnectorCommand, + items: [ + { + name: 'name', + enabled: true, + settings: { + regex: '.*' + } + } + ] + }; + ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(sqliteConnectorCommand.settings); + ctx.app.reloadService.onCreateSouth.mockReturnValue(southConnector); + ctx.app.southService.getInstalledSouthManifests.mockReturnValue([southTestManifest]); + + await southConnectorController.createSouthConnector(ctx); + expect(ctx.badRequest).toHaveBeenCalledWith('Scan mode not specified for item name'); + }); + + it('createSouthConnector() should throw error when scan mode is not found', async () => { + ctx.request.body = { + south: southConnectorCommand, + items: [ + { + name: 'name', + scanModeName: 'invalid', + enabled: true, + settings: { + regex: '.*' + } + } + ] + }; + const scanMode: ScanModeDTO = { + id: '1', + name: 'scan mode', + description: 'description', + cron: '* * * * *' + }; + ctx.app.repositoryService.scanModeRepository.getScanModes.mockReturnValue([scanMode]); + + ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(sqliteConnectorCommand.settings); + ctx.app.reloadService.onCreateSouth.mockReturnValue(southConnector); + ctx.app.southService.getInstalledSouthManifests.mockReturnValue([southTestManifest]); + + await southConnectorController.createSouthConnector(ctx); + expect(ctx.badRequest).toHaveBeenCalledWith('Scan mode invalid not found for item name'); + }); + + it('createSouthConnector() should create connector with items', async () => { + ctx.request.body = { + south: southConnectorCommand, + items: [ + { + name: 'name', + scanModeName: 'scan mode', + enabled: true, + settings: { + regex: '.*' + } + } + ] + }; + const scanMode: ScanModeDTO = { + id: '1', + name: 'scan mode', + description: 'description', + cron: '* * * * *' + }; + ctx.app.repositoryService.scanModeRepository.getScanModes.mockReturnValue([scanMode]); + ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(sqliteConnectorCommand.settings); + ctx.app.reloadService.onCreateSouth.mockReturnValue(southConnector); + ctx.app.southService.getInstalledSouthManifests.mockReturnValue([southTestManifest]); + + await southConnectorController.createSouthConnector(ctx); + expect(ctx.app.reloadService.onCreateSouth).toHaveBeenCalledWith(sqliteConnectorCommand); + expect(ctx.created).toHaveBeenCalledWith(southConnector); + }); + it('createSouthConnector() should create South connector with forceMaxInstantPerItem', async () => { ctx.request.body = { south: sqliteConnectorCommand, - items: [{}] + items: [itemCommand] }; ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(sqliteConnectorCommand.settings); ctx.app.reloadService.onCreateSouth.mockReturnValue(southConnector); diff --git a/backend/src/web-server/controllers/south-connector.controller.ts b/backend/src/web-server/controllers/south-connector.controller.ts index 7cdc649b9a..542bb0aa93 100644 --- a/backend/src/web-server/controllers/south-connector.controller.ts +++ b/backend/src/web-server/controllers/south-connector.controller.ts @@ -95,7 +95,14 @@ export default class SouthConnectorController { manifest.settings ); ctx.request.body!.name = southConnector ? southConnector.name : `${ctx.request.body!.type}:test-connection`; - const logger = ctx.app.logger.child({ scopeType: 'south', scopeId: command.id, scopeName: command.name }, { level: 'silent' }); + const logger = ctx.app.logger.child( + { + scopeType: 'south', + scopeId: command.id, + scopeName: command.name + }, + { level: 'silent' } + ); const southToTest = ctx.app.southService.createSouth(command, [], this.addValues, this.addFile, 'baseFolder', logger); await southToTest.testConnection(); @@ -118,10 +125,21 @@ export default class SouthConnectorController { return ctx.throw(404, 'South manifest not found'); } + const scanModes = ctx.app.repositoryService.scanModeRepository.getScanModes(); + await this.validator.validateSettings(manifest.settings, command.settings); // Check if item settings match the item schema, throw an error otherwise for (const item of ctx.request.body!.items) { await this.validator.validateSettings(manifest.items.settings, item.settings); + if (!item.scanModeId && !item.scanModeName) { + throw new Error(`Scan mode not specified for item ${item.name}`); + } else if (!item.scanModeId && item.scanModeName) { + const scanMode = scanModes.find(element => element.name === item.scanModeName); + if (!scanMode) { + throw new Error(`Scan mode ${item.scanModeName} not found for item ${item.name}`); + } + item.scanModeId = scanMode.id; + } } if (manifest.modes.forceMaxInstantPerItem) { @@ -140,6 +158,7 @@ export default class SouthConnectorController { manifest.settings ); const southConnector = await ctx.app.reloadService.onCreateSouth(command); + await ctx.app.reloadService.onCreateOrUpdateSouthItems(southConnector, ctx.request.body!.items, []); ctx.created(southConnector); @@ -203,7 +222,7 @@ export default class SouthConnectorController { } } - startSouthConnector = async (ctx: KoaContext) => { + async startSouthConnector(ctx: KoaContext): Promise { const southConnector = ctx.app.repositoryService.southConnectorRepository.getSouthConnector(ctx.params.id); if (!southConnector) { return ctx.notFound(); @@ -215,9 +234,9 @@ export default class SouthConnectorController { } catch (error: any) { ctx.badRequest(error.message); } - }; + } - stopSouthConnector = async (ctx: KoaContext) => { + async stopSouthConnector(ctx: KoaContext): Promise { const southConnector = ctx.app.repositoryService.southConnectorRepository.getSouthConnector(ctx.params.id); if (!southConnector) { return ctx.notFound(); @@ -229,7 +248,7 @@ export default class SouthConnectorController { } catch (error: any) { ctx.badRequest(error.message); } - }; + } async resetSouthMetrics(ctx: KoaContext): Promise { const southConnector = ctx.app.repositoryService.southConnectorRepository.getSouthConnector(ctx.params.southId); @@ -255,6 +274,10 @@ export default class SouthConnectorController { ctx.ok(southItems); } + /** + * Endpoint used to download a CSV from a list of items when creating a connector (before the items are saved on + * the database). When the items are already saved, it is downloaded with the export method + */ async southItemsToCsv(ctx: KoaContext<{ items: Array }, any>): Promise { const scanModes = ctx.app.repositoryService.scanModeRepository.getScanModes(); const southItems = ctx.request.body!.items.map(item => { diff --git a/backend/src/web-server/routes/index.ts b/backend/src/web-server/routes/index.ts index bbbde91383..666c39cb94 100644 --- a/backend/src/web-server/routes/index.ts +++ b/backend/src/web-server/routes/index.ts @@ -200,6 +200,7 @@ router.post('/api/south/:southType/items/check-import/:southId', upload.single(' ); router.post('/api/south/:southId/items/import', (ctx: KoaContext) => southConnectorController.importSouthItems(ctx)); router.get('/api/south/:southId/items/export', (ctx: KoaContext) => southConnectorController.exportSouthItems(ctx)); +router.get('/api/south/:southId/items/export', (ctx: KoaContext) => southConnectorController.exportSouthItems(ctx)); router.put('/api/south/items/to-csv', (ctx: KoaContext) => southConnectorController.southItemsToCsv(ctx)); router.get('/api/south/:southId/items/:id', (ctx: KoaContext) => southConnectorController.getSouthItem(ctx)); router.put('/api/south/:southId/items/:id', (ctx: KoaContext) => southConnectorController.updateSouthItem(ctx)); diff --git a/frontend/src/app/history-query/history-query-items/history-query-items.component.ts b/frontend/src/app/history-query/history-query-items/history-query-items.component.ts index 603e46dc9f..f533d4c688 100644 --- a/frontend/src/app/history-query/history-query-items/history-query-items.component.ts +++ b/frontend/src/app/history-query/history-query-items/history-query-items.component.ts @@ -150,7 +150,14 @@ export class HistoryQueryItemsComponent implements OnInit { if (!this.inMemory) { return this.historyQueryService.createItem(this.historyQuery!.id, command); } else { - this.allItems.push({ id: command.id ?? '', connectorId: this.historyQuery?.id ?? '', ...command }); + this.allItems.push({ + id: command.id ?? '', + name: command.name, + enabled: command.enabled, + connectorId: this.historyQuery?.id ?? '', + scanModeId: command.scanModeId!, + settings: { ...command.settings } + }); return of(null); } }) diff --git a/frontend/src/app/south/south-items/south-items.component.ts b/frontend/src/app/south/south-items/south-items.component.ts index e84896d98d..ce62d788ed 100644 --- a/frontend/src/app/south/south-items/south-items.component.ts +++ b/frontend/src/app/south/south-items/south-items.component.ts @@ -151,7 +151,14 @@ export class SouthItemsComponent implements OnInit { if (!this.inMemory) { return this.southConnectorService.createItem(this.southConnector!.id, command); } else { - this.allItems.push({ id: command.id ?? '', connectorId: this.southConnector?.id ?? '', ...command }); + this.allItems.push({ + id: command.id ?? '', + name: command.name, + enabled: command.enabled, + connectorId: this.southConnector?.id ?? '', + scanModeId: command.scanModeId!, + settings: { ...command.settings } + }); return of(null); } }) diff --git a/shared/model/north-connector.model.ts b/shared/model/north-connector.model.ts index e956a443cf..9933ffee9d 100644 --- a/shared/model/north-connector.model.ts +++ b/shared/model/north-connector.model.ts @@ -13,6 +13,17 @@ export interface NorthCacheSettingsDTO { maxSize: number; } +export interface NorthCacheSettingsCommandDTO { + scanModeId?: string; + scanModeName?: string; + retryInterval: number; + retryCount: number; + groupCount: number; + maxSendCount: number; + sendFileImmediately: boolean; + maxSize: number; +} + export interface NorthArchiveSettings { enabled: boolean; retentionDuration: number; @@ -51,7 +62,7 @@ export interface NorthConnectorCommandDTO { description: string; enabled: boolean; settings: T; - caching: NorthCacheSettingsDTO; + caching: NorthCacheSettingsCommandDTO; archive: NorthArchiveSettings; } @@ -59,7 +70,7 @@ export interface NorthConnectorCommandDTO { * Command DTO for South connector */ export interface NorthConnectorWithItemsCommandDTO<> { - north: NorthConnectorDTO; + north: NorthConnectorCommandDTO; subscriptions: Array; subscriptionsToDelete: Array; } diff --git a/shared/model/south-connector.model.ts b/shared/model/south-connector.model.ts index efa409663f..6d2816b86f 100644 --- a/shared/model/south-connector.model.ts +++ b/shared/model/south-connector.model.ts @@ -51,7 +51,7 @@ export interface SouthConnectorCommandDTO { */ export interface SouthConnectorWithItemsCommandDTO<> { south: SouthConnectorDTO; - items: Array; + items: Array; itemIdsToDelete: Array; } @@ -74,7 +74,8 @@ export interface SouthConnectorItemCommandDTO enabled: boolean; name: string; settings: T; - scanModeId: string; + scanModeId?: string; + scanModeName?: string; } export interface SouthConnectorItemSearchParam {