fix(scan-mode): allow creation of connectors with scan mode name

OptimistikSAS · Jun 5, 2024 · fb5de79 · fb5de79
1 parent ed7f7ab
commit fb5de79
Show file tree

Hide file tree

Showing 12 changed files with 249 additions and 19 deletions.
diff --git a/backend/src/repository/south-item.repository.ts b/backend/src/repository/south-item.repository.ts
@@ -167,7 +167,11 @@ export default class SouthItemRepository {
     this.database.prepare(query).run(southId);
   }
 
-  createAndUpdateSouthItems(southId: string, itemsToAdd: Array<SouthConnectorItemDTO>, itemsToUpdate: Array<SouthConnectorItemDTO>): void {
+  createAndUpdateSouthItems(
+    southId: string,
+    itemsToAdd: Array<SouthConnectorItemCommandDTO>,
+    itemsToUpdate: Array<SouthConnectorItemCommandDTO>
+  ): void {
     const insert = this.database.prepare(
       `INSERT INTO ${SOUTH_ITEMS_TABLE} (id, name, enabled, connector_id, scan_mode_id, settings) VALUES (?, ?, ?, ?, ?, ?);`
     );

diff --git a/backend/src/service/reload.service.ts b/backend/src/service/reload.service.ts
@@ -190,8 +190,8 @@ export default class ReloadService {
 
   async onCreateOrUpdateSouthItems(
     southConnector: SouthConnectorDTO,
-    itemsToAdd: Array<SouthConnectorItemDTO>,
-    itemsToUpdate: Array<SouthConnectorItemDTO>,
+    itemsToAdd: Array<SouthConnectorItemCommandDTO>,
+    itemsToUpdate: Array<SouthConnectorItemCommandDTO>,
     restart = true
   ): Promise<void> {
     await this.oibusEngine.stopSouth(southConnector.id);
@@ -425,10 +425,10 @@ export default class ReloadService {
   /**
    * Handle the change of a south item's scan mode
    */
-  private onSouthItemScanModeChange(southId: string, previousItem: SouthConnectorItemDTO, newItem: SouthConnectorItemDTO) {
+  private onSouthItemScanModeChange(southId: string, previousItem: SouthConnectorItemDTO, newItem: SouthConnectorItemCommandDTO) {
     const settings = this.repositoryService.southConnectorRepository.getSouthConnector(southId)!;
     const oldScanModeId = previousItem.scanModeId;
-    const newScanModeId = newItem.scanModeId;
+    const newScanModeId = newItem.scanModeId!;
 
     if (oldScanModeId === newScanModeId) {
       return;

diff --git a/backend/src/web-server/controllers/history-query.controller.ts b/backend/src/web-server/controllers/history-query.controller.ts
@@ -14,7 +14,7 @@ import csv from 'papaparse';
 import fs from 'node:fs/promises';
 import AbstractController from './abstract.controller';
 import Joi from 'joi';
-import { NorthConnectorCommandDTO, NorthConnectorDTO } from '../../../../shared/model/north-connector.model';
+import { NorthCacheSettingsDTO, NorthConnectorCommandDTO, NorthConnectorDTO } from '../../../../shared/model/north-connector.model';
 
 interface HistoryQueryWithItemsCommandDTO {
   historyQuery: HistoryQueryCommandDTO;
@@ -555,9 +555,14 @@ export default class HistoryQueryController extends AbstractController {
       }
       await this.validator.validateSettings(manifest.settings, ctx.request.body!.settings);
 
+      const northCaching = { ...ctx.request.body!.caching };
+      delete northCaching.scanModeName;
+      northCaching.scanModeId = '';
+
       const command: NorthConnectorDTO = {
         id: 'test',
         ...ctx.request.body!,
+        caching: northCaching as NorthCacheSettingsDTO,
         name: `${ctx.request.body!.type}:test-connection`
       };
       command.settings = await ctx.app.encryptionService.encryptConnectorSecrets(command.settings, northSettings, manifest.settings);

diff --git a/backend/src/web-server/controllers/north-connector.controller.spec.ts b/backend/src/web-server/controllers/north-connector.controller.spec.ts
@@ -2,6 +2,7 @@ import NorthConnectorController from './north-connector.controller';
 import KoaContextMock from '../../tests/__mocks__/koa-context.mock';
 import JoiValidator from './validators/joi.validator';
 import { northTestManifest } from '../../tests/__mocks__/north-service.mock';
+import { ScanModeDTO } from '../../../../shared/model/scan-mode.model';
 
 jest.mock('./validators/joi.validator');
 
@@ -163,6 +164,73 @@ describe('North connector controller', () => {
     expect(ctx.created).toHaveBeenCalledWith(northConnector);
   });
 
+  it('createNorthConnector() should throw an error if scan mode not specifiec', async () => {
+    ctx.request.body = {
+      north: JSON.parse(JSON.stringify(northConnectorCommand)),
+      subscriptions: [
+        { type: 'south', subscription: { id: 'id1' } },
+        { type: 'external-source', externalSubscription: { id: 'id2' } }
+      ]
+    };
+    delete ctx.request.body.north.caching.scanModeId;
+    ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(northConnectorCommand.settings);
+    ctx.app.reloadService.onCreateNorth.mockReturnValue(northConnector);
+
+    await northConnectorController.createNorthConnector(ctx);
+
+    expect(ctx.badRequest).toHaveBeenCalledWith('Scan mode not specified');
+  });
+
+  it('createNorthConnector() should throw an error if scan mode not found', async () => {
+    ctx.request.body = {
+      north: JSON.parse(JSON.stringify(northConnectorCommand)),
+      subscriptions: [
+        { type: 'south', subscription: { id: 'id1' } },
+        { type: 'external-source', externalSubscription: { id: 'id2' } }
+      ]
+    };
+    delete ctx.request.body.north.caching.scanModeId;
+    ctx.request.body.north.caching.scanModeName = 'invalid';
+    const scanMode: ScanModeDTO = {
+      id: '1',
+      name: 'scan mode',
+      description: 'description',
+      cron: '* * * * *'
+    };
+    ctx.app.repositoryService.scanModeRepository.getScanModes.mockReturnValue([scanMode]);
+    ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(northConnectorCommand.settings);
+    ctx.app.reloadService.onCreateNorth.mockReturnValue(northConnector);
+
+    await northConnectorController.createNorthConnector(ctx);
+
+    expect(ctx.badRequest).toHaveBeenCalledWith('Scan mode invalid not found');
+  });
+
+  it('createNorthConnector() should create North connector with found scan mode', async () => {
+    ctx.request.body = {
+      north: JSON.parse(JSON.stringify(northConnectorCommand)),
+      subscriptions: [
+        { type: 'south', subscription: { id: 'id1' } },
+        { type: 'external-source', externalSubscription: { id: 'id2' } }
+      ]
+    };
+    delete ctx.request.body.north.caching.scanModeId;
+    ctx.request.body.north.caching.scanModeName = 'scan mode';
+    const scanMode: ScanModeDTO = {
+      id: '1',
+      name: 'scan mode',
+      description: 'description',
+      cron: '* * * * *'
+    };
+    ctx.app.repositoryService.scanModeRepository.getScanModes.mockReturnValue([scanMode]);
+    ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(northConnectorCommand.settings);
+    ctx.app.reloadService.onCreateNorth.mockReturnValue(northConnector);
+
+    await northConnectorController.createNorthConnector(ctx);
+
+    expect(ctx.created).toHaveBeenCalledWith(northConnector);
+  });
+
   it('createNorthConnector() should create North connector and not start it', async () => {
     ctx.request.body = {
       north: { ...northConnectorCommand, enabled: false },

diff --git a/backend/src/web-server/controllers/north-connector.controller.ts b/backend/src/web-server/controllers/north-connector.controller.ts
@@ -1,5 +1,6 @@
 import { KoaContext } from '../koa';
 import {
+  NorthCacheSettingsDTO,
   NorthConnectorCommandDTO,
   NorthConnectorDTO,
   NorthConnectorWithItemsCommandDTO,
@@ -73,6 +74,17 @@ export default class NorthConnectorController {
 
       await this.validator.validateSettings(manifest.settings, command.settings);
 
+      if (!command.caching.scanModeId && !command.caching.scanModeName) {
+        throw new Error(`Scan mode not specified`);
+      } else if (!command.caching.scanModeId && command.caching.scanModeName) {
+        const scanModes = ctx.app.repositoryService.scanModeRepository.getScanModes();
+        const scanMode = scanModes.find(element => element.name === command.caching.scanModeName);
+        if (!scanMode) {
+          throw new Error(`Scan mode ${command.caching.scanModeName} not found`);
+        }
+        command.caching.scanModeId = scanMode.id;
+      }
+
       let duplicatedConnector: NorthConnectorDTO | null = null;
       if (ctx.query.duplicateId) {
         duplicatedConnector = ctx.app.repositoryService.northConnectorRepository.getNorthConnector(ctx.query.duplicateId);
@@ -544,9 +556,13 @@ export default class NorthConnectorController {
       }
 
       await this.validator.validateSettings(manifest.settings, ctx.request.body!.settings);
+      const northCaching = { ...ctx.request.body!.caching };
+      delete northCaching.scanModeName;
+      northCaching.scanModeId = '';
       const command: NorthConnectorDTO = {
         id: northConnector?.id || 'test',
         ...ctx.request.body!,
+        caching: northCaching as NorthCacheSettingsDTO,
         name: northConnector?.name || `${ctx.request.body!.type}:test-connection`
       };
       command.settings = await ctx.app.encryptionService.encryptConnectorSecrets(

diff --git a/backend/src/web-server/controllers/south-connector.controller.spec.ts b/backend/src/web-server/controllers/south-connector.controller.spec.ts
@@ -10,6 +10,7 @@ import {
   SouthConnectorItemDTO
 } from '../../../../shared/model/south-connector.model';
 import { southTestManifest } from '../../tests/__mocks__/south-service.mock';
+import { ScanModeDTO } from '../../../../shared/model/scan-mode.model';
 
 jest.mock('./validators/joi.validator');
 jest.mock('papaparse');
@@ -68,7 +69,12 @@ const itemCommand: SouthConnectorItemCommandDTO = {
 const item: SouthConnectorItemDTO = {
   id: 'id',
   connectorId: 'connectorId',
-  ...itemCommand
+  enabled: true,
+  name: 'name',
+  settings: {
+    regex: '.*'
+  },
+  scanModeId: 'scanModeId'
 };
 const page = {
   content: [item],
@@ -226,10 +232,91 @@ describe('South connector controller', () => {
     ctx.query.duplicateId = null;
   });
 
+  it('createSouthConnector() should throw error when scan mode is not specified in item', async () => {
+    ctx.request.body = {
+      south: southConnectorCommand,
+      items: [
+        {
+          name: 'name',
+          enabled: true,
+          settings: {
+            regex: '.*'
+          }
+        }
+      ]
+    };
+    ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(sqliteConnectorCommand.settings);
+    ctx.app.reloadService.onCreateSouth.mockReturnValue(southConnector);
+    ctx.app.southService.getInstalledSouthManifests.mockReturnValue([southTestManifest]);
+
+    await southConnectorController.createSouthConnector(ctx);
+    expect(ctx.badRequest).toHaveBeenCalledWith('Scan mode not specified for item name');
+  });
+
+  it('createSouthConnector() should throw error when scan mode is not found', async () => {
+    ctx.request.body = {
+      south: southConnectorCommand,
+      items: [
+        {
+          name: 'name',
+          scanModeName: 'invalid',
+          enabled: true,
+          settings: {
+            regex: '.*'
+          }
+        }
+      ]
+    };
+    const scanMode: ScanModeDTO = {
+      id: '1',
+      name: 'scan mode',
+      description: 'description',
+      cron: '* * * * *'
+    };
+    ctx.app.repositoryService.scanModeRepository.getScanModes.mockReturnValue([scanMode]);
+
+    ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(sqliteConnectorCommand.settings);
+    ctx.app.reloadService.onCreateSouth.mockReturnValue(southConnector);
+    ctx.app.southService.getInstalledSouthManifests.mockReturnValue([southTestManifest]);
+
+    await southConnectorController.createSouthConnector(ctx);
+    expect(ctx.badRequest).toHaveBeenCalledWith('Scan mode invalid not found for item name');
+  });
+
+  it('createSouthConnector() should create connector with items', async () => {
+    ctx.request.body = {
+      south: southConnectorCommand,
+      items: [
+        {
+          name: 'name',
+          scanModeName: 'scan mode',
+          enabled: true,
+          settings: {
+            regex: '.*'
+          }
+        }
+      ]
+    };
+    const scanMode: ScanModeDTO = {
+      id: '1',
+      name: 'scan mode',
+      description: 'description',
+      cron: '* * * * *'
+    };
+    ctx.app.repositoryService.scanModeRepository.getScanModes.mockReturnValue([scanMode]);
+    ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(sqliteConnectorCommand.settings);
+    ctx.app.reloadService.onCreateSouth.mockReturnValue(southConnector);
+    ctx.app.southService.getInstalledSouthManifests.mockReturnValue([southTestManifest]);
+
+    await southConnectorController.createSouthConnector(ctx);
+    expect(ctx.app.reloadService.onCreateSouth).toHaveBeenCalledWith(sqliteConnectorCommand);
+    expect(ctx.created).toHaveBeenCalledWith(southConnector);
+  });
+
   it('createSouthConnector() should create South connector with forceMaxInstantPerItem', async () => {
     ctx.request.body = {
       south: sqliteConnectorCommand,
-      items: [{}]
+      items: [itemCommand]
     };
     ctx.app.encryptionService.encryptConnectorSecrets.mockReturnValue(sqliteConnectorCommand.settings);
     ctx.app.reloadService.onCreateSouth.mockReturnValue(southConnector);

diff --git a/backend/src/web-server/controllers/south-connector.controller.ts b/backend/src/web-server/controllers/south-connector.controller.ts
@@ -95,7 +95,14 @@ export default class SouthConnectorController {
         manifest.settings
       );
       ctx.request.body!.name = southConnector ? southConnector.name : `${ctx.request.body!.type}:test-connection`;
-      const logger = ctx.app.logger.child({ scopeType: 'south', scopeId: command.id, scopeName: command.name }, { level: 'silent' });
+      const logger = ctx.app.logger.child(
+        {
+          scopeType: 'south',
+          scopeId: command.id,
+          scopeName: command.name
+        },
+        { level: 'silent' }
+      );
       const southToTest = ctx.app.southService.createSouth(command, [], this.addValues, this.addFile, 'baseFolder', logger);
       await southToTest.testConnection();
 
@@ -118,10 +125,21 @@ export default class SouthConnectorController {
         return ctx.throw(404, 'South manifest not found');
       }
 
+      const scanModes = ctx.app.repositoryService.scanModeRepository.getScanModes();
+
       await this.validator.validateSettings(manifest.settings, command.settings);
       // Check if item settings match the item schema, throw an error otherwise
       for (const item of ctx.request.body!.items) {
         await this.validator.validateSettings(manifest.items.settings, item.settings);
+        if (!item.scanModeId && !item.scanModeName) {
+          throw new Error(`Scan mode not specified for item ${item.name}`);
+        } else if (!item.scanModeId && item.scanModeName) {
+          const scanMode = scanModes.find(element => element.name === item.scanModeName);
+          if (!scanMode) {
+            throw new Error(`Scan mode ${item.scanModeName} not found for item ${item.name}`);
+          }
+          item.scanModeId = scanMode.id;
+        }
       }
 
       if (manifest.modes.forceMaxInstantPerItem) {
@@ -140,6 +158,7 @@ export default class SouthConnectorController {
         manifest.settings
       );
       const southConnector = await ctx.app.reloadService.onCreateSouth(command);
+
       await ctx.app.reloadService.onCreateOrUpdateSouthItems(southConnector, ctx.request.body!.items, []);
 
       ctx.created(southConnector);
@@ -203,7 +222,7 @@ export default class SouthConnectorController {
     }
   }
 
-  startSouthConnector = async (ctx: KoaContext<void, void>) => {
+  async startSouthConnector(ctx: KoaContext<void, void>): Promise<void> {
     const southConnector = ctx.app.repositoryService.southConnectorRepository.getSouthConnector(ctx.params.id);
     if (!southConnector) {
       return ctx.notFound();
@@ -215,9 +234,9 @@ export default class SouthConnectorController {
     } catch (error: any) {
       ctx.badRequest(error.message);
     }
-  };
+  }
 
-  stopSouthConnector = async (ctx: KoaContext<void, void>) => {
+  async stopSouthConnector(ctx: KoaContext<void, void>): Promise<void> {
     const southConnector = ctx.app.repositoryService.southConnectorRepository.getSouthConnector(ctx.params.id);
     if (!southConnector) {
       return ctx.notFound();
@@ -229,7 +248,7 @@ export default class SouthConnectorController {
     } catch (error: any) {
       ctx.badRequest(error.message);
     }
-  };
+  }
 
   async resetSouthMetrics(ctx: KoaContext<void, void>): Promise<void> {
     const southConnector = ctx.app.repositoryService.southConnectorRepository.getSouthConnector(ctx.params.southId);
@@ -255,6 +274,10 @@ export default class SouthConnectorController {
     ctx.ok(southItems);
   }
 
+  /**
+   * Endpoint used to download a CSV from a list of items when creating a connector (before the items are saved on
+   * the database). When the items are already saved, it is downloaded with the export method
+   */
   async southItemsToCsv(ctx: KoaContext<{ items: Array<SouthConnectorItemDTO> }, any>): Promise<void> {
     const scanModes = ctx.app.repositoryService.scanModeRepository.getScanModes();
     const southItems = ctx.request.body!.items.map(item => {

diff --git a/backend/src/web-server/routes/index.ts b/backend/src/web-server/routes/index.ts
@@ -200,6 +200,7 @@ router.post('/api/south/:southType/items/check-import/:southId', upload.single('
 );
 router.post('/api/south/:southId/items/import', (ctx: KoaContext<any, any>) => southConnectorController.importSouthItems(ctx));
 router.get('/api/south/:southId/items/export', (ctx: KoaContext<any, any>) => southConnectorController.exportSouthItems(ctx));
+router.get('/api/south/:southId/items/export', (ctx: KoaContext<any, any>) => southConnectorController.exportSouthItems(ctx));
 router.put('/api/south/items/to-csv', (ctx: KoaContext<any, any>) => southConnectorController.southItemsToCsv(ctx));
 router.get('/api/south/:southId/items/:id', (ctx: KoaContext<any, any>) => southConnectorController.getSouthItem(ctx));
 router.put('/api/south/:southId/items/:id', (ctx: KoaContext<any, any>) => southConnectorController.updateSouthItem(ctx));