feat(external-source): deprecate and remove external sources

backend/src/db/entity-migrations/v3.0-initial-setup.ts

@@ -2,7 +2,6 @@ import { Knex } from 'knex';
 import { LOG_LEVELS } from '../../../../shared/model/engine.model';
 import { ENGINES_TABLE } from '../../repository/engine.repository';
 import { USERS_TABLE } from '../../repository/user.repository';
-import { EXTERNAL_SOURCES_TABLE } from '../../repository/external-source.repository';
 import { IP_FILTERS_TABLE } from '../../repository/ip-filter.repository';
 import { SCAN_MODES_TABLE } from '../../repository/scan-mode.repository';
 import { HISTORY_QUERIES_TABLE } from '../../repository/history-query.repository';
@@ -11,7 +10,10 @@ import { SOUTH_CONNECTORS_TABLE } from '../../repository/south-connector.reposit
 import { SOUTH_ITEMS_TABLE } from '../../repository/south-item.repository';
 import { NORTH_CONNECTORS_TABLE } from '../../repository/north-connector.repository';
 import CreateTableBuilder = Knex.CreateTableBuilder;
-import { EXTERNAL_SUBSCRIPTION_TABLE, SUBSCRIPTION_TABLE } from '../../repository/subscription.repository';
+import { SUBSCRIPTION_TABLE } from '../../repository/subscription.repository';
+
+const EXTERNAL_SOURCES_TABLE = 'external_sources';
+const EXTERNAL_SUBSCRIPTION_TABLE = 'external_subscription';
 
 export async function up(knex: Knex): Promise<void> {
   await createEnginesTable(knex);

backend/src/db/entity-migrations/v3.4.0-deprecate-north-oibus.ts

@@ -6,9 +6,13 @@ import { filesExists } from '../../service/utils';
 import fs from 'node:fs/promises';
 import { HISTORY_ITEMS_TABLE } from '../../repository/history-query-item.repository';
 
+const EXTERNAL_SOURCES_TABLE = 'external_sources';
+const EXTERNAL_SUBSCRIPTION_TABLE = 'external_subscription';
+
 export async function up(knex: Knex): Promise<void> {
   await removeNorthOIBusConnectors(knex);
   await removeNorthOIBusHistoryQueries(knex);
+  await removeExternalSubscriptions(knex);
 }
 
 async function removeNorthOIBusConnectors(knex: Knex): Promise<void> {
@@ -39,4 +43,9 @@ async function removeNorthOIBusHistoryQueries(knex: Knex): Promise<void> {
   await knex(HISTORY_QUERIES_TABLE).delete().where('north_type', 'oibus');
 }
 
+async function removeExternalSubscriptions(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(EXTERNAL_SUBSCRIPTION_TABLE);
+  await knex.schema.dropTableIfExists(EXTERNAL_SOURCES_TABLE);
+}
+
 export async function down(): Promise<void> {}

backend/src/north/north-connector.spec.ts

@@ -353,36 +353,18 @@ describe('NorthConnector enabled', () => {
     expect(north.isSubscribed('southId')).toBeTruthy();
   });
 
-  it('should check if North is subscribed to all external sources', async () => {
-    (repositoryService.subscriptionRepository.getExternalNorthSubscriptions as jest.Mock).mockReturnValue([]);
-    await north.start();
-    expect(north.isSubscribedToExternalSource('externalSourceId')).toBeTruthy();
-  });
-
   it('should check if North is subscribed to South', async () => {
     (repositoryService.subscriptionRepository.getNorthSubscriptions as jest.Mock).mockReturnValue(['southId']);
     await north.start();
     expect(north.isSubscribed('southId')).toBeTruthy();
   });
 
-  it('should check if North is subscribed to external source', async () => {
-    (repositoryService.subscriptionRepository.getExternalNorthSubscriptions as jest.Mock).mockReturnValue(['externalSourceId']);
-    await north.start();
-    expect(north.isSubscribedToExternalSource('externalSourceId')).toBeTruthy();
-  });
-
   it('should check if North is not subscribed to South', async () => {
     (repositoryService.subscriptionRepository.getNorthSubscriptions as jest.Mock).mockReturnValue(['southId1']);
     await north.start();
     expect(north.isSubscribed('southId2')).toBeFalsy();
   });
 
-  it('should check if North is not subscribed to external source', async () => {
-    (repositoryService.subscriptionRepository.getExternalNorthSubscriptions as jest.Mock).mockReturnValue(['externalSourceId1']);
-    await north.start();
-    expect(north.isSubscribedToExternalSource('externalSourceId2')).toBeFalsy();
-  });
-
   it('should get error files', async () => {
     const result = await north.getErrorFiles('2022-11-11T11:11:11.111Z', '2022-11-12T11:11:11.111Z', 'file');
     expect(result).toEqual([

backend/src/north/north-connector.ts

@@ -11,7 +11,7 @@ import { EventEmitter } from 'node:events';
 import { ScanModeDTO } from '../../../shared/model/scan-mode.model';
 import DeferredPromise from '../service/deferred-promise';
 import { OIBusContent, OIBusError, OIBusRawContent, OIBusTimeValue, OIBusTimeValueContent } from '../../../shared/model/engine.model';
-import { ExternalSubscriptionDTO, SubscriptionDTO } from '../../../shared/model/subscription.model';
+import { SubscriptionDTO } from '../../../shared/model/subscription.model';
 import { DateTime } from 'luxon';
 import { PassThrough } from 'node:stream';
 import { ReadStream } from 'node:fs';
@@ -45,7 +45,6 @@ export default class NorthConnector<T extends NorthSettings = any> {
   private fileCacheService: FileCacheService;
   protected metricsService: NorthConnectorMetricsService | null = null;
   private subscribedTo: Array<SubscriptionDTO> = [];
-  private subscribedToExternalSources: Array<ExternalSubscriptionDTO> = [];
   private cacheSize = 0;
 
   private fileBeingSent: string | null = null;
@@ -142,7 +141,6 @@ export default class NorthConnector<T extends NorthSettings = any> {
         cacheSize: this.cacheSize
       });
       this.subscribedTo = this.repositoryService.subscriptionRepository.getNorthSubscriptions(this.connector.id);
-      this.subscribedToExternalSources = this.repositoryService.subscriptionRepository.getExternalNorthSubscriptions(this.connector.id);
     }
     await this.valueCacheService.start();
     await this.fileCacheService.start();
@@ -394,14 +392,6 @@ export default class NorthConnector<T extends NorthSettings = any> {
     return this.subscribedTo.length === 0 || this.subscribedTo.includes(southId);
   }
 
-  /**
-   * Check whether the North is subscribed to an external source.
-   * If subscribedToExternalSources is not defined or an empty array, the subscription is true.
-   */
-  isSubscribedToExternalSource(externalSourceId: string): boolean {
-    return this.subscribedToExternalSources.length === 0 || this.subscribedToExternalSources.includes(externalSourceId);
-  }
-
   /**
    * Check appropriate caches emptiness
    */

backend/src/repository/subscription.repository.spec.ts

@@ -1,7 +1,7 @@
 import SubscriptionRepository from './subscription.repository';
-import SqliteDatabaseMock, { run, all, get } from '../tests/__mocks__/database.mock';
+import SqliteDatabaseMock, { all, get, run } from '../tests/__mocks__/database.mock';
 import { Database } from 'better-sqlite3';
-import { ExternalSubscriptionDTO, SubscriptionDTO } from '../../../shared/model/subscription.model';
+import { SubscriptionDTO } from '../../../shared/model/subscription.model';
 
 jest.mock('../tests/__mocks__/database.mock');
 
@@ -29,16 +29,6 @@ describe('Subscription repository', () => {
     expect(subscriptions).toEqual(expectedValue);
   });
 
-  it('should properly get all external subscriptions', () => {
-    const expectedValue: Array<ExternalSubscriptionDTO> = ['external1', 'external2'];
-    all.mockReturnValueOnce(expectedValue.map(sourceId => ({ externalSourceId: sourceId })));
-    const subscriptions = repository.getExternalNorthSubscriptions('north1');
-    expect(database.prepare).toHaveBeenCalledWith(
-      'SELECT external_source_id AS externalSourceId FROM external_subscription WHERE north_connector_id = ?;'
-    );
-    expect(subscriptions).toEqual(expectedValue);
-  });
-
   it('should properly get all subscribed North connectors', () => {
     const expectedValue = ['north1', 'north2'];
     all.mockReturnValueOnce(expectedValue.map(northId => ({ northConnectorId: northId })));
@@ -59,16 +49,6 @@ describe('Subscription repository', () => {
     expect(get).toHaveBeenCalledWith('north1', 'south1');
   });
 
-  it('should check an external subscription', () => {
-    get.mockReturnValueOnce({ externalSourceId: 'external1' });
-
-    repository.checkExternalNorthSubscription('north1', 'external1');
-    expect(database.prepare).toHaveBeenCalledWith(
-      'SELECT external_source_id AS externalSourceId FROM external_subscription WHERE north_connector_id = ? AND external_source_id = ?;'
-    );
-    expect(get).toHaveBeenCalledWith('north1', 'external1');
-  });
-
   it('should create a subscription', () => {
     run.mockReturnValueOnce({ lastInsertRowid: 1 });
 
@@ -77,39 +57,15 @@ describe('Subscription repository', () => {
     expect(run).toHaveBeenCalledWith('north1', 'south1');
   });
 
-  it('should create an external subscription', () => {
-    run.mockReturnValueOnce({ lastInsertRowid: 1 });
-
-    repository.createExternalNorthSubscription('north1', 'external1');
-    expect(database.prepare).toHaveBeenCalledWith(
-      'INSERT INTO external_subscription (north_connector_id, external_source_id) VALUES (?, ?);'
-    );
-    expect(run).toHaveBeenCalledWith('north1', 'external1');
-  });
-
   it('should delete a subscription', () => {
     repository.deleteNorthSubscription('north1', 'south1');
     expect(database.prepare).toHaveBeenCalledWith('DELETE FROM subscription WHERE north_connector_id = ? AND south_connector_id = ?;');
     expect(run).toHaveBeenCalledWith('north1', 'south1');
   });
 
-  it('should delete an external subscription', () => {
-    repository.deleteExternalNorthSubscription('north1', 'external1');
-    expect(database.prepare).toHaveBeenCalledWith(
-      'DELETE FROM external_subscription WHERE north_connector_id = ? AND external_source_id = ?;'
-    );
-    expect(run).toHaveBeenCalledWith('north1', 'external1');
-  });
-
   it('should delete all North subscriptions', () => {
     repository.deleteNorthSubscriptions('north1');
     expect(database.prepare).toHaveBeenCalledWith('DELETE FROM subscription WHERE north_connector_id = ?;');
     expect(run).toHaveBeenCalledWith('north1');
   });
-
-  it('should delete all external North subscriptions', () => {
-    repository.deleteExternalNorthSubscriptions('north1');
-    expect(database.prepare).toHaveBeenCalledWith('DELETE FROM external_subscription WHERE north_connector_id = ?;');
-    expect(run).toHaveBeenCalledWith('north1');
-  });
 });