From f4156090e967e9df4a0ca32c22e10d92b2da6567 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Mon, 18 Dec 2023 17:36:06 +0200 Subject: [PATCH 1/2] fix(core): missing check for double removal of unresponsive module refactor(core): add sanity check for module existence in monitor loop --- .../service-discovery/ServiceMonitor.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/core/src/config-manager/service-discovery/ServiceMonitor.ts b/packages/core/src/config-manager/service-discovery/ServiceMonitor.ts index 6af99229f..5757f747d 100644 --- a/packages/core/src/config-manager/service-discovery/ServiceMonitor.ts +++ b/packages/core/src/config-manager/service-discovery/ServiceMonitor.ts @@ -169,15 +169,18 @@ export class ServiceMonitor { private async monitorModules() { for (const module of this._serviceRegistry.getRegisteredModules()) { - const registeredModule = this._serviceRegistry.getModule(module)!; + const registeredModule = this._serviceRegistry.getModule(module); + if (!registeredModule) continue; try { await this.healthCheckService(module, registeredModule.address); } catch (e) { - this.handleUnresponsiveModule( - module, - registeredModule.address, - HealthCheckStatus.SERVICE_UNKNOWN, - ); + if (this._serviceRegistry.getModule(module)) { + this.handleUnresponsiveModule( + module, + registeredModule.address, + HealthCheckStatus.SERVICE_UNKNOWN, + ); + } } } this.moduleRegister.emit('serving-modules-update'); From 18b1db17d84a7b4588d48e5c0324e8f387c81f19 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Mon, 18 Dec 2023 18:24:56 +0200 Subject: [PATCH 2/2] chore: add more logs in module recovery --- packages/core/src/config-manager/service-discovery/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/src/config-manager/service-discovery/index.ts b/packages/core/src/config-manager/service-discovery/index.ts index bf80bb102..fa5186be6 100644 --- a/packages/core/src/config-manager/service-discovery/index.ts +++ b/packages/core/src/config-manager/service-discovery/index.ts @@ -137,6 +137,8 @@ export class ServiceDiscovery { this.grpcSdk.createModuleClient(moduleName, moduleUrl); } } catch (e) { + ConduitGrpcSdk.Logger.error(`SD: failed to recover: ${moduleName} ${moduleUrl}`); + ConduitGrpcSdk.Logger.error(`SD: recovery error: ${e}`); throw new Error('Failed to register unresponsive module'); } const healthStatus = healthResponse.status as unknown as HealthCheckStatus;