From 4c19916b4fb2a2fbfb5c61a2f9be3f5eaa5239c4 Mon Sep 17 00:00:00 2001 From: nmlinaric Date: Tue, 21 Apr 2020 14:50:29 +0200 Subject: [PATCH 1/5] Fix property logging --- src/Services/NodeUptimeNotificationService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Services/NodeUptimeNotificationService.ts b/src/Services/NodeUptimeNotificationService.ts index e1b979d..a7e78fe 100644 --- a/src/Services/NodeUptimeNotificationService.ts +++ b/src/Services/NodeUptimeNotificationService.ts @@ -78,7 +78,7 @@ export class NodeUptimeNotificationService { if (oldNodeStatus.isReported != newNodeStatus.isReported || oldNodeStatus.isUp != newNodeStatus.isUp) { logger.info(`Found node ID ${nodeUptime.nodeId} that has old status` + - `[isReported: ${oldNodeStatus.isReported}, isUp: ${oldNodeStatus.isNewRecord}].`); + `[isReported: ${oldNodeStatus.isReported}, isUp: ${oldNodeStatus.isUp}].`); await this.nodeStatusService.updateNodeStatus( nodeUptime.nodeId, From 97cbf40f29b9ac3d68ce01cda149974741639d0a Mon Sep 17 00:00:00 2001 From: nmlinaric Date: Tue, 21 Apr 2020 14:51:49 +0200 Subject: [PATCH 2/5] Fix notification reporting when daemon app is down --- src/Scheduler/Tasks/UptimeNotificationTask.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Scheduler/Tasks/UptimeNotificationTask.ts b/src/Scheduler/Tasks/UptimeNotificationTask.ts index 49396ee..0b7e248 100644 --- a/src/Scheduler/Tasks/UptimeNotificationTask.ts +++ b/src/Scheduler/Tasks/UptimeNotificationTask.ts @@ -29,7 +29,10 @@ export class UptimeNotificationTask implements Task { // select, from table of latest uptime records for each node, // entries that are older than 1 hour or are entries in last hour that reported node is down return await database.runQuery( - `select NU.* + `select "id", "createdAt", "updatedAt", Nu."nodeId", + case WHEN ("isWorking" = true and "updatedAt" > now() - interval '16 minutes') then true + else false + end as "isWorking" from ( select "nodeId", max("createdAt") as "lastUptimeReported" from "NodeUptime" @@ -38,8 +41,8 @@ export class UptimeNotificationTask implements Task { left outer join "NodeUptime" NU on NU."nodeId" = latest_uptimes."nodeId" and NU."createdAt" = "lastUptimeReported" - where "lastUptimeReported" < now() - interval '1 hour' or - ("lastUptimeReported" > now() - interval '1 hour' and NU."isWorking" = false);`, + where "lastUptimeReported" < now() - interval '16 minutes' or + ("lastUptimeReported" > now() - interval '1 hour' and NU."isWorking" = false);`, {type: QueryTypes.SELECT}) } catch (e) { logger.error("Failed to find not working nodes in database", e) From 40976fbdc20a6cddeb6428fbaccdcbe7a7b4cf9c Mon Sep 17 00:00:00 2001 From: morrigan Date: Mon, 4 May 2020 16:47:05 +0200 Subject: [PATCH 3/5] Handle separately case when node is found as offline --- src/Models/NodeUptime.ts | 3 ++ src/Scheduler/Tasks/UptimeNotificationTask.ts | 11 ++++--- src/Services.ts | 3 +- src/Services/NodeUptimeNotificationService.ts | 32 +++++++++++++------ 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/Models/NodeUptime.ts b/src/Models/NodeUptime.ts index 624ff46..b1310ab 100644 --- a/src/Models/NodeUptime.ts +++ b/src/Models/NodeUptime.ts @@ -7,6 +7,9 @@ export class NodeUptime extends Model { public isWorking: boolean; public nodeId: number; + // Virtual property returned from query + public foundDown?: boolean; + public static initialize(sequelize: Sequelize) { this.init({ isWorking: { diff --git a/src/Scheduler/Tasks/UptimeNotificationTask.ts b/src/Scheduler/Tasks/UptimeNotificationTask.ts index 0b7e248..9b0547b 100644 --- a/src/Scheduler/Tasks/UptimeNotificationTask.ts +++ b/src/Scheduler/Tasks/UptimeNotificationTask.ts @@ -24,15 +24,16 @@ export class UptimeNotificationTask implements Task { } } + // TODO: Should possibly optimize with excluding reported downtimes private async findAllNotWorkingNodes(): Promise { try { // select, from table of latest uptime records for each node, // entries that are older than 1 hour or are entries in last hour that reported node is down return await database.runQuery( - `select "id", "createdAt", "updatedAt", Nu."nodeId", - case WHEN ("isWorking" = true and "updatedAt" > now() - interval '16 minutes') then true - else false - end as "isWorking" + `select "id", "isWorking", "createdAt", "updatedAt", Nu."nodeId", + case WHEN ("isWorking" = true and "updatedAt" > now() - interval '16 minutes') then false + else true + end as "foundDown" from ( select "nodeId", max("createdAt") as "lastUptimeReported" from "NodeUptime" @@ -49,4 +50,4 @@ export class UptimeNotificationTask implements Task { } return null; } -} \ No newline at end of file +} diff --git a/src/Services.ts b/src/Services.ts index d2da4d3..47a43c8 100644 --- a/src/Services.ts +++ b/src/Services.ts @@ -46,7 +46,8 @@ export class Services { this.emailService, this.userService, this.nodeService, - this.nodeStatusService + this.nodeStatusService, + this.nodeUptimeService, ); // define scheduled tasks this.schedulingService = new SchedulingService( diff --git a/src/Services/NodeUptimeNotificationService.ts b/src/Services/NodeUptimeNotificationService.ts index a7e78fe..51145bb 100644 --- a/src/Services/NodeUptimeNotificationService.ts +++ b/src/Services/NodeUptimeNotificationService.ts @@ -6,6 +6,7 @@ import {NodeService} from "./NodeService"; import {NodeStatusService} from "./NodeStatusService"; import config from "../Config/Config"; import {NodeStatus} from "../Models/NodeStatus"; +import {NodeUptimeService} from "./NodeUptimeService"; export class NodeUptimeNotificationService { @@ -13,45 +14,56 @@ export class NodeUptimeNotificationService { private userService: UserService; private nodeService: NodeService; private nodeStatusService: NodeStatusService; + private nodeUptimeService: NodeUptimeService; constructor( emailService: EmailService, userService: UserService, nodeService: NodeService, - nodeStatusService: NodeStatusService + nodeStatusService: NodeStatusService, + nodeUptimeService: NodeUptimeService ) { this.emailService = emailService; this.userService = userService; this.nodeService = nodeService; - this.nodeStatusService = nodeStatusService + this.nodeStatusService = nodeStatusService; + this.nodeUptimeService = nodeUptimeService; } public async processNodeUptime(nodeUptime: NodeUptime): Promise { logger.info(`Processing node uptime for node ID ${nodeUptime.nodeId}`); // check if node status already created for node const currentNodeStatus = await this.nodeStatusService.getNodeStatusByNodeId(nodeUptime.nodeId); - const newNodeStatus = {nodeId: nodeUptime.nodeId, isUp: nodeUptime.isWorking, isReported: true} as NodeStatus; + const newNodeStatus = { + nodeId: nodeUptime.nodeId, + isUp: nodeUptime.foundDown ? false : nodeUptime.isWorking, + isReported: true + } as NodeStatus; if (currentNodeStatus != null) { - logger.info(`Updating existing status to ${newNodeStatus.isUp} for node ID ${nodeUptime.nodeId}`); await this.updateExistingNodeStatus(nodeUptime, newNodeStatus, currentNodeStatus); } else { logger.info(`Creating new status (${newNodeStatus.isUp}) for node ID ${nodeUptime.nodeId}`); await this.createNewNodeStatus(nodeUptime, newNodeStatus); } + + if (nodeUptime.foundDown && (currentNodeStatus && !currentNodeStatus.isReported || !currentNodeStatus)) { + logger.info("Creating new record for node uptime with status offline."); + await this.nodeUptimeService.createNodeUptime(false, nodeUptime.nodeId); + } } private async createNewNodeStatus(nodeUptime: NodeUptime, newNodeStatus: NodeStatus): Promise { // create new node status entry await this.nodeStatusService.storeNodeStatus( - nodeUptime.nodeId, - nodeUptime.isWorking, + newNodeStatus.nodeId, + newNodeStatus.isUp, true ); - logger.info(`Stored new node status (${nodeUptime.isWorking}) for node ${nodeUptime.nodeId}.`); + logger.info(`Stored new node status (${newNodeStatus.isUp}) for node ${newNodeStatus.nodeId}.`); // send notification if node is down if (!newNodeStatus.isUp) { - logger.info(`Sending uptime notification for node ID ${nodeUptime.nodeId}`); + logger.info(`Sending uptime notification for node ID ${newNodeStatus.nodeId}`); await this.sendUptimeNotification(nodeUptime); } } @@ -62,7 +74,7 @@ export class NodeUptimeNotificationService { oldNodeStatus: NodeStatus, ): Promise { // if reported node is down - if (!nodeUptime.isWorking) { + if (!newNodeStatus.isUp) { logger.info(`Node ID ${nodeUptime.nodeId} is down.`); // send email notification if current node was up until now or was down but report wasn't sent if (oldNodeStatus.isUp) { @@ -118,4 +130,4 @@ export class NodeUptimeNotificationService { logger.error(`Failed to find user:${node.userId} for uptime entry: ${uptime.id}`); } } -} \ No newline at end of file +} From 943c5e2914e9ec304f1d00a84042a6a689c1d77c Mon Sep 17 00:00:00 2001 From: morrigan Date: Mon, 4 May 2020 16:47:45 +0200 Subject: [PATCH 4/5] Update interval --- src/Scheduler/Tasks/UptimeNotificationTask.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Scheduler/Tasks/UptimeNotificationTask.ts b/src/Scheduler/Tasks/UptimeNotificationTask.ts index 9b0547b..5aa0e0e 100644 --- a/src/Scheduler/Tasks/UptimeNotificationTask.ts +++ b/src/Scheduler/Tasks/UptimeNotificationTask.ts @@ -31,7 +31,7 @@ export class UptimeNotificationTask implements Task { // entries that are older than 1 hour or are entries in last hour that reported node is down return await database.runQuery( `select "id", "isWorking", "createdAt", "updatedAt", Nu."nodeId", - case WHEN ("isWorking" = true and "updatedAt" > now() - interval '16 minutes') then false + case WHEN ("isWorking" = true and "updatedAt" > now() - interval '59 minutes') then false else true end as "foundDown" from ( @@ -42,7 +42,7 @@ export class UptimeNotificationTask implements Task { left outer join "NodeUptime" NU on NU."nodeId" = latest_uptimes."nodeId" and NU."createdAt" = "lastUptimeReported" - where "lastUptimeReported" < now() - interval '16 minutes' or + where "lastUptimeReported" < now() - interval '59 minutes' or ("lastUptimeReported" > now() - interval '1 hour' and NU."isWorking" = false);`, {type: QueryTypes.SELECT}) } catch (e) { From e37b5d8d6df0ea6543c2a42c4a028fb47be15438 Mon Sep 17 00:00:00 2001 From: morrigan Date: Mon, 4 May 2020 17:21:16 +0200 Subject: [PATCH 5/5] Fix test --- test/unit/Services/NodeUptimeNotificationService.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/unit/Services/NodeUptimeNotificationService.test.ts b/test/unit/Services/NodeUptimeNotificationService.test.ts index fc9d5ba..4b5707f 100644 --- a/test/unit/Services/NodeUptimeNotificationService.test.ts +++ b/test/unit/Services/NodeUptimeNotificationService.test.ts @@ -13,6 +13,7 @@ import * as chai from "chai"; // eslint-disable-next-line @typescript-eslint/no-require-imports import sinonChai = require('sinon-chai'); import {NodeStatus} from "../../../src/Models/NodeStatus"; +import {NodeUptimeService} from "../../../src/Services/NodeUptimeService"; chai.should(); chai.use(sinonChai); @@ -59,6 +60,9 @@ describe("NodeUptimeNotificationService", function () { getNodeByPkStub = sinon.stub(nodeService, "getNodeByPk"); getNodeByPkStub.returns(testNode); + const nodeUptimeService = new NodeUptimeService(); + sinon.stub(nodeUptimeService, "createNodeUptime").resolves(); + nodeStatusService = new NodeStatusService(); updateNodeStatusStub = sinon.stub(nodeStatusService, "updateNodeStatus"); createNodeStatusStub = sinon.stub(nodeStatusService, "storeNodeStatus"); @@ -68,7 +72,8 @@ describe("NodeUptimeNotificationService", function () { emailService, userService, nodeService, - nodeStatusService + nodeStatusService, + nodeUptimeService, ) });