From 46d4bf315d7f54c14436fa7dfebfe841e524c434 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 30 May 2023 16:25:36 +0200 Subject: [PATCH 01/52] split findSilentEntities by mongo or API --- lib/models/entitiesStore.js | 77 +++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index b279936d..3303fda3 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -35,6 +35,83 @@ function orionServiceDb(service) { } function findSilentEntities(service, subservice, ruleData, func, callback) { + if (!config.nonSignalByAPI) { + return findSilentEntitiesByMongo(service, subservice, ruleData, func, callback); + } else { + return findSilentEntitiesByAPI(service, subservice, ruleData, func, callback); + } +} + +function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) { + var context = { op: 'checkNoSignal', comp: constants.COMPONENT_NAME }; + + var options = { + service: service, + servicepath: subservice + }; + options.headers = {}; + // if (token !== null) { + // options.headers[constants.AUTH_HEADER] = token; + // } + // Add correlator + var domain = process.domain; + if (domain && domain.context) { + options.headers[constants.CORRELATOR_HEADER] = domain.context.corr; + // Add other headers + if (domain.context.srv && options.headers[constants.SERVICE_HEADER] === undefined) { + options.headers[constants.SERVICE_HEADER] = domain.context.srv; + } + if (domain.context.subsrv && options.headers[constants.SUBSERVICE_HEADER] === undefined) { + options.headers[constants.SUBSERVICE_HEADER] = domain.context.subsrv; + } + if (domain.context.from && options.headers[constants.REALIP_HEADER] === undefined) { + options.headers[constants.REALIP_HEADER] = domain.context.from; + } + } + var connection = new NGSI.Connection(config.orion.URL, options); + + var filter = { + id: ruleData.id, + type: ruleData.type, + idPattern: ruleData.idRegexp + /* + ruleData.attribute + ruleData.reportInterval + */ + }; + // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor + connection.v2.listEntities(filter).then( + (response) => { + // Entities retrieved successfully + // response.correlator transaction id associated with the server response + // response.limit contains the used page size + // response.results is an array with the retrieved entities + // response.offset contains the offset used in the request + var count = 0; + response.results.forEach((entity) => { + logger.debug(context, 'silent entity %j', entity); + func(one); + count++; + }); + logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); + callback(null, response.results); + }, + (error) => { + // Error retrieving entities + // If the error was reported by Orion, error.correlator will be + // filled with the associated transaction id + // logger.warn( + // 'error v2.batchUpdate: %j trying update entity %j after event %j', + // error, + // changes, + // event + // ); + callback(error, null); + } + ); +} + +function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback) { var db, context = { op: 'checkNoSignal', comp: constants.COMPONENT_NAME }, criterion = {}; From cea4ca5faffbb860a73e470a1a371f5f5afd6565 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 5 Jun 2023 11:35:40 +0200 Subject: [PATCH 02/52] add logs and flag config for check non signal by API --- bin/perseo | 6 +++++- config.js | 6 ++++++ lib/models/entitiesStore.js | 13 +++++-------- test/unit/nsr_utest.js | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/bin/perseo b/bin/perseo index 4956ad37..0d6f79bb 100755 --- a/bin/perseo +++ b/bin/perseo @@ -107,7 +107,8 @@ function loadConfiguration() { 'PERSEO_AUTHENTICATION_USER', 'PERSEO_AUTHENTICATION_PASSWORD', 'PERSEO_AUTHENTICATION_SERVICE', - 'PERSEO_MAX_RULES_BY_CORR' + 'PERSEO_MAX_RULES_BY_CORR', + 'PERSEO_CHECK_NON_SIGNAL_BY_API' ]; const protectedVariables = [ @@ -234,6 +235,9 @@ function loadConfiguration() { if (process.env.PERSEO_MAX_RULES_BY_CORR) { config.maxRulesByCorr = process.env.PERSEO_MAX_RULES_BY_CORR; } + if (process.env.PERSEO_CHECK_NON_SIGNAL_BY_API) { + config.nonSignalByAPI = process.env.PERSEO_CHECK_NON_SIGNAL_BY_API; + } } loadConfiguration(); diff --git a/config.js b/config.js index ea8e1c39..c9ae4220 100644 --- a/config.js +++ b/config.js @@ -232,4 +232,10 @@ config.castTypes = false; */ config.maxRulesByCorr = 20; +/** + * Check nonSignal rules using API ngsiv2 or just access to mongo + * @type {Boolean} + */ +config.nonSignalByAPI = false; + module.exports = config; diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 3303fda3..e30103e2 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -28,7 +28,8 @@ var async = require('async'), entitiesCollectionName = require('../../config').orionDb.collection, myutils = require('../myutils'), constants = require('../constants'), - logger = require('logops'); + logger = require('logops'), + NGSI = require('ngsijs'); function orionServiceDb(service) { return appContext.OrionDb(config.orionDb.prefix + '-' + service); @@ -73,12 +74,13 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) var filter = { id: ruleData.id, type: ruleData.type, - idPattern: ruleData.idRegexp + idPattern: ruleData.idRegexp ? ruleData.idRegexp : '.*' /* ruleData.attribute ruleData.reportInterval */ }; + logger.debug(context, 'find silent entities by API ngsi using options %j and fileter %j', options, filter); // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor connection.v2.listEntities(filter).then( (response) => { @@ -100,12 +102,7 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) // Error retrieving entities // If the error was reported by Orion, error.correlator will be // filled with the associated transaction id - // logger.warn( - // 'error v2.batchUpdate: %j trying update entity %j after event %j', - // error, - // changes, - // event - // ); + logger.warn('error v2.listEntities: %j trying list entities using filter %j', error, filter); callback(error, null); } ); diff --git a/test/unit/nsr_utest.js b/test/unit/nsr_utest.js index a7a86a7a..d719a2fa 100644 --- a/test/unit/nsr_utest.js +++ b/test/unit/nsr_utest.js @@ -47,6 +47,33 @@ describe('noSignal', function() { type: null } }; + var rule2 = { + name: 'NSR2', + action: { + type: 'update', + parameters: { + id: 'alarma:${id}', + type: 'Alarm', + attributes: [ + { + name: 'msg', + value: 'El status de ${id} es ${status}' + } + ] + } + }, + subservice: '/', + service: 'unknownt', + nosignal: { + checkInterval: '1', + attribute: 'temperature', + reportInterval: '5', + id: 'thing:disp1', + idRegexp: null, + type: 'thing' + } + }; + describe('#addNSRule()', function() { it('should reject to add a rule with invalid check interval', function() { rule.nosignal.checkInterval = 'aserejé'; @@ -110,4 +137,13 @@ describe('noSignal', function() { should.deepEqual(noSignal.Nsr2Arr(rule.service, rule.subservice, rule.name, rule.nosignal), updatedRule); }); }); + + describe('#checkNoSignal()', function() { + it('check no signal ', function() { + var gotNSRule; + should.notEqual(noSignal.AddNSRule(rule2.service, rule2.subservice, rule2.name, rule2.nosignal), 0); + gotNSRule = noSignal.GetNSArrRule(rule2.service, rule2.subservice, rule2.name); + //console.log("RULE: " + JSON.stringify(gotNSRule)); + }); + }); }); From 693a3883a01c4f024d79b78b25e36ea9663b8682 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 6 Jun 2023 10:48:49 +0200 Subject: [PATCH 03/52] update filter with params for listEntity of ngsijs --- lib/models/entitiesStore.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index e30103e2..615bb302 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -44,7 +44,7 @@ function findSilentEntities(service, subservice, ruleData, func, callback) { } function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) { - var context = { op: 'checkNoSignal', comp: constants.COMPONENT_NAME }; + var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; var options = { service: service, @@ -72,14 +72,16 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) var connection = new NGSI.Connection(config.orion.URL, options); var filter = { - id: ruleData.id, + service: service, + servicepath: subservice, type: ruleData.type, - idPattern: ruleData.idRegexp ? ruleData.idRegexp : '.*' - /* - ruleData.attribute - ruleData.reportInterval - */ + mq: ruleData.attribute + '.dateModified=' + Date.now() / 1000 - ruleData.reportInterval }; + if (ruleData.id) { + filter['id'] = ruleData.id; + } else if (ruleData.idRegexp) { + filter['idPattern'] = ruleData.idRegexp; + } logger.debug(context, 'find silent entities by API ngsi using options %j and fileter %j', options, filter); // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor connection.v2.listEntities(filter).then( @@ -110,7 +112,7 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback) { var db, - context = { op: 'checkNoSignal', comp: constants.COMPONENT_NAME }, + context = { op: 'checkNoSignal.findSilentEntitiesByMongo', comp: constants.COMPONENT_NAME }, criterion = {}; db = orionServiceDb(service); From 3f87d0e9c8c546245da889cfebceb5cba46fee95 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 6 Jun 2023 11:02:29 +0200 Subject: [PATCH 04/52] add ruleData to log about rule --- lib/models/entitiesStore.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 615bb302..1f99155a 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -82,7 +82,13 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) } else if (ruleData.idRegexp) { filter['idPattern'] = ruleData.idRegexp; } - logger.debug(context, 'find silent entities by API ngsi using options %j and fileter %j', options, filter); + logger.debug( + context, + 'find silent entities by API ngsi using options %j and filter %j and rule %j', + options, + filter, + ruleData + ); // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor connection.v2.listEntities(filter).then( (response) => { From 08ca5dbb48a5e9514597dc2afe0abf8422dbdc27 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 6 Jun 2023 11:08:29 +0200 Subject: [PATCH 05/52] fix dateModified usage in filter of entites --- lib/models/entitiesStore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 1f99155a..2e4602cf 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -75,7 +75,7 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) service: service, servicepath: subservice, type: ruleData.type, - mq: ruleData.attribute + '.dateModified=' + Date.now() / 1000 - ruleData.reportInterval + mq: ruleData.attribute + '.dateModified=' + (Date.now() / 1000 - ruleData.reportInterval).toString() }; if (ruleData.id) { filter['id'] = ruleData.id; From ae7708dccb8da4e79995cb24448d2cb062ce64b7 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 8 Jun 2023 16:42:30 +0200 Subject: [PATCH 06/52] fix dateModified and func by entity --- lib/models/entitiesStore.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 2e4602cf..253618e8 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -75,7 +75,7 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) service: service, servicepath: subservice, type: ruleData.type, - mq: ruleData.attribute + '.dateModified=' + (Date.now() / 1000 - ruleData.reportInterval).toString() + mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString() }; if (ruleData.id) { filter['id'] = ruleData.id; @@ -100,7 +100,7 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) var count = 0; response.results.forEach((entity) => { logger.debug(context, 'silent entity %j', entity); - func(one); + func(entity); count++; }); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); From 2071248633f99576930037aadfefee1a5f38fb7a Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 9 Jun 2023 10:31:46 +0200 Subject: [PATCH 07/52] add expected _id subdoc to entity --- lib/models/entitiesStore.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 253618e8..fe9ea822 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -100,6 +100,11 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) var count = 0; response.results.forEach((entity) => { logger.debug(context, 'silent entity %j', entity); + // put expected subdoc 'id' by alertfunc + entity['_id'] = { + id: entity.id, + type: entity.type + }; func(entity); count++; }); From b9a40f29474f3e68d918bdf710762ebd1f0ea05c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 9 Jun 2023 12:57:34 +0200 Subject: [PATCH 08/52] update alterfunc to allow config.nonSignalByAPI --- lib/models/entitiesStore.js | 5 --- lib/models/noSignal.js | 62 +++++++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index fe9ea822..253618e8 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -100,11 +100,6 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) var count = 0; response.results.forEach((entity) => { logger.debug(context, 'silent entity %j', entity); - // put expected subdoc 'id' by alertfunc - entity['_id'] = { - id: entity.id, - type: entity.type - }; func(entity); count++; }); diff --git a/lib/models/noSignal.js b/lib/models/noSignal.js index 1de62fdc..c27db176 100644 --- a/lib/models/noSignal.js +++ b/lib/models/noSignal.js @@ -63,36 +63,63 @@ function alertFunc(nsLineRule, entity) { d.exit(); }); d.run(function() { + logger.debug(context, 'alertfunc nsLineRule %j entity %j ', nsLineRule, entity); // We duplicate info in event and event.ev for VR and non-VR action parameters var event = { service: nsLineRule[SERVICE], subservice: nsLineRule[SUBSERVICE], ruleName: nsLineRule[NAME], reportInterval: nsLineRule[REPORT_INTERVAL], - id: entity._id.id, - type: entity._id.type, internalCurrentTime: new Date().toISOString() }; - - // Search for modDate of the entity's attribute - // and copy every attribute (if not in event yet) - // for use in action template - Object.keys(entity.attrs).forEach(function(attrName) { - if (attrName === nsLineRule[ATTRIBUTE]) { + if (!config.nonSignalByAPI) { + // entity is really a entity doc obtained from mongo + event['id'] = entity._id.id; + event['type'] = entity._id.type; + logger.debug(context, 'alertfunc event %j ', event); + // Search for modDate of the entity's attribute + // and copy every attribute (if not in event yet) + // for use in action template + Object.keys(entity.attrs).forEach(function(attrName) { + if (attrName === nsLineRule[ATTRIBUTE]) { + try { + lastTime = new Date(entity.attrs[attrName].modDate * 1000).toISOString(); + } catch (ex) { + myutils.logErrorIf(ex, 'run ', d.context); + } + } + if (event[attrName] === undefined) { + if (entity.attrs[attrName].type === 'DateTime') { + event[attrName] = new Date(entity.attrs[attrName].value * 1000).toISOString(); + } else { + event[attrName] = entity.attrs[attrName].value; + } + } + }); + } else { + // entity is and NGSI object + event['id'] = entity.id; + event['type'] = entity.type; + logger.debug(context, 'alertfunc event %j ', event); + // Search for modDate of the entity's attribute + // and copy every attribute (if not in event yet) + // for use in action template + const attrName = nsLineRule[ATTRIBUTE]; + if (entity[attrName]) { try { - lastTime = new Date(entity.attrs[attrName].modDate * 1000).toISOString(); + lastTime = entity[attrName].metadata.TimeInstant.value; } catch (ex) { myutils.logErrorIf(ex, 'run ', d.context); } - } - if (event[attrName] === undefined) { - if (entity.attrs[attrName].type === 'DateTime') { - event[attrName] = new Date(entity.attrs[attrName].value * 1000).toISOString(); - } else { - event[attrName] = entity.attrs[attrName].value; + if (event[attrName] === undefined) { + if (entity[attrName].type === 'DateTime') { + event[attrName] = entity[attrName].metadata.TimeInstant.value; + } else { + event[attrName] = entity[attrName].value; + } } } - }); + } logger.debug(context, 'lastTime could be ', lastTime); if (lastTime !== undefined && lastTime !== null) { @@ -119,6 +146,7 @@ function checkNoSignal(period) { currentContext.srv = 'n/a'; currentContext.subsrv = 'n/a'; logger.debug(currentContext, 'Executing no-signal handler for period of %d (%d rules)', period, list.length); + list.forEach(function(nsrule) { currentContext.srv = nsrule[SERVICE]; currentContext.subsrv = nsrule[SUBSERVICE]; @@ -192,6 +220,7 @@ function addNSRule(service, subservice, name, nsr) { ); intervalAsNum = MIN_INTERVAL_MS; } + arrayRule = nsr2arr(service, subservice, name, nsr); nsRulesByInterval[nsr.checkInterval] = nsRulesByInterval[nsr.checkInterval] || []; nsRulesByInterval[nsr.checkInterval].forEach(function(element, index, array) { @@ -206,6 +235,7 @@ function addNSRule(service, subservice, name, nsr) { logger.debug(context, util.format('Adding no-signal rule (%s, %s, %s)', service, subservice, name)); } if (!checkers.hasOwnProperty(nsr.checkInterval)) { + logger.info(context, util.format('no-signal rule (%s, %s, %s)', service, subservice, name)); checkers[nsr.checkInterval] = setInterval(checkNoSignal, intervalAsNum, nsr.checkInterval); checkers[nsr.checkInterval].unref(); } From df171c0e25e6bcdf21651786663a5a53dc5d8c32 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 12 Jun 2023 10:04:43 +0200 Subject: [PATCH 09/52] change definition order --- lib/models/entitiesStore.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 253618e8..1d70f4be 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -35,14 +35,6 @@ function orionServiceDb(service) { return appContext.OrionDb(config.orionDb.prefix + '-' + service); } -function findSilentEntities(service, subservice, ruleData, func, callback) { - if (!config.nonSignalByAPI) { - return findSilentEntitiesByMongo(service, subservice, ruleData, func, callback); - } else { - return findSilentEntitiesByAPI(service, subservice, ruleData, func, callback); - } -} - function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) { var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; @@ -170,6 +162,14 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback ); } +function findSilentEntities(service, subservice, ruleData, func, callback) { + if (!config.nonSignalByAPI) { + return findSilentEntitiesByMongo(service, subservice, ruleData, func, callback); + } else { + return findSilentEntitiesByAPI(service, subservice, ruleData, func, callback); + } +} + module.exports = { FindSilentEntities: findSilentEntities }; From 0ac60c9b1d8d2e657b71452448b71a90b095a1ca Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 20 Jun 2023 14:09:59 +0200 Subject: [PATCH 10/52] findSilentEntities test --- .vscode/settings.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..e69de29b From c0465972a9b24495cfcec5fe7699e4f57353c99c Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 20 Jun 2023 14:10:39 +0200 Subject: [PATCH 11/52] findSilentEntities test --- lib/models/entitiesStore.js | 8 +- test/unit/entitiesStore_utest.js | 186 +++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 test/unit/entitiesStore_utest.js diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 1d70f4be..4d5242a9 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -164,12 +164,14 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback function findSilentEntities(service, subservice, ruleData, func, callback) { if (!config.nonSignalByAPI) { - return findSilentEntitiesByMongo(service, subservice, ruleData, func, callback); + return this.FindSilentEntitiesByMongo(service, subservice, ruleData, func, callback); } else { - return findSilentEntitiesByAPI(service, subservice, ruleData, func, callback); + return this.FindSilentEntitiesByAPI(service, subservice, ruleData, func, callback); } } module.exports = { - FindSilentEntities: findSilentEntities + FindSilentEntities: findSilentEntities, + FindSilentEntitiesByAPI: findSilentEntitiesByAPI, + FindSilentEntitiesByMongo: findSilentEntitiesByMongo }; diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js new file mode 100644 index 00000000..7c94280c --- /dev/null +++ b/test/unit/entitiesStore_utest.js @@ -0,0 +1,186 @@ +/* + * Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U + * + * This file is part of perseo-fe + * + * perseo-fe is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * perseo-fe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with perseo-fe. + * If not, see http://www.gnu.org/licenses/. + * + * For those usages not covered by the GNU Affero General Public License + * please contact with iot_support at tid dot es + * + * Created by: Carlos Blanco - Future Internet Consulting and Development Solutions (FICODES) + */ + +'use strict'; + +var should = require('should'); +var rewire = require('rewire'); +var entitiesStore = rewire('../../lib/models/entitiesStore.js'); +var assert = require('assert'); +var MongoClient = require('mongodb').MongoClient; +var chai = require('chai'); +var expect = chai.expect; +var sinon = require('sinon'); +var sinonChai = require('sinon-chai'); +var config = require('../../config.js'); +chai.Should(); +chai.use(sinonChai); + +/* +(service, subservice, ruleData, func, callback) +*/ + +describe('Test suit', function() { + var ruleData = { + name: 'NSR2', + action: { + type: 'update', + parameters: { + id: 'alarma:${id}', + type: 'Alarm', + attributes: [ + { + name: 'msg', + value: 'El status de ${id} es ${status}' + } + ] + } + }, + subservice: '/', + service: 'unknownt', + nosignal: { + checkInterval: '1', + attribute: 'temperature', + reportInterval: '5', + id: 'thing:disp1', + idRegexp: null, + type: 'thing' + } + }, + func = 'hola', + callback = { + function(e, request) { + should.exist(request); + should.not.exist(e); + should.equal(request.httpCode, 200); + done(); + } + }; + describe('FindSilentEntities test', () => { + it('By default should call FindSilentEntitiesByMongo', () => { + // Crear un stub de Sinon para la función findSilentEntitiesByMongo + var FindSilentEntitiesByMongoStub = sinon.stub(); + + // Configurar el módulo para que utilice el stub en lugar de la implementación real + entitiesStore.FindSilentEntitiesByMongo = FindSilentEntitiesByMongoStub; + + // Llamar a la función que se va a probar + entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); + + // Comprobar si la función FindSilentEntitiesByMongo ha sido llamada + expect(FindSilentEntitiesByMongoStub.calledOnce).to.be.true; + }); + it('If default settings are changed FindSilentEntitiesByAPI should be called', () => { + config.nonSignalByAPI = true; + // Crear un stub de Sinon para la función FindSilentEntitiesByAPI + var FindSilentEntitiesByAPIStub = sinon.stub(); + + // Configurar el módulo para que utilice el stub en lugar de la implementación real + entitiesStore.FindSilentEntitiesByAPI = FindSilentEntitiesByAPIStub; + + // Llamar a la función que se va a probar + entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); + + // Comprobar si la función FindSilentEntitiesByAPI ha sido llamada + expect(FindSilentEntitiesByAPIStub.calledOnce).to.be.true; + }); + }); + + describe('FindSilentEntitiesByMongo', function(done) { + it('should return a 200 status code and JSON object when everything is fine', function(done) { + // Mock the necessary dependencies + var dbMock = { + collection: sinon.stub().returnsThis(), + find: sinon.stub().returnsThis(), + batchSize: sinon.stub().returnsThis(), + each: function(callback) { + var entity = { _id: { id: 'entityId' } }; + callback(null, entity); + callback(null, null); // Cursor exhausted + } + }; + var appContextMock = { + OrionDb: sinon.stub().returns(dbMock) + }; + var configMock = { + orionDb: { + collection: 'entities' + } + }; + var loggerMock = { + debug: sinon.stub() + }; + + // Set up the test data + var service = 'service'; + var subservice = 'subservice'; + var ruleData = { + attribute: 'attribute', + reportInterval: 1000 + }; + var funcMock = sinon.stub(); + var expectedResponse = [{ _id: { id: 'entityId' } }]; + + // Set the mocks in the module + entitiesStore.__set__('appContext', appContextMock); + entitiesStore.__set__('config', configMock); + entitiesStore.__set__('logger', loggerMock); + done(); + + // Call the function to be tested + entitiesStore.FindSilentEntitiesByMongo(service, subservice, ruleData, funcMock, function(err, result) { + // Check the response + assert.strictEqual(err, null); + assert.deepStrictEqual(result, expectedResponse); + + // Check the function calls and arguments + assert.strictEqual(appContextMock.OrionDb.calledOnce, true); + assert.deepStrictEqual(appContextMock.OrionDb.firstCall.args, ['prefix-service']); + assert.strictEqual(dbMock.collection.calledOnce, true); + assert.deepStrictEqual(dbMock.collection.firstCall.args, ['entities']); + assert.strictEqual(dbMock.find.calledOnce, true); + assert.deepStrictEqual(dbMock.find.firstCall.args, [ + { + 'attrs.attribute.modDate': { $lt: Date.now() / 1000 - 1000 }, + '_id.servicePath': 'subservice' + } + ]); + assert.strictEqual(dbMock.batchSize.calledOnce, true); + assert.deepStrictEqual(dbMock.batchSize.firstCall.args, [100]); + assert.strictEqual(loggerMock.debug.calledOnce, true); + assert.strictEqual(loggerMock.debug.getCall(0).args[0].op, 'checkNoSignal.FindSilentEntitiesByMongo'); + assert.deepStrictEqual(loggerMock.debug.getCall(0).args[1], 'FindSilentEntities criterion %j'); + assert.deepStrictEqual(loggerMock.debug.getCall(0).args[2], { + 'attrs.attribute.modDate': { $lt: Date.now() / 1000 - 1000 }, + '_id.servicePath': 'subservice' + }); + assert.strictEqual(funcMock.calledOnce, true); + assert.deepStrictEqual(funcMock.firstCall.args[0], { _id: { id: 'entityId' } }); + + done(); + }); + }); + }); +}); From 82a49103d149f2eb90082207e4210dc115d4800c Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Wed, 21 Jun 2023 10:29:56 +0200 Subject: [PATCH 12/52] correcting linter issues --- lib/models/entitiesStore.js | 4 ++-- lib/models/noSignal.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 4d5242a9..1141762f 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -70,9 +70,9 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString() }; if (ruleData.id) { - filter['id'] = ruleData.id; + filter.id = ruleData.id; } else if (ruleData.idRegexp) { - filter['idPattern'] = ruleData.idRegexp; + filter.idPattern = ruleData.idRegexp; } logger.debug( context, diff --git a/lib/models/noSignal.js b/lib/models/noSignal.js index c27db176..a0e3cacd 100644 --- a/lib/models/noSignal.js +++ b/lib/models/noSignal.js @@ -74,8 +74,8 @@ function alertFunc(nsLineRule, entity) { }; if (!config.nonSignalByAPI) { // entity is really a entity doc obtained from mongo - event['id'] = entity._id.id; - event['type'] = entity._id.type; + event.id = entity._id.id; + event.type = entity._id.type; logger.debug(context, 'alertfunc event %j ', event); // Search for modDate of the entity's attribute // and copy every attribute (if not in event yet) @@ -98,8 +98,8 @@ function alertFunc(nsLineRule, entity) { }); } else { // entity is and NGSI object - event['id'] = entity.id; - event['type'] = entity.type; + event.id = entity.id; + event.type = entity.type; logger.debug(context, 'alertfunc event %j ', event); // Search for modDate of the entity's attribute // and copy every attribute (if not in event yet) From 801fa809a01e0b6a317cfd73305d9286fd6f5c25 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Wed, 21 Jun 2023 11:49:51 +0200 Subject: [PATCH 13/52] sugestions and improvements --- lib/models/entitiesStore.js | 8 +++----- test/unit/entitiesStore_utest.js | 21 ++++++++------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 1141762f..1f819759 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -170,8 +170,6 @@ function findSilentEntities(service, subservice, ruleData, func, callback) { } } -module.exports = { - FindSilentEntities: findSilentEntities, - FindSilentEntitiesByAPI: findSilentEntitiesByAPI, - FindSilentEntitiesByMongo: findSilentEntitiesByMongo -}; +module.exports.FindSilentEntities = findSilentEntities; +module.exports.FindSilentEntitiesByAPI = findSilentEntitiesByAPI; +module.exports.FindSilentEntitiesByMongo = findSilentEntitiesByMongo; diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 7c94280c..54f2b9dd 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -29,7 +29,6 @@ var should = require('should'); var rewire = require('rewire'); var entitiesStore = rewire('../../lib/models/entitiesStore.js'); var assert = require('assert'); -var MongoClient = require('mongodb').MongoClient; var chai = require('chai'); var expect = chai.expect; var sinon = require('sinon'); @@ -38,10 +37,6 @@ var config = require('../../config.js'); chai.Should(); chai.use(sinonChai); -/* -(service, subservice, ruleData, func, callback) -*/ - describe('Test suit', function() { var ruleData = { name: 'NSR2', @@ -80,30 +75,30 @@ describe('Test suit', function() { }; describe('FindSilentEntities test', () => { it('By default should call FindSilentEntitiesByMongo', () => { - // Crear un stub de Sinon para la función findSilentEntitiesByMongo + // Create a Sinon stub for the function findSilentEntitiesByMongo var FindSilentEntitiesByMongoStub = sinon.stub(); - // Configurar el módulo para que utilice el stub en lugar de la implementación real + // Configure the module to use the stub instead of the actual implementation entitiesStore.FindSilentEntitiesByMongo = FindSilentEntitiesByMongoStub; - // Llamar a la función que se va a probar + // Calling the function to be tested entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); - // Comprobar si la función FindSilentEntitiesByMongo ha sido llamada + // Check if the function FindSilentEntitiesByMongo has been call expect(FindSilentEntitiesByMongoStub.calledOnce).to.be.true; }); it('If default settings are changed FindSilentEntitiesByAPI should be called', () => { config.nonSignalByAPI = true; - // Crear un stub de Sinon para la función FindSilentEntitiesByAPI + // Create a Sinon stub for the function FindSilentEntitiesByAPI var FindSilentEntitiesByAPIStub = sinon.stub(); - // Configurar el módulo para que utilice el stub en lugar de la implementación real + // Configure the module to use the stub instead of the actual implementation entitiesStore.FindSilentEntitiesByAPI = FindSilentEntitiesByAPIStub; - // Llamar a la función que se va a probar + // Calling the function to be tested entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); - // Comprobar si la función FindSilentEntitiesByAPI ha sido llamada + // Check if the function FindSilentEntitiesByAPI has been call expect(FindSilentEntitiesByAPIStub.calledOnce).to.be.true; }); }); From b7168c1518f4fad09c34588c9ed9b20d7e2165eb Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Thu, 22 Jun 2023 12:07:20 +0200 Subject: [PATCH 14/52] u test and linter problems correction --- lib/models/entitiesStore.js | 8 +- test/unit/entitiesStore_utest.js | 127 +++++-------------------------- 2 files changed, 21 insertions(+), 114 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 1f819759..e16718ca 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -164,12 +164,12 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback function findSilentEntities(service, subservice, ruleData, func, callback) { if (!config.nonSignalByAPI) { - return this.FindSilentEntitiesByMongo(service, subservice, ruleData, func, callback); + return findSilentEntitiesByMongo(service, subservice, ruleData, func, callback); } else { - return this.FindSilentEntitiesByAPI(service, subservice, ruleData, func, callback); + return findSilentEntitiesByAPI(service, subservice, ruleData, func, callback); } } module.exports.FindSilentEntities = findSilentEntities; -module.exports.FindSilentEntitiesByAPI = findSilentEntitiesByAPI; -module.exports.FindSilentEntitiesByMongo = findSilentEntitiesByMongo; +module.exports.findSilentEntitiesByAPI = findSilentEntitiesByAPI; +module.exports.findSilentEntitiesByMongo = findSilentEntitiesByMongo; diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 54f2b9dd..0ab224ff 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -64,118 +64,25 @@ describe('Test suit', function() { type: 'thing' } }, - func = 'hola', - callback = { - function(e, request) { - should.exist(request); - should.not.exist(e); - should.equal(request.httpCode, 200); - done(); - } + func = 'sinon.stub()', + callback = function(e, request) { + should.exist(request); + should.not.exist(e); + should.equal(request.httpCode, 200); + done(); }; - describe('FindSilentEntities test', () => { - it('By default should call FindSilentEntitiesByMongo', () => { - // Create a Sinon stub for the function findSilentEntitiesByMongo - var FindSilentEntitiesByMongoStub = sinon.stub(); - - // Configure the module to use the stub instead of the actual implementation - entitiesStore.FindSilentEntitiesByMongo = FindSilentEntitiesByMongoStub; - - // Calling the function to be tested - entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); - // Check if the function FindSilentEntitiesByMongo has been call - expect(FindSilentEntitiesByMongoStub.calledOnce).to.be.true; - }); - it('If default settings are changed FindSilentEntitiesByAPI should be called', () => { - config.nonSignalByAPI = true; - // Create a Sinon stub for the function FindSilentEntitiesByAPI - var FindSilentEntitiesByAPIStub = sinon.stub(); - - // Configure the module to use the stub instead of the actual implementation - entitiesStore.FindSilentEntitiesByAPI = FindSilentEntitiesByAPIStub; - - // Calling the function to be tested - entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); - - // Check if the function FindSilentEntitiesByAPI has been call - expect(FindSilentEntitiesByAPIStub.calledOnce).to.be.true; - }); + it('By default should call findSilentEntitiesByMongo', function() { + var findSilentEntitiesByMongoSpy = sinon.spy(); + entitiesStore.__set__('findSilentEntitiesByMongo', findSilentEntitiesByMongoSpy); + entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); + sinon.assert.calledOnce(findSilentEntitiesByMongoSpy); }); - - describe('FindSilentEntitiesByMongo', function(done) { - it('should return a 200 status code and JSON object when everything is fine', function(done) { - // Mock the necessary dependencies - var dbMock = { - collection: sinon.stub().returnsThis(), - find: sinon.stub().returnsThis(), - batchSize: sinon.stub().returnsThis(), - each: function(callback) { - var entity = { _id: { id: 'entityId' } }; - callback(null, entity); - callback(null, null); // Cursor exhausted - } - }; - var appContextMock = { - OrionDb: sinon.stub().returns(dbMock) - }; - var configMock = { - orionDb: { - collection: 'entities' - } - }; - var loggerMock = { - debug: sinon.stub() - }; - - // Set up the test data - var service = 'service'; - var subservice = 'subservice'; - var ruleData = { - attribute: 'attribute', - reportInterval: 1000 - }; - var funcMock = sinon.stub(); - var expectedResponse = [{ _id: { id: 'entityId' } }]; - - // Set the mocks in the module - entitiesStore.__set__('appContext', appContextMock); - entitiesStore.__set__('config', configMock); - entitiesStore.__set__('logger', loggerMock); - done(); - - // Call the function to be tested - entitiesStore.FindSilentEntitiesByMongo(service, subservice, ruleData, funcMock, function(err, result) { - // Check the response - assert.strictEqual(err, null); - assert.deepStrictEqual(result, expectedResponse); - - // Check the function calls and arguments - assert.strictEqual(appContextMock.OrionDb.calledOnce, true); - assert.deepStrictEqual(appContextMock.OrionDb.firstCall.args, ['prefix-service']); - assert.strictEqual(dbMock.collection.calledOnce, true); - assert.deepStrictEqual(dbMock.collection.firstCall.args, ['entities']); - assert.strictEqual(dbMock.find.calledOnce, true); - assert.deepStrictEqual(dbMock.find.firstCall.args, [ - { - 'attrs.attribute.modDate': { $lt: Date.now() / 1000 - 1000 }, - '_id.servicePath': 'subservice' - } - ]); - assert.strictEqual(dbMock.batchSize.calledOnce, true); - assert.deepStrictEqual(dbMock.batchSize.firstCall.args, [100]); - assert.strictEqual(loggerMock.debug.calledOnce, true); - assert.strictEqual(loggerMock.debug.getCall(0).args[0].op, 'checkNoSignal.FindSilentEntitiesByMongo'); - assert.deepStrictEqual(loggerMock.debug.getCall(0).args[1], 'FindSilentEntities criterion %j'); - assert.deepStrictEqual(loggerMock.debug.getCall(0).args[2], { - 'attrs.attribute.modDate': { $lt: Date.now() / 1000 - 1000 }, - '_id.servicePath': 'subservice' - }); - assert.strictEqual(funcMock.calledOnce, true); - assert.deepStrictEqual(funcMock.firstCall.args[0], { _id: { id: 'entityId' } }); - - done(); - }); - }); + it('If default settings are changed FindSilentEntitiesByAPI should be called', function() { + config.nonSignalByAPI = true; + var findSilentEntitiesByAPISpy = sinon.spy(); + entitiesStore.__set__('findSilentEntitiesByAPI', findSilentEntitiesByAPISpy); + entitiesStore.FindSilentEntities(); + sinon.assert.calledOnce(findSilentEntitiesByAPISpy); }); }); From dcf84ddddcd0e98cca1fbaf8f4662f1d334796b7 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Thu, 22 Jun 2023 12:17:42 +0200 Subject: [PATCH 15/52] linter correction --- test/unit/entitiesStore_utest.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 0ab224ff..ff64e3dd 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -28,9 +28,8 @@ var should = require('should'); var rewire = require('rewire'); var entitiesStore = rewire('../../lib/models/entitiesStore.js'); -var assert = require('assert'); +//var assert = require('assert'); var chai = require('chai'); -var expect = chai.expect; var sinon = require('sinon'); var sinonChai = require('sinon-chai'); var config = require('../../config.js'); @@ -69,7 +68,6 @@ describe('Test suit', function() { should.exist(request); should.not.exist(e); should.equal(request.httpCode, 200); - done(); }; it('By default should call findSilentEntitiesByMongo', function() { From f71575e3607ccd74944d4d2a4a03942c02884d9b Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 26 Jun 2023 12:27:15 +0200 Subject: [PATCH 16/52] findSilentEntitiesByAPI test --- test/unit/entitiesStore_utest.js | 128 ++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 44 deletions(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index ff64e3dd..05687be9 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -28,59 +28,99 @@ var should = require('should'); var rewire = require('rewire'); var entitiesStore = rewire('../../lib/models/entitiesStore.js'); -//var assert = require('assert'); var chai = require('chai'); var sinon = require('sinon'); var sinonChai = require('sinon-chai'); +var expect = chai.expect; var config = require('../../config.js'); +var NGSI = require('ngsijs'); chai.Should(); chai.use(sinonChai); -describe('Test suit', function() { - var ruleData = { - name: 'NSR2', - action: { - type: 'update', - parameters: { - id: 'alarma:${id}', - type: 'Alarm', - attributes: [ - { - name: 'msg', - value: 'El status de ${id} es ${status}' - } - ] +describe('entitiesStore', function() { + describe('#findSilentEntitiesByAPI', function() { + var connectionMock; + var listEntitiesMock; + + beforeEach(function() { + connectionMock = sinon.stub(NGSI, 'Connection'); + listEntitiesMock = sinon.stub(); + + // Mock the Connection function + connectionMock.returns({ v2: { listEntities: listEntitiesMock } }); + }); + + afterEach(function() { + // Restore the original function after each test + connectionMock.restore(); + }); + + var ruleData = { + name: 'NSR2', + action: { + type: 'update', + parameters: { + id: 'alarma:${id}', + type: 'Alarm', + attributes: [ + { + name: 'msg', + value: 'El status de ${id} es ${status}' + } + ] + } + }, + subservice: '/', + service: 'unknownt', + nosignal: { + checkInterval: '1', + attribute: 'temperature', + reportInterval: '5', + id: 'thing:disp1', + idRegexp: null, + type: 'thing' } }, - subservice: '/', - service: 'unknownt', - nosignal: { - checkInterval: '1', - attribute: 'temperature', - reportInterval: '5', - id: 'thing:disp1', - idRegexp: null, - type: 'thing' - } - }, - func = 'sinon.stub()', - callback = function(e, request) { - should.exist(request); - should.not.exist(e); - should.equal(request.httpCode, 200); - }; + func = 'sinon.stub()', + callback = function(e, request) { + should.exist(request); + should.not.exist(e); + should.equal(request.httpCode, 200); + }; - it('By default should call findSilentEntitiesByMongo', function() { - var findSilentEntitiesByMongoSpy = sinon.spy(); - entitiesStore.__set__('findSilentEntitiesByMongo', findSilentEntitiesByMongoSpy); - entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); - sinon.assert.calledOnce(findSilentEntitiesByMongoSpy); - }); - it('If default settings are changed FindSilentEntitiesByAPI should be called', function() { - config.nonSignalByAPI = true; - var findSilentEntitiesByAPISpy = sinon.spy(); - entitiesStore.__set__('findSilentEntitiesByAPI', findSilentEntitiesByAPISpy); - entitiesStore.FindSilentEntities(); - sinon.assert.calledOnce(findSilentEntitiesByAPISpy); + it('By default should call findSilentEntitiesByMongo', function() { + var findSilentEntitiesByMongoSpy = sinon.spy(); + entitiesStore.__set__('findSilentEntitiesByMongo', findSilentEntitiesByMongoSpy); + entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); + sinon.assert.calledOnce(findSilentEntitiesByMongoSpy); + }); + + it('If default settings are changed FindSilentEntitiesByAPI should be called', function() { + config.nonSignalByAPI = true; + var findSilentEntitiesByAPISpy = sinon.spy(); + entitiesStore.__set__('findSilentEntitiesByAPI', findSilentEntitiesByAPISpy); + entitiesStore.FindSilentEntities(); + sinon.assert.calledOnce(findSilentEntitiesByAPISpy); + }); + + it('should return silent entities', async function() { + var funcM = sinon.spy(), + callbackM = sinon.spy(); + + // Mock the listEntities function to resolve with a test response + listEntitiesMock.returns(Promise.resolve({ results: [] })); + + await entitiesStore.findSilentEntitiesByAPI( + ruleData.service, + ruleData.subservice, + ruleData, + funcM, + callbackM + ); + + expect(listEntitiesMock.calledOnce).to.be.true; + expect(funcM.callCount).to.equal(0); + expect(callbackM.calledOnceWith(null, [])).to.be.true; + }); }); }); From b0958b9131ffbb58c84f5804c854c53826f2076a Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 26 Jun 2023 12:37:06 +0200 Subject: [PATCH 17/52] linter problems From cabcd0d6097b86d9d3b56fad49f6a4c9d1d5fb61 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 26 Jun 2023 13:04:00 +0200 Subject: [PATCH 18/52] linter skip line --- test/unit/entitiesStore_utest.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 05687be9..0d3f3b3d 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -104,6 +104,7 @@ describe('entitiesStore', function() { }); it('should return silent entities', async function() { + // jshint ignore:line var funcM = sinon.spy(), callbackM = sinon.spy(); @@ -121,6 +122,6 @@ describe('entitiesStore', function() { expect(listEntitiesMock.calledOnce).to.be.true; expect(funcM.callCount).to.equal(0); expect(callbackM.calledOnceWith(null, [])).to.be.true; - }); + }); // jshint ignore:line }); }); From 4fd3bf0fd4f8449fb69c2f1f735940cb896a5253 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 26 Jun 2023 13:16:32 +0200 Subject: [PATCH 19/52] linter skip line --- test/unit/entitiesStore_utest.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 0d3f3b3d..68d00617 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -102,9 +102,8 @@ describe('entitiesStore', function() { entitiesStore.FindSilentEntities(); sinon.assert.calledOnce(findSilentEntitiesByAPISpy); }); - + // eslint-disable-next-line it('should return silent entities', async function() { - // jshint ignore:line var funcM = sinon.spy(), callbackM = sinon.spy(); From b3ef4fe30740664621d3f08e4dc66865863c43bf Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 26 Jun 2023 13:28:44 +0200 Subject: [PATCH 20/52] linter, skip section --- test/unit/entitiesStore_utest.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 68d00617..438fd9c2 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -102,7 +102,8 @@ describe('entitiesStore', function() { entitiesStore.FindSilentEntities(); sinon.assert.calledOnce(findSilentEntitiesByAPISpy); }); - // eslint-disable-next-line + / jshint ignore:start /; + it('should return silent entities', async function() { var funcM = sinon.spy(), callbackM = sinon.spy(); @@ -117,10 +118,10 @@ describe('entitiesStore', function() { funcM, callbackM ); - + / jshint ignore:end /; expect(listEntitiesMock.calledOnce).to.be.true; expect(funcM.callCount).to.equal(0); expect(callbackM.calledOnceWith(null, [])).to.be.true; - }); // jshint ignore:line + }); }); }); From 3cdf6cec426fdd0c5faa67e79c9ce2cd6e031773 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 26 Jun 2023 13:44:14 +0200 Subject: [PATCH 21/52] skip section --- test/unit/entitiesStore_utest.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 438fd9c2..0636f7af 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -102,7 +102,7 @@ describe('entitiesStore', function() { entitiesStore.FindSilentEntities(); sinon.assert.calledOnce(findSilentEntitiesByAPISpy); }); - / jshint ignore:start /; + /* jshint ignore:start */ it('should return silent entities', async function() { var funcM = sinon.spy(), @@ -118,7 +118,7 @@ describe('entitiesStore', function() { funcM, callbackM ); - / jshint ignore:end /; + /* jshint ignore:end */ expect(listEntitiesMock.calledOnce).to.be.true; expect(funcM.callCount).to.equal(0); expect(callbackM.calledOnceWith(null, [])).to.be.true; From c88639f3e8e1e8471abeb74b619b1f04c313228b Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 26 Jun 2023 14:58:33 +0200 Subject: [PATCH 22/52] linter, suggested changes --- test/unit/entitiesStore_utest.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 0636f7af..0082cf50 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -31,7 +31,6 @@ var entitiesStore = rewire('../../lib/models/entitiesStore.js'); var chai = require('chai'); var sinon = require('sinon'); var sinonChai = require('sinon-chai'); -var expect = chai.expect; var config = require('../../config.js'); var NGSI = require('ngsijs'); chai.Should(); @@ -105,6 +104,7 @@ describe('entitiesStore', function() { /* jshint ignore:start */ it('should return silent entities', async function() { + var expect = chai.expect; var funcM = sinon.spy(), callbackM = sinon.spy(); @@ -118,10 +118,10 @@ describe('entitiesStore', function() { funcM, callbackM ); - /* jshint ignore:end */ expect(listEntitiesMock.calledOnce).to.be.true; expect(funcM.callCount).to.equal(0); expect(callbackM.calledOnceWith(null, [])).to.be.true; }); }); + /*jshint ignore:end */ }); From d2038d250afc088f60588fe1753c34344383ee5a Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 26 Jun 2023 15:01:52 +0200 Subject: [PATCH 23/52] unmatch { --- test/unit/entitiesStore_utest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 0082cf50..96046938 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -122,6 +122,6 @@ describe('entitiesStore', function() { expect(funcM.callCount).to.equal(0); expect(callbackM.calledOnceWith(null, [])).to.be.true; }); + /*jshint ignore:end */ }); - /*jshint ignore:end */ }); From 18c2c89094863182a06bf6fe459ac414531887da Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 27 Jun 2023 08:48:34 +0200 Subject: [PATCH 24/52] CNR and eliminate uncompleted test --- CHANGES_NEXT_RELEASE | 2 ++ test/unit/nsr_utest.js | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 8b137891..5f11ddb6 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,3 @@ +- Allow access entities using NGSIv2 API for non_signal rules (#549) + diff --git a/test/unit/nsr_utest.js b/test/unit/nsr_utest.js index d719a2fa..9816b586 100644 --- a/test/unit/nsr_utest.js +++ b/test/unit/nsr_utest.js @@ -137,13 +137,4 @@ describe('noSignal', function() { should.deepEqual(noSignal.Nsr2Arr(rule.service, rule.subservice, rule.name, rule.nosignal), updatedRule); }); }); - - describe('#checkNoSignal()', function() { - it('check no signal ', function() { - var gotNSRule; - should.notEqual(noSignal.AddNSRule(rule2.service, rule2.subservice, rule2.name, rule2.nosignal), 0); - gotNSRule = noSignal.GetNSArrRule(rule2.service, rule2.subservice, rule2.name); - //console.log("RULE: " + JSON.stringify(gotNSRule)); - }); - }); }); From bb3f154aeb6200d9e5a03d6058006ae19f612439 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 27 Jun 2023 08:52:19 +0200 Subject: [PATCH 25/52] removing unused rule --- test/unit/nsr_utest.js | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/test/unit/nsr_utest.js b/test/unit/nsr_utest.js index 9816b586..4fae2376 100644 --- a/test/unit/nsr_utest.js +++ b/test/unit/nsr_utest.js @@ -47,32 +47,6 @@ describe('noSignal', function() { type: null } }; - var rule2 = { - name: 'NSR2', - action: { - type: 'update', - parameters: { - id: 'alarma:${id}', - type: 'Alarm', - attributes: [ - { - name: 'msg', - value: 'El status de ${id} es ${status}' - } - ] - } - }, - subservice: '/', - service: 'unknownt', - nosignal: { - checkInterval: '1', - attribute: 'temperature', - reportInterval: '5', - id: 'thing:disp1', - idRegexp: null, - type: 'thing' - } - }; describe('#addNSRule()', function() { it('should reject to add a rule with invalid check interval', function() { From a289a3da12fda056d50781215a3d3dd78a50a5f7 Mon Sep 17 00:00:00 2001 From: RafaelM3 <134268825+RafaelM3@users.noreply.github.com> Date: Tue, 27 Jun 2023 09:58:28 +0200 Subject: [PATCH 26/52] Update configuration.md --- docs/admin/configuration.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/admin/configuration.md b/docs/admin/configuration.md index a9d2685e..f3be64c5 100644 --- a/docs/admin/configuration.md +++ b/docs/admin/configuration.md @@ -49,10 +49,12 @@ The following table shows the environment variables available for Perseo configu | PERSEO_AUTHENTICATION_USER | User to perform authentication | | PERSEO_AUTHENTICATION_PASSWORD | Password for the user to perform authentication | | PERSEO_AUTHENTICATION_SERVICE | Keystone User Service to perform authentication | -| PERSEO_CAST_TYPE | If true, enable attribute value casting based in NGSI-v2 attribute types if true. If false (default), the JSON native type for the attribute value is used. | +| PERSEO_CAST_TYPE | Maximum number of rule executions triggered by the same notification (with the same correlator) until refuse execute a rule. Default is 20. See [loop detection tips](https://github.com/telefonicaid/perseo-fe/blob/master/docs/architecture/architecture.md#loop-detection) | +| PERSEO_MAX_RULES_BY_CORR | If true, enable attribute value casting based in NGSI-v2 attribute types if true. If false (default), the JSON native type for the attribute value is used. | +| PERSEO_CHECK_NON_SIGNAL_BY_API | If false, the entities will be directly find using MongoDB, if true they will be fing using NGSIv2 API + -| PERSEO_MAX_RULES_BY_CORR | Maximum number of rule executions triggered by the same notification (with the same correlator) until refuse execute a rule. Default is 20. See [loop detection tips](https://github.com/telefonicaid/perseo-fe/blob/master/docs/architecture/architecture.md#loop-detection) -| + ### Basic Configuration @@ -91,6 +93,7 @@ In order to have perseo running, there are several basic pieces of information t - `config.checkDB.bufferMaxEntries`: Number of operations buffered up before giving up on getting a working connection (see [database aspects](admin.md#database-aspects) documentation for mode detail). - `config.maxRulesByCorr`: Maximum number of rule executions triggered by the same notification (with the same correlator) until refuse execute a rule. Defualt is 20 +- `config.nonSignalByAPI`: Flag value, determine the use of the NGSIv2 API or MongoDB. By default is false. Options for HA: From c036794ebdbb46e6ba954291cba21c385e5219e6 Mon Sep 17 00:00:00 2001 From: RafaelM3 <134268825+RafaelM3@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:05:42 +0200 Subject: [PATCH 27/52] Update configuration.md --- docs/admin/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/admin/configuration.md b/docs/admin/configuration.md index f3be64c5..30506f02 100644 --- a/docs/admin/configuration.md +++ b/docs/admin/configuration.md @@ -51,9 +51,9 @@ The following table shows the environment variables available for Perseo configu | PERSEO_AUTHENTICATION_SERVICE | Keystone User Service to perform authentication | | PERSEO_CAST_TYPE | Maximum number of rule executions triggered by the same notification (with the same correlator) until refuse execute a rule. Default is 20. See [loop detection tips](https://github.com/telefonicaid/perseo-fe/blob/master/docs/architecture/architecture.md#loop-detection) | | PERSEO_MAX_RULES_BY_CORR | If true, enable attribute value casting based in NGSI-v2 attribute types if true. If false (default), the JSON native type for the attribute value is used. | -| PERSEO_CHECK_NON_SIGNAL_BY_API | If false, the entities will be directly find using MongoDB, if true they will be fing using NGSIv2 API +| PERSEO_CHECK_NON_SIGNAL_BY_API | If false, the entities will be directly find using MongoDB, if true they will be fing using NGSIv2 API* - +'*' Although the idea is that now Perseo works always using NGSIv2 API to check times for non signal rules, this setting has been introduced to ease a potential rollback if some problem is found with this new approach. Thus, I has to be considered a kind of temporal debugging settings, to be removed when we get confident with the new functionality, thus you shouldn't use it with care. ### Basic Configuration From 184f6b7a3759d36dc5ee07f176925d9fdc049b42 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 27 Jun 2023 12:51:49 +0200 Subject: [PATCH 28/52] small impact suggested changes --- CHANGES_NEXT_RELEASE | 2 +- lib/models/entitiesStore.js | 6 +++--- test/unit/entitiesStore_utest.js | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 5f11ddb6..d122660d 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,3 @@ -- Allow access entities using NGSIv2 API for non_signal rules (#549) +- Allow access entities using NGSIv2 API for non_signal rules (new setting nonSignalByAPI / PERSEO_CHECK_NON_SIGNAL_BY_API) (#549) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index e16718ca..6b91277d 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -29,7 +29,7 @@ var async = require('async'), myutils = require('../myutils'), constants = require('../constants'), logger = require('logops'), - NGSI = require('ngsijs'); + ngsi = require('ngsijs'); function orionServiceDb(service) { return appContext.OrionDb(config.orionDb.prefix + '-' + service); @@ -61,7 +61,7 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) options.headers[constants.REALIP_HEADER] = domain.context.from; } } - var connection = new NGSI.Connection(config.orion.URL, options); + var connection = new ngsi.Connection(config.orion.URL, options); var filter = { service: service, @@ -74,7 +74,7 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) } else if (ruleData.idRegexp) { filter.idPattern = ruleData.idRegexp; } - logger.debug( + logger.info( context, 'find silent entities by API ngsi using options %j and filter %j and rule %j', options, diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 96046938..2fb2fbb9 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -32,7 +32,7 @@ var chai = require('chai'); var sinon = require('sinon'); var sinonChai = require('sinon-chai'); var config = require('../../config.js'); -var NGSI = require('ngsijs'); +var ngsi = require('ngsijs'); chai.Should(); chai.use(sinonChai); @@ -42,7 +42,7 @@ describe('entitiesStore', function() { var listEntitiesMock; beforeEach(function() { - connectionMock = sinon.stub(NGSI, 'Connection'); + connectionMock = sinon.stub(ngsi, 'Connection'); listEntitiesMock = sinon.stub(); // Mock the Connection function From b0ba12e4c9a979b733fe7acbd108923fa0e4c257 Mon Sep 17 00:00:00 2001 From: RafaelM3 <134268825+RafaelM3@users.noreply.github.com> Date: Tue, 27 Jun 2023 13:14:14 +0200 Subject: [PATCH 29/52] Update configuration.md --- docs/admin/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/admin/configuration.md b/docs/admin/configuration.md index 30506f02..44278929 100644 --- a/docs/admin/configuration.md +++ b/docs/admin/configuration.md @@ -51,9 +51,9 @@ The following table shows the environment variables available for Perseo configu | PERSEO_AUTHENTICATION_SERVICE | Keystone User Service to perform authentication | | PERSEO_CAST_TYPE | Maximum number of rule executions triggered by the same notification (with the same correlator) until refuse execute a rule. Default is 20. See [loop detection tips](https://github.com/telefonicaid/perseo-fe/blob/master/docs/architecture/architecture.md#loop-detection) | | PERSEO_MAX_RULES_BY_CORR | If true, enable attribute value casting based in NGSI-v2 attribute types if true. If false (default), the JSON native type for the attribute value is used. | -| PERSEO_CHECK_NON_SIGNAL_BY_API | If false, the entities will be directly find using MongoDB, if true they will be fing using NGSIv2 API* +| PERSEO_CHECK_NON_SIGNAL_BY_API | If false, the entities will be directly find using MongoDB, if true they will be fing using NGSIv2 API(*) -'*' Although the idea is that now Perseo works always using NGSIv2 API to check times for non signal rules, this setting has been introduced to ease a potential rollback if some problem is found with this new approach. Thus, I has to be considered a kind of temporal debugging settings, to be removed when we get confident with the new functionality, thus you shouldn't use it with care. +(*)Although the idea is that now Perseo works always using NGSIv2 API to check times for non signal rules, this setting has been introduced to ease a potential rollback if some problem is found with this new approach. Thus, I has to be considered a kind of temporal debugging settings, to be removed when we get confident with the new functionality, thus you shouldn't use it with care. ### Basic Configuration From cb70ed8808458d986be3ac2119e1eb90a7b67adc Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Wed, 28 Jun 2023 13:37:16 +0200 Subject: [PATCH 30/52] findSilentEntitiesByAPIWithPagination --- lib/models/entitiesStore.js | 64 ++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 6b91277d..a58f23c0 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -35,9 +35,25 @@ function orionServiceDb(service) { return appContext.OrionDb(config.orionDb.prefix + '-' + service); } -function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) { - var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; +function createFilter(ruleData, service, subservice, limit, offset) { + var filter = { + service: service, + servicepath: subservice, + type: ruleData.type, + mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString(), + options: 'count', + limit: limit, + offset: offset + }; + if (ruleData.id) { + filter.id = ruleData.id; + } else if (ruleData.idRegexp) { + filter.idPattern = ruleData.idRegexp; + } + return filter; +} +function createConnection() { var options = { service: service, servicepath: subservice @@ -61,26 +77,31 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) options.headers[constants.REALIP_HEADER] = domain.context.from; } } - var connection = new ngsi.Connection(config.orion.URL, options); + return new ngsi.Connection(config.orion.URL, options); +} + +function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) { + var limit = limit || 20; + var offset = offset || 0; + + var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; + + var connection = createConnection(service, subservice); + var filter = createFilter(ruleData, service, subservice, limit, offset); - var filter = { - service: service, - servicepath: subservice, - type: ruleData.type, - mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString() - }; - if (ruleData.id) { - filter.id = ruleData.id; - } else if (ruleData.idRegexp) { - filter.idPattern = ruleData.idRegexp; - } logger.info( context, 'find silent entities by API ngsi using options %j and filter %j and rule %j', - options, + connection, filter, ruleData ); + + // Call the pagination function + findSilentEntitiesByAPIWithPagination(connection, filter, func, callback); +} + +function findSilentEntitiesByAPIWithPagination(connection, filter, func, callback) { // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor connection.v2.listEntities(filter).then( (response) => { @@ -96,12 +117,17 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) count++; }); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); - callback(null, response.results); + // Check if there are more entities to retrieve + if (response.count > filter.limit + filter.offset) { + // Call the function again with updated offset + filter.offset += filter.limit; + findSilentEntitiesByAPIWithPagination(connection, filter, func, callback); + } else { + // All entities have been retrieved and processed, call the callback + callback(null, response.results); + } }, (error) => { - // Error retrieving entities - // If the error was reported by Orion, error.correlator will be - // filled with the associated transaction id logger.warn('error v2.listEntities: %j trying list entities using filter %j', error, filter); callback(error, null); } From 04543fafdb47ef203f6f745ad3dbabafcd2885f6 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Wed, 28 Jun 2023 14:35:08 +0200 Subject: [PATCH 31/52] createConnection test --- lib/models/entitiesStore.js | 4 +++- test/unit/entitiesStore_utest.js | 28 ++++++++++------------------ 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index a58f23c0..10ced9ba 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -53,7 +53,7 @@ function createFilter(ruleData, service, subservice, limit, offset) { return filter; } -function createConnection() { +function createConnection(service, subservice) { var options = { service: service, servicepath: subservice @@ -199,3 +199,5 @@ function findSilentEntities(service, subservice, ruleData, func, callback) { module.exports.FindSilentEntities = findSilentEntities; module.exports.findSilentEntitiesByAPI = findSilentEntitiesByAPI; module.exports.findSilentEntitiesByMongo = findSilentEntitiesByMongo; +module.exports.findSilentEntitiesByAPIWithPagination = findSilentEntitiesByAPIWithPagination; +module.exports.createConnection = createConnection; diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 2fb2fbb9..689904a9 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -101,27 +101,19 @@ describe('entitiesStore', function() { entitiesStore.FindSilentEntities(); sinon.assert.calledOnce(findSilentEntitiesByAPISpy); }); - /* jshint ignore:start */ - it('should return silent entities', async function() { - var expect = chai.expect; - var funcM = sinon.spy(), - callbackM = sinon.spy(); + it('should correctly create a connection', function() { + var service = 'Service'; + var subservice = 'Subservice'; - // Mock the listEntities function to resolve with a test response - listEntitiesMock.returns(Promise.resolve({ results: [] })); + entitiesStore.createConnection(service, subservice); - await entitiesStore.findSilentEntitiesByAPI( - ruleData.service, - ruleData.subservice, - ruleData, - funcM, - callbackM - ); - expect(listEntitiesMock.calledOnce).to.be.true; - expect(funcM.callCount).to.equal(0); - expect(callbackM.calledOnceWith(null, [])).to.be.true; + sinon.assert.calledOnce(connectionMock); + sinon.assert.calledWithExactly(connectionMock, config.orion.URL, { + service: service, + servicepath: subservice, + headers: {} + }); }); - /*jshint ignore:end */ }); }); From 7f16995ee0c30cc99585a944d7256c54b3875835 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Wed, 28 Jun 2023 17:28:26 +0200 Subject: [PATCH 32/52] findSilentEntitiesByAPI test --- package.json | 2 +- test/unit/entitiesStore_utest.js | 128 ++++++++++++++----------------- 2 files changed, 58 insertions(+), 72 deletions(-) diff --git a/package.json b/package.json index 155be63f..a3a62456 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "clean": "rm -rf package-lock.json && rm -rf node_modules && rm -rf coverage", "healthcheck": "node ./bin/healthcheck", - "test": "mocha --recursive 'test/**/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", + "test": "mocha --recursive 'test/unit/entitiesStore_utest.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", "test:watch": "npm run test -- -w ./lib", "lint": "jshint lib/ --config .jshintrc && jshint bin/ --config .jshintrc && jshint test/ --config test/.jshintrc", "lint:md": "remark -f '*.md' 'docs/*.md'", diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 689904a9..d5714566 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -37,83 +37,69 @@ chai.Should(); chai.use(sinonChai); describe('entitiesStore', function() { - describe('#findSilentEntitiesByAPI', function() { - var connectionMock; - var listEntitiesMock; - - beforeEach(function() { - connectionMock = sinon.stub(ngsi, 'Connection'); - listEntitiesMock = sinon.stub(); - - // Mock the Connection function - connectionMock.returns({ v2: { listEntities: listEntitiesMock } }); - }); - - afterEach(function() { - // Restore the original function after each test - connectionMock.restore(); - }); - - var ruleData = { - name: 'NSR2', - action: { - type: 'update', - parameters: { - id: 'alarma:${id}', - type: 'Alarm', - attributes: [ - { - name: 'msg', - value: 'El status de ${id} es ${status}' - } - ] - } - }, - subservice: '/', - service: 'unknownt', - nosignal: { - checkInterval: '1', - attribute: 'temperature', - reportInterval: '5', - id: 'thing:disp1', - idRegexp: null, - type: 'thing' + var ruleData = { + name: 'NSR2', + action: { + type: 'update', + parameters: { + id: 'alarma:${id}', + type: 'Alarm', + attributes: [ + { + name: 'msg', + value: 'El status de ${id} es ${status}' + } + ] } }, - func = 'sinon.stub()', - callback = function(e, request) { - should.exist(request); - should.not.exist(e); - should.equal(request.httpCode, 200); - }; + subservice: '/', + service: 'unknownt', + nosignal: { + checkInterval: '1', + attribute: 'temperature', + reportInterval: '5', + id: 'thing:disp1', + idRegexp: null, + type: 'thing' + } + }, + func = 'sinon.stub()', + callback = function(e, request) { + should.exist(request); + should.not.exist(e); + should.equal(request.httpCode, 200); + }; - it('By default should call findSilentEntitiesByMongo', function() { - var findSilentEntitiesByMongoSpy = sinon.spy(); - entitiesStore.__set__('findSilentEntitiesByMongo', findSilentEntitiesByMongoSpy); - entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); - sinon.assert.calledOnce(findSilentEntitiesByMongoSpy); - }); + it('By default should call findSilentEntitiesByMongo', function() { + var findSilentEntitiesByMongoSpy = sinon.spy(); + entitiesStore.__set__('findSilentEntitiesByMongo', findSilentEntitiesByMongoSpy); + entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); + sinon.assert.calledOnce(findSilentEntitiesByMongoSpy); + }); + + it('If default settings are changed FindSilentEntitiesByAPI should be called', function() { + config.nonSignalByAPI = true; + var findSilentEntitiesByAPISpy = sinon.spy(); + entitiesStore.__set__('findSilentEntitiesByAPI', findSilentEntitiesByAPISpy); + entitiesStore.FindSilentEntities(); + sinon.assert.calledOnce(findSilentEntitiesByAPISpy); + }); - it('If default settings are changed FindSilentEntitiesByAPI should be called', function() { - config.nonSignalByAPI = true; - var findSilentEntitiesByAPISpy = sinon.spy(); - entitiesStore.__set__('findSilentEntitiesByAPI', findSilentEntitiesByAPISpy); - entitiesStore.FindSilentEntities(); - sinon.assert.calledOnce(findSilentEntitiesByAPISpy); - }); + it('should call findSilentEntitiesByAPIWithPagination', function() { + var findSilentEntitiesByAPIWithPaginationSpy = sinon.spy(); + var createConnectionStub = sinon.stub().returns({}); + var createFilterStub = sinon.stub().returns({}); + var func2 = sinon.stub(); + var callback2 = sinon.stub(); - it('should correctly create a connection', function() { - var service = 'Service'; - var subservice = 'Subservice'; + entitiesStore.__set__('findSilentEntitiesByAPIWithPagination', findSilentEntitiesByAPIWithPaginationSpy); + entitiesStore.__set__('createConnection', createConnectionStub); + entitiesStore.__set__('createFilter', createFilterStub); - entitiesStore.createConnection(service, subservice); + entitiesStore.findSilentEntitiesByAPI(ruleData.service, ruleData.subservice, ruleData, func2, callback2); - sinon.assert.calledOnce(connectionMock); - sinon.assert.calledWithExactly(connectionMock, config.orion.URL, { - service: service, - servicepath: subservice, - headers: {} - }); - }); + sinon.assert.calledOnce(findSilentEntitiesByAPIWithPaginationSpy); + sinon.assert.calledOnce(createConnectionStub); + sinon.assert.calledOnce(createFilterStub); }); }); From aa79f8612ce24b6577226b69ae7d464ddd797a51 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Wed, 28 Jun 2023 17:48:57 +0200 Subject: [PATCH 33/52] changing problematic definitions --- lib/models/entitiesStore.js | 40 ++++++++++++++++---------------- package.json | 2 +- test/unit/entitiesStore_utest.js | 1 - 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 10ced9ba..29e85c2d 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -22,6 +22,8 @@ */ 'use strict'; +const { info } = require('console'); + var async = require('async'), appContext = require('../appContext'), config = require('../../config'), @@ -80,28 +82,9 @@ function createConnection(service, subservice) { return new ngsi.Connection(config.orion.URL, options); } -function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) { - var limit = limit || 20; - var offset = offset || 0; - +function findSilentEntitiesByAPIWithPagination(connection, filter, func, callback) { var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; - var connection = createConnection(service, subservice); - var filter = createFilter(ruleData, service, subservice, limit, offset); - - logger.info( - context, - 'find silent entities by API ngsi using options %j and filter %j and rule %j', - connection, - filter, - ruleData - ); - - // Call the pagination function - findSilentEntitiesByAPIWithPagination(connection, filter, func, callback); -} - -function findSilentEntitiesByAPIWithPagination(connection, filter, func, callback) { // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor connection.v2.listEntities(filter).then( (response) => { @@ -134,6 +117,23 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, func, callbac ); } +function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) { + var limit = limit || 20; + var offset = offset || 0; + var connection = createConnection(service, subservice); + var filter = createFilter(ruleData, service, subservice, limit, offset); + + logger.info( + 'find silent entities by API ngsi using options %j and filter %j and rule %j', + connection, + filter, + ruleData + ); + + // Call the pagination function + findSilentEntitiesByAPIWithPagination(connection, filter, func, callback); +} + function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback) { var db, context = { op: 'checkNoSignal.findSilentEntitiesByMongo', comp: constants.COMPONENT_NAME }, diff --git a/package.json b/package.json index a3a62456..155be63f 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "clean": "rm -rf package-lock.json && rm -rf node_modules && rm -rf coverage", "healthcheck": "node ./bin/healthcheck", - "test": "mocha --recursive 'test/unit/entitiesStore_utest.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", + "test": "mocha --recursive 'test/**/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", "test:watch": "npm run test -- -w ./lib", "lint": "jshint lib/ --config .jshintrc && jshint bin/ --config .jshintrc && jshint test/ --config test/.jshintrc", "lint:md": "remark -f '*.md' 'docs/*.md'", diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index d5714566..214fb492 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -32,7 +32,6 @@ var chai = require('chai'); var sinon = require('sinon'); var sinonChai = require('sinon-chai'); var config = require('../../config.js'); -var ngsi = require('ngsijs'); chai.Should(); chai.use(sinonChai); From 4b63d3d67a812e7c1f070f2c3e2ecb7897777719 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Wed, 28 Jun 2023 17:51:06 +0200 Subject: [PATCH 34/52] unused reference --- lib/models/entitiesStore.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 29e85c2d..e03a0180 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -22,8 +22,6 @@ */ 'use strict'; -const { info } = require('console'); - var async = require('async'), appContext = require('../appContext'), config = require('../../config'), From a0b86c35ecacfe6d11f6b260fdc33666bb5fe214 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Thu, 29 Jun 2023 09:13:04 +0200 Subject: [PATCH 35/52] renaming unclear function --- lib/models/entitiesStore.js | 24 ++++++------- test/unit/entitiesStore_utest.js | 62 +++++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 16 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index e03a0180..699ab6c8 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -80,7 +80,7 @@ function createConnection(service, subservice) { return new ngsi.Connection(config.orion.URL, options); } -function findSilentEntitiesByAPIWithPagination(connection, filter, func, callback) { +function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback) { var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor @@ -94,7 +94,7 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, func, callbac var count = 0; response.results.forEach((entity) => { logger.debug(context, 'silent entity %j', entity); - func(entity); + alterFunc(entity); count++; }); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); @@ -102,7 +102,7 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, func, callbac if (response.count > filter.limit + filter.offset) { // Call the function again with updated offset filter.offset += filter.limit; - findSilentEntitiesByAPIWithPagination(connection, filter, func, callback); + findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } else { // All entities have been retrieved and processed, call the callback callback(null, response.results); @@ -115,9 +115,9 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, func, callbac ); } -function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) { - var limit = limit || 20; - var offset = offset || 0; +function findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callback) { + var limit = 20; + var offset = 0; var connection = createConnection(service, subservice); var filter = createFilter(ruleData, service, subservice, limit, offset); @@ -129,10 +129,10 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, func, callback) ); // Call the pagination function - findSilentEntitiesByAPIWithPagination(connection, filter, func, callback); + findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } -function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback) { +function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback) { var db, context = { op: 'checkNoSignal.findSilentEntitiesByMongo', comp: constants.COMPONENT_NAME }, criterion = {}; @@ -174,7 +174,7 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback return cb(err, 'silent ones count ' + count); } logger.debug(context, 'silent entity %j', one._id); - func(one); + alterFunc(one); count++; }); } @@ -186,11 +186,11 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, func, callback ); } -function findSilentEntities(service, subservice, ruleData, func, callback) { +function findSilentEntities(service, subservice, ruleData, alterFunc, callback) { if (!config.nonSignalByAPI) { - return findSilentEntitiesByMongo(service, subservice, ruleData, func, callback); + return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback); } else { - return findSilentEntitiesByAPI(service, subservice, ruleData, func, callback); + return findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callback); } } diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 214fb492..bcbe1b65 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -62,7 +62,7 @@ describe('entitiesStore', function() { type: 'thing' } }, - func = 'sinon.stub()', + alterFunc = 'sinon.stub()', callback = function(e, request) { should.exist(request); should.not.exist(e); @@ -72,7 +72,7 @@ describe('entitiesStore', function() { it('By default should call findSilentEntitiesByMongo', function() { var findSilentEntitiesByMongoSpy = sinon.spy(); entitiesStore.__set__('findSilentEntitiesByMongo', findSilentEntitiesByMongoSpy); - entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, func, callback); + entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, alterFunc, callback); sinon.assert.calledOnce(findSilentEntitiesByMongoSpy); }); @@ -88,17 +88,71 @@ describe('entitiesStore', function() { var findSilentEntitiesByAPIWithPaginationSpy = sinon.spy(); var createConnectionStub = sinon.stub().returns({}); var createFilterStub = sinon.stub().returns({}); - var func2 = sinon.stub(); + var alterFunc2 = sinon.stub(); var callback2 = sinon.stub(); entitiesStore.__set__('findSilentEntitiesByAPIWithPagination', findSilentEntitiesByAPIWithPaginationSpy); entitiesStore.__set__('createConnection', createConnectionStub); entitiesStore.__set__('createFilter', createFilterStub); - entitiesStore.findSilentEntitiesByAPI(ruleData.service, ruleData.subservice, ruleData, func2, callback2); + entitiesStore.findSilentEntitiesByAPI(ruleData.service, ruleData.subservice, ruleData, alterFunc2, callback2); sinon.assert.calledOnce(findSilentEntitiesByAPIWithPaginationSpy); sinon.assert.calledOnce(createConnectionStub); sinon.assert.calledOnce(createFilterStub); }); + + it('should call listEntities and handle successful API response', function(done) { + var listEntitiesStub = sinon.stub().returns( + Promise.resolve({ + results: [], + count: 0 + }) + ); + var fakeConnection = { + v2: { + listEntities: listEntitiesStub + } + }; + var fakeFilter = {}; + var alterFuncStub = sinon.stub(); + var callbackStub = sinon.stub(); + + entitiesStore + .findSilentEntitiesByAPIWithPagination(fakeConnection, fakeFilter, alterFuncStub, callbackStub) + .then(() => { + sinon.assert.calledOnce(listEntitiesStub); + sinon.assert.calledWith(listEntitiesStub, fakeFilter); + sinon.assert.calledOnce(alterFuncStub); + sinon.assert.calledOnce(callbackStub); + done(); + }) + .catch(done); + }); + + it('should handle errors from listEntities', function(done) { + var listEntitiesStub = sinon.stub().returns(Promise.reject('error')); + var fakeConnection = { + v2: { + listEntities: listEntitiesStub + } + }; + var fakeFilter = {}; + var alterFuncStub = sinon.stub(); + var callbackStub = sinon.stub(); + + entitiesStore + .findSilentEntitiesByAPIWithPagination(fakeConnection, fakeFilter, alterFuncStub, callbackStub) + .then(() => { + done(new Error('Expected method to reject.')); + }) + .catch(() => { + sinon.assert.calledOnce(listEntitiesStub); + sinon.assert.calledWith(listEntitiesStub, fakeFilter); + sinon.assert.notCalled(alterFuncStub); + sinon.assert.calledOnce(callbackStub); + sinon.assert.calledWith(callbackStub, 'error', null); + done(); + }); + }); }); From e593c4cab3e7363fb3ca7d6be379428cafe2b944 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Thu, 29 Jun 2023 09:40:04 +0200 Subject: [PATCH 36/52] erease bad test --- test/unit/entitiesStore_utest.js | 54 -------------------------------- 1 file changed, 54 deletions(-) diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index bcbe1b65..70f3a32d 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -101,58 +101,4 @@ describe('entitiesStore', function() { sinon.assert.calledOnce(createConnectionStub); sinon.assert.calledOnce(createFilterStub); }); - - it('should call listEntities and handle successful API response', function(done) { - var listEntitiesStub = sinon.stub().returns( - Promise.resolve({ - results: [], - count: 0 - }) - ); - var fakeConnection = { - v2: { - listEntities: listEntitiesStub - } - }; - var fakeFilter = {}; - var alterFuncStub = sinon.stub(); - var callbackStub = sinon.stub(); - - entitiesStore - .findSilentEntitiesByAPIWithPagination(fakeConnection, fakeFilter, alterFuncStub, callbackStub) - .then(() => { - sinon.assert.calledOnce(listEntitiesStub); - sinon.assert.calledWith(listEntitiesStub, fakeFilter); - sinon.assert.calledOnce(alterFuncStub); - sinon.assert.calledOnce(callbackStub); - done(); - }) - .catch(done); - }); - - it('should handle errors from listEntities', function(done) { - var listEntitiesStub = sinon.stub().returns(Promise.reject('error')); - var fakeConnection = { - v2: { - listEntities: listEntitiesStub - } - }; - var fakeFilter = {}; - var alterFuncStub = sinon.stub(); - var callbackStub = sinon.stub(); - - entitiesStore - .findSilentEntitiesByAPIWithPagination(fakeConnection, fakeFilter, alterFuncStub, callbackStub) - .then(() => { - done(new Error('Expected method to reject.')); - }) - .catch(() => { - sinon.assert.calledOnce(listEntitiesStub); - sinon.assert.calledWith(listEntitiesStub, fakeFilter); - sinon.assert.notCalled(alterFuncStub); - sinon.assert.calledOnce(callbackStub); - sinon.assert.calledWith(callbackStub, 'error', null); - done(); - }); - }); }); From 543cfdd44d8fa50f6e32a799bf9efe9ad00a3ba5 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Thu, 29 Jun 2023 10:45:26 +0200 Subject: [PATCH 37/52] clarifying the code From cb84048fa241d6e23124cdf3b7fe164a8836ac98 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Thu, 29 Jun 2023 11:42:26 +0200 Subject: [PATCH 38/52] create filter test --- lib/models/entitiesStore.js | 1 + package.json | 2 +- test/unit/entitiesStore_utest.js | 31 +++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 699ab6c8..f3296b42 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -199,3 +199,4 @@ module.exports.findSilentEntitiesByAPI = findSilentEntitiesByAPI; module.exports.findSilentEntitiesByMongo = findSilentEntitiesByMongo; module.exports.findSilentEntitiesByAPIWithPagination = findSilentEntitiesByAPIWithPagination; module.exports.createConnection = createConnection; +module.exports.createFilter = createFilter; diff --git a/package.json b/package.json index 155be63f..a3a62456 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "clean": "rm -rf package-lock.json && rm -rf node_modules && rm -rf coverage", "healthcheck": "node ./bin/healthcheck", - "test": "mocha --recursive 'test/**/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", + "test": "mocha --recursive 'test/unit/entitiesStore_utest.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", "test:watch": "npm run test -- -w ./lib", "lint": "jshint lib/ --config .jshintrc && jshint bin/ --config .jshintrc && jshint test/ --config test/.jshintrc", "lint:md": "remark -f '*.md' 'docs/*.md'", diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 70f3a32d..7c27eb8b 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -32,6 +32,7 @@ var chai = require('chai'); var sinon = require('sinon'); var sinonChai = require('sinon-chai'); var config = require('../../config.js'); +var assert = require('chai').assert; chai.Should(); chai.use(sinonChai); @@ -101,4 +102,34 @@ describe('entitiesStore', function() { sinon.assert.calledOnce(createConnectionStub); sinon.assert.calledOnce(createFilterStub); }); + it('should correctly create filter', function() { + // Define input arguments + var service = 'testService'; + var subservice = 'testSubservice'; + var ruleData = { + type: 'testType', + attribute: 'testAttribute', + eportInterval: 3000, + id: 'testId' + }; + var limit = 20; + var offset = 0; + + var expectedFilter = { + service: service, + servicepath: subservice, + type: ruleData.type, + mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString(), + options: 'count', + limit: limit, + offset: offset, + id: ruleData.id + }; + + // Call the function + var resultFilter = entitiesStore.createFilter(ruleData, service, subservice, limit, offset); + + // Verify the result using assert.deepEqual + assert.deepStrictEqual(resultFilter, expectedFilter); + }); }); From ebdd267d9021c958d5822e6515dd751615af8114 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Fri, 30 Jun 2023 10:33:06 +0200 Subject: [PATCH 39/52] findsilententites timmer update --- lib/models/entitiesStore.js | 14 ++++++++++++++ package.json | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index f3296b42..4c353372 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -187,6 +187,20 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal } function findSilentEntities(service, subservice, ruleData, alterFunc, callback) { + // Start the timer + var hrstart = process.hrtime(); + + // Function to be called when silent entities search is completed + var timedCallback = function(err, result) { + // Stop the timer + var hrend = process.hrtime(hrstart); + + // Log the execution time in seconds and milliseconds + logger.info('Execution time (hr): %dms', hrend[1] / 1000000); + + // Call the original callback + callback(err, result); + }; if (!config.nonSignalByAPI) { return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback); } else { diff --git a/package.json b/package.json index a3a62456..155be63f 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "clean": "rm -rf package-lock.json && rm -rf node_modules && rm -rf coverage", "healthcheck": "node ./bin/healthcheck", - "test": "mocha --recursive 'test/unit/entitiesStore_utest.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", + "test": "mocha --recursive 'test/**/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", "test:watch": "npm run test -- -w ./lib", "lint": "jshint lib/ --config .jshintrc && jshint bin/ --config .jshintrc && jshint test/ --config test/.jshintrc", "lint:md": "remark -f '*.md' 'docs/*.md'", From 145e79856090b3a8336adfd4d2ccf4e6bde3a04e Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Fri, 30 Jun 2023 10:41:39 +0200 Subject: [PATCH 40/52] finsilententites timmer fix --- lib/models/entitiesStore.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 4c353372..744a0e3e 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -202,9 +202,9 @@ function findSilentEntities(service, subservice, ruleData, alterFunc, callback) callback(err, result); }; if (!config.nonSignalByAPI) { - return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback); + return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, timedCallback); } else { - return findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callback); + return findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, timedCallback); } } From 92aa5fe3c314ccb489b3c4ac63d94cc05a25b90c Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Fri, 30 Jun 2023 13:18:28 +0200 Subject: [PATCH 41/52] update findSilentEntities --- lib/models/entitiesStore.js | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 744a0e3e..276f3827 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -80,7 +80,8 @@ function createConnection(service, subservice) { return new ngsi.Connection(config.orion.URL, options); } -function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback) { +function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, total, callback) { + total = total || 0; var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor @@ -98,14 +99,16 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, ca count++; }); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); + // Add the count of this page to the total count + total += count; // Check if there are more entities to retrieve if (response.count > filter.limit + filter.offset) { // Call the function again with updated offset filter.offset += filter.limit; findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } else { - // All entities have been retrieved and processed, call the callback - callback(null, response.results); + // Pass the total count to the callback + callback(null, response.results, total); } }, (error) => { @@ -129,7 +132,14 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callb ); // Call the pagination function - findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); + findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, (err, results, total) => { + if (err) { + callback(err, null); + } else { + callback(null, results); + logger.info('Total number of silent entities retrieved: %d', total); + } + }); } function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback) { @@ -181,6 +191,7 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal ], function(err, result) { logger.debug(context, 'findSilentEntities %s', myutils.firstChars(result)); + logger.info(context, 'Number of silent entities retrieved: %d', count); // Logging the number of entities return callback(err, result); } ); @@ -202,8 +213,10 @@ function findSilentEntities(service, subservice, ruleData, alterFunc, callback) callback(err, result); }; if (!config.nonSignalByAPI) { + logger.debug('Executing findSilentEntitiesByMongo'); return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, timedCallback); } else { + logger.debug('Executing findSilentEntitiesByAPI'); return findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, timedCallback); } } From c5135227dc5b80d211c57369253a6bdafbec029e Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Fri, 30 Jun 2023 14:12:49 +0200 Subject: [PATCH 42/52] findSilentEntities total count fix --- lib/models/entitiesStore.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 276f3827..89d89e5f 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -137,7 +137,8 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callb callback(err, null); } else { callback(null, results); - logger.info('Total number of silent entities retrieved: %d', total); + // Logging the number of entities + logger.info(context, 'Number of silent entities retrieved using findSilentEntitiesByAPI: %d', count); } }); } @@ -191,7 +192,8 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal ], function(err, result) { logger.debug(context, 'findSilentEntities %s', myutils.firstChars(result)); - logger.info(context, 'Number of silent entities retrieved: %d', count); // Logging the number of entities + // Logging the number of entities + logger.info(context, 'Number of silent entities retrieved using findSilentEntitiesByMongo: %d', count); return callback(err, result); } ); From 3020ed0dd73eb97f19fa44bc3b858b4a2c529f94 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Fri, 30 Jun 2023 14:13:53 +0200 Subject: [PATCH 43/52] context fix --- lib/models/entitiesStore.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 89d89e5f..e25bfcae 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -215,10 +215,10 @@ function findSilentEntities(service, subservice, ruleData, alterFunc, callback) callback(err, result); }; if (!config.nonSignalByAPI) { - logger.debug('Executing findSilentEntitiesByMongo'); + logger.debug(context, 'Executing findSilentEntitiesByMongo'); return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, timedCallback); } else { - logger.debug('Executing findSilentEntitiesByAPI'); + logger.debug(context, 'Executing findSilentEntitiesByAPI'); return findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, timedCallback); } } From 38a29328cd73b0c9328c9308625609469e821982 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Fri, 30 Jun 2023 14:35:53 +0200 Subject: [PATCH 44/52] findSilentEntities timmer fix --- lib/models/entitiesStore.js | 47 ++++++++++++++++++++++++++++++------- package.json | 2 +- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index f3296b42..32ebb9f4 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -29,7 +29,8 @@ var async = require('async'), myutils = require('../myutils'), constants = require('../constants'), logger = require('logops'), - ngsi = require('ngsijs'); + ngsi = require('ngsijs'), + context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; function orionServiceDb(service) { return appContext.OrionDb(config.orionDb.prefix + '-' + service); @@ -80,8 +81,8 @@ function createConnection(service, subservice) { return new ngsi.Connection(config.orion.URL, options); } -function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback) { - var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; +function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, total, callback) { + total = total || 0; // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor connection.v2.listEntities(filter).then( @@ -98,14 +99,16 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, ca count++; }); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); + // Add the count of this page to the total count + total += count; // Check if there are more entities to retrieve if (response.count > filter.limit + filter.offset) { // Call the function again with updated offset filter.offset += filter.limit; findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } else { - // All entities have been retrieved and processed, call the callback - callback(null, response.results); + // Pass the total count to the callback + callback(null, response.results, total); } }, (error) => { @@ -129,12 +132,20 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callb ); // Call the pagination function - findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); + findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, total, (err, results) => { + if (err) { + callback(err, null); + } else { + callback(null, results); + // Logging the number of entities + logger.info(context, 'Number of silent entities retrieved using findSilentEntitiesByAPI: %d', count); + } + }); } function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback) { + context = { op: 'checkNoSignal.findSilentEntitiesByMongo', comp: constants.COMPONENT_NAME }; var db, - context = { op: 'checkNoSignal.findSilentEntitiesByMongo', comp: constants.COMPONENT_NAME }, criterion = {}; db = orionServiceDb(service); @@ -181,16 +192,34 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal ], function(err, result) { logger.debug(context, 'findSilentEntities %s', myutils.firstChars(result)); + // Logging the number of entities + logger.info(context, 'Number of silent entities retrieved using findSilentEntitiesByMongo: %d', count); return callback(err, result); } ); } function findSilentEntities(service, subservice, ruleData, alterFunc, callback) { + // Start the timer + var hrstart = process.hrtime(); + + // Function to be called when silent entities search is completed + var timedCallback = function(err, result) { + // Stop the timer + var hrend = process.hrtime(hrstart); + + // Log the execution time in seconds and milliseconds + logger.info('Execution time (hr): %dms', hrend[1] / 1000000); + + // Call the original callback + callback(err, result); + }; if (!config.nonSignalByAPI) { - return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback); + logger.debug(context, 'Executing findSilentEntitiesByMongo'); + return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, timedCallback); } else { - return findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callback); + logger.debug(context, 'Executing findSilentEntitiesByAPI'); + return findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, timedCallback); } } diff --git a/package.json b/package.json index a3a62456..155be63f 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "clean": "rm -rf package-lock.json && rm -rf node_modules && rm -rf coverage", "healthcheck": "node ./bin/healthcheck", - "test": "mocha --recursive 'test/unit/entitiesStore_utest.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", + "test": "mocha --recursive 'test/**/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", "test:watch": "npm run test -- -w ./lib", "lint": "jshint lib/ --config .jshintrc && jshint bin/ --config .jshintrc && jshint test/ --config test/.jshintrc", "lint:md": "remark -f '*.md' 'docs/*.md'", From e240874503efb8b172ead84133c77afd653cf34b Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 3 Jul 2023 13:56:00 +0200 Subject: [PATCH 45/52] count entities fix --- lib/models/entitiesStore.js | 48 +++++++++++--------------------- package.json | 2 +- test/unit/entitiesStore_utest.js | 2 +- 3 files changed, 19 insertions(+), 33 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index e6fad7a2..70c535dc 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -29,8 +29,7 @@ var async = require('async'), myutils = require('../myutils'), constants = require('../constants'), logger = require('logops'), - ngsi = require('ngsijs'), - context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; + ngsi = require('ngsijs'); function orionServiceDb(service) { return appContext.OrionDb(config.orionDb.prefix + '-' + service); @@ -42,9 +41,9 @@ function createFilter(ruleData, service, subservice, limit, offset) { servicepath: subservice, type: ruleData.type, mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString(), - options: 'count', limit: limit, - offset: offset + offset: offset, + count: true }; if (ruleData.id) { filter.id = ruleData.id; @@ -81,9 +80,7 @@ function createConnection(service, subservice) { return new ngsi.Connection(config.orion.URL, options); } -function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, total, callback) { - total = total || 0; - +function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback) { var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor @@ -94,23 +91,20 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, to // response.limit contains the used page size // response.results is an array with the retrieved entities // response.offset contains the offset used in the request - var count = 0; response.results.forEach((entity) => { logger.debug(context, 'silent entity %j', entity); alterFunc(entity); - count++; }); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); - // Add the count of this page to the total count - total += count; // Check if there are more entities to retrieve if (response.count > filter.limit + filter.offset) { // Call the function again with updated offset filter.offset += filter.limit; findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } else { - // Pass the total count to the callback - callback(null, response.results, total); + logger.info('%j entities found using findSilentEntitiesBy', filter.count); + // All entities have been retrieved and processed, call the callback + callback(null, response.results); } }, (error) => { @@ -134,20 +128,12 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callb ); // Call the pagination function - findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, total, (err, results) => { - if (err) { - callback(err, null); - } else { - callback(null, results); - // Logging the number of entities - logger.info(context, 'Number of silent entities retrieved using findSilentEntitiesByAPI: %d', count); - } - }); + findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback) { - context = { op: 'checkNoSignal.findSilentEntitiesByMongo', comp: constants.COMPONENT_NAME }; var db, + context = { op: 'checkNoSignal.findSilentEntitiesByMongo', comp: constants.COMPONENT_NAME }, criterion = {}; db = orionServiceDb(service); @@ -171,11 +157,14 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal criterion['_id.type'] = ruleData.type; } logger.debug(context, 'findSilentEntities criterion %j', criterion); + + // Variable to store the count of entities + var entityCount = 0; + async.waterfall( [ db.collection.bind(db, entitiesCollectionName, { strict: true }), function(col, cb) { - var count = 0; col.find(criterion) .batchSize(config.orionDb.batchSize) .each(function(err, one) { @@ -183,19 +172,18 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal return cb(err, null); } if (one === null) { - //cursor exhausted - return cb(err, 'silent ones count ' + count); + // Cursor exhausted + return cb(err, 'silent ones count ' + entityCount); } logger.debug(context, 'silent entity %j', one._id); alterFunc(one); - count++; + entityCount++; // Increment the count of entities }); } ], function(err, result) { + logger.debug('Total entities retrieved: %d', entityCount); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(result)); - // Logging the number of entities - logger.info(context, 'Number of silent entities retrieved using findSilentEntitiesByMongo: %d', count); return callback(err, result); } ); @@ -217,10 +205,8 @@ function findSilentEntities(service, subservice, ruleData, alterFunc, callback) callback(err, result); }; if (!config.nonSignalByAPI) { - logger.debug(context, 'Executing findSilentEntitiesByMongo'); return findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, timedCallback); } else { - logger.debug(context, 'Executing findSilentEntitiesByAPI'); return findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, timedCallback); } } diff --git a/package.json b/package.json index 155be63f..a3a62456 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "clean": "rm -rf package-lock.json && rm -rf node_modules && rm -rf coverage", "healthcheck": "node ./bin/healthcheck", - "test": "mocha --recursive 'test/**/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", + "test": "mocha --recursive 'test/unit/entitiesStore_utest.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", "test:watch": "npm run test -- -w ./lib", "lint": "jshint lib/ --config .jshintrc && jshint bin/ --config .jshintrc && jshint test/ --config test/.jshintrc", "lint:md": "remark -f '*.md' 'docs/*.md'", diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index 7c27eb8b..b7ebce77 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -120,7 +120,7 @@ describe('entitiesStore', function() { servicepath: subservice, type: ruleData.type, mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString(), - options: 'count', + count: true, limit: limit, offset: offset, id: ruleData.id From 2369f6b81b06dd172c60574026f76a20b39b0f47 Mon Sep 17 00:00:00 2001 From: RafaelM3 <134268825+RafaelM3@users.noreply.github.com> Date: Mon, 3 Jul 2023 15:38:39 +0200 Subject: [PATCH 46/52] Update CHANGES_NEXT_RELEASE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 079e4728..cab6522a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Allow access entities using NGSIv2 API for non_signal rules (new setting nonSignalByAPI / PERSEO_CHECK_NON_SIGNAL_BY_API) (#549) +- Add: allow access entities using NGSIv2 API for non_signal rules (new setting nonSignalByAPI / PERSEO_CHECK_NON_SIGNAL_BY_API) (#549) - Remove support for ngsv1 notifications (#714) - Remove ngsiv1 support for updateAction (#714) From f3207ec2ec72f2fa9fbeaa753525de4c9c077290 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 3 Jul 2023 16:45:01 +0200 Subject: [PATCH 47/52] count entities fix --- lib/models/entitiesStore.js | 13 ++-- package.json | 2 +- test/unit/entitiesStore_utest.js | 114 +++++++++++++++++-------------- 3 files changed, 71 insertions(+), 58 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 70c535dc..4139afc9 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -42,8 +42,7 @@ function createFilter(ruleData, service, subservice, limit, offset) { type: ruleData.type, mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString(), limit: limit, - offset: offset, - count: true + offset: offset }; if (ruleData.id) { filter.id = ruleData.id; @@ -102,7 +101,7 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, ca filter.offset += filter.limit; findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } else { - logger.info('%j entities found using findSilentEntitiesBy', filter.count); + logger.info('%d entities found using findSilentEntitiesByAPI', response.results.lenght()); // All entities have been retrieved and processed, call the callback callback(null, response.results); } @@ -177,12 +176,13 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal } logger.debug(context, 'silent entity %j', one._id); alterFunc(one); - entityCount++; // Increment the count of entities + // Increment the count of entities + entityCount++; }); } ], function(err, result) { - logger.debug('Total entities retrieved: %d', entityCount); + logger.info('%d entities found using findSilentEntitiesByMongo', entityCount); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(result)); return callback(err, result); } @@ -192,6 +192,7 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal function findSilentEntities(service, subservice, ruleData, alterFunc, callback) { // Start the timer var hrstart = process.hrtime(); + var method = !config.nonSignalByAPI ? 'findSilentEntitiesByMongo' : 'findSilentEntitiesByAPI'; // Function to be called when silent entities search is completed var timedCallback = function(err, result) { @@ -199,7 +200,7 @@ function findSilentEntities(service, subservice, ruleData, alterFunc, callback) var hrend = process.hrtime(hrstart); // Log the execution time in seconds and milliseconds - logger.info('Execution time (hr): %dms', hrend[1] / 1000000); + logger.info('%s execution time (hr): %dms', method, hrend[1] / 1000000); // Call the original callback callback(err, result); diff --git a/package.json b/package.json index a3a62456..b9ac4069 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "clean": "rm -rf package-lock.json && rm -rf node_modules && rm -rf coverage", "healthcheck": "node ./bin/healthcheck", - "test": "mocha --recursive 'test/unit/entitiesStore_utest.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", + "test": "mocha --recursive 'test/uni**t/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", "test:watch": "npm run test -- -w ./lib", "lint": "jshint lib/ --config .jshintrc && jshint bin/ --config .jshintrc && jshint test/ --config test/.jshintrc", "lint:md": "remark -f '*.md' 'docs/*.md'", diff --git a/test/unit/entitiesStore_utest.js b/test/unit/entitiesStore_utest.js index b7ebce77..b886ceb0 100644 --- a/test/unit/entitiesStore_utest.js +++ b/test/unit/entitiesStore_utest.js @@ -70,66 +70,78 @@ describe('entitiesStore', function() { should.equal(request.httpCode, 200); }; - it('By default should call findSilentEntitiesByMongo', function() { - var findSilentEntitiesByMongoSpy = sinon.spy(); - entitiesStore.__set__('findSilentEntitiesByMongo', findSilentEntitiesByMongoSpy); - entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, alterFunc, callback); - sinon.assert.calledOnce(findSilentEntitiesByMongoSpy); - }); + describe('FindSilentEntities', function() { + it('By default should call findSilentEntitiesByMongo', function() { + var findSilentEntitiesByMongoSpy = sinon.spy(); + entitiesStore.__set__('findSilentEntitiesByMongo', findSilentEntitiesByMongoSpy); + entitiesStore.FindSilentEntities(ruleData.service, ruleData.subservice, ruleData, alterFunc, callback); + sinon.assert.calledOnce(findSilentEntitiesByMongoSpy); + }); - it('If default settings are changed FindSilentEntitiesByAPI should be called', function() { - config.nonSignalByAPI = true; - var findSilentEntitiesByAPISpy = sinon.spy(); - entitiesStore.__set__('findSilentEntitiesByAPI', findSilentEntitiesByAPISpy); - entitiesStore.FindSilentEntities(); - sinon.assert.calledOnce(findSilentEntitiesByAPISpy); + it('If default settings are changed FindSilentEntitiesByAPI should be called', function() { + config.nonSignalByAPI = true; + var findSilentEntitiesByAPISpy = sinon.spy(); + entitiesStore.__set__('findSilentEntitiesByAPI', findSilentEntitiesByAPISpy); + entitiesStore.FindSilentEntities(); + sinon.assert.calledOnce(findSilentEntitiesByAPISpy); + }); }); - it('should call findSilentEntitiesByAPIWithPagination', function() { - var findSilentEntitiesByAPIWithPaginationSpy = sinon.spy(); - var createConnectionStub = sinon.stub().returns({}); - var createFilterStub = sinon.stub().returns({}); - var alterFunc2 = sinon.stub(); - var callback2 = sinon.stub(); + describe('findSilentEntitiesByAPI', function() { + it('should call findSilentEntitiesByAPIWithPagination', function() { + var findSilentEntitiesByAPIWithPaginationSpy = sinon.spy(); + var createConnectionStub = sinon.stub().returns({}); + var createFilterStub = sinon.stub().returns({}); + var alterFunc2 = sinon.stub(); + var callback2 = sinon.stub(); - entitiesStore.__set__('findSilentEntitiesByAPIWithPagination', findSilentEntitiesByAPIWithPaginationSpy); - entitiesStore.__set__('createConnection', createConnectionStub); - entitiesStore.__set__('createFilter', createFilterStub); + entitiesStore.__set__('findSilentEntitiesByAPIWithPagination', findSilentEntitiesByAPIWithPaginationSpy); + entitiesStore.__set__('createConnection', createConnectionStub); + entitiesStore.__set__('createFilter', createFilterStub); - entitiesStore.findSilentEntitiesByAPI(ruleData.service, ruleData.subservice, ruleData, alterFunc2, callback2); + entitiesStore.findSilentEntitiesByAPI( + ruleData.service, + ruleData.subservice, + ruleData, + alterFunc2, + callback2 + ); - sinon.assert.calledOnce(findSilentEntitiesByAPIWithPaginationSpy); - sinon.assert.calledOnce(createConnectionStub); - sinon.assert.calledOnce(createFilterStub); + sinon.assert.calledOnce(findSilentEntitiesByAPIWithPaginationSpy); + sinon.assert.calledOnce(createConnectionStub); + sinon.assert.calledOnce(createFilterStub); + }); }); - it('should correctly create filter', function() { - // Define input arguments - var service = 'testService'; - var subservice = 'testSubservice'; - var ruleData = { - type: 'testType', - attribute: 'testAttribute', - eportInterval: 3000, - id: 'testId' - }; - var limit = 20; - var offset = 0; - var expectedFilter = { - service: service, - servicepath: subservice, - type: ruleData.type, - mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString(), - count: true, - limit: limit, - offset: offset, - id: ruleData.id - }; + describe('createFilter', function() { + it('should correctly create filter', function() { + // Define input arguments + var service = 'testService'; + var subservice = 'testSubservice'; + var ruleData = { + type: 'testType', + attribute: 'testAttribute', + eportInterval: 3000, + id: 'testId' + }; + var limit = 20; + var offset = 0; + + var expectedFilter = { + service: service, + servicepath: subservice, + type: ruleData.type, + mq: ruleData.attribute + '.dateModified<' + (Date.now() / 1000 - ruleData.reportInterval).toString(), + limit: limit, + offset: offset, + id: ruleData.id + }; - // Call the function - var resultFilter = entitiesStore.createFilter(ruleData, service, subservice, limit, offset); + // Call the function + var resultFilter = entitiesStore.createFilter(ruleData, service, subservice, limit, offset); - // Verify the result using assert.deepEqual - assert.deepStrictEqual(resultFilter, expectedFilter); + // Verify the result using assert.deepEqual + assert.deepStrictEqual(resultFilter, expectedFilter); + }); }); }); From 00dcd96a834b32be5bf5c92ffdc4e580aa860996 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Mon, 3 Jul 2023 16:47:26 +0200 Subject: [PATCH 48/52] test fix --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b9ac4069..155be63f 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "scripts": { "clean": "rm -rf package-lock.json && rm -rf node_modules && rm -rf coverage", "healthcheck": "node ./bin/healthcheck", - "test": "mocha --recursive 'test/uni**t/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", + "test": "mocha --recursive 'test/**/*.js' --reporter spec --timeout 5000 --ui bdd --exit --color true", "test:watch": "npm run test -- -w ./lib", "lint": "jshint lib/ --config .jshintrc && jshint bin/ --config .jshintrc && jshint test/ --config test/.jshintrc", "lint:md": "remark -f '*.md' 'docs/*.md'", From b2e11dd29465022d9d28e6c6e2c152ba85282f9d Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 4 Jul 2023 08:48:58 +0200 Subject: [PATCH 49/52] context fix --- lib/models/entitiesStore.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 4139afc9..b8303de3 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -29,7 +29,8 @@ var async = require('async'), myutils = require('../myutils'), constants = require('../constants'), logger = require('logops'), - ngsi = require('ngsijs'); + ngsi = require('ngsijs'), + context = { op: 'entitiesStore', comp: constants.COMPONENT_NAME }; function orionServiceDb(service) { return appContext.OrionDb(config.orionDb.prefix + '-' + service); @@ -80,8 +81,6 @@ function createConnection(service, subservice) { } function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback) { - var context = { op: 'checkNoSignal.findSilentEntitiesByAPI', comp: constants.COMPONENT_NAME }; - // https://ficodes.github.io/ngsijs/stable/NGSI.Connection.html#.%22v2.listEntities%22__anchor connection.v2.listEntities(filter).then( (response) => { @@ -91,7 +90,7 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, ca // response.results is an array with the retrieved entities // response.offset contains the offset used in the request response.results.forEach((entity) => { - logger.debug(context, 'silent entity %j', entity); + logger.debug(contextAPI, 'silent entity %j', entity); alterFunc(entity); }); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); @@ -132,7 +131,6 @@ function findSilentEntitiesByAPI(service, subservice, ruleData, alterFunc, callb function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, callback) { var db, - context = { op: 'checkNoSignal.findSilentEntitiesByMongo', comp: constants.COMPONENT_NAME }, criterion = {}; db = orionServiceDb(service); @@ -150,7 +148,7 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal } } else { //Added default else clause - logger.debug('findSilentEntities() - Default else clause'); + logger.debug(context, 'findSilentEntities() - Default else clause'); } if (ruleData.type) { criterion['_id.type'] = ruleData.type; From 855cde029a5081fb4ce996bdcd1f484db6536527 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 4 Jul 2023 09:05:38 +0200 Subject: [PATCH 50/52] context fix --- lib/models/entitiesStore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index b8303de3..b23ed0a4 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -90,7 +90,7 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, ca // response.results is an array with the retrieved entities // response.offset contains the offset used in the request response.results.forEach((entity) => { - logger.debug(contextAPI, 'silent entity %j', entity); + logger.debug(context, 'silent entity %j', entity); alterFunc(entity); }); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(response.results)); From 53796c1fc953515347ee1b95809c7713b0f55654 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 4 Jul 2023 10:29:39 +0200 Subject: [PATCH 51/52] count entities fix --- lib/models/entitiesStore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index b23ed0a4..2c33f4b1 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -100,7 +100,7 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, ca filter.offset += filter.limit; findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } else { - logger.info('%d entities found using findSilentEntitiesByAPI', response.results.lenght()); + logger.info('%d entities found using findSilentEntitiesByAPI', response.results.length); // All entities have been retrieved and processed, call the callback callback(null, response.results); } From 3f3b8ef6042bf1fb91aaa59cb1042fb8a5fd2be8 Mon Sep 17 00:00:00 2001 From: RafaelM3 Date: Tue, 4 Jul 2023 11:18:40 +0200 Subject: [PATCH 52/52] debug fix --- lib/models/entitiesStore.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/models/entitiesStore.js b/lib/models/entitiesStore.js index 2c33f4b1..6d0f775c 100644 --- a/lib/models/entitiesStore.js +++ b/lib/models/entitiesStore.js @@ -100,7 +100,6 @@ function findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, ca filter.offset += filter.limit; findSilentEntitiesByAPIWithPagination(connection, filter, alterFunc, callback); } else { - logger.info('%d entities found using findSilentEntitiesByAPI', response.results.length); // All entities have been retrieved and processed, call the callback callback(null, response.results); } @@ -180,7 +179,6 @@ function findSilentEntitiesByMongo(service, subservice, ruleData, alterFunc, cal } ], function(err, result) { - logger.info('%d entities found using findSilentEntitiesByMongo', entityCount); logger.debug(context, 'findSilentEntities %s', myutils.firstChars(result)); return callback(err, result); } @@ -198,7 +196,7 @@ function findSilentEntities(service, subservice, ruleData, alterFunc, callback) var hrend = process.hrtime(hrstart); // Log the execution time in seconds and milliseconds - logger.info('%s execution time (hr): %dms', method, hrend[1] / 1000000); + logger.debug('%s has found %d entities in (hr): %d ms', method, result.length, hrend[1] / 1000000); // Call the original callback callback(err, result);