Merge pull request #6 from governify/periods

Periods calculation refactoring
governify · Sep 12, 2021 · 8c8f745 · 8c8f745
2 parents ee36fe9 + 1cd631d
commit 8c8f745
Show file tree

Hide file tree

Showing 10 changed files with 140 additions and 75 deletions.
diff --git a/config/index.js b/config/index.js
@@ -1,14 +1,14 @@
 'use strict';
 
-var jsyaml = require('js-yaml');
+const jsyaml = require('js-yaml');
 const fs = require('fs');
 const path = require('path');
 const mustache = require('mustache');
 mustache.escape = function (text) { return text; };
 
-var configStringTemplate = fs.readFileSync(path.resolve(__dirname, './credentials/credentials.yaml'), 'utf8');
-var configString = mustache.render(configStringTemplate, process.env, {}, ['$_[', ']']);
-var credentials, config;
+const configStringTemplate = fs.readFileSync(path.resolve(__dirname, './credentials/credentials.yaml'), 'utf8');
+const configString = mustache.render(configStringTemplate, process.env, {}, ['$_[', ']']);
+let credentials, config;
 if (configString) {
   credentials = jsyaml.safeLoad(configString);
 }

diff --git a/controllers/apiv2computationsController.js b/controllers/apiv2computationsController.js
@@ -1,6 +1,6 @@
 'use strict';
 
-var varapiv2computationsController = require('./apiv2computationsControllerService');
+const varapiv2computationsController = require('./apiv2computationsControllerService');
 
 module.exports.addComputation = function addComputation (req, res, next) {
   varapiv2computationsController.addComputation(req.swagger.params, res, next);

diff --git a/controllers/apiv2computationsControllerService.js b/controllers/apiv2computationsControllerService.js
@@ -69,8 +69,8 @@ const validateInput = (dsl) => {
       const end = dsl.metric.window.end;
 
       // ISO8601 Date validation
-      var iso8601 = /^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\.[0-9]+)?(Z)?$/;
-      var iso8601Millis = /^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\.[0-9]+)?\.([0-9][0-9][0-9])(Z)?$/;
+      const iso8601 = /^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\.[0-9]+)?(Z)?$/;
+      const iso8601Millis = /^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\.[0-9]+)?\.([0-9][0-9][0-9])(Z)?$/;
 
       if (Date.parse(end) - Date.parse(initial) < 0) {
         reject(new Error('End period date must be later than the initial.'));
@@ -92,45 +92,45 @@ const getPeriods = (dsl) => {
     try {
       const initial = dsl.metric.window.initial;
       const end = dsl.metric.window.end;
-      const windowPeriod = dsl.metric.window.period;
 
       // Translate period string to actual days and obtain number of periods
-      const periodLengths = {
-        daily: 1,
-        weekly: 7,
-        biweekly: 14,
-        monthly: 30,
-        bimonthly: 60,
-        annually: 365
-      };
-      const periodLength = periodLengths[windowPeriod];
-      if (periodLength === undefined) { reject(new Error('metric.window.period must be within these: daily, weekly, biweekly, monthly, bimonthly, annually.')); }
+      // const periodLengths = {
+      //  daily: 1,
+      //  weekly: 7,
+      //  biweekly: 14,
+      //  monthly: 30,
+      //  bimonthly: 60,
+      //  annually: 365
+      // };
+      // const periodLength = periodLengths[windowPeriod];
+      // if (periodLength === undefined) { reject(new Error('metric.window.period must be within these: daily, weekly, biweekly, monthly, bimonthly, annually.')); }
 
       // Obtain periods
       const periods = [];
 
-      let fromStr = initial;
-      let toDate;
-      let toStr;
-
-      let keepGoing = true;
-      while (keepGoing) {
-        // Set from after each iteration
-        if (toStr !== undefined) {
-          fromStr = toStr;
-        }
-
-        // Check if to is after end of periods
-        toDate = new Date(Date.parse(fromStr) + periodLength * 24 * 60 * 60 * 1000);
-        if (toDate >= new Date(Date.parse(end))) {
-          toDate = new Date(Date.parse(end));
-          keepGoing = false;
-        }
-        toStr = toDate.toISOString();
-
-        // Push into the array
-        periods.push({ from: fromStr, to: toStr });
-      }
+      // let fromStr = initial;
+      // let toDate;
+      // let toStr;
+      //
+      // let keepGoing = true;
+      // while (keepGoing) {
+      //  // Set from after each iteration
+      //  if (toStr !== undefined) {
+      //    fromStr = toStr;
+      //  }
+      //
+      //  // Check if to is after end of periods
+      //  toDate = new Date(Date.parse(fromStr) + periodLength * 24 * 60 * 60 * 1000);
+      //  if (toDate >= new Date(Date.parse(end))) {
+      //    toDate = new Date(Date.parse(end));
+      //    keepGoing = false;
+      //  }
+      //  toStr = toDate.toISOString();
+      //
+      //  // Push into the array
+      //  periods.push({ from: fromStr, to: toStr });
+      // }
+      periods.push({ from: initial, to: end });
 
       resolve(periods);
     } catch (err) {

diff --git a/controllers/apiv2computationscomputationIdController.js b/controllers/apiv2computationscomputationIdController.js
@@ -1,6 +1,6 @@
 'use strict';
 
-var varapiv2computationscomputationIdController = require('./apiv2computationscomputationIdControllerService');
+const varapiv2computationscomputationIdController = require('./apiv2computationscomputationIdControllerService');
 
 module.exports.findComputationById = function findComputationById (req, res, next) {
   varapiv2computationscomputationIdController.findComputationById(req.swagger.params, res, next);

diff --git a/controllers/computationCalculator/flavours/collectordynamic.js b/controllers/computationCalculator/flavours/collectordynamic.js
@@ -10,9 +10,9 @@ async function applyStep (dsl, period, inputs, responseList) {
   return new Promise(async function (resolve, reject) {
     logger.debug('Applying next step');
     const url = dsl.config.url + inputs.request.endpoint;
-    var periodfromms = new Date(period.from).getTime();
+    const periodfromms = new Date(period.from).getTime();
     // var periodtoms = new Date(period.to).getTime();
-    var realperiod = {};
+    let realperiod = {};
     logger.debug('DSL: ' + JSON.stringify(dsl));
     if (dsl.metric.scope.periodo_de_actividad === 'alta') {
       realperiod = {
@@ -38,17 +38,17 @@ async function applyStep (dsl, period, inputs, responseList) {
     }).then(response => { return response.data; }).catch(logger.info);
     logger.debug('RESPONSE' + JSON.stringify(res));
 
-    var resultList = Object.getPropertyByString(res, 'aggregations.services.buckets');
-    var finalResult = [];
+    const resultList = Object.getPropertyByString(res, 'aggregations.services.buckets');
+    const finalResult = [];
     resultList.forEach(rs => {
-      var newScope = Object.assign({}, dsl.metric.scope);
+      const newScope = Object.assign({}, dsl.metric.scope);
       newScope.servicio = rs.key;
-      var metricResult = rs.avgresponsetime ? rs.avgresponsetime.value : rs.doc_count;
+      const metricResult = rs.avgresponsetime ? rs.avgresponsetime.value : rs.doc_count;
       finalResult.push({ evidences: res.hits.hits, metric: metricResult != null ? metricResult : 0, scope: newScope });
     });
 
     // Add the result to the current result from previous steps.
-    var resultConcat = responseList.concat(finalResult);
+    const resultConcat = responseList.concat(finalResult);
     resolve(resultConcat);
   });
 }

diff --git a/controllers/computationCalculator/flavours/scopeGroup.js b/controllers/computationCalculator/flavours/scopeGroup.js
@@ -17,7 +17,7 @@ const logger = governify.getLogger().tag('scopeGroup');
 async function applyStep (dsl, period, inputs, responseList) {
   return new Promise(async function (resolve, reject) {
     // Create the inverted map with the assocations groups
-    var invertedMap = {};
+    const invertedMap = {};
     Object.keys(inputs.groups).forEach(groupName => {
       inputs.groups[groupName].forEach(servicio => {
         invertedMap[servicio] = groupName;

diff --git a/controllers/computationCalculator/index.js b/controllers/computationCalculator/index.js
@@ -7,13 +7,13 @@ const logger = governify.getLogger().tag('computationCalculator');
 module.exports.compute = async (dsl, period) => {
   return new Promise(async (resolve, reject) => {
     try {
-      var responseList = [];
+      let responseList = [];
 
       for (const stepKey in dsl.metric.params.steps) {
-        var stepModel = dsl.metric.params.steps[stepKey];
+        const stepModel = dsl.metric.params.steps[stepKey];
         logger.info('Executing STEP: ' + JSON.stringify(stepModel));
 
-        var stepModule = await governify.utils.requireFromFileOrURL(stepModel.script);
+        const stepModule = await governify.utils.requireFromFileOrURL(stepModel.script);
         responseList = await stepModule.applyStep(dsl, period, stepModel.inputs, responseList);
         logger.info('STEPKEY: ' + stepKey + ' R-> ' + JSON.stringify(responseList));
       }

diff --git a/objectUtils.js b/objectUtils.js
@@ -1,9 +1,9 @@
 Object.getPropertyByString = function (o, s) {
   s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
   s = s.replace(/^\./, ''); // strip a leading dot
-  var a = s.split('.');
-  for (var i = 0, n = a.length; i < n; ++i) {
-    var k = a[i];
+  const a = s.split('.');
+  for (let i = 0, n = a.length; i < n; ++i) {
+    const k = a[i];
     if (k in o) {
       o = o[k];
     } else {

diff --git a/server.js b/server.js
@@ -5,26 +5,26 @@ const { logger } = require('oas-tools/src/configurations');
 const deploy = (env, commonsMiddleware) => {
   return new Promise((resolve, reject) => {
     try {
-      var fs = require('fs');
-      var http = require('http');
-      var path = require('path');
+      const fs = require('fs');
+      const http = require('http');
+      const path = require('path');
       const governify = require('governify-commons');
       const logger = governify.getLogger().tag('initialization');
-      var express = require('express');
-      var app = express();
+      const express = require('express');
+      const app = express();
       app.use(commonsMiddleware);
-      var bodyParser = require('body-parser');
+      const bodyParser = require('body-parser');
       app.use(bodyParser.json({
         strict: false
       }));
-      var oasTools = require('oas-tools');
-      var jsyaml = require('js-yaml');
-      var serverPort = process.env.PORT || 5501;
+      const oasTools = require('oas-tools');
+      const jsyaml = require('js-yaml');
+      const serverPort = process.env.PORT || 5501;
 
-      var spec = fs.readFileSync(path.join(__dirname, '/api/oas-doc.yaml'), 'utf8');
-      var oasDoc = jsyaml.safeLoad(spec);
+      const spec = fs.readFileSync(path.join(__dirname, '/api/oas-doc.yaml'), 'utf8');
+      const oasDoc = jsyaml.safeLoad(spec);
 
-      var optionsObject = {
+      const optionsObject = {
         controllers: path.join(__dirname, './controllers'),
         loglevel: env === 'test' ? 'error' : 'info',
         strict: false,

diff --git a/tests/test.js b/tests/test.js
@@ -3,8 +3,10 @@ var request = require('request');
 var server = require('../server');
 const governify = require('governify-commons');
 const logger = governify.getLogger().tag('tests')
+const fs = require('fs');
+const path = require('path');
 
-let computationEP;
+var computationEP;
 
 describe('Array', function () {
     before((done) => {
@@ -29,7 +31,7 @@ function apiRestRequestControllersTest() {
     it('should respond with 201 Created on POST and have computation in body', function (done) {
         try {
             const options = {
-                url: 'http://localhost:8081/api/v2/computations',
+                url: 'http://localhost:5501/api/v2/computations',
                 json: postData1,
                 headers: {
                     'User-Agent': 'request',
@@ -55,7 +57,7 @@ function apiRestRequestControllersTest() {
         try {
             assert.notStrictEqual(undefined, computationEP);
 
-            getComputationV2('http://localhost:8081' + computationEP, 20000).then(computations => {
+            getComputationV2('http://localhost:5501' + computationEP, 20000).then(computations => {
                 try {
                     assert.deepStrictEqual(resData1, computations);
                     done();
@@ -86,6 +88,65 @@ const initializeDataAndServer = (done) => {
                 "period": "daily",
                 "type": "string",
                 "end": "2019-01-04T15:00:00.000Z"
+            },
+            "params": {
+                "steps": {
+                    "step1": {
+                        "type": "script",
+                        "script": "http://host.docker.internal:5200/api/v1/public/collector/steps/request.js",
+                        "inputs": {
+                            "request": {
+                                "endpoint": "filebeat-*/_search",
+                                "body": {
+                                    "size": 0,
+                                    "track_total_hits": true,
+                                    "sort": [
+                                        {
+                                            "TIMESTAMP-NEXO-DATE": "desc"
+                                        }
+                                    ],
+                                    "query": {
+                                        "bool": {
+                                            "must": 
+                                                {
+                                                    "range": {
+                                                        "TIMESTAMP-NEXO-DATE": {
+                                                            "gt": ">>>period.from<<<",
+                                                            "lt": ">>>period.to<<<"
+                                                        }
+                                                    }
+                                                },
+                                            "must_not":
+                                                {
+                                                    "prefix": {
+                                                        "CODE_ERROR": "2"
+                                                    }
+                                                }
+
+                                        }
+                                    },
+                                    "aggs": {
+                                        "services": {
+                                            "terms": {
+                                                "field": "PROXY.keyword",
+                                                "size": 999
+                                            }
+                                        }
+                                    }
+                                },
+                                "result": {
+                                    "type": "array",
+                                    "valueAddress": "aggregations.services.buckets",
+                                    "scopeVar": {
+                                        "$ref": "#/context/definitions/scopes/nexo/servicio"
+                                    },
+                                    "scopeKey": "key",
+                                    "scopeValue": "doc_count"
+                                }
+                            }
+                        }
+                    }
+                }
             }
         },
         "config": {}
@@ -144,11 +205,15 @@ const initializeDataAndServer = (done) => {
 
 
     // Server initialization
-    server.deploy('test').then(() => {
-        done();
-    }).catch((err) => {
-        logger.error(err);
-        done();
+    governify.init().then((commonsMiddleware) => {
+        server.deploy('test', commonsMiddleware).then(() => {
+            governify.httpClient.setRequestLogging(false);
+            //sinon.stub(console);
+            done();
+        }).catch(err1 => {
+            console.log(err1.message)
+            done(err1);
+        });
     });
 
 }