add fiware-total-count header

CHANGES_NEXT_RELEASE

@@ -1,3 +1,4 @@
 - Upgrade mongodb dependence to 2.2.35
 - Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431]
 - Fix: Content Disposition header encoding issue (#433)
+- Add count to query and related fiware-total-count header [#428]

doc/manuals/raw-data-retrieval.md

@@ -19,6 +19,7 @@ The requests for raw context information can use the following query parameters:
 * **dateFrom**: The starting date and time from which the raw context information is desired. It is an optional parameter.
 * **dateTo**: The final date and time until which the raw context information is desired. It is an optional parameter.
 * **filetype**: The raw context information can be requested as a file setting this query parameter to the desired file type. Currently, the only supported value and file type is `csv`.  It is an optional parameter.
+* **count**: The total count of elements could be asked using this query parameter. Supported values are `true` or `false`. As a result reesponse will include a new header: Fiware-Total-Count. It is an optional parameter.
 
 In order to avoid problems handing big results there is a restriction about the the number of results per page that could be retrieved.
 The rule is  ```hLimit <= lastN <= config.maxPageSize```

lib/configuration/sthConfiguration.js

@@ -41,6 +41,7 @@ module.exports = {
     CORRELATOR: 'fiware-correlator',
     FIWARE_SERVICE: 'fiware-service',
     FIWARE_SERVICE_PATH: 'fiware-servicepath',
+    FIWARE_TOTAL_COUNT: 'fiware-total-count',
     X_REAL_IP: 'x-real-ip'
   },
   OPERATION_TYPE_PREFIX: 'OPER_STH_',

lib/database/sthDatabase.js

@@ -459,6 +459,7 @@ function getRawData(data, callback) {
   }
 
   var cursor;
+  var totalCount=0;
   if (lastN || lastN === 0) {
     cursor = collection.find(
       findCondition,
@@ -468,15 +469,19 @@ function getRawData(data, callback) {
         attrValue: 1,
         recvTime: 1
       }
-    ).sort({'recvTime': -1}).limit(lastN);
+    ).sort({'recvTime': -1});
+    cursor.count(function(err, count) {
+      totalCount=count;
+    });
+    cursor = cursor.limit(lastN);
     if (filetype === 'csv') {
       generateCSV(attrName, cursor.stream(), callback);
     } else {
       cursor.toArray(function (err, results) {
         if (!err) {
           results.reverse();
         }
-        return process.nextTick(callback.bind(null, err, results));
+        return process.nextTick(callback.bind(null, err, results, totalCount));
       });
     }
   } else if (hOffset || hLimit) {
@@ -488,11 +493,17 @@ function getRawData(data, callback) {
         attrValue: 1,
         recvTime: 1
       }
-    ).sort({'recvTime': 1}).skip(hOffset || 0).limit(hLimit || 0);
+    ).sort({'recvTime': 1});
+    cursor.count(function(err, count) {
+      totalCount=count;
+    });
+    cursor = cursor.skip(hOffset || 0).limit(hLimit || 0);
     if (filetype === 'cvs') {
       generateCSV(attrName, cursor.stream(), callback);
     } else {
-      cursor.toArray(callback);
+      cursor.toArray(function (err, results) {
+        return process.nextTick(callback.bind(null, err, results, totalCount));
+      });
     }
   } else {
     cursor = collection.find(
@@ -504,10 +515,15 @@ function getRawData(data, callback) {
         recvTime: 1
       }
     );
+    cursor.count(function(err, count) {
+      totalCount=count;
+    });
     if (filetype === 'csv') {
       generateCSV(attrName, cursor.stream(), callback);
     } else {
-      cursor.toArray(callback);
+      cursor.toArray(function (err, results) {
+        return process.nextTick(callback.bind(null, err, results, totalCount));
+      });
     }
   }
 }

lib/server/handlers/sthGetDataHandler.js

@@ -81,6 +81,9 @@ function getRawData(request, reply) {
           emptyResponse);
         response = reply(ngsiPayload);
         sthServerUtils.addFiwareCorrelator(request, response);
+        if (request.query.count) {
+          sthServerUtils.addFiwareTotalCount(0, response);
+        }
       } else {
         sthLogger.debug(
           request.sth.context,
@@ -100,9 +103,9 @@ function getRawData(request, reply) {
             to: request.query.dateTo,
             filetype: request.query.filetype
           },
-          function (err, result) {
+          function (err, result, totalCount) {
             if (err) {
-              // Error when getting the aggregated data
+              // Error when getting the raw data
               sthLogger.error(
                 request.sth.context,
                 'Error when getting raw data from collection \'%s\'',
@@ -114,10 +117,10 @@ function getRawData(request, reply) {
               );
               response = reply(err);
             } else if (!result || !(result.length || result instanceof stream)) {
-              // No aggregated data available for the request
+              // No raw data available for the request
               sthLogger.debug(
                 request.sth.context,
-                'No aggregated data available for the request: ' + request.url.path
+                'No raw data available for the request: ' + request.url.path
               );
 
               sthLogger.debug(
@@ -181,6 +184,14 @@ function getRawData(request, reply) {
               }
             }
             sthServerUtils.addFiwareCorrelator(request, response);
+            if (request.query.count) {
+              sthLogger.debug(
+                request.sth.context,
+                'totalCount %j',
+                totalCount
+              );
+              sthServerUtils.addFiwareTotalCount(totalCount, response);
+            }
           }
         );
       }
@@ -314,7 +325,7 @@ function getAggregatedData(request, reply) {
 function getDataHandler(request, reply) {
   request.sth = request.sth || {};
   request.sth.context = sthServerUtils.getContext(request);
-
+  var message, error;
   sthLogger.debug(
     request.sth.context,
     request.method.toUpperCase() + ' ' + request.url.path
@@ -330,16 +341,17 @@ function getDataHandler(request, reply) {
         if ( (request.query.lastN && request.query.lastN > sthConfig.MAX_PAGE_SIZE) ||
              (request.query.hLimit && request.query.hLimit > sthConfig.MAX_PAGE_SIZE) ||
              (request.query.hLimit && request.query.lastN && request.query.hLimit > request.query.lastN ) ){
-            var message = 'hLimit <= lastN <= config.maxPageSize';
+            message = 'hLimit <= lastN <= config.maxPageSize';
             sthLogger.warn(
-                request.sth.context,
-                request.method.toUpperCase() + ' ' + request.url.path +
-                    ', error=' + message
+              request.sth.context,
+              request.method.toUpperCase() + ' ' + request.url.path +
+                ', error=' + message
             );
-            var error = boom.badRequest(message);
+            error = boom.badRequest(message);
             error.output.payload.validation = {
-                source: 'query',
-                keys: ['lastN', 'hLimit', 'hOffset', 'filetype', 'aggrMethod', 'aggrPeriod']
+              source: 'query',
+              keys: ['lastN', 'hLimit', 'hOffset', 'filetype', 'aggrMethod',
+                     'aggrPeriod', 'count']
             };
             return reply(error);
         }
@@ -349,17 +361,18 @@ function getDataHandler(request, reply) {
     // Aggregated data is requested
     getAggregatedData(request, reply);
   } else {
-    var message = 'A combination of the following query params is required: lastN, hLimit and hOffset, ' +
+    message = 'A combination of the following query params is required: lastN, hLimit and hOffset, ' +
       'filetype, or aggrMethod and aggrPeriod';
     sthLogger.warn(
       request.sth.context,
       request.method.toUpperCase() + ' ' + request.url.path +
-      ', error=' + message
+        ', error=' + message
     );
-    var error = boom.badRequest(message);
+    error = boom.badRequest(message);
     error.output.payload.validation = {
       source: 'query',
-      keys: ['lastN', 'hLimit', 'hOffset', 'filetype', 'aggrMethod', 'aggrPeriod']
+      keys: ['lastN', 'hLimit', 'hOffset', 'filetype', 'aggrMethod', 'aggrPeriod',
+             'count']
     };
     return reply(error);
   }

lib/server/sthServer.js

@@ -99,7 +99,8 @@ function doStartServer(host, port, callback) {
             aggrPeriod: joi.string().required().valid('month', 'day', 'hour', 'minute', 'second').optional(),
             dateFrom: joi.date().optional(),
             dateTo: joi.date().optional(),
-            filetype: joi.string().optional()
+            filetype: joi.string().optional(),
+            count: joi.boolean().optional()
           }
         }
       }

lib/server/utils/sthServerUtils.js

@@ -45,6 +45,15 @@ function addFiwareCorrelator(request, response) {
   response.header(sthConfig.HEADER.CORRELATOR, getCorrelator(request) || request.sth.context.trans);
 }
 
+/**
+ * Adds the Fiware-Total-Count header into the response object
+ * @param {object} totalCount  The totalCount
+ * @param {object} response The response
+ */
+function addFiwareTotalCount(totalCount, response) {
+  response.header(sthConfig.HEADER.FIWARE_TOTAL_COUNT, totalCount);
+}
+
 /**
  * Generates the transaction identifier to be used for logging
  * @return {string} The generated transaction
@@ -129,6 +138,7 @@ function getContext(request) {
 
 module.exports = {
   addFiwareCorrelator: addFiwareCorrelator,
+  addFiwareTotalCount: addFiwareTotalCount,
   getContext: getContext,
   getCorrelator: getCorrelator,
   getEmptyResponse: getEmptyResponse,

test/unit/sthTestUtils.js

@@ -428,6 +428,9 @@ function getURL(type, options, attrName) {
       if (options && options.dateTo) {
         url += (getQuerySeparator() + 'dateTo=' + options.dateTo);
       }
+      if (options && options.count) {
+        url += (getQuerySeparator() + 'count=' + options.count);
+      }
       break;
     case sthTestConfig.API_OPERATION.NOTIFY:
       if (options && options.invalidPath) {
@@ -535,6 +538,10 @@ function rawDataAvailableDateFilter(params, done) {
     var bodyJSON = JSON.parse(body);
     expect(err).to.equal(null);
     expect(response.statusCode).to.equal(200);
+    if (options && options.count) {
+      // Check fiware-total-count header
+      expect(response.headers['fiware-total-count']).to.not.be(undefined);
+    }
     expect(bodyJSON.contextResponses[0].contextElement.id).to.equal(sthTestConfig.ENTITY_ID);
     expect(bodyJSON.contextResponses[0].contextElement.isPattern).to.equal(false);
     expect(bodyJSON.contextResponses[0].contextElement.attributes[0].name).to.equal(
@@ -1346,6 +1353,10 @@ function status200Test(options, done) {
     var bodyJSON = JSON.parse(body);
     expect(err).to.equal(null);
     expect(response.statusCode).to.equal(200);
+    if (options && options.count) {
+      // Check fiware-total-count header
+      expect(response.headers['fiware-total-count']).to.not.be(undefined);
+    }
     expect(bodyJSON.contextResponses[0].contextElement.id).to.equal(sthTestConfig.ENTITY_ID);
     expect(bodyJSON.contextResponses[0].contextElement.type).to.equal(sthTestConfig.ENTITY_TYPE);
     expect(bodyJSON.contextResponses[0].contextElement.attributes[0].values).to.be.an(Array);
@@ -1354,6 +1365,7 @@ function status200Test(options, done) {
   });
 }
 
+
 /**
  * Test to check that in case of updating a numeric attribute value aggregated data:
  *  - If the value of the attribute is the same, it is only aggregated once

test/unit/sth_test.js

@@ -163,6 +163,9 @@ describe('sth tests', function() {
 
     it('should respond with 200 - OK if lastN query param', sthTestUtils.status200Test.bind(null, {lastN: 1}));
 
+    it('should respond with 200 - OK if lastN and count query params',
+       sthTestUtils.status200Test.bind(null, {lastN: 1, count: true}));
+
     it('should respond with 200 - OK if hLimit and hOffset query params',
       sthTestUtils.status200Test.bind(
         null,
@@ -199,6 +202,9 @@ describe('sth tests', function() {
     describe('raw data retrieval',
       sthTestUtils.rawDataRetrievalSuite.bind(null, {hLimit: 1, hOffset: i}, 'attribute-float', 'float', true));
 
+    describe('raw data retrieval',
+      sthTestUtils.rawDataRetrievalSuite.bind(null, {hLimit: 1, hOffset: i, count: true}, 'attribute-float', 'float', false));
+
     describe('aggregated data retrieval',
       sthTestUtils.aggregatedDataRetrievalSuite.bind(null, 'attribute-float', 'float', 'min'));