fix timestamp mapped attribute cases

CHANGES_NEXT_RELEASE

@@ -1,3 +1,5 @@
+- Fix: TimeInstant mapped from measure overrides system TimeInstant (#1557)
 - Fix: reduce information showed handling errors to just config flags (#1594)
 - Upgrade express dep from 4.18.1 to 4.19.2
 - Add: allow devices with the same device_id in the same service and subservice but different apikey (#1589)
+

doc/api.md

@@ -157,8 +157,8 @@ parameters defined at device level in database, the parameters are inherit from
 
 Group service uniqueness is defined by the combination of: service, subservice and apikey
 
-Device uniqueness is defined by the combination of: service, subservice, device_id and apikey. Note that several
-devices with the same device_id are allowed in the same service and subservice as long as their apikeys are different.
+Device uniqueness is defined by the combination of: service, subservice, device_id and apikey. Note that several devices
+with the same device_id are allowed in the same service and subservice as long as their apikeys are different.
 
 ## Special measures and attributes names
 
@@ -972,32 +972,37 @@ Will now generate the following NGSI v2 payload:
 
 ## Timestamp Processing
 
-The IOTA processes the entity attributes looking for a `TimeInstant` attribute. If one is found, for NGSI v2, then it
-adds a `TimeInstant` attribute as metadata for every other attribute in the same request. With NGSI-LD, the Standard
-`observedAt` property-of-a-property is used instead.
+Timestamp processing done by IOTA is as follows:
 
-If a `TimeInstant` arrives as measure but not follows [ISO_8601](https://en.wikipedia.org/wiki/ISO_8601) then measure is
-refused.
+* An attribute `TimeInstant` is added to updated entities
+* In the case of NGSI-v2, a `TimeInstant` metadata is added in each updated attribute. With NGSI-LD, the Standard
+`observedAt` property-of-a-property is used instead.
 
 Depending on the `timestamp` configuration and if the measure contains a value named `TimeInstant` with a correct value,
 the IoTA behaviour is described in the following table:
 
-| `timestamp` value | measure contains `TimeInstant` | Behaviour                                              |
-| ----------------- | ------------------------------ | ------------------------------------------------------ |
-| true              | Yes                            | TimeInstant and metadata updated with measure value    |
-| true              | No                             | TimeInstant and metadata updated with server timestamp |
-| false             | Yes                            | TimeInstant and metadata updated with measure value    |
-| false             | No                             | TimeInstant and metadata updated with server timestamp |
-| Not defined       | Yes                            | TimeInstant and metadata updated with measure value    |
-| Not defined       | No                             | TimeInstant and metadata updated with server timestamp |
+| `timestamp` conf value | measure contains `TimeInstant` | Behaviour                                              |
+| ---------------------- | ------------------------------ | ------------------------------------------------------ |
+| true                   | Yes                            | TimeInstant and metadata updated with measure value    |
+| true                   | No                             | TimeInstant and metadata updated with server timestamp |
+| false                  | Yes                            | TimeInstant and metadata updated with measure value    |
+| false                  | No                             | TimeInstant and metadata updated with server timestamp |
+| Not defined            | Yes                            | TimeInstant and metadata updated with measure value    |
+| Not defined            | No                             | TimeInstant and metadata updated with server timestamp |
 
-The `timestamp` value used is:
+The `timestamp` conf value used is:
 
 -   The one defined at device level
 -   The one defined at group level (if not defined at device level)
 -   The one defined at [IoTA configuration level](admin.md#timestamp) / `IOTA_TIMESTAMP` env var (if not defined at
     group level or device level)
 
+Some additional considerations to take into account:
+
+* If there is an attribute which maps a measure to `TimeInstant` attribute (after [expression evaluation](#expression-language-support) if any is defined), then that value will be used as `TimeInstant,
+overwriting the above rules specified in "Behaviour" column. Note that an expression in the  could be used in that mapping.
+* If the resulting `TimeInstant` not follows [ISO_8601](https://en.wikipedia.org/wiki/ISO_8601) (either from a direct measure of after a mapping, as described in the previous bullet) then it is refused (so a failover to server timestamp will take place).
+
 ## Overriding global Context Broker host
 
 **cbHost**: Context Broker host URL. This option can be used to override the global CB configuration for specific types

lib/services/ngsi/entities-NGSI-v2.js

@@ -277,7 +277,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call
     let entities = {}; //{entityName:{entityType:[attrs]}}  //SubGoal Populate entoties data striucture
     let jexlctxt = {}; //will store the whole context (not just for JEXL)
     let payload = {}; //will store the final payload
-    let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions.
     let plainMeasures = null; //will contain measures POJO
     let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation);
 
@@ -308,43 +307,16 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call
     jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt);
 
     //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on)
-    const mustInsertTimeInstant = 
-        typeInformation.timestamp !== undefined 
-            ? typeInformation.timestamp 
-            : false;
-
-    if (mustInsertTimeInstant) {
-        //remove TimeInstant from measures
-        measures = measures.filter((item) => item.name !== constants.TIMESTAMP_ATTRIBUTE);
-
-        if (plainMeasures[constants.TIMESTAMP_ATTRIBUTE]) {
-            //if it comes from a measure
-            if (moment(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], moment.ISO_8601, true).isValid()) {
-                timestamp.value = plainMeasures[constants.TIMESTAMP_ATTRIBUTE];
-            } else {
-                callback(
-                    new errors.BadTimestamp(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], entityName, typeInformation)
-                );
-                return;
-            }
-        } else if (!typeInformation.timezone) {
-            timestamp.value = new Date().toISOString();
-            jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value;
-        } else {
-            timestamp.value = moment().tz(typeInformation.timezone).format('YYYY-MM-DD[T]HH:mm:ss.SSSZ');
-            jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value;
-        }
-    }
+    const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false;
 
     logger.debug(
         context,
-        'sendUpdateValueNgsi2 called with: entityName=%s,  measures=%j,  typeInformation=%j, initial jexlContext=%j, timestamp=%j with value=%j',
+        'sendUpdateValueNgsi2 called with: entityName=%s,  measures=%j,  typeInformation=%j, initial jexlContext=%j, timestamp=%j',
         entityName,
         plainMeasures,
         typeInformation,
         jexlctxt,
-        mustInsertTimeInstant,
-        timestamp.value
+        mustInsertTimeInstant
     );
 
     //Now we can calculate the EntityName of primary entity
@@ -478,15 +450,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call
 
         currentAttr.hitted = hitted;
         currentAttr.value = valueExpression;
-
-        //add TimeInstant to attr metadata
-        if (mustInsertTimeInstant) {
-            if (!currentAttr.metadata) {
-                currentAttr.metadata = {};
-            }
-            currentAttr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp;
-        }
-
         //store de New Attributte in entity data structure
         if (hitted === true) {
             if (entities[attrEntityName] === undefined) {
@@ -527,16 +490,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call
 
     //more mesures may be added to the attribute list (unnhandled/left mesaures) l
     if (explicit === false && Object.keys(measures).length > 0) {
-        //add Timestamp to measures if needed
-        if (mustInsertTimeInstant) {
-            for (let currentMeasure of measures) {
-                if (!currentMeasure.metadata) {
-                    currentMeasure.metadata = {};
-                }
-                currentMeasure.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp;
-            }
-            //If just measures in the principal entity we missed the Timestamp.
-        }
         entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures);
     }
 
@@ -547,11 +500,40 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call
     payload.actionType = 'append';
 
     payload.entities = [];
+    const currentIsoDate = new Date().toISOString();
+    const currentMoment = moment(currentIsoDate);
     for (let ename in entities) {
         for (let etype in entities[ename]) {
             let e = {};
             e.id = String(ename);
             e.type = String(etype);
+            let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions.
+            let timestampAttrs = null;
+            if (mustInsertTimeInstant) {
+                // get timestamp for current entity
+
+                timestampAttrs = entities[ename][etype].filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE);
+                if (timestampAttrs && timestampAttrs.length > 0) {
+                    timestamp.value = timestampAttrs[0]['value'];
+                }
+
+                if (timestamp.value) {
+                    if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) {
+                        callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation));
+                        return;
+                    }
+                } else {
+                    if (!typeInformation.timezone) {
+                        timestamp.value = currentIsoDate;
+                        jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value;
+                    } else {
+                        timestamp.value = currentMoment
+                            .tz(typeInformation.timezone)
+                            .format('YYYY-MM-DD[T]HH:mm:ss.SSSZ');
+                        jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value;
+                    }
+                }
+            }
             //extract attributes
             let isEmpty = true;
             for (let attr of entities[ename][etype]) {
@@ -568,6 +550,15 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call
                                 ))))
                 ) {
                     isEmpty = false;
+                    if (mustInsertTimeInstant) {
+                        // Add TimeInstant to all attribute metadata of all entities
+                        if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) {
+                            if (!attr.metadata) {
+                                attr.metadata = {};
+                            }
+                            attr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp;
+                        }
+                    }
                     e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata };
                 }
             }