Merge e4e2850 into f732532

RELEASE_NOTES.md

@@ -1,5 +1,12 @@
 # QuantumLeap Release Notes
 
+## 0.7.4
+
+- Fix bug with Custom Time Index header handling (#247)
+- Timescale backend fixes (#243 #246)
+- Bring back /ui endpoint (#229)
+- Update dependencies (#244)
+
 ## 0.7.3
 
 - Relax Crate health check (#239)

src/reporter/reporter.py

@@ -29,16 +29,16 @@
 from flask import request
 from geocoding.geocache import GeoCodingCache
 from requests import RequestException
-from translators.crate import CrateTranslator, CrateTranslatorInstance, \
-    NGSI_TO_CRATE, NGSI_TEXT, NGSI_DATETIME, NGSI_ISO8601
+from translators.crate import NGSI_TO_CRATE, NGSI_TEXT
 from translators.factory import translator_for
-from utils.common import iter_entity_attrs
+from utils.common import iter_entity_attrs, TIME_INDEX_NAME
 import json
 import logging
 import os
 import requests
 from reporter.subscription_builder import build_subscription
-from reporter.timex import select_time_index_value_as_iso
+from reporter.timex import select_time_index_value_as_iso, \
+    TIME_INDEX_HEADER_NAME
 from geocoding.location import normalize_location
 
 
@@ -127,8 +127,9 @@ def notify():
             return error, 400
 
         # Add TIME_INDEX attribute
-        entity[CrateTranslator.TIME_INDEX_NAME] = \
-            select_time_index_value_as_iso(request.headers, entity)
+        custom_index = request.headers.get(TIME_INDEX_HEADER_NAME, None)
+        entity[TIME_INDEX_NAME] = \
+            select_time_index_value_as_iso(custom_index, entity)
 
         # Add GEO-DATE if enabled
         add_geodata(entity)

src/reporter/tests/docker-compose.yml

@@ -6,7 +6,7 @@ services:
     image: fiware/orion:${ORION_VERSION}
     ports:
       - "1026:1026"
-    command: -logLevel DEBUG -dbhost mongo
+    command: -logLevel DEBUG -noCache -dbhost mongo
     depends_on:
       - mongo
     healthcheck:

src/reporter/tests/test_integration.py

@@ -18,7 +18,7 @@ def test_integration(entity, clean_mongo, clean_crate):
     data = json.dumps(entity)
     r = requests.post('{}/entities'.format(ORION_URL), data=data, headers=h)
     assert r.ok
-    time.sleep(2)
+    time.sleep(1)
 
     # Update values in Orion
     for i in range(1, 4):
@@ -35,7 +35,7 @@ def test_integration(entity, clean_mongo, clean_crate):
         endpoint = '{}/entities/{}/attrs'.format(ORION_URL, entity['id'])
         r = requests.patch(endpoint, data=json.dumps(attrs), headers=h)
         assert r.ok
-        time.sleep(2)
+        time.sleep(1)
 
     # Query in Quantumleap
     query_params = {
@@ -48,7 +48,66 @@ def test_integration(entity, clean_mongo, clean_crate):
     r = requests.get(query_url, params=query_params)
     assert r.status_code == 200, r.text
     data = r.json()
-    assert len(data['index']) == 4
+    assert len(data['index']) > 1
     assert len(data['attributes']) == 2
-    assert data['attributes'][0]['values'] == [720.0, 721.0, 722.0, 723.0]
-    assert data['attributes'][1]['values'] == [24.2, 25.2, 26.2, 27.2]
+
+    # Note some notifications may have been lost
+    pressures = data['attributes'][0]['values']
+    assert set(pressures).issubset(set([720.0, 721.0, 722.0, 723.0]))
+    temperatures = data['attributes'][1]['values']
+    assert set(temperatures).issubset(set([24.2, 25.2, 26.2, 27.2]))
+
+
+def test_integration_custom_index(entity, clean_mongo, clean_crate):
+    # Subscribe QL to Orion
+    params = {
+        'orionUrl': ORION_URL,
+        'quantumleapUrl': QL_URL,
+        'timeIndexAttribute': 'myCustomIndex'
+    }
+    r = requests.post("{}/subscribe".format(QL_URL), params=params)
+    assert r.status_code == 201
+
+    # Insert values in Orion
+    entity['myCustomIndex'] = {
+        'value': '2019-08-22T18:22:00',
+        'type': 'DateTime',
+        'metadata': {}
+    }
+    entity.pop('temperature')
+    entity.pop('pressure')
+
+    data = json.dumps(entity)
+    h = {'Content-Type': 'application/json'}
+    r = requests.post('{}/entities'.format(ORION_URL), data=data, headers=h)
+    assert r.ok
+    time.sleep(1)
+
+    # Update values in Orion
+    for i in range(1, 4):
+        attrs = {
+            'myCustomIndex': {
+                'value': '2019-08-22T18:22:0{}'.format(i),
+                'type': 'DateTime',
+            },
+        }
+        endpoint = '{}/entities/{}/attrs'.format(ORION_URL, entity['id'])
+        r = requests.patch(endpoint, data=json.dumps(attrs), headers=h)
+        assert r.ok
+        time.sleep(1)
+
+    # Query in Quantumleap
+    query_params = {
+        'type': entity['type'],
+    }
+    query_url = "{qlUrl}/entities/{entityId}".format(
+        qlUrl=QL_URL,
+        entityId=entity['id'],
+    )
+    r = requests.get(query_url, params=query_params)
+    assert r.status_code == 200, r.text
+
+    data = r.json()
+    # Note some notifications may have been lost
+    assert data['attributes'][0]['values'] == data['index']
+    assert len(data['index']) > 1

src/reporter/tests/test_timex.py

@@ -1,5 +1,6 @@
 from datetime import *
 from reporter.timex import *
+import pytest
 
 
 def build_notification(custom, ti, ts, dm,
@@ -62,21 +63,15 @@ def build_notification_timepoints(base_point):
 
 
 def test_custom_index_takes_priority():
-    headers = {
-        TIME_INDEX_HEADER_NAME: 'customTimeIndex'
-    }
     custom_time_index_value = datetime(2019, 1, 1)
     ts = build_notification_timepoints(custom_time_index_value)
     notification = build_notification(*ts)
 
     assert custom_time_index_value == \
-        select_time_index_value(headers, notification)
+        select_time_index_value('customTimeIndex', notification)
 
 
 def test_use_latest_meta_custom_index():
-    headers = {
-        TIME_INDEX_HEADER_NAME: 'customTimeIndex'
-    }
     base_point = datetime(2019, 1, 1)
 
     ts = build_notification_timepoints(base_point)
@@ -86,13 +81,10 @@ def test_use_latest_meta_custom_index():
 
     # latest meta custom index slot = 8
     assert ts[8] == \
-        select_time_index_value(headers, notification).isoformat()
+        select_time_index_value('customTimeIndex', notification).isoformat()
 
 
 def test_skip_custom_index_if_it_has_no_value():
-    headers = {
-        TIME_INDEX_HEADER_NAME: 'customTimeIndex'
-    }
     base_point = datetime(2019, 1, 1)
     ts = build_notification_timepoints(base_point)
     ts[0] = None  # custom index slot
@@ -101,21 +93,19 @@ def test_skip_custom_index_if_it_has_no_value():
     notification = build_notification(*ts)
 
     assert ts[1] == \
-        select_time_index_value(headers, notification).isoformat()
+        select_time_index_value('customTimeIndex', notification).isoformat()
 
 
 def test_use_time_instant():
-    headers = {}
     base_point = datetime(2019, 1, 1)
     ts = build_notification_timepoints(base_point)
     notification = build_notification(*ts)
 
     assert ts[1] == \
-        select_time_index_value(headers, notification).isoformat()
+        select_time_index_value(None, notification).isoformat()
 
 
 def test_use_latest_meta_time_instant():
-    headers = {}
     base_point = datetime(2019, 1, 1)
 
     ts = build_notification_timepoints(base_point)
@@ -125,11 +115,10 @@ def test_use_latest_meta_time_instant():
 
     # latest meta time instant slot = 9
     assert ts[9] == \
-        select_time_index_value(headers, notification).isoformat()
+        select_time_index_value(None, notification).isoformat()
 
 
 def test_use_timestamp():
-    headers = {}
     base_point = datetime(2019, 1, 1)
     ts = build_notification_timepoints(base_point)
     ts[1] = None  # TimeInstant slot
@@ -138,11 +127,10 @@ def test_use_timestamp():
     notification = build_notification(*ts)
 
     assert ts[2] == \
-        select_time_index_value(headers, notification).isoformat()
+        select_time_index_value(None, notification).isoformat()
 
 
 def test_use_latest_meta_timestamp():
-    headers = {}
     base_point = datetime(2019, 1, 1)
 
     ts = build_notification_timepoints(base_point)
@@ -155,11 +143,10 @@ def test_use_latest_meta_timestamp():
 
     # latest meta time instant slot = 10
     assert ts[10] == \
-        select_time_index_value(headers, notification).isoformat()
+        select_time_index_value(None, notification).isoformat()
 
 
 def test_use_date_modified():
-    headers = {}
     base_point = datetime(2019, 1, 1)
 
     ts = build_notification_timepoints(base_point)
@@ -174,11 +161,10 @@ def test_use_date_modified():
 
     # date modified slot = 3
     assert ts[3] == \
-        select_time_index_value(headers, notification).isoformat()
+        select_time_index_value(None, notification).isoformat()
 
 
 def test_use_latest_meta_date_modified():
-    headers = {}
     base_point = datetime(2019, 1, 1)
 
     ts = build_notification_timepoints(base_point)
@@ -194,13 +180,12 @@ def test_use_latest_meta_date_modified():
 
     # latest meta date modified slot = 11
     assert ts[11] == \
-        select_time_index_value(headers, notification).isoformat()
+        select_time_index_value(None, notification).isoformat()
 
 
 def test_use_default_value():
-    headers = {}
     notification = build_notification(*([None] * 12))
 
-    actual = select_time_index_value(headers, notification)
+    actual = select_time_index_value(None, notification)
     diff = datetime.now() - actual
     assert diff < timedelta(seconds=2)

src/reporter/tests/test_version.py

@@ -7,5 +7,5 @@ def test_version():
     r = requests.get('{}'.format(version_url))
     assert r.status_code == 200, r.text
     assert r.json() == {
-        "version": "0.7.3"
+        "version": "0.7.4"
     }

src/reporter/timex.py

@@ -1,8 +1,7 @@
 from datetime import datetime
 from typing import Iterable, Union
-
 from utils.common import iter_entity_attrs
-from utils.jsondict import maybe_value
+from utils.jsondict import maybe_value, maybe_string_match
 from utils.timestr import latest_from_str_rep, to_datetime
 
 TIME_INDEX_HEADER_NAME = 'Fiware-TimeIndex-Attribute'
@@ -32,18 +31,16 @@ def _iter_metadata(notification: dict, meta_name: str) -> Iterable[MaybeString]:
         yield _meta_attribute(notification, attr_name, meta_name)
 
 
-def time_index_priority_list(headers: dict, notification: dict) -> datetime:
+def time_index_priority_list(custom_index: str, notification: dict) -> datetime:
     """
     Returns the next possible time_index value using the strategy described in
     the function select_time_index_value.
     """
-    custom_attribute = maybe_value(headers, TIME_INDEX_HEADER_NAME)
-
     # Custom time index attribute
-    yield to_datetime(_attribute(notification, custom_attribute))
+    yield to_datetime(_attribute(notification, custom_index))
 
     # The most recent custom time index metadata
-    yield latest_from_str_rep(_iter_metadata(notification, custom_attribute))
+    yield latest_from_str_rep(_iter_metadata(notification, custom_index))
 
     # TimeInstant attribute
     yield to_datetime(_attribute(notification, "TimeInstant"))
@@ -64,7 +61,7 @@ def time_index_priority_list(headers: dict, notification: dict) -> datetime:
     yield latest_from_str_rep(_iter_metadata(notification, "dateModified"))
 
 
-def select_time_index_value(headers: dict, notification: dict) -> datetime:
+def select_time_index_value(custom_index: str, notification: dict) -> datetime:
     """
     Determine which attribute or metadata value to use as a time index for the
     entity being notified.
@@ -93,23 +90,23 @@ def select_time_index_value(headers: dict, notification: dict) -> datetime:
       present or none of the values found can actually be converted to a
       ``datetime``.
 
-    :param headers: the HTTP headers as received from Orion.
+    :param custom_index: name of the custom_index (if requested, None otherwise)
     :param notification: the notification JSON payload as received from Orion.
     :return: the value to be used as time index.
     """
     current_time = datetime.now()
 
-    for time_index_candidate in time_index_priority_list(headers, notification):
-        if time_index_candidate:
-            return time_index_candidate
+    for index_candidate in time_index_priority_list(custom_index, notification):
+        if index_candidate:
+            return index_candidate
 
     # use the current time as a last resort
     return current_time
 
 
-def select_time_index_value_as_iso(headers: dict, notification: dict) -> str:
+def select_time_index_value_as_iso(custom_index: str, notification: dict) -> str:
     """
     Same as ``select_time_index_value`` but formats the returned ``datetime``
     as an ISO 8601 string.
     """
-    return select_time_index_value(headers, notification).isoformat()
+    return select_time_index_value(custom_index, notification).isoformat()

src/reporter/version.py

@@ -2,5 +2,5 @@
 
 def version():
     return {
-        'version': '0.7.3'
+        'version': '0.7.4'
     }