Merge d315d5d into 77ed420

.travis.yml

@@ -1,6 +1,6 @@
 language: python
 python:
-  - 3.6
+  - 3.8.3
 
 services:
   - docker

Dockerfile

@@ -1,19 +1,17 @@
-FROM python:3.6.8-alpine3.8 as base
+FROM python:3.8.3-alpine3.12 as base
 FROM base as builder
-RUN apk --no-cache --update-cache add gcc python python-dev py-pip build-base wget
+RUN apk --no-cache --update-cache add gcc python3 python3-dev py-pip build-base wget
 RUN ln -s /usr/include/locale.h /usr/include/xlocale.h
 RUN pip install pipenv
 RUN mkdir -p /src/ngsi-timeseries-api
 COPY Pipfile /src/ngsi-timeseries-api/Pipfile
 COPY Pipfile.lock /src/ngsi-timeseries-api/Pipfile.lock
-RUN mkdir /install
-WORKDIR /install
 RUN cd /src/ngsi-timeseries-api && { pipenv lock -r > /requirements.txt; }
-RUN pip install --install-option="--prefix=/install" -r /requirements.txt
+RUN pip install -r /requirements.txt
 
 FROM base
 RUN apk --no-cache add curl
-COPY --from=builder /install /usr/local
+COPY --from=builder /usr/local /usr/local
 COPY . /src/ngsi-timeseries-api/
 WORKDIR /src/ngsi-timeseries-api/src
 ENV PYTHONPATH=$PWD:$PYTHONPATH

Pipfile

@@ -12,11 +12,11 @@ geocoder = "~=1.33"
 geojson = "~=2.4"
 geomet = "~=0.2"
 influxdb = "~=4.0"
-pg8000 = ">=1.13"
+pg8000 = ">=1.15"
 pymongo = "~=3.4"
 pytest = "~=3.0"
 pytest-cov = "~=2.7.1"
-coveralls = "~=1.7.0"
+coveralls = "~=2.0"
 pytest-flask = "~=0.10"
 python-dateutil = ">=2.7"
 pyyaml = ">=4.2"
@@ -27,4 +27,4 @@ rethinkdb = "==2.3"
 [dev-packages]
 
 [requires]
-python_version = "3.6"
+python_version = "3.8.3"

docker/docker-compose-dev.yml

@@ -38,8 +38,6 @@ services:
     image: grafana/grafana
     ports:
       - "3000:3000"
-    environment:
-      - GF_INSTALL_PLUGINS=crate-datasource,grafana-clock-panel,grafana-worldmap-panel
     depends_on:
       - crate
 
@@ -50,6 +48,27 @@ services:
     volumes:
       - redisdata:/data
 
+  timescale:
+    image: timescale/timescaledb-postgis:latest-pg10
+    ports:
+      - "5432:5432"
+      # Don't expose container port 5432 with the same number outside of the
+      # swarm. In the Travis test env, there's already a PG instance running
+      # on port 5432!
+    environment:
+      - POSTGRES_PASSWORD=*
+
+  quantumleap-db-setup:
+    build: ../timescale-container/
+    image: quantumleap-db-setup
+    depends_on:
+      - timescale
+    environment:
+      - QL_DB_PASS=*
+      - QL_DB_INIT_DIR=/ql-db-init
+      - PG_HOST=timescale
+      - PG_PASS=*
+
 volumes:
   mongodata:
   cratedata:

docker/docker-compose.yml

@@ -53,8 +53,6 @@ services:
     image: grafana/grafana
     ports:
       - "3000:3000"
-    environment:
-      - GF_INSTALL_PLUGINS=crate-datasource,grafana-clock-panel,grafana-worldmap-panel
     depends_on:
       - crate
 

run_tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 docker pull smartsdk/quantumleap
-docker build --cache-from smartsdk/quantumleap -t quantumleap .
+docker build --cache-from smartsdk/quantumleap -t smartsdk/quantumleap .
 
 cd src/translators/tests
 sh run_tests.sh

setup_dev_env.sh

@@ -3,9 +3,12 @@
 # Usage: 
 #      $ source setup_dev_env.sh
 
+# stop at the first command that returns a non-zero exit code.
+set -e
+
 export PYTHONPATH=${PWD}/src:${PYTHONPATH}
 
-docker build -t quantumleap .
+docker build -t smartsdk/quantumleap .
 
 source deps.env
 

src/conftest.py

@@ -6,14 +6,18 @@
 import requests
 
 
-QL_HOST = os.environ.get('QL_HOST', "quantumleap")
+QL_HOST = os.environ.get('QL_HOST', 'quantumleap')
 QL_PORT = 8668
 QL_URL = "http://{}:{}/v2".format(QL_HOST, QL_PORT)
 QL_BASE_URL = "http://{}:{}".format(QL_HOST, QL_PORT)
+QL_DEFAULT_DB = os.environ.get('QL_DEFAULT_DB', 'crate')
 
 CRATE_HOST = os.environ.get('CRATE_HOST', 'crate')
 CRATE_PORT = 4200
 
+POSTGRES_HOST = os.environ.get('POSTGRES_HOST', 'timescale')
+POSTGRES_HOST = 5432
+
 REDIS_HOST = os.environ.get('REDIS_HOST', 'redis')
 REDIS_PORT = 6379
 
@@ -129,7 +133,6 @@ def delete_entities(self, entity_type=None, fiware_service=None,
     with Translator(host=CRATE_HOST, port=CRATE_PORT) as trans:
         yield trans
 
-
 @pytest.fixture
 def entity():
     entity = {
@@ -164,7 +167,7 @@ def sameEntityWithDifferentAttrs():
                 'metadata': {
                     'dateModified': {
                         'type': 'DateTime',
-                        'value': '2019-05-09T15:28:30.000'
+                        'value': '2019-05-09T15:28:30.000Z'
                     }
                 }
             },
@@ -174,7 +177,7 @@ def sameEntityWithDifferentAttrs():
                 'metadata': {
                     'dateModified': {
                         'type': 'DateTime',
-                        'value': '2019-05-09T15:28:30.000'
+                        'value': '2019-05-09T15:28:30.000Z'
                     }
                 }
             }
@@ -188,7 +191,7 @@ def sameEntityWithDifferentAttrs():
                 'metadata': {
                     'dateModified': {
                         'type': 'DateTime',
-                        'value': '2019-05-09T15:29:30.000'
+                        'value': '2019-05-09T15:29:30.000Z'
                     }
                 }
             }
@@ -216,7 +219,7 @@ def air_quality_observed():
         },
         "dateObserved": {
             "type": "DateTime",
-            "value": "2016-03-14T17:00:00"
+            "value": "2016-03-14T17:00:00Z"
         },
         "location": {
             "value": {
@@ -286,7 +289,7 @@ def traffic_flow_observed():
         },
         'dateObservedFrom': {
             'type': 'Text',
-            'value': '2017-11-22T17:17:30.352635',
+            'value': '2017-11-22T17:17:30.352635Z',
         },
         'averageVehicleLength': {
             'type': 'Number',
@@ -318,7 +321,7 @@ def traffic_flow_observed():
         },
         'dateObservedTo': {
             'type': 'Text',
-            'value': '2017-11-22T17:17:40.352652'
+            'value': '2017-11-22T17:17:40.352652Z'
         },
         'location': {
             'type': 'StructuredValue',

src/exceptions/exceptions.py

@@ -27,3 +27,12 @@ def __init__(self, entity_id=''):
         msg = "There are multiple entities with the given entity_id {}. " \
               "Please specify entity_type."
         NGSIUsageError.__init__(self, msg.format(entity_id))
+
+
+class InvalidParameterValue(QLError):
+    """
+    Passed parameter value is not valid.
+    """
+    def __init__(self, par_value='', par_name=''):
+        msg = "The parameter value '{}' for parameter {} is not valid."
+        QLError.__init__(self, msg.format(par_value, par_name))

src/geocoding/tests/run_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-docker build -t quantumleap ../../../
+docker build -t smartsdk/quantumleap ../../../
 
 docker-compose up -d
 sleep 8

src/reporter/conftest.py

@@ -29,3 +29,9 @@ def create_notification(entity_type='Room', entity_id='Room1', subs_id=None):
 @pytest.fixture
 def notification():
     return create_notification()
+
+
+@pytest.fixture()
+def clean():
+    yield
+    do_clean_crate()

src/reporter/query_1T1E1A.py

@@ -1,7 +1,7 @@
 from exceptions.exceptions import NGSIUsageError
 from flask import request
 from reporter.reporter import _validate_query_params
-from translators.crate import CrateTranslatorInstance
+from translators.factory import translator_for
 import logging
 from .geo_query_handler import handle_geo_query
 from utils.jsondict import lookup_string_match
@@ -39,7 +39,7 @@ def query_1T1E1A(attr_name,   # In Path
 
     entities = None
     try:
-        with CrateTranslatorInstance() as trans:
+        with translator_for(fiware_s) as trans:
             entities = trans.query(attr_names=[attr_name],
                                    entity_type=type_,
                                    entity_id=entity_id,

src/reporter/query_1T1ENA.py

@@ -1,7 +1,7 @@
-from exceptions.exceptions import NGSIUsageError
+from exceptions.exceptions import NGSIUsageError, InvalidParameterValue
 from flask import request
 from reporter.reporter import _validate_query_params
-from translators.crate import CrateTranslatorInstance
+from translators.factory import translator_for
 import logging
 from .geo_query_handler import handle_geo_query
 
@@ -41,7 +41,7 @@ def query_1T1ENA(entity_id,   # In Path
 
     entities = None
     try:
-        with CrateTranslatorInstance() as trans:
+        with translator_for(fiware_s) as trans:
             entities = trans.query(attr_names=attrs,
                                    entity_type=type_,
                                    entity_id=entity_id,
@@ -61,6 +61,12 @@ def query_1T1ENA(entity_id,   # In Path
             "description": str(e)
         }, 400
 
+    except InvalidParameterValue as e:
+        return {
+            "error": "{}".format(type(e)),
+            "description": str(e)
+        }, 422
+
     except Exception as e:
         # Temp workaround to debug test_not_found
         msg = "Something went wrong with QL. Error: {}".format(e)

src/reporter/query_1TNE1A.py

@@ -1,7 +1,7 @@
-from exceptions.exceptions import NGSIUsageError
+from exceptions.exceptions import NGSIUsageError, InvalidParameterValue
 from flask import request
 from reporter.reporter import _validate_query_params
-from translators.crate import CrateTranslatorInstance
+from translators.factory import translator_for
 import logging
 from .geo_query_handler import handle_geo_query
 from utils.jsondict import lookup_string_match
@@ -46,7 +46,7 @@ def query_1TNE1A(attr_name,   # In Path
     if id_:
         entity_ids = [s.strip() for s in id_.split(',') if s]
     try:
-        with CrateTranslatorInstance() as trans:
+        with translator_for(fiware_s) as trans:
             entities = trans.query(attr_names=[attr_name],
                                    entity_type=entity_type,
                                    entity_ids=entity_ids,
@@ -67,6 +67,12 @@ def query_1TNE1A(attr_name,   # In Path
             "description": str(e)
         }, 400
 
+    except InvalidParameterValue as e:
+        return {
+            "error": "{}".format(type(e)),
+            "description": str(e)
+        }, 422
+
     except Exception as e:
         # Temp workaround to debug test_not_found
         msg = "Something went wrong with QL. Error: {}".format(e)

src/reporter/query_1TNENA.py

@@ -1,10 +1,13 @@
-from exceptions.exceptions import NGSIUsageError
+from exceptions.exceptions import NGSIUsageError, InvalidParameterValue
 from flask import request
 from reporter.reporter import _validate_query_params
-from translators.crate import CrateTranslatorInstance
+from translators.factory import translator_for
 import logging
 from .geo_query_handler import handle_geo_query
 from utils.jsondict import lookup_string_match
+import dateutil.parser
+from datetime import datetime, timezone
+
 
 def query_1TNENA(entity_type=None,  # In Path
                  id_=None,  # In Query
@@ -48,7 +51,7 @@ def query_1TNENA(entity_type=None,  # In Path
     if id_:
         entity_ids = [s.strip() for s in id_.split(',') if s]
     try:
-        with CrateTranslatorInstance() as trans:
+        with translator_for(fiware_s) as trans:
             entities = trans.query(attr_names=attrs,
                                    entity_type=entity_type,
                                    entity_ids=entity_ids,
@@ -69,6 +72,12 @@ def query_1TNENA(entity_type=None,  # In Path
             "description": str(e)
         }, 400
 
+    except InvalidParameterValue as e:
+        return {
+            "error": "{}".format(type(e)),
+            "description": str(e)
+        }, 422
+
     except Exception as e:
         msg = "Something went wrong with QL. Error: {}".format(e)
         logging.getLogger().error(msg, exc_info=True)
@@ -104,7 +113,15 @@ def _prepare_response(entities, attrs, entity_type, entity_ids,
                 'attrName': at,
                 'values': e[at]['values']
             })
-        index = [from_date or '', to_date or ''] if aggr_method and not aggr_period else e['index']
+        try:
+            f_date = dateutil.parser.isoparse(from_date).replace(tzinfo=timezone.utc).isoformat()
+        except Exception as ex:
+            f_date = ''
+        try:
+            t_date = dateutil.parser.isoparse(to_date).replace(tzinfo=timezone.utc).isoformat()
+        except Exception as ex:
+            t_date = ''
+        index = [f_date, t_date] if aggr_method and not aggr_period else e['index']
         entity = {
                  'entityId': e['id'],
                  'index': index,