fix 347 (#365)

* fix #208

* added code to group entities by servicepath when multiple notifications arrive at once (#208)
* add tests and integration tests (so we have also test with orion triggering such case)
* fix notification tests to rely only on api calls (remove call to translator)
* changed error code to 400 when due to invalid request
* upgraded python to 3.8.5
* fix integration test to use new docker image with entrypoint (#362)

* minor fixes

* remove build (we can assume a build is available)
* remove volumes at the end of the test
* increase timesleep for one test
* improve code

* fix #347

* fix wrong delete logic not keeping into account correctly servicePath!=None
* add test for #347 bug
 * add test for selective service path delete

* add drop all also to single entity delete if table is empty

* fix tests to correctly delete entities with service paths

* dont drop table when deleting entities, use a separate endpoint for that.

* factor out common http code from delete module.

* make delete api work w/ any translator.

* keep timescale version in env var as done for other services.

* temp solution to automate delete tests for timescale, ditch when timescale support is complete.

* use droptable query param to drop entity table instead of separate endpoint.

Co-authored-by: c0c0n3 <andrea.falconi@gmail.com>

deps.env

@@ -6,6 +6,7 @@ export ORION_VERSION=2.2.0
 export INFLUX_VERSION=1.2.2
 export RETHINK_VERSION=2.3.5
 export CRATE_VERSION=4.1.4
+export TIMESCALE_VERSION=1.7.1-pg12
 
 export REDIS_VERSION=3
 

docker/docker-compose-dev.yml

@@ -49,7 +49,7 @@ services:
       - redisdata:/data
 
   timescale:
-    image: timescale/timescaledb-postgis:1.7.1-pg12
+    image: timescale/timescaledb-postgis:${TIMESCALE_VERSION}
     ports:
       - "5432:5432"
       # Don't expose container port 5432 with the same number outside of the

run_tests.sh

@@ -1,22 +1,40 @@
 #!/bin/bash
 
+test_suite_header () {
+  echo "======================================================================="
+  echo "        $1 TESTS"
+  echo "======================================================================="
+}
+
 docker pull smartsdk/quantumleap
 docker build --cache-from smartsdk/quantumleap -t smartsdk/quantumleap .
 
 cd src/translators/tests
+test_suite_header "TRANSLATOR"
 sh run_tests.sh
 tot=$?
 cd -
 
 cd src/reporter/tests
+test_suite_header "REPORTER (Crate)"
 sh run_tests.sh
 loc=$?
 if [ "$tot" -eq 0 ]; then
    tot=$loc
 fi
 cd -
 
+cd src/reporter/tests
+test_suite_header "REPORTER (Timescale)"
+sh run_tests.timescale.sh
+loc=$?
+if [ "$tot" -eq 0 ]; then
+   tot=$loc
+fi
+cd -
+
 cd src/geocoding/tests
+test_suite_header "GEO-CODING"
 sh run_tests.sh
 loc=$?
 if [ "$tot" -eq 0 ]; then
@@ -25,6 +43,7 @@ fi
 cd -
 
 cd src/utils/tests
+test_suite_header "UTILS"
 sh run_tests.sh
 loc=$?
 if [ "$tot" -eq 0 ]; then
@@ -33,6 +52,7 @@ fi
 cd -
 
 cd src/tests/
+test_suite_header "BACKWARD COMPAT & INTEGRATION"
 sh run_tests.sh
 loc=$?
 if [ "$tot" -eq 0 ]; then

specification/quantumleap.yml

@@ -1,7 +1,7 @@
 swagger: '2.0'  # For 3.0 see (https://github.com/zalando/connexion/issues/420)
 info:
   title: "QuantumLeap API"
-  version: "0.7.5"      # we'll keep it aligned with QL version
+  version: "0.7.6"      # we'll keep it aligned with QL version
 host: "localhost:8668"  # it'll run in the same container, hence localhost.
 produces:
   - text/plain
@@ -313,6 +313,19 @@ parameters:
     `40 42' 51'',74 0' 21''`.
     Full details can be found in the Geographical Queries section of the specification:
     http://fiware.github.io/specifications/ngsiv2/stable/."
+  dropTable:
+    in: query
+    name: dropTable
+    type: boolean
+    default: false
+    description: "Optional. Drop the table storing an entity type. When deleting
+    by entity type, setting this parameter to true will result in all entity
+    data for the given type being deleted, the entity table will be dropped
+    and the corresponding entry removed from the metadata table. This option
+    should only be used for maintenance after the devices whose data is written
+    to the table are decommissioned and no further writes are possible. In fact,
+    race conditions are possible if entities of that type are POSTed to the
+    notify endpoint while the underlying clean-up procedure is in progress."
 
 ################################################################################
 # PATHS: META
@@ -1245,6 +1258,7 @@ paths:
         # In Query...
         - $ref: '#/parameters/fromDate'
         - $ref: '#/parameters/toDate'
+        - $ref: '#/parameters/dropTable'
         # In Header...
         - $ref: '#/parameters/fiware-Service'
         - $ref: '#/parameters/fiware-ServicePath'

src/reporter/delete.py

@@ -1,20 +1,17 @@
 from exceptions.exceptions import AmbiguousNGSIIdError
-from flask import request
-from translators import crate
+from .http import fiware_s, fiware_sp
+from translators.factory import translator_for
 
 
 def delete_entity(entity_id, type_=None, from_date=None, to_date=None):
-    fiware_s = request.headers.get('fiware-service', None)
-    fiware_sp = request.headers.get('fiware-servicepath', None)
-
     try:
-        with crate.CrateTranslatorInstance() as trans:
-            deleted = trans.delete_entity(entity_id=entity_id,
-                                          entity_type=type_,
+        with translator_for(fiware_s()) as trans:
+            deleted = trans.delete_entity(eid=entity_id,
+                                          etype=type_,
                                           from_date=from_date,
                                           to_date=to_date,
-                                          fiware_service=fiware_s,
-                                          fiware_servicepath=fiware_sp,)
+                                          fiware_service=fiware_s(),
+                                          fiware_servicepath=fiware_sp(),)
     except AmbiguousNGSIIdError as e:
         return {
             "error": "AmbiguousNGSIIdError",
@@ -32,16 +29,18 @@ def delete_entity(entity_id, type_=None, from_date=None, to_date=None):
         return '{} records successfully deleted.'.format(deleted), 204
 
 
-def delete_entities(entity_type, from_date=None, to_date=None):
-    fiware_s = request.headers.get('fiware-service', None)
-    fiware_sp = request.headers.get('fiware-servicepath', None)
+def delete_entities(entity_type, from_date=None, to_date=None,
+                    drop_table=False):
+    with translator_for(fiware_s()) as trans:
+        if drop_table:
+            trans.drop_table(etype=entity_type, fiware_service=fiware_s())
+            return 'entity table dropped', 204
 
-    with crate.CrateTranslatorInstance() as trans:
-        deleted = trans.delete_entities(entity_type=entity_type,
+        deleted = trans.delete_entities(etype=entity_type,
                                         from_date=from_date,
                                         to_date=to_date,
-                                        fiware_service=fiware_s,
-                                        fiware_servicepath=fiware_sp,)
+                                        fiware_service=fiware_s(),
+                                        fiware_servicepath=fiware_sp(),)
         if deleted == 0:
             r = {
                 "error": "Not Found",

src/reporter/http.py

@@ -0,0 +1,15 @@
+from flask import request
+
+
+def fiware_s() -> str:
+    """
+    :return: The content of the FiWare service header if any.
+    """
+    return request.headers.get('fiware-service', None)
+
+
+def fiware_sp() -> str:
+    """
+    :return: The content of the FiWare service path header if any.
+    """
+    return request.headers.get('fiware-servicepath', None)