Clean test translator (#403)

* consolidate original_data_test
* consolidate insert and delete
* consolidate aggregation, multitenancy and multientities
* add comments on how to test specific translator
* add ngsi_ld test to translator
* add health test
* fix formatting
* add missing dependence
* fix pg8000 version to 1.16.5
* remove unused meta data from timescale geo query response
* unify reporter tests between timescale and crate
* remove MTU from compose
* fix cache naming bug
* allow retry for db setup
* re-insert postgres port
* fix pytest-lazy-fixture version

Pipfile

@@ -13,7 +13,7 @@ geojson = "~=2.4"
 geomet = "~=0.2"
 gunicorn = "~=20.0.4"
 influxdb = "~=4.0"
-pg8000 = ">=1.15"
+pg8000 = "==1.16.5"
 pymongo = "~=3.4"
 pytest = "~=3.0"
 pytest-cov = "~=2.7.1"
@@ -25,6 +25,7 @@ redis = "~=2.10"
 requests = ">=2.20"
 rethinkdb = "==2.3"
 pickle-mixin = "==1.0.2"
+pytest-lazy-fixture = "~=0.6.3"
 
 [dev-packages]
 

docker/docker-compose-dev.yml

@@ -96,5 +96,3 @@ volumes:
 
 networks:
     default:
-        driver_opts:
-            com.docker.network.driver.mtu: ${DOCKER_MTU:-1400}

docker/docker-compose.yml

@@ -70,5 +70,3 @@ volumes:
 
 networks:
     default:
-        driver_opts:
-            com.docker.network.driver.mtu: ${DOCKER_MTU:-1400}

run_tests.sh

@@ -16,23 +16,14 @@ tot=$?
 cd -
 
 cd src/reporter/tests
-test_suite_header "REPORTER (Crate)"
+test_suite_header "REPORTER"
 sh run_tests.sh
 loc=$?
 if [ "$tot" -eq 0 ]; then
    tot=$loc
 fi
 cd -
 
-cd src/reporter/tests
-test_suite_header "REPORTER (Timescale)"
-sh run_tests.timescale.sh
-loc=$?
-if [ "$tot" -eq 0 ]; then
-   tot=$loc
-fi
-cd -
-
 cd src/geocoding/tests
 test_suite_header "GEO-CODING"
 sh run_tests.sh

src/conftest.py

@@ -15,7 +15,7 @@
 CRATE_PORT = 4200
 
 POSTGRES_HOST = os.environ.get('POSTGRES_HOST', 'timescale')
-POSTGRES_HOST = 5432
+POSTGRES_PORT = 5432
 
 REDIS_HOST = os.environ.get('REDIS_HOST', 'redis')
 REDIS_PORT = 6379
@@ -70,6 +70,12 @@ def insert(self, entity, service=None, service_path=None):
                           headers=headers(service, service_path))
         return r
 
+    def update_attr(self, entity_id, attrs, service=None, service_path=None):
+        r = requests.patch('{}/v2/entities/{}/attrs'.format(self.url, entity_id),
+                           data=json.dumps(attrs),
+                           headers=headers(service, service_path))
+        return r
+
     def delete(self, entity_id, service=None, service_path=None):
         r = requests.delete('{}/v2/entities/{}'.format(self.url, entity_id),
                             headers=headers(service, service_path))
@@ -184,6 +190,54 @@ def clean(self, fiware_service=None, **kwargs):
         yield trans
 
 
+@pytest.fixture()
+def timescale_translator():
+    from src.translators.timescale import PostgresTranslator, \
+        PostgresConnectionData
+
+    class Translator(PostgresTranslator):
+
+        def insert(self, entities,
+                   fiware_service=None, fiware_servicepath=None):
+            r = PostgresTranslator.insert(self, entities,
+                                          fiware_service, fiware_servicepath)
+            return r
+
+        def delete_entity(self, entity_id, entity_type=None,
+                          fiware_service=None, **kwargs):
+            r = PostgresTranslator.delete_entity(self, entity_id, entity_type,
+                                                 fiware_service=fiware_service,
+                                                 **kwargs)
+            return r
+
+        def delete_entities(self, entity_type=None, fiware_service=None,
+                            **kwargs):
+            r = PostgresTranslator.delete_entities(self, entity_type,
+                                                   fiware_service=fiware_service,
+                                                   **kwargs)
+            return r
+
+        def entity_types(self, fiware_service=None, **kwargs):
+            r = PostgresTranslator.query_entity_types(self, entity_type=None,
+                                                      fiware_service=fiware_service,
+                                                      **kwargs)
+            return r
+
+        def clean(self, fiware_service=None, **kwargs):
+            types = PostgresTranslator.query_entity_types(self,
+                                                          fiware_service=fiware_service,
+                                                          **kwargs)
+            if types:
+                for t in types:
+                    PostgresTranslator.drop_table(self, t,
+                                                  fiware_service=fiware_service,
+                                                  **kwargs)
+
+    with Translator(PostgresConnectionData(host=POSTGRES_HOST,
+                                           port=POSTGRES_PORT)) as trans:
+        yield trans
+
+
 @pytest.fixture
 def entity():
     entity = {
@@ -454,3 +508,71 @@ def traffic_flow_observed():
         }
     }
     return entity
+
+
+@pytest.fixture
+def ngsi_ld():
+    """
+    :return: dict
+        The NGSI LD model as received within an Orion notification.
+    """
+    entity = {
+        "id": "urn:ngsi-ld:Streetlight:streetlight:guadalajara:4567",
+        "type": "Streetlight",
+        "location": {
+            "type": "GeoProperty",
+            "value": {
+                "type": "Point",
+                "coordinates": [-3.164485591715449, 40.62785133667262]
+            }
+        },
+        "areaServed": {
+            "type": "Property",
+            "value": "Roundabouts city entrance"
+        },
+        "status": {
+            "type": "Property",
+            "value": "ok"
+        },
+        "refStreetlightGroup": {
+            "type": "Relationship",
+            "object": "urn:ngsi-ld:StreetlightGroup:streetlightgroup:G345"
+        },
+        "refStreetlightModel": {
+            "type": "Relationship",
+            "object": "urn:ngsi-ld:StreetlightModel:streetlightmodel:STEEL_Tubular_10m"
+        },
+        "circuit": {
+            "type": "Property",
+            "value": "C-456-A467"
+        },
+        "lanternHeight": {
+            "type": "Property",
+            "value": 10
+        },
+        "locationCategory": {
+            "type": "Property",
+            "value": "centralIsland"
+        },
+        "powerState": {
+            "type": "Property",
+            "value": "off"
+        },
+        "controllingMethod": {
+            "type": "Property",
+            "value": "individual"
+        },
+        "dateLastLampChange": {
+            "type": "Property",
+            "value": {
+                "@type": "DateTime",
+                "@value": "2016-07-08T08:02:21.753Z"
+            }
+        },
+        "@context": [
+            "https://schema.lab.fiware.org/ld/context",
+            "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld"
+        ]
+    }
+
+    return entity

src/geocoding/geojson/wktcodec.py

@@ -79,7 +79,12 @@ def decode_wkb_hexstr(geom: str) -> dict:
     :return: the corresponding GeoJSON.
     """
     geom_bytes = bytes.fromhex(geom)
-    return decode_wkb(geom_bytes)
+    geojson = decode_wkb(geom_bytes)
+    if 'meta' in geojson:
+        geojson.pop('meta')
+    if 'crs' in geojson:
+        geojson.pop('crs')
+    return geojson
 
 
 # TODO. Use shapely?

src/reporter/tests/docker-compose.yml

@@ -32,12 +32,39 @@ services:
     depends_on:
       - orion
       - crate
+      - timescale
+    volumes:
+      - ${PWD}/ql-config.yml:/config/ql-config.yml
     networks:
         - reportertests
     environment:
-      - USE_GEOCODING=True
+      - USE_GEOCODING=False
       - REDIS_HOST=redis
-      - LOGLEVEL=DEBUG
+      - LOGLEVEL=INFO
+      - POSTGRES_HOST=timescale
+      - QL_CONFIG=/config/ql-config.yml
+
+  timescale:
+    image: timescale/timescaledb-postgis:${TIMESCALE_VERSION}
+    ports:
+      - "5432:5432"
+    networks:
+      - reportertests
+    environment:
+      - POSTGRES_PASSWORD=*
+
+  quantumleap-db-setup:
+    build: ../../../timescale-container/
+    image: quantumleap-db-setup
+    depends_on:
+      - timescale
+    networks:
+      - reportertests
+    environment:
+      - QL_DB_PASS=*
+      - QL_DB_INIT_DIR=/ql-db-init
+      - PG_HOST=timescale
+      - PG_PASS=*
 
   crate:
     image: crate:${CRATE_VERSION}

src/reporter/tests/ql-config.yml

@@ -0,0 +1,6 @@
+tenants:
+  t1:
+    backend: Crate
+  t2:
+    backend: Timescale
+default-backend: Crate