diff --git a/deps.env b/deps.env index bfbcbe9c..25359ea2 100644 --- a/deps.env +++ b/deps.env @@ -6,6 +6,7 @@ export ORION_VERSION=2.2.0 export INFLUX_VERSION=1.2.2 export RETHINK_VERSION=2.3.5 export CRATE_VERSION=4.1.4 +export TIMESCALE_VERSION=1.7.1-pg12 export REDIS_VERSION=3 diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml index 2242be4e..d1459d1f 100644 --- a/docker/docker-compose-dev.yml +++ b/docker/docker-compose-dev.yml @@ -49,7 +49,7 @@ services: - redisdata:/data timescale: - image: timescale/timescaledb-postgis:1.7.1-pg12 + image: timescale/timescaledb-postgis:${TIMESCALE_VERSION} ports: - "5432:5432" # Don't expose container port 5432 with the same number outside of the diff --git a/run_tests.sh b/run_tests.sh index 0691ebf1..69bfcc65 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,14 +1,22 @@ #!/bin/bash +test_suite_header () { + echo "=======================================================================" + echo " $1 TESTS" + echo "=======================================================================" +} + docker pull smartsdk/quantumleap docker build --cache-from smartsdk/quantumleap -t smartsdk/quantumleap . cd src/translators/tests +test_suite_header "TRANSLATOR" sh run_tests.sh tot=$? cd - cd src/reporter/tests +test_suite_header "REPORTER (Crate)" sh run_tests.sh loc=$? if [ "$tot" -eq 0 ]; then @@ -16,7 +24,17 @@ if [ "$tot" -eq 0 ]; then fi cd - +cd src/reporter/tests +test_suite_header "REPORTER (Timescale)" +sh run_tests.timescale.sh +loc=$? +if [ "$tot" -eq 0 ]; then + tot=$loc +fi +cd - + cd src/geocoding/tests +test_suite_header "GEO-CODING" sh run_tests.sh loc=$? if [ "$tot" -eq 0 ]; then @@ -25,6 +43,7 @@ fi cd - cd src/utils/tests +test_suite_header "UTILS" sh run_tests.sh loc=$? if [ "$tot" -eq 0 ]; then @@ -33,6 +52,7 @@ fi cd - cd src/tests/ +test_suite_header "BACKWARD COMPAT & INTEGRATION" sh run_tests.sh loc=$? if [ "$tot" -eq 0 ]; then diff --git a/specification/quantumleap.yml b/specification/quantumleap.yml index 6c54243f..5d4edbcd 100644 --- a/specification/quantumleap.yml +++ b/specification/quantumleap.yml @@ -1,7 +1,7 @@ swagger: '2.0' # For 3.0 see (https://github.com/zalando/connexion/issues/420) info: title: "QuantumLeap API" - version: "0.7.5" # we'll keep it aligned with QL version + version: "0.7.6" # we'll keep it aligned with QL version host: "localhost:8668" # it'll run in the same container, hence localhost. produces: - text/plain @@ -313,6 +313,19 @@ parameters: `40 42' 51'',74 0' 21''`. Full details can be found in the Geographical Queries section of the specification: http://fiware.github.io/specifications/ngsiv2/stable/." + dropTable: + in: query + name: dropTable + type: boolean + default: false + description: "Optional. Drop the table storing an entity type. When deleting + by entity type, setting this parameter to true will result in all entity + data for the given type being deleted, the entity table will be dropped + and the corresponding entry removed from the metadata table. This option + should only be used for maintenance after the devices whose data is written + to the table are decommissioned and no further writes are possible. In fact, + race conditions are possible if entities of that type are POSTed to the + notify endpoint while the underlying clean-up procedure is in progress." ################################################################################ # PATHS: META @@ -1245,6 +1258,7 @@ paths: # In Query... - $ref: '#/parameters/fromDate' - $ref: '#/parameters/toDate' + - $ref: '#/parameters/dropTable' # In Header... - $ref: '#/parameters/fiware-Service' - $ref: '#/parameters/fiware-ServicePath' diff --git a/src/reporter/delete.py b/src/reporter/delete.py index 4fe5f264..4012ad61 100644 --- a/src/reporter/delete.py +++ b/src/reporter/delete.py @@ -1,20 +1,17 @@ from exceptions.exceptions import AmbiguousNGSIIdError -from flask import request -from translators import crate +from .http import fiware_s, fiware_sp +from translators.factory import translator_for def delete_entity(entity_id, type_=None, from_date=None, to_date=None): - fiware_s = request.headers.get('fiware-service', None) - fiware_sp = request.headers.get('fiware-servicepath', None) - try: - with crate.CrateTranslatorInstance() as trans: - deleted = trans.delete_entity(entity_id=entity_id, - entity_type=type_, + with translator_for(fiware_s()) as trans: + deleted = trans.delete_entity(eid=entity_id, + etype=type_, from_date=from_date, to_date=to_date, - fiware_service=fiware_s, - fiware_servicepath=fiware_sp,) + fiware_service=fiware_s(), + fiware_servicepath=fiware_sp(),) except AmbiguousNGSIIdError as e: return { "error": "AmbiguousNGSIIdError", @@ -32,16 +29,18 @@ def delete_entity(entity_id, type_=None, from_date=None, to_date=None): return '{} records successfully deleted.'.format(deleted), 204 -def delete_entities(entity_type, from_date=None, to_date=None): - fiware_s = request.headers.get('fiware-service', None) - fiware_sp = request.headers.get('fiware-servicepath', None) +def delete_entities(entity_type, from_date=None, to_date=None, + drop_table=False): + with translator_for(fiware_s()) as trans: + if drop_table: + trans.drop_table(etype=entity_type, fiware_service=fiware_s()) + return 'entity table dropped', 204 - with crate.CrateTranslatorInstance() as trans: - deleted = trans.delete_entities(entity_type=entity_type, + deleted = trans.delete_entities(etype=entity_type, from_date=from_date, to_date=to_date, - fiware_service=fiware_s, - fiware_servicepath=fiware_sp,) + fiware_service=fiware_s(), + fiware_servicepath=fiware_sp(),) if deleted == 0: r = { "error": "Not Found", diff --git a/src/reporter/http.py b/src/reporter/http.py new file mode 100644 index 00000000..5e8c9734 --- /dev/null +++ b/src/reporter/http.py @@ -0,0 +1,15 @@ +from flask import request + + +def fiware_s() -> str: + """ + :return: The content of the FiWare service header if any. + """ + return request.headers.get('fiware-service', None) + + +def fiware_sp() -> str: + """ + :return: The content of the FiWare service path header if any. + """ + return request.headers.get('fiware-servicepath', None) diff --git a/src/reporter/tests/_tmptest_delete_with_timescale.py b/src/reporter/tests/_tmptest_delete_with_timescale.py new file mode 100644 index 00000000..0274bc06 --- /dev/null +++ b/src/reporter/tests/_tmptest_delete_with_timescale.py @@ -0,0 +1,383 @@ +from conftest import QL_URL +from exceptions.exceptions import AmbiguousNGSIIdError +from reporter.conftest import create_notification +from reporter.tests.utils import delete_test_data +import json +import pg8000 +import pytest +import requests + +from translators.timescale import PostgresConnectionData +from translators.sql_translator import SQLTranslator + + +notify_url = "{}/notify".format(QL_URL) + +# TODO: get rid of this file. +# This is just a stopgap solution to re-run the tests in test_delete with +# Timescale. I used my famed C&P tech to lift & tweak code from test_delete. +# Once we have fixed the Timescale queries, the (duplicated) tests in this +# file won't be needed anymore, we should just be able to run the tests in +# test_delete twice, first with Crate as a back-end and then with Timescale +# using the docker compose setup in docker-compose.timescale.yml. + + +@pytest.fixture(scope='module') +def with_pg8000(): + pg8000.paramstyle = "qmark" + t = PostgresConnectionData() + t.read_env() + + conn = pg8000.connect(host=t.host, port=t.port, + database=t.db_name, + user=t.db_user, password=t.db_pass) + conn.autocommit = True + cursor = conn.cursor() + + yield cursor + + cursor.close() + conn.close() + + +def count_entity_rows(with_pg8000, service: str, etype: str) -> int: + table_name = SQLTranslator._et2tn(etype, service) + stmt = f"SELECT COUNT(*) FROM {table_name}" + + cursor = with_pg8000 + cursor.execute(stmt) + rows = cursor.fetchall() + + return rows[0][0] if rows else 0 + + +def insert_test_data(service, service_path=None, entity_id=None): + # 3 entity types, 2 entities for each, 10 updates for each entity. + for t in ("AirQualityObserved", "Room", "TrafficFlowObserved"): + for e in range(2): + for u in range(10): + ei = entity_id or '{}{}'.format(t, e) + notification = create_notification(t, ei) + data = json.dumps(notification) + h = { + 'Content-Type': 'application/json', + 'Fiware-Service': service + } + if service_path: + h['Fiware-ServicePath'] = service_path + r = requests.post(notify_url, + data=data, + headers=h) + assert r.status_code == 200, r.text + + +def assert_have_all_entities_of_type(etype, service, with_pg8000): + assert count_entity_rows(with_pg8000, service, etype) == 20 + + +def assert_have_no_entities_of_deleted_id(etype, service, with_pg8000): + assert count_entity_rows(with_pg8000, service, etype) == 10 + + +def assert_have_no_entities_of_type(etype, service, with_pg8000): + assert count_entity_rows(with_pg8000, service, etype) == 0 + + +def assert_have_entities_of_type(etype, service, how_many, with_pg8000): + assert count_entity_rows(with_pg8000, service, etype) == how_many + + +@pytest.mark.parametrize("service", [ + "t1", "t2" +]) +def test_delete_entity(with_pg8000, service): + """ + By default, delete all records of some entity. + """ + pass + insert_test_data(service) + + entity_type = "AirQualityObserved" + params = { + 'type': entity_type, + } + h = { + 'Fiware-Service': service + } + url = '{}/entities/{}'.format(QL_URL, entity_type + '0') + + # Values are there + assert_have_all_entities_of_type(entity_type, service, with_pg8000) + + # Delete them + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 204, r.text + + # Values are gone + # But not for other entities of same type + assert_have_no_entities_of_deleted_id(entity_type, service, with_pg8000) + + for t in ("AirQualityObserved", "Room", "TrafficFlowObserved"): + delete_test_data(service, [t]) + + +@pytest.mark.parametrize("service", [ + "t1", "t2" +]) +def test_delete_entities(with_pg8000, service): + """ + By default, delete all historical records of all entities of some type. + """ + entity_type = "TrafficFlowObserved" + params = { + 'type': entity_type, + } + h = { + 'Fiware-Service': service + } + insert_test_data(service) + + # Values are there for both entities + assert_have_all_entities_of_type(entity_type, service, with_pg8000) + + # 1 Delete call + url = '{}/types/{}'.format(QL_URL, entity_type) + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 204, r.text + + # Values are gone for both entities + assert_have_no_entities_of_type(entity_type, service, with_pg8000) + + # But not for entities of other types + assert_have_all_entities_of_type('Room', service, with_pg8000) + + for t in ("AirQualityObserved", "Room"): + delete_test_data(service, [t]) + + +@pytest.mark.parametrize("service", [ + "t1", "t2" +]) +def test_not_found(service): + entity_type = "AirQualityObserved" + params = { + 'type': entity_type, + } + h = { + 'Fiware-Service': service + } + url = '{}/entities/{}'.format(QL_URL, entity_type + '0') + + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 404, r.text + assert r.json() == { + "error": "Not Found", + "description": "No records were found for such query." + } + + +@pytest.mark.parametrize("service", [ + "t1", "t2" +]) +def test_no_type_not_unique(service): + # If id is not unique across types, you must specify type. + insert_test_data(service, entity_id='repeatedId') + + url = '{}/entities/{}'.format(QL_URL, 'repeatedId') + h = { + 'Fiware-Service': service + } + + # Without type + r = requests.delete(url, params={}, headers=h) + assert r.status_code == 409, r.text + assert r.json() == { + "error": "AmbiguousNGSIIdError", + "description": str(AmbiguousNGSIIdError('repeatedId')) + } + + # With type + r = requests.delete(url, params={'type': 'AirQualityObserved'}, headers=h) + assert r.status_code == 204, r.text + + for t in ("AirQualityObserved", "Room", "TrafficFlowObserved"): + delete_test_data(service, [t]) + + +@pytest.mark.parametrize("service", [ + "t1", "t2" +]) +def test_delete_no_type_with_multitenancy(with_pg8000, service): + """ + A Car and a Truck with the same entity_id. Same thing in two different + tenants (USA an EU). + """ + # You have a car1 + car = json.dumps(create_notification("Car", "car1")) + + # In Default + h_def = { + 'Content-Type': 'application/json', + 'Fiware-Service': service + } + r = requests.post(notify_url, data=car, headers=h_def) + assert r.status_code == 200 + + # In EU + h_eu = { + 'Content-Type': 'application/json', + 'Fiware-Service': 'EU' + } + r = requests.post(notify_url, data=car, headers=h_eu) + assert r.status_code == 200 + + # In USA + h_usa = { + 'Content-Type': 'application/json', + 'Fiware-Service': 'USA' + } + r = requests.post(notify_url, data=car, headers=h_usa) + assert r.status_code == 200 + + # I could delete car1 from default without giving a type + url = '{}/entities/{}'.format(QL_URL, 'car1') + r = requests.delete(url, params={}, headers=h_def) + assert r.status_code == 204, r.text + + # But it should still be in EU. + assert_have_entities_of_type('Car', 'EU', 1, with_pg8000) + + # I could delete car1 from EU without giving a type + url = '{}/entities/{}'.format(QL_URL, 'car1') + r = requests.delete(url, params={}, headers=h_eu) + assert r.status_code == 204, r.text + + # But it should still be in USA. + assert_have_entities_of_type('Car', 'USA', 1, with_pg8000) + + delete_test_data(service, ["Car"]) + delete_test_data('USA', ["Car"]) + delete_test_data('EU', ["Car"]) + + +def test_delete_347(with_pg8000): + """ + Test to replicate issue #347. + """ + entity_type = "deletetestDuno" + service = 'bbbbb' + service_path = '/' + params = { + 'type': entity_type, + } + h = { + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + + data = { + 'subscriptionId': 'ID_FROM_SUB', + 'data': [{ + 'id': 'un3', + 'type': 'deletetestDuno', + 'batteryVoltage': { + 'type': 'Text', + 'value': 'ilariso' + } + }] + } + + hn = { + 'Content-Type': 'application/json', + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + r = requests.post(notify_url, + data=json.dumps(data), + headers=hn) + assert r.status_code == 200, r.text + + # check that value is in the database + assert_have_entities_of_type(entity_type, service, 1, with_pg8000) + + # Delete call + url = '{}/types/{}'.format(QL_URL, entity_type) + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 204, r.text + + # Values are gone + assert_have_no_entities_of_type(entity_type, service, with_pg8000) + + +def test_delete_different_servicepaths(with_pg8000): + """ + Selective delete by service Path. + """ + entity_type = "deletetestDuno" + service = 'bbbbb' + service_path = '/a' + params = { + 'type': entity_type, + } + h = { + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + + data = { + 'subscriptionId': 'ID_FROM_SUB', + 'data': [{ + 'id': 'un3', + 'type': 'deletetestDuno', + 'batteryVoltage': { + 'type': 'Text', + 'value': 'ilariso' + } + }] + } + + hn = { + 'Content-Type': 'application/json', + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + r = requests.post(notify_url, + data=json.dumps(data), + headers=hn) + assert r.status_code == 200, r.text + + # insert the same entity in a different service path + service_path = '/b' + hn = { + 'Content-Type': 'application/json', + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + r = requests.post(notify_url, + data=json.dumps(data), + headers=hn) + assert r.status_code == 200, r.text + + # check that value is in the database + assert_have_entities_of_type(entity_type, service, 2, with_pg8000) + + # Delete /a + url = '{}/types/{}'.format(QL_URL, entity_type) + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 204, r.text + + h = { + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + + # 1 entity is still there + assert_have_entities_of_type(entity_type, service, 1, with_pg8000) + + # Delete /b + url = '{}/types/{}'.format(QL_URL, entity_type) + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 204, r.text + + # No entity + assert_have_no_entities_of_type(entity_type, service, with_pg8000) diff --git a/src/reporter/tests/docker-compose.timescale.yml b/src/reporter/tests/docker-compose.timescale.yml new file mode 100644 index 00000000..3c9fb6de --- /dev/null +++ b/src/reporter/tests/docker-compose.timescale.yml @@ -0,0 +1,117 @@ +version: '3' + +services: + + timescale: + image: timescale/timescaledb-postgis:${TIMESCALE_VERSION} + ports: + - "54320:5432" + # Don't expose container port 5432 with the same number outside of the + # swarm. In the Travis test env, there's already a PG instance running + # on port 5432! + networks: + - reportertests + environment: + - POSTGRES_PASSWORD=* + + quantumleap-db-setup: + build: ../../../timescale-container/ + image: quantumleap-db-setup + depends_on: + - timescale + networks: + - reportertests + environment: + - QL_DB_PASS=* + - QL_DB_INIT_DIR=/ql-db-init + - PG_HOST=timescale + - PG_PASS=* + + quantumleap: + build: ../../../ + image: smartsdk/quantumleap + ports: + - "8668:8668" + depends_on: + - timescale + networks: + - reportertests + environment: + - USE_GEOCODING=False + - QL_DEFAULT_DB=timescale + - POSTGRES_HOST=${POSTGRES_HOST} + - POSTGRES_PORT=54320 + - LOGLEVEL=DEBUG + +networks: + reportertests: + driver: bridge + +# TODO: QL PG host. +# Setting POSTGRES_HOST=timescale doesn't work. The driver fails to connect, +# see debug session below. Why is that? Setting POSTGRES_HOST=${POSTGRES_HOST} +# as done in the above quantumleap service stanza works on my machine but +# I'm not entirely sure it's 100% portable... +# +# Here's the transcript of a debug session on my machine. +# +# $ docker-compose -f docker-compose.timescale.yml up -d +# $ docker ps +# CONTAINER ID IMAGE ... +# 3e516e0ebba4 smartsdk/quantumleap ... +# $ docker exec -it 3e516e0ebba4 sh +# +# /src/ngsi-timeseries-api/src # printenv +# ... +# POSTGRES_HOST=192.0.0.1 +# LOGLEVEL=DEBUG +# ... +# POSTGRES_PORT=54320 +# QL_DEFAULT_DB=timescale +# ... +# +# /src/ngsi-timeseries-api/src # nslookup timescale +# Server: 127.0.0.11 +# Address: 127.0.0.11:53 +# +# Non-authoritative answer: +# Non-authoritative answer: +# Name: timescale +# Address: 172.28.0.2 +# +# /src/ngsi-timeseries-api/src # ping -c 1 timescale +# PING timescale (172.28.0.2): 56 data bytes +# ... +# 1 packets transmitted, 1 packets received, 0% packet loss +# ... +# +# /src/ngsi-timeseries-api/src # python +# >>> import pg8000 +# >>> pg8000.connect(host='timescale', port=54320, +# database='quantumleap', user='quantumleap', password='*') +# ... +# pg8000.exceptions.InterfaceError: Can't create a connection to host +# timescale and port 54320 (timeout is None and source_address is None). +# >>> pg8000.connect(host='172.28.0.2', port=54320, +# database='quantumleap', user='quantumleap', password='*') +# ... +# pg8000.exceptions.InterfaceError: Can't create a connection to host +# 172.28.0.2 and port 54320 (timeout is None and source_address is None). +# >>> pg8000.connect(host='192.0.0.1', port=54320, +# database='quantumleap', user='quantumleap', password='*') +# +# +# >>> quit() +# /src/ngsi-timeseries-api/src # exit +# +# $ psql postgres://postgres:*@localhost:54320 -c 'SELECT * FROM pg_hba_file_rules' +# line_number | type | database | user_name | address | netmask | auth_method | options | error +# -------------+-------+---------------+-----------+-----------+-----------------------------------------+-------------+---------+------- +# 84 | local | {all} | {all} | | | trust | | +# 86 | host | {all} | {all} | 127.0.0.1 | 255.255.255.255 | trust | | +# 88 | host | {all} | {all} | ::1 | ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff | trust | | +# 91 | local | {replication} | {all} | | | trust | | +# 92 | host | {replication} | {all} | 127.0.0.1 | 255.255.255.255 | trust | | +# 93 | host | {replication} | {all} | ::1 | ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff | trust | | +# 95 | host | {all} | {all} | all | | md5 | | +# \ No newline at end of file diff --git a/src/reporter/tests/run_tests.timescale.sh b/src/reporter/tests/run_tests.timescale.sh new file mode 100644 index 00000000..b64fd895 --- /dev/null +++ b/src/reporter/tests/run_tests.timescale.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +docker build -t smartsdk/quantumleap ../../../ + +docker-compose -f docker-compose.timescale.yml up -d +sleep 10 + +# Set Postgres port to same value as in docker-compose.timescale.yml +export POSTGRES_PORT='54320' + +cd ../../../ + +# pytest src/reporter/ --cov-report= --cov-config=.coveragerc --cov-append --cov=src/ +# TODO: comment in above and zap line below when Timescale backend +# is fully functional. + +pytest src/reporter/tests/_tmptest_delete_with_timescale.py \ + --cov-report= --cov-config=.coveragerc --cov-append --cov=src/ +r=$? +cd - + +unset POSTGRES_PORT + +docker-compose -f docker-compose.timescale.yml down -v +exit $r diff --git a/src/reporter/tests/test_delete.py b/src/reporter/tests/test_delete.py index 7e2c6e2e..fc2b0095 100644 --- a/src/reporter/tests/test_delete.py +++ b/src/reporter/tests/test_delete.py @@ -10,7 +10,7 @@ notify_url = "{}/notify".format(QL_URL) -def insert_test_data(service, entity_id=None): +def insert_test_data(service, service_path=None, entity_id=None): # 3 entity types, 2 entities for each, 10 updates for each entity. for t in ("AirQualityObserved", "Room", "TrafficFlowObserved"): for e in range(2): @@ -22,12 +22,15 @@ def insert_test_data(service, entity_id=None): 'Content-Type': 'application/json', 'Fiware-Service': service } + if service_path: + h['Fiware-ServicePath'] = service_path r = requests.post(notify_url, data=data, headers=h) assert r.status_code == 200, r.text time.sleep(1) + @pytest.mark.parametrize("service", [ "t1", "t2" ]) @@ -44,7 +47,7 @@ def test_delete_entity(service): h = { 'Fiware-Service': service } - url = '{}/entities/{}'.format(QL_URL, entity_type+'0') + url = '{}/entities/{}'.format(QL_URL, entity_type + '0') # Values are there r = requests.get(url, params=params, headers=h) @@ -61,7 +64,7 @@ def test_delete_entity(service): assert r.status_code == 404, r.text # But not for other entities of same type - url = '{}/entities/{}'.format(QL_URL, entity_type+'1') + url = '{}/entities/{}'.format(QL_URL, entity_type + '1') r = requests.get(url, params=params, headers=h) assert r.status_code == 200, r.text assert r.text != '' @@ -124,7 +127,7 @@ def test_not_found(service): h = { 'Fiware-Service': service } - url = '{}/entities/{}'.format(QL_URL, entity_type+'0') + url = '{}/entities/{}'.format(QL_URL, entity_type + '0') r = requests.delete(url, params=params, headers=h) assert r.status_code == 404, r.text @@ -160,6 +163,7 @@ def test_no_type_not_unique(service): for t in ("AirQualityObserved", "Room", "TrafficFlowObserved"): delete_test_data(service, [t]) + @pytest.mark.parametrize("service", [ "t1", "t2" ]) @@ -216,4 +220,144 @@ def test_delete_no_type_with_multitenancy(service): assert r.status_code == 200, r.text delete_test_data(service, ["Car"]) delete_test_data('USA', ["Car"]) - delete_test_data('EU', ["Car"]) \ No newline at end of file + delete_test_data('EU', ["Car"]) + + +def test_delete_347(): + """ + Test to replicate issue #347. + """ + entity_type = "deletetestDuno" + service = 'bbbbb' + service_path = '/' + params = { + 'type': entity_type, + } + h = { + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + + data = { + 'subscriptionId': 'ID_FROM_SUB', + 'data': [{ + 'id': 'un3', + 'type': 'deletetestDuno', + 'batteryVoltage': { + 'type': 'Text', + 'value': 'ilariso' + } + }] + } + + hn = { + 'Content-Type': 'application/json', + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + r = requests.post(notify_url, + data=json.dumps(data), + headers=hn) + assert r.status_code == 200, r.text + time.sleep(1) + # check that value is in the database + url = '{}/entities/{}'.format(QL_URL, 'un3') + r = requests.get(url, params=params, headers=h) + assert r.status_code == 200, r.text + assert r.text != '' + + # Delete call + time.sleep(1) + url = '{}/types/{}'.format(QL_URL, entity_type) + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 204, r.text + + # Values are gone + time.sleep(1) + url = '{}/entities/{}'.format(QL_URL, '{}{}'.format(entity_type, 'un3')) + r = requests.get(url, params=params, headers=h) + assert r.status_code == 404, r.text + +def test_delete_different_servicepaths(): + """ + Selective delete by service Path. + """ + entity_type = "deletetestDuno" + service = 'bbbbb' + service_path = '/a' + params = { + 'type': entity_type, + } + h = { + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + + data = { + 'subscriptionId': 'ID_FROM_SUB', + 'data': [{ + 'id': 'un3', + 'type': 'deletetestDuno', + 'batteryVoltage': { + 'type': 'Text', + 'value': 'ilariso' + } + }] + } + + hn = { + 'Content-Type': 'application/json', + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + r = requests.post(notify_url, + data=json.dumps(data), + headers=hn) + assert r.status_code == 200, r.text + + #insert the same entity in a different service path + service_path = '/b' + hn = { + 'Content-Type': 'application/json', + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + r = requests.post(notify_url, + data=json.dumps(data), + headers=hn) + assert r.status_code == 200, r.text + time.sleep(1) + # check that value is in the database + url = '{}/entities/{}'.format(QL_URL, 'un3') + r = requests.get(url, params=params, headers=h) + assert r.status_code == 200, r.text + assert r.text != '' + + # Delete /a + time.sleep(2) + url = '{}/types/{}'.format(QL_URL, entity_type) + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 204, r.text + + h = { + 'Fiware-Service': service, + 'Fiware-ServicePath': service_path + } + + # 1 entity is still there + time.sleep(2) + url = '{}/entities/{}'.format(QL_URL, 'un3') + r = requests.get(url, params=params, headers=h) + assert r.status_code == 200, r.text + + # Delete /b + time.sleep(2) + url = '{}/types/{}'.format(QL_URL, entity_type) + r = requests.delete(url, params=params, headers=h) + assert r.status_code == 204, r.text + + # No entity + time.sleep(2) + url = '{}/entities/{}'.format(QL_URL, 'un3') + r = requests.get(url, params=params, headers=h) + assert r.status_code == 404, r.text \ No newline at end of file diff --git a/src/reporter/tests/test_notify.py b/src/reporter/tests/test_notify.py index 92e0ed11..049bcbbf 100644 --- a/src/reporter/tests/test_notify.py +++ b/src/reporter/tests/test_notify.py @@ -267,7 +267,7 @@ def test_integration_multiple_entities(diffEntityWithDifferentAttrs, orion_clien assert r.status_code == 200 entities = r.json() assert len(entities) == 3 - delete_entity_type("service", diffEntityWithDifferentAttrs[0]['type']) + delete_entity_type("service", diffEntityWithDifferentAttrs[0]['type'], "/Root") @pytest.mark.skip(reason="See issue #105") @pytest.mark.parametrize("service", services) @@ -327,7 +327,7 @@ def test_multiple_data_elements(service, notification, diffEntityWithDifferentAt r = requests.get(entities_url, params=None, headers=query_header(service)) entities = r.json() assert len(entities) == 3 - delete_entity_type(None, diffEntityWithDifferentAttrs[0]['type']) + delete_entity_type(service, diffEntityWithDifferentAttrs[0]['type']) @pytest.mark.parametrize("service", services) @@ -376,7 +376,7 @@ def test_multiple_data_elements_different_servicepath(service, notification, dif r = requests.get(entities_url, params=None, headers=query_headers) entities = r.json() assert len(entities) == 3 - delete_entity_type(service, diffEntityWithDifferentAttrs[0]['type']) + delete_entity_type(service, diffEntityWithDifferentAttrs[0]['type'], '/Test') @pytest.mark.parametrize("service", services) diff --git a/src/reporter/tests/utils.py b/src/reporter/tests/utils.py index b2df17e1..02faa1ce 100644 --- a/src/reporter/tests/utils.py +++ b/src/reporter/tests/utils.py @@ -84,13 +84,19 @@ def insert_test_data(service, entity_types, n_entities=1, index_size=30, time.sleep(1) -def delete_entity_type(service, entity_type): +def delete_entity_type(service, entity_type, service_path=None): h = {} if service: - h = {'Fiware-Service': service} + h['Fiware-Service'] = service + if service_path: + h['Fiware-ServicePath'] = service_path + query_params = { + 'dropTable': True + } + url = '{}/types/{}'.format(QL_URL, entity_type) - r = requests.delete(url, headers=h) + r = requests.delete(url, headers=h, params=query_params) # assert r.status_code == 204 diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index b4825364..7c876b9e 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -137,7 +137,8 @@ def _get_isoformat(self, ms_since_epoch): utc = datetime(1970, 1, 1, 0, 0, 0, 0, timezone.utc) + d return utc.isoformat(timespec='milliseconds') - def _et2tn(self, entity_type, fiware_service=None): + @staticmethod + def _et2tn(entity_type, fiware_service=None): """ Return table name based on entity type. When specified, fiware_service will define the table schema. @@ -206,7 +207,7 @@ def insert(self, entities, fiware_service=None, fiware_servicepath=None): path) else: msg = 'Multiple servicePath are allowed only ' \ - 'if their size is match the size of entities' + 'if their number match the number of entities' raise InvalidHeaderValue('Fiware-ServicePath', fiware_servicepath, msg) @@ -992,70 +993,48 @@ def _format_response(self, resultset, col_names, table_name, last_n): return [entities[k] for k in sorted(entities.keys())] - def delete_entity(self, entity_id, entity_type=None, from_date=None, + def delete_entity(self, eid, etype=None, from_date=None, to_date=None, fiware_service=None, fiware_servicepath=None): - if not entity_id: + if not eid: raise NGSIUsageError("entity_id cannot be None nor empty") - if not entity_type: - entity_type = self._get_entity_type(entity_id, fiware_service) + if not etype: + etype = self._get_entity_type(eid, fiware_service) - if not entity_type: + if not etype: return 0 - if len(entity_type.split(',')) > 1: - raise AmbiguousNGSIIdError(entity_id) + if len(etype.split(',')) > 1: + raise AmbiguousNGSIIdError(eid) - # First delete entries from table - table_name = self._et2tn(entity_type, fiware_service) - where_clause = self._get_where_clause([entity_id, ], + return self.delete_entities(etype, eid=[eid], + from_date=from_date, to_date=to_date, + fiware_service=fiware_service, + fiware_servicepath=fiware_servicepath) + + def delete_entities(self, etype, eid=None, from_date=None, to_date=None, + fiware_service=None, fiware_servicepath=None): + table_name = self._et2tn(etype, fiware_service) + where_clause = self._get_where_clause(eid, from_date, to_date, fiware_servicepath) op = "delete from {} {}".format(table_name, where_clause) - try: self.cursor.execute(op) - except Exception as e: - logging.error("{}".format(e)) - return 0 - - return self.cursor.rowcount - - def delete_entities(self, entity_type, from_date=None, to_date=None, - fiware_service=None, fiware_servicepath=None): - table_name = self._et2tn(entity_type, fiware_service) - - # Delete only requested range - if from_date or to_date or fiware_servicepath: - entity_id = None - where_clause = self._get_where_clause(entity_id, - from_date, - to_date, - fiware_servicepath) - op = "delete from {} {}".format(table_name, where_clause) - try: - self.cursor.execute(op) - except Exception as e: - logging.error("{}".format(e)) - return 0 return self.cursor.rowcount - - # Drop whole table - try: - self.cursor.execute("select count(*) from {}".format(table_name)) except Exception as e: logging.error("{}".format(e)) return 0 - count = self.cursor.fetchone()[0] + def drop_table(self, etype, fiware_service=None): + table_name = self._et2tn(etype, fiware_service) op = "drop table {}".format(table_name) try: self.cursor.execute(op) except Exception as e: logging.error("{}".format(e)) - return 0 # Delete entry from metadata table op = "delete from {} where table_name = ?".format(METADATA_TABLE_NAME) @@ -1072,8 +1051,6 @@ def delete_entities(self, entity_type, from_date=None, to_date=None, except Exception as e: logging.error("{}".format(e)) - return count - def _get_entity_type(self, entity_id, fiware_service): """ Find the type of the given entity_id. diff --git a/src/translators/tests/docker-compose.yml b/src/translators/tests/docker-compose.yml index 1e4807bc..49b52159 100644 --- a/src/translators/tests/docker-compose.yml +++ b/src/translators/tests/docker-compose.yml @@ -38,7 +38,7 @@ services: - translatorstests timescale: - image: timescale/timescaledb-postgis:1.7.1-pg12 + image: timescale/timescaledb-postgis:${TIMESCALE_VERSION} ports: - "54320:5432" # Don't expose container port 5432 with the same number outside of the diff --git a/timescale-container/test/docker-compose.yml b/timescale-container/test/docker-compose.yml index 5ae990bd..1f8c6a3f 100644 --- a/timescale-container/test/docker-compose.yml +++ b/timescale-container/test/docker-compose.yml @@ -3,7 +3,7 @@ version: '3' services: timescale: - image: timescale/timescaledb-postgis:1.7.1-pg12 + image: timescale/timescaledb-postgis:${TIMESCALE_VERSION} ports: - "5432:5432" networks: diff --git a/timescale-container/test/ql-db-setup.sh b/timescale-container/test/ql-db-setup.sh index 5902b40c..5987f776 100755 --- a/timescale-container/test/ql-db-setup.sh +++ b/timescale-container/test/ql-db-setup.sh @@ -2,7 +2,7 @@ set -e -DOCKER_IMG=timescale/timescaledb-postgis:1.7.1-pg12 +DOCKER_IMG=timescale/timescaledb-postgis:${TIMESCALE_VERSION} PORT=5432 PASS=abc123 DATA=ql-db-init/mtutenant.etdevice.csv