Timescale backend teething troubles (#237)

* cast postgres port to int

* extend jsondict to support case-insensitive path lookups.

* make translator lookup case insensitive both on keys and values.

* add logging of selected backend

* expose ssl config switch.

src/reporter/query_1T1E1A.py

@@ -4,7 +4,7 @@
 from translators.crate import CrateTranslatorInstance
 import logging
 from .geo_query_handler import handle_geo_query
-from utils.dicts import lookup_string_match
+from utils.jsondict import lookup_string_match
 
 
 def query_1T1E1A(attr_name,   # In Path

src/reporter/query_1TNE1A.py

@@ -4,7 +4,7 @@
 from translators.crate import CrateTranslatorInstance, CrateTranslator
 import logging
 from .geo_query_handler import handle_geo_query
-from utils.dicts import lookup_string_match
+from utils.jsondict import lookup_string_match
 
 
 def query_1TNE1A(attr_name,   # In Path

src/translators/factory.py

@@ -1,32 +1,50 @@
-import yaml
+import logging
 import os
+import yaml
 
 from translators.crate import CrateTranslatorInstance
 from translators.timescale import postgres_translator_instance
-from utils.jsondict import maybe_value
+from utils.jsondict import maybe_string_match
 
 
-CRATE_BACKEND = 'Crate'
-TIMESCALE_BACKEND = 'Timescale'
+CRATE_BACKEND = 'crate'
+TIMESCALE_BACKEND = 'timescale'
 
 QL_CONFIG_ENV_VAR = 'QL_CONFIG'
 
 
+def log():
+    logging.basicConfig(level=logging.INFO)
+    return logging.getLogger(__name__)
+
+
 def read_config() -> dict:
     path = os.environ.get(QL_CONFIG_ENV_VAR)
     if path:
+        log().info(f"using config file: {path}")
         file = open(path)
         return yaml.safe_load(file)
+
+    log().info(f"no config file specified, using defaults.")
     return {}
 
 
 def translator_for(fiware_service: str):
     config = read_config()
-    backend = maybe_value(config, 'tenants', fiware_service, 'backend')\
-        or maybe_value(config, 'default-backend')
+    backend = maybe_string_match(config, 'tenants', fiware_service, 'backend')\
+        or maybe_string_match(config, 'default-backend')
+    backend = backend.strip().lower() if backend is not None else ''
 
     if backend == CRATE_BACKEND:
-        return CrateTranslatorInstance()
-    if backend == TIMESCALE_BACKEND:
-        return postgres_translator_instance()
-    return CrateTranslatorInstance()
+        translator = CrateTranslatorInstance()
+        selected = CRATE_BACKEND
+    elif backend == TIMESCALE_BACKEND:
+        translator = postgres_translator_instance()
+        selected = TIMESCALE_BACKEND
+    else:
+        translator = CrateTranslatorInstance()
+        selected = CRATE_BACKEND
+
+    log().info(
+        f"Backend selected for tenant '{fiware_service}' is: {selected}")
+    return translator

src/translators/tests/ql-config.yml

@@ -3,6 +3,6 @@ tenants:
     backend: Timescale
   t2:
     backend: Crate
-  t3:
+  T3:
     backend: Timescale
 default-backend: Crate

src/translators/tests/test_timescale_insert.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timezone
+import os
 import pg8000
 import pytest
 import random
@@ -123,6 +124,9 @@ def insert(entities, fw_svc=None, fw_path=None):
 
 @pytest.fixture(scope='module')
 def with_pg8000():
+    pg_port_var = 'POSTGRES_PORT'
+    os.environ[pg_port_var] = '54320'
+
     pg8000.paramstyle = "qmark"
     t = PostgresConnectionData()
     t.read_env()
@@ -135,6 +139,7 @@ def with_pg8000():
 
     yield (pg_conn, pg_cursor)
 
+    os.environ[pg_port_var] = ''
     pg_cursor.close()
     pg_conn.close()
 

src/translators/tests/test_translator_factory.py

@@ -20,8 +20,11 @@ def write_config() -> str:
 def with_config():
     path = os.path.join(os.path.dirname(__file__), 'ql-config.yml')
     os.environ[QL_CONFIG_ENV_VAR] = path
+    pg_port_var = 'POSTGRES_PORT'
+    os.environ[pg_port_var] = '54320'
     yield {}
     os.environ[QL_CONFIG_ENV_VAR] = ''
+    os.environ[pg_port_var] = ''
 
 
 def test_tenant1(with_config):
@@ -42,3 +45,8 @@ def test_tenant3(with_config):
 def test_unknown_tenant(with_config):
     with translator_for('not-in-config') as t:
         assert isinstance(t, CrateTranslator)
+
+
+def test_no_tenant(with_config):
+    with translator_for(None) as t:
+        assert isinstance(t, CrateTranslator)

src/translators/timescale.py

@@ -69,11 +69,12 @@
 
 class PostgresConnectionData:
 
-    def __init__(self, host='timescale', port=5432,
+    def __init__(self, host='timescale', port=5432, use_ssl=False,
                  db_name='quantumleap',
                  db_user='quantumleap', db_pass='*'):
         self.host = host
         self.port = port
+        self.use_ssl = use_ssl
         self.db_name = db_name
         self.db_user = db_user
         self.db_pass = db_pass
@@ -85,7 +86,11 @@ def read_env(self):
 
         env_var = os.environ.get('POSTGRES_PORT')
         if env_var:
-            self.port = env_var
+            self.port = int(env_var)
+
+        env_var = os.environ.get('POSTGRES_USE_SSL')
+        if env_var:
+            self.use_ssl = env_var.strip().lower() in ('true', 'yes', '1', 't')
 
         env_var = os.environ.get('POSTGRES_DB_NAME')
         if env_var:
@@ -108,12 +113,13 @@ def __init__(self, conn_data=PostgresConnectionData()):
         self.logger = logging.getLogger(__name__)
         self.db_user = conn_data.db_user
         self.db_pass = conn_data.db_pass
+        self.ssl = {} if conn_data.use_ssl else None
         self.conn = None
         self.cursor = None
 
     def setup(self):
         pg8000.paramstyle = "qmark"
-        self.conn = pg8000.connect(host=self.host, port=self.port,
+        self.conn = pg8000.connect(host=self.host, port=self.port, ssl=self.ssl,
                                    database=self.db_name,
                                    user=self.db_user, password=self.db_pass)
         self.cursor = self.conn.cursor()