improved logs (#355)

* improved logs

* improved logs

* Update error log with payload

src/geocoding/factory.py

@@ -21,7 +21,7 @@ class GeoCodingEnvReader:
     """
 
     def __init__(self):
-        self.env = EnvReader(log=logging.getLogger(__name__).info)
+        self.env = EnvReader(log=logging.getLogger(__name__).debug)
 
     def use_geocoding(self) -> bool:
         return self.env.read(BoolVar('USE_GEOCODING', False))
@@ -62,9 +62,9 @@ def get_geo_cache() -> MaybeGeoCache:
     """
     env = GeoCodingEnvReader()
     if is_geo_coding_available():
-        log().info("Geo Cache env variables set, building a cache.")
+        log().debug("Geo Cache env variables set, building a cache.")
 
         return GeoCodingCache(env.redis_host(), env.redis_port())
 
-    log().info("Geo Cache env variables indicate cache should not be used.")
+    log().debug("Geo Cache env variables indicate cache should not be used.")
     return None

src/reporter/reporter.py

@@ -183,16 +183,16 @@ def notify():
             e_new = _filter_no_type_no_value_entities(e)
             res_entity.append(e_new)
     payload = res_entity
-
+    entity_id = [i["id"] for i in payload]
     # Send valid entities to translator
     try:
         with translator_for(fiware_s) as trans:
             trans.insert(payload, fiware_s, fiware_sp)
     except:
-        msg = "Notification not processed or not updated"
+        msg = "Notification not processed or not updated for payload: %s" % (payload)
         log().error(msg)
         return msg, 500
-    msg = 'Notification successfully processed'
+    msg = "Notification successfully processed for : 'tenant' %s, 'fiwareServicePath' %s, 'entity_id' %s" % (fiware_s, fiware_sp, entity_id)
     log().info(msg)
     return msg
 

src/reporter/tests/test_Headers.py

@@ -23,7 +23,7 @@ def test_for_valid_headers(notification):
                       headers=HEADERS_VALID)
     time.sleep(1)
     assert res_post.status_code == 200
-    assert res_post.json() == 'Notification successfully processed'
+    assert res_post.json().startswith('Notification successfully processed')
 
     get_url = "{}/entities/Room0".format(QL_URL)
     res_get = requests.get(get_url, headers=HEADERS_VALID)
@@ -51,7 +51,7 @@ def test_for_invalid_headers(notification):
                       headers=HEADERS_VALID)
 
     assert res_post.status_code == 200
-    assert res_post.json() == 'Notification successfully processed'
+    assert res_post.json().startswith('Notification successfully processed')
 
     get_url = "{}/entities/Room0".format(QL_URL)
     res_get = requests.get(get_url, headers=HEADERS_INVALID)

src/reporter/tests/test_notify.py

@@ -76,9 +76,8 @@ def test_valid_notification(notification, clean_mongo, clean_crate):
     r = requests.post('{}'.format(notify_url),
                       data=json.dumps(notification),
                       headers=HEADERS_PUT)
-
     assert r.status_code == 200
-    assert r.json() == 'Notification successfully processed'
+    assert r.json().startswith('Notification successfully processed')
 
 
 def test_valid_no_modified(notification, clean_mongo, clean_crate):
@@ -87,7 +86,7 @@ def test_valid_no_modified(notification, clean_mongo, clean_crate):
                       data=json.dumps(notification),
                       headers=HEADERS_PUT)
     assert r.status_code == 200
-    assert r.json() == 'Notification successfully processed'
+    assert r.json().startswith('Notification successfully processed')
 
 def do_integration(entity, notify_url, orion_client, crate_translator):
     entity_id = entity['id']
@@ -220,7 +219,7 @@ def test_multiple_data_elements(notification, sameEntityWithDifferentAttrs, clea
     r = requests.post('{}'.format(notify_url), data=json.dumps(notification),
                       headers=HEADERS_PUT)
     assert r.status_code == 200
-    assert r.json() == 'Notification successfully processed'
+    assert r.json().startswith('Notification successfully processed')
 
 
 def test_time_index(notification, clean_mongo, crate_translator):
@@ -236,7 +235,7 @@ def test_time_index(notification, clean_mongo, crate_translator):
                       data=json.dumps(notification),
                       headers=HEADERS_PUT)
     assert r.status_code == 200
-    assert r.json() == 'Notification successfully processed'
+    assert r.json().startswith('Notification successfully processed')
 
     time.sleep(1)
     entity_type = notification['data'][0]['type']
@@ -261,7 +260,7 @@ def test_time_index(notification, clean_mongo, crate_translator):
                       data=json.dumps(notification),
                       headers=HEADERS_PUT)
     assert r.status_code == 200
-    assert r.json() == 'Notification successfully processed'
+    assert r.json().startswith('Notification successfully processed')
 
     time.sleep(1)
     crate_translator._refresh([entity_type])
@@ -278,7 +277,7 @@ def test_time_index(notification, clean_mongo, crate_translator):
                       data=json.dumps(notification),
                       headers=HEADERS_PUT)
     assert r.status_code == 200
-    assert r.json() == 'Notification successfully processed'
+    assert r.json().startswith('Notification successfully processed')
 
     time.sleep(1)
     crate_translator._refresh([entity_type])

src/translators/factory.py

@@ -19,7 +19,7 @@ def log():
 
 def lookup_backend(fiware_service: str) -> MaybeString:
     cfg_reader = YamlReader(log=log().debug)
-    env_reader = EnvReader(log=log().info)
+    env_reader = EnvReader(log=log().debug)
 
     config = cfg_reader.from_env_file(QL_CONFIG_ENV_VAR, defaults={})
     tenant_backend = maybe_string_match(config, 'tenants', fiware_service,
@@ -45,6 +45,6 @@ def translator_for(fiware_service: str):
         translator = CrateTranslatorInstance()
         selected = CRATE_BACKEND
 
-    log().info(
+    log().debug(
         f"Backend selected for tenant '{fiware_service}' is: {selected}")
     return translator

src/translators/sql_translator.py

@@ -239,6 +239,7 @@ def _insert_entities_of_type(self,
 
                 col = self._ea2cn(attr)
                 original_attrs[col] = (attr, attr_t)
+                entity_id = e.get('id')
 
                 if attr_t not in self.NGSI_TO_SQL:
                     # if attribute is complex assume it as an NGSI StructuredValue
@@ -249,11 +250,13 @@ def _insert_entities_of_type(self,
                     else:
                         # TODO fallback type should be defined by actual JSON type
                         supported_types = ', '.join(self.NGSI_TO_SQL.keys())
-                        msg = ("'{}' is not a supported NGSI type. "
+                        msg = ("'{}' is not a supported NGSI type"
+                               " for Attribute:  '{}' "
+                               " and id : '{}'. "
                                "Please use any of the following: {}. "
                                "Falling back to {}.")
                         self.logger.warning(msg.format(
-                            attr_t, supported_types, NGSI_TEXT))
+                            attr_t, attr, entity_id, supported_types, NGSI_TEXT))
 
                         table[col] = self.NGSI_TO_SQL[NGSI_TEXT]