Added logging. Fixed #29, fixed #34.

src/wa_kat/bottle_index.py

@@ -72,6 +72,7 @@ def get_remote_info(url_id):  # TODO: Add timeout, print error in case of except
     return data
 
 
+# API =========================================================================
 # TODO: REMOVE
 @get("/" + settings.REMOTE_INFO_URL.split("/")[-1])
 def mock_data():

src/wa_kat/rest_api/__init__.py

@@ -50,22 +50,26 @@ def get_result(url):
         error_msg = """
             Požadovanou stránku {url} nebylo možné stáhnout během {timeout}
             vteřin. Zkuste URL zadat s `www.`, či zkontrolovat funkčnost
-            stránek."""
+            stránek.
+        """
         error_msg = error_msg.format(url=url, timeout=settings.REQUEST_TIMEOUT)
 
         return {
             "status": False,
+            "log": ri.get_log(),
             "error": error_msg,
         }
     except Exception as e:
         return {
             "status": False,
+            "log": ri.get_log(),
             "error": str(e.message) + "\n" + traceback.format_exc().strip()
         }
 
     return {
         "status": True,
         "body": ri.to_dict(),
+        "log": ri.get_log(),
         "conspect_dict": ConspectDatabase().data,
     }
 

src/wa_kat/zeo/request_info.py

@@ -74,6 +74,7 @@ def __init__(self, url):
         self.url = url
         self.domain = urlparse(url).netloc
         self.index = None
+        self._log = {}
 
         # time trackers
         self.creation_ts = time.time()
@@ -98,6 +99,17 @@ def _download(self, url):
 
         return resp.text.encode("utf-8")  # TODO: what about binaries?
 
+    @transaction_manager
+    def log(self, msg):
+        self._log[time.time()] = msg
+
+    @transaction_manager
+    def get_log(self):
+        return "\n".join(
+            "%s: %s" % (str(timestamp), self._log[timestamp])
+            for timestamp in sorted(self._log.keys())
+        )
+
     def paralel_processing(self, client_conf=None):
         """
         Lauch paralel processes (see :func:`.worker`) to fill properties with
@@ -185,14 +197,16 @@ def is_old(self):
         if not self.processing_started_ts:
             return True
 
+        if self.processing_ended_ts:
+            return self.processing_ended_ts + ZEO_CACHE_TIME < time.time()
+
         # in case that processing started, but didn't ended in
         # ZEO_MAX_WAIT_TIME
         expected_end_ts = self.creation_ts + ZEO_MAX_WAIT_TIME
-        ended = (self.processing_ended_ts and self.is_all_set())
-        if not ended and expected_end_ts < time.time():
-            return True
+        if expected_end_ts < time.time():
+            self.log("Prosessing timeouted and properites were not set!")
 
-        return self.processing_ended_ts + ZEO_CACHE_TIME < time.time()
+        return expected_end_ts < time.time()
 
     @transaction_manager
     def to_dict(self):

src/wa_kat/zeo/worker.py

@@ -5,6 +5,7 @@
 #
 # Imports =====================================================================
 import time
+import traceback
 
 import transaction
 from ZODB.POSException import ConflictError
@@ -13,15 +14,13 @@
 # Functions & classes =========================================================
 def _save_to_database(req, property_name, data):
     with transaction.manager:
-        # val = getattr(req, property_name)
-        # if val is not None:
-        #     val.extend(data)
-        # else:
         setattr(req, property_name, data)
 
         req.processing_ended_ts = time.time()
         req._p_changed = True
 
+    req.log("`%s` saved." % property_name)
+
 
 def worker(url_key, property_name, function, function_arguments,
            conf_path=None):
@@ -43,11 +42,16 @@ def worker(url_key, property_name, function, function_arguments,
             redirect the database connections to test's environment.
     """
     # this may take some time, hence outside transaction manager
+    error_msg = None
     try:
         data = function(*function_arguments)
     except Exception as e:
-        # TODO: log e
         data = []
+        error_msg = "Error: " + traceback.format_exc().strip()
+        error_msg += "\n" + str(e.message)
+
+        with open("/home/bystrousak/wa_error.log", "a") as f:
+            f.write(error_msg)
 
     # get the RequestInfo object from database
     from .request_database import RequestDatabase
@@ -58,11 +62,24 @@ def worker(url_key, property_name, function, function_arguments,
 
     # save `data` into RequestInfo object property
     for i in xrange(5):
+        # resolve the request_info object
+        request_info = db.get_request(url_key)
+
+        # handle logging of possible error message
+        if error_msg:
+            request_info.log(error_msg)
+            error_msg = None
+
+        func_name = str(function.__name__)
+        request_info.log("Attempting to save output from `%s`." % func_name)
+
+        # attempt to save the data into database
         try:
             return _save_to_database(
-                req=db.get_request(url_key),
+                req=request_info,
                 property_name=property_name,
                 data=data
             )
         except ConflictError:
+            request_info.log("Attempt to save `%s` failed." % func_name)
             time.sleep(0.1)