Use a ConcurrentRotatingFileHandler for logging

Using a normal RotatingFileHandler can cause problems if there are
multiple processes accessing the same logfile at the same time.
We run multiple threads of some of our workers and so we should be
careful when they all log to the same file.

requirements.txt

@@ -1,6 +1,6 @@
 Django==1.9.5
 BeautifulSoup==3.2.0
-#apt-get install libyaml-dev (to allow the cloader) 
+#apt-get install libyaml-dev (to allow the cloader)
 PyYAML==3.11
 pycrypto==2.6.1
 feedparser==5.1.3
@@ -36,3 +36,4 @@ raven==5.10.1
 django-multiupload==0.5
 zenpy==1.1.3
 PyJWT==1.4.2
+ConcurrentLogHandler==0.9.1

requirements_processing.txt

@@ -1,3 +1,3 @@
 numpy==1.9.0
 scikits.audiolab==0.11.0
-#essentia
+#essentia

requirements_similarity.txt

@@ -2,7 +2,8 @@ PyYAML==3.11
 Twisted==14.0.2
 graypy==0.2.11
 numpy==1.9.0
+ConcurrentLogHandler==0.9.1
 #scipy==0.10.1
 #apt-get build-dep python-scipy
 #scikit-learn==0.15.2
-#gaia2
+#gaia2

similarity/similarity_server.py

@@ -29,6 +29,7 @@
 from similarity_server_utils import parse_filter, parse_target, parse_metric_descriptors
 import json
 import yaml
+import cloghandler
 
 
 def server_interface(resource):
@@ -230,7 +231,7 @@ def save(self, request, filename=None):
         print("LOG_TO_STDOUT is False, will not log")
     logger = logging.getLogger('similarity')
     logger.setLevel(logging.DEBUG)
-    handler = RotatingFileHandler(LOGFILE, maxBytes=2*1024*1024, backupCount=5)
+    handler = cloghandler.ConcurrentRotatingFileHandler(LOGFILE, "a", maxBytes=2*1024*1024, backupCount=5)
     handler.setLevel(logging.DEBUG)
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     handler.setFormatter(formatter)

tagrecommendation/tagrecommendation_server.py

@@ -33,6 +33,7 @@
 import json
 from communityBasedTagRecommendation import CommunityBasedTagRecommender
 from utils import loadFromJson, saveToJson
+import cloghandler
 
 
 def server_interface(resource):
@@ -149,7 +150,7 @@ def add_to_index(self, sound_ids, sound_tagss):
         print("LOG_TO_STDOUT is False, will not log")
     logger = logging.getLogger('tagrecommendation')
     logger.setLevel(logging.DEBUG)
-    handler = RotatingFileHandler(LOGFILE, maxBytes=2*1024*1024, backupCount=5)
+    handler = cloghandler.ConcurrentRotatingFileHandler(LOGFILE, mode="a", maxBytes=2*1024*1024, backupCount=5)
     handler.setLevel(logging.DEBUG)
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     handler.setFormatter(formatter)