Merge pull request #121 from istresearch/scutils_log_update

Scutils LogFactory update

crawler/requirements.txt

@@ -28,7 +28,7 @@ requests==2.13.0 # Updated from 2.11.1
 requests-file==1.4.1 # Updated from 1.4
 retrying==1.3.3
 Scrapy==1.3.3
-scutils==1.2.0
+../utils # scutils==1.3.0dev0
 service-identity==16.0.0
 six==1.10.0
 testfixtures==4.13.5 # Updated from 4.10.0

docker/crawler/Dockerfile

@@ -15,8 +15,10 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # install requirements
+COPY utils /usr/src/utils
 COPY crawler/requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r requirements.txt
+RUN rm -rf /usr/src/utils
 
 # move codebase over
 COPY crawler /usr/src/app

docker/crawler/Dockerfile.py2alpine

@@ -4,6 +4,7 @@ MAINTAINER Madison Bahmer <madison.bahmer@istresearch.com>
 
 # copy crawler own requirements.txt with its dependencies
 COPY crawler/requirements.txt /usr/src/app/
+COPY utils /usr/src/utils
 
 # Combine run command to create single intermeiate image layer
 # This MANDATORY because developments dependencies are huge.
@@ -32,6 +33,7 @@ RUN mkdir -p /usr/src/app \
  && apk del build-dependencies
 
 # move codebase over
+RUN rm -rf /usr/src/utils
 COPY crawler /usr/src/app
 
 WORKDIR /usr/src/app

docker/kafka-monitor/Dockerfile

@@ -7,8 +7,10 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # install requirements
+COPY utils /usr/src/utils
 COPY kafka-monitor/requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r requirements.txt
+RUN rm -rf /usr/src/utils
 
 # move codebase over
 COPY kafka-monitor /usr/src/app

docker/kafka-monitor/Dockerfile.py2alpine

@@ -4,6 +4,7 @@ MAINTAINER Madison Bahmer <madison.bahmer@istresearch.com>
 
 # copy kafka-monitor own requirements.txt with its dependencies
 COPY kafka-monitor/requirements.txt /usr/src/app/
+COPY utils /usr/src/utils
 
 # Combine run command to create single intermeiate image layer
 # This MANDATORY because developments dependencies are huge.
@@ -24,6 +25,7 @@ RUN mkdir -p /usr/src/app \
  && apk del build-dependencies
 
 # move codebase over
+RUN rm -rf /usr/src/utils
 COPY kafka-monitor /usr/src/app
 
 WORKDIR /usr/src/app

docker/redis-monitor/Dockerfile

@@ -7,8 +7,10 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # install requirements
+COPY utils /usr/src/utils
 COPY redis-monitor/requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r requirements.txt
+RUN rm -rf /usr/src/utils
 
 # move codebase over
 COPY redis-monitor /usr/src/app

docker/redis-monitor/Dockerfile.py2alpine

@@ -3,6 +3,7 @@ MAINTAINER Madison Bahmer <madison.bahmer@istresearch.com>
 
 # copy redis-monitor own requirements.txt with its dependencies
 COPY redis-monitor/requirements.txt /usr/src/app/
+COPY utils /usr/src/utils
 
 # Combine run command to create single intermeiate image layer
 # This MANDATORY because developments dependencies are huge.
@@ -23,6 +24,7 @@ RUN mkdir -p /usr/src/app \
  && apk del build-dependencies
 
 # move codebase over
+RUN rm -rf /usr/src/utils
 COPY redis-monitor /usr/src/app
 
 WORKDIR /usr/src/app

docker/rest/Dockerfile

@@ -7,8 +7,10 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # install requirements
+COPY utils /usr/src/utils
 COPY rest/requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r requirements.txt
+RUN rm -rf /usr/src/utils
 
 # move codebase over
 COPY rest /usr/src/app

docker/rest/Dockerfile.py2alpine

@@ -2,6 +2,7 @@ FROM python:2.7.12-alpine
 MAINTAINER Madison Bahmer <madison.bahmer@istresearch.com>
 
 # copy rest own requirements.txt with its dependencies
+COPY utils /usr/src/utils
 COPY rest/requirements.txt /usr/src/app/
 
 # Combine run command to create single intermeiate image layer
@@ -23,6 +24,7 @@ RUN mkdir -p /usr/src/app \
  && apk del build-dependencies
 
 # move codebase over
+RUN rm -rf /usr/src/utils
 COPY rest /usr/src/app
 
 WORKDIR /usr/src/app

kafka-monitor/requirements.txt

@@ -15,7 +15,7 @@ redis==2.10.5
 requests==2.13.0 # Updated from 2.11.1
 requests-file==1.4.1 # Updated from 1.4
 retrying==1.3.3
-scutils==1.2.0
+../utils # scutils==1.3.0dev0
 six==1.10.0
 testfixtures==4.13.5 # Updated from 4.10.0
 tldextract==2.0.2 # Updated from 2.0.1

redis-monitor/requirements.txt

@@ -11,7 +11,7 @@ python-redis-lock==3.2.0 # Updated from 3.1.0
 PyYAML==3.12
 redis==2.10.5
 retrying==1.3.3
-scutils==1.2.0
+../utils # scutils==1.3.0dev0
 six==1.10.0
 testfixtures==4.13.5 # Updated from 4.10.0
 ujson==1.35

requirements.txt

@@ -41,7 +41,7 @@ redis==2.10.5
 requests-file==1.4.1 # Updated from 1.4
 requests==2.13.0 # Updated from 2.11.1
 retrying==1.3.3
-scutils==1.2.0
+./utils # scutils==1.3.0dev0
 service-identity==16.0.0
 six==1.10.0
 testfixtures==4.13.5 # Updated from 4.11.0

rest/requirements.txt

@@ -16,7 +16,7 @@ python-json-logger==0.1.7 # Updated from 0.1.5
 redis==2.10.5
 requests==2.13.0 # Updated from 2.11.1
 retrying==1.3.3
-scutils==1.2.0
+../utils # scutils==1.3.0dev0
 six==1.10.0
 testfixtures==4.13.5 # Updated from 4.11.0
 ujson==1.35

travis/ansible.sh

@@ -17,7 +17,7 @@ sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm /bin/bash -c "ans
 sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm /bin/bash -c "ansible-playbook -i ${PWD}/ansible/travis.inventory ${PWD}/ansible/scrapy-cluster.yml --connection=local --become"
 
 # Install coveralls and other pip requiremnts
-sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm /bin/bash -c "virtualenv ${PWD}/sc; source ${PWD}/sc/bin/activate; pip install -r ${PWD}/requirements.txt; cd ${PWD}; find . -name "*.pyc" -type f -delete;"
+sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm /bin/bash -c "virtualenv ${PWD}/sc; source ${PWD}/sc/bin/activate; cd ${PWD}; pip install -r requirements.txt; find . -name "*.pyc" -type f -delete;"
 
 # Run offline tests
 sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm /bin/bash -c "source ${PWD}/sc/bin/activate; cd ${PWD}; ./run_offline_tests.sh"

utils/scutils/log_factory.py

@@ -25,51 +25,66 @@ def get_instance(self, **kwargs):
 
         return self._instance
 
-class LogCallbackMixin:
+class LogCallbackHandler:
+
+    def __init__(self, logger):
+        self.logger = logger
+        self.callbacks = OrderedDict([
+            ("DEBUG", []),
+            ("INFO", []),
+            ("WARNING", []),
+            ("ERROR", []),
+            ("CRITICAL", []),
+        ])
+
     def parse_log_level(self, log_level):
-        MIN_LOG_LEVEL = self.level_dict['DEBUG']
-        MAX_LOG_LEVEL = self.level_dict['CRITICAL']
+        MIN_LOG_LEVEL = self.logger.level_dict['DEBUG']
+        MAX_LOG_LEVEL = self.logger.level_dict['CRITICAL']
 
         chrs = log_level[:2]
         if chrs == '<=':
             log_level = log_level[2:]
-            log_level_n = self.level_dict[log_level]
+            log_level_n = self.logger.level_dict[log_level]
             r = range(MIN_LOG_LEVEL, log_level_n + 1)
         elif chrs.startswith('<'):
             log_level = log_level[1:]
-            log_level_n = self.level_dict[log_level]
+            log_level_n = self.logger.level_dict[log_level]
             r = range(MIN_LOG_LEVEL, log_level_n)
         elif chrs == '>=':
             log_level = log_level[2:]
-            log_level_n = self.level_dict[log_level]
+            log_level_n = self.logger.level_dict[log_level]
             r = range(log_level_n, MAX_LOG_LEVEL+1)
         elif chrs.startswith('>'):
             log_level = log_level[1:]
-            log_level_n = self.level_dict[log_level]
+            log_level_n = self.logger.level_dict[log_level]
             r = range(log_level_n+1, MAX_LOG_LEVEL+1)
         elif chrs.startswith('='):
             log_level = log_level[1:]
-            log_level_n = self.level_dict[log_level]
+            log_level_n = self.logger.level_dict[log_level]
             r = range(log_level_n, log_level_n+1)
         elif chrs == '*':
             r = range(MIN_LOG_LEVEL, MAX_LOG_LEVEL+1)
         else:
-            log_level_n = self.level_dict[log_level]
+            log_level_n = self.logger.level_dict[log_level]
             r = range(log_level_n, log_level_n+1)
 
-        return r
+        return list(r)
 
     def is_subdict(self, a,b):
         '''
         Return True if a is a subdict of b
         '''
-        return all((k in b and b[k]==v) for k,v in a.iteritems())
+        return all((k in b and b[k]==v) for k,v in a.items())
 
 
     def register_callback(self, log_level, fn, criteria=None):
         criteria = criteria or {}
 
-        num_to_level_map = {v: k for k, v in self.level_dict.iteritems()}
+        num_to_level_map = {v: k for k, v in self.logger.level_dict.items()}
+
+        # Force normalization of WARN
+        num_to_level_map[2] = "WARNING"
+
         log_range = self.parse_log_level(log_level)
 
         for log_n in log_range:
@@ -88,7 +103,8 @@ def fire_callbacks(self, log_level, log_message=None, log_extra=None):
             else:
                 cb(log_message, log_extra)
 
-class LogObject(object, LogCallbackMixin):
+
+class LogObject(object):
     '''
     Easy wrapper for writing json logs to a rotating file log
     '''
@@ -129,13 +145,7 @@ def __init__(self, json=False, stdout=True, name='scrapy-cluster',
         self.log_level = level
         self.format_string = format
         self.include_extra = include_extra
-        self.callbacks = OrderedDict([
-            ("DEBUG", []),
-            ("INFO", []),
-            ("WARNING", []),
-            ("ERROR", []),
-            ("CRITICAL", []),
-        ])
+        self.cb_handler = LogCallbackHandler(self)
 
         if stdout:
             # set up to std out
@@ -198,7 +208,7 @@ def debug(self, message, extra={}):
         if self.level_dict['DEBUG'] >= self.level_dict[self.log_level]:
             extras = self.add_extras(extra, "DEBUG")
             self._write_message(message, extras)
-            self.fire_callbacks('DEBUG', message, extra)
+            self.cb_handler.fire_callbacks('DEBUG', message, extra)
 
     def info(self, message, extra={}):
         '''
@@ -210,7 +220,7 @@ def info(self, message, extra={}):
         if self.level_dict['INFO'] >= self.level_dict[self.log_level]:
             extras = self.add_extras(extra, "INFO")
             self._write_message(message, extras)
-            self.fire_callbacks('INFO', message, extra)
+            self.cb_handler.fire_callbacks('INFO', message, extra)
 
     def warn(self, message, extra={}):
         '''
@@ -231,7 +241,7 @@ def warning(self, message, extra={}):
         if self.level_dict['WARNING'] >= self.level_dict[self.log_level]:
             extras = self.add_extras(extra, "WARNING")
             self._write_message(message, extras)
-            self.fire_callbacks('WARNING', message, extra)
+            self.cb_handler.fire_callbacks('WARNING', message, extra)
 
     def error(self, message, extra={}):
         '''
@@ -243,7 +253,7 @@ def error(self, message, extra={}):
         if self.level_dict['ERROR'] >= self.level_dict[self.log_level]:
             extras = self.add_extras(extra, "ERROR")
             self._write_message(message, extras)
-            self.fire_callbacks('ERROR', message, extra)
+            self.cb_handler.fire_callbacks('ERROR', message, extra)
 
     def critical(self, message, extra={}):
         '''
@@ -255,7 +265,7 @@ def critical(self, message, extra={}):
         if self.level_dict['CRITICAL'] >= self.level_dict[self.log_level]:
             extras = self.add_extras(extra, "CRITICAL")
             self._write_message(message, extras)
-            self.fire_callbacks('CRITICAL', message, extra)
+            self.cb_handler.fire_callbacks('CRITICAL', message, extra)
 
     def _write_message(self, message, extra):
         '''
@@ -330,3 +340,10 @@ def _get_time(self):
         Returns the system time
         '''
         return datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+
+    def register_callback(self, log_level, fn, criteria=None):
+        '''
+        Passthrough for registering callbacks to preserve backwards
+        compatibility
+        '''
+        return self.cb_handler.register_callback(log_level, fn, criteria)

utils/scutils/version.py

@@ -1,2 +1,2 @@
-__version__ = '1.2.0'
+__version__ = '1.3.0dev0'
 VERSION = tuple(int(x) for x in __version__.split('.'))

utils/tests/test_argparse_helper.py

@@ -9,7 +9,11 @@
 
 # from http://stackoverflow.com/questions/4219717/how-to-assert-output-with-nosetest-unittest-in-python
 from contextlib import contextmanager
-from StringIO import StringIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 @contextmanager
 def captured_output():
     new_out, new_err = StringIO(), StringIO()