Merge pull request #197 from mkurek/feature/log-separation

added DirectoryTimedRotatingFileHandler, removed sentry workaround, impr...

src/ralph_pricing/app.py

@@ -6,26 +6,9 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import logging
-
 from ralph.app import RalphModule
 
 
-def setup_scrooge_logger(level='ERROR'):
-    from django.conf import settings
-    if 'raven.contrib.django' in settings.INSTALLED_APPS:
-        from raven import Client
-        from raven.handlers.logging import SentryHandler
-        SCROOGE_SENTRY_DSN = getattr(settings, 'SCROOGE_SENTRY_DSN', False)
-        if SCROOGE_SENTRY_DSN:
-            client = Client(SCROOGE_SENTRY_DSN)
-            handler = SentryHandler(client, level=level)
-            logger = logging.getLogger('ralph_pricing')
-            logger.addHandler(handler)
-            return True
-    return False
-
-
 class Scrooge(RalphModule):
     """Scrooge main application. The 'ralph_pricing' name is retained
     internally for historical reasons, while we try to use 'scrooge' as
@@ -35,6 +18,7 @@ class Scrooge(RalphModule):
     module_name = 'ralph_pricing'
     disp_name = 'Scrooge'
     icon = 'fugue-money-coin'
+    default_settings_module = 'ralph_pricing.settings'
 
     @property
     def required_permission(self):
@@ -49,35 +33,14 @@ def __init__(self, **kwargs):
         )
         self.append_app()
         self.insert_templates(__file__)
-        if not setup_scrooge_logger():
-            self.register_logger('ralph_pricing', {
-                'handlers': ['file'],
-                'propagate': True,
-                'level': 'DEBUG',
-            })
-            self.register_logger('ralph_pricing.plugins', {
-                'handlers': ['file', 'console'],
-                'propagate': True,
-                'level': 'DEBUG',
-            })
-            self.register_logger('ralph_pricing.management', {
-                'handlers': ['file', 'console'],
-                'propagate': True,
-                'level': 'DEBUG',
-            })
-        app_settings = {
-            'SSH_NFSEN_CREDENTIALS': {},
-            'NFSEN_CHANNELS': [],
-            'NFSEN_CLASS_ADDRESS': [],
-            'NFSEN_FILES_PATH': '',
-            'VIRTUAL_VENTURE_NAMES': {},
-            'WARNINGS_LIMIT_FOR_USAGES': 40,
-            'CLOUD_UNKNOWN_VENTURE': None,
-            'SHARE_VENTURE_SYMBOLS': {},
-            'SHARES_UNKNOWN_VENTURE': None,  # symbol
-        }
-        # workaround to not overwriting manually defined settings
-        # check if setting is in global settings - if no, add default
-        for key, value in app_settings.iteritems():
-            if key not in self.settings:
-                self.settings[key] = value
+        self.register_logger('ralph_pricing', {
+            'handlers': ['file'],
+            'propagate': True,
+            'level': 'DEBUG',
+        })
+        self.register_logger('ralph_pricing.plugins', {
+            'handlers': ['console'],
+        })
+        self.register_logger('ralph_pricing.management', {
+            'handlers': ['console'],
+        })

src/ralph_pricing/logging.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import time
+from logging.handlers import TimedRotatingFileHandler
+
+
+class DirectoryTimedRotatingFileHandler(TimedRotatingFileHandler):
+    """
+    Timed Rotating File Handler with saving files to date / time directory
+    instead of adding suffix with date / time to log file.
+
+    Based totally on TimedRotatingFileHandler, instead of rollovering log file.
+    """
+    def doRollover(self):
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+        # get the time that this sequence started at and make it a TimeTuple
+        currentTime = int(time.time())
+        dstNow = time.localtime(currentTime)[-1]
+        t = self.rolloverAt - self.interval
+        if self.utc:
+            timeTuple = time.gmtime(t)
+        else:
+            timeTuple = time.localtime(t)
+            dstThen = timeTuple[-1]
+            if dstNow != dstThen:
+                if dstNow:
+                    addend = 3600
+                else:
+                    addend = -3600
+                timeTuple = time.localtime(t + addend)
+
+        # diff
+        dirName, baseName = os.path.split(self.baseFilename)
+        time_directory = os.path.join(
+            dirName,
+            time.strftime(self.suffix, timeTuple)
+        )
+        dfn = os.path.join(time_directory, baseName)
+        if not os.path.exists(time_directory):
+            os.mkdir(time_directory)
+        # end of diff
+
+        if os.path.exists(dfn):
+            os.remove(dfn)
+        # Issue 18940: A file may not have been created if delay is True.
+        if os.path.exists(self.baseFilename):
+            os.rename(self.baseFilename, dfn)
+        if self.backupCount > 0:
+            for s in self.getFilesToDelete():
+                os.remove(s)
+        if not self.delay:
+            self.stream = self._open()
+        newRolloverAt = self.computeRollover(currentTime)
+        while newRolloverAt <= currentTime:
+            newRolloverAt = newRolloverAt + self.interval
+        # If DST changes and midnight or weekly rollover, adjust for this.
+        if (
+            (self.when == 'MIDNIGHT' or self.when.startswith('W'))
+            and not self.utc
+        ):
+            dstAtRollover = time.localtime(newRolloverAt)[-1]
+            if dstNow != dstAtRollover:
+                if not dstNow:
+                    addend = -3600
+                else:
+                    addend = 3600
+                newRolloverAt += addend
+        self.rolloverAt = newRolloverAt

src/ralph_pricing/management/commands/pricing_sync.py

@@ -11,9 +11,9 @@
 from optparse import make_option
 
 from django.core.management.base import BaseCommand
+from django.conf import settings
 
 from ralph.util import plugin
-from ralph_pricing.app import setup_scrooge_logger
 
 
 logger = logging.getLogger(__name__)
@@ -59,7 +59,6 @@ def _run_plugin(self, name, today):
             logger.exception("{0}: {1}".format(name, e))
 
     def handle(self, today, run_only, *args, **options):
-        setup_scrooge_logger()
         from ralph_pricing.plugins import collects  # noqa
         if today:
             today = datetime.datetime.strptime(today, '%Y-%m-%d').date()
@@ -75,7 +74,8 @@ def handle(self, today, run_only, *args, **options):
                     break
                 name = plugin.highest_priority('pricing', to_run)
                 tried.add(name)
-                if self._run_plugin(name, today):
+                if (name in settings.COLLECT_PLUGINS and
+                        self._run_plugin(name, today)):
                     done.add(name)
         else:
             self._run_plugin(run_only, today)

src/ralph_pricing/plugins/collects/assets_part.py

@@ -32,7 +32,7 @@ def update_assets_parts(data, date):
 
 
 @plugin.register(chain='pricing', requires=['assets'])
-def parts(**kwargs):
+def assets_parts(**kwargs):
     """Updates the devices from Ralph Assets."""
     date = kwargs['today']
     count = sum(update_assets_parts(data, date) for data in get_asset_parts())

src/ralph_pricing/plugins/collects/extra_cost.py

@@ -33,7 +33,7 @@ def update_extra_cost(data, date):
 
 
 @plugin.register(chain='pricing', requires=['ventures'])
-def extracost(**kwargs):
+def extra_cost(**kwargs):
     """
     Main method of daily imprint create.
     """

src/ralph_pricing/plugins/collects/virtual.py

@@ -85,7 +85,7 @@ def get_or_create_usages(usage_names):
 
 # virtual usages requires assets plugin to get proper devices
 @plugin.register(chain='pricing', requires=['assets'])
-def virtual_usages(**kwargs):
+def virtual(**kwargs):
     """Updates the virtual usages from Ralph."""
 
     date = kwargs['today']

src/ralph_pricing/settings.py

@@ -0,0 +1,27 @@
+# NFSEN (network) plugin default config
+SSH_NFSEN_CREDENTIALS = {}
+NFSEN_CHANNELS = []
+NFSEN_CLASS_ADDRESS = []
+NFSEN_FILES_PATH = ''
+
+# Virtual Usages plugin default config
+VIRTUAL_VENTURE_NAMES = {}
+
+# Cloud (from Ralph) plugin default config
+CLOUD_UNKNOWN_VENTURE = None
+
+# Shares plugin default config
+SHARE_VENTURE_SYMBOLS = {}
+SHARES_UNKNOWN_VENTURE = None  # symbol
+
+# Pricing statistics default config
+WARNINGS_LIMIT_FOR_USAGES = 40
+
+# Default collect plugins to run
+COLLECT_PLUGINS = set([
+    'assets',
+    'extra_cost',
+    'ventures',
+    'virtual',
+    'warehouse',
+])

src/ralph_pricing/tests/collect_plugins/test_extra_cost.py

@@ -10,7 +10,7 @@
 from django.test import TestCase
 from ralph_pricing.plugins.collects.extra_cost import (
     update_extra_cost,
-    extracost,
+    extra_cost,
 )
 from ralph_pricing.models import ExtraCost, DailyExtraCost
 from ralph_pricing.tests.utils import (
@@ -39,5 +39,5 @@ def test_update_extra_cost_calculate_price(self):
     def test_extracost(self):
         self.assertEqual(
             (True, u'1 new extracosts', {u'today': datetime.date(2014, 5, 1)}),
-            extracost(**{'today': self.date})
+            extra_cost(**{'today': self.date})
         )