Add handling of logs for fuel statistics collectors and sender

Separate logger added for statistic module. Logs for collecting process
of each oswl resource now are written to separate files. This is
possible because collecting for particular resource is started in
separate process by systemd and so logs separation is achieved by
just using filehandler with specific name.

Change-Id: I35a1d03441639821233d068fbde57a251d39a721
Closes-Bug: #1523476

nailgun/nailgun/settings.yaml

@@ -105,6 +105,8 @@ REMOVE_IMAGES_TIMEOUT: 60
 # statistics send interval
 STATS_SEND_INTERVAL: 3600
 
+STATS_LOGS_PATH: "/var/log/nailgun/"
+
 # statistics send enabled check interval
 STATS_ENABLE_CHECK_INTERVAL: 300
 

nailgun/nailgun/statistics/oswl/collector.py

@@ -14,13 +14,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import six
 import sys
 import time
 
 from nailgun import consts
 from nailgun.db import db
-from nailgun.logger import logger
 from nailgun.objects import ClusterCollection
 from nailgun.objects import MasterNodeSettings
 from nailgun.objects import OpenStackWorkloadStatsCollection
@@ -31,6 +31,9 @@
 from nailgun.statistics import utils
 
 
+logger = logging.getLogger('statistics')
+
+
 def collect(resource_type):
     try:
         operational_clusters = ClusterCollection.filter_by(
@@ -94,6 +97,11 @@ def collect(resource_type):
 
 def run():
     resource_type = sys.argv[1]
+
+    # add file handler to log collecting for particular resource
+    log_file = "oswl_{0}_collectord.log".format(resource_type)
+    utils.prepare_logger(logger, log_file)
+
     poll_interval = settings.OSWL_COLLECTORS_POLLING_INTERVAL[resource_type]
     logger.info("Starting OSWL collector for {0} resource"
                 .format(resource_type))

nailgun/nailgun/statistics/oswl/helpers.py

@@ -11,6 +11,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import six
 
 from cinderclient import client as cinder_client
@@ -21,14 +22,16 @@
 
 from nailgun import consts
 from nailgun.db import db
-from nailgun.logger import logger
 from nailgun import objects
 from nailgun.settings import settings
 from nailgun.statistics.oswl.resources_description \
     import resources_description
 from nailgun.statistics import utils
 
 
+logger = logging.getLogger('statistics')
+
+
 class ClientProvider(object):
     """Initialize clients for OpenStack component and expose them as attrs"""
 

nailgun/nailgun/statistics/statsenderd.py

@@ -14,6 +14,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import requests
 import six
 import time
@@ -24,12 +25,15 @@
 from nailgun import consts
 from nailgun.db import db
 from nailgun.db.sqlalchemy import models
-from nailgun.logger import logger
 from nailgun import objects
 from nailgun.settings import settings
 from nailgun.statistics.fuel_statistics.installation_info \
     import InstallationInfo
 from nailgun.statistics.utils import dithered
+from nailgun.statistics.utils import prepare_logger
+
+
+logger = logging.getLogger('statistics')
 
 
 class StatsSender(object):
@@ -250,6 +254,9 @@ def send_stats_once(self):
 
 
 def run():
+    # add file handler to logger
+    prepare_logger(logger, file_name='statsenderd.log')
+
     logger.info("Starting standalone stats sender...")
     try:
         while True:

nailgun/nailgun/statistics/utils.py

@@ -12,18 +12,22 @@
 #    under the License.
 
 from collections import namedtuple
+import logging
 import os
 import random
 
 from contextlib import contextmanager
 
 from nailgun import consts
-from nailgun.logger import logger
+from nailgun.logger import set_logger
 from nailgun.network import manager
 from nailgun.settings import settings
 from nailgun.statistics import errors
 
 
+logger = logging.getLogger('statistics')
+
+
 WhiteListRule = namedtuple(
     'WhiteListItem', ['path', 'map_to_name', 'transform_func'])
 
@@ -159,3 +163,12 @@ def get_version_info(cluster):
     except Exception:
         logger.exception("Fetching version info for cluster '%s' failed",
                          cluster)
+
+
+def prepare_logger(logger, file_name):
+    handler = logging.FileHandler(
+        os.path.join(settings.STATS_LOGS_PATH, file_name)
+    )
+    set_logger(logger, handler)
+
+    return logger

nailgun/tools/env.sh

@@ -63,6 +63,7 @@ API_LOG: "${NAILGUN_LOGS}/api.log"
 APP_LOG: "${NAILGUN_LOGS}/app.log"
 RPC_CONSUMER_LOG_PATH: "${NAILGUN_LOGS}/receiverd.log"
 ASSASSIN_LOG_PATH: "${NAILGUN_LOGS}/assassind.log"
+STATS_LOGS_PATH: ${NAILGUN_LOGS}
 EOL
 }