separate base manager from pipeline

common agent for all

Change-Id: I19a83d3d0e5c91ab5cb6e792ab7389e36f8ede55

ceilometer/agent.py

@@ -0,0 +1,151 @@
+#
+# Copyright 2013 Intel Corp.
+# Copyright 2014 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import hashlib
+import os
+import pkg_resources
+
+from oslo_log import log
+from oslo_utils import fnmatch
+import six
+import yaml
+
+LOG = log.getLogger(__name__)
+
+
+class ConfigException(Exception):
+    def __init__(self, cfg_type, message, cfg):
+        self.cfg_type = cfg_type
+        self.msg = message
+        self.cfg = cfg
+
+    def __str__(self):
+        return '%s %s: %s' % (self.cfg_type, self.cfg, self.msg)
+
+
+class SourceException(Exception):
+    def __init__(self, message, cfg):
+        self.msg = message
+        self.cfg = cfg
+
+    def __str__(self):
+        return 'Source definition invalid: %s (%s)' % (self.msg, self.cfg)
+
+
+class ConfigManagerBase(object):
+    """Base class for managing configuration file refresh"""
+
+    def __init__(self, conf):
+        self.conf = conf
+        self.cfg_loc = None
+
+    def load_config(self, cfg_file, fallback_cfg_prefix='pipeline/data/'):
+        """Load a configuration file and set its refresh values."""
+        if os.path.exists(cfg_file):
+            self.cfg_loc = cfg_file
+        else:
+            self.cfg_loc = self.conf.find_file(cfg_file)
+        if not self.cfg_loc and fallback_cfg_prefix is not None:
+            LOG.debug("No pipeline definitions configuration file found! "
+                      "Using default config.")
+            self.cfg_loc = pkg_resources.resource_filename(
+                __name__, fallback_cfg_prefix + cfg_file)
+        with open(self.cfg_loc) as fap:
+            data = fap.read()
+        conf = yaml.safe_load(data)
+        self.cfg_mtime = self.get_cfg_mtime()
+        self.cfg_hash = self.get_cfg_hash()
+        LOG.info("Config file: %s", conf)
+        return conf
+
+    def get_cfg_mtime(self):
+        """Return modification time of cfg file"""
+        return os.path.getmtime(self.cfg_loc) if self.cfg_loc else None
+
+    def get_cfg_hash(self):
+        """Return hash of configuration file"""
+        if not self.cfg_loc:
+            return None
+
+        with open(self.cfg_loc) as fap:
+            data = fap.read()
+        if six.PY3:
+            data = data.encode('utf-8')
+
+        file_hash = hashlib.md5(data).hexdigest()
+        return file_hash
+
+    def cfg_changed(self):
+        """Returns hash of changed cfg else False."""
+        mtime = self.get_cfg_mtime()
+        if mtime > self.cfg_mtime:
+            LOG.info('Configuration file has been updated.')
+            self.cfg_mtime = mtime
+            _hash = self.get_cfg_hash()
+            if _hash != self.cfg_hash:
+                LOG.info("Detected change in configuration.")
+                return _hash
+        return False
+
+
+class Source(object):
+    """Represents a generic source"""
+
+    def __init__(self, cfg):
+        self.cfg = cfg
+        try:
+            self.name = cfg['name']
+        except KeyError as err:
+            raise SourceException(
+                "Required field %s not specified" % err.args[0], cfg)
+
+    def __str__(self):
+        return self.name
+
+    def check_source_filtering(self, data, d_type):
+        """Source data rules checking
+
+        - At least one meaningful datapoint exist
+        - Included type and excluded type can't co-exist on the same pipeline
+        - Included type meter and wildcard can't co-exist at same pipeline
+        """
+        if not data:
+            raise SourceException('No %s specified' % d_type, self.cfg)
+
+        if ([x for x in data if x[0] not in '!*'] and
+           [x for x in data if x[0] == '!']):
+            raise SourceException(
+                'Both included and excluded %s specified' % d_type,
+                self.cfg)
+
+        if '*' in data and [x for x in data if x[0] not in '!*']:
+            raise SourceException(
+                'Included %s specified with wildcard' % d_type,
+                self.cfg)
+
+    @staticmethod
+    def is_supported(dataset, data_name):
+        # Support wildcard like storage.* and !disk.*
+        # Start with negation, we consider that the order is deny, allow
+        if any(fnmatch.fnmatch(data_name, datapoint[1:])
+               for datapoint in dataset if datapoint[0] == '!'):
+            return False
+
+        if any(fnmatch.fnmatch(data_name, datapoint)
+               for datapoint in dataset if datapoint[0] != '!'):
+            return True
+
+        # if we only have negation, we suppose the default is allow
+        return all(datapoint.startswith('!') for datapoint in dataset)

ceilometer/cmd/polling.py

@@ -19,7 +19,7 @@
 from oslo_config import cfg
 from oslo_log import log
 
-from ceilometer.agent import manager
+from ceilometer.polling import manager
 from ceilometer import service
 
 LOG = log.getLogger(__name__)

ceilometer/compute/discovery.py

@@ -30,9 +30,9 @@
 except ImportError:
     libvirt = None
 
-from ceilometer.agent import plugin_base
 from ceilometer.compute.virt.libvirt import utils as libvirt_utils
 from ceilometer import nova_client
+from ceilometer.polling import plugin_base
 
 OPTS = [
     cfg.StrOpt('instance_discovery_method',

ceilometer/compute/pollsters/__init__.py

@@ -19,9 +19,9 @@
 from oslo_utils import timeutils
 
 import ceilometer
-from ceilometer.agent import plugin_base
 from ceilometer.compute.pollsters import util
 from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.polling import plugin_base
 from ceilometer import sample
 
 LOG = log.getLogger(__name__)

ceilometer/hardware/discovery.py

@@ -16,9 +16,8 @@
 from oslo_log import log
 from oslo_utils import timeutils
 
-from ceilometer.agent import plugin_base
 from ceilometer import nova_client
-
+from ceilometer.polling import plugin_base
 
 LOG = log.getLogger(__name__)
 

ceilometer/hardware/pollsters/generic.py

@@ -21,11 +21,11 @@
 from oslo_utils import netutils
 import six
 
-from ceilometer.agent import plugin_base
 from ceilometer import declarative
 from ceilometer.hardware import inspector as insloader
 from ceilometer.hardware.pollsters import util
 from ceilometer.i18n import _
+from ceilometer.polling import plugin_base
 from ceilometer import sample
 
 OPTS = [

ceilometer/image/discovery.py

@@ -14,8 +14,8 @@
 import glanceclient
 from oslo_config import cfg
 
-from ceilometer.agent import plugin_base
 from ceilometer import keystone_client
+from ceilometer.polling import plugin_base
 
 SERVICE_OPTS = [
     cfg.StrOpt('glance',

ceilometer/image/glance.py

@@ -17,7 +17,7 @@
 
 from __future__ import absolute_import
 
-from ceilometer.agent import plugin_base
+from ceilometer.polling import plugin_base
 from ceilometer import sample
 
 

ceilometer/ipmi/pollsters/node.py

@@ -17,10 +17,10 @@
 from oslo_log import log
 import six
 
-from ceilometer.agent import plugin_base
 from ceilometer.i18n import _
 from ceilometer.ipmi.platform import exception as nmexcept
 from ceilometer.ipmi.platform import intel_node_manager as node_manager
+from ceilometer.polling import plugin_base
 from ceilometer import sample
 
 LOG = log.getLogger(__name__)

ceilometer/ipmi/pollsters/sensor.py

@@ -14,11 +14,11 @@
 
 from oslo_log import log
 
-from ceilometer.agent import plugin_base
 from ceilometer.i18n import _
 from ceilometer.ipmi.notifications import ironic as parser
 from ceilometer.ipmi.platform import exception as ipmiexcept
 from ceilometer.ipmi.platform import ipmi_sensor
+from ceilometer.polling import plugin_base
 from ceilometer import sample
 
 LOG = log.getLogger(__name__)

ceilometer/network/services/base.py

@@ -13,7 +13,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from ceilometer.agent import plugin_base
+from ceilometer.polling import plugin_base
 
 
 # status map for converting metric status to volume int

ceilometer/network/services/discovery.py

@@ -13,8 +13,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from ceilometer.agent import plugin_base
 from ceilometer import neutron_client
+from ceilometer.polling import plugin_base
 
 
 class _BaseServicesDiscovery(plugin_base.DiscoveryBase):

ceilometer/network/statistics/__init__.py

@@ -20,7 +20,7 @@
 from six.moves.urllib import parse as urlparse
 from stevedore import driver as _driver
 
-from ceilometer.agent import plugin_base
+from ceilometer.polling import plugin_base
 from ceilometer import sample
 
 

ceilometer/objectstore/rgw.py

@@ -20,8 +20,8 @@
 from oslo_log import log
 import six.moves.urllib.parse as urlparse
 
-from ceilometer.agent import plugin_base
 from ceilometer import keystone_client
+from ceilometer.polling import plugin_base
 from ceilometer import sample
 
 LOG = log.getLogger(__name__)

ceilometer/objectstore/swift.py

@@ -24,8 +24,8 @@
 from swiftclient import client as swift
 from swiftclient.exceptions import ClientException
 
-from ceilometer.agent import plugin_base
 from ceilometer import keystone_client
+from ceilometer.polling import plugin_base
 from ceilometer import sample
 
 

ceilometer/opts.py

@@ -17,7 +17,6 @@
 from keystoneauth1 import loading
 from oslo_config import cfg
 
-import ceilometer.agent.manager
 import ceilometer.compute.discovery
 import ceilometer.compute.virt.inspector
 import ceilometer.compute.virt.libvirt.utils
@@ -38,6 +37,7 @@
 import ceilometer.objectstore.rgw
 import ceilometer.objectstore.swift
 import ceilometer.pipeline
+import ceilometer.polling.manager
 import ceilometer.publisher.messaging
 import ceilometer.publisher.utils
 import ceilometer.sample
@@ -70,7 +70,7 @@ def list_opts():
     # This have been removed due to a recursive import issue
     return [
         ('DEFAULT',
-         itertools.chain(ceilometer.agent.manager.OPTS,
+         itertools.chain(ceilometer.polling.manager.OPTS,
                          ceilometer.compute.virt.inspector.OPTS,
                          ceilometer.compute.virt.libvirt.utils.OPTS,
                          ceilometer.objectstore.swift.OPTS,
@@ -126,7 +126,7 @@ def list_opts():
         ('notification',
          itertools.chain(ceilometer.notification.OPTS,
                          ceilometer.notification.EXCHANGES_OPTS)),
-        ('polling', ceilometer.agent.manager.POLLING_OPTS),
+        ('polling', ceilometer.polling.manager.POLLING_OPTS),
         ('publisher', ceilometer.publisher.utils.OPTS),
         ('publisher_notifier', ceilometer.publisher.messaging.NOTIFIER_OPTS),
         ('rgw_admin_credentials', ceilometer.objectstore.rgw.CREDENTIAL_OPTS),