Use stevedore to load extractors

Rather than using importutils and alike, lets rely on stevedore to
handle loading the extractors. We need to also modify how the conf
options are being declared, as otherwise we will not be able to load the
extractors.

Co-authored-by: Alvaro Lopez Garcia <aloga@ifca.unican.es>
Fixes #31

caso/extract/base.py

@@ -24,8 +24,33 @@
 
 from caso import keystone_client
 
+opts = [
+    cfg.StrOpt('site_name',
+               help='Site name as in GOCDB.'),
+    cfg.StrOpt('service_name',
+               default='$site_name',
+               help='Service name within the site'),
+    cfg.StrOpt('benchmark_name_key',
+               default='accounting:benchmark_type',
+               help='Metadata key used to retrieve the benchmark type '
+                    'from the flavor properties.'),
+    cfg.StrOpt('benchmark_value_key',
+               default='accounting:benchmark_value',
+               help='Metadata key used to retrieve the benchmark value '
+                    'from the flavor properties.'),
+    cfg.ListOpt('projects',
+                default=[],
+                deprecated_name='tenants',
+                help='List of projects to extract accounting records from.'),
+    cfg.StrOpt('mapping_file',
+               default='/etc/caso/voms.json',
+               deprecated_group="extractor",
+               help='File containing the VO <-> project mapping as used '
+               'in Keystone-VOMS.'),
+]
+
 CONF = cfg.CONF
-CONF.import_opt("mapping_file", "caso.extract.manager")
+CONF.register_opts(opts)
 
 LOG = log.getLogger(__name__)
 

caso/extract/manager.py

@@ -20,35 +20,9 @@
 from dateutil import tz
 from oslo_config import cfg
 from oslo_log import log
-from oslo_utils import importutils
 import six
 
-SUPPORTED_EXTRACTORS = {
-    'nova': 'caso.extract.nova.OpenStackExtractor',
-    'ceilometer': 'caso.extract.ceilometer.CeilometerExtractor',
-}
-
-opts = [
-    cfg.StrOpt('site_name',
-               help='Site name as in GOCDB.'),
-    cfg.StrOpt('service_name',
-               default='$site_name',
-               help='Service name within the site'),
-    cfg.StrOpt('mapping_file',
-               default='/etc/caso/voms.json',
-               deprecated_group="extractor",
-               help='File containing the VO <-> project mapping as used '
-               'in Keystone-VOMS.'),
-    cfg.StrOpt('benchmark_name_key',
-               default='accounting:benchmark_type',
-               help='Metadata key used to retrieve the benchmark type '
-                    'from the flavor properties.'),
-    cfg.StrOpt('benchmark_value_key',
-               default='accounting:benchmark_value',
-               help='Metadata key used to retrieve the benchmark value '
-                    'from the flavor properties.'),
-
-]
+from caso import loading
 
 cli_opts = [
     cfg.ListOpt('projects',
@@ -73,7 +47,7 @@
                     'it will extract records from the beginning of time. '
                     'If no time zone is specified, UTC will be used.'),
     cfg.StrOpt('extractor',
-               choices=SUPPORTED_EXTRACTORS.keys(),
+               choices=loading.get_available_extractor_names(),
                default='nova',
                help='Which extractor to use for getting the data. '
                     'If you do not specify anything, nova will be '
@@ -82,18 +56,17 @@
 
 CONF = cfg.CONF
 
-CONF.register_opts(opts)
 CONF.register_cli_opts(cli_opts)
+CONF.import_opt("projects", "caso.extract.base")
 
 LOG = log.getLogger(__name__)
 
 
 class Manager(object):
     def __init__(self):
-        extractor_class = importutils.import_class(
-            SUPPORTED_EXTRACTORS[CONF.extractor])
-        self.extractor = extractor_class()
-
+        extractor = loading.get_available_extractors()[CONF.extractor]
+        self.extractor = extractor
+        self.records = None
         self.last_run_base = os.path.join(CONF.spooldir, "lastrun")
 
     def get_lastrun(self, project):

caso/extract/nova.py

@@ -28,10 +28,10 @@
 from caso import record
 
 CONF = cfg.CONF
-CONF.import_opt("extractor", "caso.extract.manager")
-CONF.import_opt("site_name", "caso.extract.manager")
-CONF.import_opt("benchmark_name_key", "caso.extract.manager")
-CONF.import_opt("benchmark_value_key", "caso.extract.manager")
+
+CONF.import_opt("site_name", "caso.extract.base")
+CONF.import_opt("benchmark_name_key", "caso.extract.base")
+CONF.import_opt("benchmark_value_key", "caso.extract.base")
 
 LOG = log.getLogger(__name__)
 

caso/loading.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Spanish National Research Council (CSIC)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import stevedore
+
+EXTRACTOR_NAMESPACE = "caso.extractors"
+
+
+def get_available_extractor_names():
+    """Get the names of all the extractors that are available on the system.
+
+    :returns: A list of names.
+    :rtype: frozenset
+    """
+    mgr = stevedore.ExtensionManager(namespace=EXTRACTOR_NAMESPACE)
+    return frozenset(mgr.names())
+
+
+def get_available_extractors():
+    """Retrieve all the extractors available on the system.
+
+    :returns: A dict with the entrypoint name as the key and the extractor
+              as the value.
+    :rtype: dict
+    """
+    mgr = stevedore.ExtensionManager(namespace=EXTRACTOR_NAMESPACE,
+                                     propagate_map_exceptions=True)
+
+    return dict(mgr.map(lambda ext: (ext.entry_point.name, ext.plugin)))

caso/opts.py

@@ -16,6 +16,7 @@
 
 import itertools
 
+import caso.extract.base
 import caso.extract.manager
 import caso.keystone_client
 import caso.manager
@@ -27,7 +28,7 @@ def list_opts():
     return [
         ('DEFAULT', itertools.chain(caso.manager.opts,
                                     caso.manager.cli_opts,
-                                    caso.extract.manager.opts,
+                                    caso.extract.base.opts,
                                     caso.extract.manager.cli_opts)
          ),
         ('keystone_auth', caso.keystone_client.opts),

etc/caso/caso.conf.sample

@@ -30,14 +30,6 @@
 # Service name within the site (string value)
 #service_name = $site_name
 
-# List of projects to extract accounting records from. (list value)
-# Deprecated group/name - [DEFAULT]/tenants
-#projects =
-
-# File containing the VO <-> project mapping as used in Keystone-VOMS. (string
-# value)
-#mapping_file = /etc/caso/voms.json
-
 # Metadata key used to retrieve the benchmark type from the flavor properties.
 # (string value)
 #benchmark_name_key = accounting:benchmark_type
@@ -46,6 +38,14 @@
 # (string value)
 #benchmark_value_key = accounting:benchmark_value
 
+# List of projects to extract accounting records from. (list value)
+# Deprecated group/name - [DEFAULT]/tenants
+#projects =
+
+# File containing the VO <-> project mapping as used in Keystone-VOMS. (string
+# value)
+#mapping_file = /etc/caso/voms.json
+
 # Extract record changes until this date. If it is not set, we use now. If a
 # server has ended after this date, it will be included, but the consuption
 # reported will end on this date. If no time zone is specified, UTC will be
@@ -65,8 +65,8 @@
 # Which extractor to use for getting the data. If you do not specify anything,
 # nova will be used. (string value)
 # Possible values:
-# nova - <No description provided>
 # ceilometer - <No description provided>
+# nova - <No description provided>
 #extractor = nova
 
 #

setup.cfg

@@ -24,17 +24,21 @@ classifier =
 [files]
 packages =
     caso
-data_files = 
+data_files =
     etc/caso/ = etc/caso/*
 
 [entry_points]
 
 oslo.config.opts =
     caso = caso.opts:list_opts
 
-console_scripts = 
+console_scripts =
     caso-extract = caso.cmd.extract:main
 
+caso.extractors =
+    nova = caso.extract.nova:OpenStackExtractor
+    ceilometer = caso.extract.ceilometer:CeilometerExtractor
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build