Create entry point to load unit models

ref #853

common/pulp/common/error_codes.py

@@ -89,6 +89,12 @@
 PLP0036 = Error("PLP0036", _("The source_location: %(source_location) specified for the content unit is invalid."),
                 ['source_location'])
 PLP0037 = Error("PLP0037", _("The relative_path specified for the content unit is empty."), [])
+PLP0038 = Error("PLP0038", _("The unit model with id %(model_id)s and class "
+                             "%(model_class)s failed to register. Another model has already "
+                             "been registered with the same id."), ['model_id', 'model_class'])
+PLP0039 = Error("PLP0039", _("The unit model with the id %(model_id)s failed to register. The "
+                             "class %(model_class)s is not a subclass of ContentUnit."),
+                ['model_id', 'model_class'])
 
 # Create a section for general validation errors (PLP1000 - PLP2999)
 # Validation problems should be reported with a general PLP1000 error with a more specific

server/pulp/plugins/loader/api.py

@@ -40,6 +40,7 @@ def initialize(validate=True):
     if _is_initialized():
         return
 
+    # Initialize the plugin manager, this includes initialization of the unit_model entry point
     _create_manager()
 
     plugin_entry_points = (
@@ -81,7 +82,21 @@ def list_content_types():
     :rtype: list of str
     """
     assert _is_initialized()
-    return database.all_type_ids()
+    types_list = _MANAGER.unit_models.keys()
+    legacy_types = database.all_type_ids()
+    types_list.extend(legacy_types)
+    return types_list
+
+
+def list_unit_models():
+    """
+    Get the id's of the supported unit_models.
+
+    :return: list of unit model content type IDs
+    :rtype: list of str
+    """
+    assert _is_initialized()
+    return _MANAGER.unit_models.keys()
 
 
 def list_group_distributors():
@@ -282,6 +297,20 @@ def is_valid_cataloger(cataloger_id):
 
 # plugin api -------------------------------------------------------------------
 
+def get_unit_model_by_id(model_id):
+    """
+    Get the ContentUnit model class that corresponds to the given id.
+
+    :param model_id: id of the model
+    :type model_id: str
+
+    :return: the Model class or None
+    :rtype: pulp.server.db.model.ContentUnit
+    """
+    assert _is_initialized()
+    return _MANAGER.unit_models.get(model_id)
+
+
 def get_distributor_by_id(distributor_id):
     """
     Get a distributor instance that corresponds to the given id.

server/pulp/plugins/loader/manager.py

@@ -2,12 +2,20 @@
 from pprint import pformat
 import copy
 import logging
+import pkg_resources
 
+from mongoengine import signals
+
+from pulp.common import error_codes
 from pulp.plugins.loader import exceptions as loader_exceptions
+from pulp.server.db.model import ContentUnit
+from pulp.server.exceptions import PulpCodedException
 
 
 _logger = logging.getLogger(__name__)
 
+ENTRY_POINT_UNIT_MODELS = 'pulp.unit_models'
+
 
 class PluginManager(object):
     """
@@ -22,6 +30,45 @@ def __init__(self):
         self.importers = _PluginMap()
         self.profilers = _PluginMap()
         self.catalogers = _PluginMap()
+        self.unit_models = dict()
+
+        # Load the unit models
+        self._load_unit_models()
+
+    def _load_unit_models(self):
+        """"
+        Load all of the Unit Models from the ENTRY_POINT_UNIT_MODELS entry point
+
+        Attach the signals to the models here since the mongoengine signals will not be
+        sent correctly if they are attached to the base class.
+
+        :raises: PLP0038 if two models are defined with the same id
+        :raises: PLP0039 if a model is not a subclass of ContentUnit
+        """
+        _logger.debug(_("Loading Unit Models"))
+        for entry_point in pkg_resources.iter_entry_points(ENTRY_POINT_UNIT_MODELS):
+            msg = _('Loading unit model: %s' % str(entry_point))
+            _logger.info(msg)
+            model_id = entry_point.name
+            model_class = entry_point.load()
+            class_name = model_class.__class__.__module__ + "." + model_class.__class__.__name__
+            if not issubclass(model_class, ContentUnit):
+                raise PulpCodedException(error_code=error_codes.PLP0039,
+                                         model_id=model_id,
+                                         model_class=class_name)
+
+            if model_id in self.unit_models:
+                raise PulpCodedException(error_code=error_codes.PLP0038,
+                                         model_id=model_id,
+                                         model_class=class_name)
+            self.unit_models[model_id] = model_class
+
+            # Attach all the signals
+            model_class.attach_signals()
+            signals.post_init.connect(model_class.post_init_signal, sender=model_class)
+            signals.pre_save.connect(model_class.pre_save_signal, sender=model_class)
+
+        _logger.debug(_("Unit Model Loading Completed"))
 
 
 class _PluginMap(object):

server/pulp/server/db/manage.py

@@ -9,6 +9,7 @@
 import traceback
 
 from pulp.plugins.loader.api import load_content_types
+from pulp.plugins.loader.manager import PluginManager
 from pulp.server import logs
 from pulp.server.db import connection
 from pulp.server.db.migrate import models
@@ -120,6 +121,11 @@ def ensure_database_indexes():
     model.TaskStatus.ensure_indexes()
     model.Worker.ensure_indexes()
 
+    # Load all the model classes that the server knows about and ensure their inexes as well
+    plugin_manager = PluginManager()
+    for model_class in plugin_manager.unit_models.itervalues():
+        model_class.ensure_indexes()
+
 
 def main():
     """

server/pulp/server/db/model/__init__.py

@@ -354,6 +354,17 @@ def __init__(self, *args, **kwargs):
         self._source_location = None
         self._relative_path = None
 
+    @classmethod
+    def attach_signals(cls):
+        """
+        Attach the signals to this class.
+
+        This is provided as a class method so it can be called on subclasses
+        and all the correct signals will be applied.
+        """
+        signals.post_init.connect(cls.post_init_signal, sender=cls)
+        signals.pre_save.connect(cls.pre_save_signal, sender=cls)
+
     @classmethod
     def post_init_signal(cls, sender, document):
         """

server/test/unit/plugins/loader/test_api.py

@@ -252,3 +252,56 @@ def test_catalogers(self):
     def test_finalize(self):
         api.finalize()
         self.assertEqual(api._MANAGER, None)
+
+
+class TestAPIModels(unittest.TestCase):
+
+    @mock.patch('pulp.plugins.loader.api._is_initialized')
+    @mock.patch('pulp.plugins.loader.api._MANAGER')
+    def test_get_unit_model_by_id(self, mock_manager, mock_is_initialized):
+        mock_is_initialized.return_value = True
+        mock_manager.unit_models.get.return_value = 'apples'
+        return_val = api.get_unit_model_by_id('foo')
+        self.assertEquals(return_val, 'apples')
+        mock_manager.unit_models.get.assert_called_once_with('foo')
+
+    @mock.patch('pulp.plugins.loader.api._is_initialized')
+    @mock.patch('pulp.plugins.loader.api._MANAGER')
+    def test_list_unit_models(self, mock_manager, mock_is_initialized):
+        mock_is_initialized.return_value = True
+        mock_manager.unit_models = {'foo': 'apples', 'bar': 'pears'}
+        return_val = api.list_unit_models()
+        self.assertEquals(return_val, ['foo', 'bar'])
+
+
+class TestAPIContentTypes(unittest.TestCase):
+
+    @mock.patch('pulp.plugins.loader.api.database')
+    @mock.patch('pulp.plugins.loader.api._is_initialized')
+    @mock.patch('pulp.plugins.loader.api._MANAGER')
+    def test_list_content_types(self, mock_manager, mock_is_initialized, mock_db):
+        mock_is_initialized.return_value = True
+        mock_manager.unit_models = {'foo': 'apples', 'bar': 'pears'}
+        mock_db.all_type_ids.return_value = ['baz', 'qux']
+        return_val = api.list_content_types()
+        self.assertEquals(return_val, ['foo', 'bar', 'baz', 'qux'])
+
+    @mock.patch('pulp.plugins.loader.api.database')
+    @mock.patch('pulp.plugins.loader.api._is_initialized')
+    @mock.patch('pulp.plugins.loader.api._MANAGER')
+    def test_list_content_types_no_legacy(self, mock_manager, mock_is_initialized, mock_db):
+        mock_is_initialized.return_value = True
+        mock_manager.unit_models = {'foo': 'apples', 'bar': 'pears'}
+        mock_db.all_type_ids.return_value = []
+        return_val = api.list_content_types()
+        self.assertEquals(return_val, ['foo', 'bar'])
+
+    @mock.patch('pulp.plugins.loader.api.database')
+    @mock.patch('pulp.plugins.loader.api._is_initialized')
+    @mock.patch('pulp.plugins.loader.api._MANAGER')
+    def test_list_content_types_no_models(self, mock_manager, mock_is_initialized, mock_db):
+        mock_is_initialized.return_value = True
+        mock_manager.unit_models = {}
+        mock_db.all_type_ids.return_value = ['baz', 'quux']
+        return_val = api.list_content_types()
+        self.assertEquals(return_val, ['baz', 'quux'])

server/test/unit/plugins/loader/test_manager.py

@@ -0,0 +1,83 @@
+import pkg_resources
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+import mock
+
+from pulp.common import error_codes
+from pulp.plugins.loader import manager
+from pulp.server import exceptions
+from pulp.server.db.model import ContentUnit
+
+
+class ContentUnitHelper(ContentUnit):
+    pass
+
+
+class BadContentUnit(object):
+    pass
+
+
+class TestPluginManager(unittest.TestCase):
+
+    @mock.patch.object(ContentUnitHelper, 'attach_signals')
+    @mock.patch('pulp.plugins.loader.manager.pkg_resources.iter_entry_points')
+    def test_load_unit_models(self, mock_entry_points_iter, mock_signals):
+        """
+        Test loading of the unit models entry points
+        """
+        req = pkg_resources.Requirement.parse('pulp-devel')
+        dist = pkg_resources.working_set.find(req)
+        entry_string = 'foo=unit.plugins.loader.test_manager:ContentUnitHelper'
+        entry = pkg_resources.EntryPoint.parse(entry_string, dist=dist)
+        mock_entry_points_iter.return_value = [entry]
+
+        plugin_manager = manager.PluginManager()
+
+        self.assertTrue('foo' in plugin_manager.unit_models)
+        test_model = ContentUnitHelper
+        found_model = plugin_manager.unit_models.get('foo')
+
+        self.assertTrue(test_model == found_model)
+        mock_signals.assert_called_once_with()
+
+    @mock.patch('pulp.plugins.loader.manager.pkg_resources.iter_entry_points')
+    def test_load_unit_models_id_reused(self, mock_entry_points_iter):
+        """
+        Test loading of the unit models when the same model id is used twice raises
+        PLP0038
+        """
+        req = pkg_resources.Requirement.parse('pulp-devel')
+        dist = pkg_resources.working_set.find(req)
+        entry_string = 'foo=unit.plugins.loader.test_manager:ContentUnitHelper'
+        entry1 = pkg_resources.EntryPoint.parse(entry_string, dist=dist)
+        entry_string = 'foo=unit.plugins.loader.test_manager:ContentUnitHelper'
+        entry2 = pkg_resources.EntryPoint.parse(entry_string, dist=dist)
+
+        mock_entry_points_iter.return_value = [entry1, entry2]
+
+        try:
+            manager.PluginManager()
+            self.fail("This should have raised PLP0038")
+        except exceptions.PulpCodedException, e:
+            self.assertEquals(e.error_code, error_codes.PLP0038)
+
+    @mock.patch('pulp.plugins.loader.manager.pkg_resources.iter_entry_points')
+    def test_load_unit_models_non_content_unit(self, mock_entry_points_iter):
+        """
+        Test loading of the unit models that don't subclass ContentUnit
+        raise PLP0039
+        """
+        req = pkg_resources.Requirement.parse('pulp-devel')
+        dist = pkg_resources.working_set.find(req)
+        entry_string = 'foo=unit.plugins.loader.test_manager:BadContentUnit'
+        entry1 = pkg_resources.EntryPoint.parse(entry_string, dist=dist)
+        mock_entry_points_iter.return_value = [entry1]
+
+        try:
+            manager.PluginManager()
+            self.fail("This should have raised PLP0039")
+        except exceptions.PulpCodedException, e:
+            self.assertEquals(e.error_code, error_codes.PLP0039)

server/test/unit/server/db/test_manage.py

@@ -105,18 +105,22 @@ def clean(self):
         super(self.__class__, self).clean()
         types_db.clean()
 
+    @patch.object(manage, 'PluginManager')
     @patch.object(manage, 'model')
-    def test_ensure_index(self, mock_model):
+    def test_ensure_index(self, mock_model, mock_plugin_manager):
         """
         Make sure that the ensure_indexes method is called for all
         the appropriate platform models
         """
+        test_model = MagicMock()
+        mock_plugin_manager.return_value.unit_models.itervalues.return_value = [test_model]
         manage.ensure_database_indexes()
         self.assertTrue(mock_model.Repository.ensure_indexes.called)
         self.assertTrue(mock_model.RepositoryContentUnit.ensure_indexes.called)
         self.assertTrue(mock_model.ReservedResource.ensure_indexes.called)
         self.assertTrue(mock_model.TaskStatus.ensure_indexes.called)
         self.assertTrue(mock_model.Worker.ensure_indexes.called)
+        test_model.ensure_indexes.assert_called_once_with()
 
     @patch.object(manage, 'ensure_database_indexes')
     @patch('logging.config.fileConfig')

server/test/unit/server/db/test_model.py

@@ -55,6 +55,18 @@ def test_model_fields(self):
     def test_meta_abstract(self):
         self.assertEquals(model.ContentUnit._meta['abstract'], True)
 
+    @patch('pulp.server.db.model.signals')
+    def test_attach_signals(self, mock_signals):
+        class ContentUnitHelper(model.ContentUnit):
+            pass
+
+        ContentUnitHelper.attach_signals()
+
+        mock_signals.post_init.connect.assert_called_once_with(ContentUnitHelper.post_init_signal,
+                                                               sender=ContentUnitHelper)
+        mock_signals.pre_save.connect.assert_called_once_with(ContentUnitHelper.pre_save_signal,
+                                                              sender=ContentUnitHelper)
+
     def test_post_init_signal_with_unit_key_fields_defined(self):
         """
         Test the init signal handler that validates the existence of the