REST API initial documentation and base classes

closes #1873
https://pulp.plan.io/issues/1873

app/pulp/app/apps.py

@@ -1,9 +1,77 @@
+from importlib import import_module
+
 from django import apps
+from django.utils.module_loading import module_has_submodule
+
+VIEWSETS_MODULE_NAME = 'viewsets'
+
+
+def pulp_plugin_configs():
+    """
+    A generator of Pulp plugin AppConfigs
+
+    This makes it easy to iterate over just the installed Pulp plugins when working
+    with discovered plugin components.
+    """
+    for app_config in apps.apps.get_app_configs():
+        if isinstance(app_config, PulpPluginAppConfig):
+            yield app_config
+
+
+class PulpPluginAppConfig(apps.AppConfig):
+    """AppConfig class. Use this in plugins to identify your app as a Pulp plugin."""
+
+    # Plugin behavior loading should happen in ready(), not in __init__().
+    # ready() is called after all models are initialized, and at that point we should
+    # be able to safely inspect the plugin modules to look for any components we need
+    # to "register" with platform. The viewset registration below is based on Django's
+    # own model importing method.
 
+    def __init__(self, app_name, app_module):
+        super(PulpPluginAppConfig, self).__init__(app_name, app_module)
+
+        # Module containing viewsets eg. <module 'pulp_plugin.app.viewsets'
+        # from 'pulp_plugin/app/viewsets.pyc'>. Set by import_viewsets().
+        # None if the application doesn't have a viewsets module, automatically set
+        # when this app becomes ready.
+        self.viewsets_module = None
+
+        # Mapping of model names to viewsets (viewsets unrelated to models are excluded)
+        self.named_viewsets = None
+
+    def ready(self):
+        self.import_viewsets()
+
+    def import_viewsets(self):
+        # circular import avoidance
+        from pulp.app.viewsets import NamedModelViewSet
+
+        self.named_viewsets = {}
+        if module_has_submodule(self.module, VIEWSETS_MODULE_NAME):
+            # import the viewsets module and track any interesting viewsets
+            viewsets_module_name = '%s.%s' % (self.name, VIEWSETS_MODULE_NAME)
+            self.viewsets_module = import_module(viewsets_module_name)
+            for objname in dir(self.viewsets_module):
+                obj = getattr(self.viewsets_module, objname)
+                try:
+                    # Any subclass of NamedModelViewSet that isn't itself NamedModelViewSet
+                    # gets registered in the named_viewsets registry.
+                    if (obj is not NamedModelViewSet and
+                            issubclass(obj, NamedModelViewSet)):
+                        model = obj.queryset.model
+                        self.named_viewsets[model] = obj
+                except TypeError:
+                    # obj isn't a class, issubclass exploded but obj can be safely filtered out
+                    continue
+
+
+class PulpAppConfig(PulpPluginAppConfig):
+    # The pulp platform app is itself a pulp plugin so that it can benefit from
+    # the component discovery mechanisms provided by that superclass.
 
-class PulpAppConfig(apps.AppConfig):
     # The app's importable name
     name = 'pulp.app'
+
     # The app label to be used when creating tables, registering models,
     # referencing this app with manage.py, etc. This cannot contain a dot,
     # so for brevity's sake it's just "pulp", rather than e.g. "pulp_app".

app/pulp/app/db-reset.sh

@@ -1,15 +1,20 @@
 #!/bin/bash
 
-cd `dirname "$0"`
+pushd `dirname "$0"`
+
 if [ -d migrations ]
 then
-    echo "Platform 'migrations' dir already exists (`pwd`/migrations)"
-    echo "If resetting the DB fails, migrations for pulp apps (including platform)"
-    echo "may need to be removed for the DB reset to succeed."
-    echo ""
-    echo "Continuing in 3 seconds."
-    sleep 3
+# weird indentation here because the heredoc EOF terminator can't be indented
+cat <<EOF
+Platform 'migrations' dir already exists (`pwd`/migrations)
+If resetting the DB fails, migrations for pulp apps (including platform)
+may need to be removed for the DB reset to succeed.
+EOF
+sleep 1
+
 fi
+
 python manage.py reset_db --noinput
 python manage.py makemigrations pulp --noinput
 python manage.py migrate --noinput
+popd

app/pulp/app/models/__init__.py

@@ -1,6 +1,7 @@
 # https://docs.djangoproject.com/en/1.10/topics/db/models/#organizing-models-in-a-package
 from .base import Model, MasterModel  # NOQA
-from .generic import GenericRelationModel, GenericKeyValueStore, Config, Notes, Scratchpad  # NOQA
+from .generic import (GenericRelationModel, GenericKeyValueManager, GenericKeyValueRelation,  # NOQA
+                      GenericKeyValueModel, Config, Notes, Scratchpad)  # NOQA
 
 from .consumer import Consumer, ConsumerContent  # NOQA
 from .content import Content, Artifact  # NOQA

app/pulp/app/models/base.py

@@ -102,10 +102,7 @@ def __repr__(self):
 
 
 # Add properties to model _meta info to support master/detail models
-# These are handy for registering API ModelViewSets and for other mechanisms that want to
-# introspect a model's master/detail status. Due to the fact that multiple modules can
-# inherit from a single master, it is impossible to go the other way with this:
-# We can't return a single "detail model" for a given master model.
+# If this property is not None on a Model, then that Model is a Detail Model.
 # Doing this in a non-monkeypatch way would mean a lot of effort to achieve the same result
 # (e.g. custom model metaclass, custom Options implementation, etc). These could be classmethods
 # on Model classes, but it's easy enough to use the model's _meta namespace to do this, since
@@ -132,19 +129,3 @@ def master_model(options):
         # Also None if this model is itself the master.
         return None
 options.Options.master_model = property(master_model)
-
-
-def master_model_name(options):
-    """
-    The name of the Master model class of this Model's Master/Detail relationship.
-
-    Accessible at <model_class>._meta.master_model_name
-
-    If this model is not a detail model, None will be returned.
-    """
-    try:
-        return options.master_model._meta.model_name
-    except AttributeError:
-        # _meta was None, no master model
-        return None
-options.Options.master_model_name = property(master_model_name)

app/pulp/app/models/consumer.py

@@ -2,10 +2,9 @@
 Django models related to content consumers.
 """
 
-from django.contrib.contenttypes import fields
 from django.db import models
 
-from pulp.app.models import Model, Notes
+from pulp.app.models import Model, Notes, GenericKeyValueRelation
 
 
 class Consumer(Model):
@@ -23,15 +22,15 @@ class Consumer(Model):
     Relations:
 
     :cvar notes: Arbitrary information about the consumer.
-    :type notes: fields.GenericRelation
+    :type notes: GenericKeyValueRelation
 
     :cvar publishers: Associated publishers.
     :type publishers: models.ManyToManyField
     """
     name = models.TextField(db_index=True, unique=True)
     description = models.TextField(blank=True)
 
-    notes = fields.GenericRelation(Notes)
+    notes = GenericKeyValueRelation(Notes)
     publishers = models.ManyToManyField('Publisher', related_name='consumers')
 
     def natural_key(self):

app/pulp/app/models/content.py

@@ -3,11 +3,10 @@
 """
 import hashlib
 
-from django.contrib.contenttypes import fields
 from django.db import models
 
 
-from pulp.app.models import Model, MasterModel, Notes
+from pulp.app.models import Model, MasterModel, Notes, GenericKeyValueRelation
 from pulp.app.models.storage import StoragePath
 
 
@@ -21,13 +20,16 @@ class Content(MasterModel):
     Relations:
 
     :cvar notes: Arbitrary information stored with the content.
-    :type notes: fields.GenericRelation
+    :type notes: GenericKeyValueRelation
     """
     TYPE = 'content'
 
     natural_key_fields = ()
 
-    notes = fields.GenericRelation(Notes)
+    notes = GenericKeyValueRelation(Notes)
+
+    class Meta:
+        verbose_name_plural = 'content'
 
     def natural_key(self):
         """

app/pulp/app/models/generic.py

@@ -6,11 +6,11 @@
 """
 from collections import MutableMapping
 
-from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
 from django.contrib.contenttypes.models import ContentType
-from django.db import models
+from django.db import models, transaction
 
-from pulp.app.models import Model
+from pulp.app.models.base import Model
 
 
 class GenericRelationModel(Model):
@@ -88,6 +88,29 @@ def values(self):
     def __repr__(self):
         return '{}({})'.format(self.manager.model._meta.object_name, repr(dict(self)))
 
+    def replace(self, mapping):
+        """Efficiently replace the contents of this mapping with the provided mapping
+
+        Since this mapping is actually DB-backed, this provides a mechanism to atomically
+        update related key/value pairs with as few database calls as possible.
+
+        """
+        with transaction.atomic():
+            # no need to deepcopy since these are flat dicts by design
+            current_mapping = dict(self)
+
+            # update keys with new values only
+            for key, value in mapping.items():
+                # force value to string to force the aforementioned flatness
+                value = str(value)
+                if current_mapping.get(key) != value:
+                    self[key] = value
+
+            # remove keys that aren't in the provided mapping
+            for key in current_mapping:
+                if key not in mapping:
+                    del(self[key])
+
 
 class GenericKeyValueManager(models.Manager):
     """A normal Django Manager with a mapping attribute providing the MutableMapping interface.
@@ -102,7 +125,7 @@ def mapping(self):
         return GenericKeyValueMutableMapping(self)
 
 
-class GenericKeyValueStore(GenericRelationModel):
+class GenericKeyValueModel(GenericRelationModel):
     """Generic model providing a Key/Value store that can be related to any other model."""
     # Because we have multiple types of Key/Value stores, this is an abstract base class that
     # can be easily subclasses to create the specific types. We could potentially support multiple
@@ -112,7 +135,7 @@ class GenericKeyValueStore(GenericRelationModel):
     key = models.TextField()
     value = models.TextField()
 
-    # Use the GenericKeyValueManager by default to let anything using a GenericKeyValueStore
+    # Use the GenericKeyValueManager by default to let anything using a GenericKeyValueModel
     # have access to the mapping attr
     objects = GenericKeyValueManager()
 
@@ -121,16 +144,26 @@ class Meta:
         unique_together = ('key', 'content_type', 'object_id')
 
 
-class Config(GenericKeyValueStore):
+class Config(GenericKeyValueModel):
     """Used by pulp and users to store k/v config data on a model"""
-    pass
+    class Meta:
+        pass
 
 
-class Notes(GenericKeyValueStore):
+class Notes(GenericKeyValueModel):
     """Used by users to store arbitrary k/v data on a model"""
     pass
 
 
-class Scratchpad(GenericKeyValueStore):
+class Scratchpad(GenericKeyValueModel):
     """Used by pulp to store arbitrary k/v data on a model."""
     pass
+
+
+class GenericKeyValueRelation(GenericRelation):
+    """
+    A version of GenericReleation used when relating to GenericKeyValueModels.
+    """
+    # This mainly exists so that we can add a field to the serializer field mappings to make the
+    # pulp base serializer know how to serialize these without having to declare the fields on
+    # every serializer that might use them.

app/pulp/app/models/repository.py

@@ -2,10 +2,9 @@
 Repository related Django models.
 """
 from django.db import models
-from django.contrib.contenttypes import fields
 from django.utils import timezone
 
-from pulp.app.models import Model, Notes, Scratchpad, MasterModel
+from pulp.app.models import Model, Notes, Scratchpad, MasterModel, GenericKeyValueRelation
 
 
 class Repository(Model):
@@ -29,10 +28,10 @@ class Repository(Model):
     Relations:
 
     :cvar scratchpad: Arbitrary information stashed on the repository.
-    :type scratchpad: fields.GenericRelation
+    :type scratchpad: GenericKeyValueRelation
 
     :cvar notes: Arbitrary repository properties.
-    :type notes: fields.GenericRelation
+    :type notes: GenericKeyValueRelation
 
     :cvar content: Associated content.
     :type content: models.ManyToManyField
@@ -43,10 +42,11 @@ class Repository(Model):
     last_content_added = models.DateTimeField(blank=True, null=True)
     last_content_removed = models.DateTimeField(blank=True, null=True)
 
-    scratchpad = fields.GenericRelation(Scratchpad)
-    notes = fields.GenericRelation(Notes)
+    scratchpad = GenericKeyValueRelation(Scratchpad)
+    notes = GenericKeyValueRelation(Notes)
 
-    content = models.ManyToManyField('Content', through='RepositoryContent')
+    content = models.ManyToManyField('Content', through='RepositoryContent',
+                                     related_name='repositories')
 
     @property
     def content_summary(self):
@@ -68,6 +68,9 @@ def natural_key(self):
         """
         return (self.name,)
 
+    def __str__(self):
+        return "<{}: {}>".format(self._meta.model.__name__, self.name)
+
 
 class RepositoryGroup(Model):
     """
@@ -83,7 +86,7 @@ class RepositoryGroup(Model):
 
     Relations:
     :cvar notes: Arbitrary group properties.
-    :type notes: fields.GenericRelation
+    :type notes: GenericKeyValueRelation
 
     :cvar members: Repositories associated with the group.
     :type members: models.ManyToManyField
@@ -92,8 +95,8 @@ class RepositoryGroup(Model):
     description = models.TextField(blank=True)
 
     members = models.ManyToManyField('Repository')
-    scratchpad = fields.GenericRelation(Scratchpad)
-    notes = fields.GenericRelation(Notes)
+    scratchpad = GenericKeyValueRelation(Scratchpad)
+    notes = GenericKeyValueRelation(Notes)
 
     def natural_key(self):
         """
@@ -191,7 +194,7 @@ class Importer(ContentAdaptor):
     Relations:
 
     :cvar scratchpad: Arbitrary information stashed by the importer.
-    :type scratchpad: fields.GenericRelation
+    :type scratchpad: GenericKeyValueRelation
     """
     TYPE = 'importer'
 
@@ -223,7 +226,7 @@ class Importer(ContentAdaptor):
     download_policy = models.TextField(choices=DOWNLOAD_POLICIES)
     last_sync = models.DateTimeField(blank=True, null=True)
 
-    scratchpad = fields.GenericRelation(Scratchpad)
+    scratchpad = GenericKeyValueRelation(Scratchpad)
 
     class Meta(ContentAdaptor.Meta):
         default_related_name = 'importers'