Added the SharedAttributeManager feature.

fixes #2824.
[nocoverage]

CHANGES/2824.feature

@@ -0,0 +1,5 @@
+Added a new concept, `SharedAttributeManager`, to Pulp.
+
+This feature allows attributes whose values are intended to
+be the same, across multiple instances, to be managed from
+one point. The feature can be accessed at `/pulp/api/v3/sams/`.

docs/workflows/index.rst

@@ -23,4 +23,5 @@ assumes that the reader is familiar with the fundamentals discussed in the :doc:
    import-export
    labels
    plugin-removal
+   shared-attribute-management
    troubleshooting

docs/workflows/shared-attribute-management.rst

@@ -0,0 +1,273 @@
+.. _shared-attribute-management:
+
+Managing Shared Attributes for Pulp Entities
+--------------------------------------------
+
+.. note::
+
+    Pulp shared-attribute-management features are in tech-preview and may change in backwards incompatible
+    ways in future releases.
+
+
+Pulp provides the ability keep the attributes of entities "in sync" from a central manager.
+For example, one may have many Remote objects that should all share the same concurrency,
+socket-timeout, and rate-limit options. This feature lets you define those options in a
+``SharedAttributeManager`` (SAM), add a list of entities that should have those attributes be
+in sync, and arrange for all managed entities to be updated to the managed values.
+
+A SAM consists of the following attributes:
+
+    * ``name``, a unique identifier
+    * ``managed_attributes``, a JSON structure defining the attributes being managed and the
+      desired values
+    * ``managed_entities``, a list of pulp-HREFs of the entities whose attributes should be
+      set to the values stored in ``managed_attributes``
+
+The ``managed_entities`` of a SAM can be a heterogeneous set of entities (e.g., several
+different kinds of Detail Remotes), and its ``managed_attributes`` can be an arbitrary set.
+The SAM will only attempt to apply attributes from ``managed_attributes``, to specific
+entity, if that entity defines the attribute, skipping any which do not apply.
+
+If there are ``managed_attributes``, and an entity to-be-managed is added (or the ``managed_entities`` list is reset),
+the ``managed_attributes`` will be applied to the new entity or list.
+
+The entities being managed are not tied to a managing SAM in any direct way; being added to
+or removed from SAM-management does not impact the behavior of the specified entities.
+
+Full documentation for the SAM REST API can be found at
+`docs.pulpproject.org <https://docs.pulpproject.org/pulpcore/restapi.html/#tag/Shared-Attribute-Managers>`_. Examples
+of the SAM workflow follow.
+
+To create a SAM:
+
+.. code-block::
+
+    $ http POST :/pulp/api/v3/shared-attribute-managers/ name="MyManager"
+    {
+        "managed_attributes": null,
+        "managed_entities": null,
+        "name": "MyManager",
+        "pulp_created": "2022-08-31T13:33:23.891003Z",
+        "pulp_href": "/pulp/api/v3/shared-attribute-managers/ac6d3093-ac1e-4b3a-b30f-117e31abacad/"
+    }
+
+To show all existing SAMs:
+
+.. code-block::
+
+    $ http GET :/pulp/api/v3/shared-attribute-managers/
+    {
+        "count": 2,
+        "next": null,
+        "previous": null,
+        "results": [
+            {
+                "managed_attributes": null,
+                "managed_entities": null,
+                "name": "SecondManager",
+                "pulp_created": "2022-08-31T14:02:36.110474Z",
+                "pulp_href": "/pulp/api/v3/shared-attribute-managers/1d84bb18-e3c6-4399-ba37-8d765fc9a8af/"
+            },
+            {
+                "managed_attributes": null,
+                "managed_entities": null,
+                "name": "MyManager",
+                "pulp_created": "2022-08-31T13:33:23.891003Z",
+                "pulp_href": "/pulp/api/v3/shared-attribute-managers/ac6d3093-ac1e-4b3a-b30f-117e31abacad/"
+            }
+        ]
+    }
+
+To show a specific SAM:
+
+.. code-block::
+
+    $ http GET :/pulp/api/v3/shared-attribute-managers/ac6d3093-ac1e-4b3a-b30f-117e31abacad/
+    {
+        "managed_attributes": null,
+        "managed_entities": null,
+        "name": "MyManager",
+        "pulp_created": "2022-08-31T13:33:23.891003Z",
+        "pulp_href": "/pulp/api/v3/shared-attribute-managers/ac6d3093-ac1e-4b3a-b30f-117e31abacad/"
+    }
+
+To define the set of attributes to be shared, update a specific SAM with a JSON string defining
+the attributes to be managed. This endpoint replaces the existing set of managed-attributes, and applies the new set
+to any managed-entities:
+
+.. code-block::
+
+    # SAM_HREF=/pulp/api/v3/shared-attribute-managers/949a0807-d79f-4821-b8de-8aadd32ac9ae/
+    $ PATCH_TASK=$(http --json PATCH :${SAM_HREF} \
+        managed_attributes:='{"bar": "blech", "description": "This is a description", "retain_package_versions": 5, "sqlite_metadata": true, "url": "http://THIS-ONE-WORKS"}' | jq -r .task)
+    $ http :${PATCH_TASK}
+    {
+        "child_tasks": [],
+        "created_resources": [],
+        "error": null,
+        "finished_at": "2022-08-31T14:13:53.415482Z",
+        "logging_cid": "1eaa76991faa4b32bb41cb96d6a6a84a",
+        "name": "pulpcore.app.tasks.base.general_update",
+        "parent_task": null,
+        "progress_reports": [],
+        "pulp_created": "2022-08-31T14:13:53.335209Z",
+        "pulp_href": "/pulp/api/v3/tasks/8fbc9fb6-ebad-49ab-a5fb-4cac5fa6c51c/",
+        "reserved_resources_record": [
+            "/pulp/api/v3/shared-attribute-managers/949a0807-d79f-4821-b8de-8aadd32ac9ae/"
+        ],
+        "started_at": "2022-08-31T14:13:53.379429Z",
+        "state": "completed",
+        "task_group": null,
+        "worker": "/pulp/api/v3/workers/11161a5b-653a-4496-886b-50f4700c1cde/"
+    }
+    $ http :${SAM_HREF}
+    {
+        "managed_attributes": {
+            "bar": "blech",
+            "description": "This is a description",
+            "retain_package_versions": 5,
+            "sqlite_metadata": true,
+            "url": "http://THIS-ONE-WORKS"
+        },
+        "managed_entities": null,
+        "name": "foo",
+        "pulp_created": "2022-08-29T20:35:06.090490Z",
+        "pulp_href": "/pulp/api/v3/shared-attribute-managers/949a0807-d79f-4821-b8de-8aadd32ac9ae/"
+    }
+
+
+To add a set of entities to be managed, send an update to a specific SAM along with a list of the entity-hrefs. This
+endpoint replaces the existing set of managed-entities, and applies the current set of managed-attributes to the
+new entities-being-managed:
+
+.. code-block::
+
+    $ PATCH_TASK=$(http --json PATCH :/pulp/api/v3/shared-attribute-managers/949a0807-d79f-4821-b8de-8aadd32ac9ae/ \
+      managed_entities:='["/pulp/api/v3/repositories/rpm/rpm/47a78cce-d947-45fe-a618-c139f288dd7f/", \
+      "/pulp/api/v3/remotes/rpm/rpm/4507f854-e334-4f99-9c75-ee7621d64352/"]'  | jq -r '.task')
+    $ http :${SAM_HREF}
+    {
+        "managed_attributes": {
+            "bar": "blech",
+            "description": "This is a description",
+            "retain_package_versions": 5,
+            "sqlite_metadata": true,
+            "url": "http://THIS-ONE-WORKS"
+        },
+        "managed_entities": [
+            "/pulp/api/v3/repositories/rpm/rpm/47a78cce-d947-45fe-a618-c139f288dd7f/",
+            "/pulp/api/v3/remotes/rpm/rpm/4507f854-e334-4f99-9c75-ee7621d64352/"
+        ],
+        "name": "foo",
+        "pulp_created": "2022-08-29T20:35:06.090490Z",
+        "pulp_href": "/pulp/api/v3/shared-attribute-managers/949a0807-d79f-4821-b8de-8aadd32ac9ae/"
+    }
+
+To add a new entity to be managed:
+
+.. code-block::
+
+    $ http -b POST :${SAM_HREF}add/ entity_href="/pulp/api/v3/repositories/file/file/daea11e9-b673-49f2-984a-b0de9278de01/"
+    {
+        "task": "/pulp/api/v3/tasks/5f8632b3-2919-402f-8111-f71f8f92e0f1/"
+    }
+    $ http -b :${SAM_HREF} | jq -r .managed_entities
+    [
+      "/pulp/api/v3/repositories/rpm/rpm/47a78cce-d947-45fe-a618-c139f288dd7f/",
+      "/pulp/api/v3/remotes/rpm/rpm/4507f854-e334-4f99-9c75-ee7621d64352/"
+      "/pulp/api/v3/repositories/file/file/daea11e9-b673-49f2-984a-b0de9278de01/"
+    ]
+
+
+To (re)apply the specified attributes to the current managed-entities list:
+
+.. code-block::
+
+    $ http -b POST :${SAM_HREF}apply/
+    {
+    "task": "/pulp/api/v3/tasks/7000fcae-99ae-4c76-bcdf-15ed7b277789/"
+    }
+    $ http :/pulp/api/v3/tasks/7000fcae-99ae-4c76-bcdf-15ed7b277789/
+    {
+        "child_tasks": [],
+        "created_resources": [],
+        "error": null,
+        "finished_at": "2022-08-31T18:58:06.423795Z",
+        "logging_cid": "b2a0f86bcafe4c92a321fa441ffc1baa",
+        "name": "pulpcore.app.tasks.sam.update_managed_entities",
+        "parent_task": null,
+        "progress_reports": [
+            {
+                "code": "sam.apply",
+                "done": 3,
+                "message": "Updating Managed Entities",
+                "state": "completed",
+                "suffix": null,
+                "total": 3
+            },
+            {
+                "code": "sam.apply_success",
+                "done": 3,
+                "message": "Successful Updates",
+                "state": "completed",
+                "suffix": null,
+                "total": 3
+            },
+            {
+                "code": "sam.apply_failures",
+                "done": 0,
+                "message": "Failed Updates",
+                "state": "completed",
+                "suffix": null,
+                "total": 3
+            }
+        ],
+        "pulp_created": "2022-08-31T18:58:06.187955Z",
+        "pulp_href": "/pulp/api/v3/tasks/7000fcae-99ae-4c76-bcdf-15ed7b277789/",
+        "reserved_resources_record": [
+            "/pulp/api/v3/repositories/rpm/rpm/47a78cce-d947-45fe-a618-c139f288dd7f/",
+            "/pulp/api/v3/remotes/rpm/rpm/4507f854-e334-4f99-9c75-ee7621d64352/"
+            "/pulp/api/v3/repositories/file/file/daea11e9-b673-49f2-984a-b0de9278de01/"
+            "/pulp/api/v3/shared-attribute-managers/0754a264-06e4-4de9-a14f-b0a88e6a7ac1/"
+        ],
+        "started_at": "2022-09-01T18:58:06.227120Z",
+        "state": "completed",
+        "task_group": null,
+        "worker": "/pulp/api/v3/workers/56cc8fa3-23f8-4b23-9c89-9d629471c217/"
+    }
+
+.. note::
+
+    Any error while attempting to apply-changes will be recorded in the "sam.apply_failures" progress report. Specific
+    failure, per entity, can be found in the server-log output.
+
+    Example:
+
+    .. code-block::
+
+        Sep 01 19:13:59 pulp2-nightly-pulp3-source-centos7
+            pulpcore-worker[17332]: pulp [273027f2706b4f54bb85991a4d920173]:
+            pulpcore.app.tasks.sam:WARNING: Unexpected error [
+                ErrorDetail(string='URI /pulp/api/v3/repositories/file/file/daea11e9-b673-49f2-984a-b0de9278de01/ not found for filerepository.', code='invalid')
+            ] trying to update attributes for /pulp/api/v3/repositories/file/file/daea11e9-b673-49f2-984a-b0de9278de01/
+
+To remove an entity from being managed:
+
+.. code-block::
+
+    $ http -b POST :${SAM_HREF}remove/ entity_href="/pulp/api/v3/repositories/rpm/rpm/47a78cce-d947-45fe-a618-c139f288dd7f/"
+    "Removed /pulp/api/v3/repositories/rpm/rpm/47a78cce-d947-45fe-a618-c139f288dd7f/ from managed entities."
+
+    $ http -b :${SAM_HREF} | jq -r .managed_entities
+    [
+      "/pulp/api/v3/remotes/rpm/rpm/4507f854-e334-4f99-9c75-ee7621d64352/"
+      "/pulp/api/v3/repositories/file/file/daea11e9-b673-49f2-984a-b0de9278de01/"
+    ]
+
+Finally, to remove a SAM:
+
+.. code-block::
+
+    $ http DELETE :/pulp/api/v3/shared-attribute-managers/ac6d3093-ac1e-4b3a-b30f-117e31abacad/
+    $
+

pulpcore/app/migrations/0095_sharedattributemanager.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.15 on 2022-09-02 14:58
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django_lifecycle.mixins
+import pulpcore.app.models.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0094_protect_repository_content'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SharedAttributeManager',
+            fields=[
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.TextField(db_index=True, unique=True)),
+                ('managed_attributes', pulpcore.app.models.fields.EncryptedJSONField(blank=True, null=True)),
+                ('managed_entities', django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), null=True, size=None)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model),
+        ),
+    ]

pulpcore/app/models/__init__.py

@@ -5,6 +5,7 @@
     BaseModel,
     Label,
     MasterModel,
+    SharedAttributeManager,
 )
 
 from .access_policy import (  # noqa

pulpcore/app/models/base.py

@@ -3,11 +3,14 @@
 
 from django.contrib.contenttypes.fields import GenericRelation, GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
+from django.contrib.postgres.fields import ArrayField
 from django.db import models
 from django.db.models import options
 from django.db.models.base import ModelBase
 from django_lifecycle import LifecycleModel
 
+from pulpcore.app.models.fields import EncryptedJSONField
+
 
 class Label(LifecycleModel):
     """Model for handling resource labels.
@@ -79,6 +82,22 @@ def __repr__(self):
         return str(self)
 
 
+class SharedAttributeManager(BaseModel):
+    """
+    Manage the shared-attributes of a set of managed entities.
+
+    Fields:
+        pulp_id (models.UUIDField): Primary key identifier
+        name (models.TextField): Unique name of this SAM
+        managed_attributes (models.JSONField): attr:value JSON defining the attributes being managed
+        managed_entities (ArrayField(models.TextField()): list of managed HREFs
+    """
+
+    name = models.TextField(db_index=True, unique=True)
+    managed_attributes = EncryptedJSONField(blank=True, null=True)
+    managed_entities = ArrayField(models.TextField(), null=True)
+
+
 class MasterModelMeta(ModelBase):
     def __new__(cls, name, bases, attrs, **kwargs):
         """Override __new__ to set the default_related_name."""