Adding PulpExporter model and endpoints

Also adding a new Export model and refactoring existing Exporters.

fixes #6328
https://pulp.plan.io/issues/6328

Required PR: pulp/pulp_file#366

CHANGES/6328.feature

@@ -0,0 +1 @@
+Added a new APIs for PulpExorpters and Exports.

CHANGES/plugin_api/6328.feature

@@ -0,0 +1 @@
+Added a new Export model, serializer, and viewset.

CHANGES/plugin_api/6328.removal

@@ -0,0 +1,3 @@
+Changed master model from FileSystemExporter to Exporter. Plugins will still need to extend
+FileSystemExporter but the master table is now core_exporter. This will require that plugins drop
+and recreate their filesystem exporter tables.

pulpcore/app/migrations/0023_change_exporter_models.py

@@ -0,0 +1,82 @@
+# Generated by Django 2.2.6 on 2020-03-17 19:43
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('core', '0022_rename_last_version'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Export',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('params', django.contrib.postgres.fields.jsonb.JSONField(null=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ExportedResource',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('object_id', models.UUIDField()),
+                ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+                ('export', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='exported_resources', to='core.Export')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Exporter',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('pulp_type', models.TextField(default=None)),
+                ('name', models.TextField(db_index=True, unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.DeleteModel(
+            name='FileSystemExporter',
+        ),
+        migrations.AddField(
+            model_name='export',
+            name='exporter',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.Exporter'),
+        ),
+        migrations.AddField(
+            model_name='export',
+            name='task',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='core.Task'),
+        ),
+        migrations.CreateModel(
+            name='PulpExporter',
+            fields=[
+                ('exporter_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='core_pulp_exporter', serialize=False, to='core.Exporter')),
+                ('path', models.TextField()),
+                ('last_export', models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='core_pulp_exporter', to='core.Export')),
+                ('repositories', models.ManyToManyField(related_name='core_pulp_exporter', to='core.Repository')),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_pulp_exporter',
+            },
+            bases=('core.exporter',),
+        ),
+    ]

pulpcore/app/models/__init__.py

@@ -11,9 +11,11 @@
 )
 from .generic import GenericRelationModel  # noqa
 from .exporter import (  # noqa
+    Export,
+    ExportedResource,
+    Exporter,
     FileSystemExporter,
-    FileSystemPublicationExporter,
-    FileSystemRepositoryVersionExporter,
+    PulpExporter,
 )
 from .publication import (  # noqa
     BaseDistribution,

pulpcore/app/models/exporter.py

@@ -2,23 +2,99 @@
 from gettext import gettext as _
 
 from django.conf import settings
+from django.contrib.postgres.fields import JSONField
 from django.db import models
 
-from pulpcore.app.models import ContentArtifact, MasterModel
+from pulpcore.app.models import (
+    BaseModel,
+    ContentArtifact,
+    GenericRelationModel,
+    MasterModel,
+)
+from .repository import Repository
+from .task import CreatedResource, Task
 
 
-class FileSystemExporter(MasterModel):
+class Export(BaseModel):
     """
-    A base class that provides logic to export a set of content. This set of content can be a
-    publication, repo version, etc.
+    A class that represents an Export.
+
+    Fields:
+
+        params (models.JSONField): A set of parameters used to create the export
+
+    Relations:
+
+        task (models.ForeignKey): The Task that created the export
+        exporter (models.ForeignKey): The Exporter that exported the resource.
+
+    Relations:
+
+        exporter (models.ForeignKey): The Exporter that created this Export.
+    """
+    params = JSONField(null=True)
+    task = models.ForeignKey("Task", on_delete=models.PROTECT)
+    exporter = models.ForeignKey("Exporter", on_delete=models.CASCADE)
+
+
+class ExportedResource(GenericRelationModel):
+    """
+    A class to represent anything that was exported in an Export. This includes repo versions,
+    publications, etc.
+
+    Relations:
+
+        export (models.ForeignKey): The Export that exported the resource.
+    """
+    export = models.ForeignKey(
+        Export,
+        related_name='exported_resources',
+        on_delete=models.CASCADE
+    )
+
+
+class Exporter(MasterModel):
+    """
+    A base class that provides logic to export a set of content and keep track of Exports.
 
     Fields:
 
         name (models.TextField): The exporter unique name.
+    """
+    name = models.TextField(db_index=True, unique=True)
+
+
+class PulpExporter(Exporter):
+    """
+    A class that provides exports that can be imported into other Pulp instances.
+
+    Fields:
+
         path (models.TextField): a full path where the export will go.
+
+    Relations:
+
+        repositories (models.ManyToManyField): Repos to be exported.
+        last_export (models.ForeignKey): The last Export from the Exporter.
     """
+    TYPE = 'pulp'
 
-    name = models.TextField(db_index=True, unique=True)
+    path = models.TextField()
+    repositories = models.ManyToManyField(Repository)
+    last_export = models.ForeignKey(Export, on_delete=models.PROTECT, null=True)
+
+    class Meta:
+        default_related_name = '%(app_label)s_pulp_exporter'
+
+
+class FileSystemExporter(Exporter):
+    """
+    A base class that provides logic to export a set of content to the filesystem.
+
+    Fields:
+
+        path (models.TextField): a full path where the export will go.
+    """
     path = models.TextField()
 
     def _export_to_file_system(self, content_artifacts):
@@ -31,32 +107,33 @@ def _export_to_file_system(self, content_artifacts):
         if content_artifacts.filter(artifact=None).exists():
             RuntimeError(_("Remote artifacts cannot be exported."))
 
-        if settings.DEFAULT_FILE_STORAGE == 'pulpcore.app.models.storage.FileSystem':
-            for ca in content_artifacts:
-                artifact = ca.artifact
-                src = os.path.join(settings.MEDIA_ROOT, artifact.file.name)
-                dest = os.path.join(self.path, ca.relative_path)
+        for ca in content_artifacts:
+            artifact = ca.artifact
+            dest = os.path.join(self.path, ca.relative_path)
 
-                try:
-                    os.makedirs(os.path.split(dest)[0])
-                except FileExistsError:
-                    pass
+            try:
+                os.makedirs(os.path.split(dest)[0])
+            except FileExistsError:
+                pass
 
+            if settings.DEFAULT_FILE_STORAGE == 'pulpcore.app.models.storage.FileSystem':
+                src = os.path.join(settings.MEDIA_ROOT, artifact.file.name)
                 os.link(src, dest)
+            else:
+                with open(dest, "wb") as f:
+                    f.write(artifact.file.read())
 
-
-class FileSystemPublicationExporter(FileSystemExporter):
-    """
-    A publication file system exporter.
-    """
-
-    def export(self, publication):
+    def export_publication(self, publication):
         """
         Export a publication to the file system
 
         Args:
             publication (pulpcore.app.models.Publication): a publication to export
         """
+        export = Export.objects.create(exporter=self, task=Task.current())
+        ExportedResource.objects.create(export=export, content_object=publication)
+        CreatedResource.objects.create(content_object=export)
+
         content_artifacts = ContentArtifact.objects.filter(
             pk__in=publication.published_artifact.values_list("content_artifact__pk", flat=True))
 
@@ -66,22 +143,17 @@ def export(self, publication):
 
         self._export_to_file_system(content_artifacts)
 
-    class Meta:
-        abstract = True
-
-
-class FileSystemRepositoryVersionExporter(FileSystemExporter):
-    """
-    A repo version file system exporter.
-    """
-
-    def export(self, repository_version):
+    def export_repository_version(self, repository_version):
         """
         Export a repository version to the file system
 
         Args:
             repository_version (pulpcore.app.models.RepositoryVersion): a repo version to export
         """
+        export = Export.objects.create(exporter=self, task=Task.current())
+        ExportedResource.objects.create(export=export, content_object=repository_version)
+        CreatedResource.objects.create(content_object=export)
+
         content_artifacts = ContentArtifact.objects.filter(
             content__in=repository_version.content)
 

pulpcore/app/serializers/__init__.py

@@ -20,9 +20,11 @@
     SingleArtifactContentSerializer,
     SigningServiceSerializer,
 )
-from .exporter import FileSystemExporterSerializer, PublicationExportSerializer  # noqa
 from .fields import (  # noqa
     BaseURLField,
+    ExportsIdentityFromExporterField,
+    ExportRelatedField,
+    ExportIdentityField,
     LatestVersionField,
     SecretCharField,
     SingleContentArtifactField,
@@ -31,6 +33,13 @@
     RepositoryVersionIdentityField,
     relative_path_validator,
 )
+from .exporter import (  # noqa
+    ExportSerializer,
+    ExporterSerializer,
+    FileSystemExporterSerializer,
+    PublicationExportSerializer,
+    PulpExporterSerializer,
+)
 from .progress import ProgressReportSerializer  # noqa
 from .publication import (  # noqa
     BaseDistributionSerializer,