Adding PulpExporter model and endpoints

Also adding a new Export model and refactoring existing Exporters.

fixes #6328
https://pulp.plan.io/issues/6328

Required PR: pulp/pulp_file#366

pulpcore/app/migrations/0023_auto_20200313_1618.py

@@ -0,0 +1,79 @@
+# Generated by Django 2.2.6 on 2020-03-13 16:18
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('file', '0006_delete_filefilesystemexporter'),
+        ('core', '0022_rename_last_version'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Export',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('state', models.TextField(choices=[('waiting', 'Waiting'), ('skipped', 'Skipped'), ('running', 'Running'), ('completed', 'Completed'), ('failed', 'Failed'), ('canceled', 'Canceled')])),
+                ('started_at', models.DateTimeField(null=True)),
+                ('finished_at', models.DateTimeField(null=True)),
+                ('error', django.contrib.postgres.fields.jsonb.JSONField(null=True)),
+                ('user_params', django.contrib.postgres.fields.jsonb.JSONField()),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ExportedResource',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('object_id', models.UUIDField()),
+                ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+                ('export', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='exported_resources', to='core.Export')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Exporter',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('pulp_type', models.TextField(default=None)),
+                ('name', models.TextField(db_index=True, unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.DeleteModel(
+            name='FileSystemExporter',
+        ),
+        migrations.CreateModel(
+            name='PulpExporter',
+            fields=[
+                ('exporter_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='core_pulp_exporter', serialize=False, to='core.Exporter')),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_pulp_exporter',
+            },
+            bases=('core.exporter',),
+        ),
+        migrations.AddField(
+            model_name='export',
+            name='exporter',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.Exporter'),
+        ),
+    ]

pulpcore/app/models/__init__.py

@@ -11,9 +11,8 @@
 )
 from .generic import GenericRelationModel  # noqa
 from .exporter import (  # noqa
+    Exporter,
     FileSystemExporter,
-    FileSystemPublicationExporter,
-    FileSystemRepositoryVersionExporter,
 )
 from .publication import (  # noqa
     BaseDistribution,

pulpcore/app/models/exporter.py

@@ -2,23 +2,86 @@
 from gettext import gettext as _
 
 from django.conf import settings
+from django.contrib.postgres.fields import JSONField
 from django.db import models
 
-from pulpcore.app.models import ContentArtifact, MasterModel
+from pulpcore.app.models import (
+    BaseModel,
+    ContentArtifact,
+    GenericRelationModel,
+    MasterModel,
+)
+from .repository import Repository, RepositoryVersion
+from pulpcore.constants import TASK_CHOICES
 
 
-class FileSystemExporter(MasterModel):
+class Export(BaseModel):
     """
-    A base class that provides logic to export a set of content. This set of content can be a
-    publication, repo version, etc.
+    A class that represents an Export.
+    """
+    state = models.TextField(choices=TASK_CHOICES)
+    started_at = models.DateTimeField(null=True)
+    finished_at = models.DateTimeField(null=True)
+    error = JSONField(null=True)
+    user_params = JSONField()
+    exporter = models.ForeignKey("Exporter", on_delete=models.CASCADE)
+
+
+class ExportedResource(GenericRelationModel):
+    """
+    A class to represent anything that was exported in an Export. This includes repo versions,
+    publications, etc.
+
+    Relations:
+        export (models.ForeignKey): The Export that exported the resource.
+    """
+    export = models.ForeignKey(
+        Export,
+        related_name='exported_resources',
+        on_delete=models.CASCADE
+    )
+
+
+class Exporter(MasterModel):
+    """
+    A base class that provides logic to export a set of content and keep track of Exports.
 
     Fields:
 
         name (models.TextField): The exporter unique name.
+    """
+    name = models.TextField(db_index=True, unique=True)
+
+
+class PulpExporter(Exporter):
+    """
+    A class that provides exports that can be imported into other Pulp instances.
+
+    Fields:
+
         path (models.TextField): a full path where the export will go.
     """
 
-    name = models.TextField(db_index=True, unique=True)
+    TYPE = 'pulp'
+
+    path = models.TextField()
+    repositories = models.ManyToManyField(Repository)
+    repository_versions = models.ManyToManyField(RepositoryVersion)
+    last_export = models.ForeignKey(Export, on_delete=models.CASCADE)
+
+    class Meta:
+        default_related_name = '%(app_label)s_pulp_exporter'
+
+
+class FileSystemExporter(Exporter):
+    """
+    A base class that provides logic to export a set of content to the filesystem.
+
+    Fields:
+
+        name (models.TextField): The exporter unique name.
+        path (models.TextField): a full path where the export will go.
+    """
     path = models.TextField()
 
     def _export_to_file_system(self, content_artifacts):
@@ -28,29 +91,29 @@ def _export_to_file_system(self, content_artifacts):
         Args:
             content_artifacts (django.db.models.QuerySet): Set of ContentArtifacts to export
         """
+
+        # TODO: we need to create an Export
+
         if content_artifacts.filter(artifact=None).exists():
             RuntimeError(_("Remote artifacts cannot be exported."))
 
-        if settings.DEFAULT_FILE_STORAGE == 'pulpcore.app.models.storage.FileSystem':
-            for ca in content_artifacts:
-                artifact = ca.artifact
-                src = os.path.join(settings.MEDIA_ROOT, artifact.file.name)
-                dest = os.path.join(self.path, ca.relative_path)
+        for ca in content_artifacts:
+            artifact = ca.artifact
+            dest = os.path.join(self.path, ca.relative_path)
 
-                try:
-                    os.makedirs(os.path.split(dest)[0])
-                except FileExistsError:
-                    pass
+            try:
+                os.makedirs(os.path.split(dest)[0])
+            except FileExistsError:
+                pass
 
+            if settings.DEFAULT_FILE_STORAGE == 'pulpcore.app.models.storage.FileSystem':
+                src = os.path.join(settings.MEDIA_ROOT, artifact.file.name)
                 os.link(src, dest)
+            else:
+                with open(dest, "wb") as f:
+                    f.write(artifact.file.read())
 
-
-class FileSystemPublicationExporter(FileSystemExporter):
-    """
-    A publication file system exporter.
-    """
-
-    def export(self, publication):
+    def export_publication(self, publication):
         """
         Export a publication to the file system
 
@@ -66,16 +129,7 @@ def export(self, publication):
 
         self._export_to_file_system(content_artifacts)
 
-    class Meta:
-        abstract = True
-
-
-class FileSystemRepositoryVersionExporter(FileSystemExporter):
-    """
-    A repo version file system exporter.
-    """
-
-    def export(self, repository_version):
+    def export_repository_version(self, repository_version):
         """
         Export a repository version to the file system
 

pulpcore/app/serializers/__init__.py

@@ -20,7 +20,11 @@
     SingleArtifactContentSerializer,
     SigningServiceSerializer,
 )
-from .exporter import FileSystemExporterSerializer, PublicationExportSerializer  # noqa
+from .exporter import (  # noqa
+    ExporterSerializer,
+    FileSystemExporterSerializer,
+    PublicationExportSerializer,
+)
 from .fields import (  # noqa
     BaseURLField,
     LatestVersionField,

pulpcore/app/serializers/exporter.py

@@ -11,25 +11,32 @@
 )
 
 
-class FileSystemExporterSerializer(ModelSerializer):
+class ExporterSerializer(ModelSerializer):
     """
-    Base serializer for FileSystemExporters.
+    Base serializer for Exporters.
     """
     pulp_href = DetailIdentityField()
     name = serializers.CharField(
         help_text=_("Unique name of the file system exporter."),
         validators=[UniqueValidator(queryset=models.BaseDistribution.objects.all())]
     )
+
+    class Meta:
+        model = models.Exporter
+        fields = ModelSerializer.Meta.fields + ('name',)
+
+
+class FileSystemExporterSerializer(ExporterSerializer):
+    """
+    Base serializer for FileSystemExporters.
+    """
     path = serializers.CharField(
         help_text=_("File system location to export to.")
     )
 
     class Meta:
         model = models.FileSystemExporter
-        fields = ModelSerializer.Meta.fields + (
-            'path',
-            'name',
-        )
+        fields = ExporterSerializer.Meta.fields + ('path',)
 
 
 class PublicationExportSerializer(serializers.Serializer):

pulpcore/app/tasks/__init__.py

@@ -1,3 +1,5 @@
 from pulpcore.app.tasks import base, repository, upload  # noqa
 
+from .export import file_system_export  # noqa
+
 from .orphan import orphan_cleanup  # noqa

pulpcore/app/tasks/export.py

@@ -0,0 +1,40 @@
+import logging
+
+from gettext import gettext as _
+
+from pulpcore.app.models import Exporter, Publication, RepositoryVersion
+
+
+log = logging.getLogger(__name__)
+
+
+def file_system_export(exporter_pk, publication_pk=None, repository_version_pk=None):
+    """
+    Export to the file system.
+
+    Args:
+        exporter_pk (str): FileSystemExporter pk
+        publication_pk (str): Publication pk
+        publication_pk (str): RepositoryVersion pk
+
+    """
+    exporter = Exporter.objects.get(pk=exporter_pk).cast()
+
+    if publication_pk:
+        publication = Publication.objects.get(pk=publication_pk).cast()
+
+        log.info(
+            _(
+                "Exporting: file_system_exporter={exporter}, publication={publication}, path=path"
+            ).format(exporter=exporter.name, publication=publication.pk, path=exporter.path)
+        )
+        exporter.export_publication(publication)
+    elif repository_version_pk:
+        repository_version = RepositoryVersion.objects.get(pk=repository_version_pk)
+
+        log.info(
+            _(
+                "Exporting: file_system_exporter={exporter}, publication={publication}, path=path"
+            ).format(exporter=exporter.name, publication=publication.pk, path=exporter.path)
+        )
+        exporter.export_repository_version(repository_version)

pulpcore/app/viewsets/__init__.py

@@ -19,7 +19,7 @@
     RepositoryVersionFilter,
 )
 from .exporter import (  # noqa
-    FileSystemExporterViewSet
+    ExporterViewSet
 )
 from .publication import (  # noqa
     BaseDistributionViewSet,