Add custom repo metadata migration.

closes #6283
https://pulp.plan.io/issues/6283

CHANGES/6283.feature

@@ -0,0 +1 @@
+Add custom repo metadata migration.

pulp_2to3_migration/app/migrations/0004_pulp2yumrepometadatafile.py

@@ -0,0 +1,32 @@
+# Generated by Django 2.2.10 on 2020-03-10 16:18
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pulp_2to3_migration', '0003_pulp2erratum_pulp2rpm'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Pulp2YumRepoMetadataFile',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('data_type', models.CharField(max_length=20)),
+                ('checksum', models.CharField(max_length=128)),
+                ('checksum_type', models.CharField(max_length=6)),
+                ('repo_id', models.TextField()),
+                ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='yum_repo_metadata_file_detail_model', to='pulp_2to3_migration.Pulp2Content')),
+            ],
+            options={
+                'default_related_name': 'yum_repo_metadata_file_detail_model',
+                'unique_together': {('data_type', 'repo_id', 'pulp2content')},
+            },
+        ),
+    ]

pulp_2to3_migration/app/plugin/rpm/migrator.py

@@ -9,10 +9,12 @@
 from .pulp2_models import (
     Errata,
     RPM,
+    YumMetadataFile,
 )
 from .pulp_2to3_models import (
     Pulp2Erratum,
     Pulp2Rpm,
+    Pulp2YumRepoMetadataFile,
 )
 
 from .repository import (
@@ -39,13 +41,15 @@ class RpmMigrator(Pulp2to3PluginMigrator):
     pulp2_content_models = {
         'rpm': RPM,
         'erratum': Errata,
+        'yum_repo_metadata_file': YumMetadataFile,
     }
     pulp2_collection = 'units_rpm'
     pulp3_plugin = 'pulp_rpm'
     pulp3_repository = RpmRepository
     content_models = {
         'rpm': Pulp2Rpm,
         'erratum': Pulp2Erratum,
+        'yum_repo_metadata_file': Pulp2YumRepoMetadataFile,
     }
     importer_migrators = {
         'yum_importer': RpmImporter,

pulp_2to3_migration/app/plugin/rpm/pulp2_models.py

@@ -156,3 +156,32 @@ class Errata(ContentUnit):
         'collection': 'units_erratum',
         'allow_inheritance': False,
     }
+
+
+class YumMetadataFile(FileContentUnit):
+    """
+    A model for Pulp 2 YumMetadataFile content type.
+
+    It will become a RepoMetadataFile content type in Pulp 3 world.
+    """
+
+    data_type = StringField(required=True)
+    repo_id = StringField(required=True)
+
+    checksum = StringField()
+    checksum_type = StringField()
+
+    # For backward compatibility
+    _ns = StringField(default='units_yum_repo_metadata_file')
+    _content_type_id = StringField(required=True, default='yum_repo_metadata_file')
+
+    unit_key_fields = ('data_type', 'repo_id')
+    unit_display_name = 'YUM Repository Metadata File'
+    unit_description = 'YUM Repository Metadata File'
+
+    TYPE_ID = 'yum_repo_metadata_file'
+
+    meta = {
+        'indexes': ['data_type'],
+        'collection': 'units_yum_repo_metadata_file',
+        'allow_inheritance': False}

pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py

@@ -5,12 +5,14 @@
 
 from pulp_rpm.app.models import (
     Package,
+    RepoMetadataFile,
     UpdateRecord,
 )
 
 from .pulp2_models import (
     Errata,
     RPM,
+    YumMetadataFile,
 )
 
 from .xml_utils import get_cr_obj
@@ -203,3 +205,63 @@ async def create_pulp3_content(self):
         advisory = UpdateRecord(**cr_update)
         # advisory.digest = digest
         return advisory, relations
+
+
+class Pulp2YumRepoMetadataFile(Pulp2to3Content):
+    """
+    Pulp 2to3 detail content model to store pulp 2 yum_repo_metadata_file
+    content details for Pulp 3 content creation.
+    """
+    data_type = models.CharField(max_length=20)
+    checksum = models.CharField(max_length=128)
+    checksum_type = models.CharField(max_length=6)
+    repo_id = models.TextField()
+
+    pulp2_type = 'yum_repo_metadata_file'
+
+    class Meta:
+        unique_together = ('data_type', 'repo_id', 'pulp2content')
+        default_related_name = 'yum_repo_metadata_file_detail_model'
+
+    @property
+    def expected_digests(self):
+        """Return expected digests."""
+        return {self.checksum_type: self.checksum}
+
+    @property
+    def expected_size(self):
+        """Return expected size."""
+        return
+
+    @property
+    def relative_path_for_content_artifact(self):
+        """Return relative path."""
+        return self.data_type
+
+    @classmethod
+    async def pre_migrate_content_detail(cls, content_batch):
+        """
+        Pre-migrate Pulp 2 YumMetadataFile with all the fields needed to create a Pulp 3 content
+
+        Args:
+             content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content.
+
+        """
+        pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch}
+        pulp2_ids = pulp2_id_obj_map.keys()
+        pulp2_metadata_content_batch = YumMetadataFile.objects.filter(id__in=pulp2_ids)
+        pulp2metadata_to_save = [cls(data_type=meta.data_type,
+                                     checksum=meta.checksum,
+                                     checksum_type=meta.checksum_type,
+                                     repo_id=meta.repo_id,
+                                     pulp2content=pulp2_id_obj_map[meta.id])
+                                 for meta in pulp2_metadata_content_batch]
+        cls.objects.bulk_create(pulp2metadata_to_save, ignore_conflicts=True)
+
+    async def create_pulp3_content(self):
+        """
+        Create a Pulp 3 RepoMetadataFile unit for saving it later in a bulk operation.
+        """
+        return RepoMetadataFile(data_type=self.data_type,
+                                checksum=self.checksum,
+                                checksum_type=self.checksum_type)