From bbf641ed2b1e5d86f3c260f8957ac792b1a849cf Mon Sep 17 00:00:00 2001 From: Ina Panova Date: Thu, 5 Mar 2020 19:46:39 +0100 Subject: [PATCH] Add custom repo metadata migration. closes #6283 https://pulp.plan.io/issues/6283 --- CHANGES/6283.feature | 1 + .../0004_pulp2yumrepometadatafile.py | 32 ++++++++++ .../app/plugin/rpm/migrator.py | 4 ++ .../app/plugin/rpm/pulp2_models.py | 29 +++++++++ .../app/plugin/rpm/pulp_2to3_models.py | 62 +++++++++++++++++++ 5 files changed, 128 insertions(+) create mode 100644 CHANGES/6283.feature create mode 100644 pulp_2to3_migration/app/migrations/0004_pulp2yumrepometadatafile.py diff --git a/CHANGES/6283.feature b/CHANGES/6283.feature new file mode 100644 index 00000000..1b7e6a73 --- /dev/null +++ b/CHANGES/6283.feature @@ -0,0 +1 @@ +Add custom repo metadata migration. diff --git a/pulp_2to3_migration/app/migrations/0004_pulp2yumrepometadatafile.py b/pulp_2to3_migration/app/migrations/0004_pulp2yumrepometadatafile.py new file mode 100644 index 00000000..93db889d --- /dev/null +++ b/pulp_2to3_migration/app/migrations/0004_pulp2yumrepometadatafile.py @@ -0,0 +1,32 @@ +# Generated by Django 2.2.10 on 2020-03-10 16:18 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('pulp_2to3_migration', '0003_pulp2erratum_pulp2rpm'), + ] + + operations = [ + migrations.CreateModel( + name='Pulp2YumRepoMetadataFile', + fields=[ + ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('pulp_created', models.DateTimeField(auto_now_add=True)), + ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), + ('data_type', models.CharField(max_length=20)), + ('checksum', models.CharField(max_length=128)), + ('checksum_type', models.CharField(max_length=6)), + ('repo_id', models.TextField()), + ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='yum_repo_metadata_file_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ], + options={ + 'default_related_name': 'yum_repo_metadata_file_detail_model', + 'unique_together': {('data_type', 'repo_id', 'pulp2content')}, + }, + ), + ] diff --git a/pulp_2to3_migration/app/plugin/rpm/migrator.py b/pulp_2to3_migration/app/plugin/rpm/migrator.py index 6b325560..0905b4bd 100644 --- a/pulp_2to3_migration/app/plugin/rpm/migrator.py +++ b/pulp_2to3_migration/app/plugin/rpm/migrator.py @@ -9,10 +9,12 @@ from .pulp2_models import ( Errata, RPM, + YumMetadataFile, ) from .pulp_2to3_models import ( Pulp2Erratum, Pulp2Rpm, + Pulp2YumRepoMetadataFile, ) from .repository import ( @@ -39,6 +41,7 @@ class RpmMigrator(Pulp2to3PluginMigrator): pulp2_content_models = { 'rpm': RPM, 'erratum': Errata, + 'yum_repo_metadata_file': YumMetadataFile, } pulp2_collection = 'units_rpm' pulp3_plugin = 'pulp_rpm' @@ -46,6 +49,7 @@ class RpmMigrator(Pulp2to3PluginMigrator): content_models = { 'rpm': Pulp2Rpm, 'erratum': Pulp2Erratum, + 'yum_repo_metadata_file': Pulp2YumRepoMetadataFile, } importer_migrators = { 'yum_importer': RpmImporter, diff --git a/pulp_2to3_migration/app/plugin/rpm/pulp2_models.py b/pulp_2to3_migration/app/plugin/rpm/pulp2_models.py index 612efeae..a82a5225 100644 --- a/pulp_2to3_migration/app/plugin/rpm/pulp2_models.py +++ b/pulp_2to3_migration/app/plugin/rpm/pulp2_models.py @@ -156,3 +156,32 @@ class Errata(ContentUnit): 'collection': 'units_erratum', 'allow_inheritance': False, } + + +class YumMetadataFile(FileContentUnit): + """ + A model for Pulp 2 YumMetadataFile content type. + + It will become a RepoMetadataFile content type in Pulp 3 world. + """ + + data_type = StringField(required=True) + repo_id = StringField(required=True) + + checksum = StringField() + checksum_type = StringField() + + # For backward compatibility + _ns = StringField(default='units_yum_repo_metadata_file') + _content_type_id = StringField(required=True, default='yum_repo_metadata_file') + + unit_key_fields = ('data_type', 'repo_id') + unit_display_name = 'YUM Repository Metadata File' + unit_description = 'YUM Repository Metadata File' + + TYPE_ID = 'yum_repo_metadata_file' + + meta = { + 'indexes': ['data_type'], + 'collection': 'units_yum_repo_metadata_file', + 'allow_inheritance': False} diff --git a/pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py b/pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py index bc57ea3c..890dc9b2 100644 --- a/pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py +++ b/pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py @@ -5,12 +5,14 @@ from pulp_rpm.app.models import ( Package, + RepoMetadataFile, UpdateRecord, ) from .pulp2_models import ( Errata, RPM, + YumMetadataFile, ) from .xml_utils import get_cr_obj @@ -203,3 +205,63 @@ async def create_pulp3_content(self): advisory = UpdateRecord(**cr_update) # advisory.digest = digest return advisory, relations + + +class Pulp2YumRepoMetadataFile(Pulp2to3Content): + """ + Pulp 2to3 detail content model to store pulp 2 yum_repo_metadata_file + content details for Pulp 3 content creation. + """ + data_type = models.CharField(max_length=20) + checksum = models.CharField(max_length=128) + checksum_type = models.CharField(max_length=6) + repo_id = models.TextField() + + pulp2_type = 'yum_repo_metadata_file' + + class Meta: + unique_together = ('data_type', 'repo_id', 'pulp2content') + default_related_name = 'yum_repo_metadata_file_detail_model' + + @property + def expected_digests(self): + """Return expected digests.""" + return {self.checksum_type: self.checksum} + + @property + def expected_size(self): + """Return expected size.""" + return + + @property + def relative_path_for_content_artifact(self): + """Return relative path.""" + return self.data_type + + @classmethod + async def pre_migrate_content_detail(cls, content_batch): + """ + Pre-migrate Pulp 2 YumMetadataFile with all the fields needed to create a Pulp 3 content + + Args: + content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. + + """ + pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_ids = pulp2_id_obj_map.keys() + pulp2_metadata_content_batch = YumMetadataFile.objects.filter(id__in=pulp2_ids) + pulp2metadata_to_save = [cls(data_type=meta.data_type, + checksum=meta.checksum, + checksum_type=meta.checksum_type, + repo_id=meta.repo_id, + pulp2content=pulp2_id_obj_map[meta.id]) + for meta in pulp2_metadata_content_batch] + cls.objects.bulk_create(pulp2metadata_to_save, ignore_conflicts=True) + + async def create_pulp3_content(self): + """ + Create a Pulp 3 RepoMetadataFile unit for saving it later in a bulk operation. + """ + return RepoMetadataFile(data_type=self.data_type, + checksum=self.checksum, + checksum_type=self.checksum_type)