As a user I can migrate modules and modules-defaults.

closes #6321

CHANGES/6321.feature

@@ -0,0 +1 @@
+As a user I can migrate modules and modules-defaults

pulp_2to3_migration/app/migrations/0005_pulp2modulemd_pulp2modulemddefaults.py

@@ -0,0 +1,55 @@
+# Generated by Django 2.2.10 on 2020-03-13 19:28
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pulp_2to3_migration', '0004_pulp2yumrepometadatafile'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Pulp2ModulemdDefaults',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('module', models.TextField()),
+                ('stream', models.TextField()),
+                ('profiles', django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict)),
+                ('digest', models.TextField()),
+                ('repo_id', models.TextField()),
+                ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='modulemd_defaults_detail_model', to='pulp_2to3_migration.Pulp2Content')),
+            ],
+            options={
+                'default_related_name': 'modulemd_defaults_detail_model',
+                'unique_together': {('digest', 'repo_id', 'pulp2content')},
+            },
+        ),
+        migrations.CreateModel(
+            name='Pulp2Modulemd',
+            fields=[
+                ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pulp_created', models.DateTimeField(auto_now_add=True)),
+                ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('name', models.TextField()),
+                ('stream', models.TextField()),
+                ('version', models.BigIntegerField()),
+                ('context', models.TextField()),
+                ('arch', models.TextField()),
+                ('artifacts', django.contrib.postgres.fields.jsonb.JSONField()),
+                ('checksum', models.TextField()),
+                ('dependencies', django.contrib.postgres.fields.jsonb.JSONField()),
+                ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='modulemd_detail_model', to='pulp_2to3_migration.Pulp2Content')),
+            ],
+            options={
+                'default_related_name': 'modulemd_detail_model',
+                'unique_together': {('name', 'stream', 'version', 'context', 'arch', 'pulp2content')},
+            },
+        ),
+    ]

pulp_2to3_migration/app/plugin/rpm/migrator.py

@@ -1,26 +1,46 @@
+from django.db import transaction
+from django.db.models import Q
+
+from collections import OrderedDict
+
 from pulp_2to3_migration.app.plugin.api import (
     ContentMigrationFirstStage,
     DeclarativeContentMigration,
     Pulp2to3PluginMigrator,
+    RelatePulp2to3Content,
 )
 
-from pulp_rpm.app.models import RpmRepository
+from pulp_rpm.app import models as pulp3_models
 
 from .pulp2_models import (
     Errata,
+    Modulemd,
+    ModulemdDefaults,
     RPM,
     YumMetadataFile,
 )
 from .pulp_2to3_models import (
     Pulp2Erratum,
+    Pulp2Modulemd,
+    Pulp2ModulemdDefaults,
     Pulp2Rpm,
     Pulp2YumRepoMetadataFile,
 )
 
 from .repository import (
+    RpmDistributor,
     RpmImporter,
-    RpmDistributor
 )
+from pulpcore.plugin.stages import (
+    ArtifactSaver,
+    ContentSaver,
+    RemoteArtifactSaver,
+    Stage,
+    QueryExistingArtifacts,
+    QueryExistingContents,
+)
+
+from . import package_utils
 
 
 class RpmMigrator(Pulp2to3PluginMigrator):
@@ -41,15 +61,23 @@ class RpmMigrator(Pulp2to3PluginMigrator):
     pulp2_content_models = {
         'rpm': RPM,
         'erratum': Errata,
+        'modulemd': Modulemd,
+        'modulemd_defaults': ModulemdDefaults,
         'yum_repo_metadata_file': YumMetadataFile,
     }
     pulp2_collection = 'units_rpm'
     pulp3_plugin = 'pulp_rpm'
-    pulp3_repository = RpmRepository
-    content_models = {
-        'rpm': Pulp2Rpm,
-        'erratum': Pulp2Erratum,
-        'yum_repo_metadata_file': Pulp2YumRepoMetadataFile,
+    pulp3_repository = pulp3_models.RpmRepository
+    content_models = OrderedDict([
+        ('rpm', Pulp2Rpm),
+        ('erratum', Pulp2Erratum),
+        ('modulemd', Pulp2Modulemd),
+        ('modulemd_defaults', Pulp2ModulemdDefaults),
+        ('yum_repo_metadata_file', Pulp2YumRepoMetadataFile),
+    ])
+    mutable_content_models = {
+        'modulemd': Pulp2Modulemd,
+        'modulemd_defaults': Pulp2ModulemdDefaults,
     }
     importer_migrators = {
         'yum_importer': RpmImporter,
@@ -64,5 +92,96 @@ async def migrate_content_to_pulp3(cls):
         Migrate pre-migrated Pulp 2 RPM plugin content.
         """
         first_stage = ContentMigrationFirstStage(cls)
-        dm = DeclarativeContentMigration(first_stage=first_stage)
+        dm = RpmDeclarativeContentMigration(first_stage=first_stage)
         await dm.create()
+
+
+class RpmDeclarativeContentMigration(DeclarativeContentMigration):
+    """
+    A pipeline that migrates pre-migrated Pulp 2 rpm content into Pulp 3.
+    """
+
+    def pipeline_stages(self):
+        """
+        Build a list of stages.
+
+        This defines the "architecture" of the content migration to Pulp 3.
+
+        Returns:
+            list: List of :class:`~pulpcore.plugin.stages.Stage` instances
+
+        """
+        pipeline = [
+            self.first_stage,
+            QueryExistingArtifacts(),
+            ArtifactSaver(),
+            QueryExistingContents(),
+            ContentSaver(),
+            RemoteArtifactSaver(),
+            InterrelateContent(),
+            RelatePulp2to3Content(),
+        ]
+
+        return pipeline
+
+
+class InterrelateContent(Stage):
+    """
+    Stage for relating Content to other Content.
+    """
+
+    async def run(self):
+        """
+        Relate each item in the input queue to objects specified on the DeclarativeContent.
+        """
+        async for batch in self.batches():
+            modulemd_packages_batch = []
+            with transaction.atomic():
+                for dc in batch:
+                    if type(dc.content) == pulp3_models.Modulemd:
+                        thru = self.relate_packages_to_module(dc)
+                        modulemd_packages_batch.extend(thru)
+
+                ModulemdPackages = pulp3_models.Modulemd.packages.through
+                ModulemdPackages.objects.bulk_create(objs=modulemd_packages_batch,
+                                                     ignore_conflicts=True,
+                                                     batch_size=1000)
+
+            for dc in batch:
+                await self.put(dc)
+
+    def relate_packages_to_module(self, module_dc):
+        """
+        Relate Packages to a Module.
+
+        Args:
+            module_dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Module
+        """
+        ModulemdPackages = pulp3_models.Modulemd.packages.through
+        artifacts_list = module_dc.content.artifacts
+        # find rpm by nevra
+        # We are relying on the order of the processed DC
+        # RPMs should have passed through ContentSaver stage already
+        pq = Q()
+        for artifact in artifacts_list:
+            nevra = package_utils.nevra(artifact)
+            pq |= Q(
+                name=nevra[0],
+                epoch=nevra[1],
+                version=nevra[2],
+                release=nevra[3],
+                arch=nevra[4],
+                is_modular=True)
+        packages_list = []
+        if pq:
+            packages_list = pulp3_models.Package.objects.filter(pq).only('pk')
+        thru = []
+        # keep track of rpm nevra for which we already created a relation with module.
+        # it can happen that we have 2 rpms with same nevra but different checksum
+        # in that case just skip the second occurrence of rpm and do not create the relation
+        already_related = []
+        for pkg in packages_list:
+            if pkg.nevra not in already_related:
+                thru.append(ModulemdPackages(package_id=pkg.pk, modulemd_id=module_dc.content.pk))
+                already_related.append(pkg.nevra)
+        return thru

pulp_2to3_migration/app/plugin/rpm/package_utils.py

@@ -0,0 +1,71 @@
+import logging
+
+from gettext import gettext as _
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def nevra(name):
+    """
+    Parse NEVRA.
+    Inspired by:
+    https://github.com/rpm-software-management/hawkey/blob/d61bf52871fcc8e41c92921c8cd92abaa4dfaed5/src/util.c#L157. # NOQA
+    We don't use hawkey because it is not available on all platforms we support.
+
+    Args:
+        name(string): NEVRA (jay-3:3.10-4.fc3.x86_64)
+
+    Returns:
+        tuple: parsed NEVRA (name, epoch, version, release, architecture)
+
+    """
+    if name.count(".") < 1:
+        msg = _("failed to parse nevra '%s' not a valid nevra") % name
+        _LOGGER.exception(msg)
+        raise ValueError(msg)
+
+    arch_dot_pos = name.rfind(".")
+    arch = name[arch_dot_pos + 1:]
+
+    return nevr(name[:arch_dot_pos]) + (arch, )
+
+
+def nevr(name):
+    """
+    Parse NEVR.
+    Inspired by:
+    https://github.com/rpm-software-management/hawkey/blob/d61bf52871fcc8e41c92921c8cd92abaa4dfaed5/src/util.c#L157. # NOQA
+
+    Args:
+        name(string): NEVR "jay-test-3:3.10-4.fc3"
+
+    Returns:
+       tuple: parsed NEVR (name, epoch, version, release)
+
+    """
+    if name.count("-") < 2:  # release or name is missing
+        msg = _("failed to parse nevr '%s' not a valid nevr") % name
+        _LOGGER.exception(msg)
+        raise ValueError(msg)
+
+    release_dash_pos = name.rfind("-")
+    release = name[release_dash_pos + 1:]
+    name_epoch_version = name[:release_dash_pos]
+    name_dash_pos = name_epoch_version.rfind("-")
+    package_name = name_epoch_version[:name_dash_pos]
+
+    epoch_version = name_epoch_version[name_dash_pos + 1:].split(":")
+    if len(epoch_version) == 1:
+        epoch = 0
+        version = epoch_version[0]
+    elif len(epoch_version) == 2:
+        epoch = int(epoch_version[0])
+        version = epoch_version[1]
+    else:
+        # more than one ':'
+        msg = _("failed to parse nevr '%s' not a valid nevr") % name
+        _LOGGER.exception(msg)
+        raise ValueError(msg)
+
+    return package_name, epoch, version, release

pulp_2to3_migration/app/plugin/rpm/pulp2_models.py

@@ -185,3 +185,64 @@ class YumMetadataFile(FileContentUnit):
         'indexes': ['data_type'],
         'collection': 'units_yum_repo_metadata_file',
         'allow_inheritance': False}
+
+
+class Modulemd(FileContentUnit):
+    """
+    A model for Pulp 2 Modulemd content type.
+    """
+    TYPE_ID = 'modulemd'
+
+    # Unit key fields NSVCA
+    name = StringField(required=True)
+    stream = StringField(required=True)
+    version = IntField(required=True)
+    context = StringField(required=True)
+    arch = StringField(required=True)
+
+    summary = StringField()
+    description = StringField()
+    profiles = DictField()
+    artifacts = ListField()
+    checksum = StringField()
+    dependencies = ListField()
+
+    # For backward compatibility
+    _ns = StringField(default='units_modulemd')
+    _content_type_id = StringField(required=True, default=TYPE_ID)
+
+    unit_key_fields = ('name', 'stream', 'version', 'context', 'arch', )
+    unit_display_name = 'Modulemd'
+    unit_description = 'Modulemd'
+
+    meta = {'collection': 'units_modulemd',
+            'indexes': ['artifacts'],
+            'allow_inheritance': False}
+
+
+class ModulemdDefaults(FileContentUnit):
+    """
+    A model for Pulp 2 Modulemd content type.
+    """
+    TYPE_ID = 'modulemd_defaults'
+
+    # Unit key fields
+    name = StringField(required=True)
+    repo_id = StringField(required=True)
+
+    stream = StringField()
+    profiles = StringField()
+
+    checksum = StringField()
+
+    # For backward compatibility
+    _ns = StringField(default='units_modulemd_defaults')
+    _content_type_id = StringField(required=True, default=TYPE_ID)
+
+    unit_key_fields = ('name', 'repo_id',)
+    unit_display_name = 'ModulemdDefaults'
+    unit_description = 'ModulemdDefaults'
+
+    meta = {'collection': 'units_modulemd_defaults',
+            'indexes': ['repo_id'],
+            'allow_inheritance': False}