Fix distribution serialization.

closes #7809
https://pulp.plan.io/issues/7809

CHANGES/7809.bugfix

@@ -0,0 +1 @@
+Fixed distribution serialization.

docs/workflows.rst

@@ -44,6 +44,7 @@ migration. Optionally, skip corrupted or missing Pulp 2 content by specifying th
     It is possible to re-run migration as many times as needed (if the Pulp 3 plugin which is
     being migrated is not used).
 
+
 3. List the mapping for Pulp 2 and Pulp 3 repositories if needed.
 
 .. code:: bash

pulp_2to3_migration/app/migration.py

@@ -276,6 +276,9 @@ def complex_repo_migration(plugin, pulp3_repo_setup, repo_name):
                     dist_migrator, progress_dist, dist,
                     migrated_repo.pulp3_repository_version
                 )
+                # add distirbutors specified in the complex plan
+                # these can be native and not native distributors
+                migrated_repo.pulp2_dists.add(dist)
 
 
 def create_repoversions_publications_distributions(plan, parallel=True):

pulp_2to3_migration/app/migrations/0018_pulp2distributor_pulp2_repos.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.16 on 2020-11-12 09:59
+
+from django.db import migrations, models
+
+
+def make_pulp2repos_m2m(apps, schema_editor):
+    """
+    Add many2many relation between pulp2repo and pulp2dist.
+    """
+    Pulp2Distributor = apps.get_model('pulp_2to3_migration', 'Pulp2Distributor')
+    pulp2_dists_qs = Pulp2Distributor.objects.all()
+
+    for dist in pulp2_dists_qs.iterator():
+        if dist.pulp2_repository:
+            dist.pulp2_repos.add(dist.pulp2_repository)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pulp_2to3_migration', '0017_pulp2debpackage'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pulp2distributor',
+            name='pulp2_repos',
+            field=models.ManyToManyField(related_name='pulp2_dists', to='pulp_2to3_migration.Pulp2Repository'),
+        ),
+        migrations.RunPython(make_pulp2repos_m2m),
+    ]

pulp_2to3_migration/app/migrations/0019_remove_pulp2_repository.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.16 on 2020-11-12 10:05
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pulp_2to3_migration', '0018_pulp2distributor_pulp2_repos'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='pulp2distributor',
+            unique_together={('pulp2_object_id',)},
+        ),
+        migrations.RemoveField(
+            model_name='pulp2distributor',
+            name='pulp2_repository',
+        ),
+    ]

pulp_2to3_migration/app/models/repository.py

@@ -137,6 +137,7 @@ class Pulp2Distributor(BaseModel):
     Information about Pulp 2 distributor.
 
     Fields:
+        pulp2_object_id (models.CharField): Object id of a distributor in Pulp 2
         pulp2_id (models.TextField): Id of distributor in Pulp 2
         pulp2_type_id (models.CharField): Id of distributor type in Pulp 2
         pulp2_config (JSONField): Pulp 2 distributor config in JSON format
@@ -147,7 +148,7 @@ class Pulp2Distributor(BaseModel):
         not_in_plan (models.BooleanField): True if a resource is not a part of the migration plan.
 
     Relations:
-        pulp2_repository (models.ForeignKey): Pulp 2 repository this distributor belongs to
+        pulp2_repos (models.ManyToManyField): Pulp 2 repository that is getting distributed
         pulp3_publication (models.ForeignKey): Pulp 3 publication this distributor was
             migrated to
         pulp3_distribution (models.OneToOneField): Pulp 3 distribution this distributor was
@@ -163,7 +164,7 @@ class Pulp2Distributor(BaseModel):
     not_in_plan = models.BooleanField(default=False)
 
     # each pulp2 repository can have multiple distributors
-    pulp2_repository = models.ForeignKey(Pulp2Repository, on_delete=models.CASCADE, null=True)
+    pulp2_repos = models.ManyToManyField(Pulp2Repository, related_name='pulp2_dists')
 
     # the same publication/repo version can be published by multiple distributors
     pulp3_publication = models.ForeignKey(Publication, on_delete=models.SET_NULL, null=True)
@@ -175,7 +176,7 @@ class Pulp2Distributor(BaseModel):
                                               null=True)
 
     class Meta:
-        unique_together = ('pulp2_repository', 'pulp2_id')
+        unique_together = ('pulp2_object_id',)
         indexes = [
             models.Index(fields=['pulp2_type_id']),
         ]

pulp_2to3_migration/app/plugin/deb/repository.py

@@ -61,8 +61,10 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version):
             created(bool): True if Distribution has just been created;
                            False if Distribution is an existing one;
         """
+        # this will go away with the simple-complex plan conversion work
         if not repo_version:
-            repo_version = pulp2distributor.pulp2_repository.pulp3_repository_version
+            repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True)
+            repo_version = repo[0].pulp3_repository_version
         publication = repo_version.publication_set.first()
         if not publication:
             # create publication

pulp_2to3_migration/app/plugin/docker/repository.py

@@ -48,8 +48,10 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version):
             created(bool): True if Distribution has just been created;
                            False if Distribution is an existing one
         """
+        # this will go away with the simple-complex plan conversion work
         if not repo_version:
-            repo_version = pulp2distributor.pulp2_repository.pulp3_repository_version
+            repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True)
+            repo_version = repo[0].pulp3_repository_version
         pulp2_config = pulp2distributor.pulp2_config
         base_config = cls.parse_base_config(pulp2distributor, pulp2_config)
         base_config['base_path'] = pulp2_config.get(

pulp_2to3_migration/app/plugin/iso/repository.py

@@ -59,8 +59,10 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version):
                            is an existing one
         """
 
+        # this will go away with the simple-complex plan conversion work
         if not repo_version:
-            repo_version = pulp2distributor.pulp2_repository.pulp3_repository_version
+            repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True)
+            repo_version = repo[0].pulp3_repository_version
         publication = repo_version.publication_set.first()
         if not publication:
             # create publication

pulp_2to3_migration/app/plugin/rpm/repository.py

@@ -62,8 +62,10 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version):
         """
         pulp2_config = pulp2distributor.pulp2_config
 
+        # this will go away with the simple-complex plan conversion work
         if not repo_version:
-            repo_version = pulp2distributor.pulp2_repository.pulp3_repository_version
+            repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True)
+            repo_version = repo[0].pulp3_repository_version
         publication = repo_version.publication_set.first()
         if not publication:
             pulp2_checksum_type = pulp2_config.get('checksum_type')

pulp_2to3_migration/app/pre_migration.py

@@ -535,10 +535,15 @@ def pre_migrate_distributor(repo_id, distributors, distributor_migrators, repo=N
                       'pulp2_type_id': dist_data.distributor_type_id,
                       'pulp2_last_updated': last_updated,
                       'pulp2_config': dist_data.config,
-                      'pulp2_repository': repo,
                       'pulp2_repo_id': repo_id,
                       'is_migrated': False})
 
+        # this is the case for the simple plan
+        # add native distributor to the repo
+        # this will go away with the simple-complex plan conversion work
+        if not distributors and repo:
+            repo.pulp2_dists.add(distributor)
+
         if not created:
             # if it was marked as such because it was not present in the migration plan
             distributor.not_in_plan = False
@@ -675,7 +680,7 @@ def handle_outdated_resources(plan, type_to_repo_ids):
     )
 
     old_dist_query = Q(pulp3_distribution__isnull=False) | Q(pulp3_publication__isnull=False)
-    old_dist_query &= Q(pulp2_repository__in=repos_with_old_distributions_qs) | Q(not_in_plan=True)
+    old_dist_query &= Q(pulp2_repos__in=repos_with_old_distributions_qs) | Q(not_in_plan=True)
 
     with transaction.atomic():
         pulp2distributors_with_old_distributions_qs = Pulp2Distributor.objects.filter(

pulp_2to3_migration/app/serializers.py

@@ -236,30 +236,8 @@ def get_pulp3_distribution_hrefs(self, obj):
         """
         Get pulp3_distribution_hrefs from pulp3_repository_version
         """
-        dist_list = []
-        rv = obj.pulp3_repository_version
-        if not rv:
-            return dist_list
-        result_qs = rv.publication_set.all()
-        if result_qs:
-            # repo_version  has publication, therefore is a PublicationDistribution
-            for publication in result_qs:
-                distribution_type = f'{publication.pulp_type.replace(".", "_")}distribution'
-                plugin_distribution = getattr(publication, distribution_type)
-                dist_list.extend(plugin_distribution.all())
-        else:
-            # empty result_qs means that repo_version does not need publication,
-            # or repo_version was not published/distributed
-            pulp_type = rv.repository.pulp_type
-            distribution_type = f'{pulp_type.replace(".", "_")}distribution'
-            if hasattr(rv, distribution_type):
-                # repo_version is distributed directly trough RepositoryDistribution
-                plugin_distribution = getattr(rv, distribution_type)
-                dist_list = plugin_distribution.all()
-            else:
-                # repo_version was not published/distributed
-                return dist_list
-        return [get_pulp_href(dist) for dist in dist_list]
+        pulp2dists = obj.pulp2_dists.filter(not_in_plan=False, is_migrated=True)
+        return [get_pulp_href(dist.pulp3_distribution) for dist in pulp2dists]
 
     class Meta:
         fields = ModelSerializer.Meta.fields + (