Fully migrate importers and distributors on every run

Also inspect all importers, distributors in case some of them are requested
to be migrated without a repository.
Make sure RemoteArtifacts are created for already existing content
when old Remotes are removed.

Migrations are squashed due to changes in the importer/distributor models.

closes #5980
https://pulp.plan.io/issues/5980

closes #5852
https://pulp.plan.io/issues/5852

CHANGES/5852.bugfix

@@ -0,0 +1 @@
+Importer or distributor can be migrated even if their repository is not.

CHANGES/5980.bugfix

@@ -0,0 +1 @@
+All requested repositories are migrated regardless of the time of the last run or a migration plan change.

pulp_2to3_migration/app/migration.py

@@ -55,7 +55,7 @@ async def migrate_repositories(plan):
     )
     with ProgressReport(**progress_data) as pb:
         pulp2repos_qs = Pulp2Repository.objects.filter(pulp3_repository_version=None,
-                                                       not_in_pulp2=False)
+                                                       not_in_plan=False)
 
         for plugin in plan.get_plugin_plans():
             repos_to_create = plugin.get_repo_creation_setup()
@@ -119,7 +119,7 @@ async def migrate_importers(plan):
     with ProgressReport(**progress_data) as pb:
         pulp2importers_qs = Pulp2Importer.objects.filter(
             pulp3_remote=None,
-            not_in_pulp2=False)
+            not_in_plan=False)
         pb.total += pulp2importers_qs.count()
         pb.save()
 
@@ -179,7 +179,7 @@ async def migrate_repo_distributor(pb, dist_migrator, pulp2dist, repo_version=No
         pulp2distributors_qs = Pulp2Distributor.objects.filter(
             pulp3_distribution=None,
             pulp3_publication=None,
-            not_in_pulp2=False)
+            not_in_plan=False)
         pb.total = pulp2distributors_qs.count()
         pb.save()
 
@@ -221,14 +221,28 @@ async def create_repo_versions(plan):
     """
     def create_repo_version(migrator, pulp3_repo_name, pulp2_repo, pulp3_remote=None):
         """
-        Create a repo version based on a pulp2 repository
+        Create a repo version based on a pulp2 repository.
+
+        Add a remote to a corresponding pulp 2 repository.
 
         Args:
             migrator: migrator to use, provides repo type information
             pulp3_repo_name(str): repository name in Pulp 3
             pulp2_repo(Pulp2Repository): a pre-migrated repository to create a repo version for
             pulp3_remote(remote): a pulp3 remote
         """
+
+        # Add a remote to every repo, even the migrated one, because remotes are migrated on
+        # every run
+        if pulp3_remote:
+            pulp2_repo.pulp3_repository_remote = pulp3_remote
+        # pulp2importer might not be migrated, e.g. config was empty
+        elif hasattr(pulp2_repo, 'pulp2importer'):
+            pulp2_repo.pulp3_repository_remote = pulp2_repo.pulp2importer.pulp3_remote
+        if pulp2_repo.is_migrated:
+            pulp2_repo.save()
+            return
+
         repository_class = migrator.pulp3_repository
         pulp3_repo = repository_class.objects.get(name=pulp3_repo_name)
         unit_ids = Pulp2RepoContent.objects.filter(pulp2_repository=pulp2_repo).values_list(
@@ -246,42 +260,35 @@ def create_repo_version(migrator, pulp3_repo_name, pulp2_repo, pulp3_remote=None
             pulp2_repo.pulp3_repository_version = new_version
         if not pulp2_repo.pulp3_repository_version:
             pulp2_repo.pulp3_repository_version = pulp3_repo.latest_version()
-        if pulp3_remote:
-            pulp2_repo.pulp3_repository_remote = pulp3_remote
-        # pulp2importer might not be migrated, e.g. config was empty
-        elif hasattr(pulp2_repo, 'pulp2importer'):
-            pulp2_repo.pulp3_repository_remote = pulp2_repo.pulp2importer.pulp3_remote
         pulp2_repo.is_migrated = True
         pulp2_repo.save()
 
     for plugin in plan.get_plugin_plans():
         pulp3_repo_setup = plugin.get_repo_creation_setup()
         if not pulp3_repo_setup:
-            # create one repo version for each pulp 2 repo
-            repos_to_migrate = Pulp2Repository.objects.filter(is_migrated=False,
-                                                              type=plugin.type,
-                                                              not_in_pulp2=False)
+            repos_to_migrate = Pulp2Repository.objects.filter(type=plugin.type,
+                                                              not_in_plan=False)
             for pulp2_repo in repos_to_migrate:
+                # Create one repo version for each pulp 2 repo if needed.
                 create_repo_version(plugin.migrator, pulp2_repo.pulp2_repo_id, pulp2_repo)
         else:
             for repo_name in pulp3_repo_setup:
                 repo_versions_setup = pulp3_repo_setup[repo_name]['versions']
                 pulp2_importer_repo_id = \
                     pulp3_repo_setup[repo_name]['pulp2_importer_repository_id']
-                pulp2_importer_repo = Pulp2Repository.objects.get(
+                pulp2_importer = Pulp2Importer.objects.get(
                     pulp2_repo_id=pulp2_importer_repo_id
                 )
                 for pulp2_repo_id in repo_versions_setup:
                     try:
-                        repo_to_migrate = Pulp2Repository.objects.get(pulp2_repo_id=pulp2_repo_id,
-                                                                      is_migrated=False,
-                                                                      not_in_pulp2=False)
+                        pulp2_repo = Pulp2Repository.objects.get(pulp2_repo_id=pulp2_repo_id,
+                                                                 not_in_plan=False)
                     except Pulp2Repository.DoesNotExist:
-                        # already migrated
+                        # not in Pulp 2 anymore
                         continue
                     else:
                         # it's possible to have a random order of the repo versions (after migration
                         # re-run, a repo can be changed in pulp 2 and it might not be for the last
                         # repo version)
-                        create_repo_version(plugin.migrator, repo_name, repo_to_migrate,
-                                            pulp2_importer_repo.pulp2importer.pulp3_remote)
+                        create_repo_version(plugin.migrator, repo_name, pulp2_repo,
+                                            pulp2_importer.pulp3_remote)

pulp_2to3_migration/app/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 2.2.7 on 2019-12-02 20:44
+# Generated by Django 2.2.9 on 2020-01-22 15:53
 
 import django.contrib.postgres.fields
 import django.contrib.postgres.fields.jsonb
@@ -12,7 +12,7 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('core', '0018_auto_20191127_2350'),
+        ('core', '0019_add_signing_service_model'),
     ]
 
     operations = [
@@ -57,7 +57,9 @@ class Migration(migrations.Migration):
                 ('pulp2_last_unit_added', models.DateTimeField(null=True)),
                 ('pulp2_last_unit_removed', models.DateTimeField(null=True)),
                 ('is_migrated', models.BooleanField(default=False)),
+                ('not_in_plan', models.BooleanField(default=False)),
                 ('type', models.CharField(max_length=25)),
+                ('pulp3_repository_remote', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Remote')),
                 ('pulp3_repository_version', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.RepositoryVersion')),
             ],
             options={
@@ -91,8 +93,10 @@ class Migration(migrations.Migration):
                 ('pulp2_type_id', models.CharField(max_length=255)),
                 ('pulp2_config', django.contrib.postgres.fields.jsonb.JSONField()),
                 ('pulp2_last_updated', models.DateTimeField()),
+                ('pulp2_repo_id', models.TextField()),
                 ('is_migrated', models.BooleanField(default=False)),
-                ('pulp2_repository', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='pulp_2to3_migration.Pulp2Repository')),
+                ('not_in_plan', models.BooleanField(default=False)),
+                ('pulp2_repository', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='pulp_2to3_migration.Pulp2Repository')),
                 ('pulp3_remote', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Remote')),
             ],
             options={
@@ -192,10 +196,12 @@ class Migration(migrations.Migration):
                 ('pulp2_config', django.contrib.postgres.fields.jsonb.JSONField()),
                 ('pulp2_auto_publish', models.BooleanField()),
                 ('pulp2_last_updated', models.DateTimeField()),
+                ('pulp2_repo_id', models.TextField()),
                 ('is_migrated', models.BooleanField(default=False)),
-                ('pulp2_repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pulp_2to3_migration.Pulp2Repository')),
+                ('not_in_plan', models.BooleanField(default=False)),
+                ('pulp2_repository', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='pulp_2to3_migration.Pulp2Repository')),
                 ('pulp3_distribution', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.BaseDistribution')),
-                ('pulp3_publication', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Publication')),
+                ('pulp3_publication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Publication')),
             ],
             options={
                 'unique_together': {('pulp2_repository', 'pulp2_id')},

pulp_2to3_migration/app/models/repository.py

@@ -26,8 +26,7 @@ class Pulp2Repository(BaseModel):
             a repository in Pulp 2
         is_migrated (models.BooleanField): True if a resource has been migrated to Pulp 3; False
             if it's never been migrated or if it's been updated since the last migration run.
-        not_in_pulp2 (models.BooleanField): True if a resource is no longer present in Pulp 2 at
-            the time of the last migration run; False if it's present in Pulp2.
+        not_in_plan (models.BooleanField): True if a resource is not a part of the migration plan.
         type (models.CharField): repo type in Pulp 2
 
     Relations:
@@ -41,7 +40,7 @@ class Pulp2Repository(BaseModel):
     pulp2_last_unit_added = models.DateTimeField(null=True)
     pulp2_last_unit_removed = models.DateTimeField(null=True)
     is_migrated = models.BooleanField(default=False)
-    not_in_pulp2 = models.BooleanField(default=False)
+    not_in_plan = models.BooleanField(default=False)
     type = models.CharField(max_length=25)
 
     pulp3_repository_version = models.OneToOneField(RepositoryVersion,
@@ -86,10 +85,10 @@ class Pulp2Importer(BaseModel):
         pulp2_type_id (models.CharField): Id of importer type in Pulp 2
         pulp2_config (JSONField): Pulp 2 importer config in JSON format
         pulp2_last_updated (models.DateTimeField): Last time the importer was updated
+        pulp2_repo_id (models.TextField): Id of a repo in Pulp 2 an importer belongs to
         is_migrated (models.BooleanField): True if a resource has been migrated to Pulp 3; False
             if it's never been migrated or if it's been updated since the last migration run.
-        not_in_pulp2 (models.BooleanField): True if a resource is no longer present in Pulp 2 at
-            the time of the last migration run; False if it's present in Pulp2.
+        not_in_plan (models.BooleanField): True if a resource is not a part of the migration plan.
 
     Relations:
         pulp2_repository (models.OneToOneField): Pulp 2 repository this importer belongs to
@@ -99,10 +98,11 @@ class Pulp2Importer(BaseModel):
     pulp2_type_id = models.CharField(max_length=255)
     pulp2_config = JSONField()
     pulp2_last_updated = models.DateTimeField()
+    pulp2_repo_id = models.TextField()
     is_migrated = models.BooleanField(default=False)
-    not_in_pulp2 = models.BooleanField(default=False)
+    not_in_plan = models.BooleanField(default=False)
 
-    pulp2_repository = models.OneToOneField(Pulp2Repository, on_delete=models.CASCADE)
+    pulp2_repository = models.OneToOneField(Pulp2Repository, on_delete=models.CASCADE, null=True)
     pulp3_remote = models.OneToOneField(Remote, on_delete=models.SET_NULL, null=True)
 
 
@@ -116,10 +116,10 @@ class Pulp2Distributor(BaseModel):
         pulp2_config (JSONField): Pulp 2 distributor config in JSON format
         pulp2_auto_publish (models.BooleanField): A flag to determine if auto-publish is enabled
         pulp2_last_updated (models.DateTimeField): Last time the distributor was updated
+        pulp2_repo_id (models.TextField): Id of a repo in Pulp 2 a distributor belongs to
         is_migrated (models.BooleanField): True if a resource has been migrated to Pulp 3; False
             if it's never been migrated or if it's been updated since the last migration run.
-        not_in_pulp2 (models.BooleanField): True if a resource is no longer present in Pulp 2 at
-            the time of the last migration run; False if it's present in Pulp2.
+        not_in_plan (models.BooleanField): True if a resource is not a part of the migration plan.
 
     Relations:
         pulp2_repository (models.ForeignKey): Pulp 2 repository this distributor belongs to
@@ -134,10 +134,11 @@ class Pulp2Distributor(BaseModel):
     pulp2_config = JSONField()
     pulp2_auto_publish = models.BooleanField()
     pulp2_last_updated = models.DateTimeField()
+    pulp2_repo_id = models.TextField()
     is_migrated = models.BooleanField(default=False)
-    not_in_pulp2 = models.BooleanField(default=False)
+    not_in_plan = models.BooleanField(default=False)
 
-    pulp2_repository = models.ForeignKey(Pulp2Repository, on_delete=models.CASCADE)
+    pulp2_repository = models.ForeignKey(Pulp2Repository, on_delete=models.CASCADE, null=True)
     pulp3_publication = models.ForeignKey(Publication, on_delete=models.SET_NULL, null=True)
     pulp3_distribution = models.OneToOneField(BaseDistribution,
                                               on_delete=models.SET_NULL,

pulp_2to3_migration/app/plugin/content.py

@@ -159,9 +159,8 @@ async def run(self):
 
         content_types = self.migrator.content_models.keys()
         for ctype in content_types:
-
-            pulp2content_qs = Pulp2Content.objects.filter(pulp2_content_type_id=ctype,
-                                                          pulp3_content=None)
+            # we need to go through all content in case any of Remotes changed
+            pulp2content_qs = Pulp2Content.objects.filter(pulp2_content_type_id=ctype)
             total_pulp2content = pulp2content_qs.count()
 
             # determine the batch size if we can have up to 36 coroutines and the number
@@ -195,6 +194,8 @@ async def migrate_to_pulp3(self, batch, pb=None):
 
         Plugin writers might want to override this method if it doesn't satisfy their needs as is.
 
+        In this implementation there is an assumption that each content has one artifact.
+
         Args:
             batch: A batch of Pulp2Content objects to migrate to Pulp 3
         """
@@ -215,8 +216,6 @@ def get_remote_by_importer_id(importer_id):
 
         for pulp2content in batch:
             pulp_2to3_detail_content = pulp2content.detail_model
-            pulp3content = pulp_2to3_detail_content.create_pulp3_content()
-            future_relations = {'pulp2content': pulp2content}
 
             # get all Lazy Catalog Entries (LCEs) for this content
             pulp2lazycatalog = Pulp2LazyCatalog.objects.filter(
@@ -227,11 +226,13 @@ def get_remote_by_importer_id(importer_id):
                                'catalog, pulp2 unit_id: {}'.format(pulp2content.pulp2_id)))
                 break
 
+            pulp3content = pulp_2to3_detail_content.create_pulp3_content()
+            future_relations = {'pulp2content': pulp2content}
+
             artifact = await self.create_artifact(pulp2content.pulp2_storage_path,
                                                   pulp_2to3_detail_content.expected_digests,
                                                   pulp_2to3_detail_content.expected_size,
                                                   downloaded=pulp2content.downloaded)
-
             # Downloaded content with no LCE
             if not pulp2lazycatalog and pulp2content.downloaded:
                 da = DeclarativeArtifact(

pulp_2to3_migration/app/plugin/docker/repository.py

@@ -7,6 +7,7 @@ class DockerImporter(Pulp2to3Importer):
     """
     Interface to migrate Pulp 2 Docker importer
     """
+    pulp3_remote_models = [ContainerRemote]
 
     @classmethod
     async def migrate_to_pulp3(cls, pulp2importer):
@@ -32,6 +33,7 @@ class DockerDistributor(Pulp2to3Distributor):
     """
     Interface to migrate Pulp 2 Docker distributor
     """
+    pulp3_distribution_models = [ContainerDistribution]
 
     @classmethod
     async def migrate_to_pulp3(cls, pulp2distributor, repo_version):

pulp_2to3_migration/app/plugin/iso/repository.py

@@ -13,6 +13,8 @@ class IsoImporter(Pulp2to3Importer):
     """
     Interface to migrate Pulp 2 ISO importer
     """
+    pulp3_remote_models = [FileRemote]
+
     @classmethod
     async def migrate_to_pulp3(cls, pulp2importer):
         """
@@ -34,6 +36,9 @@ class IsoDistributor(Pulp2to3Distributor):
     """
     Interface to migrate Pulp 2 ISO distributor
     """
+    pulp3_publication_models = [FilePublication]
+    pulp3_distribution_models = [FileDistribution]
+
     @classmethod
     async def migrate_to_pulp3(cls, pulp2distributor, repo_version):
         """
@@ -47,6 +52,7 @@ async def migrate_to_pulp3(cls, pulp2distributor, repo_version):
             created(bool): True if Distribution has just been created; False if Distribution
                            is an existing one
         """
+
         if not repo_version:
             repo_version = pulp2distributor.pulp2_repository.pulp3_repository_version
         publication = repo_version.publication_set.first()