Merge pull request #122 from goosemania/perf-cp

Performance cherry-picks for 0.1

pulp_2to3_migration/app/migrations/0003_pulp2lazycatalog_indices.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.11 on 2020-03-20 13:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pulp_2to3_migration', '0002_remove_pulp2distributor_pulp2_auto_publish'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='pulp2lazycatalog',
+            index=models.Index(fields=['pulp2_unit_id'], name='pulp_2to3_m_pulp2_u_c60485_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='pulp2lazycatalog',
+            index=models.Index(fields=['pulp2_content_type_id'], name='pulp_2to3_m_pulp2_c_766098_idx'),
+        ),
+    ]

pulp_2to3_migration/app/models/content.py

@@ -30,11 +30,6 @@ class Pulp2Content(BaseModel):
     class Meta:
         unique_together = ('pulp2_id', 'pulp2_content_type_id')
 
-    @property
-    def detail_model(self):
-        """Return detail_model."""
-        return getattr(self, f'{self.pulp2_content_type_id}_detail_model').get()
-
 
 class Pulp2to3Content(BaseModel):
     """
@@ -99,3 +94,7 @@ class Pulp2LazyCatalog(BaseModel):
 
     class Meta:
         unique_together = ('pulp2_storage_path', 'pulp2_importer_id', 'pulp2_revision')
+        indexes = [
+            models.Index(fields=['pulp2_unit_id']),
+            models.Index(fields=['pulp2_content_type_id'])
+        ]

pulp_2to3_migration/app/plugin/content.py

@@ -1,4 +1,5 @@
 import asyncio
+import functools
 import logging
 import os
 import shutil
@@ -29,7 +30,6 @@
 
 from pulp_2to3_migration.app.constants import NOT_USED
 from pulp_2to3_migration.app.models import (
-    Pulp2Content,
     Pulp2Importer,
     Pulp2LazyCatalog,
 )
@@ -155,11 +155,9 @@ async def run(self):
         If a plugin needs to have more control over the order of content migration, it should
         override this method.
         """
-
-        content_types = self.migrator.content_models.keys()
-        for ctype in content_types:
+        for ctype, cmodel in self.migrator.content_models.items():
             # we need to go through all content in case any of Remotes changed
-            pulp2content_qs = Pulp2Content.objects.filter(pulp2_content_type_id=ctype)
+            pulp2content_qs = cmodel.objects.all().prefetch_related('pulp2content')
             total_pulp2content = pulp2content_qs.count()
 
             with ProgressReport(
@@ -184,6 +182,7 @@ async def migrate_to_pulp3(self, batch, pb=None):
         Args:
             batch: A batch of Pulp2Content objects to migrate to Pulp 3
         """
+        @functools.lru_cache(maxsize=20)
         def get_remote_by_importer_id(importer_id):
             """
             Args:
@@ -199,14 +198,14 @@ def get_remote_by_importer_id(importer_id):
                 return
             return pulp2importer.pulp3_remote
 
-        for pulp2content in batch:
-            pulp_2to3_detail_content = pulp2content.detail_model
+        for pulp_2to3_detail_content in batch:
+            pulp2content = pulp_2to3_detail_content.pulp2content
 
             # get all Lazy Catalog Entries (LCEs) for this content
             pulp2lazycatalog = Pulp2LazyCatalog.objects.filter(
-                pulp2_unit_id=pulp2content.pulp2_id)
+                pulp2_unit_id=pulp2content.pulp2_id).only('pulp2_importer_id', 'pulp2_url')
 
-            if not pulp2lazycatalog and not pulp2content.downloaded:
+            if not pulp2content.downloaded and not pulp2lazycatalog:
                 _logger.warn(_('On_demand content cannot be migrated without an entry in the lazy '
                                'catalog, pulp2 unit_id: {}'.format(pulp2content.pulp2_id)))
                 continue
@@ -219,7 +218,7 @@ def get_remote_by_importer_id(importer_id):
                                                   pulp_2to3_detail_content.expected_size,
                                                   downloaded=pulp2content.downloaded)
             # Downloaded content with no LCE
-            if not pulp2lazycatalog and pulp2content.downloaded:
+            if pulp2content.downloaded and not pulp2lazycatalog:
                 da = DeclarativeArtifact(
                     artifact=artifact,
                     url=NOT_USED,

pulp_2to3_migration/app/plugin/docker/migrator.py

@@ -132,8 +132,9 @@ async def migrate_to_pulp3(self, batch, pb=None):
             batch: A batch of Pulp2Content objects to migrate to Pulp 3
         """
 
-        for pulp2content in batch:
-            pulp_2to3_detail_content = pulp2content.detail_model
+        for pulp_2to3_detail_content in batch:
+            pulp2content = pulp_2to3_detail_content.pulp2content
+
             pulp3content = await pulp_2to3_detail_content.create_pulp3_content()
             future_relations = {'pulp2content': pulp2content}
             # store digests for future pulp3 content relations

pulp_2to3_migration/app/pre_migration.py

@@ -113,7 +113,7 @@ async def pre_migrate_content(content_model, mutable_type, premigrate_hook):
     fields = set(['id', '_storage_path', '_last_updated', '_content_type_id'])
     if hasattr(content_model.pulp2, 'downloaded'):
         fields.add('downloaded')
-    for i, record in enumerate(mongo_content_qs.only(*fields).batch_size(batch_size)):
+    for i, record in enumerate(mongo_content_qs.only(*fields).no_cache().batch_size(batch_size)):
         if record._last_updated == last_updated:
             # corner case - content with the last``last_updated`` date might be pre-migrated;
             # check if this content is already pre-migrated
@@ -167,7 +167,7 @@ async def pre_migrate_content(content_model, mutable_type, premigrate_hook):
             pulp2detail_pb.done += content_saved
             pulp2detail_pb.save()
 
-            pulp2content = []
+            pulp2content.clear()
             existing_count = 0
     if pulp2mutatedcontent:
         # when we flip the is_migrated flag to False, we base this decision on the last_unit_added