diff --git a/pulp_2to3_migration/app/migrations/0003_pulp2lazycatalog_indices.py b/pulp_2to3_migration/app/migrations/0003_pulp2lazycatalog_indices.py new file mode 100644 index 00000000..14fc0ab4 --- /dev/null +++ b/pulp_2to3_migration/app/migrations/0003_pulp2lazycatalog_indices.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.11 on 2020-03-20 13:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pulp_2to3_migration', '0002_remove_pulp2distributor_pulp2_auto_publish'), + ] + + operations = [ + migrations.AddIndex( + model_name='pulp2lazycatalog', + index=models.Index(fields=['pulp2_unit_id'], name='pulp_2to3_m_pulp2_u_c60485_idx'), + ), + migrations.AddIndex( + model_name='pulp2lazycatalog', + index=models.Index(fields=['pulp2_content_type_id'], name='pulp_2to3_m_pulp2_c_766098_idx'), + ), + ] diff --git a/pulp_2to3_migration/app/models/content.py b/pulp_2to3_migration/app/models/content.py index f6f65acb..ed340396 100644 --- a/pulp_2to3_migration/app/models/content.py +++ b/pulp_2to3_migration/app/models/content.py @@ -30,11 +30,6 @@ class Pulp2Content(BaseModel): class Meta: unique_together = ('pulp2_id', 'pulp2_content_type_id') - @property - def detail_model(self): - """Return detail_model.""" - return getattr(self, f'{self.pulp2_content_type_id}_detail_model').get() - class Pulp2to3Content(BaseModel): """ @@ -99,3 +94,7 @@ class Pulp2LazyCatalog(BaseModel): class Meta: unique_together = ('pulp2_storage_path', 'pulp2_importer_id', 'pulp2_revision') + indexes = [ + models.Index(fields=['pulp2_unit_id']), + models.Index(fields=['pulp2_content_type_id']) + ] diff --git a/pulp_2to3_migration/app/plugin/content.py b/pulp_2to3_migration/app/plugin/content.py index 590f0114..8946d82e 100644 --- a/pulp_2to3_migration/app/plugin/content.py +++ b/pulp_2to3_migration/app/plugin/content.py @@ -1,4 +1,5 @@ import asyncio +import functools import logging import os import shutil @@ -29,7 +30,6 @@ from pulp_2to3_migration.app.constants import NOT_USED from pulp_2to3_migration.app.models import ( - Pulp2Content, Pulp2Importer, Pulp2LazyCatalog, ) @@ -155,11 +155,9 @@ async def run(self): If a plugin needs to have more control over the order of content migration, it should override this method. """ - - content_types = self.migrator.content_models.keys() - for ctype in content_types: + for ctype, cmodel in self.migrator.content_models.items(): # we need to go through all content in case any of Remotes changed - pulp2content_qs = Pulp2Content.objects.filter(pulp2_content_type_id=ctype) + pulp2content_qs = cmodel.objects.all().prefetch_related('pulp2content') total_pulp2content = pulp2content_qs.count() with ProgressReport( @@ -184,6 +182,7 @@ async def migrate_to_pulp3(self, batch, pb=None): Args: batch: A batch of Pulp2Content objects to migrate to Pulp 3 """ + @functools.lru_cache(maxsize=20) def get_remote_by_importer_id(importer_id): """ Args: @@ -199,14 +198,14 @@ def get_remote_by_importer_id(importer_id): return return pulp2importer.pulp3_remote - for pulp2content in batch: - pulp_2to3_detail_content = pulp2content.detail_model + for pulp_2to3_detail_content in batch: + pulp2content = pulp_2to3_detail_content.pulp2content # get all Lazy Catalog Entries (LCEs) for this content pulp2lazycatalog = Pulp2LazyCatalog.objects.filter( - pulp2_unit_id=pulp2content.pulp2_id) + pulp2_unit_id=pulp2content.pulp2_id).only('pulp2_importer_id', 'pulp2_url') - if not pulp2lazycatalog and not pulp2content.downloaded: + if not pulp2content.downloaded and not pulp2lazycatalog: _logger.warn(_('On_demand content cannot be migrated without an entry in the lazy ' 'catalog, pulp2 unit_id: {}'.format(pulp2content.pulp2_id))) continue @@ -219,7 +218,7 @@ def get_remote_by_importer_id(importer_id): pulp_2to3_detail_content.expected_size, downloaded=pulp2content.downloaded) # Downloaded content with no LCE - if not pulp2lazycatalog and pulp2content.downloaded: + if pulp2content.downloaded and not pulp2lazycatalog: da = DeclarativeArtifact( artifact=artifact, url=NOT_USED, diff --git a/pulp_2to3_migration/app/plugin/docker/migrator.py b/pulp_2to3_migration/app/plugin/docker/migrator.py index cdfceee4..4628850f 100644 --- a/pulp_2to3_migration/app/plugin/docker/migrator.py +++ b/pulp_2to3_migration/app/plugin/docker/migrator.py @@ -132,8 +132,9 @@ async def migrate_to_pulp3(self, batch, pb=None): batch: A batch of Pulp2Content objects to migrate to Pulp 3 """ - for pulp2content in batch: - pulp_2to3_detail_content = pulp2content.detail_model + for pulp_2to3_detail_content in batch: + pulp2content = pulp_2to3_detail_content.pulp2content + pulp3content = await pulp_2to3_detail_content.create_pulp3_content() future_relations = {'pulp2content': pulp2content} # store digests for future pulp3 content relations diff --git a/pulp_2to3_migration/app/pre_migration.py b/pulp_2to3_migration/app/pre_migration.py index 7da50f85..7be478fb 100644 --- a/pulp_2to3_migration/app/pre_migration.py +++ b/pulp_2to3_migration/app/pre_migration.py @@ -113,7 +113,7 @@ async def pre_migrate_content(content_model, mutable_type, premigrate_hook): fields = set(['id', '_storage_path', '_last_updated', '_content_type_id']) if hasattr(content_model.pulp2, 'downloaded'): fields.add('downloaded') - for i, record in enumerate(mongo_content_qs.only(*fields).batch_size(batch_size)): + for i, record in enumerate(mongo_content_qs.only(*fields).no_cache().batch_size(batch_size)): if record._last_updated == last_updated: # corner case - content with the last``last_updated`` date might be pre-migrated; # check if this content is already pre-migrated @@ -167,7 +167,7 @@ async def pre_migrate_content(content_model, mutable_type, premigrate_hook): pulp2detail_pb.done += content_saved pulp2detail_pb.save() - pulp2content = [] + pulp2content.clear() existing_count = 0 if pulp2mutatedcontent: # when we flip the is_migrated flag to False, we base this decision on the last_unit_added