From acb5be67d0a548f5ea72140b8d866e9207519eed Mon Sep 17 00:00:00 2001 From: Grant Gainey Date: Mon, 31 Jul 2023 12:40:14 -0400 Subject: [PATCH] Reset export to using a set() for Artifacts. The QuerySet approach introduced a subtle bug the breaks pulp_rpm export. If we want to retain the 'good parts' of that approach, it will need to be its own investigation/PR. fixes #4210. (cherry picked from commit 3194be5b76ba4a97cb07759de507e7ecbdef0b38) --- CHANGES/4210.bugfix | 1 + pulpcore/app/importexport.py | 5 ++--- pulpcore/app/tasks/export.py | 11 ++++------- 3 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 CHANGES/4210.bugfix diff --git a/CHANGES/4210.bugfix b/CHANGES/4210.bugfix new file mode 100644 index 0000000000..8fe562e209 --- /dev/null +++ b/CHANGES/4210.bugfix @@ -0,0 +1 @@ +Fix a subtle export bug introduced from the optimizations in #4159. diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py index 0f97107794..3960f4226b 100644 --- a/pulpcore/app/importexport.py +++ b/pulpcore/app/importexport.py @@ -94,14 +94,14 @@ def export_artifacts(export, artifacts): Args: export (django.db.models.PulpExport): export instance that's doing the export - artifacts (django.db.models.Artifacts): QuerySet of artifacts in all repos being exported + artifacts (django.db.models.Artifacts): Set of artifacts in all repos being exported Raises: ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting """ data = dict(message="Exporting Artifacts", code="export.artifacts", total=len(artifacts)) with ProgressReport(**data) as pb: - for artifact in artifacts.iterator(): # chunk_size= defaults to 2000 at a fetch + for artifact in pb.iter(artifacts): dest = artifact.file.name if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": with tempfile.TemporaryDirectory(dir=".") as temp_dir: @@ -112,7 +112,6 @@ def export_artifacts(export, artifacts): export.tarfile.add(temp_file.name, dest) else: export.tarfile.add(artifact.file.path, dest) - pb.increment() resource = ArtifactResource() resource.queryset = artifacts diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py index 3245c42fbe..03a99f0d0b 100644 --- a/pulpcore/app/tasks/export.py +++ b/pulpcore/app/tasks/export.py @@ -500,7 +500,7 @@ def _do_export(pulp_exporter, tar, the_export): starting_versions = _get_starting_versions(do_incremental, pulp_exporter, the_export) vers_match = _version_match(ending_versions, starting_versions) # Gather up versions and artifacts - artifacts = None # Will be a QuerySet selecting the Artifacts that need to be exported + artifacts = set() for version in ending_versions: # Check version-content to make sure we're not being asked to export # an on_demand repo @@ -509,14 +509,11 @@ def _do_export(pulp_exporter, tar, the_export): raise RuntimeError(_("Remote artifacts cannot be exported.")) if do_incremental: - vers_artifacts = version.artifacts.difference(vers_match[version].artifacts) + vers_artifacts = version.artifacts.difference(vers_match[version].artifacts).all() else: - vers_artifacts = version.artifacts + vers_artifacts = version.artifacts.all() - if artifacts: - artifacts.union(vers_artifacts) - else: - artifacts = vers_artifacts + artifacts.update(vers_artifacts) # export plugin-version-info export_versions(the_export, plugin_version_info)