Sync with metadata mirroring

When synced with "mirror=True", we take the manifest we downloaded and
immediately create a publication with it, rather than regenerating the
whole manifest.

closes: #8851
https://pulp.plan.io/issues/8851

CHANGES/8851.feature

@@ -0,0 +1 @@
+Performing a sync with "mirror=True" will automatically generate a publication at sync-time.

pulp_file/app/tasks/synchronizing.py

@@ -4,21 +4,26 @@
 from gettext import gettext as _
 from urllib.parse import urlparse, urlunparse
 
-from pulpcore.plugin.models import Artifact, ProgressReport, Remote
+from django.core.files import File
+
+from pulpcore.plugin.models import Artifact, ProgressReport, Remote, PublishedMetadata
 from pulpcore.plugin.stages import (
     DeclarativeArtifact,
     DeclarativeContent,
     DeclarativeVersion,
     Stage,
 )
 
-from pulp_file.app.models import FileContent, FileRemote, FileRepository
+from pulp_file.app.models import FileContent, FileRemote, FileRepository, FilePublication
 from pulp_file.manifest import Manifest
 
 
 log = logging.getLogger(__name__)
 
 
+metadata_files = []
+
+
 def synchronize(remote_pk, repository_pk, mirror):
     """
     Sync content from the remote repository.
@@ -42,7 +47,24 @@ def synchronize(remote_pk, repository_pk, mirror):
 
     first_stage = FileFirstStage(remote)
     dv = DeclarativeVersion(first_stage, repository, mirror=mirror)
-    return dv.create()
+    rv = dv.create()
+    if mirror:
+        # TODO: this is awful, we really should rewrite the DeclarativeVersion API to
+        # accomodate this use case
+        global metadata_files
+        with FilePublication.create(rv, pass_through=True) as publication:
+            (mdfile_path, relative_path) = metadata_files.pop()
+            PublishedMetadata.create_from_file(
+                file=File(open(mdfile_path, "rb")),
+                relative_path=relative_path,
+                publication=publication,
+            )
+            publication.manifest = relative_path
+            publication.save()
+
+        log.info(_("Publication: {publication} created").format(publication=publication.pk))
+
+    return rv
 
 
 class FileFirstStage(Stage):
@@ -65,13 +87,16 @@ async def run(self):
         """
         Build and emit `DeclarativeContent` from the Manifest data.
         """
+        global metadata_files
+
         deferred_download = self.remote.policy != Remote.IMMEDIATE  # Interpret download policy
         with ProgressReport(message="Downloading Metadata", code="sync.downloading.metadata") as pb:
             parsed_url = urlparse(self.remote.url)
             root_dir = os.path.dirname(parsed_url.path)
             downloader = self.remote.get_downloader(url=self.remote.url)
             result = await downloader.run()
             pb.increment()
+            metadata_files.append((result.path, self.remote.url.split("/")[-1]))
 
         with ProgressReport(message="Parsing Metadata Lines", code="sync.parsing.metadata") as pb:
             manifest = Manifest(result.path)

pulp_file/tests/functional/api/test_auto_publish.py

@@ -37,7 +37,7 @@ def setUpClass(cls):
         cls.CUSTOM_MANIFEST = "TEST_MANIFEST"
 
     def setUp(self):
-        """Create remote, repo, publish settings, and distribution."""
+        """Create remote, repo, and distribution."""
         self.remote = self.remote_api.create(gen_file_remote())
         self.repo = self.repo_api.create(gen_repo(manifest=self.CUSTOM_MANIFEST, autopublish=True))
         response = self.distributions_api.create(
@@ -48,11 +48,9 @@ def setUp(self):
 
     def tearDown(self):
         """Clean up."""
-        tasks = []
-        tasks.append(self.repo_api.delete(self.repo.pulp_href).task)
-        tasks.append(self.remote_api.delete(self.remote.pulp_href).task)
-        tasks.append(self.distributions_api.delete(self.distribution.pulp_href).task)
-        [monitor_task(task) for task in tasks]
+        monitor_task(self.repo_api.delete(self.repo.pulp_href).task)
+        monitor_task(self.remote_api.delete(self.remote.pulp_href).task)
+        monitor_task(self.distributions_api.delete(self.distribution.pulp_href).task)
 
     def test_01_sync(self):
         """Assert that syncing the repository triggers auto-publish and auto-distribution."""

pulp_file/tests/functional/api/test_sync.py

@@ -18,6 +18,8 @@
     RepositoriesFileApi,
     RepositorySyncURL,
     RemotesFileApi,
+    ContentFilesApi,
+    PublicationsFileApi,
 )
 
 
@@ -82,6 +84,43 @@ def test_sync(self):
         self.assertDictEqual(get_content_summary(repo.to_dict()), FILE_FIXTURE_SUMMARY)
 
 
+class MirrorSyncTestCase(unittest.TestCase):
+    """Do a mirrored sync a repository with the file plugin."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Create class-wide variables."""
+        cls.cfg = config.get_config()
+        cls.client = gen_file_client()
+
+        cls.content_api = ContentFilesApi(cls.client)
+        cls.repo_api = RepositoriesFileApi(cls.client)
+        cls.remote_api = RemotesFileApi(cls.client)
+        cls.publications_api = PublicationsFileApi(cls.client)
+
+    def setUp(self):
+        """Create remote, repo, and distribution."""
+        self.remote = self.remote_api.create(gen_file_remote())
+        self.repo = self.repo_api.create(gen_repo())
+
+    def tearDown(self):
+        """Clean up."""
+        monitor_task(self.repo_api.delete(self.repo.pulp_href).task)
+        monitor_task(self.remote_api.delete(self.remote.pulp_href).task)
+
+    def test_01_sync(self):
+        """Assert that syncing the repository w/ mirror=True creates a publication."""
+        # Sync the repository.
+        repository_sync_data = RepositorySyncURL(remote=self.remote.pulp_href, mirror=True)
+        sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data)
+        task = monitor_task(sync_response.task)
+
+        # Check that all the appropriate resources were created
+        self.assertEqual(len(task.created_resources), 2)
+        self.assertTrue(any(["publication" in resource for resource in task.created_resources]))
+        self.assertTrue(any(["version" in resource for resource in task.created_resources]))
+
+
 class SyncInvalidTestCase(unittest.TestCase):
     """Sync a repository with a given url on the remote."""