Merge pull request #287 from negillett/24551

Don't flush UD cache for repos without eng ID [RHELDST-24551]

src/pubtools/_pulp/tasks/common.py

@@ -96,8 +96,10 @@ def flush_ud(self, repos, errata=None):
             return out
 
         for repo in repos:
-            out.append(client.flush_repo(repo.id))
+            # RHELDST-24551: UD can't flush cache of repos that have no eng product ID.
+            # Ensure this condition is met before flushing.
             if repo.eng_product_id:
+                out.append(client.flush_repo(repo.id))
                 out.append(client.flush_product(repo.eng_product_id))
 
         out.extend([client.flush_erratum(erratum.id) for erratum in (errata or [])])

tests/logs/copy_repo/test_copy_repo/test_copy_invalid_content_type.jsonl

@@ -0,0 +1,4 @@
+{"event": {"type": "check-repos-start"}}
+{"event": {"type": "check-repos-end"}}
+{"event": {"type": "copy-content-start"}}
+{"event": {"type": "copy-content-error"}}

tests/logs/copy_repo/test_copy_repo/test_copy_invalid_content_type.txt

@@ -0,0 +1,6 @@
+[    INFO] Check repos: started
+[    INFO] Check repos: finished
+[    INFO] Copy content: started
+[   ERROR] Unsupported content type: container
+[   ERROR] Copy content: failed
+# Raised: 30

tests/logs/publish/test_publish/test_repo_publish_cache_cleanup_skip_ud.jsonl

@@ -0,0 +1,10 @@
+{"event": {"type": "check-repos-start"}}
+{"event": {"type": "check-repos-end"}}
+{"event": {"type": "publish-start"}}
+{"event": {"type": "publish-end"}}
+{"event": {"type": "flush-cdn-cache-start"}}
+{"event": {"type": "flush-cdn-cache-end"}}
+{"event": {"type": "set-cdn_published-start"}}
+{"event": {"type": "set-cdn_published-end"}}
+{"event": {"type": "flush-ud-cache-start"}}
+{"event": {"type": "flush-ud-cache-end"}}

tests/logs/publish/test_publish/test_repo_publish_cache_cleanup_skip_ud.txt

@@ -0,0 +1,19 @@
+[    INFO] Check repos: started
+[    INFO] Check repos: finished
+[    INFO] Publish: started
+[    INFO] Publishing repo1
+[    INFO] Publishing repo4
+[    INFO] Publish: finished
+[    INFO] Flush CDN cache: started
+[    INFO] Flushing cache for repo1:
+[    INFO]    https://cdn.example.com/content/unit/1/client/mutable1
+[    INFO]    https://cdn.example.com/content/unit/1/client/mutable2
+[    INFO] Flushing cache for repo4:
+[    INFO]    https://cdn.example.com/content/unit/4/client/mutable1
+[    INFO]    https://cdn.example.com/content/unit/4/client/mutable2
+[    INFO] Flush CDN cache: finished
+[    INFO] Set cdn_published: started
+[    INFO] Set cdn_published: finished
+[    INFO] Flush UD cache: started
+[    INFO] Flush UD cache: finished
+[    INFO] Publishing repositories completed

tests/publish/test_publish.py

@@ -150,9 +150,26 @@ def _add_repo(controller):
         relative_url="content/unit/3/client",
     )
 
+    dt4 = datetime(2019, 9, 9, 0, 0, 0)
+    d4 = Distributor(
+        id="cdn_distributor",
+        type_id="rpm_rsync_distributor",
+        repo_id="repo4",
+        last_publish=dt4,
+        relative_url="content/unit/4/client",
+    )
+    repo4 = Repository(
+        id="repo4",
+        # omit eng ID -- UD can't flush this repo
+        distributors=[d4],
+        relative_url="content/unit/4/client",
+        mutable_urls=["mutable1", "mutable2"],
+    )
+
     controller.insert_repository(repo1)
     controller.insert_repository(repo2)
     controller.insert_repository(repo3)
+    controller.insert_repository(repo4)
 
 
 def test_nonexist_repos(command_tester):
@@ -238,6 +255,51 @@ def test_repo_publish_cache_cleanup(command_tester):
     assert fake_publish.udcache_client.flushed_repos == ["repo1"]
 
 
+def test_repo_publish_cache_cleanup_skip_ud(command_tester):
+    """publishes the repo provided, doesn't clean up UD cache if repo missing eng ID"""
+    with FakePublish() as fake_publish:
+        fake_pulp = fake_publish.pulp_client_controller
+        _add_repo(fake_pulp)
+
+        command_tester.test(
+            fake_publish.main,
+            [
+                "test-publish",
+                "--pulp-url",
+                "https://pulp.example.com",
+                "--fastpurge-host",
+                "fakehost-xxx.example.net",
+                "--fastpurge-client-secret",
+                "abcdef",
+                "--fastpurge-client-token",
+                "efg",
+                "--fastpurge-access-token",
+                "tok",
+                "--fastpurge-root-url",
+                "https://cdn.example.com/",
+                "--udcache-url",
+                "https://ud.example.com/",
+                "--repo-ids",
+                "repo1,repo4",
+            ],
+        )
+
+    # pulp repo is published
+    assert [hist.repository.id for hist in fake_pulp.publish_history] == [
+        "repo1",
+        "repo4",
+    ]
+    # flushed the urls
+    assert sorted(fake_publish.fastpurge_client.purged_urls) == [
+        "https://cdn.example.com/content/unit/1/client/mutable1",
+        "https://cdn.example.com/content/unit/1/client/mutable2",
+        "https://cdn.example.com/content/unit/4/client/mutable1",
+        "https://cdn.example.com/content/unit/4/client/mutable2",
+    ]
+    # should not flush the UD object for repo4 because it's missing an eng ID
+    assert fake_publish.udcache_client.flushed_repos == ["repo1"]
+
+
 def test_repo_publish_cache_cleanup_with_arl(command_tester):
     """publishes the repo provided, cleans up UD cache and CDN cache also by ARLs"""
     with FakePublish() as fake_publish: