Merge 5ff0684 into 34d6e4d

tests/test_pulp.py

@@ -40,6 +40,7 @@ def fixture_search_repo_response():
         "notes": {
             "arch": "x86_64",
             "content_set": "test_repo-source-rpms",
+            "platform": "ubi",
             "platform_full_version": "7",
         },
         "distributors": [{
@@ -60,6 +61,7 @@ def fixture_mock_repo():
             ("dist_id_1", "dist_type_id_1"),
             ("dist_id_2", "dist_type_id_2"),
         ],
+        ubi_population=None,
     )
 
 
@@ -181,6 +183,8 @@ def test_search_repo_by_cs(mock_pulp, mock_search_repos):
     assert repo.content_set == "test_repo-source-rpms"
     assert repo.platform_full_version == "7"
     assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id")
+    # ubi_population note was unset, so it should default to True
+    assert repo.ubi_population is True
 
 
 def test_search_repo_by_id(mock_pulp, mock_search_repos):
@@ -194,6 +198,8 @@ def test_search_repo_by_id(mock_pulp, mock_search_repos):
     assert repo.content_set == "test_repo-source-rpms"
     assert repo.platform_full_version == "7"
     assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id")
+    # ubi_population note was unset, so it should default to True
+    assert repo.ubi_population is True
 
 
 def test_publish_repo(mock_pulp, mock_publish, mock_repo):

tests/test_ubipop.py

@@ -67,6 +67,7 @@ def get_test_repo(**kwargs):
         kwargs.get('content_set'),
         kwargs.get('platform_full_version'),
         kwargs.get('distributors_ids_type_ids'),
+        kwargs.get('ubi_population'),
     )
 
 
@@ -105,6 +106,86 @@ def test_get_output_repo_ids_no_debug(ubi_repo_set_no_debug):
     assert repo_ids == set(["ubi-foo-rpms", "ubi-foo-source"])
 
 
+@patch("ubipop.UbiPopulateRunner")
+@patch("ubipop._pulp_client.Pulp.search_repo_by_cs")
+def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner, caplog):
+    # Don't actually query Pulp for repos
+    mocked_search_repo_by_cs.side_effect = [
+        # Input repos - rhel-8-for-x86_64-appstream
+        [get_test_repo(
+            repo_id="rhel-8-for-x86_64-appstream-rpms",
+            content_set="rhel-8-for-x86_64-appstream-rpms",
+        ), ],
+        [get_test_repo(
+            repo_id="rhel-8-for-x86_64-appstream-source-rpms",
+            content_set="rhel-8-for-x86_64-appstream-source-rpms",
+        ), ],
+        [get_test_repo(
+            repo_id="rhel-8-for-x86_64-appstream-debug-rpms",
+            content_set="rhel-8-for-x86_64-appstream-debug-rpms",
+        ), ],
+
+        # Output repos - rhel-8-for-x86_64-appstream
+        [get_test_repo(
+            repo_id="ubi-8-for-x86_64-appstream-rpms",
+            content_set="ubi-8-for-x86_64-appstream-rpms",
+            ubi_population=True
+        ), ],
+        [get_test_repo(
+            repo_id="ubi-8-for-x86_64-appstream-source-rpms",
+            content_set="ubi-8-for-x86_64-appstream-source-rpms",
+            ubi_population=True
+        ), ],
+        [get_test_repo(
+            repo_id="ubi-8-for-x86_64-appstream-debug-rpms",
+            content_set="ubi-8-for-x86_64-appstream-debug-rpms",
+            ubi_population=True
+        ), ],
+
+        # Input repos - rhel-7-server
+        [get_test_repo(
+            repo_id="rhel-7-server-rpms__7_DOT_2__x86_64",
+            content_set="rhel-7-server-rpms",
+        ), ],
+        [get_test_repo(
+            repo_id="rhel-7-server-source-rpms__7_DOT_2__x86_64",
+            content_set="rhel-7-server-source-rpms",
+        ), ],
+        [get_test_repo(
+            repo_id="rhel-7-server-debuginfo-rpms__7_DOT_2__x86_64",
+            content_set="rhel-7-server-debuginfo-rpms",
+        ), ],
+
+        # Output repos - rhel-7-server
+        [get_test_repo(
+            repo_id="ubi-7-server-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-rpms",
+            ubi_population=True
+        ), ],
+        [get_test_repo(
+            repo_id="ubi-7-server-source-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-source-rpms",
+            ubi_population=False
+        ), ],
+        [get_test_repo(
+            repo_id="ubi-7-server-debuginfo-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-debuginfo-rpms",
+            ubi_population=False
+        ), ],
+    ]
+
+    # Attempt to populate both invalid and valid repo sets
+    ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=TEST_DATA_DIR)
+    ubipop.populate_ubi_repos()
+
+    # Should've only run once
+    assert mocked_ubipop_runner.call_count == 1
+    # For rhel-8-for-x86_64-appstream
+    assert "Skipping rhel-8-for-x86_64-appstream" not in caplog.text
+    # Not for rhel-7-server
+    assert "Skipping rhel-7-server-rpms" in caplog.text
+
+
 def test_get_packages_from_module_by_name(mock_ubipop_runner):
     package_name = "postgresql"
     input_modules = [

tests/test_utils.py

@@ -16,15 +16,15 @@
 
 def test_raise_not_implemented_pulp_action():
     units = ["unit1", "unit2"]
-    repo = Repo("test", "test-rpms", "1", "2", None)
+    repo = Repo("test", "1", "test-rpms", "2", None, None)
     action = PulpAction(units, repo)
     pytest.raises(NotImplementedError, action.get_action, None)
 
 
 def test_raise_not_implemented_associate_action():
     units = ["unit1", "unit2"]
-    repo = Repo("test", "test-rpms", "1", "2", None)
-    src_repo = Repo("test", "test-rpms", "1", "2", None)
+    repo = Repo("test", "1", "test-rpms", "2", None, None)
+    src_repo = Repo("test", "1", "test-rpms", "2", None, None)
     action = AssociateAction(units, repo, src_repo)
     pytest.raises(NotImplementedError, action.get_action, None)
 
@@ -36,8 +36,8 @@ def test_raise_not_implemented_associate_action():
 ])
 def test_get_action_associate(klass, method):
     units = ["unit1", "unit2"]
-    dst_repo = Repo("test_dst", "test_dst-rpms", "1", "2", None)
-    src_repo = Repo("test_src", "test_src-rpms", "1", "2", None)
+    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None)
+    src_repo = Repo("test_src", "1", "test_src-rpms", "2", None, None)
     action = klass(units, dst_repo, src_repo)
     associate_action, src_repo_current, dst_repo_current, current_units = \
         action.get_action(MagicMock())
@@ -55,7 +55,7 @@ def test_get_action_associate(klass, method):
 ])
 def test_get_action_unassociate(klass, method):
     units = ["unit1", "unit2"]
-    dst_repo = Repo("test_dst", "test_dst-rpms", "1", "2", None)
+    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None)
     action = klass(units, dst_repo)
     associate_action, dst_repo_current, current_units = action.get_action(MagicMock())
 

ubipop/__init__.py

@@ -162,6 +162,7 @@ def _get_input_and_output_repo_pairs(self, ubiconfig_item):
         rpms_cs = ubiconfig_item.content_sets.rpm
         source_cs = ubiconfig_item.content_sets.srpm
         debug_cs = ubiconfig_item.content_sets.debuginfo
+
         _LOG.info(
             "Getting input repos for input content sets:\n\t%s\n\t%s\n\t%s",
             rpms_cs.input, source_cs.input, debug_cs.input)
@@ -180,19 +181,26 @@ def _get_input_and_output_repo_pairs(self, ubiconfig_item):
 
         repo_pairs = []
         for input_repo in in_repos_ft.result():
-            rpm = input_repo
-            source = self._get_repo_counterpart(input_repo, in_source_repos_ft.result())
-            debug_info = self._get_repo_counterpart(input_repo, in_debug_repos_ft.result())
-
-            rhel_repo_set = RepoSet(rpm, source, debug_info)
-
-            rpm = self._get_repo_counterpart(input_repo, out_repos_ft.result())
-            source = self._get_repo_counterpart(input_repo, out_source_repos_ft.result())
-            debug_info = self._get_repo_counterpart(input_repo, out_debug_repos_ft.result())
-
-            ubi_repo_set = RepoSet(rpm, source, debug_info)
+            in_rpm = input_repo
+            in_source = self._get_repo_counterpart(input_repo, in_source_repos_ft.result())
+            in_debug_info = self._get_repo_counterpart(input_repo, in_debug_repos_ft.result())
+
+            out_rpm = self._get_repo_counterpart(input_repo, out_repos_ft.result())
+            out_source = self._get_repo_counterpart(input_repo, out_source_repos_ft.result())
+            out_debug_info = self._get_repo_counterpart(input_repo, out_debug_repos_ft.result())
+
+            in_repos = (in_rpm, in_source, in_debug_info)
+            out_repos = (out_rpm, out_source, out_debug_info)
+
+            # Skip repos sets containing output repos which should not be populated
+            if not all([r.ubi_population is True for r in out_repos]):
+                _LOG.debug(
+                    "Skipping %s, population disabled for output repo(s):\n\t%s",
+                    in_rpm.content_set,
+                    "\n\t".join(r.content_set for r in out_repos if r.ubi_population is False))
+                continue
 
-            repo_pairs.append(UbiRepoSet(rhel_repo_set, ubi_repo_set))
+            repo_pairs.append(UbiRepoSet(RepoSet(*in_repos), RepoSet(*out_repos)))
 
         return repo_pairs
 

ubipop/_pulp_client.py

@@ -87,12 +87,16 @@ def _search_repo(self, criteria):
         for item in ret.json():
             notes = item['notes']
             dist_info = [(dist['id'], dist['distributor_type_id']) for dist in item['distributors']]
+            # Only UBI repos should have a ubi_population note, set to None for other platforms
+            # If the UBI repo does not have this note, assume it is okay to populate
+            ubi_population = notes.get('ubi_population', True) if notes['platform'] == 'ubi' else None
             repos.append(Repo(
                 repo_id=item['id'],
                 arch=notes['arch'],
                 content_set=notes['content_set'],
                 platform_full_version=notes['platform_full_version'],
                 dist_ids_type_ids=dist_info,
+                ubi_population=ubi_population,
             ))
 
         return repos
@@ -306,12 +310,14 @@ def publish_repo(self, repo):
 
 
 class Repo(object):
-    def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids):
+    def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids,
+                 ubi_population):
         self.repo_id = repo_id
         self.arch = arch
         self.content_set = content_set
         self.platform_full_version = platform_full_version
         self.distributors_ids_type_ids_tuples = dist_ids_type_ids
+        self.ubi_population = ubi_population
 
 
 class Package(object):