Merge fe8e5c0 into 93eabff

requirements.txt

@@ -1,4 +1,4 @@
 requests
 more-executors
-ubi-config
+ubi-config>=2.2.0
 rpm-py-installer

tests/data/ubi8/ubiconf_golang2.yaml

@@ -0,0 +1,52 @@
+content_sets:
+  rpm:
+    output: ubi-8-for-x86_64-appstream-rpms
+    input: rhel-8-for-x86_64-appstream-rpms
+  srpm:
+    output: ubi-8-for-x86_64-appstream-source-rpms
+    input: rhel-8-for-x86_64-appstream-source-rpms
+  debuginfo:
+    output: ubi-8-for-x86_64-appstream-debug-rpms
+    input: rhel-8-for-x86_64-appstream-debug-rpms
+
+arches:
+  - src
+  - x86_64
+
+packages:
+  include:
+  - go-srpm-macros.*
+  - go-toolset.*
+  - golang.*
+  - golang-bin.*
+  - golang-docs.*
+  - golang-misc.*
+  - golang-race.*
+  - golang-src.*
+  - golang-tests.*
+  exclude:
+  - python36
+  - kernel
+  - kernel-doc
+  - kmod-kvdo
+  - kernel.src
+  - kernel-debug-debuginfo
+  - kernel-debuginfo
+  - kernel-debuginfo-common-aarch64
+  - kernel-debuginfo-common-ppc64le
+  - kernel-debuginfo-common-s390x
+  - kernel-debuginfo-common-x86_64
+  - kernel-tools-debuginfo
+  - linux-firmware
+  - grub2
+  - grub2-common
+  - grub2-tools
+  - grub2-tools-minimal
+  - grubby
+  - redhat-release-eula
+  - s390utils-base
+
+modules:
+  include:
+  - name: go-toolset
+    stream: rhel8

tests/integration/test_association.py

@@ -19,7 +19,7 @@
 PULP_SECURE = os.getenv('TEST_PULP_SECURE', '0').lower() in ['true', 'yes', '1']
 PULP_RPM_REPO_PREFIX = '/pulp/rpmrepos/'
 
-TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), './data')
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), './data/ubi7')
 
 INTEGRATION_NOT_SETUP = PULP_HOSTNAME is None
 

tests/test_pulp.py

@@ -43,6 +43,8 @@ def fixture_search_repo_response():
             "content_set": "test_repo-source-rpms",
             "platform": "ubi",
             "platform_full_version": "7",
+            "platform_major_version": "7",
+            "ubi_config_version": "7",
         },
         "distributors": [{
             "id": "dist_id",
@@ -58,6 +60,8 @@ def fixture_mock_repo():
         arch="x86_64",
         content_set="test_repo-source-rpms",
         platform_full_version="7",
+        platform_major_version="7",
+        ubi_config_version="7",
         dist_ids_type_ids=[
             ("dist_id_1", "dist_type_id_1"),
             ("dist_id_2", "dist_type_id_2"),
@@ -184,6 +188,7 @@ def test_search_repo_by_cs(mock_pulp, mock_search_repos):
     assert repo.arch == "x86_64"
     assert repo.content_set == "test_repo-source-rpms"
     assert repo.platform_full_version == "7"
+    assert repo.platform_major_version == "7"
     assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id")
     # ubi_population note was unset, so it should default to True
     assert repo.ubi_population is True
@@ -199,6 +204,7 @@ def test_search_repo_by_id(mock_pulp, mock_search_repos):
     assert repo.arch == "x86_64"
     assert repo.content_set == "test_repo-source-rpms"
     assert repo.platform_full_version == "7"
+    assert repo.ubi_config_version == "7"
     assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id")
     # ubi_population note was unset, so it should default to True
     assert repo.ubi_population is True

tests/test_ubipop.py

@@ -12,7 +12,7 @@
 
 from mock import MagicMock, patch, call
 from more_executors import Executors
-from ubipop import UbiPopulateRunner, UbiRepoSet, RepoSet, UbiPopulate
+from ubipop import UbiPopulateRunner, UbiRepoSet, RepoSet, UbiPopulate, ConfigMissing
 from ubipop._pulp_client import Module, ModuleDefaults, Package, Repo
 from ubipop._utils import (
     AssociateActionModules,
@@ -27,12 +27,18 @@
 
 @pytest.fixture(name='ubi_repo_set')
 def fixture_ubi_repo_set():
-    yield UbiRepoSet(RepoSet([get_test_repo(repo_id="foo-rpms")],
-                             [get_test_repo(repo_id="foo-source")],
-                             [get_test_repo(repo_id="foo-debug")]),
-                     RepoSet(get_test_repo(repo_id="ubi-foo-rpms"),
-                             get_test_repo(repo_id="ubi-foo-source"),
-                             get_test_repo(repo_id="ubi-foo-debug")))
+    yield UbiRepoSet(RepoSet([get_test_repo(repo_id="foo-rpms",
+                                            ubi_config_version="7")],
+                             [get_test_repo(repo_id="foo-source",
+                                            ubi_config_version="7")],
+                             [get_test_repo(repo_id="foo-debug",
+                                            ubi_config_version="7")]),
+                     RepoSet(get_test_repo(repo_id="ubi-foo-rpms",
+                                           ubi_config_version="7"),
+                             get_test_repo(repo_id="ubi-foo-source",
+                                           ubi_config_version="7"),
+                             get_test_repo(repo_id="ubi-foo-debug",
+                                           ubi_config_version="7")))
 
 
 @pytest.fixture(name='ubi_repo_set_no_debug')
@@ -47,7 +53,7 @@ def fixture_ubi_repo_set_no_debug():
 
 @pytest.fixture(name='test_ubiconfig')
 def fixture_test_ubiconfig():
-    yield ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml")
+    yield ubiconfig.get_loader(TEST_DATA_DIR).load("ubi7/conf.yaml")
 
 
 @pytest.fixture(name='executor')
@@ -66,9 +72,11 @@ def get_test_repo(**kwargs):
         kwargs.get('arch'),
         kwargs.get('content_set'),
         kwargs.get('platform_full_version'),
+        kwargs.get('platform_major_version'),
         kwargs.get('distributors_ids_type_ids'),
         kwargs.get('ubi_population'),
         kwargs.get('population_sources'),
+        kwargs.get('ubi_config_version'),
     )
 
 
@@ -118,21 +126,49 @@ def test_get_output_repo_ids_no_debug(ubi_repo_set_no_debug):
 def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner, caplog):
     # Don't actually query Pulp for repos
     mocked_search_repo_by_cs.side_effect = [
+        # Output repos - rhel-7-server
+        [get_test_repo(
+            repo_id="ubi-7-server-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-rpms",
+            ubi_population=False,
+            platform_full_version="7.2",
+        ), ],
+        [get_test_repo(
+            repo_id="ubi-7-server-source-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-source-rpms",
+            ubi_population=True,
+            # doesn't matter here, it sufficient to have ubi_population==False at rpm binary repo
+            # for skipping whole repo triplet
+            platform_full_version="7.2",
+        ), ],
+        [get_test_repo(
+            repo_id="ubi-7-server-debuginfo-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-debuginfo-rpms",
+            ubi_population=False,
+            platform_full_version="7.2"
+        ), ],
+
         # Output repos - rhel-8-for-x86_64-appstream
         [get_test_repo(
             repo_id="ubi-8-for-x86_64-appstream-rpms",
             content_set="ubi-8-for-x86_64-appstream-rpms",
-            ubi_population=True
+            ubi_population=True,
+            ubi_config_version="8",
+            platform_full_version="8"
         ), ],
         [get_test_repo(
             repo_id="ubi-8-for-x86_64-appstream-source-rpms",
             content_set="ubi-8-for-x86_64-appstream-source-rpms",
-            ubi_population=True
+            ubi_population=True,
+            ubi_config_version="8",
+            platform_full_version="8"
         ), ],
         [get_test_repo(
             repo_id="ubi-8-for-x86_64-appstream-debug-rpms",
             content_set="ubi-8-for-x86_64-appstream-debug-rpms",
-            ubi_population=True
+            ubi_population=True,
+            ubi_config_version="8",
+            platform_full_version="8"
         ), ],
 
         # Input repos - rhel-8-for-x86_64-appstream
@@ -148,51 +184,71 @@ def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner,
             repo_id="rhel-8-for-x86_64-appstream-debug-rpms",
             content_set="rhel-8-for-x86_64-appstream-debug-rpms",
         ), ],
+    ]
 
-        # Output repos - rhel-7-server
+    # Attempt to populate both invalid and valid repo sets
+    ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=TEST_DATA_DIR)
+    ubipop.populate_ubi_repos()
+
+    # Should've only run once
+    assert mocked_ubipop_runner.call_count == 1
+    # For ubi-8
+    assert "ubi-8-for-x86_64-appstream-rpms" not in caplog.text
+    # Not for ubi-7-server
+    assert "ubi-7-server-rpms__7_DOT_2__x86_64" in caplog.text
+    # Used content sets won't be used again
+    assert "Skipping ubiconf_golang2.yaml, since it's been used already" in caplog.text
+
+
+@patch("ubipop.UbiPopulateRunner")
+@patch("ubipop._pulp_client.Pulp.search_repo_by_cs")
+def test_raise_config_missing(mocked_search_repo_by_cs, mocked_ubipop_runner, caplog):
+    mocked_search_repo_by_cs.side_effect = [
+        # Output repos - rhel-7-for-x86_64-appstream
         [get_test_repo(
-            repo_id="ubi-7-server-rpms__7_DOT_2__x86_64",
-            content_set="ubi-7-server-rpms",
-            ubi_population=False
+            repo_id="ubi-7-for-x86_64-appstream-rpms",
+            content_set="ubi-7-for-x86_64-appstream-rpms",
+            ubi_population=True,
+            platform_full_version="7",
+            platform_major_version="7"
         ), ],
         [get_test_repo(
-            repo_id="ubi-7-server-source-rpms__7_DOT_2__x86_64",
-            content_set="ubi-7-server-source-rpms",
-            ubi_population=True
-            # doesn't matter here, it sufficient to have ubi_population==False at rpm binary repo
-            # for skipping whole repo triplet
+            repo_id="ubi-7-for-x86_64-appstream-source-rpms",
+            content_set="ubi-7-for-x86_64-appstream-source-rpms",
+            ubi_population=True,
+            platform_full_version="7",
+            platform_major_version="7"
         ), ],
         [get_test_repo(
-            repo_id="ubi-7-server-debuginfo-rpms__7_DOT_2__x86_64",
-            content_set="ubi-7-server-debuginfo-rpms",
-            ubi_population=False
+            repo_id="ubi-7-for-x86_64-appstream-debug-rpms",
+            content_set="ubi-7-for-x86_64-appstream-debug-rpms",
+            ubi_population=True,
+            platform_full_version="7",
+            platform_major_version="7"
         ), ],
 
-        # Input repos - rhel-7-server
+        # Input repos - rhel-7-for-x86_64-appstream
         [get_test_repo(
-            repo_id="rhel-7-server-rpms__7_DOT_2__x86_64",
-            content_set="rhel-7-server-rpms",
+            repo_id="rhel-7-for-x86_64-appstream-rpms",
+            content_set="rhel-7-for-x86_64-appstream-rpms",
         ), ],
         [get_test_repo(
-            repo_id="rhel-7-server-source-rpms__7_DOT_2__x86_64",
-            content_set="rhel-7-server-source-rpms",
+            repo_id="rhel-7-for-x86_64-appstream-source-rpms",
+            content_set="rhel-7-for-x86_64-appstream-source-rpms",
         ), ],
         [get_test_repo(
-            repo_id="rhel-7-server-debuginfo-rpms__7_DOT_2__x86_64",
-            content_set="rhel-7-server-debuginfo-rpms",
+            repo_id="rhel-7-for-x86_64-appstream-debug-rpms",
+            content_set="rhel-7-for-x86_64-appstream-debug-rpms",
         ), ],
     ]
+    config_path = os.path.join(TEST_DATA_DIR, 'ubi8')
+    ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=config_path)
+    with pytest.raises(ConfigMissing):
+        ubipop.populate_ubi_repos()
 
-    # Attempt to populate both invalid and valid repo sets
-    ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=TEST_DATA_DIR)
-    ubipop.populate_ubi_repos()
+    assert mocked_ubipop_runner.call_count == 0
 
-    # Should've only run once
-    assert mocked_ubipop_runner.call_count == 1
-    # For ubi-8
-    assert "ubi-8-for-x86_64-appstream-rpms" not in caplog.text
-    # Not for ubi-7-server
-    assert "ubi-7-server-rpms__7_DOT_2__x86_64" in caplog.text
+    assert "Config file ubiconf_golang.yaml missing from 7 and default 7 branches" in caplog.text
 
 
 @patch("ubipop._pulp_client.Pulp.search_repo_by_id")
@@ -522,7 +578,7 @@ def test_match_modules(mock_ubipop_runner):
 
 
 def test_match_modules_without_profile(ubi_repo_set, executor):
-    test_ubiconf = ubiconfig.get_loader(TEST_DATA_DIR).load('ubiconf_golang.yaml')
+    test_ubiconf = ubiconfig.get_loader(TEST_DATA_DIR).load('ubi8/ubiconf_golang.yaml')
     mocked_ubipop_runner = UbiPopulateRunner(
         MagicMock(),
         ubi_repo_set,
@@ -771,15 +827,18 @@ def test_ubi_repo_set(rhel_repo_set, ubi_repo_set, fail, caplog):
 @pytest.fixture(name='mocked_ubiconfig_load')
 def fixture_mocked_ubiconfig_load():
     with patch('ubiconfig.get_loader') as get_loader:
-        get_loader.return_value.load.return_value = "test"
+        m = MagicMock()
+        m.file_name = "test"
+        m.version = "7.7"
+        get_loader.return_value.load.return_value = m
         yield get_loader
 
 
 def test_ubipopulate_load_ubiconfig(mocked_ubiconfig_load):
     # pylint: disable=unused-argument
     ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ['cfg.yaml'])
     assert len(ubipop.ubiconfig_list) == 1
-    assert ubipop.ubiconfig_list[0] == "test"
+    assert ubipop.ubiconfig_list[0].file_name == "test"
 
 
 def test_load_ubiconfig_by_content_set_labels():
@@ -805,15 +864,18 @@ def test_load_ubiconfig_by_repo_ids(mocked_search_repo_by_id):
 @pytest.fixture(name='mocked_ubiconfig_load_all')
 def fixture_mocked_ubiconfig_load_all():
     with patch('ubiconfig.get_loader') as get_loader:
-        get_loader.return_value.load_all.return_value = ["test"]
+        m = MagicMock()
+        m.file_name = "test"
+        m.version = "7"
+        get_loader.return_value.load_all.return_value = [m]
         yield get_loader
 
 
 def test_ubipopulate_load_all_ubiconfig(mocked_ubiconfig_load_all):
     # pylint: disable=unused-argument
     ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False)
     assert len(ubipop.ubiconfig_list) == 1
-    assert ubipop.ubiconfig_list[0] == "test"
+    assert ubipop.ubiconfig_list[0].file_name == "test"
 
 
 def test_create_srpms_output_set(mock_ubipop_runner):