Use correct version of ubi config file for population

This resloves release-engineering#138
A new attr is added to Repo class and it will be populated while
searching repos.

After all config files are loaded, it now creates a config map, which
helps find the right config file for the repo to populate.

tests/test_pulp.py

@@ -43,6 +43,7 @@ def fixture_search_repo_response():
             "content_set": "test_repo-source-rpms",
             "platform": "ubi",
             "platform_full_version": "7",
+            "ubi_config_version": "7",
         },
         "distributors": [{
             "id": "dist_id",

tests/test_ubipop.py

@@ -47,8 +47,9 @@ def fixture_ubi_repo_set_no_debug():
 
 @pytest.fixture(name='test_ubiconfig')
 def fixture_test_ubiconfig():
-    yield ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml")
-
+    config = ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml")
+    config.version = '7.7'
+    yield config
 
 @pytest.fixture(name='executor')
 def fixture_executor():
@@ -68,6 +69,7 @@ def get_test_repo(**kwargs):
         kwargs.get('platform_full_version'),
         kwargs.get('distributors_ids_type_ids'),
         kwargs.get('ubi_population'),
+        kwargs.get('ubi_config_version'),
     )
 
 

ubipop/__init__.py

@@ -90,6 +90,7 @@ def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=No
         self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url,
                                                    content_sets=kwargs.get('content_sets', None),
                                                    repo_ids=kwargs.get('repo_ids', None))
+        self.ubiconfig_map = self._create_config_map()
 
     def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None):
         loader = ubiconfig.get_loader(ubiconfig_dir_or_url)
@@ -132,18 +133,65 @@ def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids):
 
         return filtered_conf_list
 
+    def _create_config_map(self):
+        """Create a config map from self.ubiconfig_list, it has the form in:
+            {
+                "7.7":
+                    {
+                        "config_filename1": config1,
+                        "config_filename2": config2,
+                        ...,
+                    },
+                "8.1":
+                    {
+                        "config_filename1": config1,
+                        ...,
+                    },
+                ....
+            }
+        """
+
+        config_map = {}
+        for config in self.ubiconfig_list:
+            config_map.setdefault(config.version, {})\
+                .setdefault(config.filename, config)
+
+        return config_map
+
     def populate_ubi_repos(self):
         out_repos = set()
-
+        used_content_sets = set()
+        # since repos are searched by content sets, same repo could be searched and ppopulated
+        # multiple times, to avoid that, cache the content sets already used and skip the config
+        # whose content sets are all in the cache
         for config in self.ubiconfig_list:
+            content_sets = [
+                config.content_sets.rpm,
+                config.content_sets.srpm,
+                config.content_sets.debuginfo
+            ]
+            to_use = [cs for cs in content_sets if cs not in used_content_sets]
+            if to_use:
+                for cs in to_use:
+                    used_content_sets.add(cs)
+            else:
+                _LOG.debug("Skipping %s, since its been used already")
+                continue
+
             try:
                 repo_pairs = self._get_input_and_output_repo_pairs(config)
             except RepoMissing:
                 _LOG.warning("Skipping current content triplet, some repos are missing")
                 continue
 
             for repo_set in repo_pairs:
-                UbiPopulateRunner(self.pulp, repo_set, config, self.dry_run,
+                # get the right config file by ubi_config_version attr, if it's None,
+                # then it's not a mainline repo, use platform_full_version instead.
+                version = repo_set.out_repos.rpm.ubi_config_version \
+                    or repo_set.out_repos.rpm.platform_full_version
+                right_config = self.ubiconfig_map.get(version, {}).get(config.filename)
+
+                UbiPopulateRunner(self.pulp, repo_set, right_config, self.dry_run,
                                   self._executor).run_ubi_population()
 
                 out_repos.update(repo_set.get_output_repo_ids())

ubipop/_pulp_client.py

@@ -90,13 +90,15 @@ def _search_repo(self, criteria):
             # Only UBI repos should have a ubi_population note, set to None for other platforms
             # If the UBI repo does not have this note, assume it is okay to populate
             ubi_population = notes.get('ubi_population', True) if notes['platform'] == 'ubi' else None
+            ubi_config_version = notes.get('ubi_config_version', None)
             repos.append(Repo(
                 repo_id=item['id'],
                 arch=notes['arch'],
                 content_set=notes['content_set'],
                 platform_full_version=notes['platform_full_version'],
                 dist_ids_type_ids=dist_info,
                 ubi_population=ubi_population,
+                ubi_config_version=ubi_config_version,
             ))
 
         return repos
@@ -311,11 +313,12 @@ def publish_repo(self, repo):
 
 class Repo(object):
     def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids,
-                 ubi_population):
+                 ubi_population, ubi_config_version):
         self.repo_id = repo_id
         self.arch = arch
         self.content_set = content_set
         self.platform_full_version = platform_full_version
+        self.ubi_config_version = ubi_config_version
         self.distributors_ids_type_ids_tuples = dist_ids_type_ids
         self.ubi_population = ubi_population