Code to merge image/rpm metadata with assembly overrides

doozerlib/assembly.py

@@ -0,0 +1,106 @@
+import copy
+
+from doozerlib.model import Model
+
+
+def merger(a, b):
+    """
+    Merges two, potentially deep, objects into a new one and returns the result.
+    Conceptually, 'a' is layered over 'b' and is dominant when
+    necessary. The output is 'c'.
+    1. if 'a' specifies a primitive value, regardless of depth, 'c' will contain that value.
+    2. if a key in 'a' specifies a list and 'b' has the same key/list, a's list will be appended to b's for c's list.
+       Duplicates entries will be removed and primitive (str, int, ..) lists will be returned in sorted order).
+    3. if a key ending with '!' in 'a' specifies a value, c's key-! will be to set that value exactly.
+    4. if a key ending with a '?' in 'a' specifies a value, c's key-? will be set to that value is 'c' does not contain the key.
+    """
+
+    if type(a) in [bool, int, float, str, bytes, type(None)]:
+        return a
+
+    c = copy.deepcopy(b)
+
+    if type(a) is list:
+        if type(c) is not list:
+            return a
+        for entry in a:
+            if entry not in c:  # do not include duplicates
+                c.append(entry)
+
+        if c and type(c[0]) in [str, int, float]:
+            return sorted(c)
+        return c
+
+    if type(a) is dict:
+        if type(c) is not dict:
+            return a
+        for k, v in a.items():
+            if k.endswith('!'):  # full dominant key
+                k = k[:-1]
+                c[k] = v
+            elif k.endswith('?'):  # default value key
+                k = k[:-1]
+                if k not in c:
+                    c[k] = v
+            else:
+                if k in c:
+                    c[k] = merger(a[k], c[k])
+                else:
+                    c[k] = a[k]
+        return c
+
+    raise IOError(f'Unexpected value type: {type(a)}: {a}')
+
+
+def group_for_assembly(releases_config: Model, assembly: str, group_config: Model):
+    """
+    Returns a group config based on the assembly information
+    and the input group config.
+    :param releases_config: A Model for releases.yaml.
+    :param assembly: The name of the assembly
+    :param group_config: The group config to merge into a new group config (original Model will not be altered)
+    """
+    if not assembly or not isinstance(releases_config, Model):
+        return group_config
+
+    target_assembly = releases_config.releases[assembly].assembly
+
+    if target_assembly.basis.assembly:  # Does this assembly inherit from another?
+        # Recursive apply ancestor assemblies
+        group_config = group_for_assembly(releases_config, target_assembly.basis.assembly, group_config)
+
+    target_assembly_group = target_assembly.group
+    if not target_assembly_group:
+        return group_config
+
+    return Model(dict_to_model=merger(target_assembly_group.primitive(), group_config.primitive()))
+
+
+def metadata_config_for_assembly(releases_config: Model, assembly: str, meta_type: str, distgit_key: str, meta_config: Model):
+    """
+    Returns a group config based on the assembly information
+    and the input group config.
+    :param releases_config: A Model for releases.yaml.
+    :param assembly: The name of the assembly
+    :param meta_type: 'rpm' or 'image'
+    :param distgit_key: The name of the underlying component
+    :param meta_config: The meta's config object
+    :return: Returns a computed config for the metadata (e.g. value for meta.config).
+    """
+    if not assembly or not isinstance(releases_config, Model):
+        return meta_config
+
+    target_assembly = releases_config.releases[assembly].assembly
+
+    if target_assembly.basis.assembly:  # Does this assembly inherit from another?
+        # Recursive apply ancestor assemblies
+        meta_config = metadata_config_for_assembly(releases_config, target_assembly.basis.assembly, meta_type, distgit_key, meta_config)
+
+    config_dict = meta_config.primitive()
+
+    component_list = target_assembly.members[f'{meta_type}s']
+    for component_entry in component_list:
+        if component_entry.distgit_key == '*' or component_entry.distgit_key == distgit_key:
+            config_dict = merger(component_entry.metadata.primitive(), config_dict)
+
+    return Model(dict_to_model=config_dict)

doozerlib/image.py

@@ -95,29 +95,6 @@ def is_payload(self):
     def for_release(self):
         return self.config.get('for_release', True)
 
-    def get_component_name(self, default=-1):
-        """
-        :param default: Not used. Here to stay consistent with similar rpmcfg.get_component_name
-        :return: Returns the component name of the image. This is the name in the nvr
-        that brew assigns to this image's build. Component name is synonymous with package name.
-        """
-        # By default, the bugzilla component is the name of the distgit,
-        # but this can be overridden in the config yaml.
-        component_name = self.name
-
-        # For apbs, component name seems to have -apb appended.
-        # ex. http://dist-git.host.prod.eng.bos.redhat.com/cgit/apbs/openshift-enterprise-mediawiki/tree/Dockerfile?h=rhaos-3.7-rhel-7
-        if self.namespace == "apbs":
-            component_name = "%s-apb" % component_name
-
-        if self.namespace == "containers":
-            component_name = "%s-container" % component_name
-
-        if self.config.distgit.component is not Missing:
-            component_name = self.config.distgit.component
-
-        return component_name
-
     def get_brew_image_name_short(self):
         # Get image name in the Brew pullspec. e.g. openshift3/ose-ansible --> openshift3-ose-ansible
         return self.image_name.replace("/", "-")

doozerlib/metadata.py

@@ -6,6 +6,7 @@
 import json
 import traceback
 import datetime
+import time
 import re
 from enum import Enum
 
@@ -19,8 +20,8 @@
 from . import logutil
 from .brew import BuildStates
 
-
 from .model import Model, Missing
+from doozerlib.assembly import metadata_config_for_assembly
 
 
 class CgitAtomFeedEntry(NamedTuple):
@@ -111,30 +112,18 @@ def __init__(self, meta_type, runtime, data_obj: Dict, commitish: Optional[str]
 
         self.runtime.logger.debug("Loading metadata from {}".format(self.full_config_path))
 
-        self.config = Model(data_obj.data)
+        self.raw_config = Model(data_obj.data)  # Config straight from ocp-build-data
+        assert (self.raw_config.name is not Missing)
+
+        self.config = metadata_config_for_assembly(runtime.get_releases_config(), runtime.assembly, meta_type, self.distgit_key, self.raw_config)
+        self.namespace, self._component_name = Metadata.extract_component_info(meta_type, self.name, self.config)
 
         self.mode = self.config.get('mode', CONFIG_MODE_DEFAULT).lower()
         if self.mode not in CONFIG_MODES:
             raise ValueError('Invalid mode for {}'.format(self.config_filename))
 
         self.enabled = (self.mode == CONFIG_MODE_DEFAULT)
 
-        # Basic config validation. All images currently required to have a name in the metadata.
-        # This is required because from.member uses these data to populate FROM in images.
-        # It would be possible to query this data from the distgit Dockerflie label, but
-        # not implementing this until we actually need it.
-        assert (self.config.name is not Missing)
-
-        # Choose default namespace for config data
-        if meta_type == "image":
-            self.namespace = "containers"
-        else:
-            self.namespace = "rpms"
-
-        # Allow config data to override
-        if self.config.distgit.namespace is not Missing:
-            self.namespace = self.config.distgit.namespace
-
         self.qualified_name = "%s/%s" % (self.namespace, self.name)
         self.qualified_key = "%s/%s" % (self.namespace, self.distgit_key)
 
@@ -145,7 +134,8 @@ def __init__(self, meta_type, runtime, data_obj: Dict, commitish: Optional[str]
 
         # List of Brew targets.
         # The first target is the primary target, against which tito will direct build.
-        # Others are secondary targets. We will use Brew API to build against secondary targets with the same distgit commit as the primary target.
+        # Others are secondary targets. We will use Brew API to build against secondary
+        # targets with the same distgit commit as the primary target.
         self.targets = self.determine_targets()
 
     def determine_targets(self) -> List[str]:
@@ -371,7 +361,14 @@ def latest_build_list(pattern_suffix):
                     # a rebuild). If we find the latest build is not tagged appropriately, blow up
                     # and let a human figure out what happened.
                     check_nvr = refined[0]['nvr']
-                    tags = {tag['name'] for tag in koji_api.listTags(build=check_nvr)}
+                    for i in range(2):
+                        tags = {tag['name'] for tag in koji_api.listTags(build=check_nvr)}
+                        if tags:
+                            break
+                        # Observed that a complete build needs some time before it gets tagged. Give it some
+                        # time if not immediately available.
+                        time.sleep(60)
+
                     # RPMS have multiple targets, so our self.branch() isn't perfect.
                     # We should permit rhel-8/rhel-7/etc.
                     tag_prefix = self.branch().rsplit('-', 1)[0] + '-'   # String off the rhel version.
@@ -420,15 +417,56 @@ def get_latest_build_info(self, default=-1, **kwargs):
             return default
         return build['name'], build['version'], build['release']
 
-    def get_component_name(self, default=-1) -> str:
+    @classmethod
+    def extract_component_info(cls, meta_type: str, meta_name: str, config_model: Model) -> Tuple[str, str]:
+        """
+        Determine the component information for either RPM or Image metadata
+        configs.
+        :param meta_type: 'rpm' or 'image'
+        :param meta_name: The name of the component's distgit
+        :param config_model: The configuration for the metadata.
+        :return: Return (namespace, component_name)
+        """
+
+        # Choose default namespace for config data
+        if meta_type == "image":
+            namespace = "containers"
+        else:
+            namespace = "rpms"
+
+        # Allow config data to override namespace
+        if config_model.distgit.namespace is not Missing:
+            namespace = config_model.distgit.namespace
+
+        if namespace == "rpms":
+            # For RPMS, component names must match package name and be in metadata config
+            return namespace, config_model.name
+
+        # For RPMs, by default, the component is the name of the distgit,
+        # but this can be overridden in the config yaml.
+        component_name = meta_name
+
+        # For apbs, component name seems to have -apb appended.
+        # ex. http://dist-git.host.prod.eng.bos.redhat.com/cgit/apbs/openshift-enterprise-mediawiki/tree/Dockerfile?h=rhaos-3.7-rhel-7
+        if namespace == "apbs":
+            component_name = "%s-apb" % component_name
+
+        if namespace == "containers":
+            component_name = "%s-container" % component_name
+
+        if config_model.distgit.component is not Missing:
+            component_name = config_model.distgit.component
+
+        return namespace, component_name
+
+    def get_component_name(self) -> str:
         """
-        :param default: If the component name cannot be determined,
-        :return: Returns the component name of the image. This is the name in the nvr
+        :return: Returns the component name of the metadata. This is the name in the nvr
         that brew assigns to component build. Component name is synonymous with package name.
         For RPMs, spec files declare the package name. For images, it is usually based on
         the distgit repo name + '-container'.
         """
-        raise IOError('Subclass must implement')
+        return self._component_name
 
     def needs_rebuild(self):
         if self.config.targets:
@@ -455,11 +493,7 @@ def _target_needs_rebuild(self, el_target=None) -> RebuildHint:
         # If a build fails, how long will we wait before trying again
         rebuild_interval = self.runtime.group_config.scan_freshness.threshold_hours or 6
 
-        component_name = self.get_component_name(default='')
-        if not component_name:
-            # This can happen for RPMs if they have never been rebased into distgit.
-            return RebuildHint(code=RebuildHintCode.NO_COMPONENT,
-                               reason='Could not determine component name; assuming it has never been built')
+        component_name = self.get_component_name()
 
         latest_build = self.get_latest_build(default=None, el_target=el_target)
 

doozerlib/rpmcfg.py

@@ -207,20 +207,16 @@ def assert_golang_versions(self):
                 if (check_mode == "exact" and nevr[2] != first_nevr[2]) or (check_mode == "x.y" and nevr[2].split(".")[:2] != first_nevr[2].split(".")[:2]):
                     raise DoozerFatalError(f"Buildroot for target {target} has inconsistent golang compiler version {nevr[2]} while target {first_target} has {first_nevr[2]}.")
 
-    def get_package_name_from_spec(self, default=-1):
+    def get_package_name_from_spec(self):
         """
         Returns the brew package name for the distgit repository. Method requires
         a local clone. This differs from get_package_name because it will actually
         parse the .spec file in the distgit clone vs trusting the ocp-build-data.
-        :param default: If specified, this value will be returned if package name could not be found. If
-                        not specified, an exception will be raised instead.
         :return: The package name if detected in the distgit spec file. Otherwise, the specified default.
                 If default is not passed in, an exception will be raised if the package name can't be found.
         """
         specs = glob.glob(f'{self.distgit_repo().distgit_dir}/*.spec')
         if len(specs) != 1:
-            if default != -1:
-                return default
             raise IOError('Unable to find .spec file in RPM distgit: ' + self.qualified_name)
 
         spec_path = specs[0]
@@ -229,32 +225,18 @@ def get_package_name_from_spec(self, default=-1):
                 if line.lower().startswith('name:'):
                     return line[5:].strip()  # Exclude "Name:" and then remove whitespace
 
-        if default != -1:
-            return default
-
         raise IOError(f'Unable to find Name: field in rpm spec: {spec_path}')
 
-    def get_package_name(self, default=-1):
+    def get_package_name(self) -> str:
         """
         Returns the brew package name for the distgit repository. Package names for RPMs
         do not necessarily match the distgit component name. The package name is controlled
         by the RPM's spec file. The 'name' field in the doozer metadata should match what is
         in the RPM's spec file.
-        :param default: If specified, this value will be returned if package name could not be found.
         :return: The package name - Otherwise, the specified default.
                 If default is not passed in, an exception will be raised if the package name can't be found.
         """
-        package_name = self.config.name  # This should be the package name for the RPM
-        if package_name:
-            return package_name
-
-        if default != -1:
-            return default
-
-        raise IOError(f'Missing package name in RPM doozer metadata: {self.distgit_key}')
-
-    def get_component_name(self, default=-1):
-        return self.get_package_name(default=default)
+        return self.get_component_name()
 
     def candidate_brew_tag(self):
         return self.targets[0]

doozerlib/runtime.py

@@ -23,7 +23,6 @@
 import signal
 import io
 import pathlib
-import koji
 from typing import Optional, List, Dict, Tuple
 import time
 
@@ -45,6 +44,7 @@
 from doozerlib import constants
 from doozerlib import util
 from doozerlib import brew
+from doozerlib.assembly import group_for_assembly
 
 # Values corresponds to schema for group.yml: freeze_automation. When
 # 'yes', doozer itself will inhibit build/rebase related activity
@@ -259,6 +259,12 @@ def get_named_semaphore(self, lock_name, is_dir=False, count=1):
                 self.named_semaphores[p] = new_semaphore
                 return new_semaphore
 
+    def get_releases_config(self):
+        load = self.gitdata.load_data(key='releases')
+        if not load:
+            return Model()
+        return Model(load.data)
+
     def get_group_config(self):
         # group.yml can contain a `vars` section which should be a
         # single level dict containing keys to str.format(**dict) replace
@@ -276,7 +282,7 @@ def get_group_config(self):
             except KeyError as e:
                 raise ValueError('group.yml contains template key `{}` but no value was provided'.format(e.args[0]))
 
-        return tmp_config
+        return group_for_assembly(self.get_releases_config(), self.assembly, tmp_config)
 
     def init_state(self):
         self.state = dict(state.TEMPLATE_BASE_STATE)