Allow to match product versions from Koji build target

There is a new option for subject type definitions that allows Greenwave
to "guess" multiple product versions from Koji build target for given
Koji build.

Example:

    --- !SubjectType
    id: koji_build
    is_koji_build: true
    product_version_from_koji_build_target:
      - match: '^(rhel-\d+\.\d+).*'
        product_version: '\1'
      - match: '^(rhel-\d+).*'
        product_version: '\1'

Multiple product versions can now also be matched from subject
identifier with `product_version_match`.

JIRA: RHELWF-383

conf/subject_types/koji_build.yaml

@@ -13,3 +13,8 @@ result_queries:
       type: "koji_build,brew-build"
   # {"original_spec_nvr": ITEM}
   - item_key: "original_spec_nvr"
+product_version_from_koji_build_target:
+  - match: '^(rhel-\d+\.\d+).*'
+    product_version: '\1'
+  - match: '^(rhel-\d+).*'
+    product_version: '\1'

conftest.py

@@ -1,8 +1,15 @@
 # SPDX-License-Identifier: GPL-2.0+
-
+import mock
 import pytest
 
 
 @pytest.fixture(autouse=True)
 def set_environment_variable(monkeypatch):
     monkeypatch.setenv('TEST', 'true')
+
+
+@pytest.fixture
+def koji_proxy():
+    mock_proxy = mock.Mock()
+    with mock.patch('greenwave.resources.get_server_proxy', return_value=mock_proxy):
+        yield mock_proxy

functional-tests/consumers/test_resultsdb.py

@@ -21,7 +21,7 @@ def create_resultdb_handler(greenwave_server, cache_config=None):
 @mock.patch('greenwave.consumers.consumer.fedora_messaging.api.publish')
 def test_consume_new_result(
         mock_fedora_messaging, requests_session, greenwave_server,
-        testdatabuilder):
+        testdatabuilder, koji_proxy):
     nvr = testdatabuilder.unique_nvr(product_version='fc26')
     result = testdatabuilder.create_result(item=nvr,
                                            testcase_name='dist.rpmdeplint',
@@ -296,7 +296,7 @@ def test_consume_compose_id_result(
 @mock.patch('greenwave.consumers.consumer.fedora_messaging.api.publish')
 def test_consume_legacy_result(
         mock_fedora_messaging, requests_session, greenwave_server,
-        testdatabuilder):
+        testdatabuilder, koji_proxy):
     """ Test that we can still handle the old legacy "taskotron" format.
 
     We should be using resultsdb.result.new everywhere now, but we also

greenwave/consumers/resultsdb.py

@@ -12,7 +12,7 @@
 import logging
 
 from greenwave.consumers.consumer import Consumer
-from greenwave.product_versions import subject_product_version
+from greenwave.product_versions import subject_product_versions
 from greenwave.subjects.factory import (
     create_subject_from_data,
     UnknownSubjectDataError,
@@ -126,18 +126,22 @@ def _consume_message(self, message):
 
         log.debug('Considering subject: %r', subject)
 
-        product_version = subject_product_version(
+        product_versions = subject_product_versions(
             subject,
             self.koji_base_url,
             brew_task_id,
         )
 
-        log.debug('Guessed product version: %r', product_version)
+        log.debug('Guessed product versions: %r', product_versions)
 
-        self._publish_decision_change(
-            submit_time=submit_time,
-            subject=subject,
-            testcase=testcase,
-            product_version=product_version,
-            publish_testcase=False,
-        )
+        if not product_versions:
+            product_versions = [None]
+
+        for product_version in product_versions:
+            self._publish_decision_change(
+                submit_time=submit_time,
+                subject=subject,
+                testcase=testcase,
+                product_version=product_version,
+                publish_testcase=False,
+            )

greenwave/listeners/resultsdb.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0+
 from greenwave.listeners.base import BaseListener
-from greenwave.product_versions import subject_product_version
+from greenwave.product_versions import subject_product_versions
 from greenwave.subjects.factory import (
     create_subject_from_data,
     UnknownSubjectDataError,
@@ -94,19 +94,24 @@ def _consume_message(self, msg):
 
         self.app.logger.debug("Considering subject: %r", subject)
 
-        product_version = subject_product_version(
+        product_versions = subject_product_versions(
             subject,
             self.koji_base_url,
             brew_task_id,
         )
 
-        self.app.logger.debug("Guessed product version: %r", product_version)
+        self.app.logger.debug("Guessed product versions: %r", product_versions)
+
+        if not product_versions:
+            product_versions = [None]
+
+        for product_version in product_versions:
+            self._publish_decision_change(
+                submit_time=submit_time,
+                subject=subject,
+                testcase=testcase,
+                product_version=product_version,
+                publish_testcase=False,
+            )
 
-        self._publish_decision_change(
-            submit_time=submit_time,
-            subject=subject,
-            testcase=testcase,
-            product_version=product_version,
-            publish_testcase=False,
-        )
         return True

greenwave/product_versions.py

@@ -6,6 +6,7 @@
 import logging
 import re
 import socket
+from typing import List
 
 from defusedxml.xmlrpc import xmlrpc_client
 from werkzeug.exceptions import NotFound
@@ -18,9 +19,9 @@
 log = logging.getLogger(__name__)
 
 
-def _guess_product_version(toparse, koji_build=False):
+def _guess_product_versions(toparse, koji_build=False) -> List[str]:
     if toparse == 'rawhide' or toparse.startswith('Fedora-Rawhide'):
-        return 'fedora-rawhide'
+        return ['fedora-rawhide']
 
     product_version = None
     if toparse.startswith('f') and koji_build:
@@ -41,52 +42,58 @@ def _guess_product_version(toparse, koji_build=False):
             try:
                 int(result[1])
                 product_version += result[1]
-                return product_version
+                return [product_version]
             except ValueError:
                 pass
 
     log.warning("Failed to guess the product version for %s", toparse)
-    return None
+    return []
 
 
-def _guess_koji_build_product_version(
-        subject_identifier, koji_base_url, koji_task_id=None):
+def _guess_koji_build_product_versions(
+        subject, koji_base_url, koji_task_id=None) -> List[str]:
     try:
         if not koji_task_id:
             try:
                 koji_task_id = retrieve_koji_build_task_id(
-                    subject_identifier, koji_base_url
+                    subject.identifier, koji_base_url
                 )
             except NotFound:
                 koji_task_id = None
 
             if not koji_task_id:
-                return None
+                return []
 
         target = retrieve_koji_build_target(koji_task_id, koji_base_url)
         if target:
-            return _guess_product_version(target, koji_build=True)
+            pvs = subject.product_versions_from_koji_build_target(target)
+            if pvs:
+                return pvs
+            return _guess_product_versions(target, koji_build=True)
 
-        return None
+        return []
     except (xmlrpc_client.ProtocolError, socket.error) as err:
         raise ConnectionError('Could not reach Koji: {}'.format(err))
     except xmlrpc_client.Fault:
         log.exception('Unexpected Koji XML RPC fault')
+        return []
 
 
-def subject_product_version(
+def subject_product_versions(
         subject,
         koji_base_url=None,
-        koji_task_id=None):
-    if subject.product_version:
-        return subject.product_version
+        koji_task_id=None) -> List[str]:
+    if subject.product_versions:
+        return subject.product_versions
+
+    if koji_base_url and subject.is_koji_build:
+        pvs = _guess_koji_build_product_versions(
+            subject, koji_base_url, koji_task_id)
+        if pvs:
+            return pvs
 
     if subject.short_product_version:
-        product_version = _guess_product_version(
+        return _guess_product_versions(
             subject.short_product_version, koji_build=subject.is_koji_build)
-        if product_version:
-            return product_version
 
-    if koji_base_url and subject.is_koji_build:
-        return _guess_koji_build_product_version(
-            subject.identifier, koji_base_url, koji_task_id)
+    return []

greenwave/resources.py

@@ -170,10 +170,10 @@ class NoSourceException(RuntimeError):
 
 
 @cached
-def retrieve_koji_build_target(nvr: str, koji_url: str):
-    log.debug('Getting Koji task request ID %r', nvr)
+def retrieve_koji_build_target(task_id, koji_url: str):
+    log.debug('Getting Koji task request ID %r', task_id)
     proxy = _koji(koji_url)
-    task_request = proxy.getTaskRequest(nvr)
+    task_request = proxy.getTaskRequest(task_id)
     if isinstance(task_request, list) and len(task_request) > 1:
         target = task_request[1]
         if isinstance(target, str):

greenwave/subjects/subject.py

@@ -35,6 +35,10 @@ def __init__(self, type_: Union[GenericSubjectType, SubjectType], item: str):
         self._type = type_
         self.item = item
 
+    def product_versions_from_koji_build_target(self, target):
+        return sorted(self._matching_product_versions(
+            target, self._type.product_version_from_koji_build_target))
+
     @property
     def type(self):
         """Subject type string."""
@@ -67,13 +71,19 @@ def short_product_version(self):
         return None
 
     @property
-    def product_version(self):
-        for pv_match in self._type.product_version_match:
-            pv = re.sub(pv_match['match'], pv_match['product_version'], self.item)
-            if pv and pv != self.item:
-                return pv.lower()
+    def product_versions(self):
+        pvs = sorted({
+            pv.lower()
+            for pv in self._matching_product_versions(
+                self.item, self._type.product_version_match)
+        })
+        if pvs:
+            return pvs
+
+        if self._type.product_version:
+            return [self._type.product_version]
 
-        return self._type.product_version
+        return []
 
     @property
     def is_koji_build(self):
@@ -106,6 +116,13 @@ def result_queries(self):
         else:
             yield self.to_dict()
 
+    def _matching_product_versions(self, text, match_dict):
+        return {
+            re.sub(pv_match['match'], pv_match['product_version'], text)
+            for pv_match in match_dict
+            if re.match(pv_match['match'], text)
+        }
+
     def __str__(self):
         return "subject_type {!r}, subject_identifier {!r}".format(
             self.type, self.item

greenwave/subjects/subject_type.py

@@ -46,6 +46,10 @@ class SubjectType(SafeYAMLObject):
         #   '\1', '\2' etc, expanded to matched groups)
         'product_version_match': SafeYAMLList(item_type=dict, optional=True),
 
+        # Same as product_version_match, but to match build target from Koji
+        # instead of subject ID.
+        'product_version_from_koji_build_target': SafeYAMLList(item_type=dict, optional=True),
+
         # Fixed product version for the subject type used if
         # product_version_match is undefined or does not match subject ID.
         'product_version': SafeYAMLString(optional=True),

greenwave/tests/conftest.py

@@ -1,4 +1,4 @@
-import mock
+# SPDX-License-Identifier: GPL-2.0+
 import pytest
 
 from greenwave.app_factory import create_app
@@ -19,10 +19,3 @@ def app():
 @pytest.fixture
 def client(app):
     yield app.test_client()
-
-
-@pytest.fixture
-def koji_proxy():
-    mock_proxy = mock.Mock()
-    with mock.patch('greenwave.resources.get_server_proxy', return_value=mock_proxy):
-        yield mock_proxy