Skip to content

Commit

Permalink
Allow to match product versions from Koji build target
Browse files Browse the repository at this point in the history
There is a new option for subject type definitions that allows Greenwave
to "guess" multiple product versions from Koji build target for given
Koji build.

Example:

    --- !SubjectType
    id: koji_build
    is_koji_build: true
    product_version_from_koji_build_target:
      - match: '^(rhel-\d+\.\d+).*'
        product_version: '\1'
      - match: '^(rhel-\d+).*'
        product_version: '\1'

Multiple product versions can now also be matched from subject
identifier with `product_version_match`.

JIRA: RHELWF-383
  • Loading branch information
hluk committed May 3, 2023
1 parent 027d8d7 commit 2a90b2f
Show file tree
Hide file tree
Showing 14 changed files with 160 additions and 108 deletions.
5 changes: 5 additions & 0 deletions conf/subject_types/koji_build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,8 @@ result_queries:
type: "koji_build,brew-build"
# {"original_spec_nvr": ITEM}
- item_key: "original_spec_nvr"
product_version_from_koji_build_target:
- match: '^(rhel-\d+\.\d+).*'
product_version: '\1'
- match: '^(rhel-\d+).*'
product_version: '\1'
9 changes: 8 additions & 1 deletion conftest.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
# SPDX-License-Identifier: GPL-2.0+

import mock
import pytest


@pytest.fixture(autouse=True)
def set_environment_variable(monkeypatch):
monkeypatch.setenv('TEST', 'true')


@pytest.fixture
def koji_proxy():
mock_proxy = mock.Mock()
with mock.patch('greenwave.resources.get_server_proxy', return_value=mock_proxy):
yield mock_proxy
4 changes: 2 additions & 2 deletions functional-tests/consumers/test_resultsdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def create_resultdb_handler(greenwave_server, cache_config=None):
@mock.patch('greenwave.consumers.consumer.fedora_messaging.api.publish')
def test_consume_new_result(
mock_fedora_messaging, requests_session, greenwave_server,
testdatabuilder):
testdatabuilder, koji_proxy):
nvr = testdatabuilder.unique_nvr(product_version='fc26')
result = testdatabuilder.create_result(item=nvr,
testcase_name='dist.rpmdeplint',
Expand Down Expand Up @@ -296,7 +296,7 @@ def test_consume_compose_id_result(
@mock.patch('greenwave.consumers.consumer.fedora_messaging.api.publish')
def test_consume_legacy_result(
mock_fedora_messaging, requests_session, greenwave_server,
testdatabuilder):
testdatabuilder, koji_proxy):
""" Test that we can still handle the old legacy "taskotron" format.
We should be using resultsdb.result.new everywhere now, but we also
Expand Down
24 changes: 14 additions & 10 deletions greenwave/consumers/resultsdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import logging

from greenwave.consumers.consumer import Consumer
from greenwave.product_versions import subject_product_version
from greenwave.product_versions import subject_product_versions
from greenwave.subjects.factory import (
create_subject_from_data,
UnknownSubjectDataError,
Expand Down Expand Up @@ -126,18 +126,22 @@ def _consume_message(self, message):

log.debug('Considering subject: %r', subject)

product_version = subject_product_version(
product_versions = subject_product_versions(
subject,
self.koji_base_url,
brew_task_id,
)

log.debug('Guessed product version: %r', product_version)
log.debug('Guessed product versions: %r', product_versions)

self._publish_decision_change(
submit_time=submit_time,
subject=subject,
testcase=testcase,
product_version=product_version,
publish_testcase=False,
)
if not product_versions:
product_versions = [None]

for product_version in product_versions:
self._publish_decision_change(
submit_time=submit_time,
subject=subject,
testcase=testcase,
product_version=product_version,
publish_testcase=False,
)
25 changes: 15 additions & 10 deletions greenwave/listeners/resultsdb.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0+
from greenwave.listeners.base import BaseListener
from greenwave.product_versions import subject_product_version
from greenwave.product_versions import subject_product_versions
from greenwave.subjects.factory import (
create_subject_from_data,
UnknownSubjectDataError,
Expand Down Expand Up @@ -94,19 +94,24 @@ def _consume_message(self, msg):

self.app.logger.debug("Considering subject: %r", subject)

product_version = subject_product_version(
product_versions = subject_product_versions(
subject,
self.koji_base_url,
brew_task_id,
)

self.app.logger.debug("Guessed product version: %r", product_version)
self.app.logger.debug("Guessed product versions: %r", product_versions)

if not product_versions:
product_versions = [None]

for product_version in product_versions:
self._publish_decision_change(
submit_time=submit_time,
subject=subject,
testcase=testcase,
product_version=product_version,
publish_testcase=False,
)

self._publish_decision_change(
submit_time=submit_time,
subject=subject,
testcase=testcase,
product_version=product_version,
publish_testcase=False,
)
return True
47 changes: 27 additions & 20 deletions greenwave/product_versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import logging
import re
import socket
from typing import List

from defusedxml.xmlrpc import xmlrpc_client
from werkzeug.exceptions import NotFound
Expand All @@ -18,9 +19,9 @@
log = logging.getLogger(__name__)


def _guess_product_version(toparse, koji_build=False):
def _guess_product_versions(toparse, koji_build=False) -> List[str]:
if toparse == 'rawhide' or toparse.startswith('Fedora-Rawhide'):
return 'fedora-rawhide'
return ['fedora-rawhide']

product_version = None
if toparse.startswith('f') and koji_build:
Expand All @@ -41,52 +42,58 @@ def _guess_product_version(toparse, koji_build=False):
try:
int(result[1])
product_version += result[1]
return product_version
return [product_version]
except ValueError:
pass

log.warning("Failed to guess the product version for %s", toparse)
return None
return []


def _guess_koji_build_product_version(
subject_identifier, koji_base_url, koji_task_id=None):
def _guess_koji_build_product_versions(
subject, koji_base_url, koji_task_id=None) -> List[str]:
try:
if not koji_task_id:
try:
koji_task_id = retrieve_koji_build_task_id(
subject_identifier, koji_base_url
subject.identifier, koji_base_url
)
except NotFound:
koji_task_id = None

if not koji_task_id:
return None
return []

target = retrieve_koji_build_target(koji_task_id, koji_base_url)
if target:
return _guess_product_version(target, koji_build=True)
pvs = subject.product_versions_from_koji_build_target(target)
if pvs:
return pvs
return _guess_product_versions(target, koji_build=True)

return None
return []
except (xmlrpc_client.ProtocolError, socket.error) as err:
raise ConnectionError('Could not reach Koji: {}'.format(err))
except xmlrpc_client.Fault:
log.exception('Unexpected Koji XML RPC fault')
return []


def subject_product_version(
def subject_product_versions(
subject,
koji_base_url=None,
koji_task_id=None):
if subject.product_version:
return subject.product_version
koji_task_id=None) -> List[str]:
if subject.product_versions:
return subject.product_versions

if koji_base_url and subject.is_koji_build:
pvs = _guess_koji_build_product_versions(
subject, koji_base_url, koji_task_id)
if pvs:
return pvs

if subject.short_product_version:
product_version = _guess_product_version(
return _guess_product_versions(
subject.short_product_version, koji_build=subject.is_koji_build)
if product_version:
return product_version

if koji_base_url and subject.is_koji_build:
return _guess_koji_build_product_version(
subject.identifier, koji_base_url, koji_task_id)
return []
6 changes: 3 additions & 3 deletions greenwave/resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,10 @@ class NoSourceException(RuntimeError):


@cached
def retrieve_koji_build_target(nvr: str, koji_url: str):
log.debug('Getting Koji task request ID %r', nvr)
def retrieve_koji_build_target(task_id, koji_url: str):
log.debug('Getting Koji task request ID %r', task_id)
proxy = _koji(koji_url)
task_request = proxy.getTaskRequest(nvr)
task_request = proxy.getTaskRequest(task_id)
if isinstance(task_request, list) and len(task_request) > 1:
target = task_request[1]
if isinstance(target, str):
Expand Down
29 changes: 23 additions & 6 deletions greenwave/subjects/subject.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ def __init__(self, type_: Union[GenericSubjectType, SubjectType], item: str):
self._type = type_
self.item = item

def product_versions_from_koji_build_target(self, target):
return sorted(self._matching_product_versions(
target, self._type.product_version_from_koji_build_target))

@property
def type(self):
"""Subject type string."""
Expand Down Expand Up @@ -67,13 +71,19 @@ def short_product_version(self):
return None

@property
def product_version(self):
for pv_match in self._type.product_version_match:
pv = re.sub(pv_match['match'], pv_match['product_version'], self.item)
if pv and pv != self.item:
return pv.lower()
def product_versions(self):
pvs = sorted({
pv.lower()
for pv in self._matching_product_versions(
self.item, self._type.product_version_match)
})
if pvs:
return pvs

if self._type.product_version:
return [self._type.product_version]

return self._type.product_version
return []

@property
def is_koji_build(self):
Expand Down Expand Up @@ -106,6 +116,13 @@ def result_queries(self):
else:
yield self.to_dict()

def _matching_product_versions(self, text, match_dict):
return {
re.sub(pv_match['match'], pv_match['product_version'], text)
for pv_match in match_dict
if re.match(pv_match['match'], text)
}

def __str__(self):
return "subject_type {!r}, subject_identifier {!r}".format(
self.type, self.item
Expand Down
4 changes: 4 additions & 0 deletions greenwave/subjects/subject_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ class SubjectType(SafeYAMLObject):
# '\1', '\2' etc, expanded to matched groups)
'product_version_match': SafeYAMLList(item_type=dict, optional=True),

# Same as product_version_match, but to match build target from Koji
# instead of subject ID.
'product_version_from_koji_build_target': SafeYAMLList(item_type=dict, optional=True),

# Fixed product version for the subject type used if
# product_version_match is undefined or does not match subject ID.
'product_version': SafeYAMLString(optional=True),
Expand Down
9 changes: 1 addition & 8 deletions greenwave/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import mock
# SPDX-License-Identifier: GPL-2.0+
import pytest

from greenwave.app_factory import create_app
Expand All @@ -19,10 +19,3 @@ def app():
@pytest.fixture
def client(app):
yield app.test_client()


@pytest.fixture
def koji_proxy():
mock_proxy = mock.Mock()
with mock.patch('greenwave.resources.get_server_proxy', return_value=mock_proxy):
yield mock_proxy
Loading

0 comments on commit 2a90b2f

Please sign in to comment.