Skip to content

Commit

Permalink
Fix showing correct scenario for failed results
Browse files Browse the repository at this point in the history 
Also adds other important values for distinguishing results.
  • Loading branch information
@hluk
hluk committed Apr 6, 2022
1 parent b8f9b97 commit 9a7cd28
Show file tree
Hide file tree
Showing 10 changed files with 284 additions and 32 deletions.
20 changes: 20 additions & 0 deletions docs/decision_requirements.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ requirement.
Unsatisfied requirements containing ``testcase`` property can be waived (using
this value in a new waiver).

Requirements related to an existing result contain ``result_id`` attribute that
refers to the result ID in ResultsDB, and also ``scenario``,
``system_architecture`` and ``system_variant`` from the result data.

See :ref:`decision_requirements_examples` to get an idea about the data of
various requirements.

Expand Down Expand Up @@ -76,6 +80,22 @@ latest outcome is ``QUEUED`` or ``RUNNING`` (this can be overridden by
"scenario": null
}
Unsatisfied requirement for an incomplete result would be indicated by
additional attributes from the queued/running result data:

.. code-block:: json
{
"type": "test-result-missing",
"testcase": "example.test.case",
"subject_type": "koji-build",
"subject_identifier": "nethack-1.2.3-1.rawhide",
"result_id": 1004,
"scenario": null,
"system_architecture": null,
"system_variant": null
}
.. _failed_test_result:

Failed test result
Expand Down
18 changes: 18 additions & 0 deletions functional-tests/consumers/test_resultsdb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ def test_consume_new_result(
'subject_identifier': nvr,
'result_id': result['id'],
'testcase': 'dist.rpmdeplint',
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
'type': 'test-result-passed',
},
Expand Down Expand Up @@ -138,6 +141,9 @@ def test_consume_new_result(
'subject_identifier': nvr,
'result_id': result['id'],
'testcase': 'dist.rpmdeplint',
'scenario': None,
'system_architecture': None,
'system_variant': None,
'type': 'test-result-passed',
'source': None,
},
Expand Down Expand Up @@ -257,6 +263,9 @@ def test_consume_compose_id_result(
'subject_type': 'compose',
'subject_identifier': compose_id,
'result_id': result['id'],
'scenario': 'scenario1',
'system_architecture': None,
'system_variant': None,
'testcase': 'compose.install_no_user',
'source': None,
'type': 'test-result-passed'
Expand Down Expand Up @@ -329,6 +338,9 @@ def test_consume_legacy_result(
'subject_type': 'koji_build',
'subject_identifier': nvr,
'result_id': result['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'testcase': 'dist.rpmdeplint',
'source': None,
'type': 'test-result-passed'
Expand Down Expand Up @@ -391,6 +403,9 @@ def test_consume_legacy_result(
'subject_type': 'koji_build',
'subject_identifier': nvr,
'result_id': result['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'testcase': 'dist.rpmdeplint',
'source': None,
'type': 'test-result-passed'
Expand Down Expand Up @@ -596,6 +611,9 @@ def test_consume_new_result_container_image(
'subject_type': 'container-image',
'subject_identifier': item_hash,
'result_id': result['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'testcase': 'baseos-qe.baseos-ci.tier1.functional',
'source': None,
'type': 'test-result-passed'
Expand Down
18 changes: 18 additions & 0 deletions functional-tests/consumers/test_waiverdb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ def test_consume_new_waiver(
'subject_type': 'koji_build',
'subject_identifier': nvr,
'result_id': results[0]['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'testcase': passing_tests[0],
'source': None,
'type': 'test-result-passed'
Expand All @@ -88,6 +91,9 @@ def test_consume_new_waiver(
'subject_type': 'koji_build',
'subject_identifier': nvr,
'result_id': results[1]['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'testcase': passing_tests[1],
'source': None,
'type': 'test-result-passed'
Expand All @@ -97,9 +103,13 @@ def test_consume_new_waiver(
{
'result_id': result['id'],
'item': {'item': nvr, 'type': 'koji_build'},
'subject_identifier': nvr,
'subject_type': 'koji_build',
'testcase': failing_test,
'type': 'test-result-failed',
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
},
],
Expand All @@ -118,12 +128,17 @@ def test_consume_new_waiver(
'testcase': failing_test,
'type': 'test-result-failed-waived',
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
},
{
'subject_type': 'koji_build',
'subject_identifier': nvr,
'result_id': results[0]['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'testcase': passing_tests[0],
'source': None,
'type': 'test-result-passed'
Expand All @@ -132,6 +147,9 @@ def test_consume_new_waiver(
'subject_type': 'koji_build',
'subject_identifier': nvr,
'result_id': results[1]['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'testcase': passing_tests[1],
'source': None,
'type': 'test-result-passed'
Expand Down
40 changes: 40 additions & 0 deletions functional-tests/test_api_v1.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,9 @@ def test_make_a_decision_with_verbose_flag(requests_session, greenwave_server, t
'subject_type': 'koji_build',
'subject_identifier': nvr,
'source': None,
'scenario': None,
'system_architecture': None,
'system_variant': None,
} for result in results
]
assert res_data['satisfied_requirements'] == expected_satisfied_requirements
Expand Down Expand Up @@ -380,9 +383,13 @@ def test_make_a_decision_on_failed_result(requests_session, greenwave_server, te
expected_unsatisfied_requirements = [
{
'item': {'item': nvr, 'type': 'koji_build'},
'subject_type': 'koji_build',
'subject_identifier': nvr,
'result_id': result['id'],
'testcase': TASKTRON_RELEASE_CRITICAL_TASKS[0],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
'type': 'test-result-failed'
},
Expand Down Expand Up @@ -425,7 +432,10 @@ def test_make_a_decision_on_queued_result(requests_session, greenwave_server, te
'subject_identifier': result['data']['item'][0],
'subject_type': result['data']['type'][0],
'testcase': TASKTRON_RELEASE_CRITICAL_TASKS[0],
'result_id': result['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
'type': 'test-result-missing'
},
Expand Down Expand Up @@ -468,7 +478,10 @@ def test_make_a_decision_on_running_result(requests_session, greenwave_server, t
'subject_identifier': result['data']['item'][0],
'subject_type': result['data']['type'][0],
'testcase': TASKTRON_RELEASE_CRITICAL_TASKS[0],
'result_id': result['id'],
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
'type': 'test-result-missing'
},
Expand Down Expand Up @@ -550,18 +563,26 @@ def test_make_a_decision_on_redhat_cont_image(requests_session, greenwave_server
expected_unsatisfied_requirements = [
{
'item': {'item': item1_nvr, 'type': 'redhat-container-image'},
'subject_identifier': result1['data']['item'][0],
'subject_type': 'redhat-container-image',
'result_id': result1['id'],
'testcase': 'test.testcase1',
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
'type': 'test-result-failed'
},
{
'item': {'item': item1_nvr, 'type': 'redhat-container-image'},
'subject_identifier': result2['data']['item'][0],
'subject_type': 'redhat-container-image',
'result_id': result2['id'],
'testcase': 'test.testcase2',
'type': 'test-result-failed',
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
}
]
Expand Down Expand Up @@ -693,10 +714,14 @@ def test_multiple_results_in_a_subject(
expected_unsatisfied_requirements = [
{
'item': {'item': nvr, 'type': 'koji_build'},
'subject_identifier': nvr,
'subject_type': 'koji_build',
'result_id': result['id'],
'testcase': 'dist.abicheck',
'type': 'test-result-failed',
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
},
]
Expand Down Expand Up @@ -825,10 +850,14 @@ def test_make_a_decision_on_failing_result_with_scenario(
assert res_data['summary'] == expected_summary
expected_unsatisfied_requirements = [{
'item': {'productmd.compose.id': compose_id},
'subject_identifier': compose_id,
'subject_type': 'compose',
'result_id': results[-1]['id'],
'testcase': testcase_name,
'type': 'test-result-failed',
'scenario': 'scenario2',
'system_architecture': None,
'system_variant': None,
'source': None,
}]
assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements
Expand Down Expand Up @@ -871,10 +900,14 @@ def test_ignore_waiver(requests_session, greenwave_server, testdatabuilder):
expected_unsatisfied_requirements = [
{
'item': {'item': nvr, 'type': 'koji_build'},
'subject_identifier': nvr,
'subject_type': 'koji_build',
'result_id': result['id'],
'testcase': TASKTRON_RELEASE_CRITICAL_TASKS[0],
'type': 'test-result-failed',
'scenario': None,
'system_architecture': None,
'system_variant': None,
'source': None,
},
]
Expand Down Expand Up @@ -1148,9 +1181,13 @@ def test_make_a_decision_about_compose_all_variants_architectures(
assert not res_data['policies_satisfied']
assert res_data['unsatisfied_requirements'] == [{
'item': {'productmd.compose.id': compose_id},
'subject_identifier': compose_id,
'subject_type': 'compose',
'result_id': failed_results['id'],
'scenario': None,
'source': None,
'system_architecture': variant1['architecture'],
'system_variant': variant1['variant'],
'testcase': 'rtt.acceptance.validation',
'type': 'test-result-failed'
}]
Expand Down Expand Up @@ -1676,6 +1713,9 @@ def test_make_a_decision_on_passed_result_with_custom_scenario(
'subject_type': 'koji_build',
'result_id': result2['id'],
'testcase': 'test.testcase1',
'scenario': 'scenario1',
'system_architecture': None,
'system_variant': None,
'source': None,
'type': 'test-result-passed'
}
Expand Down
6 changes: 6 additions & 0 deletions greenwave/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@ class Config(object):
OUTCOMES_ERROR = ('ERROR',)
OUTCOMES_INCOMPLETE = ('QUEUED', 'RUNNING')

DISTINCT_LATEST_RESULTS_ON = (
'scenario',
'system_architecture',
'system_variant',
)


class ProductionConfig(Config):
DEBUG = False
Expand Down
Loading

0 comments on commit 9a7cd28

Please sign in to comment.