diff --git a/docs/examples.rst b/docs/examples.rst index e5bb3fa4e2..1a81185db1 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -328,6 +328,10 @@ directory:: url: https://github.com/psss/tmt.git Test case 'TC#0603489' successfully exported to nitrate. +Use the ``--bugzilla`` option together with ``--nitrate`` to link +bugs marked as ``verifies`` in the :ref:`/spec/core/link` +attribute with the corresponding Nitrate test case. + Test Libraries ------------------------------------------------------------------ diff --git a/setup.py b/setup.py index e9647c0840..e5307aae62 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ 'docs': ['sphinx', 'sphinx_rtd_theme', 'mock'], 'tests': ['pytest', 'python-coveralls', 'mock', 'requre', 'pre-commit'], 'provision': ['testcloud>=0.6.1'], - 'convert': ['nitrate', 'markdown'], + 'convert': ['nitrate', 'markdown', 'python-bugzilla'], 'report-html': ['jinja2'], 'report-junit': ['junit_xml'], } @@ -78,6 +78,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Topic :: Utilities', ], keywords=['metadata', 'testing'], diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index cf786c8b5f..242e384f61 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,7 +1,12 @@ +import xmlrpc.client + import nitrate +from bugzilla._backendxmlrpc import _BugzillaXMLRPCTransport +from requests import sessions from requre import cassette from requre.cassette import StorageKeysInspectSimple from requre.helpers.guess_object import Guess +from requre.helpers.requests_response import RequestResponseHandling # decorate functions what communicates with nitrate nitrate.xmlrpc_driver.GSSAPITransport.single_request = Guess.decorator_plain()( @@ -9,5 +14,12 @@ nitrate.xmlrpc_driver.GSSAPITransport.single_request_with_cookies = Guess.decorator_plain()( nitrate.xmlrpc_driver.GSSAPITransport.single_request_with_cookies) +# decorate functions that communicate with bugzilla (xmlrpc) +_BugzillaXMLRPCTransport.single_request = Guess.decorator_plain()( + _BugzillaXMLRPCTransport.single_request) +sessions.Session.send = RequestResponseHandling.decorator( + item_list=[1])( + sessions.Session.send) + # use storage simple strategy to avoid use full stack info for keys cassette.StorageKeysInspectDefault = StorageKeysInspectSimple diff --git a/tests/integration/data/nitrate/existing_testcase/main.fmf b/tests/integration/data/nitrate/existing_testcase/main.fmf index 3ea022d9ff..b9b924221e 100644 --- a/tests/integration/data/nitrate/existing_testcase/main.fmf +++ b/tests/integration/data/nitrate/existing_testcase/main.fmf @@ -1,2 +1,5 @@ summary: This is case what already exists inside nitrate extra-nitrate: TC#0609686 +link: +- verifies: https://bugzilla.redhat.com/show_bug.cgi?id=1925518 +- https://bugzilla.redhat.com/show_bug.cgi?id=1923314 diff --git a/tests/integration/main.fmf b/tests/integration/main.fmf index 0b2366cbfc..1b12e2cc08 100644 --- a/tests/integration/main.fmf +++ b/tests/integration/main.fmf @@ -35,7 +35,6 @@ description: | cd tests/integration sudo unshare -n sudo -u `whoami` pytest -v test_nitrate.py -test: "python3 -m pytest -v" environment: REQURE_MODE: read framework: shell @@ -44,3 +43,9 @@ require: tag: [integration] tier: null link: https://github.com/packit/requre + +/coverage_bugzilla: + test: "python3 -m pytest -v -k 'test_coverage_bugzilla'" + +/the_rest: + test: "python3 -m pytest -v -k 'not test_coverage_bugzilla'" diff --git a/tests/integration/test_data/test_nitrate/NitrateExport.test_coverage_bugzilla.yaml b/tests/integration/test_data/test_nitrate/NitrateExport.test_coverage_bugzilla.yaml new file mode 100644 index 0000000000..b3081e5ac9 --- /dev/null +++ b/tests/integration/test_data/test_nitrate/NitrateExport.test_coverage_bugzilla.yaml @@ -0,0 +1,537 @@ +_requre: + DataTypes: 1 + key_strategy: StorageKeysInspectSimple + version_storage_file: 3 +nitrate.xmlrpc_driver: + single_request_with_cookies: + - metadata: + guess_type: Tuple + latency: 1.021317720413208 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - nitrate.base + - nitrate.mutable + - nitrate.base + - nitrate.xmlrpc_driver + - xmlrpc.client + - requre.objects + - requre.cassette + - nitrate.xmlrpc_driver + - single_request_with_cookies + output: + - oqybpy20lj0xhzgikwe2x6k077fh6rox + - metadata: + guess_type: Tuple + latency: 0.9072721004486084 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - nitrate.base + - nitrate.mutable + - xmlrpc.client + - requre.objects + - requre.cassette + - nitrate.xmlrpc_driver + - single_request_with_cookies + output: + - alias: '' + arguments: ' ' + attachment: [] + author: jscotka + author_id: 2150 + case_id: 609686 + case_status: CONFIRMED + case_status_id: 2 + category: Sanity + category_id: 518 + component: [] + create_date: '2021-02-24 01:07:39' + default_tester: null + default_tester_id: null + estimated_time: 00:05:00 + extra_link: '' + is_automated: 1 + is_automated_proposed: false + notes: 'Test case has been migrated to git. Any changes made here might be overwritten. + + See: https://tmt.readthedocs.io/en/latest/questions.html#nitrate-migration + + + [structured-field-start] + + This is StructuredField version 1. Please, edit with care. + + + [description] + + This is case what already exists inside nitrate + + + [fmf] + + name: /existing_testcase + + url: https://github.com/psss/tmt.git + + ref: lzachar-export-bz + + path: /tests/integration_5dy2mg9 + + + [structured-field-end] + + ' + plan: + - 29309 + priority: P3 + priority_id: 3 + requirement: '' + reviewer: null + reviewer_id: null + script: null + summary: This is case what already exists inside nitrate + tag: + - fmf-export + text: + action: '' + action_checksum: d41d8cd98f00b204e9800998ecf8427e + author: jscotka + author_id: 2150 + breakdown: '' + breakdown_checksum: d41d8cd98f00b204e9800998ecf8427e + case: This is case what already exists inside nitrate + case_id: 609686 + case_text_version: 1 + create_date: '2021-02-24 01:07:39' + effect: '' + effect_checksum: d41d8cd98f00b204e9800998ecf8427e + id: 878121 + setup: '' + setup_checksum: d41d8cd98f00b204e9800998ecf8427e + - metadata: + guess_type: Tuple + latency: 0.9036877155303955 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - nitrate.containers + - xmlrpc.client + - requre.objects + - requre.cassette + - nitrate.xmlrpc_driver + - single_request_with_cookies + output: + - [] + - metadata: + guess_type: Tuple + latency: 0.9099326133728027 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - nitrate.containers + - xmlrpc.client + - requre.objects + - requre.cassette + - nitrate.xmlrpc_driver + - single_request_with_cookies + output: + - - id: 10700 + name: fmf-export + - metadata: + guess_type: Tuple + latency: 0.9249658584594727 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - nitrate.containers + - xmlrpc.client + - requre.objects + - requre.cassette + - nitrate.xmlrpc_driver + - single_request_with_cookies + output: + - - bug_id: '1925518' + bug_system: Bugzilla + bug_system_id: 1 + case: This is case what already exists inside nitrate + case_id: 609686 + case_run: null + case_run_id: null + description: '' + id: 250305 + summary: '' + - metadata: + guess_type: Tuple + latency: 0.8624577522277832 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - nitrate.mutable + - nitrate.base + - nitrate.immutable + - xmlrpc.client + - requre.objects + - requre.cassette + - nitrate.xmlrpc_driver + - single_request_with_cookies + output: + - description: '' + id: 518 + name: Sanity + product: RHEL Tests + product_id: 182 + - metadata: + guess_type: Tuple + latency: 1.9875798225402832 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - nitrate.mutable + - xmlrpc.client + - requre.objects + - requre.cassette + - nitrate.xmlrpc_driver + - single_request_with_cookies + output: + - - alias: '' + arguments: ' ' + attachment: [] + author: jscotka + author_id: 2150 + case_id: 609686 + case_status: CONFIRMED + case_status_id: 2 + category: Sanity + category_id: 518 + component: [] + create_date: '2021-02-24 01:07:39' + default_tester: null + default_tester_id: null + estimated_time: 00:05:00 + extra_link: '' + is_automated: 1 + is_automated_proposed: false + notes: 'Test case has been migrated to git. Any changes made here might be + overwritten. + + See: https://tmt.readthedocs.io/en/latest/questions.html#nitrate-migration + + + [structured-field-start] + + This is StructuredField version 1. Please, edit with care. + + + [description] + + This is case what already exists inside nitrate + + + [fmf] + + name: /existing_testcase + + url: https://github.com/psss/tmt.git + + ref: lzachar-export-bz + + path: /tests/integrationqk67wvkc + + + [structured-field-end] + + ' + plan: + - 29309 + priority: P3 + priority_id: 3 + requirement: '' + reviewer: null + reviewer_id: null + script: null + summary: This is case what already exists inside nitrate + tag: + - 10700 +requests.sessions: + send: + POST: + https://bugzilla.redhat.com/xmlrpc.cgi: + - metadata: + latency: 0.8715624809265137 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - bugzilla.base + - bugzilla._backendxmlrpc + - xmlrpc.client + - bugzilla._backendxmlrpc + - xmlrpc.client + - bugzilla._backendxmlrpc + - bugzilla._session + - requests.sessions + - requre.objects + - requre.cassette + - requests.sessions + - send + output: + __store_indicator: 1 + _content: version5.0.4.rh56 + _next: null + elapsed: 0.870112 + encoding: ISO-8859-1 + headers: + Cache-Control: no-cache, no-store + Connection: close + Content-Encoding: gzip + Content-Type: text/xml + Date: Fri, 18 Jun 2021 10:46:18 GMT + ETag: 7tA/qr9x3KS7RnQJUgmnpw + SOAPServer: SOAP::Lite/Perl/1.11 + Server: Apache + Set-Cookie: Bugzilla_login_request_cookie=EOGYkfRr0N; domain=bugzilla.redhat.com; + path=/; HttpOnly; SameSite=Lax + Transfer-Encoding: chunked + Vary: Accept-Encoding,User-Agent + X-content-type-options: nosniff + X-frame-options: SAMEORIGIN + X-xss-protection: 1; mode=block + raw: !!binary "" + reason: OK + status_code: 200 + - metadata: + latency: 1.1678190231323242 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - bugzilla.base + - bugzilla._backendxmlrpc + - xmlrpc.client + - bugzilla._backendxmlrpc + - xmlrpc.client + - bugzilla._backendxmlrpc + - bugzilla._session + - requests.sessions + - requre.objects + - requre.cassette + - requests.sessions + - send + output: + __store_indicator: 1 + _content: usersreal_nameNeed + Real Nameemailaander07@packetmaster.comnameaander07@packetmaster.comid1can_login1 + _next: null + elapsed: 1.167032 + encoding: ISO-8859-1 + headers: + Cache-Control: no-cache, no-store + Connection: close + Content-Encoding: gzip + Content-Type: text/xml + Date: Fri, 18 Jun 2021 10:46:11 GMT + ETag: O6zBQsTG+AbgcJDlcPMIsg + SOAPServer: SOAP::Lite/Perl/1.11 + Server: Apache + Transfer-Encoding: chunked + Vary: Accept-Encoding,User-Agent + X-content-type-options: nosniff + X-frame-options: SAMEORIGIN + X-xss-protection: 1; mode=block + raw: !!binary "" + reason: OK + status_code: 200 + - metadata: + latency: 1.3847901821136475 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - xmlrpc.client + - bugzilla._backendxmlrpc + - xmlrpc.client + - bugzilla._backendxmlrpc + - bugzilla._session + - requests.sessions + - requre.objects + - requre.cassette + - requests.sessions + - send + output: + __store_indicator: 1 + _content: faultsbugsexternal_bugsflagsid1925518 + _next: null + elapsed: 1.383814 + encoding: ISO-8859-1 + headers: + Cache-Control: no-cache, no-store + Connection: close + Content-Encoding: gzip + Content-Type: text/xml + Date: Fri, 18 Jun 2021 10:46:28 GMT + ETag: wtTJ8Uk+3VST17ep4YNK/w + SOAPServer: SOAP::Lite/Perl/1.11 + Server: Apache + Transfer-Encoding: chunked + Vary: Accept-Encoding,User-Agent + X-content-type-options: nosniff + X-frame-options: SAMEORIGIN + X-xss-protection: 1; mode=block + raw: !!binary "" + reason: OK + status_code: 200 + - metadata: + latency: 1.312206506729126 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - xmlrpc.client + - bugzilla._backendxmlrpc + - xmlrpc.client + - bugzilla._backendxmlrpc + - bugzilla._session + - requests.sessions + - requre.objects + - requre.cassette + - requests.sessions + - send + output: + __store_indicator: 1 + _content: faultStringNo + data given for change or flag name invalid.faultCode700 + _next: null + elapsed: 1.311867 + encoding: ISO-8859-1 + headers: + Cache-Control: no-cache, no-store + Connection: close + Content-Encoding: gzip + Content-Type: text/xml + Date: Fri, 18 Jun 2021 10:46:29 GMT + ETag: EO13om6PW6n8rJLIjuuntQ + SOAPServer: SOAP::Lite/Perl/1.11 + Server: Apache + Transfer-Encoding: chunked + Vary: Accept-Encoding,User-Agent + X-content-type-options: nosniff + X-frame-options: SAMEORIGIN + X-xss-protection: 1; mode=block + raw: !!binary "" + reason: OK + status_code: 200 + - metadata: + latency: 3.484098196029663 + module_call_list: + - unittest.case + - integration.test_nitrate + - click.testing + - click.core + - click.decorators + - tmt.cli + - tmt.base + - tmt.export + - xmlrpc.client + - bugzilla._backendxmlrpc + - xmlrpc.client + - bugzilla._backendxmlrpc + - bugzilla._session + - requests.sessions + - requre.objects + - requre.cassette + - requests.sessions + - send + output: + __store_indicator: 1 + _content: bugschangesextra_versionsremovedaddedextra_componentsremovedaddedext_bz_bug_map.ext_bz_bug_idremovedaddedTCMS Test + Case 609686last_change_time20210618T10:46:31id1925518alias + _next: null + elapsed: 3.483175 + encoding: ISO-8859-1 + headers: + Cache-Control: no-cache, no-store + Connection: close + Content-Encoding: gzip + Content-Type: text/xml + Date: Fri, 18 Jun 2021 10:46:23 GMT + ETag: mQAR7Oe6VHOWnNHZQP3U4w + SOAPServer: SOAP::Lite/Perl/1.11 + Server: Apache + Transfer-Encoding: chunked + Vary: Accept-Encoding,User-Agent + X-content-type-options: nosniff + X-frame-options: SAMEORIGIN + X-xss-protection: 1; mode=block + raw: !!binary "" + reason: OK + status_code: 200 diff --git a/tests/integration/test_nitrate.py b/tests/integration/test_nitrate.py index e482301ec0..b648d0c6a7 100644 --- a/tests/integration/test_nitrate.py +++ b/tests/integration/test_nitrate.py @@ -64,6 +64,17 @@ def test_existing(self): self.assertEqual(fmf_node.data["extra-nitrate"], "TC#0609686") + def test_coverage_bugzilla(self): + fmf_node = Tree(self.tmpdir).find("/existing_testcase") + self.assertEqual(fmf_node.data["extra-nitrate"], "TC#0609686") + + os.chdir(self.tmpdir / "existing_testcase") + runner = CliRunner() + self.runner_output = runner.invoke(tmt.cli.main, + ["test", "export", "--nitrate", + "--bugzilla", "."]) + assert self.runner_output.exit_code == 0 + class NitrateImport(Base): diff --git a/tmt.spec b/tmt.spec index 806bfbd127..e026382cee 100644 --- a/tmt.spec +++ b/tmt.spec @@ -84,6 +84,7 @@ Dependencies required to run tests in a local virtual machine. %package test-convert Summary: Test import and export dependencies Requires: make python3-nitrate python3-html2text python3-markdown +Requires: python3-bugzilla %description test-convert Additional dependencies needed for test metadata import and export. diff --git a/tmt/cli.py b/tmt/cli.py index f9e26baa3f..0ae547ae0d 100644 --- a/tmt/cli.py +++ b/tmt/cli.py @@ -501,6 +501,10 @@ def import_( @click.option( '--nitrate', is_flag=True, help='Export test metadata to Nitrate.') +@click.option( + '--bugzilla', is_flag=True, + help="Link Nitrate case to Bugzilla specified in the 'link' attribute " + "with the relation 'verifies'.") @click.option( '--create', is_flag=True, help="Create test cases in nitrate if they don't exist.") @@ -521,7 +525,7 @@ def import_( @click.option( '-d', '--debug', is_flag=True, help='Provide as much debugging details as possible.') -def export(context, format_, nitrate, **kwargs): +def export(context, format_, nitrate, bugzilla, **kwargs): """ Export test data into the desired format. @@ -529,6 +533,9 @@ def export(context, format_, nitrate, **kwargs): Use '.' to select tests under the current working directory. """ tmt.Test._save_context(context) + if bugzilla and not nitrate: + raise tmt.utils.GeneralError( + "The --bugzilla option is supported only with --nitrate for now.") for test in context.obj.tree.tests(): if nitrate: test.export(format_='nitrate') diff --git a/tmt/export.py b/tmt/export.py index f8e50be1c6..5edd311700 100644 --- a/tmt/export.py +++ b/tmt/export.py @@ -6,6 +6,8 @@ import email import os import re +import traceback +import xmlrpc.client from functools import lru_cache import fmf @@ -21,6 +23,11 @@ See: https://tmt.readthedocs.io/en/latest/questions.html#nitrate-migration """.lstrip() +# For linking bugs +BUGZILLA_XMLRPC_URL = "https://bugzilla.redhat.com/xmlrpc.cgi" +EXTERNAL_TRACKER_ID = 69 # ID of nitrate in RH's bugzilla +RE_BUGZILLA_URL = r'bugzilla.redhat.com/show_bug.cgi\?id=(\d+)' + def import_nitrate(): """ Conditionally import the nitrate module """ @@ -40,6 +47,16 @@ def import_nitrate(): raise ConvertError(error) +def import_bugzilla(): + """ Conditionally import the bugzilla module """ + try: + global bugzilla + import bugzilla + except ImportError: + raise ConvertError( + "Install 'tmt-test-convert' to link test to the bugzilla.") + + def _nitrate_find_fmf_testcases(test): """ Find all Nitrate test cases with the same fmf identifier @@ -155,6 +172,56 @@ def check_section_exists(text): return step, expect, setup, cleanup +def bz_set_coverage(bz_instance, bug_ids, case_id): + """ Set coverage in Bugzilla """ + overall_pass = True + no_email = 1 # Do not send emails about the change + get_bz_dict = { + 'ids': bug_ids, + 'include_fields': ['id', 'external_bugs', 'flags']} + bugs_data = bz_instance._proxy.Bug.get(get_bz_dict) + for bug in bugs_data['bugs']: + # Process flag (might fail for some types) + bug_id = bug['id'] + if 'qe_test_coverage+' not in set( + [x['name'] + x['status'] for x in bug['flags']]): + try: + bz_instance._proxy.Flag.update({ + 'ids': [bug_id], + 'nomail': no_email, + 'updates': [{ + 'name': 'qe_test_coverage', + 'status': '+' + }] + }) + except xmlrpc.client.Fault as err: + log.debug(f"Update flag failed: {err}") + echo(style( + f"Failed to set qe_test_coverage+ flag for BZ#{bug_id}", + fg='red')) + # Process external tracker - should succeed + current = set([int(b['ext_bz_bug_id']) for b in bug['external_bugs'] + if b['ext_bz_id'] == EXTERNAL_TRACKER_ID]) + if case_id not in current: + query = { + 'bug_ids': [bug_id], + 'nomail': no_email, + 'external_bugs': [{ + 'ext_type_id': EXTERNAL_TRACKER_ID, + 'ext_bz_bug_id': case_id, + 'ext_description': '', + }] + } + try: + bz_instance._proxy.ExternalBugs.add_external_bug(query) + except Exception as err: + log.debug(f"Link case failed: {err}") + echo(style(f"Failed to link to BZ#{bug_id}", fg='red')) + overall_pass = False + if not overall_pass: + raise ConvertError("Failed to link the case to bugs.") + + def export_to_nitrate(test): """ Export fmf metadata to nitrate test cases """ import_nitrate() @@ -164,6 +231,20 @@ def export_to_nitrate(test): create = test.opt('create') general = test.opt('general') duplicate = test.opt('duplicate') + link_bugzilla = test.opt('bugzilla') + + if link_bugzilla: + import_bugzilla() + try: + bz_instance = bugzilla.Bugzilla(url=BUGZILLA_XMLRPC_URL) + except Exception as exc: + log.debug(traceback.format_exc()) + raise ConvertError( + "Couldn't initialize the Bugzilla client.", original=exc) + if not bz_instance.logged_in: + raise ConvertError( + "Not logged to Bugzilla, check `man bugzilla` section " + "'AUTHENTICATION CACHE AND API KEYS'.") # Check nitrate test case try: @@ -350,11 +431,35 @@ def export_to_nitrate(test): except IOError: raise ConvertError("Unable to open '{0}'.".format(file_path)) + # List of bugs test verifies + verifies_bug_ids = [] + for link in test.link: + try: + verifies_bug_ids.append( + int(re.search(RE_BUGZILLA_URL, link['verifies']).group(1))) + except Exception as err: + log.debug(err) + + # Add bugs to the Nitrate case + for bug_id in verifies_bug_ids: + nitrate_case.bugs.add(nitrate.Bug(bug=int(bug_id))) + # Update nitrate test case nitrate_case.update() echo(style("Test case '{0}' successfully exported to nitrate.".format( nitrate_case.identifier), fg='magenta')) + # Optionally link Bugzilla to Nitrate case + if link_bugzilla and verifies_bug_ids: + try: + bz_set_coverage( + bz_instance, verifies_bug_ids, int( + nitrate_case.id)) + echo(style("Linked to Bugzilla: {}.".format( + " ".join([f"BZ#{bz_id}" for bz_id in verifies_bug_ids])), fg='magenta')) + except Exception as err: + raise ConvertError("Couldn't update bugs", original=err) + def create_nitrate_case(test): """ Create new nitrate case """