From f34b0023732901fb2a06dc8dc19d45a48fa9244e Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 28 Oct 2022 18:41:47 +0800 Subject: [PATCH] HBASE-27450 Update all our python scripts to use python3 (#4851) Signed-off-by: Guanghao Zhang (cherry picked from commit cdabfd3ca811d1c5ef79e9b6bd8106b2d6d12cd5) --- dev-support/Dockerfile | 10 +- dev-support/Jenkinsfile | 12 +- dev-support/Jenkinsfile_GitHub | 8 +- dev-support/checkcompatibility.py | 54 +-- dev-support/checkstyle_report.py | 56 ++-- dev-support/flaky-tests/findHangingTests.py | 38 +-- .../flaky-tests/flaky-reporting.Jenkinsfile | 2 +- dev-support/flaky-tests/report-flakies.py | 28 +- dev-support/gen_redirect_html.py | 8 +- dev-support/submit-patch.py | 311 ------------------ 10 files changed, 112 insertions(+), 415 deletions(-) delete mode 100755 dev-support/submit-patch.py diff --git a/dev-support/Dockerfile b/dev-support/Dockerfile index 923e26563b3a..44822e9fb44b 100644 --- a/dev-support/Dockerfile +++ b/dev-support/Dockerfile @@ -20,16 +20,14 @@ # # Specifically, it's used for the flaky test reporting job defined in # dev-support/flaky-tests/flaky-reporting.Jenkinsfile -FROM ubuntu:18.04 +FROM ubuntu:22.04 COPY . /hbase/dev-support RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \ && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \ - curl='7.58.0-*' \ - python2.7='2.7.17-*' \ - python-pip='9.0.1-*' \ - python-setuptools='39.0.1-*' \ + curl='7.81.0-*' \ + python3-pip='22.0.2+dfsg-*' \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -RUN pip install -r /hbase/dev-support/python-requirements.txt +RUN pip3 install -r /hbase/dev-support/python-requirements.txt diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index f287f01f637d..2098fff80085 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -267,14 +267,14 @@ pipeline { if [ -d "${OUTPUT_DIR}/branch-site" ]; then echo "Remove ${OUTPUT_DIR}/branch-site for saving space" rm -rf "${OUTPUT_DIR}/branch-site" - python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html" else echo "No branch-site, skipping" fi if [ -d "${OUTPUT_DIR}/patch-site" ]; then echo "Remove ${OUTPUT_DIR}/patch-site for saving space" rm -rf "${OUTPUT_DIR}/patch-site" - python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html" else echo "No patch-site, skipping" fi @@ -384,7 +384,7 @@ pipeline { if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -493,7 +493,7 @@ pipeline { if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -604,7 +604,7 @@ pipeline { if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space" rm -rf "${OUTPUT_DIR}/test_logs.zip" - python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -773,7 +773,7 @@ pipeline { if [ -f "${SRC_TAR}" ]; then echo "Remove ${SRC_TAR} for saving space" rm -rf "${SRC_TAR}" - python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html" + python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html" else echo "No hbase-src.tar.gz, skipping" fi diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub index 4eee610b8b13..351f1f30b544 100644 --- a/dev-support/Jenkinsfile_GitHub +++ b/dev-support/Jenkinsfile_GitHub @@ -143,14 +143,14 @@ pipeline { if [ -d "${PATCHDIR}/branch-site" ]; then echo "Remove ${PATCHDIR}/branch-site for saving space" rm -rf "${PATCHDIR}/branch-site" - python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html" + python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html" else echo "No branch-site, skipping" fi if [ -d "${PATCHDIR}/patch-site" ]; then echo "Remove ${PATCHDIR}/patch-site for saving space" rm -rf "${PATCHDIR}/patch-site" - python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html" + python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html" else echo "No patch-site, skipping" fi @@ -282,7 +282,7 @@ pipeline { if [ -f "${PATCHDIR}/test_logs.zip" ]; then echo "Remove ${PATCHDIR}/test_logs.zip for saving space" rm -rf "${PATCHDIR}/test_logs.zip" - python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" + python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi @@ -415,7 +415,7 @@ pipeline { if [ -f "${PATCHDIR}/test_logs.zip" ]; then echo "Remove ${PATCHDIR}/test_logs.zip for saving space" rm -rf "${PATCHDIR}/test_logs.zip" - python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" + python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html" else echo "No test_logs.zip, skipping" fi diff --git a/dev-support/checkcompatibility.py b/dev-support/checkcompatibility.py index 67acbb35442a..914f8dd42f17 100755 --- a/dev-support/checkcompatibility.py +++ b/dev-support/checkcompatibility.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -32,7 +32,7 @@ # --annotation org.apache.yetus.audience.InterfaceAudience.LimitedPrivate \ # --include-file "hbase-*" \ # --known_problems_path ~/known_problems.json \ -# rel/1.0.0 branch-1.2 +# rel/1.3.0 branch-1.4 import json import logging @@ -41,7 +41,9 @@ import shutil import subprocess import sys -import urllib2 +import urllib.request +import urllib.error +import urllib.parse from collections import namedtuple try: import argparse @@ -55,11 +57,11 @@ def check_output(*popenargs, **kwargs): - """ Run command with arguments and return its output as a byte string. - Backported from Python 2.7 as it's implemented as pure python on stdlib. - >>> check_output(['/usr/bin/python', '--version']) - Python 2.6.2 """ - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + """ Run command with arguments and return its output as a byte string. """ + process = subprocess.Popen(stdout=subprocess.PIPE, + universal_newlines=True, + *popenargs, + **kwargs) output, _ = process.communicate() retcode = process.poll() if retcode: @@ -69,7 +71,7 @@ def check_output(*popenargs, **kwargs): error = subprocess.CalledProcessError(retcode, cmd) error.output = output raise error - return output + return output.strip() def get_repo_dir(): @@ -161,7 +163,7 @@ def checkout_java_acc(force): url = "https://github.com/lvc/japi-compliance-checker/archive/2.4.tar.gz" scratch_dir = get_scratch_dir() path = os.path.join(scratch_dir, os.path.basename(url)) - jacc = urllib2.urlopen(url) + jacc = urllib.request.urlopen(url) with open(path, 'wb') as w: w.write(jacc.read()) @@ -196,8 +198,8 @@ def ascii_encode_dict(data): """ Iterate through a dictionary of data and convert all unicode to ascii. This method was taken from stackoverflow.com/questions/9590382/forcing-python-json-module-to-work-with-ascii """ - ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x - return dict(map(ascii_encode, pair) for pair in data.items()) + ascii_encode = lambda x: x.encode('ascii') if isinstance(x, str) else x + return dict(list(map(ascii_encode, pair)) for pair in list(data.items())) def process_json(path): @@ -229,9 +231,9 @@ def compare_results(tool_results, known_issues, compare_warnings): unexpected_issues = [unexpected_issue(check=check, issue_type=issue_type, known_count=known_count, observed_count=tool_results[check][issue_type]) - for check, known_issue_counts in known_issues.items() - for issue_type, known_count in known_issue_counts.items() - if tool_results[check][issue_type] > known_count] + for check, known_issue_counts in list(known_issues.items()) + for issue_type, known_count in list(known_issue_counts.items()) + if compare_tool_results_count(tool_results, check, issue_type, known_count)] if not compare_warnings: unexpected_issues = [tup for tup in unexpected_issues @@ -243,6 +245,14 @@ def compare_results(tool_results, known_issues, compare_warnings): return bool(unexpected_issues) +def compare_tool_results_count(tool_results, check, issue_type, known_count): + """ Check problem counts are no more than the known count. + (This function exists just so can add in logging; previous was inlined + one-liner but this made it hard debugging) + """ + # logging.info("known_count=%s, check key=%s, tool_results=%s, issue_type=%s", + # str(known_count), str(check), str(tool_results), str(issue_type)) + return tool_results[check][issue_type] > known_count def process_java_acc_output(output): """ Process the output string to find the problems and warnings in both the @@ -301,14 +311,14 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations, skip_annot logging.info("Annotations are: %s", annotations) annotations_path = os.path.join(get_scratch_dir(), "annotations.txt") logging.info("Annotations path: %s", annotations_path) - with file(annotations_path, "w") as f: + with open(annotations_path, "w") as f: f.write('\n'.join(annotations)) args.extend(["-annotations-list", annotations_path]) if skip_annotations is not None: skip_annotations_path = os.path.join( get_scratch_dir(), "skip_annotations.txt") - with file(skip_annotations_path, "w") as f: + with open(skip_annotations_path, "w") as f: f.write('\n'.join(skip_annotations)) args.extend(["-skip-annotations-list", skip_annotations_path]) @@ -327,14 +337,14 @@ def get_known_problems(json_path, src_rev, dst_rev): keys in the format source_branch/destination_branch and the values dictionaries with binary and source problems and warnings Example: - {'branch-1.0.0': { - 'rel/1.0.0': {'binary': {'problems': 123, 'warnings': 16}, + {'branch-1.3': { + 'rel/1.3.0': {'binary': {'problems': 123, 'warnings': 16}, 'source': {'problems': 167, 'warnings': 1}}, - 'branch-1.2.0': {'binary': {'problems': 0, 'warnings': 0}, + 'branch-1.4': {'binary': {'problems': 0, 'warnings': 0}, 'source': {'problems': 0, 'warnings': 0}} }, - 'branch-1.2.0': { - 'rel/1.2.1': {'binary': {'problems': 13, 'warnings': 1}, + 'branch-1.4': { + 'rel/1.4.1': {'binary': {'problems': 13, 'warnings': 1}, 'source': {'problems': 23, 'warnings': 0}} } } """ diff --git a/dev-support/checkstyle_report.py b/dev-support/checkstyle_report.py index c5810be152b3..99092c23b8ca 100755 --- a/dev-support/checkstyle_report.py +++ b/dev-support/checkstyle_report.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python3 ## # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -29,8 +29,8 @@ from collections import defaultdict if len(sys.argv) != 3 : - print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0] - exit(1) + print("usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]) + sys.exit(1) def path_key(x): path = x.attrib['name'] @@ -40,8 +40,8 @@ def error_name(x): error_class = x.attrib['source'] return error_class[error_class.rfind(".") + 1:] -def print_row(path, error, master_errors, patch_errors): - print '%s\t%s\t%s\t%s' % (path,error, master_errors,patch_errors) +def print_row(path, err, master_errors, patch_errors): + print('%s\t%s\t%s\t%s' % (path, err, master_errors, patch_errors)) master = etree.parse(sys.argv[1]) patch = etree.parse(sys.argv[2]) @@ -49,32 +49,32 @@ def print_row(path, error, master_errors, patch_errors): master_dict = defaultdict(int) ret_value = 0 -for child in master.getroot().getchildren(): +for child in list(master.getroot()): if child.tag != 'file': - continue + continue file = path_key(child) - for error_tag in child.getchildren(): - error = error_name(error_tag) - if (file, error) in master_dict: - master_dict[(file, error)] += 1 - else: - master_dict[(file, error)] = 1 + for error_tag in list(child): + error = error_name(error_tag) + if (file, error) in master_dict: + master_dict[(file, error)] += 1 + else: + master_dict[(file, error)] = 1 -for child in patch.getroot().getchildren(): - if child.tag != 'file': - continue - temp_dict = defaultdict(int) - for error_tag in child.getchildren(): - error = error_name(error_tag) - if error in temp_dict: - temp_dict[error] += 1 - else: - temp_dict[error] = 1 +for child in list(patch.getroot()): + if child.tag != 'file': + continue + temp_dict = defaultdict(int) + for error_tag in list(child): + error = error_name(error_tag) + if error in temp_dict: + temp_dict[error] += 1 + else: + temp_dict[error] = 1 - file = path_key(child) - for error, count in temp_dict.iteritems(): - if count > master_dict[(file, error)]: - print_row(file, error, master_dict[(file, error)], count) - ret_value = 1 + file = path_key(child) + for error, count in temp_dict.items(): + if count > master_dict[(file, error)]: + print_row(file, error, master_dict[(file, error)], count) + ret_value = 1 sys.exit(ret_value) diff --git a/dev-support/flaky-tests/findHangingTests.py b/dev-support/flaky-tests/findHangingTests.py index f8416e0a1e11..bb0a9a5be619 100755 --- a/dev-support/flaky-tests/findHangingTests.py +++ b/dev-support/flaky-tests/findHangingTests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 ## # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -45,8 +45,8 @@ def get_bad_tests(console_url): """ response = requests.get(console_url) if response.status_code != 200: - print "Error getting consoleText. Response = {} {}".format( - response.status_code, response.reason) + print("Error getting consoleText. Response = {} {}".format( + response.status_code, response.reason)) return # All tests: All testcases which were run. @@ -59,13 +59,13 @@ def get_bad_tests(console_url): hanging_tests_set = set() failed_tests_set = set() timeout_tests_set = set() - for line in response.content.splitlines(): + for line in response.content.decode("utf-8").splitlines(): result1 = re.findall("Running org.apache.hadoop.hbase.(.*)", line) if len(result1) == 1: test_case = result1[0] if test_case in all_tests_set: - print ("ERROR! Multiple tests with same name '{}'. Might get wrong results " - "for this test.".format(test_case)) + print(("ERROR! Multiple tests with same name '{}'. Might get wrong results " + "for this test.".format(test_case))) else: hanging_tests_set.add(test_case) all_tests_set.add(test_case) @@ -75,9 +75,9 @@ def get_bad_tests(console_url): if "FAILURE!" in line: failed_tests_set.add(test_case) if test_case not in hanging_tests_set: - print ("ERROR! No test '{}' found in hanging_tests. Might get wrong results " + print(("ERROR! No test '{}' found in hanging_tests. Might get wrong results " "for this test. This may also happen if maven is set to retry failing " - "tests.".format(test_case)) + "tests.".format(test_case))) else: hanging_tests_set.remove(test_case) result3 = re.match("^\\s+(\\w*).*\\sTestTimedOut", line) @@ -86,30 +86,30 @@ def get_bad_tests(console_url): timeout_tests_set.add(test_case) for bad_string in BAD_RUN_STRINGS: if re.match(".*" + bad_string + ".*", line): - print "Bad string found in build:\n > {}".format(line) - print "Result > total tests: {:4} failed : {:4} timedout : {:4} hanging : {:4}".format( - len(all_tests_set), len(failed_tests_set), len(timeout_tests_set), len(hanging_tests_set)) + print("Bad string found in build:\n > {}".format(line)) + print("Result > total tests: {:4} failed : {:4} timedout : {:4} hanging : {:4}".format( + len(all_tests_set), len(failed_tests_set), len(timeout_tests_set), len(hanging_tests_set))) return [all_tests_set, failed_tests_set, timeout_tests_set, hanging_tests_set] if __name__ == "__main__": if len(sys.argv) != 2: - print "ERROR : Provide the jenkins job console URL as the only argument." + print("ERROR : Provide the jenkins job console URL as the only argument.") sys.exit(1) - print "Fetching {}".format(sys.argv[1]) + print("Fetching {}".format(sys.argv[1])) result = get_bad_tests(sys.argv[1]) if not result: sys.exit(1) [all_tests, failed_tests, timedout_tests, hanging_tests] = result - print "Found {} hanging tests:".format(len(hanging_tests)) + print("Found {} hanging tests:".format(len(hanging_tests))) for test in hanging_tests: - print test - print "\n" - print "Found {} failed tests of which {} timed out:".format( - len(failed_tests), len(timedout_tests)) + print(test) + print("\n") + print("Found {} failed tests of which {} timed out:".format( + len(failed_tests), len(timedout_tests))) for test in failed_tests: - print "{0} {1}".format(test, ("(Timed Out)" if test in timedout_tests else "")) + print("{0} {1}".format(test, ("(Timed Out)" if test in timedout_tests else ""))) print ("\nA test may have had 0 or more atomic test failures before it timed out. So a " "'Timed Out' test may have other errors too.") diff --git a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile index b37045b3a6c2..f50900954532 100644 --- a/dev-support/flaky-tests/flaky-reporting.Jenkinsfile +++ b/dev-support/flaky-tests/flaky-reporting.Jenkinsfile @@ -47,7 +47,7 @@ pipeline { flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 50) docker build -t hbase-dev-support dev-support docker run --ulimit nproc=12500 -v "${WORKSPACE}":/hbase -u `id -u`:`id -g` --workdir=/hbase hbase-dev-support \ - python dev-support/flaky-tests/report-flakies.py --mvn -v -o output "${flaky_args[@]}" + ./dev-support/flaky-tests/report-flakies.py --mvn -v -o output "${flaky_args[@]}" ''' sshPublisher(publishers: [ sshPublisherDesc(configName: 'Nightlies', diff --git a/dev-support/flaky-tests/report-flakies.py b/dev-support/flaky-tests/report-flakies.py index e2d30b0d6dc3..16096e3344a5 100755 --- a/dev-support/flaky-tests/report-flakies.py +++ b/dev-support/flaky-tests/report-flakies.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 ## # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -140,7 +140,7 @@ def expand_multi_config_projects(cli_args): raise Exception("Failed to get job information from jenkins for url '" + job_url + "'. Jenkins returned HTTP status " + str(request.status_code)) response = request.json() - if response.has_key("activeConfigurations"): + if "activeConfigurations" in response: for config in response["activeConfigurations"]: final_expanded_urls.append({'url':config["url"], 'max_builds': max_builds, 'excludes': excluded_builds, 'is_yetus': is_yetus}) @@ -167,7 +167,7 @@ def expand_multi_config_projects(cli_args): url = url_max_build["url"] excludes = url_max_build["excludes"] json_response = requests.get(url + "/api/json?tree=id,builds%5Bnumber,url%5D").json() - if json_response.has_key("builds"): + if "builds" in json_response: builds = json_response["builds"] logger.info("Analyzing job: %s", url) else: @@ -238,27 +238,27 @@ def expand_multi_config_projects(cli_args): # Sort tests in descending order by flakyness. sorted_test_to_build_ids = OrderedDict( - sorted(test_to_build_ids.iteritems(), key=lambda x: x[1]['flakyness'], reverse=True)) + sorted(iter(test_to_build_ids.items()), key=lambda x: x[1]['flakyness'], reverse=True)) url_to_bad_test_results[url] = sorted_test_to_build_ids if len(sorted_test_to_build_ids) > 0: - print "URL: {}".format(url) - print "{:>60} {:10} {:25} {}".format( - "Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness") + print("URL: {}".format(url)) + print("{:>60} {:10} {:25} {}".format( + "Test Name", "Total Runs", "Bad Runs(failed/timeout/hanging)", "Flakyness")) for bad_test in sorted_test_to_build_ids: test_status = sorted_test_to_build_ids[bad_test] - print "{:>60} {:10} {:7} ( {:4} / {:5} / {:5} ) {:2.0f}%".format( + print("{:>60} {:10} {:7} ( {:4} / {:5} / {:5} ) {:2.0f}%".format( bad_test, len(test_status['all']), test_status['bad_count'], len(test_status['failed']), len(test_status['timeout']), - len(test_status['hanging']), test_status['flakyness']) + len(test_status['hanging']), test_status['flakyness'])) else: - print "No flaky tests founds." + print("No flaky tests founds.") if len(url_to_build_ids[url]) == len(build_ids_without_tests_run): - print "None of the analyzed builds have test result." + print("None of the analyzed builds have test result.") - print "Builds analyzed: {}".format(url_to_build_ids[url]) - print "Builds without any test runs: {}".format(build_ids_without_tests_run) - print "" + print("Builds analyzed: {}".format(url_to_build_ids[url])) + print("Builds without any test runs: {}".format(build_ids_without_tests_run)) + print("") all_bad_tests = all_hanging_tests.union(all_failed_tests) diff --git a/dev-support/gen_redirect_html.py b/dev-support/gen_redirect_html.py index 8a24a81ac65a..2689fd8aa4a3 100755 --- a/dev-support/gen_redirect_html.py +++ b/dev-support/gen_redirect_html.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python3 ## # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,8 +20,8 @@ from string import Template if len(sys.argv) != 2 : - print "usage: %s " % sys.argv[0] - exit(1) + print("usage: %s " % sys.argv[0]) + sys.exit(1) url = sys.argv[1].replace(" ", "%20") template = Template(""" @@ -34,4 +34,4 @@ """) output = template.substitute(url = url) -print output +print(output) diff --git a/dev-support/submit-patch.py b/dev-support/submit-patch.py deleted file mode 100755 index 8d13e20c84dd..000000000000 --- a/dev-support/submit-patch.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python2 -## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Makes a patch for the current branch, creates/updates the review board request and uploads new -# patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus' naming -# rules. If no jira is specified, patch will be named (branch name).patch and jira and review board -# are not updated. Review board id is retrieved from the remote link in the jira. -# Print help: submit-patch.py --h -import argparse -from builtins import input, str -import getpass -import git -import json -import logging -import os -import re -import requests -import subprocess -import sys - -parser = argparse.ArgumentParser( - epilog = "To avoid having to enter jira/review board username/password every time, setup an " - "encrypted ~/.apache-cred files as follows:\n" - "1) Create a file with following single " - "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", " - "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n" - "2) Encrypt it with openssl.\n" - "openssl enc -aes-256-cbc -in -out ~/.apache-creds\n" - "3) Delete original file.\n" - "Now onwards, you'll need to enter this encryption key only once per run. If you " - "forget the key, simply regenerate ~/.apache-cred file again.", - formatter_class=argparse.RawTextHelpFormatter -) -parser.add_argument("-b", "--branch", - help = "Branch to use for generating diff. If not specified, tracking branch " - "is used. If there is no tracking branch, error will be thrown.") - -# Arguments related to Jira. -parser.add_argument("-jid", "--jira-id", - help = "Jira id of the issue. If set, we deduce next patch version from " - "attachments in the jira and also upload the new patch. Script will " - "ask for jira username/password for authentication. If not set, " - "patch is named .patch.") - -# Arguments related to Review Board. -parser.add_argument("-srb", "--skip-review-board", - help = "Don't create/update the review board.", - default = False, action = "store_true") -parser.add_argument("--reviewers", - help = "Comma separated list of users to add as reviewers.") - -# Misc arguments -parser.add_argument("--patch-dir", default = "~/patches", - help = "Directory to store patch files. If it doesn't exist, it will be " - "created. Default: ~/patches") -parser.add_argument("--rb-repo", default = "hbase-git", - help = "Review board repository. Default: hbase-git") -args = parser.parse_args() - -# Setup logger -logging.basicConfig() -logger = logging.getLogger("submit-patch") -logger.setLevel(logging.INFO) - - -def log_fatal_and_exit(*arg): - logger.fatal(*arg) - sys.exit(1) - - -def assert_status_code(response, expected_status_code, description): - if response.status_code != expected_status_code: - log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..", - description, response.status_code, response.reason) - - -# Make repo instance to interact with git repo. -try: - repo = git.Repo(os.getcwd()) - git = repo.git -except git.exc.InvalidGitRepositoryError as e: - log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of " - "HBase's git repo.", e) - -logger.info(" Active branch: %s", repo.active_branch.name) -# Do not proceed if there are uncommitted changes. -if repo.is_dirty(): - log_fatal_and_exit(" Git status is dirty. Commit locally first.") - - -# Returns base branch for creating diff. -def get_base_branch(): - # if --branch is set, use it as base branch for computing diff. Also check that it's a valid branch. - if args.branch is not None: - base_branch = args.branch - # Check that given branch exists. - for ref in repo.refs: - if ref.name == base_branch: - return base_branch - log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch) - else: - # if --branch is not set, use tracking branch as base branch for computing diff. - # If there is no tracking branch, log error and quit. - tracking_branch = repo.active_branch.tracking_branch() - if tracking_branch is None: - log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify base " - " branch for computing diff using --branch flag.") - logger.info(" Using tracking branch as base branch") - return tracking_branch.name - - -# Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira is -# specified, patch is name (branch name).patch. -def get_patch_name(branch): - if args.jira_id is None: - return branch + ".patch" - - patch_name_prefix = args.jira_id.upper() + "." + branch - return get_patch_name_with_version(patch_name_prefix) - - -# Fetches list of attachments from the jira, deduces next version for the patch and returns final -# patch name. -def get_patch_name_with_version(patch_name_prefix): - # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637. - # Using crude way to get list of attachments. - url = "https://issues.apache.org/jira/browse/" + args.jira_id - logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url) - html = requests.get(url) - if html.status_code == 404: - log_fatal_and_exit(" Invalid jira id : %s", args.jira_id) - if html.status_code != 200: - log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code) - # Iterate over patch names starting from version 1 and return when name is not already used. - content = str(html.content, 'utf-8') - for i in range(1, 1000): - name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch" - if name not in content: - return name - - -# Validates that patch directory exists, if not, creates it. -def validate_patch_dir(patch_dir): - # Create patch_dir if it doesn't exist. - if not os.path.exists(patch_dir): - logger.warn(" Patch directory doesn't exist. Creating it.") - os.mkdir(patch_dir) - else: - # If patch_dir exists, make sure it's a directory. - if not os.path.isdir(patch_dir): - log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.", - patch_dir) - - -# Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if -# - base_branch has commits not in current branch -# - current branch is same as base branch -# - current branch is ahead of base_branch by more than 1 commits -def check_diff_between_branches(base_branch): - only_in_base_branch = list(repo.iter_commits("HEAD.." + base_branch)) - only_in_active_branch = list(repo.iter_commits(base_branch + "..HEAD")) - if len(only_in_base_branch) != 0: - log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase " - "and try again.", base_branch, len(only_in_base_branch)) - if len(only_in_active_branch) == 0: - log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch) - if len(only_in_active_branch) > 1: - log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single " - "commit and try again.", base_branch, len(only_in_active_branch)) - - -# If ~/.apache-creds is present, load credentials from it otherwise prompt user. -def get_credentials(): - creds = dict() - creds_filepath = os.path.expanduser("~/.apache-creds") - if os.path.exists(creds_filepath): - try: - logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials") - content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath, - shell=True) - except subprocess.CalledProcessError as e: - log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..") - creds = json.loads(content) - else: - creds['jira_username'] = input("Jira username:") - creds['jira_password'] = getpass.getpass("Jira password:") - if not args.skip_review_board: - creds['rb_username'] = input("Review Board username:") - creds['rb_password'] = getpass.getpass("Review Board password:") - return creds - - -def attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds): - # Upload patch to jira using REST API. - headers = {'X-Atlassian-Token': 'no-check'} - files = {'file': (patch_filename, open(patch_filepath, 'rb'), 'text/plain')} - jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password']) - attachment_url = issue_url + "/attachments" - r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth) - assert_status_code(r, 200, "uploading patch to jira") - - -def get_jira_summary(issue_url): - r = requests.get(issue_url + "?fields=summary") - assert_status_code(r, 200, "fetching jira summary") - return json.loads(r.content)["fields"]["summary"] - - -def get_review_board_id_if_present(issue_url, rb_link_title): - r = requests.get(issue_url + "/remotelink") - assert_status_code(r, 200, "fetching remote links") - links = json.loads(r.content) - for link in links: - if link["object"]["title"] == rb_link_title: - res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"]) - return res.group(1) - return None - - -base_branch = get_base_branch() -# Remove remote repo name from branch name if present. This assumes that we don't use '/' in -# actual branch names. -base_branch_without_remote = base_branch.split('/')[-1] -logger.info(" Base branch: %s", base_branch) - -check_diff_between_branches(base_branch) - -patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir)) -logger.info(" Patch directory: %s", patch_dir) -validate_patch_dir(patch_dir) - -patch_filename = get_patch_name(base_branch_without_remote) -logger.info(" Patch name: %s", patch_filename) -patch_filepath = os.path.join(patch_dir, patch_filename) - -diff = git.format_patch(base_branch, stdout = True) -with open(patch_filepath, "w") as f: - f.write(diff.encode('utf8')) - -if args.jira_id is not None: - creds = get_credentials() - issue_url = "https://issues.apache.org/jira/rest/api/2/issue/" + args.jira_id - - attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds) - - if not args.skip_review_board: - rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password']) - - rb_link_title = "Review Board (" + base_branch_without_remote + ")" - rb_id = get_review_board_id_if_present(issue_url, rb_link_title) - - # If no review board link found, create new review request and add its link to jira. - if rb_id is None: - reviews_url = "https://reviews.apache.org/api/review-requests/" - data = {"repository" : "hbase-git"} - r = requests.post(reviews_url, data = data, auth = rb_auth) - assert_status_code(r, 201, "creating new review request") - review_request = json.loads(r.content)["review_request"] - absolute_url = review_request["absolute_url"] - logger.info(" Created new review request: %s", absolute_url) - - # Use jira summary as review's summary too. - summary = get_jira_summary(issue_url) - # Use commit message as description. - description = repo.head.commit.message - update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups" : "hbase", - "target_people" : args.reviewers, "summary" : summary, - "description" : description } - draft_url = review_request["links"]["draft"]["href"] - r = requests.put(draft_url, data = update_draft_data, auth = rb_auth) - assert_status_code(r, 200, "updating review draft") - - draft_request = json.loads(r.content)["draft"] - diff_url = draft_request["links"]["draft_diffs"]["href"] - files = {'path' : (patch_filename, open(patch_filepath, 'rb'))} - r = requests.post(diff_url, files = files, auth = rb_auth) - assert_status_code(r, 201, "uploading diff to review draft") - - r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth) - assert_status_code(r, 200, "publishing review request") - - # Add link to review board in the jira. - remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}}) - jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password']) - r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth, - headers={'Content-Type':'application/json'}) - else: - logger.info(" Updating existing review board: https://reviews.apache.org/r/%s", rb_id) - draft_url = "https://reviews.apache.org/api/review-requests/" + rb_id + "/draft/" - diff_url = draft_url + "diffs/" - files = {'path' : (patch_filename, open(patch_filepath, 'rb'))} - r = requests.post(diff_url, files = files, auth = rb_auth) - assert_status_code(r, 201, "uploading diff to review draft") - - r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth) - assert_status_code(r, 200, "publishing review request")