Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HBASE-27450 Update all our python scripts to use python3 #4851

Merged
merged 2 commits into from Oct 28, 2022
Merged

HBASE-27450 Update all our python scripts to use python3 #4851

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5506f88
HBASE-27450 Update all our python scripts to use python3
Apache9 Oct 27, 2022
c0faa0a
fix pylint
Apache9 Oct 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 4 additions & 6 deletions dev-support/Dockerfile
View file
Open in desktop
Expand Up @@ -20,16 +20,14 @@
#
# Specifically, it's used for the flaky test reporting job defined in
# dev-support/flaky-tests/flaky-reporting.Jenkinsfile
FROM ubuntu:18.04
FROM ubuntu:22.04

COPY . /hbase/dev-support

RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
&& DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
curl='7.58.0-*' \
python2.7='2.7.17-*' \
python-pip='9.0.1-*' \
python-setuptools='39.0.1-*' \
curl='7.81.0-*' \
python3-pip='22.0.2+dfsg-*' \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN pip install -r /hbase/dev-support/python-requirements.txt
RUN pip3 install -r /hbase/dev-support/python-requirements.txt
12 changes: 6 additions & 6 deletions dev-support/Jenkinsfile
View file
Open in desktop
Expand Up @@ -267,14 +267,14 @@ pipeline {
if [ -d "${OUTPUT_DIR}/branch-site" ]; then
echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
rm -rf "${OUTPUT_DIR}/branch-site"
python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
else
echo "No branch-site, skipping"
fi
if [ -d "${OUTPUT_DIR}/patch-site" ]; then
echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
rm -rf "${OUTPUT_DIR}/patch-site"
python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
else
echo "No patch-site, skipping"
fi
Expand Down Expand Up @@ -384,7 +384,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
Expand Down Expand Up @@ -493,7 +493,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
Expand Down Expand Up @@ -604,7 +604,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
Expand Down Expand Up @@ -773,7 +773,7 @@ pipeline {
if [ -f "${SRC_TAR}" ]; then
echo "Remove ${SRC_TAR} for saving space"
rm -rf "${SRC_TAR}"
python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
else
echo "No hbase-src.tar.gz, skipping"
fi
Expand Down
8 changes: 4 additions & 4 deletions dev-support/Jenkinsfile_GitHub
View file
Open in desktop
Expand Up @@ -143,14 +143,14 @@ pipeline {
if [ -d "${PATCHDIR}/branch-site" ]; then
echo "Remove ${PATCHDIR}/branch-site for saving space"
rm -rf "${PATCHDIR}/branch-site"
python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
else
echo "No branch-site, skipping"
fi
if [ -d "${PATCHDIR}/patch-site" ]; then
echo "Remove ${PATCHDIR}/patch-site for saving space"
rm -rf "${PATCHDIR}/patch-site"
python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
else
echo "No patch-site, skipping"
fi
Expand Down Expand Up @@ -282,7 +282,7 @@ pipeline {
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
rm -rf "${PATCHDIR}/test_logs.zip"
python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
Expand Down Expand Up @@ -414,7 +414,7 @@ pipeline {
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
rm -rf "${PATCHDIR}/test_logs.zip"
python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
Expand Down
32 changes: 17 additions & 15 deletions dev-support/checkcompatibility.py
View file
Open in desktop
@@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
Expand Down Expand Up @@ -41,7 +41,9 @@
import shutil
import subprocess
import sys
import urllib2
import urllib.request
import urllib.error
import urllib.parse
from collections import namedtuple
try:
import argparse
Expand All @@ -55,11 +57,11 @@


def check_output(*popenargs, **kwargs):
""" Run command with arguments and return its output as a byte string.
Backported from Python 2.7 as it's implemented as pure python on stdlib.
>>> check_output(['/usr/bin/python', '--version'])
Python 2.6.2 """
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
""" Run command with arguments and return its output as a byte string. """
process = subprocess.Popen(stdout=subprocess.PIPE,
universal_newlines=True,
*popenargs,
**kwargs)
output, _ = process.communicate()
retcode = process.poll()
if retcode:
Expand All @@ -69,7 +71,7 @@ def check_output(*popenargs, **kwargs):
error = subprocess.CalledProcessError(retcode, cmd)
error.output = output
raise error
return output
return output.strip()


def get_repo_dir():
Expand Down Expand Up @@ -161,7 +163,7 @@ def checkout_java_acc(force):
url = "https://github.com/lvc/japi-compliance-checker/archive/2.4.tar.gz"
scratch_dir = get_scratch_dir()
path = os.path.join(scratch_dir, os.path.basename(url))
jacc = urllib2.urlopen(url)
jacc = urllib.request.urlopen(url)
with open(path, 'wb') as w:
w.write(jacc.read())

Expand Down Expand Up @@ -196,8 +198,8 @@ def ascii_encode_dict(data):
""" Iterate through a dictionary of data and convert all unicode to ascii.
This method was taken from
stackoverflow.com/questions/9590382/forcing-python-json-module-to-work-with-ascii """
ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x
return dict(map(ascii_encode, pair) for pair in data.items())
ascii_encode = lambda x: x.encode('ascii') if isinstance(x, str) else x
return dict(list(map(ascii_encode, pair)) for pair in list(data.items()))


def process_json(path):
Expand Down Expand Up @@ -229,8 +231,8 @@ def compare_results(tool_results, known_issues, compare_warnings):
unexpected_issues = [unexpected_issue(check=check, issue_type=issue_type,
known_count=known_count,
observed_count=tool_results[check][issue_type])
for check, known_issue_counts in known_issues.items()
for issue_type, known_count in known_issue_counts.items()
for check, known_issue_counts in list(known_issues.items())
for issue_type, known_count in list(known_issue_counts.items())
if compare_tool_results_count(tool_results, check, issue_type, known_count)]

if not compare_warnings:
Expand Down Expand Up @@ -309,14 +311,14 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations, skip_annot
logging.info("Annotations are: %s", annotations)
annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
logging.info("Annotations path: %s", annotations_path)
with file(annotations_path, "w") as f:
with open(annotations_path, "w") as f:
f.write('\n'.join(annotations))
args.extend(["-annotations-list", annotations_path])

if skip_annotations is not None:
skip_annotations_path = os.path.join(
get_scratch_dir(), "skip_annotations.txt")
with file(skip_annotations_path, "w") as f:
with open(skip_annotations_path, "w") as f:
f.write('\n'.join(skip_annotations))
args.extend(["-skip-annotations-list", skip_annotations_path])

Expand Down
56 changes: 28 additions & 28 deletions dev-support/checkstyle_report.py
View file
Open in desktop
@@ -1,4 +1,4 @@
#!/usr/bin/python2
#!/usr/bin/env python3
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
Expand Down Expand Up @@ -29,8 +29,8 @@
from collections import defaultdict

if len(sys.argv) != 3 :
print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
exit(1)
print("usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0])
sys.exit(1)

def path_key(x):
path = x.attrib['name']
Expand All @@ -40,41 +40,41 @@ def error_name(x):
error_class = x.attrib['source']
return error_class[error_class.rfind(".") + 1:]

def print_row(path, error, master_errors, patch_errors):
print '%s\t%s\t%s\t%s' % (path,error, master_errors,patch_errors)
def print_row(path, err, master_errors, patch_errors):
print('%s\t%s\t%s\t%s' % (path, err, master_errors, patch_errors))

master = etree.parse(sys.argv[1])
patch = etree.parse(sys.argv[2])

master_dict = defaultdict(int)
ret_value = 0

for child in master.getroot().getchildren():
for child in list(master.getroot()):
if child.tag != 'file':
continue
continue
file = path_key(child)
for error_tag in child.getchildren():
error = error_name(error_tag)
if (file, error) in master_dict:
master_dict[(file, error)] += 1
else:
master_dict[(file, error)] = 1
for error_tag in list(child):
error = error_name(error_tag)
if (file, error) in master_dict:
master_dict[(file, error)] += 1
else:
master_dict[(file, error)] = 1

for child in patch.getroot().getchildren():
if child.tag != 'file':
continue
temp_dict = defaultdict(int)
for error_tag in child.getchildren():
error = error_name(error_tag)
if error in temp_dict:
temp_dict[error] += 1
else:
temp_dict[error] = 1
for child in list(patch.getroot()):
if child.tag != 'file':
continue
temp_dict = defaultdict(int)
for error_tag in list(child):
error = error_name(error_tag)
if error in temp_dict:
temp_dict[error] += 1
else:
temp_dict[error] = 1

file = path_key(child)
for error, count in temp_dict.iteritems():
if count > master_dict[(file, error)]:
print_row(file, error, master_dict[(file, error)], count)
ret_value = 1
file = path_key(child)
for error, count in temp_dict.items():
if count > master_dict[(file, error)]:
print_row(file, error, master_dict[(file, error)], count)
ret_value = 1

sys.exit(ret_value)
4 changes: 2 additions & 2 deletions dev-support/create-release/prepend_changes.py
View file
Open in desktop
@@ -1,4 +1,4 @@
#!/usr/bin/python3
#!/usr/bin/env python3
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
Expand All @@ -22,7 +22,7 @@

if len(sys.argv) != 3:
print("usage: %s <NEW_CHANGES.md> <PREV_CHANGES.md>" % sys.argv[0])
exit(1)
sys.exit(1)

pattern = re.compile(r'^## Release .+ - Unreleased .+$')
with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
Expand Down
4 changes: 2 additions & 2 deletions dev-support/create-release/prepend_releasenotes.py
View file
Open in desktop
@@ -1,4 +1,4 @@
#!/usr/bin/python3
#!/usr/bin/env python3
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
Expand All @@ -22,7 +22,7 @@

if len(sys.argv) != 3:
print("usage: %s <NEW_RELEASENOTES.md> <PREV_RELEASENOTES.md>" % sys.argv[0])
exit(1)
sys.exit(1)

pattern = re.compile(r'^# .+ Release Notes$')
with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
Expand Down