HBASE-27450 Update all our python scripts to use python3 (#4851)

Signed-off-by: Guanghao Zhang <zghao@apache.org>
(cherry picked from commit cdabfd3)

dev-support/Dockerfile

@@ -20,16 +20,14 @@
 #
 # Specifically, it's used for the flaky test reporting job defined in
 # dev-support/flaky-tests/flaky-reporting.Jenkinsfile
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 COPY . /hbase/dev-support
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
     && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
-      curl='7.58.0-*' \
-      python2.7='2.7.17-*' \
-      python-pip='9.0.1-*' \
-      python-setuptools='39.0.1-*' \
+      curl='7.81.0-*' \
+      python3-pip='22.0.2+dfsg-*' \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
-RUN pip install -r /hbase/dev-support/python-requirements.txt
+RUN pip3 install -r /hbase/dev-support/python-requirements.txt

dev-support/Jenkinsfile

@@ -267,14 +267,14 @@ pipeline {
               if [ -d "${OUTPUT_DIR}/branch-site" ]; then
                 echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
                 rm -rf "${OUTPUT_DIR}/branch-site"
-                python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
+                python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
               else
                 echo "No branch-site, skipping"
               fi
               if [ -d "${OUTPUT_DIR}/patch-site" ]; then
                 echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
                 rm -rf "${OUTPUT_DIR}/patch-site"
-                python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
+                python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
               else
                 echo "No patch-site, skipping"
               fi
@@ -384,7 +384,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -493,7 +493,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -604,7 +604,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -773,7 +773,7 @@ pipeline {
                 if [ -f "${SRC_TAR}" ]; then
                   echo "Remove ${SRC_TAR} for saving space"
                   rm -rf "${SRC_TAR}"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
                 else
                   echo "No hbase-src.tar.gz, skipping"
                 fi

dev-support/Jenkinsfile_GitHub

@@ -143,14 +143,14 @@ pipeline {
                             if [ -d "${PATCHDIR}/branch-site" ]; then
                               echo "Remove ${PATCHDIR}/branch-site for saving space"
                               rm -rf "${PATCHDIR}/branch-site"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
                             else
                               echo "No branch-site, skipping"
                             fi
                             if [ -d "${PATCHDIR}/patch-site" ]; then
                               echo "Remove ${PATCHDIR}/patch-site for saving space"
                               rm -rf "${PATCHDIR}/patch-site"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
                             else
                               echo "No patch-site, skipping"
                             fi
@@ -282,7 +282,7 @@ pipeline {
                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
                               rm -rf "${PATCHDIR}/test_logs.zip"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
                             else
                               echo "No test_logs.zip, skipping"
                             fi
@@ -415,7 +415,7 @@ pipeline {
                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
                               rm -rf "${PATCHDIR}/test_logs.zip"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
                             else
                               echo "No test_logs.zip, skipping"
                             fi

dev-support/checkcompatibility.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -32,7 +32,7 @@
 #      --annotation org.apache.yetus.audience.InterfaceAudience.LimitedPrivate \
 #      --include-file "hbase-*" \
 #      --known_problems_path ~/known_problems.json \
-#      rel/1.0.0 branch-1.2
+#      rel/1.3.0 branch-1.4
 
 import json
 import logging
@@ -41,7 +41,9 @@
 import shutil
 import subprocess
 import sys
-import urllib2
+import urllib.request
+import urllib.error
+import urllib.parse
 from collections import namedtuple
 try:
     import argparse
@@ -55,11 +57,11 @@
 
 
 def check_output(*popenargs, **kwargs):
-    """ Run command with arguments and return its output as a byte string.
-    Backported from Python 2.7 as it's implemented as pure python on stdlib.
-    >>> check_output(['/usr/bin/python', '--version'])
-    Python 2.6.2 """
-    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+    """ Run command with arguments and return its output as a byte string. """
+    process = subprocess.Popen(stdout=subprocess.PIPE,
+                               universal_newlines=True,
+                               *popenargs,
+                               **kwargs)
     output, _ = process.communicate()
     retcode = process.poll()
     if retcode:
@@ -69,7 +71,7 @@ def check_output(*popenargs, **kwargs):
         error = subprocess.CalledProcessError(retcode, cmd)
         error.output = output
         raise error
-    return output
+    return output.strip()
 
 
 def get_repo_dir():
@@ -159,7 +161,7 @@ def checkout_java_acc(force):
     url = "https://github.com/lvc/japi-compliance-checker/archive/2.4.tar.gz"
     scratch_dir = get_scratch_dir()
     path = os.path.join(scratch_dir, os.path.basename(url))
-    jacc = urllib2.urlopen(url)
+    jacc = urllib.request.urlopen(url)
     with open(path, 'wb') as w:
         w.write(jacc.read())
 
@@ -194,8 +196,8 @@ def ascii_encode_dict(data):
     """ Iterate through a dictionary of data and convert all unicode to ascii.
     This method was taken from
     stackoverflow.com/questions/9590382/forcing-python-json-module-to-work-with-ascii """
-    ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x
-    return dict(map(ascii_encode, pair) for pair in data.items())
+    ascii_encode = lambda x: x.encode('ascii') if isinstance(x, str) else x
+    return dict(list(map(ascii_encode, pair)) for pair in list(data.items()))
 
 
 def process_json(path):
@@ -227,9 +229,9 @@ def compare_results(tool_results, known_issues, compare_warnings):
     unexpected_issues = [unexpected_issue(check=check,  issue_type=issue_type,
                                       known_count=known_count,
                                       observed_count=tool_results[check][issue_type])
-                     for check, known_issue_counts in known_issues.items()
-                        for issue_type, known_count in known_issue_counts.items()
-                            if tool_results[check][issue_type] > known_count]
+                     for check, known_issue_counts in list(known_issues.items())
+                        for issue_type, known_count in list(known_issue_counts.items())
+                           if compare_tool_results_count(tool_results, check, issue_type, known_count)]
 
     if not compare_warnings:
         unexpected_issues = [tup for tup in unexpected_issues
@@ -241,6 +243,14 @@ def compare_results(tool_results, known_issues, compare_warnings):
 
     return bool(unexpected_issues)
 
+def compare_tool_results_count(tool_results, check, issue_type, known_count):
+    """ Check problem counts are no more than the known count.
+    (This function exists just so can add in logging; previous was inlined
+    one-liner but this made it hard debugging)
+    """
+    # logging.info("known_count=%s, check key=%s, tool_results=%s, issue_type=%s",
+    #        str(known_count), str(check), str(tool_results), str(issue_type))
+    return tool_results[check][issue_type] > known_count
 
 def process_java_acc_output(output):
     """ Process the output string to find the problems and warnings in both the
@@ -299,14 +309,14 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations, skip_annot
         logging.info("Annotations are: %s", annotations)
         annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
         logging.info("Annotations path: %s", annotations_path)
-        with file(annotations_path, "w") as f:
+        with open(annotations_path, "w") as f:
             f.write('\n'.join(annotations))
         args.extend(["-annotations-list", annotations_path])
 
     if skip_annotations is not None:
         skip_annotations_path = os.path.join(
             get_scratch_dir(), "skip_annotations.txt")
-        with file(skip_annotations_path, "w") as f:
+        with open(skip_annotations_path, "w") as f:
             f.write('\n'.join(skip_annotations))
         args.extend(["-skip-annotations-list", skip_annotations_path])
 
@@ -325,14 +335,14 @@ def get_known_problems(json_path, src_rev, dst_rev):
     keys in the format source_branch/destination_branch and the values
     dictionaries with binary and source problems and warnings
     Example:
-    {'branch-1.0.0': {
-      'rel/1.0.0': {'binary': {'problems': 123, 'warnings': 16},
+    {'branch-1.3': {
+      'rel/1.3.0': {'binary': {'problems': 123, 'warnings': 16},
                       'source': {'problems': 167, 'warnings': 1}},
-      'branch-1.2.0': {'binary': {'problems': 0, 'warnings': 0},
+      'branch-1.4': {'binary': {'problems': 0, 'warnings': 0},
                       'source': {'problems': 0, 'warnings': 0}}
       },
-    'branch-1.2.0': {
-      'rel/1.2.1': {'binary': {'problems': 13, 'warnings': 1},
+    'branch-1.4': {
+      'rel/1.4.1': {'binary': {'problems': 13, 'warnings': 1},
                       'source': {'problems': 23, 'warnings': 0}}
       }
     } """

dev-support/checkstyle_report.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python3
 ##
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -29,8 +29,8 @@
 from collections import defaultdict
 
 if len(sys.argv) != 3 :
-  print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
-  exit(1)
+  print("usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0])
+  sys.exit(1)
 
 def path_key(x):
   path = x.attrib['name']
@@ -40,41 +40,41 @@ def error_name(x):
   error_class = x.attrib['source']
   return error_class[error_class.rfind(".") + 1:]
 
-def print_row(path, error, master_errors, patch_errors):
-    print '%s\t%s\t%s\t%s' % (path,error, master_errors,patch_errors)
+def print_row(path, err, master_errors, patch_errors):
+  print('%s\t%s\t%s\t%s' % (path, err, master_errors, patch_errors))
 
 master = etree.parse(sys.argv[1])
 patch = etree.parse(sys.argv[2])
 
 master_dict = defaultdict(int)
 ret_value = 0
 
-for child in master.getroot().getchildren():
+for child in list(master.getroot()):
     if child.tag != 'file':
-        continue
+      continue
     file = path_key(child)
-    for error_tag in child.getchildren():
-        error = error_name(error_tag)
-        if (file, error) in master_dict:
-            master_dict[(file, error)] += 1
-        else:
-            master_dict[(file, error)] = 1
+    for error_tag in list(child):
+      error = error_name(error_tag)
+      if (file, error) in master_dict:
+        master_dict[(file, error)] += 1
+      else:
+        master_dict[(file, error)] = 1
 
-for child in patch.getroot().getchildren():
-    if child.tag != 'file':
-        continue
-    temp_dict = defaultdict(int)
-    for error_tag in child.getchildren():
-        error = error_name(error_tag)
-        if error in temp_dict:
-            temp_dict[error] += 1
-        else:
-            temp_dict[error] = 1
+for child in list(patch.getroot()):
+  if child.tag != 'file':
+    continue
+  temp_dict = defaultdict(int)
+  for error_tag in list(child):
+    error = error_name(error_tag)
+    if error in temp_dict:
+      temp_dict[error] += 1
+    else:
+      temp_dict[error] = 1
 
-    file = path_key(child)
-    for error, count in temp_dict.iteritems():
-        if count > master_dict[(file, error)]:
-            print_row(file, error, master_dict[(file, error)], count)
-            ret_value = 1
+  file = path_key(child)
+  for error, count in temp_dict.items():
+    if count > master_dict[(file, error)]:
+      print_row(file, error, master_dict[(file, error)], count)
+      ret_value = 1
 
 sys.exit(ret_value)