diff --git a/flink-python/pom.xml b/flink-python/pom.xml index 3e62acf94c60a..2a9e02f6dc8c3 100644 --- a/flink-python/pom.xml +++ b/flink-python/pom.xml @@ -205,6 +205,14 @@ under the License. flink-test-utils-junit + + + org.apache.flink + flink-sql-avro + ${project.version} + test + + diff --git a/flink-python/pyflink/pyflink_gateway_server.py b/flink-python/pyflink/pyflink_gateway_server.py index 60301da3843a3..dfa1dd0f3a4ec 100644 --- a/flink-python/pyflink/pyflink_gateway_server.py +++ b/flink-python/pyflink/pyflink_gateway_server.py @@ -20,14 +20,12 @@ import glob import os import platform -import re import signal import socket import sys -import time from collections import namedtuple from string import Template -from subprocess import Popen, PIPE, check_output, CalledProcessError +from subprocess import Popen, PIPE from pyflink.find_flink_home import _find_flink_home, _find_flink_source_root @@ -213,47 +211,12 @@ def construct_hadoop_classpath(env): read_from_config(KEY_ENV_HBASE_CONF_DIR, hbase_conf_dir, flink_conf_file))]) -def download_apache_avro(): - """ - Currently we need to download the Apache Avro manually to avoid test failure caused by the avro - format sql jar. See https://issues.apache.org/jira/browse/FLINK-17417. If the issue is fixed, - this method could be removed. Using maven command copy the jars in repository to avoid accessing - external network. - """ - flink_source_root = _find_flink_source_root() - avro_jar_pattern = os.path.join( - flink_source_root, "flink-formats", "flink-avro", "target", "avro*.jar") - if len(glob.glob(avro_jar_pattern)) > 0: - # the avro jar already existed, just return. - return - mvn = "mvn.cmd" if on_windows() else "mvn" - avro_version_output = check_output( - [mvn, "help:evaluate", "-Dexpression=avro.version"], - cwd=flink_source_root).decode("utf-8") - lines = avro_version_output.replace("\r", "").split("\n") - avro_version = None - for line in lines: - if line.strip() != "" and re.match(r'^[0-9]+\.[0-9]+(\.[0-9]+)?$', line.strip()): - avro_version = line - break - if avro_version is None: - raise Exception("The Apache Avro version is not found in the maven command output:\n %s" % - avro_version_output) - check_output( - [mvn, - "org.apache.maven.plugins:maven-dependency-plugin:3.2.0:copy", - "-Dartifact=org.apache.avro:avro:%s:jar" % avro_version, - "-DoutputDirectory=%s/flink-formats/flink-avro/target" % flink_source_root], - cwd=flink_source_root) - - def construct_test_classpath(): test_jar_patterns = [ "flink-runtime/target/flink-runtime*tests.jar", "flink-streaming-java/target/flink-streaming-java*tests.jar", "flink-formats/flink-csv/target/flink-csv*.jar", - "flink-formats/flink-avro/target/flink-avro*.jar", - "flink-formats/flink-avro/target/avro*.jar", + "flink-formats/flink-sql-avro/target/flink-sql-avro*.jar", "flink-formats/flink-json/target/flink-json*.jar", "flink-python/target/artifacts/testDataStream.jar", "flink-python/target/flink-python*-tests.jar", @@ -291,25 +254,6 @@ def launch_gateway_server_process(env, args): classpath = os.pathsep.join( [construct_flink_classpath(env), construct_hadoop_classpath(env)]) if "FLINK_TESTING" in env: - total_retry_times = 3 - retry_times = 0 - status = 0 - error = None - while retry_times < total_retry_times and not status: - retry_times += 1 - try: - download_apache_avro() - status = 1 - except CalledProcessError as e: - status = 0 - error = e - print("{0} retry download, {1} retries remaining".format( - retry_times, total_retry_times - retry_times)) - # sleep 3 seconds and then re-download. - time.sleep(3) - if retry_times == total_retry_times and not status: - raise error - classpath = os.pathsep.join([classpath, construct_test_classpath()]) command = [java_executable, jvm_args, jvm_opts] + log_settings \ + ["-cp", classpath, program_args.main_class] + program_args.other_args