Skip to content
Permalink
Browse files
[FLINK-25188][python][build] Support m1 chip.
This closes #18769.
  • Loading branch information
deadwind4 authored and HuangXingBo committed May 23, 2022
1 parent 9920140 commit 7e9be78974f95e4eb3c8bb442564c81ea61c563e
Showing 27 changed files with 102 additions and 83 deletions.
2 NOTICE
@@ -17,7 +17,7 @@ See bundled license files for details.
This project bundles the following dependencies under the BSD license.
See bundled license files for details.

- cloudpickle:1.2.2
- cloudpickle:2.0.0
- net.sf.py4j:py4j:0.10.9.3

This project bundles the following dependencies under SIL OFL 1.1 license (https://opensource.org/licenses/OFL-1.1).
@@ -14,17 +14,18 @@
# limitations under the License.
setuptools>=18.0
wheel
apache-beam==2.27.0
apache-beam==2.38.0
cython==0.29.24
py4j==0.10.9.3
python-dateutil==2.8.0
cloudpickle==1.2.2
cloudpickle==2.0.0
avro-python3>=1.8.1,!=1.9.2,<1.10.0
pandas>=1.0,<1.2.0
pyarrow>=0.15.1,<3.0.0
pandas>=1.3.0
pyarrow>=5.0.0
pytz>=2018.3
numpy>=1.14.3,<1.20
numpy>=1.21.4
fastavro>=0.21.4,<0.24
grpcio>=1.29.0,<2
grpcio-tools>=1.3.5,<=1.14.2
pemja==0.1.4; python_version >= '3.7'
pemja==0.1.5; python_version >= '3.7'
httplib2>=0.8,<0.19.0
@@ -209,6 +209,12 @@ function install_miniconda() {
if [ ! -d "$CURRENT_DIR/.conda" ]; then
print_function "STEP" "installing conda..."
$CONDA_INSTALL_SH -b -p $CURRENT_DIR/.conda 2>&1 >/dev/null
# orjson depend on pip >= 20.3
print_function "STEP" "upgrade pip..."
$CURRENT_DIR/.conda/bin/python -m pip install --upgrade pip 2>&1 >/dev/null
print_function "STEP" "upgrade pip... [SUCCESS]"
if [ $? -ne 0 ]; then
echo "install miniconda failed"
exit $CONDA_INSTALL_STATUS
Binary file not shown.
Binary file not shown.
@@ -34,7 +34,7 @@ under the License.
<packaging>jar</packaging>

<properties>
<arrow.version>0.16.0</arrow.version>
<arrow.version>5.0.0</arrow.version>
</properties>

<dependencies>
@@ -88,6 +88,10 @@ under the License.
<groupId>org.apache.beam</groupId>
<artifactId>beam-vendor-bytebuddy-1_10_8</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
</exclusion>
</exclusions>
</dependency>

@@ -100,7 +104,7 @@ under the License.
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>pemja</artifactId>
<version>0.1.4</version>
<version>0.1.5</version>
</dependency>

<!-- Protobuf dependencies -->
@@ -141,6 +145,11 @@ under the License.
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
<version>${arrow.version}</version>
</dependency>

<!-- test dependencies -->

@@ -368,11 +377,11 @@ under the License.
</excludes>
</filter>
<filter>
<artifact>org.apache.beam:beam-vendor-grpc-1_26_0</artifact>
<artifact>org.apache.beam:beam-vendor-grpc-1_43_2</artifact>
<excludes>
<exclude>org/apache/beam/vendor/grpc/v1p26p0/org/jboss/**</exclude>
<exclude>org/apache/beam/vendor/grpc/v1p43p2/org/jboss/**</exclude>
<exclude>schema/**</exclude>
<exclude>org/apache/beam/vendor/grpc/v1p26p0/org/eclipse/jetty/**</exclude>
<exclude>org/apache/beam/vendor/grpc/v1p43p2/org/eclipse/jetty/**</exclude>
</excludes>
</filter>
<filter>
@@ -17,6 +17,7 @@
################################################################################
from apache_beam.portability import common_urns
from apache_beam.portability.api import beam_runner_api_pb2
from apache_beam.runners import common
from apache_beam.runners.worker import bundle_processor, operation_specs
from apache_beam.utils import proto_utils

@@ -150,6 +151,7 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers,
input=None,
side_inputs=None,
output_coders=[output_coders[tag] for tag in output_tags])
name = common.NameContext(transform_proto.unique_name)

serialized_fn = spec.serialized_fn
if hasattr(serialized_fn, "key_type"):
@@ -172,7 +174,7 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers,
serialized_fn.map_state_write_cache_size)

return beam_operation_cls(
transform_proto.unique_name,
name,
spec,
factory.counter_factory,
factory.state_sampler,
@@ -189,7 +191,7 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers,
serialized_fn.map_state_read_cache_size,
serialized_fn.map_state_write_cache_size)
return beam_operation_cls(
transform_proto.unique_name,
name,
spec,
factory.counter_factory,
factory.state_sampler,
@@ -198,7 +200,7 @@ def _create_user_defined_function_operation(factory, transform_proto, consumers,
keyed_state_backend)
else:
return beam_operation_cls(
transform_proto.unique_name,
name,
spec,
factory.counter_factory,
factory.state_sampler,
@@ -119,7 +119,7 @@ def setUp(self):
def test_set_requirements_without_cached_directory(self):
requirements_txt_path = os.path.join(self.tempdir, str(uuid.uuid4()))
with open(requirements_txt_path, 'w') as f:
f.write("cloudpickle==1.2.2")
f.write("cloudpickle==2.0.0")
self.st_env.set_python_requirements(requirements_txt_path)

def check_requirements(i):
@@ -310,12 +310,12 @@ def extracted_output_files(base_dir, file_path, output_directory):
author='Apache Software Foundation',
author_email='dev@flink.apache.org',
python_requires='>=3.6',
install_requires=['py4j==0.10.9.3', 'python-dateutil==2.8.0', 'apache-beam==2.27.0',
'cloudpickle==1.2.2', 'avro-python3>=1.8.1,!=1.9.2,<1.10.0',
'pandas>=1.0,<1.2.0', 'pyarrow>=0.15.1,<3.0.0',
'pytz>=2018.3', 'numpy>=1.14.3,<1.20', 'fastavro>=0.21.4,<0.24',
install_requires=['py4j==0.10.9.3', 'python-dateutil==2.8.0', 'apache-beam==2.38.0',
'cloudpickle==2.0.0', 'avro-python3>=1.8.1,!=1.9.2,<1.10.0',
'pandas>=1.3.0', 'pyarrow>=5.0.0',
'pytz>=2018.3', 'numpy>=1.21.4', 'fastavro>=0.21.4,<0.24',
'requests>=2.26.0', 'protobuf<3.18',
'pemja==0.1.4;python_full_version >= "3.7"',
'pemja==0.1.5;python_full_version >= "3.7"', 'httplib2>=0.8,<0.19.0',
apache_flink_libraries_dependency],
cmdclass={'build_ext': build_ext},
tests_require=['pytest==4.4.1'],
@@ -20,9 +20,9 @@
import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
import org.apache.beam.model.fnexecution.v1.BeamFnStateGrpc;
import org.apache.beam.runners.fnexecution.FnService;
import org.apache.beam.vendor.grpc.v1p26p0.io.grpc.stub.ServerCallStreamObserver;
import org.apache.beam.vendor.grpc.v1p26p0.io.grpc.stub.StreamObserver;
import org.apache.beam.sdk.fn.server.FnService;
import org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.ServerCallStreamObserver;
import org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.StreamObserver;

import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
@@ -14,9 +14,9 @@
* limitations under the License.
*/

package org.apache.beam.vendor.grpc.v1p26p0.io.grpc.internal;
package org.apache.beam.vendor.grpc.v1p43p2.io.grpc.internal;

import org.apache.beam.vendor.grpc.v1p26p0.com.google.common.base.Preconditions;
import org.apache.beam.vendor.grpc.v1p43p2.com.google.common.base.Preconditions;

import javax.annotation.concurrent.ThreadSafe;

@@ -179,7 +179,7 @@ private RunnerApi.ParDoPayload createRevisePayload() {
RunnerApi.FunctionSpec.newBuilder()
.setUrn(STATELESS_FUNCTION_URN)
.setPayload(
org.apache.beam.vendor.grpc.v1p26p0.com.google
org.apache.beam.vendor.grpc.v1p43p2.com.google
.protobuf.ByteString.copyFrom(
proto.toByteArray()))
.build());
@@ -196,7 +196,7 @@ private RunnerApi.ParDoPayload createUdfPayload(
RunnerApi.FunctionSpec.newBuilder()
.setUrn(urn)
.setPayload(
org.apache.beam.vendor.grpc.v1p26p0.com.google
org.apache.beam.vendor.grpc.v1p43p2.com.google
.protobuf.ByteString.copyFrom(
proto.toByteArray()))
.build());
@@ -71,7 +71,7 @@
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -542,15 +542,15 @@ private ExecutableStage createExecutableStage(RunnerApi.Environment environment)
RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
.setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
.setPayload(
org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString
org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString
.copyFrom(baos.toByteArray()))
.setInputOrOutputId(INPUT_COLLECTION_ID)
.build());
settings.add(
RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
.setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
.setPayload(
org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString
org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString
.copyFrom(baos.toByteArray()))
.setInputOrOutputId(OUTPUT_COLLECTION_ID)
.build());
@@ -560,7 +560,7 @@ private ExecutableStage createExecutableStage(RunnerApi.Environment environment)
RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
.setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
.setPayload(
org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf
org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf
.ByteString.copyFrom(baos.toByteArray()))
.setInputOrOutputId(entry.getKey())
.build());
@@ -26,7 +26,7 @@
import org.apache.flink.util.Preconditions;

import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;

import javax.annotation.Nullable;

@@ -28,7 +28,7 @@
import org.apache.flink.streaming.api.utils.ByteArrayWrapper;

import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;

import java.util.HashMap;
import java.util.Iterator;
@@ -28,8 +28,8 @@

import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.runners.fnexecution.state.StateRequestHandler;
import org.apache.beam.vendor.grpc.v1p26p0.com.google.common.base.Charsets;
import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
import org.apache.beam.vendor.grpc.v1p43p2.com.google.common.base.Charsets;
import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;

import javax.annotation.Nullable;

@@ -284,7 +284,7 @@ public static RunnerApi.Coder createCoderProto(
RunnerApi.FunctionSpec.newBuilder()
.setUrn(FLINK_CODER_URN)
.setPayload(
org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf
org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf
.ByteString.copyFrom(
coderInfoDescriptor.toByteArray()))
.build())
@@ -92,7 +92,7 @@ protected void buildTransforms(RunnerApi.Components.Builder componentsBuilder) {
RunnerApi.FunctionSpec.newBuilder()
.setUrn(functionUrn)
.setPayload(
org.apache.beam.vendor.grpc.v1p26p0.com.google
org.apache.beam.vendor.grpc.v1p43p2.com.google
.protobuf.ByteString.copyFrom(
userDefinedFunctionProto.toByteArray()))
.build())
@@ -9,25 +9,26 @@ This project bundles the following dependencies under the Apache Software Licens
- com.fasterxml.jackson.core:jackson-annotations:2.13.2
- com.fasterxml.jackson.core:jackson-core:2.13.2
- com.fasterxml.jackson.core:jackson-databind:2.13.2.2
- com.google.flatbuffers:flatbuffers-java:1.9.0
- io.netty:netty-buffer:4.1.70.Final
- io.netty:netty-common:4.1.70.Final
- com.google.flatbuffers:flatbuffers-java:1.12.0
- joda-time:joda-time:2.5
- org.apache.arrow:arrow-format:0.16.0
- org.apache.arrow:arrow-memory:0.16.0
- org.apache.arrow:arrow-vector:0.16.0
- org.apache.beam:beam-model-fn-execution:2.27.0
- org.apache.beam:beam-model-job-management:2.27.0
- org.apache.beam:beam-model-pipeline:2.27.0
- org.apache.beam:beam-runners-core-construction-java:2.27.0
- org.apache.beam:beam-runners-core-java:2.27.0
- org.apache.beam:beam-runners-java-fn-execution:2.27.0
- org.apache.beam:beam-sdks-java-core:2.27.0
- org.apache.beam:beam-sdks-java-fn-execution:2.27.0
- org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.27.0
- org.apache.arrow:arrow-format:5.0.0
- org.apache.arrow:arrow-memory:5.0.0
- org.apache.arrow:arrow-memory-core:5.0.0
- org.apache.arrow:arrow-memory-netty:5.0.0
- org.apache.arrow:arrow-vector:5.0.0
- org.apache.beam:beam-model-fn-execution:2.38.0
- org.apache.beam:beam-model-job-management:2.38.0
- org.apache.beam:beam-model-pipeline:2.38.0
- org.apache.beam:beam-runners-core-construction-java:2.38.0
- org.apache.beam:beam-runners-core-java:2.38.0
- org.apache.beam:beam-runners-java-fn-execution:2.38.0
- org.apache.beam:beam-sdks-java-core:2.38.0
- org.apache.beam:beam-sdks-java-fn-execution:2.38.0
- org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
- org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.38.0
- org.apache.beam:beam-vendor-guava-26_0-jre:0.1
- org.apache.beam:beam-vendor-grpc-1_26_0:0.3
- com.alibaba:pemja:0.1.4
- org.apache.beam:beam-vendor-grpc-1_43_2:0.1
- com.alibaba:pemja:0.1.5

This project bundles the following dependencies under the BSD license.
See bundled license files for details
@@ -47,26 +48,26 @@ The bundled Apache Beam dependencies bundle the following dependencies under the
- com.google.code.gson:gson:2.8.6
- com.google.guava:guava:26.0-jre
- com.ning:compress-lzf:1.0.3
- io.grpc:grpc-auth:1.26.0
- io.grpc:grpc-core:1.26.0
- io.grpc:grpc-context:1.26.0
- io.grpc:grpc-netty:1.26.0
- io.grpc:grpc-protobuf:1.26.0
- io.grpc:grpc-stub:1.26.0
- io.grpc:grpc-testing:1.26.0
- io.netty:netty-buffer:4.1.51.Final
- io.netty:netty-codec:4.1.51.Final
- io.netty:netty-codec-http:4.1.51.Final
- io.netty:netty-codec-http2:4.1.51.Final
- io.netty:netty-codec-socks:4.1.51.Final
- io.netty:netty-common:4.1.51.Final
- io.netty:netty-handler:4.1.51.Final
- io.netty:netty-handler-proxy:4.1.51.Final
- io.netty:netty-resolver:4.1.51.Final
- io.netty:netty-transport:4.1.51.Final
- io.netty:netty-transport-native-epoll:4.1.51.Final
- io.netty:netty-transport-native-unix-common:4.1.51.Final
- io.netty:netty-tcnative-boringssl-static:2.0.33.Final
- io.grpc:grpc-auth:1.43.2
- io.grpc:grpc-core:1.43.2
- io.grpc:grpc-context:1.43.2
- io.grpc:grpc-netty:1.43.2
- io.grpc:grpc-protobuf:1.43.2
- io.grpc:grpc-stub:1.43.2
- io.grpc:grpc-testing:1.43.2
- io.netty:netty-buffer:4.1.70.Final
- io.netty:netty-codec:4.1.70.Final
- io.netty:netty-codec-http:4.1.70.Final
- io.netty:netty-codec-http2:4.1.70.Final
- io.netty:netty-codec-socks:4.1.70.Final
- io.netty:netty-common:4.1.70.Final
- io.netty:netty-handler:4.1.70.Final
- io.netty:netty-handler-proxy:4.1.70.Final
- io.netty:netty-resolver:4.1.70.Final
- io.netty:netty-transport:4.1.70.Final
- io.netty:netty-transport-native-epoll:4.1.70.Final
- io.netty:netty-transport-native-unix-common:4.1.70.Final
- io.netty:netty-tcnative-boringssl-static:2.0.44.Final
- io.opencensus:opencensus-api:0.24.0
- io.opencensus:opencensus-contrib-grpc-metrics:0.24.0
- io.perfmark:perfmark-api:0.19.0
@@ -33,7 +33,7 @@
import org.apache.flink.table.types.logical.RowType;

import org.apache.beam.runners.fnexecution.control.JobBundleFactory;
import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;

import java.util.ArrayList;
import java.util.LinkedList;

0 comments on commit 7e9be78

Please sign in to comment.