diff --git a/NOTICE b/NOTICE index 88121c1d7e28b8..30aa331903d6fa 100644 --- a/NOTICE +++ b/NOTICE @@ -33,6 +33,7 @@ This project bundles the following dependencies under the BSD license. See bundled license files for details. - d3:3.5.12 +- cloudpickle:1.2.2 This project bundles the following dependencies under SIL OFL 1.1 license (https://opensource.org/licenses/OFL-1.1). See bundled license files for details. diff --git a/NOTICE-binary b/NOTICE-binary index f794ace44ba396..e05d9e83d59e59 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -8035,6 +8035,7 @@ See bundled license files for details - com.google.protobuf:protobuf-java:3.7.1 - com.google.protobuf:protobuf-java-util:3.7.1 - com.google.auth:google-auth-library-credentials:0.13.0 +- cloudpickle:1.2.2 This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) See bundled license files for details. diff --git a/flink-python/README.md b/flink-python/README.md index 7f8c9361441160..db4915038144fe 100644 --- a/flink-python/README.md +++ b/flink-python/README.md @@ -27,4 +27,4 @@ We can enter the directory where this README.md file is located and run test cas ## Python Requirements -PyFlink depends on Py4J (currently version 0.10.8.1). +PyFlink depends on Py4J (currently version 0.10.8.1) and CloudPickle (currently version 1.2.2). diff --git a/flink-python/bin/pyflink-shell.sh b/flink-python/bin/pyflink-shell.sh index 2c83cc76af5753..8722e208318116 100755 --- a/flink-python/bin/pyflink-shell.sh +++ b/flink-python/bin/pyflink-shell.sh @@ -34,10 +34,11 @@ PYFLINK_PYTHON="${PYFLINK_PYTHON:-"python"}" export FLINK_BIN_DIR=$FLINK_BIN_DIR export FLINK_HOME -# Add pyflink & py4j to PYTHONPATH +# Add pyflink & py4j & cloudpickle to PYTHONPATH export PYTHONPATH="$FLINK_OPT_DIR/python/pyflink.zip:$PYTHONPATH" PY4J_ZIP=`echo "$FLINK_OPT_DIR"/python/py4j-*-src.zip` -export PYTHONPATH="$PY4J_ZIP:$PYTHONPATH" +CLOUDPICKLE_ZIP=`echo "$FLINK_OPT_DIR"/python/cloudpickle-*-src.zip` +export PYTHONPATH="$PY4J_ZIP:$CLOUDPICKLE_ZIP:$PYTHONPATH" PARSER="org.apache.flink.client.python.PythonShellParser" function parse_options() { diff --git a/flink-python/bin/pyflink-udf-runner.sh b/flink-python/bin/pyflink-udf-runner.sh index 38c38cd3147601..edfb85c718ab88 100755 --- a/flink-python/bin/pyflink-udf-runner.sh +++ b/flink-python/bin/pyflink-udf-runner.sh @@ -58,6 +58,12 @@ if [[ ! ${PYTHONPATH} =~ ${PY4J_ZIP} ]]; then export PYTHONPATH="$PY4J_ZIP:$PYTHONPATH" fi +# Add cloudpickle to PYTHONPATH +CLOUDPICKLE_ZIP=`echo "$FLINK_OPT_DIR"/python/cloudpickle-*-src.zip` +if [[ ! ${PYTHONPATH} =~ ${CLOUDPICKLE_ZIP} ]]; then + export PYTHONPATH="$CLOUDPICKLE_ZIP:$PYTHONPATH" +fi + log="$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-python-udf-boot-$HOSTNAME.log" ${python} -m pyflink.fn_execution.boot $@ 2>&1 | tee -a ${log} diff --git a/flink-python/lib/cloudpickle-1.2.2-src.zip b/flink-python/lib/cloudpickle-1.2.2-src.zip new file mode 100644 index 00000000000000..82bc04ec274021 Binary files /dev/null and b/flink-python/lib/cloudpickle-1.2.2-src.zip differ diff --git a/flink-python/lib/cloudpickle-LICENSE.txt b/flink-python/lib/cloudpickle-LICENSE.txt new file mode 100644 index 00000000000000..d112c4806aa90b --- /dev/null +++ b/flink-python/lib/cloudpickle-LICENSE.txt @@ -0,0 +1,32 @@ +This module was extracted from the `cloud` package, developed by +PiCloud, Inc. + +Copyright (c) 2015, Cloudpickle contributors. +Copyright (c) 2012, Regents of the University of California. +Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-python/pyflink/gen_protos.py b/flink-python/pyflink/gen_protos.py index 8051bd6df18282..ce4ddd4010a405 100644 --- a/flink-python/pyflink/gen_protos.py +++ b/flink-python/pyflink/gen_protos.py @@ -106,7 +106,7 @@ def generate_proto_files(force=False): # Though wheels are available for grpcio-tools, setup_requires uses -# easy_install which doesn't understand them. This means that it is +# easy_install which doesn't understand them. This means that it is # compiled from scratch (which is expensive as it compiles the full # protoc compiler). Instead, we attempt to install a wheel in a temporary # directory and add it to the path as needed. @@ -124,15 +124,18 @@ def _install_grpcio_tools_and_generate_proto_files(): [sys.executable, '-m', 'pip', 'install', '--prefix', install_path, '--build', build_path, '--upgrade', GRPC_TOOLS, "-I"]) - install_path = glob.glob(os.path.join( - install_path, "lib", "python%d.%d" % (sys.version_info.major, sys.version_info.minor), - "site-packages")).pop() + from distutils.dist import Distribution + install_obj = Distribution().get_command_obj('install', create=True) + install_obj.prefix = install_path + install_obj.finalize_options() logging.warning( 'Installing grpcio-tools took %0.2f seconds.', time.time() - start) finally: sys.stderr.flush() shutil.rmtree(build_path, ignore_errors=True) - sys.path.append(install_path) + sys.path.append(install_obj.install_purelib) + if install_obj.install_purelib != install_obj.install_platlib: + sys.path.append(install_obj.install_platlib) try: generate_proto_files() finally: diff --git a/flink-python/src/main/resources/META-INF/NOTICE b/flink-python/src/main/resources/META-INF/NOTICE index b02bb4d2bb5363..b2107533d628c8 100644 --- a/flink-python/src/main/resources/META-INF/NOTICE +++ b/flink-python/src/main/resources/META-INF/NOTICE @@ -53,6 +53,7 @@ See bundled license files for details - com.google.protobuf:protobuf-java:3.7.1 - com.google.protobuf:protobuf-java-util:3.7.1 - com.google.auth:google-auth-library-credentials:0.13.0 +- cloudpickle:1.2.2 This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) See bundled license files for details. diff --git a/flink-python/src/main/resources/META-INF/licenses/LICENSE.cloudpickle b/flink-python/src/main/resources/META-INF/licenses/LICENSE.cloudpickle new file mode 100644 index 00000000000000..d112c4806aa90b --- /dev/null +++ b/flink-python/src/main/resources/META-INF/licenses/LICENSE.cloudpickle @@ -0,0 +1,32 @@ +This module was extracted from the `cloud` package, developed by +PiCloud, Inc. + +Copyright (c) 2015, Cloudpickle contributors. +Copyright (c) 2012, Regents of the University of California. +Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.cloudpickle b/licenses-binary/LICENSE.cloudpickle new file mode 100644 index 00000000000000..d112c4806aa90b --- /dev/null +++ b/licenses-binary/LICENSE.cloudpickle @@ -0,0 +1,32 @@ +This module was extracted from the `cloud` package, developed by +PiCloud, Inc. + +Copyright (c) 2015, Cloudpickle contributors. +Copyright (c) 2012, Regents of the University of California. +Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/LICENSE.cloudpickle b/licenses/LICENSE.cloudpickle new file mode 100644 index 00000000000000..d112c4806aa90b --- /dev/null +++ b/licenses/LICENSE.cloudpickle @@ -0,0 +1,32 @@ +This module was extracted from the `cloud` package, developed by +PiCloud, Inc. + +Copyright (c) 2015, Cloudpickle contributors. +Copyright (c) 2012, Regents of the University of California. +Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.