Skip to content

Commit

Permalink
[cherry-pick] support python3.8 (#27450)
Browse files Browse the repository at this point in the history 
* Fix test_gast_with_compatibility.py due to the problem of gast in python3.8 (#27433)

test=develop

* fix dll load bug on windows from python3.8 (#27324)

* Support python3.8 (#26850)

* Support python3.8

test=notest

* Replace the 'spawn' start method with 'fork' start method for multiprocessing, on MacOS with python>=3.8 (#27317)

* Replace the 'spawn' start method with 'fork' start method for multiprocessing, on MacOs when python>=3.8

test=develop

* Correct the error in decorator.py (#27409)

test=develop

Co-authored-by: Zhou Wei <52485244+zhouwei25@users.noreply.github.com>
  • Loading branch information
@gfwm2013 @zhwesky2010
gfwm2013 and zhwesky2010 committed Sep 22, 2020
1 parent 442e95c commit 5260e47
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 46 deletions.
45 changes: 44 additions & 1 deletion paddle/scripts/paddle_build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,18 @@ function cmake_base() {
else
exit 1
fi
elif [ "$1" == "cp38-cp38" ]; then
if [ -d "/Library/Frameworks/Python.framework/Versions/3.8" ]; then
export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/
export DYLD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/
export PATH=/Library/Frameworks/Python.framework/Versions/3.8/bin/:${PATH}
PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.8/bin/python3
-DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.8/include/python3.8/
-DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.8/lib/libpython3.8.dylib"
pip3.8 install --user -r ${PADDLE_ROOT}/python/requirements.txt
else
exit 1
fi
fi
# delete `gym` to avoid modifying requirements.txt in *.whl
sed -i .bak "/^gym$/d" ${PADDLE_ROOT}/python/requirements.txt
Expand Down Expand Up @@ -176,6 +188,13 @@ function cmake_base() {
-DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.7.0/include/python3.7m
-DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.7.0/lib/libpython3.so"
pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt
elif [ "$1" == "cp38-cp38" ]; then
export LD_LIBRARY_PATH=/opt/_internal/cpython-3.8.0/lib/:${LD_LIBRARY_PATH}
export PATH=/opt/_internal/cpython-3.8.0/bin/:${PATH}
export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.8.0/bin/python3.8
-DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.8.0/include/python3.8
-DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.8.0/lib/libpython3.so"
pip3.8 install -r ${PADDLE_ROOT}/python/requirements.txt
fi
else
pip install -r ${PADDLE_ROOT}/python/requirements.txt
Expand Down Expand Up @@ -506,6 +525,8 @@ EOF
pip3.6 uninstall -y paddlepaddle
elif [ "$1" == "cp37-cp37m" ]; then
pip3.7 uninstall -y paddlepaddle
elif [ "$1" == "cp38-cp38" ]; then
pip3.8 uninstall -y paddlepaddle
fi
set -ex

Expand All @@ -519,6 +540,8 @@ EOF
pip3.6 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
elif [ "$1" == "cp37-cp37m" ]; then
pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
elif [ "$1" == "cp38-cp38" ]; then
pip3.8 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
fi
ut_startTime_s=`date +%s`
ctest --output-on-failure -j $2
Expand Down Expand Up @@ -592,7 +615,7 @@ function generate_api_spec() {

awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc
awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api
if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ]; then
if [ "$1" == "cp35-cp35m" ] || [ "$1" == "cp36-cp36m" ] || [ "$1" == "cp37-cp37m" ] || [ "$1" == "cp38-cp38" ]; then
# Use sed to make python2 and python3 sepc keeps the same
sed -i 's/arg0: str/arg0: unicode/g' $spec_path
sed -i "s/\(.*Transpiler.*\).__init__ (ArgSpec(args=\['self'].*/\1.__init__ /g" $spec_path
Expand Down Expand Up @@ -1075,21 +1098,25 @@ EOF
ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl
ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl
ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl
ref_paddle38=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp38-cp38-linux_x86_64.whl

ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp27-cp27mu-linux_x86_64.whl
ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp35-cp35m-linux_x86_64.whl
ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp36-cp36m-linux_x86_64.whl
ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp37-cp37m-linux_x86_64.whl
ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}-cp38-cp38-linux_x86_64.whl

if [[ ${PADDLE_BRANCH} != "0.0.0" && ${WITH_MKL} == "ON" && ${WITH_GPU} == "ON" ]]; then
ref_paddle2=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl
ref_paddle35=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl
ref_paddle36=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl
ref_paddle37=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl
ref_paddle38=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp38-cp38-linux_x86_64.whl
ref_paddle2_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp27-cp27mu-linux_x86_64.whl
ref_paddle35_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp35-cp35m-linux_x86_64.whl
ref_paddle36_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp36-cp36m-linux_x86_64.whl
ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp37-cp37m-linux_x86_64.whl
ref_paddle38_whl=paddlepaddle${install_gpu}-${PADDLE_BRANCH}.post${ref_CUDA_MAJOR}${CUDNN_MAJOR}-cp38-cp38-linux_x86_64.whl
fi

#ref_paddle2_mv1=""
Expand Down Expand Up @@ -1194,6 +1221,22 @@ EOF
apt-get clean -y && \
rm -f ${ref_paddle37} && \
ldconfig
EOF
cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
# run paddle version to install python packages first
RUN apt-get update && ${NCCL_DEPS}
RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
xz-utils tk-dev libffi-dev liblzma-dev
RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \
tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \
CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
make -j8 > /dev/null && make altinstall > /dev/null && cd ../ && rm Python-3.8.0.tgz
RUN apt-get install -y libgtk2.0-dev dmidecode python3-tk && ldconfig && \
pip3.8 install opencv-python && wget ${ref_web}/${ref_paddle38} && pip3.8 install ${ref_paddle38_whl}; apt-get install -f -y && \
apt-get clean -y && \
rm -f ${ref_paddle38} && \
ldconfig
EOF
cat >> ${PADDLE_ROOT}/build/Dockerfile <<EOF
# run paddle version to install python packages first
Expand Down
5 changes: 5 additions & 0 deletions python/paddle/fluid/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@
third_lib_path = current_path + os.sep + '..' + os.sep + 'libs'
os.environ['path'] = third_lib_path + ';' + os.environ['path']
sys.path.insert(0, third_lib_path)
# Note: from python3.8, PATH will not take effect
# https://github.com/python/cpython/pull/12302
# Use add_dll_directory to specify dll resolution path
if sys.version_info[:2] >= (3, 8):
os.add_dll_directory(third_lib_path)

except ImportError as e:
from .. import compat as cpt
Expand Down
96 changes: 55 additions & 41 deletions python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import ast
import gast
import sys
import textwrap
import unittest

Expand Down Expand Up @@ -143,47 +144,60 @@ def foo(x_new, y_new):
"""
self._check_compatibility(source, target)

def test_with(self):
"""
The fileds `context_expr/optional_vars` of `ast.With` in PY2
is moved into `ast.With.items.withitem` in PY3.
"""
source = """
with guard():
a = 1
"""
target = """
with guard_new():
a = 1
"""
self._check_compatibility(source, target)

def test_subscript_Index(self):
source = """
x = y()[10]
"""
target = """
x = y()[20]
"""
self._check_compatibility(source, target)

def test_subscript_Slice(self):
source = """
x = y()[10:20]
"""
target = """
x = y()[20:40]
"""
self._check_compatibility(source, target)

def test_call(self):
source = """
y = foo(*arg)
"""
target = """
y = foo(*arg_new)
"""
self._check_compatibility(source, target)
# The 0.3.3 version of gast has a bug in python3.8 that
# would cause the following tests to fail. But this
# problem doesn't affect the use of Paddle's related
# functions, therefore, the following tests would be
# disable in python3.8.
#
# This problem had been fixed and updated to version
# 0.4.1 of gast.
#
# More information please refer to:
# https://github.com/serge-sans-paille/gast/issues/49
if sys.version_info < (3, 8):

def test_with(self):
"""
The fileds `context_expr/optional_vars` of `ast.With` in PY2
is moved into `ast.With.items.withitem` in PY3.
"""
source = """
with guard():
a = 1
"""
target = """
with guard_new():
a = 1
"""
self._check_compatibility(source, target)

def test_subscript_Index(self):
source = """
x = y()[10]
"""
target = """
x = y()[20]
"""
self._check_compatibility(source, target)

def test_subscript_Slice(self):
source = """
x = y()[10:20]
"""
target = """
x = y()[20:40]
"""
self._check_compatibility(source, target)

def test_call(self):
source = """
y = foo(*arg)
"""
target = """
y = foo(*arg_new)
"""
self._check_compatibility(source, target)


if __name__ == '__main__':
Expand Down
23 changes: 19 additions & 4 deletions python/paddle/reader/decorator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,21 @@
import zlib
import paddle.compat as cpt

# On macOS, the 'spawn' start method is now the default in Python3.8 multiprocessing,
# Paddle is currently unable to solve this, so forces the process to start using
# the 'fork' start method.
#
# TODO: This solution is not good, because the fork start method could lead to
# crashes of the subprocess. Figure out how to make 'spawn' work.
#
# For more details, please refer to
# https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
# https://bugs.python.org/issue33725
if sys.version_info >= (3, 8) and sys.platform == 'darwin':
fork_context = multiprocessing.get_context('fork')
else:
fork_context = multiprocessing


def cache(reader):
"""
Expand Down Expand Up @@ -556,9 +571,9 @@ def _read_into_queue(reader, queue):
six.reraise(*sys.exc_info())

def queue_reader():
queue = multiprocessing.Queue(queue_size)
queue = fork_context.Queue(queue_size)
for reader in readers:
p = multiprocessing.Process(
p = fork_context.Process(
target=_read_into_queue, args=(reader, queue))
p.start()

Expand Down Expand Up @@ -589,9 +604,9 @@ def _read_into_pipe(reader, conn):
def pipe_reader():
conns = []
for reader in readers:
parent_conn, child_conn = multiprocessing.Pipe()
parent_conn, child_conn = fork_context.Pipe()
conns.append(parent_conn)
p = multiprocessing.Process(
p = fork_context.Process(
target=_read_into_pipe, args=(reader, child_conn))
p.start()

Expand Down

2 comments on commit 5260e47

@paddle-bot-old
Copy link

@paddle-bot-old paddle-bot-old bot commented on 5260e47 Sep 22, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🕵️ CI failures summary

🔍 Commit ID: 5260e47 contains failed CI.

  • Failed: PR_Windows_CI_Release_1.8 (Paddle Windows)
  • Failed: MAC_Python3_Build (Paddle Mac Build)

@paddle-bot-old
Copy link

@paddle-bot-old paddle-bot-old bot commented on 5260e47 Sep 22, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🕵️ CI failures summary

🔍 Commit ID: 5260e47 contains failed CI.

  • Failed: PR_Windows_CI_Release_1.8 (Paddle Windows)
  • Failed: MAC_Python3_Build (Paddle Mac Build)

Please sign in to comment.