From b78e94ee62ec85d9456ae3e1acb1c79679d59ddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 8 Oct 2020 12:03:30 -0500 Subject: [PATCH 01/24] [DEBUG] Check wheel relocation issues --- packaging/build_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 60adf38f901..1235b84471f 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -17,6 +17,7 @@ if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then bin_path=$(dirname $python_exec) env_path=$(dirname $bin_path) if [[ "$(uname)" == Darwin ]]; then + echo "Debugging" # Include LibPNG cp "$env_path/lib/libpng16.dylib" torchvision # Include LibJPEG From 4ad8921367a00de6901b17250312c2a7b1aa51ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 8 Oct 2020 13:54:36 -0500 Subject: [PATCH 02/24] Call delocate on Mac --- packaging/build_wheel.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 1235b84471f..776e9da6151 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -17,16 +17,15 @@ if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then bin_path=$(dirname $python_exec) env_path=$(dirname $bin_path) if [[ "$(uname)" == Darwin ]]; then - echo "Debugging" - # Include LibPNG - cp "$env_path/lib/libpng16.dylib" torchvision - # Include LibJPEG - cp "$env_path/lib/libjpeg.dylib" torchvision + # Install delocate to relocate the required binaries + pip_install delocate else cp "$bin_path/Library/bin/libpng16.dll" torchvision cp "$bin_path/Library/bin/libjpeg.dll" torchvision fi else + # Install patchelf to relocate the required binaries + conda install patchelf # Include LibPNG cp "/usr/lib64/libpng.so" torchvision # Include LibJPEG @@ -40,3 +39,14 @@ if [[ "$OSTYPE" == "msys" ]]; then else IS_WHEEL=1 python setup.py bdist_wheel fi + + +if [[ "$(uname)" == Darwin ]]; then + pushd dist/ + python_exec="$(which python)" + bin_path=$(dirname $python_exec) + env_path=$(dirname $bin_path) + for whl in *.whl; do + DYLD_LIBRARY_PATH="$env_path/lib/:$DYLD_LIBRARY_PATH" delocate-wheel -v $whl + done +fi From 76f3b0a89f91e68a8b7b3fec135e4df3892c02f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 8 Oct 2020 14:03:32 -0500 Subject: [PATCH 03/24] Use yum instead of conda --- packaging/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 776e9da6151..ad30a21ad93 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -25,7 +25,7 @@ if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then fi else # Install patchelf to relocate the required binaries - conda install patchelf + yum install -y patchelf # Include LibPNG cp "/usr/lib64/libpng.so" torchvision # Include LibJPEG From d1f1bc5ed169fcfb0c81b00e51d650fc8e0fcb0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 8 Oct 2020 14:52:24 -0500 Subject: [PATCH 04/24] Do not copy ffmpeg dylibs --- packaging/pkg_helpers.bash | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index da77fe57047..c43bd61d647 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -367,11 +367,11 @@ download_copy_ffmpeg() { conda install -yq wget wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/osx-64/ffmpeg-4.2-h0a44026_0.tar.bz2 tar -xjvf ffmpeg-4.2-h0a44026_0.tar.bz2 - for f in lib/*.dylib; do - if [[ $f =~ ([a-z])+\.dylib ]]; then - cp $f ../torchvision - fi - done + # for f in lib/*.dylib; do + # if [[ $f =~ ([a-z])+\.dylib ]]; then + # cp $f ../torchvision + # fi + # done else wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2 tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2 From 6888d146f2b5bef255e108a512562b6b639cf49e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 8 Oct 2020 18:30:45 -0500 Subject: [PATCH 05/24] Linux dry run --- packaging/build_wheel.sh | 13 +- packaging/pkg_helpers.bash | 63 ++++++++- packaging/wheel/relocate.py | 261 ++++++++++++++++++++++++++++++++++++ 3 files changed, 324 insertions(+), 13 deletions(-) create mode 100644 packaging/wheel/relocate.py diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index ad30a21ad93..aedbe513fb4 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -24,12 +24,13 @@ if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then cp "$bin_path/Library/bin/libjpeg.dll" torchvision fi else - # Install patchelf to relocate the required binaries - yum install -y patchelf - # Include LibPNG - cp "/usr/lib64/libpng.so" torchvision - # Include LibJPEG - cp "/usr/lib64/libjpeg.so" torchvision + # Install auditwheel to get some inspection utilities + pip_install auditwheel + + # Point to custom libraries + export LD_LIBRARY_PATH=$(pwd)/ext_libraries/lib:$LD_LIBRARY_PATH + export TORCHVISION_INCLUDE=$(pwd)/ext_libraries/include + export TORCHVISION_LIBRARY=$(pwd)/ext_libraries/lib fi download_copy_ffmpeg diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index c43bd61d647..d0f7f63f358 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -184,8 +184,8 @@ setup_wheel_python() { # Install libpng from Anaconda (defaults) conda install libpng jpeg -y else - # Install native CentOS libPNG - yum install -y libpng-devel libjpeg-turbo-devel + # Install native CentOS libJPEG + yum install -y libjpeg-turbo-devel case "$PYTHON_VERSION" in 2.7) if [[ -n "$UNICODE_ABI" ]]; then @@ -203,7 +203,54 @@ setup_wheel_python() { exit 1 ;; esac - export PATH="/opt/python/$python_abi/bin:$PATH" + # Download all the dependencies required to compile image and video_reader + # extensions + + mkdir -p ext_libraries + pushd ext_libraries + + # --------------- ZLib ------------------------ + wget https://anaconda.org/anaconda/zlib/1.2.11/download/linux-64/zlib-1.2.11-h7b6447c_3.tar.bz2 + tar -xjvf zlib-1.2.11-h7b6447c_3.tar.bz2 + rm -rf zlib-1.2.11-h7b6447c_3.tar.bz2 + + # --------------- LibPNG ------------------------ + wget https://anaconda.org/anaconda/libpng/1.6.32/download/linux-64/libpng-1.6.32-hbd3595f_4.tar.bz2 + tar -xjvf libpng-1.6.32-hbd3595f_4.tar.bz2 + rm -rf libpng-1.6.32-hbd3595f_4.tar.bz2 + + # --------------- OpenH264 ---------------------- + wget https://anaconda.org/anaconda/openh264/2.1.0/download/linux-64/openh264-2.1.0-hd408876_0.tar.bz2 + tar -xjvf openh264-2.1.0-hd408876_0.tar.bz2 + rm -rf openh264-2.1.0-hd408876_0.tar.bz2 + + # --------------- LAME -------------------------- + wget https://anaconda.org/anaconda/lame/3.100/download/linux-64/lame-3.100-h7b6447c_0.tar.bz2 + tar -xjvf lame-3.100-h7b6447c_0.tar.bz2 + rm -rf lame-3.100-h7b6447c_0.tar.bz2 + + # --------------- Nettle ------------------------ + wget https://anaconda.org/anaconda/nettle/3.4.1/download/linux-64/nettle-3.4.1-hbb512f6_0.tar.bz2 + tar -xjvf nettle-3.4.1-hbb512f6_0.tar.bz2 + rm -rf nettle-3.4.1-hbb512f6_0.tar.bz2 + + # --------------- GMP --------------------------- + wget https://anaconda.org/anaconda/gmp/6.1.2/download/linux-64/gmp-6.1.2-h6c8ec71_1.tar.bz2 + tar -xjvf gmp-6.1.2-h6c8ec71_1.tar.bz2 + rm -rf gmp-6.1.2-h6c8ec71_1.tar.bz2 + + # --------------- libiconv ---------------------- + wget https://anaconda.org/anaconda/libiconv/1.15/download/linux-64/libiconv-1.15-h63c8f33_5.tar.bz2 + tar -xjvf libiconv-1.15-h63c8f33_5.tar.bz2 + rm -rf libiconv-1.15-h63c8f33_5.tar.bz2 + + # --------------- gnutls ------------------------ + wget https://anaconda.org/anaconda/gnutls/3.6.5/download/linux-64/gnutls-3.6.5-h71b1129_1002.tar.bz2 + tar -xvjf gnutls-3.6.5-h71b1129_1002.tar.bz2 + rm -rf gnutls-3.6.5-h71b1129_1002.tar.bz2 + + export PATH="/opt/python/$python_abi/bin:$(pwd)/bin:$PATH" + popd fi } @@ -373,12 +420,14 @@ download_copy_ffmpeg() { # fi # done else + pushd ext_libraries wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2 tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2 - cp lib/*.so ../torchvision - cp -r lib/* /usr/lib - cp -r bin/* /usr/bin - cp -r include/* /usr/include + rm -rf ffmpeg-4.2-hf484d3e_0.tar.bz2 + # cp lib/*.so ../torchvision + # cp -r lib/* /usr/lib + # cp -r bin/* /usr/bin + # cp -r include/* /usr/include ldconfig which ffmpeg fi diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py new file mode 100644 index 00000000000..27de304fcb0 --- /dev/null +++ b/packaging/wheel/relocate.py @@ -0,0 +1,261 @@ +# -*- coding: utf-8 -*- + +"""Helper script to package wheels and relocate binaries.""" + +# Standard library imports +import os +import io +import sys +import glob +import shutil +import zipfile +import hashlib +import platform +import subprocess +import os.path as osp +from base64 import urlsafe_b64encode + +# Third party imports +import toml +from auditwheel.lddtree import lddtree +from wheel.bdist_wheel import get_abi_tag + + +HERE = osp.dirname(osp.abspath(__file__)) +PACKAGE_ROOT = osp.dirname(osp.dirname(HERE)) +PLATFORM_ARCH = platform.machine() +PYTHON_VERSION = sys.version_info + + +def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): + """Yield pieces of data from a file-like object until EOF.""" + while True: + chunk = file.read(size) + if not chunk: + break + yield chunk + + +def rehash(path, blocksize=1 << 20): + """Return (hash, length) for path using hashlib.sha256()""" + h = hashlib.sha256() + length = 0 + with open(path, 'rb') as f: + for block in read_chunks(f, size=blocksize): + length += len(block) + h.update(block) + digest = 'sha256=' + urlsafe_b64encode( + h.digest() + ).decode('latin1').rstrip('=') + # unicode/str python2 issues + return (digest, str(length)) # type: ignore + + +def unzip_file(file, dest): + """Decompress zip `file` into directory `dest`.""" + with zipfile.ZipFile(file, 'r') as zip_ref: + zip_ref.extractall(dest) + + +def is_program_installed(basename): + """ + Return program absolute path if installed in PATH. + Otherwise, return None + On macOS systems, a .app is considered installed if + it exists. + """ + if (sys.platform == 'darwin' and basename.endswith('.app') and + osp.exists(basename)): + return basename + + for path in os.environ["PATH"].split(os.pathsep): + abspath = osp.join(path, basename) + if osp.isfile(abspath): + return abspath + + +def find_program(basename): + """ + Find program in PATH and return absolute path + Try adding .exe or .bat to basename on Windows platforms + (return None if not found) + """ + names = [basename] + if os.name == 'nt': + # Windows platforms + extensions = ('.exe', '.bat', '.cmd') + if not basename.endswith(extensions): + names = [basename + ext for ext in extensions] + [basename] + for name in names: + path = is_program_installed(name) + if path: + return path + + +def patch_new_path(library_path, new_dir): + library = osp.basename(library_path) + name, *rest = library.split('.') + rest = '.'.join(rest) + hash_id = hashlib.sha256(library_path.encode('utf-8')).hexdigest()[:8] + new_name = '.'.join([name, hash_id, rest]) + return osp.join(new_dir, new_name) + + +def patch_linux(): + # Get patchelf location + patchelf = find_program('patchelf') + if patchelf is None: + raise FileNotFoundError('Patchelf was not found in the system, please' + ' make sure that is available on the PATH.') + + # Find wheel + print('Finding wheels...') + wheels = glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl')) + output_dir = osp.join(PACKAGE_ROOT, 'dist', '.wheel-process') + + if osp.exists(output_dir): + shutil.rmtree(output_dir) + + os.makedirs(output_dir) + + image_binary = 'image.so' + video_binary = 'video_reader.so' + for wheel in wheels: + print('Unzipping wheel...') + print('{0}'.format(osp.basename(wheel))) + unzip_file(wheel, output_dir) + + print('Finding ELF dependencies...') + + output_library = osp.join(output_dir, 'torchvision') + image_binary_path = osp.join(output_library, image_binary) + + ld_tree = lddtree(image_binary_path) + tree_libs = ld_tree['libs'] + + binary_queue = [(n, image_binary) for n in ld_tree['needed']] + binary_paths = {image_binary: image_binary_path} + binary_dependencies = {} + + while binary_queue != []: + library, parent = binary_queue.pop(0) + library_info = tree_libs[library] + print(library) + print(library_info) + if (library_info['path'].startswith('/lib')): + # Omit glibc/gcc/system libraries + continue + + parent_dependencies = binary_dependencies.get(parent, []) + parent_dependencies.append(library) + binary_dependencies[parent] = parent_dependencies + + if library in binary_paths: + continue + + binary_paths[library] = library_info['path'] + binary_queue += [(n, library) for n in library_info['needed']] + + print('Copying dependencies to wheel directory') + new_libraries_path = osp.join(output_dir, 'duckling.libs') + os.makedirs(new_libraries_path) + new_names = {main_binary: binary_path} + + for library in binary_paths: + if library != main_binary: + library_path = binary_paths[library] + new_library_path = patch_new_path(library_path, new_libraries_path) + print('{0} -> {1}'.format(library, new_library_path)) + shutil.copyfile(library_path, new_library_path) + new_names[library] = new_library_path + + print('Updating dependency names by new files') + for library in binary_paths: + if library != main_binary: + if library not in binary_dependencies: + continue + library_dependencies = binary_dependencies[library] + new_library_name = new_names[library] + for dep in library_dependencies: + new_dep = osp.basename(new_names[dep]) + print('{0}: {1} -> {2}'.format(library, dep, new_dep)) + subprocess.check_output( + [ + patchelf, + '--replace-needed', + dep, + new_dep, + new_library_name + ], + cwd=new_libraries_path) + + print('Updating library rpath') + subprocess.check_output( + [ + patchelf, + '--set-rpath', + "$ORIGIN", + new_library_name + ], + cwd=new_libraries_path) + + subprocess.check_output( + [ + patchelf, + '--print-rpath', + new_library_name + ], + cwd=new_libraries_path) + + print("Update main library dependencies") + library_dependencies = binary_dependencies[main_binary] + for dep in library_dependencies: + new_dep = osp.basename(new_names[dep]) + print('{0}: {1} -> {2}'.format(main_binary, dep, new_dep)) + subprocess.check_output( + [ + patchelf, + '--replace-needed', + dep, + new_dep, + main_binary + ], + cwd=output_library) + + print('Update main library rpath') + subprocess.check_output( + [ + patchelf, + '--set-rpath', + "$ORIGIN:$ORIGIN/../duckling.libs", + binary_path + ], + cwd=output_library + ) + + print('Update RECORD file in wheel') + dist_info = osp.join( + output_dir, 'pyduckling_native-{0}.dist-info'.format(version)) + record_file = osp.join(dist_info, 'RECORD') + + with open(record_file, 'w') as f: + for root, _, files in os.walk(output_dir): + for this_file in files: + full_file = osp.join(root, this_file) + rel_file = osp.relpath(full_file, output_dir) + if full_file == record_file: + f.write('{0},,\n'.format(rel_file)) + else: + digest, size = rehash(full_file) + f.write('{0},{1},{2}\n'.format(rel_file, digest, size)) + + print('Compressing wheel') + shutil.make_archive(dist, 'zip', output_dir) + os.remove(dist) + shutil.move('{0}.zip'.format(dist), dist) + shutil.rmtree(output_dir) + + +if __name__ == '__main__': + if sys.platform == 'linux': + patch_linux() From c9627b1852dbb4cd18f5be7e3ca060eea9f54a1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 8 Oct 2020 18:35:10 -0500 Subject: [PATCH 06/24] Do not download FFmpeg on Mac --- packaging/pkg_helpers.bash | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index d0f7f63f358..575627eca07 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -400,8 +400,6 @@ setup_junit_results_folder() { download_copy_ffmpeg() { - mkdir ffmpeg_tmp - cd ffmpeg_tmp if [[ "$OSTYPE" == "msys" ]]; then # conda install -yq ffmpeg -c pytorch # curl -L -q https://anaconda.org/pytorch/ffmpeg/4.3/download/win-64/ffmpeg-4.3-ha925a31_0.tar.bz2 --output ffmpeg-4.3-ha925a31_0.tar.bz2 @@ -412,13 +410,6 @@ download_copy_ffmpeg() { if [[ "$(uname)" == Darwin ]]; then conda install -yq ffmpeg=4.2 -c pytorch conda install -yq wget - wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/osx-64/ffmpeg-4.2-h0a44026_0.tar.bz2 - tar -xjvf ffmpeg-4.2-h0a44026_0.tar.bz2 - # for f in lib/*.dylib; do - # if [[ $f =~ ([a-z])+\.dylib ]]; then - # cp $f ../torchvision - # fi - # done else pushd ext_libraries wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2 From 90611efea07691da4a215c290ea36f68d146b0cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 8 Oct 2020 18:44:58 -0500 Subject: [PATCH 07/24] Install bzip2 --- packaging/pkg_helpers.bash | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 575627eca07..2b556487d28 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -244,6 +244,11 @@ setup_wheel_python() { tar -xjvf libiconv-1.15-h63c8f33_5.tar.bz2 rm -rf libiconv-1.15-h63c8f33_5.tar.bz2 + # --------------- BZip2 ------------------------- + wget https://anaconda.org/anaconda/bzip2/1.0.8/download/linux-64/bzip2-1.0.8-h7b6447c_0.tar.bz2 + tar -xvjf bzip2-1.0.8-h7b6447c_0.tar.bz2 + rm -rf bzip2-1.0.8-h7b6447c_0.tar.bz2 + # --------------- gnutls ------------------------ wget https://anaconda.org/anaconda/gnutls/3.6.5/download/linux-64/gnutls-3.6.5-h71b1129_1002.tar.bz2 tar -xvjf gnutls-3.6.5-h71b1129_1002.tar.bz2 @@ -415,14 +420,9 @@ download_copy_ffmpeg() { wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2 tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2 rm -rf ffmpeg-4.2-hf484d3e_0.tar.bz2 - # cp lib/*.so ../torchvision - # cp -r lib/* /usr/lib - # cp -r bin/* /usr/bin - # cp -r include/* /usr/include ldconfig which ffmpeg + popd fi fi - cd .. - rm -rf ffmpeg_tmp } From 3364818ba82fdd7c60021a210aab0b4b65cd6bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 17:48:29 -0500 Subject: [PATCH 08/24] Restore FFmpeg on Windows --- .circleci/unittest/windows/scripts/environment.yml | 1 + packaging/pkg_helpers.bash | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 9f4348ebb26..59dc62ebfd5 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -9,6 +9,7 @@ dependencies: - pip - libpng - jpeg + - ffmpeg=4.2 - ca-certificates - pip: - future diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 2b556487d28..328e924c49b 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -254,8 +254,8 @@ setup_wheel_python() { tar -xvjf gnutls-3.6.5-h71b1129_1002.tar.bz2 rm -rf gnutls-3.6.5-h71b1129_1002.tar.bz2 - export PATH="/opt/python/$python_abi/bin:$(pwd)/bin:$PATH" popd + export PATH="/opt/python/$python_abi/bin:$(pwd)/ext_libraries/bin:$PATH" fi } @@ -406,11 +406,10 @@ setup_junit_results_folder() { download_copy_ffmpeg() { if [[ "$OSTYPE" == "msys" ]]; then - # conda install -yq ffmpeg -c pytorch + conda install -yq ffmpeg=4.2 -c pytorch # curl -L -q https://anaconda.org/pytorch/ffmpeg/4.3/download/win-64/ffmpeg-4.3-ha925a31_0.tar.bz2 --output ffmpeg-4.3-ha925a31_0.tar.bz2 # bzip2 --decompress --stdout ffmpeg-4.3-ha925a31_0.tar.bz2 | tar -x --file=- # cp Library/bin/*.dll ../torchvision - echo "FFmpeg is disabled currently on Windows" else if [[ "$(uname)" == Darwin ]]; then conda install -yq ffmpeg=4.2 -c pytorch From be5460595556a9589d693764e33ba41d167e6cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 17:57:22 -0500 Subject: [PATCH 09/24] Remove ffmpeg temporarily --- .circleci/unittest/windows/scripts/environment.yml | 2 +- packaging/pkg_helpers.bash | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 59dc62ebfd5..e9f3503b3df 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -9,7 +9,7 @@ dependencies: - pip - libpng - jpeg - - ffmpeg=4.2 + # - ffmpeg=4.2 - ca-certificates - pip: - future diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 328e924c49b..43e25c4f0e3 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -406,10 +406,11 @@ setup_junit_results_folder() { download_copy_ffmpeg() { if [[ "$OSTYPE" == "msys" ]]; then - conda install -yq ffmpeg=4.2 -c pytorch + # conda install -yq ffmpeg=4.2 -c pytorch # curl -L -q https://anaconda.org/pytorch/ffmpeg/4.3/download/win-64/ffmpeg-4.3-ha925a31_0.tar.bz2 --output ffmpeg-4.3-ha925a31_0.tar.bz2 # bzip2 --decompress --stdout ffmpeg-4.3-ha925a31_0.tar.bz2 | tar -x --file=- # cp Library/bin/*.dll ../torchvision + echo "FFmpeg is disabled currently on Windows" else if [[ "$(uname)" == Darwin ]]; then conda install -yq ffmpeg=4.2 -c pytorch From ffc6e2fec02ce723f5fc5ed5ada84ce136a96962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 18:01:56 -0500 Subject: [PATCH 10/24] Do not remove dependencies --- packaging/pkg_helpers.bash | 46 -------------------------------------- 1 file changed, 46 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 43e25c4f0e3..c5f367b5f5e 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -208,52 +208,6 @@ setup_wheel_python() { mkdir -p ext_libraries pushd ext_libraries - - # --------------- ZLib ------------------------ - wget https://anaconda.org/anaconda/zlib/1.2.11/download/linux-64/zlib-1.2.11-h7b6447c_3.tar.bz2 - tar -xjvf zlib-1.2.11-h7b6447c_3.tar.bz2 - rm -rf zlib-1.2.11-h7b6447c_3.tar.bz2 - - # --------------- LibPNG ------------------------ - wget https://anaconda.org/anaconda/libpng/1.6.32/download/linux-64/libpng-1.6.32-hbd3595f_4.tar.bz2 - tar -xjvf libpng-1.6.32-hbd3595f_4.tar.bz2 - rm -rf libpng-1.6.32-hbd3595f_4.tar.bz2 - - # --------------- OpenH264 ---------------------- - wget https://anaconda.org/anaconda/openh264/2.1.0/download/linux-64/openh264-2.1.0-hd408876_0.tar.bz2 - tar -xjvf openh264-2.1.0-hd408876_0.tar.bz2 - rm -rf openh264-2.1.0-hd408876_0.tar.bz2 - - # --------------- LAME -------------------------- - wget https://anaconda.org/anaconda/lame/3.100/download/linux-64/lame-3.100-h7b6447c_0.tar.bz2 - tar -xjvf lame-3.100-h7b6447c_0.tar.bz2 - rm -rf lame-3.100-h7b6447c_0.tar.bz2 - - # --------------- Nettle ------------------------ - wget https://anaconda.org/anaconda/nettle/3.4.1/download/linux-64/nettle-3.4.1-hbb512f6_0.tar.bz2 - tar -xjvf nettle-3.4.1-hbb512f6_0.tar.bz2 - rm -rf nettle-3.4.1-hbb512f6_0.tar.bz2 - - # --------------- GMP --------------------------- - wget https://anaconda.org/anaconda/gmp/6.1.2/download/linux-64/gmp-6.1.2-h6c8ec71_1.tar.bz2 - tar -xjvf gmp-6.1.2-h6c8ec71_1.tar.bz2 - rm -rf gmp-6.1.2-h6c8ec71_1.tar.bz2 - - # --------------- libiconv ---------------------- - wget https://anaconda.org/anaconda/libiconv/1.15/download/linux-64/libiconv-1.15-h63c8f33_5.tar.bz2 - tar -xjvf libiconv-1.15-h63c8f33_5.tar.bz2 - rm -rf libiconv-1.15-h63c8f33_5.tar.bz2 - - # --------------- BZip2 ------------------------- - wget https://anaconda.org/anaconda/bzip2/1.0.8/download/linux-64/bzip2-1.0.8-h7b6447c_0.tar.bz2 - tar -xvjf bzip2-1.0.8-h7b6447c_0.tar.bz2 - rm -rf bzip2-1.0.8-h7b6447c_0.tar.bz2 - - # --------------- gnutls ------------------------ - wget https://anaconda.org/anaconda/gnutls/3.6.5/download/linux-64/gnutls-3.6.5-h71b1129_1002.tar.bz2 - tar -xvjf gnutls-3.6.5-h71b1129_1002.tar.bz2 - rm -rf gnutls-3.6.5-h71b1129_1002.tar.bz2 - popd export PATH="/opt/python/$python_abi/bin:$(pwd)/ext_libraries/bin:$PATH" fi From 5f439f8a19a35a8ad66df81f0f60c339b69c18e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 18:07:16 -0500 Subject: [PATCH 11/24] Disable FFmpeg temporarily on Linux wheels --- packaging/pkg_helpers.bash | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index c5f367b5f5e..7fabbd47bb8 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -370,13 +370,14 @@ download_copy_ffmpeg() { conda install -yq ffmpeg=4.2 -c pytorch conda install -yq wget else - pushd ext_libraries - wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2 - tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2 - rm -rf ffmpeg-4.2-hf484d3e_0.tar.bz2 - ldconfig - which ffmpeg - popd + # pushd ext_libraries + # wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2 + # tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2 + # rm -rf ffmpeg-4.2-hf484d3e_0.tar.bz2 + # ldconfig + # which ffmpeg + # popd + echo "FFmpeg is disabled currently on Linux" fi fi } From 690b8729f242a71c352f917fdc7c3794da2417ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 19:05:30 -0500 Subject: [PATCH 12/24] Test relocation in Linux --- packaging/wheel/relocate.py | 247 ++++++++++++++++++++---------------- 1 file changed, 135 insertions(+), 112 deletions(-) diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index 27de304fcb0..04e1490a16b 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -16,11 +16,20 @@ from base64 import urlsafe_b64encode # Third party imports -import toml from auditwheel.lddtree import lddtree from wheel.bdist_wheel import get_abi_tag +WHITELIST = { + 'libgcc_s.so.1', 'libstdc++.so.6', 'libm.so.6', + 'libdl.so.2', 'librt.so.1', 'libc.so.6', + 'libnsl.so.1', 'libutil.so.1', 'libpthread.so.0', + 'libresolv.so.2', 'libX11.so.6', 'libXext.so.6', + 'libXrender.so.1', 'libICE.so.6', 'libSM.so.6', + 'libGL.so.1', 'libgobject-2.0.so.0', 'libgthread-2.0.so.0', + 'libglib-2.0.so.0', 'ld-linux-x86-64.so.2', 'ld-2.17.so' +} + HERE = osp.dirname(osp.abspath(__file__)) PACKAGE_ROOT = osp.dirname(osp.dirname(HERE)) PLATFORM_ARCH = platform.machine() @@ -101,142 +110,156 @@ def patch_new_path(library_path, new_dir): return osp.join(new_dir, new_name) -def patch_linux(): - # Get patchelf location - patchelf = find_program('patchelf') - if patchelf is None: - raise FileNotFoundError('Patchelf was not found in the system, please' - ' make sure that is available on the PATH.') +def relocate_library(patchelf, output_dir, output_library, binary): + print('Relocating {0}'.format(binary)) + binary_path = osp.join(output_library, binary) - # Find wheel - print('Finding wheels...') - wheels = glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl')) - output_dir = osp.join(PACKAGE_ROOT, 'dist', '.wheel-process') + ld_tree = lddtree(binary_path) + tree_libs = ld_tree['libs'] - if osp.exists(output_dir): - shutil.rmtree(output_dir) + binary_queue = [(n, binary) for n in ld_tree['needed']] + binary_paths = {binary: binary_path} + binary_dependencies = {} - os.makedirs(output_dir) + while binary_queue != []: + library, parent = binary_queue.pop(0) + library_info = tree_libs[library] + print(library) - image_binary = 'image.so' - video_binary = 'video_reader.so' - for wheel in wheels: - print('Unzipping wheel...') - print('{0}'.format(osp.basename(wheel))) - unzip_file(wheel, output_dir) + if library_info['path'] is None: + print('Omitting {0}'.format(library)) + continue - print('Finding ELF dependencies...') + if library in WHITELIST: + # Omit glibc/gcc/system libraries + print('Omitting {0}'.format(library)) + continue - output_library = osp.join(output_dir, 'torchvision') - image_binary_path = osp.join(output_library, image_binary) - - ld_tree = lddtree(image_binary_path) - tree_libs = ld_tree['libs'] - - binary_queue = [(n, image_binary) for n in ld_tree['needed']] - binary_paths = {image_binary: image_binary_path} - binary_dependencies = {} - - while binary_queue != []: - library, parent = binary_queue.pop(0) - library_info = tree_libs[library] - print(library) - print(library_info) - if (library_info['path'].startswith('/lib')): - # Omit glibc/gcc/system libraries - continue + parent_dependencies = binary_dependencies.get(parent, []) + parent_dependencies.append(library) + binary_dependencies[parent] = parent_dependencies - parent_dependencies = binary_dependencies.get(parent, []) - parent_dependencies.append(library) - binary_dependencies[parent] = parent_dependencies + if library in binary_paths: + continue - if library in binary_paths: - continue + binary_paths[library] = library_info['path'] + binary_queue += [(n, library) for n in library_info['needed']] - binary_paths[library] = library_info['path'] - binary_queue += [(n, library) for n in library_info['needed']] - - print('Copying dependencies to wheel directory') - new_libraries_path = osp.join(output_dir, 'duckling.libs') - os.makedirs(new_libraries_path) - new_names = {main_binary: binary_path} - - for library in binary_paths: - if library != main_binary: - library_path = binary_paths[library] - new_library_path = patch_new_path(library_path, new_libraries_path) - print('{0} -> {1}'.format(library, new_library_path)) - shutil.copyfile(library_path, new_library_path) - new_names[library] = new_library_path - - print('Updating dependency names by new files') - for library in binary_paths: - if library != main_binary: - if library not in binary_dependencies: - continue - library_dependencies = binary_dependencies[library] - new_library_name = new_names[library] - for dep in library_dependencies: - new_dep = osp.basename(new_names[dep]) - print('{0}: {1} -> {2}'.format(library, dep, new_dep)) - subprocess.check_output( - [ - patchelf, - '--replace-needed', - dep, - new_dep, - new_library_name - ], - cwd=new_libraries_path) - - print('Updating library rpath') - subprocess.check_output( - [ - patchelf, - '--set-rpath', - "$ORIGIN", - new_library_name - ], - cwd=new_libraries_path) + print('Copying dependencies to wheel directory') + new_libraries_path = osp.join(output_dir, 'torchvision.libs') + os.makedirs(new_libraries_path) + new_names = {binary: binary_path} + + for library in binary_paths: + if library != binary: + library_path = binary_paths[library] + new_library_path = patch_new_path(library_path, new_libraries_path) + print('{0} -> {1}'.format(library, new_library_path)) + shutil.copyfile(library_path, new_library_path) + new_names[library] = new_library_path + + print('Updating dependency names by new files') + for library in binary_paths: + if library != binary: + if library not in binary_dependencies: + continue + library_dependencies = binary_dependencies[library] + new_library_name = new_names[library] + for dep in library_dependencies: + new_dep = osp.basename(new_names[dep]) + print('{0}: {1} -> {2}'.format(library, dep, new_dep)) subprocess.check_output( [ patchelf, - '--print-rpath', + '--replace-needed', + dep, + new_dep, new_library_name ], cwd=new_libraries_path) - print("Update main library dependencies") - library_dependencies = binary_dependencies[main_binary] - for dep in library_dependencies: - new_dep = osp.basename(new_names[dep]) - print('{0}: {1} -> {2}'.format(main_binary, dep, new_dep)) + print('Updating library rpath') subprocess.check_output( [ patchelf, - '--replace-needed', - dep, - new_dep, - main_binary + '--set-rpath', + "$ORIGIN", + new_library_name ], - cwd=output_library) + cwd=new_libraries_path) - print('Update main library rpath') + subprocess.check_output( + [ + patchelf, + '--print-rpath', + new_library_name + ], + cwd=new_libraries_path) + + print("Update main library dependencies") + library_dependencies = binary_dependencies[binary] + for dep in library_dependencies: + new_dep = osp.basename(new_names[dep]) + print('{0}: {1} -> {2}'.format(binary, dep, new_dep)) subprocess.check_output( [ patchelf, - '--set-rpath', - "$ORIGIN:$ORIGIN/../duckling.libs", - binary_path + '--replace-needed', + dep, + new_dep, + binary ], - cwd=output_library - ) + cwd=output_library) + + print('Update main library rpath') + subprocess.check_output( + [ + patchelf, + '--set-rpath', + "$ORIGIN:$ORIGIN/../torchvision.libs", + binary_path + ], + cwd=output_library + ) + + +def patch_linux(): + # Get patchelf location + patchelf = find_program('patchelf') + if patchelf is None: + raise FileNotFoundError('Patchelf was not found in the system, please' + ' make sure that is available on the PATH.') + + # Find wheel + print('Finding wheels...') + wheels = glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl')) + output_dir = osp.join(PACKAGE_ROOT, 'dist', '.wheel-process') + + if osp.exists(output_dir): + shutil.rmtree(output_dir) + + os.makedirs(output_dir) + + image_binary = 'image.so' + video_binary = 'video_reader.so' + torchvision_binaries = [image_binary, video_binary] + for wheel in wheels: + print('Unzipping wheel...') + wheel_file = osp.basename(wheel) + wheel_dir = osp.dirname(wheel) + print('{0}'.format(wheel_file)) + wheel_name, _ = osp.splitext(wheel_file) + unzip_file(wheel, output_dir) + + print('Finding ELF dependencies...') + output_library = osp.join(output_dir, 'torchvision') + for binary in torchvision_binaries: + if osp.exists(osp.join(output_library, binary)): + relocate_library(patchelf, output_dir, output_library, binary) print('Update RECORD file in wheel') - dist_info = osp.join( - output_dir, 'pyduckling_native-{0}.dist-info'.format(version)) - record_file = osp.join(dist_info, 'RECORD') + record_file = glob.glob(osp.join(output_dir, '*.dist-info'))[0] with open(record_file, 'w') as f: for root, _, files in os.walk(output_dir): @@ -250,9 +273,9 @@ def patch_linux(): f.write('{0},{1},{2}\n'.format(rel_file, digest, size)) print('Compressing wheel') - shutil.make_archive(dist, 'zip', output_dir) - os.remove(dist) - shutil.move('{0}.zip'.format(dist), dist) + shutil.make_archive(wheel_name, 'zip', output_dir) + os.remove(wheel) + shutil.move('{0}.zip'.format(osp.join(wheel_dir, wheel_name)), wheel) shutil.rmtree(output_dir) From ba1cac4e5886478a47868f8ed474d45a765ccd6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 19:09:00 -0500 Subject: [PATCH 13/24] Add docstring --- packaging/wheel/relocate.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index 04e1490a16b..57ce3116619 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -111,6 +111,12 @@ def patch_new_path(library_path, new_dir): def relocate_library(patchelf, output_dir, output_library, binary): + """ + Relocate a shared library to be packaged on a wheel. + + Given a shared library, find the transitive closure of its dependencies, + rename and copy them into the wheel while updating their respective rpaths. + """ print('Relocating {0}'.format(binary)) binary_path = osp.join(output_library, binary) @@ -197,7 +203,7 @@ def relocate_library(patchelf, output_dir, output_library, binary): ], cwd=new_libraries_path) - print("Update main library dependencies") + print("Update library dependencies") library_dependencies = binary_dependencies[binary] for dep in library_dependencies: new_dep = osp.basename(new_names[dep]) @@ -212,7 +218,7 @@ def relocate_library(patchelf, output_dir, output_library, binary): ], cwd=output_library) - print('Update main library rpath') + print('Update library rpath') subprocess.check_output( [ patchelf, From 104ed586783a07e21180b4045122421cf8f6235d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 19:33:15 -0500 Subject: [PATCH 14/24] Call relocation script --- packaging/build_wheel.sh | 6 ++++++ packaging/wheel/relocate.py | 8 +++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index aedbe513fb4..6ac4c92fa8d 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -50,4 +50,10 @@ if [[ "$(uname)" == Darwin ]]; then for whl in *.whl; do DYLD_LIBRARY_PATH="$env_path/lib/:$DYLD_LIBRARY_PATH" delocate-wheel -v $whl done +else + if [[ "$OSTYPE" == "msys" ]]; then + "$script_dir/windows/internal/vc_env_helper.bat" python $script_dir/wheel/relocate.py + else + LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python $script_dir/wheel/relocate.py + done fi diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index 57ce3116619..a7a74f0a783 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -265,7 +265,8 @@ def patch_linux(): relocate_library(patchelf, output_dir, output_library, binary) print('Update RECORD file in wheel') - record_file = glob.glob(osp.join(output_dir, '*.dist-info'))[0] + dist_info = glob.glob(osp.join(output_dir, '*.dist-info'))[0] + record_file = osp.join(dist_info, 'RECORD') with open(record_file, 'w') as f: for root, _, files in os.walk(output_dir): @@ -279,9 +280,10 @@ def patch_linux(): f.write('{0},{1},{2}\n'.format(rel_file, digest, size)) print('Compressing wheel') - shutil.make_archive(wheel_name, 'zip', output_dir) + base_wheel_name = osp.join(wheel_dir, wheel_name) + shutil.make_archive(base_wheel_name, 'zip', output_dir) os.remove(wheel) - shutil.move('{0}.zip'.format(osp.join(wheel_dir, wheel_name)), wheel) + shutil.move('{0}.zip'.format(base_wheel_name), wheel) shutil.rmtree(output_dir) From db0d453983b3a45f39282b6d08741ca0dc20a1e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 19:36:04 -0500 Subject: [PATCH 15/24] Minor error correction --- packaging/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh index 6ac4c92fa8d..47d03e4f9c2 100755 --- a/packaging/build_wheel.sh +++ b/packaging/build_wheel.sh @@ -55,5 +55,5 @@ else "$script_dir/windows/internal/vc_env_helper.bat" python $script_dir/wheel/relocate.py else LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python $script_dir/wheel/relocate.py - done + fi fi From a1b08239d8c719b19cf65050f1260f73e49a1ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 12 Oct 2020 19:42:14 -0500 Subject: [PATCH 16/24] Import auditwheel only on Linux --- packaging/wheel/relocate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index a7a74f0a783..67a2add996b 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -16,7 +16,8 @@ from base64 import urlsafe_b64encode # Third party imports -from auditwheel.lddtree import lddtree +if sys.platform == 'linux': + from auditwheel.lddtree import lddtree from wheel.bdist_wheel import get_abi_tag @@ -117,6 +118,7 @@ def relocate_library(patchelf, output_dir, output_library, binary): Given a shared library, find the transitive closure of its dependencies, rename and copy them into the wheel while updating their respective rpaths. """ + print('Relocating {0}'.format(binary)) binary_path = osp.join(output_library, binary) From 09e151be03009665d69706a1809fd438a8341361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 13 Oct 2020 11:54:57 -0500 Subject: [PATCH 17/24] Restore ffmpeg again on Windows --- .circleci/unittest/windows/scripts/environment.yml | 2 +- packaging/pkg_helpers.bash | 4 ++-- packaging/torchvision/meta.yaml | 4 ++-- packaging/wheel/relocate.py | 12 ++++++++++++ 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index e9f3503b3df..59dc62ebfd5 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -9,7 +9,7 @@ dependencies: - pip - libpng - jpeg - # - ffmpeg=4.2 + - ffmpeg=4.2 - ca-certificates - pip: - future diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 7fabbd47bb8..cff142bff2b 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -360,11 +360,11 @@ setup_junit_results_folder() { download_copy_ffmpeg() { if [[ "$OSTYPE" == "msys" ]]; then - # conda install -yq ffmpeg=4.2 -c pytorch + conda install -yq ffmpeg=4.2 -c pytorch # curl -L -q https://anaconda.org/pytorch/ffmpeg/4.3/download/win-64/ffmpeg-4.3-ha925a31_0.tar.bz2 --output ffmpeg-4.3-ha925a31_0.tar.bz2 # bzip2 --decompress --stdout ffmpeg-4.3-ha925a31_0.tar.bz2 | tar -x --file=- # cp Library/bin/*.dll ../torchvision - echo "FFmpeg is disabled currently on Windows" + # echo "FFmpeg is disabled currently on Windows" else if [[ "$(uname)" == Darwin ]]; then conda install -yq ffmpeg=4.2 -c pytorch diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index fadd9b47f72..d39251d138c 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -10,7 +10,7 @@ requirements: - {{ compiler('c') }} # [win] - libpng - jpeg - - ffmpeg =4.2 # [not win] + - ffmpeg =4.2 host: - python @@ -22,7 +22,7 @@ requirements: run: - python - libpng - - ffmpeg =4.2 # [not win] + - ffmpeg =4.2 - jpeg - pillow >=4.1.1 - numpy >=1.11 diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index 67a2add996b..8ef131051fa 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -31,6 +31,18 @@ 'libglib-2.0.so.0', 'ld-linux-x86-64.so.2', 'ld-2.17.so' } +WINDOWS_WHITELIST = { + 'MSVCP140.dll', 'KERNEL32.dll', + 'VCRUNTIME140_1.dll', 'VCRUNTIME140.dll', + 'api-ms-win-crt-heap-l1-1-0.dll', + 'api-ms-win-crt-runtime-l1-1-0.dll', + 'api-ms-win-crt-stdio-l1-1-0.dll', + 'api-ms-win-crt-filesystem-l1-1-0.dll', + 'api-ms-win-crt-string-l1-1-0.dll', + 'api-ms-win-crt-environment-l1-1-0.dll', +} + + HERE = osp.dirname(osp.abspath(__file__)) PACKAGE_ROOT = osp.dirname(osp.dirname(HERE)) PLATFORM_ARCH = platform.machine() From 154522f6e56d4ce04a18459d6332a16000cf1f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 13 Oct 2020 12:01:08 -0500 Subject: [PATCH 18/24] Return *device --- torchvision/csrc/cpu/video/Video.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/csrc/cpu/video/Video.cpp b/torchvision/csrc/cpu/video/Video.cpp index f3c55fd6dea..bb9e3044bf4 100644 --- a/torchvision/csrc/cpu/video/Video.cpp +++ b/torchvision/csrc/cpu/video/Video.cpp @@ -64,7 +64,7 @@ std::pair const* _parse_type( return p.first == stream_string; }); if (device != types.end()) { - return device; + return *device; } AT_ERROR("Expected one of [audio, video, subtitle, cc] ", stream_string); } From 2111d86fd07926efa83d3db7725b9d8c15f5cec9 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Tue, 13 Oct 2020 20:18:21 +0200 Subject: [PATCH 19/24] Try fix Windows --- torchvision/csrc/cpu/video/Video.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchvision/csrc/cpu/video/Video.cpp b/torchvision/csrc/cpu/video/Video.cpp index 66108c44881..778d4cce790 100644 --- a/torchvision/csrc/cpu/video/Video.cpp +++ b/torchvision/csrc/cpu/video/Video.cpp @@ -49,7 +49,7 @@ size_t fillAudioTensor(DecoderOutputMessage& msgs, torch::Tensor& audioFrame) { return fillTensorList(msgs, audioFrame); } -std::pair const* _parse_type( +std::array, 4>::const_iterator _parse_type( const std::string& stream_string) { static const std::array, 4> types = {{ {"video", TYPE_VIDEO}, @@ -64,7 +64,7 @@ std::pair const* _parse_type( return p.first == stream_string; }); if (device != types.end()) { - return *device; + return device; } TORCH_CHECK( false, "Expected one of [audio, video, subtitle, cc] ", stream_string); From f5917e2acdce09360aaf2119ca74370dd2fac8c6 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Tue, 13 Oct 2020 20:22:00 +0200 Subject: [PATCH 20/24] Fix clang-format --- torchvision/csrc/cpu/video/Video.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchvision/csrc/cpu/video/Video.cpp b/torchvision/csrc/cpu/video/Video.cpp index 778d4cce790..a97cf5010d4 100644 --- a/torchvision/csrc/cpu/video/Video.cpp +++ b/torchvision/csrc/cpu/video/Video.cpp @@ -49,8 +49,8 @@ size_t fillAudioTensor(DecoderOutputMessage& msgs, torch::Tensor& audioFrame) { return fillTensorList(msgs, audioFrame); } -std::array, 4>::const_iterator _parse_type( - const std::string& stream_string) { +std::array, 4>::const_iterator +_parse_type(const std::string& stream_string) { static const std::array, 4> types = {{ {"video", TYPE_VIDEO}, {"audio", TYPE_AUDIO}, From a6edd4d78376c69c6d64b8d0e47a7e929d432e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 13 Oct 2020 17:26:04 -0500 Subject: [PATCH 21/24] Start windows patchwork --- packaging/pkg_helpers.bash | 4 +-- packaging/wheel/relocate.py | 66 ++++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index cff142bff2b..19f9ebfac06 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -184,8 +184,8 @@ setup_wheel_python() { # Install libpng from Anaconda (defaults) conda install libpng jpeg -y else - # Install native CentOS libJPEG - yum install -y libjpeg-turbo-devel + # Install native CentOS libJPEG, LAME, freetype and GnuTLS + yum install -y libjpeg-turbo-devel lame freetype gnutls case "$PYTHON_VERSION" in 2.7) if [[ -n "$UNICODE_ABI" ]]; then diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index 8ef131051fa..f70b38c5344 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -123,9 +123,16 @@ def patch_new_path(library_path, new_dir): return osp.join(new_dir, new_name) -def relocate_library(patchelf, output_dir, output_library, binary): +def find_dll_dependencies(dumpbin, binary): + out = subprocess.run([dumpbin, "/dependents", binary], + stdout=subprocess.PIPE) + out = out.stdout.strip().decode('utf-8') + print(out) + + +def relocate_elf_library(patchelf, output_dir, output_library, binary): """ - Relocate a shared library to be packaged on a wheel. + Relocate an ELF shared library to be packaged on a wheel. Given a shared library, find the transitive closure of its dependencies, rename and copy them into the wheel while updating their respective rpaths. @@ -244,6 +251,10 @@ def relocate_library(patchelf, output_dir, output_library, binary): ) +def relocate_dll_library(dumpbin, output_dir, output_library, binary): + pass + + def patch_linux(): # Get patchelf location patchelf = find_program('patchelf') @@ -256,15 +267,15 @@ def patch_linux(): wheels = glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl')) output_dir = osp.join(PACKAGE_ROOT, 'dist', '.wheel-process') - if osp.exists(output_dir): - shutil.rmtree(output_dir) - - os.makedirs(output_dir) - image_binary = 'image.so' video_binary = 'video_reader.so' torchvision_binaries = [image_binary, video_binary] for wheel in wheels: + if osp.exists(output_dir): + shutil.rmtree(output_dir) + + os.makedirs(output_dir) + print('Unzipping wheel...') wheel_file = osp.basename(wheel) wheel_dir = osp.dirname(wheel) @@ -276,7 +287,8 @@ def patch_linux(): output_library = osp.join(output_dir, 'torchvision') for binary in torchvision_binaries: if osp.exists(osp.join(output_library, binary)): - relocate_library(patchelf, output_dir, output_library, binary) + relocate_elf_library( + patchelf, output_dir, output_library, binary) print('Update RECORD file in wheel') dist_info = glob.glob(osp.join(output_dir, '*.dist-info'))[0] @@ -301,6 +313,44 @@ def patch_linux(): shutil.rmtree(output_dir) +def patch_win(): + # Get dumpbin location + dumpbin = find_program('dumpbin') + if dumpbin is None: + raise FileNotFoundError('Dumpbin was not found in the system, please' + ' make sure that is available on the PATH.') + + # Find wheel + print('Finding wheels...') + wheels = glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl')) + output_dir = osp.join(PACKAGE_ROOT, 'dist', '.wheel-process') + + image_binary = 'image.pyd' + video_binary = 'video_reader.pyd' + torchvision_binaries = [image_binary, video_binary] + for wheel in wheels: + if osp.exists(output_dir): + shutil.rmtree(output_dir) + + os.makedirs(output_dir) + + print('Unzipping wheel...') + wheel_file = osp.basename(wheel) + # wheel_dir = osp.dirname(wheel) + print('{0}'.format(wheel_file)) + wheel_name, _ = osp.splitext(wheel_file) + unzip_file(wheel, output_dir) + + print('Finding DLL/PE dependencies...') + output_library = osp.join(output_dir, 'torchvision') + for binary in torchvision_binaries: + if osp.exists(osp.join(output_library, binary)): + relocate_dll_library( + dumpbin, output_dir, output_library, binary) + + if __name__ == '__main__': if sys.platform == 'linux': patch_linux() + elif sys.platform == 'win32': + patch_win() From d08e3b9b609efcd5b63f1a32fcf84a9b18d1ac63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 13 Oct 2020 18:38:21 -0500 Subject: [PATCH 22/24] Copy all DLLs --- packaging/wheel/relocate.py | 105 +++++++++++++++++++++++++++--------- 1 file changed, 80 insertions(+), 25 deletions(-) diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index f70b38c5344..aa2ecb6036a 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -40,6 +40,8 @@ 'api-ms-win-crt-filesystem-l1-1-0.dll', 'api-ms-win-crt-string-l1-1-0.dll', 'api-ms-win-crt-environment-l1-1-0.dll', + 'api-ms-win-crt-math-l1-1-0.dll', + 'api-ms-win-crt-convert-l1-1-0.dll' } @@ -105,7 +107,7 @@ def find_program(basename): names = [basename] if os.name == 'nt': # Windows platforms - extensions = ('.exe', '.bat', '.cmd') + extensions = ('.exe', '.bat', '.cmd', '.dll') if not basename.endswith(extensions): names = [basename + ext for ext in extensions] + [basename] for name in names: @@ -127,7 +129,12 @@ def find_dll_dependencies(dumpbin, binary): out = subprocess.run([dumpbin, "/dependents", binary], stdout=subprocess.PIPE) out = out.stdout.strip().decode('utf-8') - print(out) + start_index = out.find('dependencies:') + len('dependencies:') + end_index = out.find('Summary') + dlls = out[start_index:end_index].strip() + dlls = dlls.split(os.linesep) + dlls = [dll.strip() for dll in dlls] + return dlls def relocate_elf_library(patchelf, output_dir, output_library, binary): @@ -252,7 +259,73 @@ def relocate_elf_library(patchelf, output_dir, output_library, binary): def relocate_dll_library(dumpbin, output_dir, output_library, binary): - pass + print('Relocating {0}'.format(binary)) + binary_path = osp.join(output_library, binary) + + library_dlls = find_dll_dependencies(dumpbin, binary_path) + binary_queue = [(dll, binary) for dll in library_dlls] + binary_paths = {binary: binary_path} + binary_dependencies = {} + + while binary_queue != []: + library, parent = binary_queue.pop(0) + if library in WINDOWS_WHITELIST or library.startswith('api-ms-win'): + print('Omitting {0}'.format(library)) + continue + + library_path = find_program(library) + if library_path is None: + print('{0} not found'.format(library)) + continue + + if osp.basename(osp.dirname(library_path)) == 'system32': + continue + + print('{0}: {1}'.format(library, library_path)) + parent_dependencies = binary_dependencies.get(parent, []) + parent_dependencies.append(library) + binary_dependencies[parent] = parent_dependencies + + if library in binary_paths: + continue + + binary_paths[library] = library_path + downstream_dlls = find_dll_dependencies(dumpbin, library_path) + binary_queue += [(n, library) for n in downstream_dlls] + + print('Copying dependencies to wheel directory') + package_dir = osp.join(output_dir, 'torchvision') + for library in binary_paths: + if library != binary: + library_path = binary_paths[library] + new_library_path = osp.join(package_dir, library) + print('{0} -> {1}'.format(library, new_library_path)) + shutil.copyfile(library_path, new_library_path) + + +def compress_wheel(output_dir, wheel, wheel_dir, wheel_name): + """Create RECORD file and compress wheel distribution.""" + print('Update RECORD file in wheel') + dist_info = glob.glob(osp.join(output_dir, '*.dist-info'))[0] + record_file = osp.join(dist_info, 'RECORD') + + with open(record_file, 'w') as f: + for root, _, files in os.walk(output_dir): + for this_file in files: + full_file = osp.join(root, this_file) + rel_file = osp.relpath(full_file, output_dir) + if full_file == record_file: + f.write('{0},,\n'.format(rel_file)) + else: + digest, size = rehash(full_file) + f.write('{0},{1},{2}\n'.format(rel_file, digest, size)) + + print('Compressing wheel') + base_wheel_name = osp.join(wheel_dir, wheel_name) + shutil.make_archive(base_wheel_name, 'zip', output_dir) + os.remove(wheel) + shutil.move('{0}.zip'.format(base_wheel_name), wheel) + shutil.rmtree(output_dir) def patch_linux(): @@ -290,27 +363,7 @@ def patch_linux(): relocate_elf_library( patchelf, output_dir, output_library, binary) - print('Update RECORD file in wheel') - dist_info = glob.glob(osp.join(output_dir, '*.dist-info'))[0] - record_file = osp.join(dist_info, 'RECORD') - - with open(record_file, 'w') as f: - for root, _, files in os.walk(output_dir): - for this_file in files: - full_file = osp.join(root, this_file) - rel_file = osp.relpath(full_file, output_dir) - if full_file == record_file: - f.write('{0},,\n'.format(rel_file)) - else: - digest, size = rehash(full_file) - f.write('{0},{1},{2}\n'.format(rel_file, digest, size)) - - print('Compressing wheel') - base_wheel_name = osp.join(wheel_dir, wheel_name) - shutil.make_archive(base_wheel_name, 'zip', output_dir) - os.remove(wheel) - shutil.move('{0}.zip'.format(base_wheel_name), wheel) - shutil.rmtree(output_dir) + compress_wheel(output_dir, wheel, wheel_dir, wheel_name) def patch_win(): @@ -336,7 +389,7 @@ def patch_win(): print('Unzipping wheel...') wheel_file = osp.basename(wheel) - # wheel_dir = osp.dirname(wheel) + wheel_dir = osp.dirname(wheel) print('{0}'.format(wheel_file)) wheel_name, _ = osp.splitext(wheel_file) unzip_file(wheel, output_dir) @@ -348,6 +401,8 @@ def patch_win(): relocate_dll_library( dumpbin, output_dir, output_library, binary) + compress_wheel(output_dir, wheel, wheel_dir, wheel_name) + if __name__ == '__main__': if sys.platform == 'linux': From c7f0f1277efd8c2fe43bc94fc609734725489f6a Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Wed, 14 Oct 2020 12:05:48 +0200 Subject: [PATCH 23/24] Disable FFmpeg on Windows for now --- .circleci/unittest/windows/scripts/environment.yml | 1 - packaging/pkg_helpers.bash | 4 ++-- packaging/torchvision/meta.yaml | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 59dc62ebfd5..9f4348ebb26 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -9,7 +9,6 @@ dependencies: - pip - libpng - jpeg - - ffmpeg=4.2 - ca-certificates - pip: - future diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index 19f9ebfac06..1e9dd90a44d 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -360,11 +360,11 @@ setup_junit_results_folder() { download_copy_ffmpeg() { if [[ "$OSTYPE" == "msys" ]]; then - conda install -yq ffmpeg=4.2 -c pytorch + # conda install -yq ffmpeg=4.2 -c pytorch # curl -L -q https://anaconda.org/pytorch/ffmpeg/4.3/download/win-64/ffmpeg-4.3-ha925a31_0.tar.bz2 --output ffmpeg-4.3-ha925a31_0.tar.bz2 # bzip2 --decompress --stdout ffmpeg-4.3-ha925a31_0.tar.bz2 | tar -x --file=- # cp Library/bin/*.dll ../torchvision - # echo "FFmpeg is disabled currently on Windows" + echo "FFmpeg is disabled currently on Windows" else if [[ "$(uname)" == Darwin ]]; then conda install -yq ffmpeg=4.2 -c pytorch diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index d39251d138c..68aced34343 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -10,7 +10,7 @@ requirements: - {{ compiler('c') }} # [win] - libpng - jpeg - - ffmpeg =4.2 + - ffmpeg =4.2 # [win] host: - python @@ -22,7 +22,7 @@ requirements: run: - python - libpng - - ffmpeg =4.2 + - ffmpeg =4.2 # [win] - jpeg - pillow >=4.1.1 - numpy >=1.11 From 76545347c03c850466559e38e42b4a42beb62087 Mon Sep 17 00:00:00 2001 From: Francisco Massa Date: Wed, 14 Oct 2020 12:06:34 +0200 Subject: [PATCH 24/24] Bugfix --- packaging/torchvision/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index 68aced34343..fadd9b47f72 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -10,7 +10,7 @@ requirements: - {{ compiler('c') }} # [win] - libpng - jpeg - - ffmpeg =4.2 # [win] + - ffmpeg =4.2 # [not win] host: - python @@ -22,7 +22,7 @@ requirements: run: - python - libpng - - ffmpeg =4.2 # [win] + - ffmpeg =4.2 # [not win] - jpeg - pillow >=4.1.1 - numpy >=1.11