diff --git a/.github/workflows/nightly-conda.yml b/.github/workflows/nightly-conda.yml index 287cce13c..373de3131 100644 --- a/.github/workflows/nightly-conda.yml +++ b/.github/workflows/nightly-conda.yml @@ -136,6 +136,29 @@ jobs: scripts/build_mac_dep.sh ranges_v3 fmt double_conversion folly re2 ./packaging/build_conda.sh + - name: Fix Velox Dylib Paths + shell: bash -l {0} + env: + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + cd conda-bld/osx-64 + pkg_name=`ls ./torcharrow-* | sed -n -e 's/.*\(torcharrow.*\).tar.bz2/\1/p'` + mkdir ./${pkg_name} + tar -xf ./${pkg_name}.tar.bz2 -C ./${pkg_name} + rm ./${pkg_name}.tar.bz2 + cd ./${pkg_name} + + source ${GITHUB_WORKSPACE}/packaging/fix_conda_dylib_paths.sh + conda_lib_folder=lib/python${PYTHON_VERSION}/site-packages/torcharrow + so_name=`ls ${conda_lib_folder}/_torcharrow.* | sed -n -e 's/.*\(_torcharrow.*\.so\)/\1/p'` + fix_velox_dylib_paths ${conda_lib_folder}/${so_name} + otool -L ${conda_lib_folder}/${so_name} + + tar -cjf ../${pkg_name}.tar.bz2 * + cd .. + rm -rf ./${pkg_name} + cd ../.. + - name: Conda Upload shell: bash -l {0} env: diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh index 9882efaf2..5d9bf070a 100755 --- a/packaging/build_conda.sh +++ b/packaging/build_conda.sh @@ -19,4 +19,4 @@ setup_env 0.1 setup_conda_pytorch_constraint mkdir -p conda-bld -conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --output-folder conda-bld --python "$PYTHON_VERSION" packaging/torcharrow +conda build $CONDA_CHANNEL_FLAGS -c conda-forge --no-anaconda-upload --output-folder conda-bld --python "$PYTHON_VERSION" packaging/torcharrow diff --git a/packaging/fix_conda_dylib_paths.sh b/packaging/fix_conda_dylib_paths.sh new file mode 100644 index 000000000..f58a85606 --- /dev/null +++ b/packaging/fix_conda_dylib_paths.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +function fix_velox_dylib_paths() { + velox_so=$1 + + # other libs + libs="libgflags libglog libz libssl libcrypto libbz2 liblz4 libzstd libsodium" + + for libx in ${libs} + do + liby=$(otool -L "${velox_so}" | sed -n -e "s/\\(.*${libx}\..*dylib\\).*/\\1/p" | tr -d '[:blank:]') + + install_name_tool -change "${liby}" "@loader_path/../../../${libx}.dylib" "${velox_so}" + done + + libx=libevent + # libevent + liby=$(otool -L "${velox_so}" | sed -n -e "s/\\(.*${libx}-.*dylib\\).*/\\1/p" | tr -d '[:blank:]') + install_name_tool -change "${liby}" "@loader_path/../../../${libx}.dylib" "${velox_so}" + + # boost libs + boost_libs="libboost_context libboost_filesystem libboost_atomic libboost_regex libboost_system libboost_thread" + + for libx in ${boost_libs} + do + liby=$(otool -L "${velox_so}" | sed -n -e "s/\\(.*${libx}.*dylib\\).*/\\1/p" | tr -d '[:blank:]') + + install_name_tool -change "${liby}" "@loader_path/../../../${libx}.dylib" "${velox_so}" + done + + libx=libboost_program + liby=$(otool -L "${velox_so}" | sed -n -e "s/\\(.*${libx}.*dylib\\).*/\\1/p" | tr -d '[:blank:]') + install_name_tool -change "${liby}" "@loader_path/../../../${libx}_options.dylib" "${velox_so}" +} diff --git a/packaging/torcharrow/meta.yaml b/packaging/torcharrow/meta.yaml index 0c03d3247..529b51dda 100644 --- a/packaging/torcharrow/meta.yaml +++ b/packaging/torcharrow/meta.yaml @@ -20,20 +20,22 @@ requirements: - typing - tabulate - pyarrow - - arrow + - arrow-cpp==8.0.0 - cffi - - glog==0.4.0 + - glog==0.6.0 - libsodium - {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} + - libboost + - libevent + - bzip2 + - lz4-c build: string: py{{py}} script_env: - BUILD_VERSION - CPU_TARGET + - PYTHON_VERSION -#GitHub Actions usually provide Mac without AVX2 support, for now just skip the test until Velox can be run without AVX2 -#pytest --no-header -v torcharrow/test test: imports: - torcharrow