From 51010baee732da8fbfe4ea1999c4afeb73939261 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 18 Sep 2024 06:15:44 -0700 Subject: [PATCH] Build Mac wheels in CI --- .github/workflows/linux_wheel.yaml | 2 +- .github/workflows/macos_wheel.yaml | 76 +++++++++++++++++++++++++----- packaging/check_glibcxx.py | 5 +- packaging/post_build_script.sh | 48 ++++++++++++------- packaging/wheel_env.sh | 1 + 5 files changed, 99 insertions(+), 33 deletions(-) create mode 100644 packaging/wheel_env.sh diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 81e8df2a3..357776d5f 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -38,7 +38,7 @@ jobs: needs: generate-matrix strategy: fail-fast: false - name: Build and Upload wheel + name: Build and Upload Linux wheel uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main with: repository: pytorch/torchcodec diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index 9060b18c7..366cdb4de 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -1,4 +1,4 @@ -name: Build and test MacOS +name: Build and test MacOS wheels on: pull_request: @@ -24,6 +24,35 @@ defaults: shell: bash -l -eo pipefail {0} jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-xpu: disable + with-rocm: disable + with-cuda: disable + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload Mac wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/fake_smoke_test.py + runner-type: macos-m1-stable + package-name: torchcodec + trigger-event: ${{ github.event_name }} + env-var-script: packaging/wheel_env.sh + install-and-test: runs-on: macos-m1-stable strategy: @@ -32,7 +61,12 @@ jobs: python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] if: ${{ always() }} + needs: build steps: + - uses: actions/download-artifact@v3 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ + path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v3 with: @@ -45,19 +79,14 @@ jobs: - name: Install PyTorch run: | python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv + - name: Check out repo uses: actions/checkout@v3 - - name: Install compile from source dependencies - run: | - conda install cmake pkg-config -c conda-forge - - name: Install test dependencies - run: | - python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow - - name: Install torchcodec from source, building against non-GPL FFmpeg - run: | - BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 pip install -e ".[dev]" --no-build-isolation - name: Install ffmpeg, post build run: | # Ideally we would have checked for that before installing the wheel, @@ -69,9 +98,30 @@ jobs: conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge ffmpeg -version + + - name: Install test dependencies + run: | + python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow + + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls - name: Smoke test run: | python test/decoders/manual_smoke_test.py - name: Run Python tests run: | - pytest test -vvv + pytest test diff --git a/packaging/check_glibcxx.py b/packaging/check_glibcxx.py index 0bf435ff6..93141138d 100644 --- a/packaging/check_glibcxx.py +++ b/packaging/check_glibcxx.py @@ -38,14 +38,15 @@ MAX_ALLOWED = (3, 4, 19) +libraries = sys.argv[1].split("\n") all_symbols = set() -for line in sys.argv[1].split("\n"): +for line in libraries: # We search for GLIBCXX_major.minor.micro if match := re.search(r"GLIBCXX_\d+\.\d+\.\d+", line): all_symbols.add(match.group(0)) if not all_symbols: - raise ValueError("No GLIBCXX symbols found. Something is wrong.") + raise ValueError(f"No GLIBCXX symbols found in {libraries}. Something is wrong.") all_versions = (symbol.split("_")[1].split(".") for symbol in all_symbols) all_versions = (tuple(int(v) for v in version) for version in all_versions) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 84686d84f..3dbe4d9d9 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,18 +1,30 @@ #!/bin/bash +set -eux + source packaging/helpers.sh wheel_path=$(pwd)/$(find dist -type f -name "*.whl") echo "Wheel content:" unzip -l $wheel_path +unamestr=$(uname) +if [[ "$unamestr" == 'Linux' ]]; then + ext="so" +elif [[ "$unamestr" == 'Darwin' ]]; then + ext="dylib" +else + echo "Unknown operating system: $unamestr" + exit 1 +fi + for ffmpeg_major_version in 4 5 6 7; do - assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so + assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.${ext} done -assert_not_in_wheel $wheel_path libtorchcodec.so +assert_not_in_wheel $wheel_path libtorchcodec.${ext} -for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do - assert_not_in_wheel $wheel_path $ffmpeg_so +for ffmpeg_ext in libavcodec.${ext} libavfilter.${ext} libavformat.${ext} libavutil.${ext} libavdevice.${ext} ; do + assert_not_in_wheel $wheel_path $ffmpeg_ext done assert_not_in_wheel $wheel_path "^test" @@ -20,19 +32,21 @@ assert_not_in_wheel $wheel_path "^doc" assert_not_in_wheel $wheel_path "^benchmarks" assert_not_in_wheel $wheel_path "^packaging" -# See invoked python script below for details about this check. -extracted_wheel_dir=$(mktemp -d) -unzip -q $wheel_path -d $extracted_wheel_dir -symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) -python packaging/check_glibcxx.py "$symbols_matches" - -echo "ls dist" -ls dist - -old="linux_x86_64" -new="manylinux_2_17_x86_64.manylinux2014_x86_64" -echo "Replacing ${old} with ${new} in wheel name" -mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/") +if [[ "$unamestr" == 'Linux' ]]; then + # See invoked python script below for details about this check. + extracted_wheel_dir=$(mktemp -d) + unzip -q $wheel_path -d $extracted_wheel_dir + symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) + python packaging/check_glibcxx.py "$symbols_matches" + + echo "ls dist" + ls dist + + old="linux_x86_64" + new="manylinux_2_17_x86_64.manylinux2014_x86_64" + echo "Replacing ${old} with ${new} in wheel name" + mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/") +fi echo "ls dist" ls dist diff --git a/packaging/wheel_env.sh b/packaging/wheel_env.sh new file mode 100644 index 000000000..0ff9e074e --- /dev/null +++ b/packaging/wheel_env.sh @@ -0,0 +1 @@ +export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1