From 51010baee732da8fbfe4ea1999c4afeb73939261 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 18 Sep 2024 06:15:44 -0700 Subject: [PATCH 01/11] Build Mac wheels in CI --- .github/workflows/linux_wheel.yaml | 2 +- .github/workflows/macos_wheel.yaml | 76 +++++++++++++++++++++++++----- packaging/check_glibcxx.py | 5 +- packaging/post_build_script.sh | 48 ++++++++++++------- packaging/wheel_env.sh | 1 + 5 files changed, 99 insertions(+), 33 deletions(-) create mode 100644 packaging/wheel_env.sh diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 81e8df2a3..357776d5f 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -38,7 +38,7 @@ jobs: needs: generate-matrix strategy: fail-fast: false - name: Build and Upload wheel + name: Build and Upload Linux wheel uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main with: repository: pytorch/torchcodec diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index 9060b18c7..366cdb4de 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -1,4 +1,4 @@ -name: Build and test MacOS +name: Build and test MacOS wheels on: pull_request: @@ -24,6 +24,35 @@ defaults: shell: bash -l -eo pipefail {0} jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-xpu: disable + with-rocm: disable + with-cuda: disable + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload Mac wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/fake_smoke_test.py + runner-type: macos-m1-stable + package-name: torchcodec + trigger-event: ${{ github.event_name }} + env-var-script: packaging/wheel_env.sh + install-and-test: runs-on: macos-m1-stable strategy: @@ -32,7 +61,12 @@ jobs: python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] if: ${{ always() }} + needs: build steps: + - uses: actions/download-artifact@v3 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ + path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v3 with: @@ -45,19 +79,14 @@ jobs: - name: Install PyTorch run: | python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv + - name: Check out repo uses: actions/checkout@v3 - - name: Install compile from source dependencies - run: | - conda install cmake pkg-config -c conda-forge - - name: Install test dependencies - run: | - python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow - - name: Install torchcodec from source, building against non-GPL FFmpeg - run: | - BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 pip install -e ".[dev]" --no-build-isolation - name: Install ffmpeg, post build run: | # Ideally we would have checked for that before installing the wheel, @@ -69,9 +98,30 @@ jobs: conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge ffmpeg -version + + - name: Install test dependencies + run: | + python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow + + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls - name: Smoke test run: | python test/decoders/manual_smoke_test.py - name: Run Python tests run: | - pytest test -vvv + pytest test diff --git a/packaging/check_glibcxx.py b/packaging/check_glibcxx.py index 0bf435ff6..93141138d 100644 --- a/packaging/check_glibcxx.py +++ b/packaging/check_glibcxx.py @@ -38,14 +38,15 @@ MAX_ALLOWED = (3, 4, 19) +libraries = sys.argv[1].split("\n") all_symbols = set() -for line in sys.argv[1].split("\n"): +for line in libraries: # We search for GLIBCXX_major.minor.micro if match := re.search(r"GLIBCXX_\d+\.\d+\.\d+", line): all_symbols.add(match.group(0)) if not all_symbols: - raise ValueError("No GLIBCXX symbols found. Something is wrong.") + raise ValueError(f"No GLIBCXX symbols found in {libraries}. Something is wrong.") all_versions = (symbol.split("_")[1].split(".") for symbol in all_symbols) all_versions = (tuple(int(v) for v in version) for version in all_versions) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 84686d84f..3dbe4d9d9 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,18 +1,30 @@ #!/bin/bash +set -eux + source packaging/helpers.sh wheel_path=$(pwd)/$(find dist -type f -name "*.whl") echo "Wheel content:" unzip -l $wheel_path +unamestr=$(uname) +if [[ "$unamestr" == 'Linux' ]]; then + ext="so" +elif [[ "$unamestr" == 'Darwin' ]]; then + ext="dylib" +else + echo "Unknown operating system: $unamestr" + exit 1 +fi + for ffmpeg_major_version in 4 5 6 7; do - assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so + assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.${ext} done -assert_not_in_wheel $wheel_path libtorchcodec.so +assert_not_in_wheel $wheel_path libtorchcodec.${ext} -for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do - assert_not_in_wheel $wheel_path $ffmpeg_so +for ffmpeg_ext in libavcodec.${ext} libavfilter.${ext} libavformat.${ext} libavutil.${ext} libavdevice.${ext} ; do + assert_not_in_wheel $wheel_path $ffmpeg_ext done assert_not_in_wheel $wheel_path "^test" @@ -20,19 +32,21 @@ assert_not_in_wheel $wheel_path "^doc" assert_not_in_wheel $wheel_path "^benchmarks" assert_not_in_wheel $wheel_path "^packaging" -# See invoked python script below for details about this check. -extracted_wheel_dir=$(mktemp -d) -unzip -q $wheel_path -d $extracted_wheel_dir -symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) -python packaging/check_glibcxx.py "$symbols_matches" - -echo "ls dist" -ls dist - -old="linux_x86_64" -new="manylinux_2_17_x86_64.manylinux2014_x86_64" -echo "Replacing ${old} with ${new} in wheel name" -mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/") +if [[ "$unamestr" == 'Linux' ]]; then + # See invoked python script below for details about this check. + extracted_wheel_dir=$(mktemp -d) + unzip -q $wheel_path -d $extracted_wheel_dir + symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) + python packaging/check_glibcxx.py "$symbols_matches" + + echo "ls dist" + ls dist + + old="linux_x86_64" + new="manylinux_2_17_x86_64.manylinux2014_x86_64" + echo "Replacing ${old} with ${new} in wheel name" + mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/") +fi echo "ls dist" ls dist diff --git a/packaging/wheel_env.sh b/packaging/wheel_env.sh new file mode 100644 index 000000000..0ff9e074e --- /dev/null +++ b/packaging/wheel_env.sh @@ -0,0 +1 @@ +export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 From 03110172cc342fd3a6d7f976f14ea7c2f289a011 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Wed, 9 Oct 2024 07:26:28 -0700 Subject: [PATCH 02/11] Update .github/workflows/macos_wheel.yaml --- .github/workflows/macos_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index ac8d594f8..e7de0aaca 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -25,7 +25,7 @@ defaults: jobs: generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@macbuildwheel with: package-type: wheel os: macos-arm64 From e72bd01df1b9f2081a6d2cc49f1eaa48503b2d72 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Wed, 9 Oct 2024 07:34:04 -0700 Subject: [PATCH 03/11] Update .github/workflows/macos_wheel.yaml --- .github/workflows/macos_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index e7de0aaca..d8f22447a 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -39,7 +39,7 @@ jobs: strategy: fail-fast: false name: Build and Upload Mac wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@macbuildwheel with: repository: pytorch/torchcodec ref: "" From 275116e58566f51cd883e97723fd1d147497f170 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 9 Oct 2024 10:13:18 -0700 Subject: [PATCH 04/11] Update macos_wheel.yaml to run build command with ${CONDA_RUN} --- .github/workflows/macos_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index d8f22447a..a0feca41d 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -52,7 +52,7 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python3 -m build --wheel -vvv --no-isolation" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ${CONDA_RUN} python3 -m build --wheel -vvv --no-isolation" install-and-test: runs-on: macos-m1-stable From 176a977f6d95ce5100ef00278e739862948808a3 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 9 Oct 2024 11:08:29 -0700 Subject: [PATCH 05/11] Update macos_wheel.yaml to use macbuildwheel branch --- .github/workflows/macos_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index a0feca41d..13ba4ca25 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -44,7 +44,7 @@ jobs: repository: pytorch/torchcodec ref: "" test-infra-repository: pytorch/test-infra - test-infra-ref: main + test-infra-ref: macbuildwheel build-matrix: ${{ needs.generate-matrix.outputs.matrix }} post-script: packaging/post_build_script.sh smoke-test-script: packaging/fake_smoke_test.py From b0de6bddb8efbe6d3d47c9f349549acded71816d Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Wed, 9 Oct 2024 15:46:15 -0700 Subject: [PATCH 06/11] Update macos_wheel.yaml --- .github/workflows/macos_wheel.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index 13ba4ca25..b441afb95 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -75,6 +75,8 @@ jobs: miniconda-version: "latest" activate-environment: test python-version: ${{ matrix.python-version }} + - name: Activate conda + run: conda activate test - name: Update pip run: python3 -m pip install --upgrade pip - name: Install PyTorch From 7d81193538616c7b51fa614f227acb16757494d2 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Wed, 9 Oct 2024 17:21:26 -0700 Subject: [PATCH 07/11] Use test-infra action for miniconda setup --- .github/workflows/macos_wheel.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index b441afb95..c1e5492c2 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -68,15 +68,10 @@ jobs: with: name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ path: pytorch/torchcodec/dist/ - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v3 + - name: Setup miniconda + uses: pytorch/test-infra/.github/actions/setup-miniconda@macbuildwheel with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test python-version: ${{ matrix.python-version }} - - name: Activate conda - run: conda activate test - name: Update pip run: python3 -m pip install --upgrade pip - name: Install PyTorch From 4a8eb56fb50c5c85fec30cc004e5b39e91da560b Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 9 Oct 2024 19:03:17 -0700 Subject: [PATCH 08/11] Set Python3 root. --- .github/workflows/macos_wheel.yaml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index c1e5492c2..559f4abaf 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -73,12 +73,22 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Update pip - run: python3 -m pip install --upgrade pip + run: | + export PATH=${CONDA_ENV}/bin:${CONDA_ENV}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH + export Python3_ROOT_DIR=${CONDA_ENV} + + python3 -m pip install --upgrade pip - name: Install PyTorch run: | + export PATH=${CONDA_ENV}/bin:${CONDA_ENV}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH + export Python3_ROOT_DIR=${CONDA_ENV} + python3 -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - name: Install torchcodec from the wheel run: | + export PATH=${CONDA_ENV}/bin:${CONDA_ENV}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH + export Python3_ROOT_DIR=${CONDA_ENV} + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` echo Installing $wheel_path python3 -m pip install $wheel_path -vvv @@ -92,13 +102,18 @@ jobs: # want to checkout the repo before installing the wheel to avoid any # side-effect. It's OK. source packaging/helpers.sh - assert_ffmpeg_not_installed + + # below was failing, but when I just try to call "ffmpeg -version" that also fails? + #assert_ffmpeg_not_installed conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge ffmpeg -version - name: Install test dependencies run: | + export PATH=${CONDA_ENV}/bin:${CONDA_ENV}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH + export Python3_ROOT_DIR=${CONDA_ENV} + python3 -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu # Ideally we would find a way to get those dependencies from pyproject.toml python3 -m pip install numpy pytest pillow @@ -119,7 +134,13 @@ jobs: ls - name: Smoke test run: | + export PATH=${CONDA_ENV}/bin:${CONDA_ENV}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH + export Python3_ROOT_DIR=${CONDA_ENV} + python3 test/decoders/manual_smoke_test.py - name: Run Python tests run: | + export PATH=${CONDA_ENV}/bin:${CONDA_ENV}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH + export Python3_ROOT_DIR=${CONDA_ENV} + pytest test From 0916aa45bf83b6b240e2a4f590db44229cd51791 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 9 Oct 2024 19:03:17 -0700 Subject: [PATCH 09/11] Set Python3 root. --- .github/workflows/macos_wheel.yaml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index c1e5492c2..de7f8e596 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -54,6 +54,16 @@ jobs: build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ${CONDA_RUN} python3 -m build --wheel -vvv --no-isolation" + validate-binaries: + uses: pytorch/test-infra/.github/workflows/validate-domain-library.yml@main + with: + package_type: "wheel" + os: "macos-arm64" + channel: "nightly" + repository: "pytorch/torchcodec" + smoke_test: "source ./packaging/validate_wheel.sh" + install_torch: true + install-and-test: runs-on: macos-m1-stable strategy: @@ -73,7 +83,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Update pip - run: python3 -m pip install --upgrade pip + run: | + python3 -m pip install --upgrade pip - name: Install PyTorch run: | python3 -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu @@ -92,7 +103,9 @@ jobs: # want to checkout the repo before installing the wheel to avoid any # side-effect. It's OK. source packaging/helpers.sh - assert_ffmpeg_not_installed + + # below was failing, but when I just try to call "ffmpeg -version" that also fails? + #assert_ffmpeg_not_installed conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge ffmpeg -version From 2733862c9a8aa2c590917945aac223422bfd2567 Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Wed, 16 Oct 2024 10:56:47 -0700 Subject: [PATCH 10/11] Add validation framework job. --- packaging/validate_wheel.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 packaging/validate_wheel.sh diff --git a/packaging/validate_wheel.sh b/packaging/validate_wheel.sh new file mode 100644 index 000000000..578acf8e6 --- /dev/null +++ b/packaging/validate_wheel.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -eux + +conda install ffmpeg -c conda-forge +pip install ${PYTORCH_PIP_PREFIX} torchcodec --extra-index-url ${PYTORCH_PIP_DOWNLOAD_URL} +pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + +python3 test/decoders/manual_smoke_test.py +pytest test -vvv From 571ed17eaa0086aa7f82ebc8804ada505a6fdfcb Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Mon, 21 Oct 2024 11:00:06 -0700 Subject: [PATCH 11/11] Change logging --- .github/workflows/linux_wheel.yaml | 8 +- .github/workflows/macos_wheel.yaml | 132 ++++-------------- .../decoders/_core/VideoDecoder.cpp | 24 ---- .../decoders/_core/VideoDecoderOps.cpp | 9 -- 4 files changed, 29 insertions(+), 144 deletions(-) diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 872aeeb7f..38f257336 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -1,4 +1,4 @@ -name: Build and test Linux wheels +name: Build and test Linux wheel on: pull_request: @@ -24,6 +24,7 @@ defaults: shell: bash -l -eo pipefail {0} jobs: + generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: @@ -34,6 +35,8 @@ jobs: with-xpu: disable with-rocm: disable with-cuda: disable + build-python-only: "disable" + build: needs: generate-matrix strategy: @@ -60,7 +63,6 @@ jobs: matrix: python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - if: ${{ always() }} needs: build steps: - uses: actions/download-artifact@v3 @@ -121,7 +123,7 @@ jobs: ls - name: Smoke test run: | - python test/decoders/manual_smoke_test.py + python -X faulthandler test/decoders/manual_smoke_test.py - name: Run Python tests run: | pytest test -vvv diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index 3a894fa74..48a77bcf5 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -1,4 +1,4 @@ -name: Build and test MacOS wheels +name: Build and test MacOS wheel on: pull_request: @@ -24,6 +24,7 @@ defaults: shell: bash -l -eo pipefail {0} jobs: + generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@macbuildwheel with: @@ -34,6 +35,8 @@ jobs: with-xpu: disable with-rocm: disable with-cuda: disable + build-python-only: "disable" + build: needs: generate-matrix strategy: @@ -61,134 +64,47 @@ jobs: matrix: python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - if: ${{ always() }} needs: build steps: - - name: Setup miniconda - uses: pytorch/test-infra/.github/actions/setup-miniconda@macbuildwheel - with: - python-version: ${{ matrix.python-version }} - - name: Check out torchcodec repo - uses: actions/checkout@v3 - - name: Check out tet-infra repo - uses: actions/checkout@v3 - with: - repository: pytorch/test-infra - ref: macbuildwheel - path: test-infra - name: Download wheel uses: actions/download-artifact@v3 with: name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ path: pytorch/torchcodec/dist/ - -## start copying from test-infra/.github/actions/setup-binary-builds/action.yaml - - name: Clean conda environment - shell: bash -l {0} - run: | - set -euxo pipefail - conda info | grep -i 'base environment' - conda clean --all --quiet --yes - - name: Reset channel priority - shell: bash -l {0} - run: | - set -euxo pipefail - conda config --set channel_priority false - - name: Discover dir structure - shell: bash -l {0} - run: | - echo "pwd" - pwd - echo "ls -lh" - ls -lh - echo "ls -lh pytorch/torchcodec" - ls -lh pytorch/torchcodec - echo "ls -lh pytorch/torchcodec/dist" - ls -lh pytorch/torchcodec/dist - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo "unzip $wheel_path -d unzipped_wheel" - unzip $wheel_path -d unzipped_wheel - echo "ls -lh unzipped_wheel" - ls -lh unzipped_wheel - - name: Generate file from pytorch_pkg_helpers - # below does not seem necessary in our context - #working-directory: "pytorch/torchcodec" - shell: bash -l {0} - run: | - set -euxo pipefail - CONDA_ENV="${RUNNER_TEMP}/pytorch_pkg_helpers_${GITHUB_RUN_ID}" - conda create \ - --yes --quiet \ - --prefix "${CONDA_ENV}" \ - "python=3.9" - CONDA_ENV="${CONDA_ENV}" - CONDA_RUN="conda run -p ${CONDA_ENV}" - ${CONDA_RUN} python -m pip install ${GITHUB_WORKSPACE}/test-infra/tools/pkg-helpers - BUILD_ENV_FILE="${RUNNER_TEMP}/build_env_${GITHUB_RUN_ID}" - ${CONDA_RUN} python -m pytorch_pkg_helpers > "${BUILD_ENV_FILE}" - cat "${BUILD_ENV_FILE}" - echo "BUILD_ENV_FILE=${BUILD_ENV_FILE}" >> "${GITHUB_ENV}" - - name: Setup conda environment for build - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - set -euxo pipefail - CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}" - if [[ "${PACKAGE_TYPE:-}" == "conda" ]]; then - # For conda package host python version is irrelevant - export PYTHON_VERSION=3.9 - export CONDA_BUILD_EXTRA="conda=24.4.0 conda-build=24.3.0 python-libarchive-c=2.9" - else - # For wheel builds we don't need neither conda nor conda-build - export CONDA_BUILD_EXTRA="" - fi - - conda create \ - --yes --quiet \ - --prefix "${CONDA_ENV}" \ - "python=${PYTHON_VERSION}" \ - cmake=3.26 \ - ninja=1.10 \ - pkg-config=0.29 \ - ${CONDA_BUILD_EXTRA} \ - wheel=0.37 - - echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}" - echo "CONDA_RUN=conda run -p ${CONDA_ENV}" >> "${GITHUB_ENV}" -## end copying - - name: Run otool - shell: bash -l {0} - run: | - echo "otool -L unzipped_wheel/torchcodec/libtorchcodec*" - otool -L unzipped_wheel/torchcodec/libtorchcodec* - + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} - name: Update pip run: python -m pip install --upgrade pip - - name: Install ffmpeg - run: | - ${CONDA_RUN} conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ${CONDA_RUN} ffmpeg -version - name: Install PyTorch run: | - ${CONDA_RUN} python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - name: Install torchcodec from the wheel run: | wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv + python -m pip install $wheel_path -vvv + - name: Check out torchcodec repo + uses: actions/checkout@v3 + - name: Install ffmpeg + run: | + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version - name: Install test dependencies run: | - ${CONDA_RUN} python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - ${CONDA_RUN} python -m pip install numpy pytest pillow + python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + python -m pip install numpy pytest pillow - name: Delete the src/ folder just for fun run: | rm -r src/ - ls + ls -lh - name: Smoke test run: | - ${CONDA_RUN} python -X faulthandler test/decoders/manual_smoke_test.py - ls -lh + python -X faulthandler test/decoders/manual_smoke_test.py - name: Run Python tests run: | - ${CONDA_RUN} pytest test + pytest test -vvv diff --git a/src/torchcodec/decoders/_core/VideoDecoder.cpp b/src/torchcodec/decoders/_core/VideoDecoder.cpp index 4f0cf991a..818aeaf79 100644 --- a/src/torchcodec/decoders/_core/VideoDecoder.cpp +++ b/src/torchcodec/decoders/_core/VideoDecoder.cpp @@ -657,7 +657,6 @@ bool VideoDecoder::canWeAvoidSeekingForStream( // AVFormatContext if it is needed. We can skip seeking in certain cases. See // the comment of canWeAvoidSeeking() for details. void VideoDecoder::maybeSeekToBeforeDesiredPts() { - std::cerr << "maybeSeekToBeforeDesiredPts" << std::endl; if (activeStreamIndices_.size() == 0) { return; } @@ -677,9 +676,6 @@ void VideoDecoder::maybeSeekToBeforeDesiredPts() { int64_t desiredPtsForStream = *maybeDesiredPts_ * streamInfo.timeBase.den; if (!canWeAvoidSeekingForStream( streamInfo, streamInfo.currentPts, desiredPtsForStream)) { - VLOG(5) << "Seeking is needed for streamIndex=" << streamIndex - << " desiredPts=" << desiredPtsForStream - << " currentPts=" << streamInfo.currentPts; mustSeek = true; break; } @@ -724,20 +720,14 @@ void VideoDecoder::maybeSeekToBeforeDesiredPts() { VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( std::function filterFunction) { - std::cerr << "getDecodedOutputWithFilter" << std::endl; auto start = std::chrono::high_resolution_clock::now(); if (activeStreamIndices_.size() == 0) { throw std::runtime_error("No active streams configured."); } - VLOG(9) << "Starting getDecodedOutputWithFilter()"; resetDecodeStats(); if (maybeDesiredPts_.has_value()) { - std::cerr << "\thas value" << std::endl; - VLOG(9) << "maybeDesiredPts_=" << *maybeDesiredPts_; - std::cerr << "\tlogged pts" << std::endl; maybeSeekToBeforeDesiredPts(); maybeDesiredPts_ = std::nullopt; - VLOG(9) << "seeking done"; } auto seekDone = std::chrono::high_resolution_clock::now(); // Need to get the next frame or error from PopFrame. @@ -752,13 +742,9 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( StreamInfo& streamInfo = streams_[streamIndex]; ffmpegStatus = avcodec_receive_frame(streamInfo.codecContext.get(), frame.get()); - VLOG(9) << "received frame" << " status=" << ffmpegStatus - << " streamIndex=" << streamInfo.stream->index; bool gotNonRetriableError = ffmpegStatus != AVSUCCESS && ffmpegStatus != AVERROR(EAGAIN); if (gotNonRetriableError) { - VLOG(9) << "Got non-retriable error from decoder: " - << getFFMPEGErrorStringFromErrorCode(ffmpegStatus); gotPermanentErrorOnAnyActiveStream = true; break; } @@ -788,7 +774,6 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( UniqueAVPacket packet(av_packet_alloc()); ffmpegStatus = av_read_frame(formatContext_.get(), packet.get()); decodeStats_.numPacketsRead++; - VLOG(9) << "av_read_frame returned status: " << ffmpegStatus; if (ffmpegStatus == AVERROR_EOF) { // End of file reached. We must drain all codecs by sending a nullptr // packet. @@ -811,8 +796,6 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( "Could not read frame from input file: " + getFFMPEGErrorStringFromErrorCode(ffmpegStatus)); } - VLOG(9) << "Got packet: stream_index=" << packet->stream_index - << " pts=" << packet->pts << " size=" << packet->size; if (activeStreamIndices_.count(packet->stream_index) == 0) { // This packet is not for any of the active streams. continue; @@ -850,10 +833,6 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( std::chrono::duration_cast(seekDone - start); auto seekToDecodeDone = std::chrono::duration_cast( decodeDone - seekDone); - VLOG(3) << "Got frame: stream_index=" << activeStream.stream->index - << " pts=" << frame->pts << " stats=" << decodeStats_ - << " startToSeekDone=" << startToSeekDone.count() << "ms" - << " seekToDecodeDone=" << seekToDecodeDone.count() << "ms"; RawDecodedOutput rawOutput; rawOutput.streamIndex = frameStreamIndex; rawOutput.frame = std::move(frame); @@ -1021,7 +1000,6 @@ VideoDecoder::DecodedOutput VideoDecoder::getFrameAtIndex( int streamIndex, int64_t frameIndex, std::optional preAllocatedOutputTensor) { - std::cerr << "getFrameAtIndex" << std::endl; validateUserProvidedStreamIndex(streamIndex); validateScannedAllStreams("getFrameAtIndex"); @@ -1196,7 +1174,6 @@ VideoDecoder::getFramesDisplayedByTimestampInRange( } VideoDecoder::RawDecodedOutput VideoDecoder::getNextRawDecodedOutputNoDemux() { - std::cerr << "getNextRawDecodedOutputNoDemux" << std::endl; auto rawOutput = getDecodedOutputWithFilter([this](int frameStreamIndex, AVFrame* frame) { StreamInfo& activeStream = streams_[frameStreamIndex]; @@ -1207,7 +1184,6 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getNextRawDecodedOutputNoDemux() { VideoDecoder::DecodedOutput VideoDecoder::getNextDecodedOutputNoDemux( std::optional preAllocatedOutputTensor) { - std::cerr << "getNextDecodedOutputNoDemux" << std::endl; auto rawOutput = getNextRawDecodedOutputNoDemux(); return convertAVFrameToDecodedOutput(rawOutput, preAllocatedOutputTensor); } diff --git a/src/torchcodec/decoders/_core/VideoDecoderOps.cpp b/src/torchcodec/decoders/_core/VideoDecoderOps.cpp index edc0aa9da..1bf090362 100644 --- a/src/torchcodec/decoders/_core/VideoDecoderOps.cpp +++ b/src/torchcodec/decoders/_core/VideoDecoderOps.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include "c10/core/SymIntArrayRef.h" #include "c10/util/Exception.h" #include "src/torchcodec/decoders/_core/VideoDecoder.h" @@ -77,7 +76,6 @@ VideoDecoder* unwrapTensorToGetDecoder(at::Tensor& tensor) { } OpsDecodedOutput makeOpsDecodedOutput(VideoDecoder::DecodedOutput& frame) { - std::cerr << "makeOpsDecodedOutput" << std::endl; return std::make_tuple( frame.frame, torch::tensor(frame.ptsSeconds, torch::dtype(torch::kFloat64)), @@ -215,7 +213,6 @@ OpsDecodedOutput get_frame_at_index( at::Tensor& decoder, int64_t stream_index, int64_t frame_index) { - std::cerr << "get_frame_at_index" << std::endl; auto videoDecoder = unwrapTensorToGetDecoder(decoder); auto result = videoDecoder->getFrameAtIndex(stream_index, frame_index); return makeOpsDecodedOutput(result); @@ -283,12 +280,6 @@ bool _test_frame_pts_equality( int64_t frame_index, double pts_seconds_to_test) { auto videoDecoder = unwrapTensorToGetDecoder(decoder); - LOG(INFO) << "pts_seconds_to_test: " << std::setprecision(15) - << pts_seconds_to_test << std::endl; - LOG(INFO) << "frame pts : " << std::setprecision(15) - << videoDecoder->getPtsSecondsForFrame(stream_index, frame_index) - << std::endl - << std::endl; return pts_seconds_to_test == videoDecoder->getPtsSecondsForFrame(stream_index, frame_index); }