From 3b55fb9094aaebc22ab958e5a6fce64538fd36f9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:35:20 +0100 Subject: [PATCH 01/57] Windows makefile --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 233 ++++++++++-------- 1 file changed, 137 insertions(+), 96 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 35d3d06f8..5ecc23864 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -38,44 +38,44 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") ) set( - f4_library_file_names - libavutil.so.56 - libavcodec.so.58 - libavformat.so.58 - libavdevice.so.58 - libavfilter.so.7 - libswscale.so.5 - libswresample.so.3 - ) - set( - f5_library_file_names - libavutil.so.57 - libavcodec.so.59 - libavformat.so.59 - libavdevice.so.59 - libavfilter.so.8 - libswscale.so.6 - libswresample.so.4 - ) - set( - f6_library_file_names - libavutil.so.58 - libavcodec.so.60 - libavformat.so.60 - libavdevice.so.60 - libavfilter.so.9 - libswscale.so.7 - libswresample.so.4 - ) - set( - f7_library_file_names - libavutil.so.59 - libavcodec.so.61 - libavformat.so.61 - libavdevice.so.61 - libavfilter.so.10 - libswscale.so.8 - libswresample.so.5 + f4_library_paths + ${f4_SOURCE_DIR}/lib/libavutil.so.56 + ${f4_SOURCE_DIR}/lib/libavcodec.so.58 + ${f4_SOURCE_DIR}/lib/libavformat.so.58 + ${f4_SOURCE_DIR}/lib/libavdevice.so.58 + ${f4_SOURCE_DIR}/lib/libavfilter.so.7 + ${f4_SOURCE_DIR}/lib/libswscale.so.5 + ${f4_SOURCE_DIR}/lib/libswresample.so.3 + ) + set( + f5_library_paths + ${f5_SOURCE_DIR}/lib/libavutil.so.57 + ${f5_SOURCE_DIR}/lib/libavcodec.so.59 + ${f5_SOURCE_DIR}/lib/libavformat.so.59 + ${f5_SOURCE_DIR}/lib/libavdevice.so.59 + ${f5_SOURCE_DIR}/lib/libavfilter.so.8 + ${f5_SOURCE_DIR}/lib/libswscale.so.6 + ${f5_SOURCE_DIR}/lib/libswresample.so.4 + ) + set( + f6_library_paths + ${f6_SOURCE_DIR}/lib/libavutil.so.58 + ${f6_SOURCE_DIR}/lib/libavcodec.so.60 + ${f6_SOURCE_DIR}/lib/libavformat.so.60 + ${f6_SOURCE_DIR}/lib/libavdevice.so.60 + ${f6_SOURCE_DIR}/lib/libavfilter.so.9 + ${f6_SOURCE_DIR}/lib/libswscale.so.7 + ${f6_SOURCE_DIR}/lib/libswresample.so.4 + ) + set( + f7_library_paths + ${f7_SOURCE_DIR}/lib/libavutil.so.59 + ${f7_SOURCE_DIR}/lib/libavcodec.so.61 + ${f7_SOURCE_DIR}/lib/libavformat.so.61 + ${f7_SOURCE_DIR}/lib/libavdevice.so.61 + ${f7_SOURCE_DIR}/lib/libavfilter.so.10 + ${f7_SOURCE_DIR}/lib/libswscale.so.8 + ${f7_SOURCE_DIR}/lib/libswresample.so.5 ) elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") set( @@ -99,44 +99,105 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a ) set( - f4_library_file_names - libavutil.56.dylib - libavcodec.58.dylib - libavformat.58.dylib - libavdevice.58.dylib - libavfilter.7.dylib - libswscale.5.dylib - libswresample.3.dylib - ) - set( - f5_library_file_names - libavutil.57.dylib - libavcodec.59.dylib - libavformat.59.dylib - libavdevice.59.dylib - libavfilter.8.dylib - libswscale.6.dylib - libswresample.4.dylib - ) - set( - f6_library_file_names - libavutil.58.dylib - libavcodec.60.dylib - libavformat.60.dylib - libavdevice.60.dylib - libavfilter.9.dylib - libswscale.7.dylib - libswresample.4.dylib - ) - set( - f7_library_file_names - libavutil.59.dylib - libavcodec.61.dylib - libavformat.61.dylib - libavdevice.61.dylib - libavfilter.10.dylib - libswscale.8.dylib - libswresample.5.dylib + f4_library_paths + ${f4_SOURCE_DIR}/lib/libavutil.56.dylib + ${f4_SOURCE_DIR}/lib/libavcodec.58.dylib + ${f4_SOURCE_DIR}/lib/libavformat.58.dylib + ${f4_SOURCE_DIR}/lib/libavdevice.58.dylib + ${f4_SOURCE_DIR}/lib/libavfilter.7.dylib + ${f4_SOURCE_DIR}/lib/libswscale.5.dylib + ${f4_SOURCE_DIR}/lib/libswresample.3.dylib + ) + set( + f5_library_paths + ${f6_SOURCE_DIR}/lib/libavutil.57.dylib + ${f6_SOURCE_DIR}/lib/libavcodec.59.dylib + ${f6_SOURCE_DIR}/lib/libavformat.59.dylib + ${f6_SOURCE_DIR}/lib/libavdevice.59.dylib + ${f6_SOURCE_DIR}/lib/libavfilter.8.dylib + ${f6_SOURCE_DIR}/lib/libswscale.6.dylib + ${f6_SOURCE_DIR}/lib/libswresample.4.dylib + ) + set( + f6_library_paths + ${f6_SOURCE_DIR}/lib/libavutil.58.dylib + ${f6_SOURCE_DIR}/lib/libavcodec.60.dylib + ${f6_SOURCE_DIR}/lib/libavformat.60.dylib + ${f6_SOURCE_DIR}/lib/libavdevice.60.dylib + ${f6_SOURCE_DIR}/lib/libavfilter.9.dylib + ${f6_SOURCE_DIR}/lib/libswscale.7.dylib + ${f6_SOURCE_DIR}/lib/libswresample.4.dylib + ) + set( + f7_library_paths + ${f7_SOURCE_DIR}/lib/libavutil.59.dylib + ${f7_SOURCE_DIR}/lib/libavcodec.61.dylib + ${f7_SOURCE_DIR}/lib/libavformat.61.dylib + ${f7_SOURCE_DIR}/lib/libavdevice.61.dylib + ${f7_SOURCE_DIR}/lib/libavfilter.10.dylib + ${f7_SOURCE_DIR}/lib/libswscale.8.dylib + ${f7_SOURCE_DIR}/lib/libswresample.5.dylib + ) +elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set( + platform_url + ${base_url}/windows_x86_64 + ) + set( + f4_sha256 + 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb + ) + set( + f5_sha256 + b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 + ) + set( + f6_sha256 + 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a + ) + set( + f7_sha256 + ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 + ) + set( + f4_library_paths + ${f4_SOURCE_DIR}/bin/avutil.lib + ${f4_SOURCE_DIR}/bin/avcodec.lib + ${f4_SOURCE_DIR}/bin/avformat.lib + ${f4_SOURCE_DIR}/bin/avdevice.lib + ${f4_SOURCE_DIR}/bin/avfilter.lib + ${f4_SOURCE_DIR}/bin/swscale.lib + ${f4_SOURCE_DIR}/bin/swresample.lib + ) + set( + f5_library_paths + ${f5_SOURCE_DIR}/bin/avutil.lib + ${f5_SOURCE_DIR}/bin/avcodec.lib + ${f5_SOURCE_DIR}/bin/avformat.lib + ${f5_SOURCE_DIR}/bin/avdevice.lib + ${f5_SOURCE_DIR}/bin/avfilter.lib + ${f5_SOURCE_DIR}/bin/swscale.lib + ${f5_SOURCE_DIR}/bin/swresample.lib + ) + set( + f6_library_paths + ${f6_SOURCE_DIR}/bin/avutil.lib + ${f6_SOURCE_DIR}/bin/avcodec.lib + ${f6_SOURCE_DIR}/bin/avformat.lib + ${f6_SOURCE_DIR}/bin/avdevice.lib + ${f6_SOURCE_DIR}/bin/avfilter.lib + ${f6_SOURCE_DIR}/bin/swscale.lib + ${f6_SOURCE_DIR}/bin/swresample.lib + ) + set( + f7_library_paths + ${f7_SOURCE_DIR}/bin/avutil.lib + ${f7_SOURCE_DIR}/bin/avcodec.lib + ${f7_SOURCE_DIR}/bin/avformat.lib + ${f7_SOURCE_DIR}/bin/avdevice.lib + ${f7_SOURCE_DIR}/bin/avfilter.lib + ${f7_SOURCE_DIR}/bin/swscale.lib + ${f7_SOURCE_DIR}/bin/swresample.lib ) else() message( @@ -183,26 +244,6 @@ target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include) target_include_directories(ffmpeg7 INTERFACE ${f7_SOURCE_DIR}/include) -list( - TRANSFORM f4_library_file_names - PREPEND ${f4_SOURCE_DIR}/lib/ - OUTPUT_VARIABLE f4_library_paths -) -list( - TRANSFORM f5_library_file_names - PREPEND ${f5_SOURCE_DIR}/lib/ - OUTPUT_VARIABLE f5_library_paths -) -list( - TRANSFORM f6_library_file_names - PREPEND ${f6_SOURCE_DIR}/lib/ - OUTPUT_VARIABLE f6_library_paths -) -list( - TRANSFORM f7_library_file_names - PREPEND ${f7_SOURCE_DIR}/lib/ - OUTPUT_VARIABLE f7_library_paths -) target_link_libraries( ffmpeg4 From 3a20aceca5d26fbae4b06415096c1cda6d4b6add Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:49:58 +0100 Subject: [PATCH 02/57] Fix --- .../fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 5ecc23864..1b3ae6b4d 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -110,13 +110,13 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") ) set( f5_library_paths - ${f6_SOURCE_DIR}/lib/libavutil.57.dylib - ${f6_SOURCE_DIR}/lib/libavcodec.59.dylib - ${f6_SOURCE_DIR}/lib/libavformat.59.dylib - ${f6_SOURCE_DIR}/lib/libavdevice.59.dylib - ${f6_SOURCE_DIR}/lib/libavfilter.8.dylib - ${f6_SOURCE_DIR}/lib/libswscale.6.dylib - ${f6_SOURCE_DIR}/lib/libswresample.4.dylib + ${f5_SOURCE_DIR}/lib/libavutil.57.dylib + ${f5_SOURCE_DIR}/lib/libavcodec.59.dylib + ${f5_SOURCE_DIR}/lib/libavformat.59.dylib + ${f5_SOURCE_DIR}/lib/libavdevice.59.dylib + ${f5_SOURCE_DIR}/lib/libavfilter.8.dylib + ${f5_SOURCE_DIR}/lib/libswscale.6.dylib + ${f5_SOURCE_DIR}/lib/libswresample.4.dylib ) set( f6_library_paths From 02a998df229c3b4bc66e584cdd7e431ceb2833be Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:52:01 +0100 Subject: [PATCH 03/57] Add Windows workflow --- .github/workflows/windows_wheel.yaml | 127 +++++++++++++++++++++++++++ packaging/vc_env_helper.bat | 41 +++++++++ 2 files changed, 168 insertions(+) create mode 100644 .github/workflows/windows_wheel.yaml create mode 100644 packaging/vc_env_helper.bat diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml new file mode 100644 index 000000000..b29d2ea0f --- /dev/null +++ b/.github/workflows/windows_wheel.yaml @@ -0,0 +1,127 @@ +name: Build and test Windows wheel + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +permissions: + id-token: write + contents: write + +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: windows + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-xpu: disable + with-rocm: disable + with-cuda: disable + build-python-only: "disable" + + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload Windows wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + env-script: packaging/vc_env_helper.bat + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + + install-and-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + needs: build + steps: + - uses: actions/download-artifact@v4 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_windows_cpu_x86_64 + path: pytorch/torchcodec/dist/ + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install PyTorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv + + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + source packaging/helpers.sh + assert_ffmpeg_not_installed + + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version + + - name: Install test dependencies + run: | + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow + + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Run Python tests + run: | + pytest test -vvv diff --git a/packaging/vc_env_helper.bat b/packaging/vc_env_helper.bat new file mode 100644 index 000000000..1f50b1b05 --- /dev/null +++ b/packaging/vc_env_helper.bat @@ -0,0 +1,41 @@ +@echo on + +set VC_VERSION_LOWER=17 +set VC_VERSION_UPPER=18 + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15INSTALLDIR=%%i" + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto vswhere + ) +) + +:vswhere +if "%VSDEVCMD_ARGS%" == "" ( + call "%VS15VCVARSALL%" x64 || exit /b 1 +) else ( + call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 +) + +@echo on + +if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + +set DISTUTILS_USE_SDK=1 + +set args=%1 +shift +:start +if [%1] == [] goto done +set args=%args% %1 +shift +goto start + +:done +if "%args%" == "" ( + echo Usage: vc_env_helper.bat [command] [args] + echo e.g. vc_env_helper.bat cl /c test.cpp +) + +%args% || exit /b 1 From 4af56adf14735428275d46d56d48033c162a8083 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:57:35 +0100 Subject: [PATCH 04/57] Fix ? --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 1b3ae6b4d..80fc9fca8 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -14,6 +14,42 @@ set( https://pytorch.s3.amazonaws.com/torchcodec/ffmpeg/2025-03-14 ) + + +FetchContent_Declare( + f4 + URL ${platform_url}/4.4.4.tar.gz + URL_HASH + SHA256=${f4_sha256} +) +FetchContent_Declare( + f5 + URL ${platform_url}/5.1.4.tar.gz + URL_HASH + SHA256=${f5_sha256} +) +FetchContent_Declare( + f6 + URL ${platform_url}/6.1.1.tar.gz + URL_HASH + SHA256=${f6_sha256} +) +FetchContent_Declare( + f7 + URL ${platform_url}/7.0.1.tar.gz + URL_HASH + SHA256=${f7_sha256} +) + +FetchContent_MakeAvailable(f4 f5 f6 f7) + +add_library(ffmpeg4 INTERFACE) +add_library(ffmpeg5 INTERFACE) +add_library(ffmpeg6 INTERFACE) +add_library(ffmpeg7 INTERFACE) + +# Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable + if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") set( platform_url @@ -205,40 +241,6 @@ else() "Unsupported operating system: ${CMAKE_SYSTEM_NAME}" ) endif() - -FetchContent_Declare( - f4 - URL ${platform_url}/4.4.4.tar.gz - URL_HASH - SHA256=${f4_sha256} -) -FetchContent_Declare( - f5 - URL ${platform_url}/5.1.4.tar.gz - URL_HASH - SHA256=${f5_sha256} -) -FetchContent_Declare( - f6 - URL ${platform_url}/6.1.1.tar.gz - URL_HASH - SHA256=${f6_sha256} -) -FetchContent_Declare( - f7 - URL ${platform_url}/7.0.1.tar.gz - URL_HASH - SHA256=${f7_sha256} -) - -FetchContent_MakeAvailable(f4 f5 f6 f7) - -add_library(ffmpeg4 INTERFACE) -add_library(ffmpeg5 INTERFACE) -add_library(ffmpeg6 INTERFACE) -add_library(ffmpeg7 INTERFACE) - -# Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable target_include_directories(ffmpeg4 INTERFACE ${f4_SOURCE_DIR}/include) target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include) From 95bd9b21bdf0784d1140a8b60a1304cdd45d9901 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 22:59:35 +0100 Subject: [PATCH 05/57] tmp --- .github/workflows/windows_wheel.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index b29d2ea0f..89e949851 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -55,8 +55,9 @@ jobs: smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + # TODO Probably needs to enable this + # build-platform: "python-build-package" + # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: ubuntu-latest From 9b3d6e4471acba573847957733a38525e81b3107 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:13:38 +0100 Subject: [PATCH 06/57] Try to use custom test-infra branch --- .github/workflows/windows_wheel.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 89e949851..a5613b43b 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -42,12 +42,12 @@ jobs: strategy: fail-fast: false name: Build and Upload Windows wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows with: repository: pytorch/torchcodec ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main + test-infra-repository: nicolashug/test-infra + test-infra-ref: build-platform-windows build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh @@ -55,9 +55,8 @@ jobs: smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} - # TODO Probably needs to enable this - # build-platform: "python-build-package" - # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + build-platform: "python-build-package" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: ubuntu-latest From 18f78ab0201699e65609543214cfa5a5ee985c5b Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:35:59 +0100 Subject: [PATCH 07/57] dont use env helper? --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index a5613b43b..854540b74 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -51,7 +51,7 @@ jobs: build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh - env-script: packaging/vc_env_helper.bat + # env-script: packaging/vc_env_helper.bat smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} From c814edf0fdf2bc5be8fef0caae1c9f9c9cff5e4c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:40:55 +0100 Subject: [PATCH 08/57] Hmm? --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 854540b74..1f581d5a6 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -56,7 +56,7 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + build-command: "set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: ubuntu-latest From a25882e1e8ede1bac5750f8c2fafa680a5a7af93 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:45:49 +0100 Subject: [PATCH 09/57] reorder --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 138 +++++++++--------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 80fc9fca8..81f115ff4 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -15,6 +15,77 @@ set( ) +if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + set( + platform_url + ${base_url}/linux_x86_64 + ) + + set( + f4_sha256 + 1a083f1922443bedb5243d04896383b8c606778a7ddb9d886c8303e55339fe0c + ) + set( + f5_sha256 + 65d6ad54082d94dcb3f801d73df2265e0e1bb303c7afbce7723e3b77ccd0e207 + ) + set( + f6_sha256 + 8bd5939c2f4a4b072e837e7870c13fe7d13824e5ff087ab534e4db4e90b7be9c + ) + set( + f7_sha256 + 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 + ) +elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + set( + platform_url + ${base_url}/macos_arm64 + ) + set( + f4_sha256 + f0335434529d9e19359eae0fe912dd9e747667534a1c92e662f5219a55dfad8c + ) + set( + f5_sha256 + cfc3449c9af6863731a431ce89e32c08c5f8ece94b306fb6b695828502a76166 + ) + set( + f6_sha256 + ec47b4783c342038e720e33b2fdfa55a9a490afb1cf37a26467733983688647e + ) + set( + f7_sha256 + 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a + ) + +elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set( + platform_url + ${base_url}/windows_x86_64 + ) + set( + f4_sha256 + 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb + ) + set( + f5_sha256 + b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 + ) + set( + f6_sha256 + 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a + ) + set( + f7_sha256 + ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 + ) +else() + message( + FATAL_ERROR + "Unsupported operating system: ${CMAKE_SYSTEM_NAME}" + ) +endif() FetchContent_Declare( f4 @@ -51,28 +122,6 @@ add_library(ffmpeg7 INTERFACE) # Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") - set( - platform_url - ${base_url}/linux_x86_64 - ) - - set( - f4_sha256 - 1a083f1922443bedb5243d04896383b8c606778a7ddb9d886c8303e55339fe0c - ) - set( - f5_sha256 - 65d6ad54082d94dcb3f801d73df2265e0e1bb303c7afbce7723e3b77ccd0e207 - ) - set( - f6_sha256 - 8bd5939c2f4a4b072e837e7870c13fe7d13824e5ff087ab534e4db4e90b7be9c - ) - set( - f7_sha256 - 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 - ) - set( f4_library_paths ${f4_SOURCE_DIR}/lib/libavutil.so.56 @@ -114,26 +163,6 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") ${f7_SOURCE_DIR}/lib/libswresample.so.5 ) elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") - set( - platform_url - ${base_url}/macos_arm64 - ) - set( - f4_sha256 - f0335434529d9e19359eae0fe912dd9e747667534a1c92e662f5219a55dfad8c - ) - set( - f5_sha256 - cfc3449c9af6863731a431ce89e32c08c5f8ece94b306fb6b695828502a76166 - ) - set( - f6_sha256 - ec47b4783c342038e720e33b2fdfa55a9a490afb1cf37a26467733983688647e - ) - set( - f7_sha256 - 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a - ) set( f4_library_paths ${f4_SOURCE_DIR}/lib/libavutil.56.dylib @@ -175,26 +204,6 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") ${f7_SOURCE_DIR}/lib/libswresample.5.dylib ) elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - set( - platform_url - ${base_url}/windows_x86_64 - ) - set( - f4_sha256 - 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb - ) - set( - f5_sha256 - b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 - ) - set( - f6_sha256 - 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a - ) - set( - f7_sha256 - ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 - ) set( f4_library_paths ${f4_SOURCE_DIR}/bin/avutil.lib @@ -235,11 +244,6 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") ${f7_SOURCE_DIR}/bin/swscale.lib ${f7_SOURCE_DIR}/bin/swresample.lib ) -else() - message( - FATAL_ERROR - "Unsupported operating system: ${CMAKE_SYSTEM_NAME}" - ) endif() target_include_directories(ffmpeg4 INTERFACE ${f4_SOURCE_DIR}/include) target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) From b46e5ac3eba2d3f9c3a418ef8dbcbcf5979e421f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 4 Jul 2025 23:47:58 +0100 Subject: [PATCH 10/57] Try something --- .github/workflows/windows_wheel.yaml | 5 +++-- setup.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 1f581d5a6..8297688f3 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -51,12 +51,13 @@ jobs: build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh - # env-script: packaging/vc_env_helper.bat + env-script: packaging/vc_env_helper.bat smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation" + # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + build-command: "python -m build --wheel -vvv --no-isolation" install-and-test: runs-on: ubuntu-latest diff --git a/setup.py b/setup.py index 5074ef478..8321a8262 100644 --- a/setup.py +++ b/setup.py @@ -169,14 +169,14 @@ def copy_extensions_to_source(self): build_against_all_ffmpeg_from_s3 = ( os.getenv(BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR) is not None ) -if "bdist_wheel" in sys.argv and not ( - build_against_all_ffmpeg_from_s3 or not_a_license_violation -): - raise ValueError( - "It looks like you're trying to build a wheel. " - f"You probably want to set {BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR}. " - f"If you have a good reason *not* to, then set {NOT_A_LICENSE_VIOLATION_VAR}." - ) +# if "bdist_wheel" in sys.argv and not ( +# build_against_all_ffmpeg_from_s3 or not_a_license_violation +# ): +# raise ValueError( +# "It looks like you're trying to build a wheel. " +# f"You probably want to set {BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR}. " +# f"If you have a good reason *not* to, then set {NOT_A_LICENSE_VIOLATION_VAR}." +# ) # See `CMakeBuild.build_extension()`. fake_extension = Extension(name="FAKE_NAME", sources=[]) From c6c30cefb7df58185020e3d8de3c5bbdde16755f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 10:01:49 +0100 Subject: [PATCH 11/57] Comment-out some workflows --- .github/workflows/cpp_tests.yaml | 136 +++++++-------- .github/workflows/docs.yaml | 218 ++++++++++++------------ .github/workflows/lint.yaml | 142 +++++++-------- .github/workflows/linux_cuda_wheel.yaml | 162 +++++++++--------- .github/workflows/linux_wheel.yaml | 126 +++++++------- .github/workflows/macos_wheel.yaml | 130 +++++++------- 6 files changed, 457 insertions(+), 457 deletions(-) diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index e08d90754..44dd288d6 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -1,73 +1,73 @@ -name: CPP tests +# name: CPP tests -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -concurrency: - group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} - cancel-in-progress: true +# concurrency: +# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} +# cancel-in-progress: true -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - Cpp-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v3 - with: - auto-update-conda: true - # Using miniforge instead of miniconda ensures that the default - # conda channel is conda-forge instead of main/default. This ensures - # ABI consistency between dependencies: - # https://conda-forge.org/docs/user/transitioning_from_defaults/ - miniforge-version: latest - activate-environment: test - python-version: '3.12' - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install torch dependencies - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install ffmpeg, pkg-config and pybind11 - run: | - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config pybind11 -c conda-forge - ffmpeg -version - - name: Build and run C++ tests - run: | - # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so - # we're building libtorchcodec against the installed FFmpeg version - # (from conda-forge) instead of building against our pre-built non-GPL - # FFmpeg libraries. - # The reason we need this is because the C++ tests decode x264 files. - # x264 support is not LGPL, os it is not supported by our - # pre-built non-GPL FFmpeg libraries. And if we were to build against - # those, this is also what the tests would be loading at run time, - # then failing when we try to decode x264. - # To remediate that, we build against the FFmpeg that we installed - # from conda-forge (which is able to decode x264), and that's also - # what gets loaded at run time. - # The Python tests are also decoding x264 files, and are built against - # our non-GPL FFmpeg. And yet they pass. This is because in Python - # we're able to distinguish between build-time (non-GPL FFmpeg) and - # run time (conda-forge FFmpeg). +# jobs: +# Cpp-tests: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v3 +# with: +# auto-update-conda: true +# # Using miniforge instead of miniconda ensures that the default +# # conda channel is conda-forge instead of main/default. This ensures +# # ABI consistency between dependencies: +# # https://conda-forge.org/docs/user/transitioning_from_defaults/ +# miniforge-version: latest +# activate-environment: test +# python-version: '3.12' +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install torch dependencies +# run: | +# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +# - name: Install ffmpeg, pkg-config and pybind11 +# run: | +# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config pybind11 -c conda-forge +# ffmpeg -version +# - name: Build and run C++ tests +# run: | +# # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so +# # we're building libtorchcodec against the installed FFmpeg version +# # (from conda-forge) instead of building against our pre-built non-GPL +# # FFmpeg libraries. +# # The reason we need this is because the C++ tests decode x264 files. +# # x264 support is not LGPL, os it is not supported by our +# # pre-built non-GPL FFmpeg libraries. And if we were to build against +# # those, this is also what the tests would be loading at run time, +# # then failing when we try to decode x264. +# # To remediate that, we build against the FFmpeg that we installed +# # from conda-forge (which is able to decode x264), and that's also +# # what gets loaded at run time. +# # The Python tests are also decoding x264 files, and are built against +# # our non-GPL FFmpeg. And yet they pass. This is because in Python +# # we're able to distinguish between build-time (non-GPL FFmpeg) and +# # run time (conda-forge FFmpeg). - build_tests_dir="${PWD}/build_tests" - mkdir $build_tests_dir - pushd $build_tests_dir - TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") - Torch_DIR="${TORCH_PATH}/share/cmake/Torch" - cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON - cmake --build . - ctest --output-on-failure - popd +# build_tests_dir="${PWD}/build_tests" +# mkdir $build_tests_dir +# pushd $build_tests_dir +# TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") +# Torch_DIR="${TORCH_PATH}/share/cmake/Torch" +# cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON +# cmake --build . +# ctest --output-on-failure +# popd diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 60bfbfa2e..39aaec1a1 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,116 +1,116 @@ -name: Docs +# name: Docs -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -permissions: - id-token: write - contents: write +# permissions: +# id-token: write +# contents: write -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-cpu: disable - with-xpu: disable - with-rocm: disable - with-cuda: enable - build-python-only: "disable" - build: - needs: generate-matrix - strategy: - fail-fast: false - name: Build and Upload wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: pytorch/torchcodec - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec - trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" +# jobs: +# generate-matrix: +# uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main +# with: +# package-type: wheel +# os: linux +# test-infra-repository: pytorch/test-infra +# test-infra-ref: main +# with-cpu: disable +# with-xpu: disable +# with-rocm: disable +# with-cuda: enable +# build-python-only: "disable" +# build: +# needs: generate-matrix +# strategy: +# fail-fast: false +# name: Build and Upload wheel +# uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main +# with: +# repository: pytorch/torchcodec +# ref: "" +# test-infra-repository: pytorch/test-infra +# test-infra-ref: main +# build-matrix: ${{ needs.generate-matrix.outputs.matrix }} +# pre-script: packaging/pre_build_script.sh +# post-script: packaging/post_build_script.sh +# smoke-test-script: packaging/fake_smoke_test.py +# package-name: torchcodec +# trigger-event: ${{ github.event_name }} +# build-platform: "python-build-package" +# build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" - build-docs: - runs-on: linux.4xlarge.nvidia.gpu - strategy: - fail-fast: false - matrix: - # 3.9 corresponds to the minimum python version for which we build - # the wheel unless the label cliflow/binaries/all is present in the - # PR. - python-version: ['3.9'] - cuda-version: ['12.6'] - ffmpeg-version-for-tests: ['7'] - container: - image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - needs: build - steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') - echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - with: - python-version: ${{ matrix.python-version }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" - - name: Check env - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip - - name: Install PyTorch - run: | - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` - echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv +# build-docs: +# runs-on: linux.4xlarge.nvidia.gpu +# strategy: +# fail-fast: false +# matrix: +# # 3.9 corresponds to the minimum python version for which we build +# # the wheel unless the label cliflow/binaries/all is present in the +# # PR. +# python-version: ['3.9'] +# cuda-version: ['12.6'] +# ffmpeg-version-for-tests: ['7'] +# container: +# image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" +# options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" +# needs: build +# steps: +# - name: Setup env vars +# run: | +# cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') +# echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV +# python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') +# echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV +# - uses: actions/download-artifact@v4 +# with: +# name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 +# path: pytorch/torchcodec/dist/ +# - name: Setup miniconda using test-infra +# uses: pytorch/test-infra/.github/actions/setup-miniconda@main +# with: +# python-version: ${{ matrix.python-version }} +# # We install conda packages at the start because otherwise conda may have conflicts with dependencies. +# default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" +# - name: Check env +# run: | +# ${CONDA_RUN} env +# ${CONDA_RUN} conda info +# ${CONDA_RUN} nvidia-smi +# ${CONDA_RUN} conda list +# - name: Assert ffmpeg exists +# run: | +# ${CONDA_RUN} ffmpeg -buildconf +# - name: Update pip +# run: ${CONDA_RUN} python -m pip install --upgrade pip +# - name: Install PyTorch +# run: | +# ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} +# ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' +# - name: Install torchcodec from the wheel +# run: | +# wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` +# echo Installing $wheel_path +# ${CONDA_RUN} python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 +# - name: Check out repo +# uses: actions/checkout@v3 - - name: Install doc dependencies - run: | - cd docs - ${CONDA_RUN} python -m pip install -r requirements.txt - - name: Build docs - run: | - cd docs - ${CONDA_RUN} make html - - uses: actions/upload-artifact@v4 - with: - name: Built-Docs - path: docs/build/html/ +# - name: Install doc dependencies +# run: | +# cd docs +# ${CONDA_RUN} python -m pip install -r requirements.txt +# - name: Build docs +# run: | +# cd docs +# ${CONDA_RUN} make html +# - uses: actions/upload-artifact@v4 +# with: +# name: Built-Docs +# path: docs/build/html/ diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c156a833c..6ed79d88c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,76 +1,76 @@ -name: Lint +# name: Lint -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -concurrency: - group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} - cancel-in-progress: true +# concurrency: +# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} +# cancel-in-progress: true -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - pre-commit-checks: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install pre-commit - run: | - python -m pip install pre-commit - - name: Run pre-commit checks - run: | - pre-commit run --all-files - - name: Check to see what files pre-commit modified - run: | - git diff +# jobs: +# pre-commit-checks: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# python-version: ['3.12'] +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v2 +# with: +# auto-update-conda: true +# miniconda-version: "latest" +# activate-environment: test +# python-version: ${{ matrix.python-version }} +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install pre-commit +# run: | +# python -m pip install pre-commit +# - name: Run pre-commit checks +# run: | +# pre-commit run --all-files +# - name: Check to see what files pre-commit modified +# run: | +# git diff - mypy: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install dependencies and FFmpeg - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - conda install "ffmpeg=7.0.1" pkg-config pybind11 -c conda-forge - ffmpeg -version - - name: Build and install torchcodec - run: | - python -m pip install -e ".[dev]" --no-build-isolation -vvv - - name: Install mypy - run: | - python -m pip install mypy - - name: Run mypy - run: | - mypy --install-types --non-interactive --config-file mypy.ini +# mypy: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# python-version: ['3.12'] +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v2 +# with: +# auto-update-conda: true +# miniconda-version: "latest" +# activate-environment: test +# python-version: ${{ matrix.python-version }} +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install dependencies and FFmpeg +# run: | +# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +# conda install "ffmpeg=7.0.1" pkg-config pybind11 -c conda-forge +# ffmpeg -version +# - name: Build and install torchcodec +# run: | +# python -m pip install -e ".[dev]" --no-build-isolation -vvv +# - name: Install mypy +# run: | +# python -m pip install mypy +# - name: Run mypy +# run: | +# mypy --install-types --non-interactive --config-file mypy.ini diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index bd57cac5e..ffc21ab46 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -56,88 +56,88 @@ jobs: build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" - install-and-test: - runs-on: linux.g5.4xlarge.nvidia.gpu - strategy: - fail-fast: false - matrix: - # 3.9 corresponds to the minimum python version for which we build - # the wheel unless the label cliflow/binaries/all is present in the - # PR. - # For the actual release we should add that label and change this to - # include more python versions. - python-version: ['3.9'] - cuda-version: ['12.6', '12.8'] - # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 - ffmpeg-version-for-tests: ['4.4.2', '6', '7'] + # install-and-test: + # runs-on: linux.g5.4xlarge.nvidia.gpu + # strategy: + # fail-fast: false + # matrix: + # # 3.9 corresponds to the minimum python version for which we build + # # the wheel unless the label cliflow/binaries/all is present in the + # # PR. + # # For the actual release we should add that label and change this to + # # include more python versions. + # python-version: ['3.9'] + # cuda-version: ['12.6', '12.8'] + # # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 + # ffmpeg-version-for-tests: ['4.4.2', '6', '7'] - container: - image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - needs: build - steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') - echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - with: - python-version: ${{ matrix.python-version }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" - - name: Check env - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip - - name: Install PyTorch - run: | - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` - echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv + # container: + # image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" + # options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" + # needs: build + # steps: + # - name: Setup env vars + # run: | + # cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') + # echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV + # python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') + # echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV + # - uses: actions/download-artifact@v4 + # with: + # name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 + # path: pytorch/torchcodec/dist/ + # - name: Setup miniconda using test-infra + # uses: pytorch/test-infra/.github/actions/setup-miniconda@main + # with: + # python-version: ${{ matrix.python-version }} + # # We install conda packages at the start because otherwise conda may have conflicts with dependencies. + # default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" + # - name: Check env + # run: | + # ${CONDA_RUN} env + # ${CONDA_RUN} conda info + # ${CONDA_RUN} nvidia-smi + # ${CONDA_RUN} conda list + # - name: Assert ffmpeg exists + # run: | + # ${CONDA_RUN} ffmpeg -buildconf + # - name: Update pip + # run: ${CONDA_RUN} python -m pip install --upgrade pip + # - name: Install PyTorch + # run: | + # ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} + # ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` + # echo Installing $wheel_path + # ${CONDA_RUN} python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 + # - name: Check out repo + # uses: actions/checkout@v3 - - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - ${CONDA_RUN} python -m pip install numpy pytest pillow + # - name: Install test dependencies + # run: | + # # Ideally we would find a way to get those dependencies from pyproject.toml + # ${CONDA_RUN} python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Run Python tests - run: | - ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest test -v --tb=short - - name: Run Python benchmark - run: | - ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls + # - name: Run Python tests + # run: | + # ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest test -v --tb=short + # - name: Run Python benchmark + # run: | + # ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 1855e904e..58086b0bc 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -57,70 +57,70 @@ jobs: build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" - install-and-test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.9'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - needs: build - steps: - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv + # install-and-test: + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # matrix: + # python-version: ['3.9'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # needs: build + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 + # path: pytorch/torchcodec/dist/ + # - name: Setup conda env + # uses: conda-incubator/setup-miniconda@v2 + # with: + # auto-update-conda: true + # miniconda-version: "latest" + # activate-environment: test + # python-version: ${{ matrix.python-version }} + # - name: Update pip + # run: python -m pip install --upgrade pip + # - name: Install PyTorch + # run: | + # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + # echo Installing $wheel_path + # python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 - - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - source packaging/helpers.sh - assert_ffmpeg_not_installed + # - name: Check out repo + # uses: actions/checkout@v3 + # - name: Install ffmpeg, post build + # run: | + # # Ideally we would have checked for that before installing the wheel, + # # but we need to checkout the repo to access this file, and we don't + # # want to checkout the repo before installing the wheel to avoid any + # # side-effect. It's OK. + # source packaging/helpers.sh + # assert_ffmpeg_not_installed - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version + # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + # ffmpeg -version - - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow + # - name: Install test dependencies + # run: | + # # Ideally we would find a way to get those dependencies from pyproject.toml + # python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Run Python tests - run: | - pytest test -vvv + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls + # - name: Run Python tests + # run: | + # pytest test -vvv diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index ee436b7a4..ad6c00fee 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -58,68 +58,68 @@ jobs: build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" - install-and-test: - runs-on: macos-14-xlarge - strategy: - fail-fast: false - matrix: - python-version: ['3.9'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - needs: build - steps: - - name: Download wheel - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ - path: pytorch/torchcodec/dist/ - - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v3 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv - - - name: Check out torchcodec repo - uses: actions/checkout@v3 - - - name: Install ffmpeg - run: | - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version - - - name: Install test dependencies - run: | - python -m pip install numpy pytest pillow - - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls -lh - - - name: Run Python tests - run: | - pytest test -vvv + # install-and-test: + # runs-on: macos-14-xlarge + # strategy: + # fail-fast: false + # matrix: + # python-version: ['3.9'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # needs: build + # steps: + # - name: Download wheel + # uses: actions/download-artifact@v4 + # with: + # name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ + # path: pytorch/torchcodec/dist/ + + # - name: Setup conda env + # uses: conda-incubator/setup-miniconda@v3 + # with: + # auto-update-conda: true + # miniconda-version: "latest" + # activate-environment: test + # python-version: ${{ matrix.python-version }} + # - name: Update pip + # run: python -m pip install --upgrade pip + + # - name: Install PyTorch + # run: | + # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + # echo Installing $wheel_path + # python -m pip install $wheel_path -vvv + + # - name: Check out torchcodec repo + # uses: actions/checkout@v3 + + # - name: Install ffmpeg + # run: | + # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + # ffmpeg -version + + # - name: Install test dependencies + # run: | + # python -m pip install numpy pytest pillow + + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls -lh + + # - name: Run Python tests + # run: | + # pytest test -vvv From 8930df5214d7f8a849a7803dae5b919bd3f141f1 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 10:11:07 +0100 Subject: [PATCH 12/57] Remove compilation flags --- src/torchcodec/_core/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 7196d0487..cce8e4a2a 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -14,7 +14,8 @@ else() set(TORCHCODEC_WERROR_OPTION "-Werror") endif() -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") +# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") function(make_torchcodec_sublibrary library_name From db0d342310fc1e5274835479f7f5ac3ff0a6f4d9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 10:35:35 +0100 Subject: [PATCH 13/57] rework flags --- src/torchcodec/_core/CMakeLists.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index cce8e4a2a..803323b62 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -14,8 +14,19 @@ else() set(TORCHCODEC_WERROR_OPTION "-Werror") endif() +if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + # Avoid warnings about non-ASCII characters in source files. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4819") + # Important for when we add Windows CUDA: exporting all symbols is aparently + # limited to 65535 symbols, which (aparently) will not work for CUDA. + # https://github.com/pytorch/pytorch/pull/3650 + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +endif() + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") +# TODO put back previous line and set proper flags for windows. e.g. Wall is W4. +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") + function(make_torchcodec_sublibrary library_name From 059c925bc2a51788e94d9809f02d6fa4d3cc707c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 10:54:36 +0100 Subject: [PATCH 14/57] Remove some flags --- src/torchcodec/_core/CMakeLists.txt | 30 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 803323b62..2cf44280e 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -158,19 +158,23 @@ function(make_torchcodec_libraries # stray initialization of py::objects. The rest of the object code must # match. See: # https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes - target_compile_options( - ${pybind_ops_library_name} - PUBLIC - "-fvisibility=hidden" - ) - # If we don't make sure this flag is set, we run into segfauls at import - # time on Mac. See: - # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 - target_link_options( - ${pybind_ops_library_name} - PUBLIC - "LINKER:-undefined,dynamic_lookup" - ) + if(NOT WIN32) # TODO unify WIN detection + target_compile_options( + ${pybind_ops_library_name} + PUBLIC + "-fvisibility=hidden" + ) + endif() + if(APPLE) # TODO unify APPLE detection + # If we don't make sure this flag is set, we run into segfauls at import + # time on Mac. See: + # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 + target_link_options( + ${pybind_ops_library_name} + PUBLIC + "LINKER:-undefined,dynamic_lookup" + ) + endif() # Install all libraries. set( From 46d4df54910ef87d750e81f26d8fa3cfba4a1f40 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 11:30:29 +0100 Subject: [PATCH 15/57] Add RUNTIME destination for windows --- src/torchcodec/_core/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 2cf44280e..516ffa950 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -192,6 +192,7 @@ function(make_torchcodec_libraries install( TARGETS ${all_libraries} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} + RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs ) endfunction() From 4ebd78a2ac755519795598a3ccae3c3363ca27f8 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 11:47:29 +0100 Subject: [PATCH 16/57] Set output dirs to CMAKE_CURRENT_BINARY_DIR for Windows --- src/torchcodec/_core/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 516ffa950..e7394d020 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -46,6 +46,18 @@ function(make_torchcodec_sublibrary # Avoid adding the "lib" prefix which we already add explicitly. set_target_properties(${library_name} PROPERTIES PREFIX "") + # On Windows, avoid the Release/Debug subdirectories in output + if(WIN32) + set_target_properties(${library_name} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} + RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} + LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} + LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} + ARCHIVE_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} + ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} + ) + endif() + target_link_libraries( ${library_name} PUBLIC From 43c9e4f672c6a0bba6c9d9a1dd9dcd01093ffbad Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 11:59:56 +0100 Subject: [PATCH 17/57] Add comment --- src/torchcodec/_core/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index e7394d020..1a665b955 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -46,8 +46,10 @@ function(make_torchcodec_sublibrary # Avoid adding the "lib" prefix which we already add explicitly. set_target_properties(${library_name} PROPERTIES PREFIX "") - # On Windows, avoid the Release/Debug subdirectories in output - if(WIN32) + if(WIN32) # TODO unify WIN detection + # On Windows, the built artifacts are put in Release/Debug + # subdirectories by default. We want to avoid that, otherwise our + # install() step would not know where to find those. set_target_properties(${library_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} From 41b90f4a1cebb79d5d72ea30d9489ba26dd51209 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 12:02:01 +0100 Subject: [PATCH 18/57] Disable post-build-script for Windows --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 8297688f3..562eddfec 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -50,7 +50,7 @@ jobs: test-infra-ref: build-platform-windows build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh + # post-script: packaging/post_build_script.sh # TODO enable post-build checks? env-script: packaging/vc_env_helper.bat smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec From 8c0bc380c8c496f8e66dbb64e5d4586fdd481f31 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 12:16:12 +0100 Subject: [PATCH 19/57] Fix expected wheel artifact name --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 562eddfec..b51de3671 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -70,7 +70,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: pytorch_torchcodec__${{ matrix.python-version }}_windows_cpu_x86_64 + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_64 path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v2 From cbbd34f1c1f16f01f0e09d76290b83d27dd29174 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 12:31:40 +0100 Subject: [PATCH 20/57] Update .github/workflows/windows_wheel.yaml --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index b51de3671..6bfaae2de 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -70,7 +70,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_64 + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v2 From 2ccb7ee9b831e0ca35a118aa83b7851f9a2f15f4 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 12:51:14 +0100 Subject: [PATCH 21/57] Update .github/workflows/windows_wheel.yaml --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 6bfaae2de..c8d8d85ff 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -60,7 +60,7 @@ jobs: build-command: "python -m build --wheel -vvv --no-isolation" install-and-test: - runs-on: ubuntu-latest + runs-on: windows-latest strategy: fail-fast: false matrix: From a06060ad9afd0ab1ebdff921d28a6b61102edf39 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 5 Jul 2025 13:13:37 +0100 Subject: [PATCH 22/57] fix extension finder --- src/torchcodec/_internally_replaced_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/torchcodec/_internally_replaced_utils.py b/src/torchcodec/_internally_replaced_utils.py index a5a3ffa39..39b04d5c1 100644 --- a/src/torchcodec/_internally_replaced_utils.py +++ b/src/torchcodec/_internally_replaced_utils.py @@ -18,9 +18,11 @@ def _get_extension_path(lib_name: str) -> str: extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES elif sys.platform == "darwin": extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES + [".dylib"] + elif sys.platform in ("win32", "cygwin"): + extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES + [".dll"] else: raise NotImplementedError( - "Platforms other than linux/darwin are not supported yet" + f"{sys.platform = } is not not supported" ) loader_details = ( importlib.machinery.ExtensionFileLoader, From 7fb6536e4f96e991c4afa87c95344148350b8b15 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:06:12 +0100 Subject: [PATCH 23/57] maybe fix something --- .github/workflows/windows_wheel.yaml | 3 +++ setup.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index c8d8d85ff..a96f4be27 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -42,6 +42,8 @@ jobs: strategy: fail-fast: false name: Build and Upload Windows wheel + # TODO + # We shouldn't use this, but this is needed for now. Note that this is where we are setting the BUILD_AGAINST_ALL_FFMPEG_FROM_S3 variable currently, we'll have to find a way around that. uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows with: repository: pytorch/torchcodec @@ -57,6 +59,7 @@ jobs: trigger-event: ${{ github.event_name }} build-platform: "python-build-package" # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + # TODO see comment above build-command: "python -m build --wheel -vvv --no-isolation" install-and-test: diff --git a/setup.py b/setup.py index 8321a8262..8a583b3b5 100644 --- a/setup.py +++ b/setup.py @@ -150,9 +150,11 @@ def copy_extensions_to_source(self): # dynamically loaded module. For more, see: # https://stackoverflow.com/a/2339910 extensions = ["dylib", "so"] + elif sys.platform in ("win32", "cygwin"): + extensions = ["dll"] else: raise NotImplementedError( - "Platforms other than linux/darwin are not supported yet" + f"Platform {sys.platform} is not supported" ) for ext in extensions: From 0be6336d0b55855f6ecef8596729e59615989336 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:17:17 +0100 Subject: [PATCH 24/57] try to add Library/bin to PATH?? --- .github/workflows/windows_wheel.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index a96f4be27..e4c0bc35e 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -107,6 +107,26 @@ jobs: conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge ffmpeg -version + # On Windows, ensure the conda Library/bin directory is in PATH + # This is needed for Windows DLL loading to find FFmpeg DLLs + if [[ "$RUNNER_OS" == "Windows" ]]; then + conda_env_path=$(conda info --base)/envs/test + library_bin_path="$conda_env_path/Library/bin" + echo "Adding conda Library/bin to PATH: $library_bin_path" + echo "$library_bin_path" >> $GITHUB_PATH + + # Verify FFmpeg DLLs are accessible + echo "Checking if FFmpeg DLLs are in PATH:" + where avutil.dll || echo "avutil.dll not found in PATH" + where avcodec.dll || echo "avcodec.dll not found in PATH" + where avformat.dll || echo "avformat.dll not found in PATH" + fi + + - name: Test torchcodec import after FFmpeg installation + run: | + echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." + python -c "import torchcodec; print('TorchCodec import successful!')" + - name: Install test dependencies run: | # Ideally we would find a way to get those dependencies from pyproject.toml From 5b9827599914a47c536b1318ccbd7c42a7c249b3 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:27:28 +0100 Subject: [PATCH 25/57] Add build and test from source workflow --- .github/workflows/windows_source_build.yaml | 103 ++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 .github/workflows/windows_source_build.yaml diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml new file mode 100644 index 000000000..13b72b2b2 --- /dev/null +++ b/.github/workflows/windows_source_build.yaml @@ -0,0 +1,103 @@ +name: Build and test Windows from source + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + build-and-test: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9'] + ffmpeg-version: ['4.4.4', '5.1.4', '6.1.1', '7.0.1'] + steps: + - name: Check out repo + uses: actions/checkout@v3 + + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: build + python-version: ${{ matrix.python-version }} + + - name: Install FFmpeg + run: | + conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge + ffmpeg -version + + # On Windows, ensure the conda Library/bin directory is in PATH + # This is needed for both pkg-config to find FFmpeg and for DLL loading + conda_env_path=$(conda info --base)/envs/build + library_bin_path="$conda_env_path/Library/bin" + echo "Adding conda Library/bin to PATH: $library_bin_path" + echo "$library_bin_path" >> $GITHUB_PATH + + # Also add Library/lib/pkgconfig to PKG_CONFIG_PATH for cmake to find FFmpeg + pkg_config_path="$conda_env_path/Library/lib/pkgconfig" + echo "Adding to PKG_CONFIG_PATH: $pkg_config_path" + echo "PKG_CONFIG_PATH=$pkg_config_path" >> $GITHUB_ENV + + # Verify FFmpeg DLLs are accessible + echo "Checking if FFmpeg DLLs are in PATH:" + where avutil.dll || echo "avutil.dll not found in PATH" + where avcodec.dll || echo "avcodec.dll not found in PATH" + where avformat.dll || echo "avformat.dll not found in PATH" + + - name: Install build dependencies + run: | + # Install pkg-config and pybind11 which are needed for Windows builds + conda install -y pkg-config pybind11 -c conda-forge + + # Verify pkg-config can find FFmpeg + echo "Testing pkg-config for FFmpeg libraries:" + pkg-config --exists libavcodec && echo "libavcodec found" || echo "libavcodec NOT found" + pkg-config --exists libavformat && echo "libavformat found" || echo "libavformat NOT found" + pkg-config --exists libavutil && echo "libavutil found" || echo "libavutil NOT found" + + - name: Update pip and install PyTorch + run: | + python -m pip install --upgrade pip + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + + - name: Build torchcodec from source + run: | + # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg + echo "Building torchcodec from source using system FFmpeg..." + python -m pip install -e . --no-build-isolation -v + + - name: Test torchcodec import + run: | + echo "Testing torchcodec import..." + python -c "import torchcodec; print('TorchCodec import successful!')" + python -c "import torchcodec; print('FFmpeg versions:', torchcodec.get_ffmpeg_library_versions())" + + - name: Install test dependencies + run: | + python -m pip install numpy pytest pillow + + - name: Run Python tests + run: | + pytest test -v From a1ba873f50095ddbabad33776d20b968858d7470 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:33:26 +0100 Subject: [PATCH 26/57] Fix FFmpeg versions for source build --- .github/workflows/windows_source_build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 13b72b2b2..15c3ae84b 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -30,7 +30,7 @@ jobs: fail-fast: false matrix: python-version: ['3.9'] - ffmpeg-version: ['4.4.4', '5.1.4', '6.1.1', '7.0.1'] + ffmpeg-version: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] steps: - name: Check out repo uses: actions/checkout@v3 From f61971b0d4554f0791950fc9e035a28b454c2a07 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 6 Jul 2025 16:43:11 +0100 Subject: [PATCH 27/57] Try random things --- .github/workflows/windows_source_build.yaml | 35 ++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 15c3ae84b..1061c7abf 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -45,14 +45,26 @@ jobs: - name: Install FFmpeg run: | + # Install FFmpeg and its development headers conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge + + # Try to install FFmpeg development package if available + conda install ffmpeg-dev -c conda-forge || echo "ffmpeg-dev not available, continuing..." + ffmpeg -version # On Windows, ensure the conda Library/bin directory is in PATH # This is needed for both pkg-config to find FFmpeg and for DLL loading conda_env_path=$(conda info --base)/envs/build library_bin_path="$conda_env_path/Library/bin" - echo "Adding conda Library/bin to PATH: $library_bin_path" + library_lib_path="$conda_env_path/Library/lib" + library_include_path="$conda_env_path/Library/include" + + echo "Adding conda Library paths:" + echo " bin: $library_bin_path" + echo " lib: $library_lib_path" + echo " include: $library_include_path" + echo "$library_bin_path" >> $GITHUB_PATH # Also add Library/lib/pkgconfig to PKG_CONFIG_PATH for cmake to find FFmpeg @@ -60,6 +72,11 @@ jobs: echo "Adding to PKG_CONFIG_PATH: $pkg_config_path" echo "PKG_CONFIG_PATH=$pkg_config_path" >> $GITHUB_ENV + # Set additional environment variables that CMake might need + echo "CMAKE_PREFIX_PATH=$conda_env_path/Library" >> $GITHUB_ENV + echo "LIBRARY_PATH=$library_lib_path" >> $GITHUB_ENV + echo "INCLUDE_PATH=$library_include_path" >> $GITHUB_ENV + # Verify FFmpeg DLLs are accessible echo "Checking if FFmpeg DLLs are in PATH:" where avutil.dll || echo "avutil.dll not found in PATH" @@ -71,11 +88,23 @@ jobs: # Install pkg-config and pybind11 which are needed for Windows builds conda install -y pkg-config pybind11 -c conda-forge + # Install FFmpeg development dependencies that are needed for linking on Windows + conda install -y ffmpeg-dev harfbuzz-dev freetype-dev fontconfig-dev -c conda-forge || true + + # Alternative: install common FFmpeg dependencies individually + conda install -y harfbuzz freetype fontconfig expat libxml2 zlib libbz2 openssl -c conda-forge || true + # Verify pkg-config can find FFmpeg echo "Testing pkg-config for FFmpeg libraries:" pkg-config --exists libavcodec && echo "libavcodec found" || echo "libavcodec NOT found" pkg-config --exists libavformat && echo "libavformat found" || echo "libavformat NOT found" pkg-config --exists libavutil && echo "libavutil found" || echo "libavutil NOT found" + + # Show what libraries pkg-config thinks we need + echo "pkg-config --libs libavcodec:" + pkg-config --libs libavcodec || true + echo "pkg-config --libs libavformat:" + pkg-config --libs libavformat || true - name: Update pip and install PyTorch run: | @@ -86,6 +115,10 @@ jobs: run: | # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg echo "Building torchcodec from source using system FFmpeg..." + + # On Windows, we may need to disable some CMake flags that cause linking issues + export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON" + python -m pip install -e . --no-build-isolation -v - name: Test torchcodec import From 2f2a38aaba2f3d70b7aa9f347a1dd37757b39407 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 10:59:53 +0100 Subject: [PATCH 28/57] Add quiet flags --- .github/workflows/windows_source_build.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 1061c7abf..295adf438 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -46,10 +46,10 @@ jobs: - name: Install FFmpeg run: | # Install FFmpeg and its development headers - conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge + conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge --quiet # Try to install FFmpeg development package if available - conda install ffmpeg-dev -c conda-forge || echo "ffmpeg-dev not available, continuing..." + conda install ffmpeg-dev -c conda-forge --quiet || echo "ffmpeg-dev not available, continuing..." ffmpeg -version @@ -86,13 +86,13 @@ jobs: - name: Install build dependencies run: | # Install pkg-config and pybind11 which are needed for Windows builds - conda install -y pkg-config pybind11 -c conda-forge + conda install -y pkg-config pybind11 -c conda-forge --quiet # Install FFmpeg development dependencies that are needed for linking on Windows - conda install -y ffmpeg-dev harfbuzz-dev freetype-dev fontconfig-dev -c conda-forge || true + conda install -y ffmpeg-dev harfbuzz-dev freetype-dev fontconfig-dev -c conda-forge --quiet || true # Alternative: install common FFmpeg dependencies individually - conda install -y harfbuzz freetype fontconfig expat libxml2 zlib libbz2 openssl -c conda-forge || true + conda install -y harfbuzz freetype fontconfig expat libxml2 zlib libbz2 openssl -c conda-forge --quiet || true # Verify pkg-config can find FFmpeg echo "Testing pkg-config for FFmpeg libraries:" From c570edbeb53c41a85cc4c985536e4e596791b236 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 11:08:43 +0100 Subject: [PATCH 29/57] print dependencies of all libs --- .github/workflows/windows_source_build.yaml | 24 ++++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 295adf438..d8570e879 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -48,9 +48,6 @@ jobs: # Install FFmpeg and its development headers conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge --quiet - # Try to install FFmpeg development package if available - conda install ffmpeg-dev -c conda-forge --quiet || echo "ffmpeg-dev not available, continuing..." - ffmpeg -version # On Windows, ensure the conda Library/bin directory is in PATH @@ -88,11 +85,6 @@ jobs: # Install pkg-config and pybind11 which are needed for Windows builds conda install -y pkg-config pybind11 -c conda-forge --quiet - # Install FFmpeg development dependencies that are needed for linking on Windows - conda install -y ffmpeg-dev harfbuzz-dev freetype-dev fontconfig-dev -c conda-forge --quiet || true - - # Alternative: install common FFmpeg dependencies individually - conda install -y harfbuzz freetype fontconfig expat libxml2 zlib libbz2 openssl -c conda-forge --quiet || true # Verify pkg-config can find FFmpeg echo "Testing pkg-config for FFmpeg libraries:" @@ -103,9 +95,25 @@ jobs: # Show what libraries pkg-config thinks we need echo "pkg-config --libs libavcodec:" pkg-config --libs libavcodec || true + echo "pkg-config --libs libavformat:" pkg-config --libs libavformat || true + echo "pkg-config --libs libavutil:" + pkg-config --libs libavutil || true + + echo "pkg-config --libs libswscale:" + pkg-config --libs libswscale|| true + + pkg-config --libs libswresample|| true + echo "pkg-config --libs libswresample:" + + echo "pkg-config --libs libavfilter:" + pkg-config --libs libavfilter || true + + echo "pkg-config --libs libavdevice:" + pkg-config --libs libavdevice || true + - name: Update pip and install PyTorch run: | python -m pip install --upgrade pip From 821982f9254af84175d370b5df7a5025eb456362 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 11:28:19 +0100 Subject: [PATCH 30/57] Crazy stuff --- .github/workflows/windows_source_build.yaml | 40 +++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index d8570e879..14f4c9ca7 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -45,7 +45,7 @@ jobs: - name: Install FFmpeg run: | - # Install FFmpeg and its development headers + # Install FFmpeg and try to get development packages conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge --quiet ffmpeg -version @@ -73,6 +73,10 @@ jobs: echo "CMAKE_PREFIX_PATH=$conda_env_path/Library" >> $GITHUB_ENV echo "LIBRARY_PATH=$library_lib_path" >> $GITHUB_ENV echo "INCLUDE_PATH=$library_include_path" >> $GITHUB_ENV + + # Add the Library/lib directory to the linker path for Visual Studio + echo "LIB=$library_lib_path;$LIB" >> $GITHUB_ENV + echo "LIBPATH=$library_lib_path;$LIBPATH" >> $GITHUB_ENV # Verify FFmpeg DLLs are accessible echo "Checking if FFmpeg DLLs are in PATH:" @@ -85,6 +89,23 @@ jobs: # Install pkg-config and pybind11 which are needed for Windows builds conda install -y pkg-config pybind11 -c conda-forge --quiet + # Inspect what's actually in the Library directory + echo "Contents of Library/lib:" + ls -la "$CONDA_PREFIX/Library/lib/" | head -20 + echo "Looking for harfbuzz files:" + find "$CONDA_PREFIX/Library/" -name "*harfbuzz*" 2>/dev/null || echo "No harfbuzz files found" + echo "Looking for freetype files:" + find "$CONDA_PREFIX/Library/" -name "*freetype*" 2>/dev/null || echo "No freetype files found" + + # Check if the .lib files exist with the exact names pkg-config expects + conda_env_path=$(conda info --base)/envs/build + library_lib_path="$conda_env_path/Library/lib" + + echo "Checking for specific .lib files in $library_lib_path:" + ls -la "$library_lib_path/harfbuzz.lib" 2>/dev/null || echo "harfbuzz.lib not found" + ls -la "$library_lib_path/freetype.lib" 2>/dev/null || echo "freetype.lib not found" + ls -la "$library_lib_path/"*harfbuzz* 2>/dev/null || echo "No harfbuzz files in lib" + ls -la "$library_lib_path/"*freetype* 2>/dev/null || echo "No freetype files in lib" # Verify pkg-config can find FFmpeg echo "Testing pkg-config for FFmpeg libraries:" @@ -124,9 +145,24 @@ jobs: # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg echo "Building torchcodec from source using system FFmpeg..." - # On Windows, we may need to disable some CMake flags that cause linking issues + # On Windows, try to work around the dependency linking issues + # by setting CMAKE_ARGS to use dynamic linking where possible export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON" + # Try to create symlinks for missing .lib files if the DLLs exist + conda_env_path=$(conda info --base)/envs/build + library_lib_path="$conda_env_path/Library/lib" + library_bin_path="$conda_env_path/Library/bin" + + # Check if we can create import libraries from DLLs for missing dependencies + if [ ! -f "$library_lib_path/harfbuzz.lib" ] && [ -f "$library_bin_path/harfbuzz.dll" ]; then + echo "Attempting to create harfbuzz.lib from harfbuzz.dll" + # This is a workaround - we'll try to use lib.exe to create an import library + # lib.exe /def:harfbuzz.def /out:harfbuzz.lib /machine:x64 + # But this requires a .def file which we don't have + echo "harfbuzz.lib missing but harfbuzz.dll exists" + fi + python -m pip install -e . --no-build-isolation -v - name: Test torchcodec import From af7a98af28c2bc9df3d7fc1e609179129deff9bc Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 11:52:53 +0100 Subject: [PATCH 31/57] crazy cmake stuff --- .github/workflows/windows_source_build.yaml | 7 ++- src/torchcodec/_core/CMakeLists.txt | 60 +++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 14f4c9ca7..868dd64f8 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -147,7 +147,12 @@ jobs: # On Windows, try to work around the dependency linking issues # by setting CMAKE_ARGS to use dynamic linking where possible - export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON" + conda_env_path=$(conda info --base)/envs/build + library_lib_path="$conda_env_path/Library/lib" + + export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=$library_lib_path" + + echo "CMAKE_ARGS: $CMAKE_ARGS" # Try to create symlinks for missing .lib files if the DLLs exist conda_env_path=$(conda info --base)/envs/build diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 1a665b955..54411add3 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -65,6 +65,12 @@ function(make_torchcodec_sublibrary PUBLIC ${library_dependencies} ) + + # Debug output for Windows builds + if(WIN32) + message(STATUS "Created library: ${library_name}") + message(STATUS "Library dependencies: ${library_dependencies}") + endif() endfunction() function(make_torchcodec_libraries @@ -198,6 +204,45 @@ function(make_torchcodec_libraries ${pybind_ops_library_name} ) + # The install step is invoked within CMakeBuild.build_library() in + # setup.py and just copies the built files from the temp + # cmake/setuptools build folder into the CMAKE_INSTALL_PREFIX folder. We + # still need to manually pass "DESTINATION ..." for cmake to copy those + # files in CMAKE_INSTALL_PREFIX instead of CMAKE_INSTALL_PREFIX/lib. + install( + TARGETS ${all_libraries} + LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} + RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs +ibrary to prevent + # stray initialization of py::objects. The rest of the object code must + # match. See: + # https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes + if(NOT WIN32) # TODO unify WIN detection + target_compile_options( + ${pybind_ops_library_name} + PUBLIC + "-fvisibility=hidden" + ) + endif() + if(APPLE) # TODO unify APPLE detection + # If we don't make sure this flag is set, we run into segfauls at import + # time on Mac. See: + # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 + target_link_options( + ${pybind_ops_library_name} + PUBLIC + "LINKER:-undefined,dynamic_lookup" + ) + endif() + + # Install all libraries. + set( + all_libraries + ${decoder_library_name} + ${custom_ops_library_name} + ${pybind_ops_library_name} + ) + # The install step is invoked within CMakeBuild.build_library() in # setup.py and just copies the built files from the temp # cmake/setuptools build folder into the CMAKE_INSTALL_PREFIX folder. We @@ -246,6 +291,21 @@ else() libswscale ) + # On Windows, explicitly add the conda library directory to the linker search path + if(WIN32 AND DEFINED ENV{CONDA_PREFIX}) + link_directories("$ENV{CONDA_PREFIX}/Library/lib") + message(STATUS "Added conda library directory to linker search path: $ENV{CONDA_PREFIX}/Library/lib") + + # Also add the library path to CMAKE_PREFIX_PATH for better library finding + list(APPEND CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX}/Library") + message(STATUS "Added conda prefix to CMAKE_PREFIX_PATH: $ENV{CONDA_PREFIX}/Library") + + # Debug: Show what LIBAV variables are set + message(STATUS "LIBAV_INCLUDE_DIRS: ${LIBAV_INCLUDE_DIRS}") + message(STATUS "LIBAV_LIBRARY_DIRS: ${LIBAV_LIBRARY_DIRS}") + message(STATUS "LIBAV_LIBRARIES: ${LIBAV_LIBRARIES}") + endif() + # Split libavcodec's version string by '.' and convert it to a list string(REPLACE "." ";" libavcodec_version_list ${LIBAV_libavcodec_VERSION}) # Get the first element of the list, which is the major version From 1413666da05ff8bf8a94b526782b8a8a017da6b9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 12:00:23 +0100 Subject: [PATCH 32/57] ???? --- .github/workflows/windows_source_build.yaml | 76 ++++++++++++++------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index 868dd64f8..d70bcb077 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -89,6 +89,10 @@ jobs: # Install pkg-config and pybind11 which are needed for Windows builds conda install -y pkg-config pybind11 -c conda-forge --quiet + # Try to install missing dependencies that FFmpeg might need + echo "Installing additional dependencies that might be missing..." + conda install -y harfbuzz freetype zlib bzip2 -c conda-forge --quiet || echo "Some dependencies might already be installed" + # Inspect what's actually in the Library directory echo "Contents of Library/lib:" ls -la "$CONDA_PREFIX/Library/lib/" | head -20 @@ -124,10 +128,10 @@ jobs: pkg-config --libs libavutil || true echo "pkg-config --libs libswscale:" - pkg-config --libs libswscale|| true + pkg-config --libs libswscale || true - pkg-config --libs libswresample|| true echo "pkg-config --libs libswresample:" + pkg-config --libs libswresample || true echo "pkg-config --libs libavfilter:" pkg-config --libs libavfilter || true @@ -140,35 +144,59 @@ jobs: python -m pip install --upgrade pip python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Build torchcodec from source + - name: Create Windows build script run: | - # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg - echo "Building torchcodec from source using system FFmpeg..." + # Create a build script that sets up the VS environment and runs the build + cat > build_with_vs.bat << 'EOF' + @echo off - # On Windows, try to work around the dependency linking issues - # by setting CMAKE_ARGS to use dynamic linking where possible - conda_env_path=$(conda info --base)/envs/build - library_lib_path="$conda_env_path/Library/lib" + :: Get conda environment info + for /f "tokens=*" %%i in ('conda info --base') do set CONDA_BASE=%%i + set CONDA_ENV_PATH=%CONDA_BASE%\envs\build + set LIBRARY_LIB_PATH=%CONDA_ENV_PATH%\Library\lib + set LIBRARY_BIN_PATH=%CONDA_ENV_PATH%\Library\bin + set LIBRARY_INCLUDE_PATH=%CONDA_ENV_PATH%\Library\include - export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=$library_lib_path" + :: Add conda Library/bin to PATH for DLL loading + set PATH=%LIBRARY_BIN_PATH%;%PATH% - echo "CMAKE_ARGS: $CMAKE_ARGS" + :: Set PKG_CONFIG_PATH + set PKG_CONFIG_PATH=%CONDA_ENV_PATH%\Library\lib\pkgconfig - # Try to create symlinks for missing .lib files if the DLLs exist - conda_env_path=$(conda info --base)/envs/build - library_lib_path="$conda_env_path/Library/lib" - library_bin_path="$conda_env_path/Library/bin" + :: Set additional paths for the linker + set LIB=%LIBRARY_LIB_PATH%;%LIB% + set LIBPATH=%LIBRARY_LIB_PATH%;%LIBPATH% + set INCLUDE=%LIBRARY_INCLUDE_PATH%;%INCLUDE% - # Check if we can create import libraries from DLLs for missing dependencies - if [ ! -f "$library_lib_path/harfbuzz.lib" ] && [ -f "$library_bin_path/harfbuzz.dll" ]; then - echo "Attempting to create harfbuzz.lib from harfbuzz.dll" - # This is a workaround - we'll try to use lib.exe to create an import library - # lib.exe /def:harfbuzz.def /out:harfbuzz.lib /machine:x64 - # But this requires a .def file which we don't have - echo "harfbuzz.lib missing but harfbuzz.dll exists" - fi + :: Set CMAKE_ARGS with more comprehensive settings + set CMAKE_ARGS=-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=%LIBRARY_LIB_PATH% -DCMAKE_INCLUDE_PATH=%LIBRARY_INCLUDE_PATH% -DCMAKE_PREFIX_PATH=%LIBRARY_LIB_PATH%;%LIBRARY_INCLUDE_PATH% -DCMAKE_VERBOSE_MAKEFILE=ON + + echo Building with CMAKE_ARGS: %CMAKE_ARGS% + echo LIB path: %LIB% + echo PKG_CONFIG_PATH: %PKG_CONFIG_PATH% + echo PATH (first few entries): + echo %PATH% | cut -d; -f1-3 + + :: Debug: Show what libraries are available + echo Available .lib files in conda lib directory: + dir "%LIBRARY_LIB_PATH%\*.lib" 2>nul || echo No .lib files found + + :: Debug: Test pkg-config within the script + echo Testing pkg-config from within build script: + pkg-config --exists libavcodec && echo libavcodec found || echo libavcodec NOT found + + :: Use vc_env_helper.bat to set up VS environment and build + call packaging\vc_env_helper.bat python -m pip install -e . --no-build-isolation -v + EOF + + - name: Verify Visual Studio environment and build + shell: cmd + run: | + echo Checking Visual Studio installation... + call packaging\vc_env_helper.bat echo Visual Studio environment setup complete - python -m pip install -e . --no-build-isolation -v + echo Running build with Visual Studio environment... + call build_with_vs.bat - name: Test torchcodec import run: | From eb8b31824c8850fc50f718a77d83b8cbad966b8c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 12:38:15 +0100 Subject: [PATCH 33/57] ??? --- .github/workflows/windows_source_build.yaml | 85 +++++++++------------ 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml index d70bcb077..04c359251 100644 --- a/.github/workflows/windows_source_build.yaml +++ b/.github/workflows/windows_source_build.yaml @@ -144,59 +144,42 @@ jobs: python -m pip install --upgrade pip python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Create Windows build script - run: | - # Create a build script that sets up the VS environment and runs the build - cat > build_with_vs.bat << 'EOF' - @echo off - - :: Get conda environment info - for /f "tokens=*" %%i in ('conda info --base') do set CONDA_BASE=%%i - set CONDA_ENV_PATH=%CONDA_BASE%\envs\build - set LIBRARY_LIB_PATH=%CONDA_ENV_PATH%\Library\lib - set LIBRARY_BIN_PATH=%CONDA_ENV_PATH%\Library\bin - set LIBRARY_INCLUDE_PATH=%CONDA_ENV_PATH%\Library\include - - :: Add conda Library/bin to PATH for DLL loading - set PATH=%LIBRARY_BIN_PATH%;%PATH% - - :: Set PKG_CONFIG_PATH - set PKG_CONFIG_PATH=%CONDA_ENV_PATH%\Library\lib\pkgconfig - - :: Set additional paths for the linker - set LIB=%LIBRARY_LIB_PATH%;%LIB% - set LIBPATH=%LIBRARY_LIB_PATH%;%LIBPATH% - set INCLUDE=%LIBRARY_INCLUDE_PATH%;%INCLUDE% - - :: Set CMAKE_ARGS with more comprehensive settings - set CMAKE_ARGS=-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=%LIBRARY_LIB_PATH% -DCMAKE_INCLUDE_PATH=%LIBRARY_INCLUDE_PATH% -DCMAKE_PREFIX_PATH=%LIBRARY_LIB_PATH%;%LIBRARY_INCLUDE_PATH% -DCMAKE_VERBOSE_MAKEFILE=ON - - echo Building with CMAKE_ARGS: %CMAKE_ARGS% - echo LIB path: %LIB% - echo PKG_CONFIG_PATH: %PKG_CONFIG_PATH% - echo PATH (first few entries): - echo %PATH% | cut -d; -f1-3 - - :: Debug: Show what libraries are available - echo Available .lib files in conda lib directory: - dir "%LIBRARY_LIB_PATH%\*.lib" 2>nul || echo No .lib files found - - :: Debug: Test pkg-config within the script - echo Testing pkg-config from within build script: - pkg-config --exists libavcodec && echo libavcodec found || echo libavcodec NOT found - - :: Use vc_env_helper.bat to set up VS environment and build - call packaging\vc_env_helper.bat python -m pip install -e . --no-build-isolation -v - EOF - - - name: Verify Visual Studio environment and build - shell: cmd + - name: Setup Visual Studio environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Build torchcodec from source run: | - echo Checking Visual Studio installation... - call packaging\vc_env_helper.bat echo Visual Studio environment setup complete + # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg + echo "Building torchcodec from source using system FFmpeg..." + + # Verify that torch is available + python -c "import torch; print('PyTorch found:', torch.__version__)" + + # Set up environment variables for the build + conda_env_path=$(conda info --base)/envs/build + library_lib_path="$conda_env_path/Library/lib" + library_bin_path="$conda_env_path/Library/bin" + library_include_path="$conda_env_path/Library/include" + + # Set CMAKE_ARGS with proper library paths + export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=$library_lib_path -DCMAKE_INCLUDE_PATH=$library_include_path -DCMAKE_PREFIX_PATH=$library_lib_path;$library_include_path -DCMAKE_VERBOSE_MAKEFILE=ON" + + echo "CMAKE_ARGS: $CMAKE_ARGS" + echo "Python executable: $(which python)" + echo "Conda environment: $CONDA_PREFIX" + + # Debug: Show what libraries are available + echo "Available .lib files in conda lib directory:" + ls -la "$library_lib_path"/*.lib 2>/dev/null || echo "No .lib files found" + + # Debug: Test that torch is still available + echo "Testing PyTorch availability:" + python -c "import torch; print('PyTorch version:', torch.__version__)" - echo Running build with Visual Studio environment... - call build_with_vs.bat + # Build using pip install with the current conda environment + python -m pip install -e . --no-build-isolation -v - name: Test torchcodec import run: | From edca0c371b7f9e53090692e88e82812140b3270a Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 7 Jul 2025 12:56:25 +0100 Subject: [PATCH 34/57] typo --- src/torchcodec/_core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 54411add3..0facc37cf 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -213,7 +213,7 @@ function(make_torchcodec_libraries TARGETS ${all_libraries} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs -ibrary to prevent + ) # stray initialization of py::objects. The rest of the object code must # match. See: # https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes From fd1d8b905a8979f0a97f8e8b7327d6f54c91218e Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:19:06 +0100 Subject: [PATCH 35/57] Remove some stuff --- .github/workflows/cpp_tests.yaml | 136 ++++++------ .github/workflows/docs.yaml | 218 ++++++++++---------- .github/workflows/lint.yaml | 142 ++++++------- .github/workflows/windows_source_build.yaml | 196 ------------------ 4 files changed, 248 insertions(+), 444 deletions(-) delete mode 100644 .github/workflows/windows_source_build.yaml diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index 44dd288d6..227e40e12 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -1,73 +1,73 @@ -# name: CPP tests +name: CPP tests -# on: -# push: -# branches: [ main ] -# pull_request: +on: + push: + branches: [ main ] + # pull_request: -# concurrency: -# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} -# cancel-in-progress: true +concurrency: + group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} + cancel-in-progress: true -# defaults: -# run: -# shell: bash -l -eo pipefail {0} +defaults: + run: + shell: bash -l -eo pipefail {0} -# jobs: -# Cpp-tests: -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] -# steps: -# - name: Check out repo -# uses: actions/checkout@v3 -# - name: Setup conda env -# uses: conda-incubator/setup-miniconda@v3 -# with: -# auto-update-conda: true -# # Using miniforge instead of miniconda ensures that the default -# # conda channel is conda-forge instead of main/default. This ensures -# # ABI consistency between dependencies: -# # https://conda-forge.org/docs/user/transitioning_from_defaults/ -# miniforge-version: latest -# activate-environment: test -# python-version: '3.12' -# - name: Update pip -# run: python -m pip install --upgrade pip -# - name: Install torch dependencies -# run: | -# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -# - name: Install ffmpeg, pkg-config and pybind11 -# run: | -# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config pybind11 -c conda-forge -# ffmpeg -version -# - name: Build and run C++ tests -# run: | -# # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so -# # we're building libtorchcodec against the installed FFmpeg version -# # (from conda-forge) instead of building against our pre-built non-GPL -# # FFmpeg libraries. -# # The reason we need this is because the C++ tests decode x264 files. -# # x264 support is not LGPL, os it is not supported by our -# # pre-built non-GPL FFmpeg libraries. And if we were to build against -# # those, this is also what the tests would be loading at run time, -# # then failing when we try to decode x264. -# # To remediate that, we build against the FFmpeg that we installed -# # from conda-forge (which is able to decode x264), and that's also -# # what gets loaded at run time. -# # The Python tests are also decoding x264 files, and are built against -# # our non-GPL FFmpeg. And yet they pass. This is because in Python -# # we're able to distinguish between build-time (non-GPL FFmpeg) and -# # run time (conda-forge FFmpeg). +jobs: + Cpp-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + steps: + - name: Check out repo + uses: actions/checkout@v3 + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + # Using miniforge instead of miniconda ensures that the default + # conda channel is conda-forge instead of main/default. This ensures + # ABI consistency between dependencies: + # https://conda-forge.org/docs/user/transitioning_from_defaults/ + miniforge-version: latest + activate-environment: test + python-version: '3.12' + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install torch dependencies + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install ffmpeg, pkg-config and pybind11 + run: | + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config pybind11 -c conda-forge + ffmpeg -version + - name: Build and run C++ tests + run: | + # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so + # we're building libtorchcodec against the installed FFmpeg version + # (from conda-forge) instead of building against our pre-built non-GPL + # FFmpeg libraries. + # The reason we need this is because the C++ tests decode x264 files. + # x264 support is not LGPL, os it is not supported by our + # pre-built non-GPL FFmpeg libraries. And if we were to build against + # those, this is also what the tests would be loading at run time, + # then failing when we try to decode x264. + # To remediate that, we build against the FFmpeg that we installed + # from conda-forge (which is able to decode x264), and that's also + # what gets loaded at run time. + # The Python tests are also decoding x264 files, and are built against + # our non-GPL FFmpeg. And yet they pass. This is because in Python + # we're able to distinguish between build-time (non-GPL FFmpeg) and + # run time (conda-forge FFmpeg). -# build_tests_dir="${PWD}/build_tests" -# mkdir $build_tests_dir -# pushd $build_tests_dir -# TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") -# Torch_DIR="${TORCH_PATH}/share/cmake/Torch" -# cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON -# cmake --build . -# ctest --output-on-failure -# popd + build_tests_dir="${PWD}/build_tests" + mkdir $build_tests_dir + pushd $build_tests_dir + TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") + Torch_DIR="${TORCH_PATH}/share/cmake/Torch" + cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON + cmake --build . + ctest --output-on-failure + popd diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 39aaec1a1..e042ab75e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,116 +1,116 @@ -# name: Docs +name: Docs -# on: -# push: -# branches: [ main ] -# pull_request: +on: + push: + branches: [ main ] + # pull_request: -# permissions: -# id-token: write -# contents: write +permissions: + id-token: write + contents: write -# defaults: -# run: -# shell: bash -l -eo pipefail {0} +defaults: + run: + shell: bash -l -eo pipefail {0} -# jobs: -# generate-matrix: -# uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main -# with: -# package-type: wheel -# os: linux -# test-infra-repository: pytorch/test-infra -# test-infra-ref: main -# with-cpu: disable -# with-xpu: disable -# with-rocm: disable -# with-cuda: enable -# build-python-only: "disable" -# build: -# needs: generate-matrix -# strategy: -# fail-fast: false -# name: Build and Upload wheel -# uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main -# with: -# repository: pytorch/torchcodec -# ref: "" -# test-infra-repository: pytorch/test-infra -# test-infra-ref: main -# build-matrix: ${{ needs.generate-matrix.outputs.matrix }} -# pre-script: packaging/pre_build_script.sh -# post-script: packaging/post_build_script.sh -# smoke-test-script: packaging/fake_smoke_test.py -# package-name: torchcodec -# trigger-event: ${{ github.event_name }} -# build-platform: "python-build-package" -# build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cpu: disable + with-xpu: disable + with-rocm: disable + with-cuda: enable + build-python-only: "disable" + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" -# build-docs: -# runs-on: linux.4xlarge.nvidia.gpu -# strategy: -# fail-fast: false -# matrix: -# # 3.9 corresponds to the minimum python version for which we build -# # the wheel unless the label cliflow/binaries/all is present in the -# # PR. -# python-version: ['3.9'] -# cuda-version: ['12.6'] -# ffmpeg-version-for-tests: ['7'] -# container: -# image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" -# options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" -# needs: build -# steps: -# - name: Setup env vars -# run: | -# cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') -# echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV -# python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') -# echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV -# - uses: actions/download-artifact@v4 -# with: -# name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 -# path: pytorch/torchcodec/dist/ -# - name: Setup miniconda using test-infra -# uses: pytorch/test-infra/.github/actions/setup-miniconda@main -# with: -# python-version: ${{ matrix.python-version }} -# # We install conda packages at the start because otherwise conda may have conflicts with dependencies. -# default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -# - name: Check env -# run: | -# ${CONDA_RUN} env -# ${CONDA_RUN} conda info -# ${CONDA_RUN} nvidia-smi -# ${CONDA_RUN} conda list -# - name: Assert ffmpeg exists -# run: | -# ${CONDA_RUN} ffmpeg -buildconf -# - name: Update pip -# run: ${CONDA_RUN} python -m pip install --upgrade pip -# - name: Install PyTorch -# run: | -# ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} -# ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' -# - name: Install torchcodec from the wheel -# run: | -# wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` -# echo Installing $wheel_path -# ${CONDA_RUN} python -m pip install $wheel_path -vvv + build-docs: + runs-on: linux.4xlarge.nvidia.gpu + strategy: + fail-fast: false + matrix: + # 3.9 corresponds to the minimum python version for which we build + # the wheel unless the label cliflow/binaries/all is present in the + # PR. + python-version: ['3.9'] + cuda-version: ['12.6'] + ffmpeg-version-for-tests: ['7'] + container: + image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" + options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" + needs: build + steps: + - name: Setup env vars + run: | + cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') + echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV + python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') + echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV + - uses: actions/download-artifact@v4 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 + path: pytorch/torchcodec/dist/ + - name: Setup miniconda using test-infra + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + with: + python-version: ${{ matrix.python-version }} + # We install conda packages at the start because otherwise conda may have conflicts with dependencies. + default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" + - name: Check env + run: | + ${CONDA_RUN} env + ${CONDA_RUN} conda info + ${CONDA_RUN} nvidia-smi + ${CONDA_RUN} conda list + - name: Assert ffmpeg exists + run: | + ${CONDA_RUN} ffmpeg -buildconf + - name: Update pip + run: ${CONDA_RUN} python -m pip install --upgrade pip + - name: Install PyTorch + run: | + ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} + ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` + echo Installing $wheel_path + ${CONDA_RUN} python -m pip install $wheel_path -vvv -# - name: Check out repo -# uses: actions/checkout@v3 + - name: Check out repo + uses: actions/checkout@v3 -# - name: Install doc dependencies -# run: | -# cd docs -# ${CONDA_RUN} python -m pip install -r requirements.txt -# - name: Build docs -# run: | -# cd docs -# ${CONDA_RUN} make html -# - uses: actions/upload-artifact@v4 -# with: -# name: Built-Docs -# path: docs/build/html/ + - name: Install doc dependencies + run: | + cd docs + ${CONDA_RUN} python -m pip install -r requirements.txt + - name: Build docs + run: | + cd docs + ${CONDA_RUN} make html + - uses: actions/upload-artifact@v4 + with: + name: Built-Docs + path: docs/build/html/ diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 6ed79d88c..456bc2126 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,76 +1,76 @@ -# name: Lint +name: Lint -# on: -# push: -# branches: [ main ] -# pull_request: +on: + push: + branches: [ main ] + # pull_request: -# concurrency: -# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} -# cancel-in-progress: true +concurrency: + group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} + cancel-in-progress: true -# defaults: -# run: -# shell: bash -l -eo pipefail {0} +defaults: + run: + shell: bash -l -eo pipefail {0} -# jobs: -# pre-commit-checks: -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# python-version: ['3.12'] -# steps: -# - name: Check out repo -# uses: actions/checkout@v3 -# - name: Setup conda env -# uses: conda-incubator/setup-miniconda@v2 -# with: -# auto-update-conda: true -# miniconda-version: "latest" -# activate-environment: test -# python-version: ${{ matrix.python-version }} -# - name: Update pip -# run: python -m pip install --upgrade pip -# - name: Install pre-commit -# run: | -# python -m pip install pre-commit -# - name: Run pre-commit checks -# run: | -# pre-commit run --all-files -# - name: Check to see what files pre-commit modified -# run: | -# git diff +jobs: + pre-commit-checks: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.12'] + steps: + - name: Check out repo + uses: actions/checkout@v3 + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install pre-commit + run: | + python -m pip install pre-commit + - name: Run pre-commit checks + run: | + pre-commit run --all-files + - name: Check to see what files pre-commit modified + run: | + git diff -# mypy: -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# python-version: ['3.12'] -# steps: -# - name: Check out repo -# uses: actions/checkout@v3 -# - name: Setup conda env -# uses: conda-incubator/setup-miniconda@v2 -# with: -# auto-update-conda: true -# miniconda-version: "latest" -# activate-environment: test -# python-version: ${{ matrix.python-version }} -# - name: Update pip -# run: python -m pip install --upgrade pip -# - name: Install dependencies and FFmpeg -# run: | -# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -# conda install "ffmpeg=7.0.1" pkg-config pybind11 -c conda-forge -# ffmpeg -version -# - name: Build and install torchcodec -# run: | -# python -m pip install -e ".[dev]" --no-build-isolation -vvv -# - name: Install mypy -# run: | -# python -m pip install mypy -# - name: Run mypy -# run: | -# mypy --install-types --non-interactive --config-file mypy.ini + mypy: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.12'] + steps: + - name: Check out repo + uses: actions/checkout@v3 + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install dependencies and FFmpeg + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + conda install "ffmpeg=7.0.1" pkg-config pybind11 -c conda-forge + ffmpeg -version + - name: Build and install torchcodec + run: | + python -m pip install -e ".[dev]" --no-build-isolation -vvv + - name: Install mypy + run: | + python -m pip install mypy + - name: Run mypy + run: | + mypy --install-types --non-interactive --config-file mypy.ini diff --git a/.github/workflows/windows_source_build.yaml b/.github/workflows/windows_source_build.yaml deleted file mode 100644 index 04c359251..000000000 --- a/.github/workflows/windows_source_build.yaml +++ /dev/null @@ -1,196 +0,0 @@ -name: Build and test Windows from source - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -defaults: - run: - shell: bash -l -eo pipefail {0} - -jobs: - build-and-test: - runs-on: windows-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.9'] - ffmpeg-version: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: build - python-version: ${{ matrix.python-version }} - - - name: Install FFmpeg - run: | - # Install FFmpeg and try to get development packages - conda install "ffmpeg=${{ matrix.ffmpeg-version }}" -c conda-forge --quiet - - ffmpeg -version - - # On Windows, ensure the conda Library/bin directory is in PATH - # This is needed for both pkg-config to find FFmpeg and for DLL loading - conda_env_path=$(conda info --base)/envs/build - library_bin_path="$conda_env_path/Library/bin" - library_lib_path="$conda_env_path/Library/lib" - library_include_path="$conda_env_path/Library/include" - - echo "Adding conda Library paths:" - echo " bin: $library_bin_path" - echo " lib: $library_lib_path" - echo " include: $library_include_path" - - echo "$library_bin_path" >> $GITHUB_PATH - - # Also add Library/lib/pkgconfig to PKG_CONFIG_PATH for cmake to find FFmpeg - pkg_config_path="$conda_env_path/Library/lib/pkgconfig" - echo "Adding to PKG_CONFIG_PATH: $pkg_config_path" - echo "PKG_CONFIG_PATH=$pkg_config_path" >> $GITHUB_ENV - - # Set additional environment variables that CMake might need - echo "CMAKE_PREFIX_PATH=$conda_env_path/Library" >> $GITHUB_ENV - echo "LIBRARY_PATH=$library_lib_path" >> $GITHUB_ENV - echo "INCLUDE_PATH=$library_include_path" >> $GITHUB_ENV - - # Add the Library/lib directory to the linker path for Visual Studio - echo "LIB=$library_lib_path;$LIB" >> $GITHUB_ENV - echo "LIBPATH=$library_lib_path;$LIBPATH" >> $GITHUB_ENV - - # Verify FFmpeg DLLs are accessible - echo "Checking if FFmpeg DLLs are in PATH:" - where avutil.dll || echo "avutil.dll not found in PATH" - where avcodec.dll || echo "avcodec.dll not found in PATH" - where avformat.dll || echo "avformat.dll not found in PATH" - - - name: Install build dependencies - run: | - # Install pkg-config and pybind11 which are needed for Windows builds - conda install -y pkg-config pybind11 -c conda-forge --quiet - - # Try to install missing dependencies that FFmpeg might need - echo "Installing additional dependencies that might be missing..." - conda install -y harfbuzz freetype zlib bzip2 -c conda-forge --quiet || echo "Some dependencies might already be installed" - - # Inspect what's actually in the Library directory - echo "Contents of Library/lib:" - ls -la "$CONDA_PREFIX/Library/lib/" | head -20 - echo "Looking for harfbuzz files:" - find "$CONDA_PREFIX/Library/" -name "*harfbuzz*" 2>/dev/null || echo "No harfbuzz files found" - echo "Looking for freetype files:" - find "$CONDA_PREFIX/Library/" -name "*freetype*" 2>/dev/null || echo "No freetype files found" - - # Check if the .lib files exist with the exact names pkg-config expects - conda_env_path=$(conda info --base)/envs/build - library_lib_path="$conda_env_path/Library/lib" - - echo "Checking for specific .lib files in $library_lib_path:" - ls -la "$library_lib_path/harfbuzz.lib" 2>/dev/null || echo "harfbuzz.lib not found" - ls -la "$library_lib_path/freetype.lib" 2>/dev/null || echo "freetype.lib not found" - ls -la "$library_lib_path/"*harfbuzz* 2>/dev/null || echo "No harfbuzz files in lib" - ls -la "$library_lib_path/"*freetype* 2>/dev/null || echo "No freetype files in lib" - - # Verify pkg-config can find FFmpeg - echo "Testing pkg-config for FFmpeg libraries:" - pkg-config --exists libavcodec && echo "libavcodec found" || echo "libavcodec NOT found" - pkg-config --exists libavformat && echo "libavformat found" || echo "libavformat NOT found" - pkg-config --exists libavutil && echo "libavutil found" || echo "libavutil NOT found" - - # Show what libraries pkg-config thinks we need - echo "pkg-config --libs libavcodec:" - pkg-config --libs libavcodec || true - - echo "pkg-config --libs libavformat:" - pkg-config --libs libavformat || true - - echo "pkg-config --libs libavutil:" - pkg-config --libs libavutil || true - - echo "pkg-config --libs libswscale:" - pkg-config --libs libswscale || true - - echo "pkg-config --libs libswresample:" - pkg-config --libs libswresample || true - - echo "pkg-config --libs libavfilter:" - pkg-config --libs libavfilter || true - - echo "pkg-config --libs libavdevice:" - pkg-config --libs libavdevice || true - - - name: Update pip and install PyTorch - run: | - python -m pip install --upgrade pip - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - - name: Setup Visual Studio environment - uses: ilammy/msvc-dev-cmd@v1 - with: - arch: x64 - - - name: Build torchcodec from source - run: | - # Build without BUILD_AGAINST_ALL_FFMPEG_FROM_S3 to use system FFmpeg - echo "Building torchcodec from source using system FFmpeg..." - - # Verify that torch is available - python -c "import torch; print('PyTorch found:', torch.__version__)" - - # Set up environment variables for the build - conda_env_path=$(conda info --base)/envs/build - library_lib_path="$conda_env_path/Library/lib" - library_bin_path="$conda_env_path/Library/bin" - library_include_path="$conda_env_path/Library/include" - - # Set CMAKE_ARGS with proper library paths - export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DTORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_LIBRARY_PATH=$library_lib_path -DCMAKE_INCLUDE_PATH=$library_include_path -DCMAKE_PREFIX_PATH=$library_lib_path;$library_include_path -DCMAKE_VERBOSE_MAKEFILE=ON" - - echo "CMAKE_ARGS: $CMAKE_ARGS" - echo "Python executable: $(which python)" - echo "Conda environment: $CONDA_PREFIX" - - # Debug: Show what libraries are available - echo "Available .lib files in conda lib directory:" - ls -la "$library_lib_path"/*.lib 2>/dev/null || echo "No .lib files found" - - # Debug: Test that torch is still available - echo "Testing PyTorch availability:" - python -c "import torch; print('PyTorch version:', torch.__version__)" - - # Build using pip install with the current conda environment - python -m pip install -e . --no-build-isolation -v - - - name: Test torchcodec import - run: | - echo "Testing torchcodec import..." - python -c "import torchcodec; print('TorchCodec import successful!')" - python -c "import torchcodec; print('FFmpeg versions:', torchcodec.get_ffmpeg_library_versions())" - - - name: Install test dependencies - run: | - python -m pip install numpy pytest pillow - - - name: Run Python tests - run: | - pytest test -v From 775753e4891a03e634b9c1258d70f183e7f3b734 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:23:14 +0100 Subject: [PATCH 36/57] Fixed botched merge? --- src/torchcodec/_core/CMakeLists.txt | 53 ----------------------------- 1 file changed, 53 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index ba2fe40e6..fbcc13af7 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -222,45 +222,7 @@ function(make_torchcodec_libraries LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs ) - # stray initialization of py::objects. The rest of the object code must - # match. See: - # https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes - if(NOT WIN32) # TODO unify WIN detection - target_compile_options( - ${pybind_ops_library_name} - PUBLIC - "-fvisibility=hidden" - ) - endif() - if(APPLE) # TODO unify APPLE detection - # If we don't make sure this flag is set, we run into segfauls at import - # time on Mac. See: - # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 - target_link_options( - ${pybind_ops_library_name} - PUBLIC - "LINKER:-undefined,dynamic_lookup" - ) - endif() - # Install all libraries. - set( - all_libraries - ${decoder_library_name} - ${custom_ops_library_name} - ${pybind_ops_library_name} - ) - - # The install step is invoked within CMakeBuild.build_library() in - # setup.py and just copies the built files from the temp - # cmake/setuptools build folder into the CMAKE_INSTALL_PREFIX folder. We - # still need to manually pass "DESTINATION ..." for cmake to copy those - # files in CMAKE_INSTALL_PREFIX instead of CMAKE_INSTALL_PREFIX/lib. - install( - TARGETS ${all_libraries} - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs - ) endfunction() if(DEFINED ENV{BUILD_AGAINST_ALL_FFMPEG_FROM_S3}) @@ -299,21 +261,6 @@ else() libswscale ) - # On Windows, explicitly add the conda library directory to the linker search path - if(WIN32 AND DEFINED ENV{CONDA_PREFIX}) - link_directories("$ENV{CONDA_PREFIX}/Library/lib") - message(STATUS "Added conda library directory to linker search path: $ENV{CONDA_PREFIX}/Library/lib") - - # Also add the library path to CMAKE_PREFIX_PATH for better library finding - list(APPEND CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX}/Library") - message(STATUS "Added conda prefix to CMAKE_PREFIX_PATH: $ENV{CONDA_PREFIX}/Library") - - # Debug: Show what LIBAV variables are set - message(STATUS "LIBAV_INCLUDE_DIRS: ${LIBAV_INCLUDE_DIRS}") - message(STATUS "LIBAV_LIBRARY_DIRS: ${LIBAV_LIBRARY_DIRS}") - message(STATUS "LIBAV_LIBRARIES: ${LIBAV_LIBRARIES}") - endif() - # Split libavcodec's version string by '.' and convert it to a list string(REPLACE "." ";" libavcodec_version_list ${LIBAV_libavcodec_VERSION}) # Get the first element of the list, which is the major version From 1098307f2dedd7ffb1fb11ad3340ca50bf17b3eb Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:24:02 +0100 Subject: [PATCH 37/57] Fix cudawheelstuff --- .github/workflows/linux_cuda_wheel.yaml | 174 +----------------------- 1 file changed, 6 insertions(+), 168 deletions(-) diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index d8fd920dc..100e95523 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -56,37 +56,6 @@ jobs: build-platform: "python-build-package" build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" -<<<<<<< HEAD - # install-and-test: - # runs-on: linux.g5.4xlarge.nvidia.gpu - # strategy: - # fail-fast: false - # matrix: - # # 3.9 corresponds to the minimum python version for which we build - # # the wheel unless the label cliflow/binaries/all is present in the - # # PR. - # # For the actual release we should add that label and change this to - # # include more python versions. - # python-version: ['3.9'] - # cuda-version: ['12.6', '12.8'] - # # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 - # ffmpeg-version-for-tests: ['4.4.2', '6', '7'] -||||||| 7945e6a - install-and-test: - runs-on: linux.g5.4xlarge.nvidia.gpu - strategy: - fail-fast: false - matrix: - # 3.9 corresponds to the minimum python version for which we build - # the wheel unless the label cliflow/binaries/all is present in the - # PR. - # For the actual release we should add that label and change this to - # include more python versions. - python-version: ['3.9'] - cuda-version: ['12.6', '12.8'] - # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 - ffmpeg-version-for-tests: ['4.4.2', '6', '7'] -======= install-and-test: runs-on: linux.g5.4xlarge.nvidia.gpu strategy: @@ -103,93 +72,7 @@ jobs: cuda-version: ['12.6', '12.9'] # TODO: put back ffmpeg 5 https://github.com/pytorch/torchcodec/issues/325 ffmpeg-version-for-tests: ['4.4.2', '6', '7'] ->>>>>>> 3c865b251bc90678621d5249083ea651c477ee8d -<<<<<<< HEAD - # container: - # image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" - # options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - # needs: build - # steps: - # - name: Setup env vars - # run: | - # cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') - # echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - # python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') - # echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - # - uses: actions/download-artifact@v4 - # with: - # name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 - # path: pytorch/torchcodec/dist/ - # - name: Setup miniconda using test-infra - # uses: pytorch/test-infra/.github/actions/setup-miniconda@main - # with: - # python-version: ${{ matrix.python-version }} - # # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - # default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" - # - name: Check env - # run: | - # ${CONDA_RUN} env - # ${CONDA_RUN} conda info - # ${CONDA_RUN} nvidia-smi - # ${CONDA_RUN} conda list - # - name: Assert ffmpeg exists - # run: | - # ${CONDA_RUN} ffmpeg -buildconf - # - name: Update pip - # run: ${CONDA_RUN} python -m pip install --upgrade pip - # - name: Install PyTorch - # run: | - # ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} - # ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - # - name: Install torchcodec from the wheel - # run: | - # wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` - # echo Installing $wheel_path - # ${CONDA_RUN} python -m pip install $wheel_path -vvv -||||||| 7945e6a - container: - image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - needs: build - steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - python_version_without_periods=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') - echo python_version_without_periods=${python_version_without_periods} >> $GITHUB_ENV - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cu${{ env.cuda_version_without_periods }}_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - with: - python-version: ${{ matrix.python-version }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - default-packages: "nvidia/label/cuda-${{ matrix.cuda-version }}.0::libnpp nvidia::cuda-nvrtc=${{ matrix.cuda-version }} nvidia::cuda-toolkit=${{ matrix.cuda-version }} nvidia::cuda-cudart=${{ matrix.cuda-version }} nvidia::cuda-driver-dev=${{ matrix.cuda-version }} conda-forge::ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" - - name: Check env - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip - - name: Install PyTorch - run: | - ${CONDA_RUN} python -m pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu${{ env.cuda_version_without_periods }} - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` - echo Installing $wheel_path - ${CONDA_RUN} python -m pip install $wheel_path -vvv -======= container: image: "pytorch/manylinux2_28-builder:cuda${{ matrix.cuda-version }}" options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" @@ -232,59 +115,15 @@ jobs: wheel_path=`find pytorch/torchcodec/dist -type f -name "*cu${{ env.cuda_version_without_periods }}-cp${{ env.python_version_without_periods }}*.whl"` echo Installing $wheel_path ${CONDA_RUN} python -m pip install $wheel_path -vvv ->>>>>>> 3c865b251bc90678621d5249083ea651c477ee8d - # - name: Check out repo - # uses: actions/checkout@v3 + - name: Check out repo + uses: actions/checkout@v3 - # - name: Install test dependencies - # run: | - # # Ideally we would find a way to get those dependencies from pyproject.toml - # ${CONDA_RUN} python -m pip install numpy pytest pillow - -<<<<<<< HEAD - # - name: Delete the src/ folder just for fun - # run: | - # # The only reason we checked-out the repo is to get access to the - # # tests. We don't care about the rest. Out of precaution, we delete - # # the src/ folder to be extra sure that we're running the code from - # # the installed wheel rather than from the source. - # # This is just to be extra cautious and very overkill because a) - # # there's no way the `torchcodec` package from src/ can be found from - # # the PythonPath: the main point of `src/` is precisely to protect - # # against that and b) if we ever were to execute code from - # # `src/torchcodec`, it would fail loudly because the built .so files - # # aren't present there. - # rm -r src/ - # ls - # - name: Run Python tests - # run: | - # ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest test -v --tb=short - # - name: Run Python benchmark - # run: | - # ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none -||||||| 7945e6a - - name: Delete the src/ folder just for fun + - name: Install test dependencies run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Run Python tests - run: | - ${CONDA_RUN} FAIL_WITHOUT_CUDA=1 pytest test -v --tb=short - - name: Run Python benchmark - run: | - ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none -======= + # Ideally we would find a way to get those dependencies from pyproject.toml + ${CONDA_RUN} python -m pip install numpy pytest pillow + - name: Delete the src/ folder just for fun run: | # The only reason we checked-out the repo is to get access to the @@ -305,4 +144,3 @@ jobs: - name: Run Python benchmark run: | ${CONDA_RUN} time python benchmarks/decoders/gpu_benchmark.py --devices=cuda:0,cpu --resize_devices=none ->>>>>>> 3c865b251bc90678621d5249083ea651c477ee8d From 57f91fb7c728d7feb8f31a9200c628f976df5108 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:41:00 +0100 Subject: [PATCH 38/57] remove test job --- .github/workflows/windows_wheel.yaml | 88 ---------------------------- src/torchcodec/_core/CMakeLists.txt | 11 +--- 2 files changed, 3 insertions(+), 96 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index e4c0bc35e..c2723c436 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -61,91 +61,3 @@ jobs: # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" # TODO see comment above build-command: "python -m build --wheel -vvv --no-isolation" - - install-and-test: - runs-on: windows-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.9'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - needs: build - steps: - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 - path: pytorch/torchcodec/dist/ - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv - - - name: Check out repo - uses: actions/checkout@v3 - - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - source packaging/helpers.sh - assert_ffmpeg_not_installed - - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version - - # On Windows, ensure the conda Library/bin directory is in PATH - # This is needed for Windows DLL loading to find FFmpeg DLLs - if [[ "$RUNNER_OS" == "Windows" ]]; then - conda_env_path=$(conda info --base)/envs/test - library_bin_path="$conda_env_path/Library/bin" - echo "Adding conda Library/bin to PATH: $library_bin_path" - echo "$library_bin_path" >> $GITHUB_PATH - - # Verify FFmpeg DLLs are accessible - echo "Checking if FFmpeg DLLs are in PATH:" - where avutil.dll || echo "avutil.dll not found in PATH" - where avcodec.dll || echo "avcodec.dll not found in PATH" - where avformat.dll || echo "avformat.dll not found in PATH" - fi - - - name: Test torchcodec import after FFmpeg installation - run: | - echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." - python -c "import torchcodec; print('TorchCodec import successful!')" - - - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow - - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Run Python tests - run: | - pytest test -vvv diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index fbcc13af7..fbb4890c6 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -14,7 +14,7 @@ else() set(TORCHCODEC_WERROR_OPTION "-Werror") endif() -if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") +if (WIN32) # Avoid warnings about non-ASCII characters in source files. set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4819") # Important for when we add Windows CUDA: exporting all symbols is aparently @@ -46,7 +46,7 @@ function(make_torchcodec_sublibrary # Avoid adding the "lib" prefix which we already add explicitly. set_target_properties(${library_name} PROPERTIES PREFIX "") - if(WIN32) # TODO unify WIN detection + if(WIN32) # On Windows, the built artifacts are put in Release/Debug # subdirectories by default. We want to avoid that, otherwise our # install() step would not know where to find those. @@ -66,11 +66,6 @@ function(make_torchcodec_sublibrary ${library_dependencies} ) - # Debug output for Windows builds - if(WIN32) - message(STATUS "Created library: ${library_name}") - message(STATUS "Library dependencies: ${library_dependencies}") - endif() endfunction() function(make_torchcodec_libraries @@ -176,7 +171,7 @@ function(make_torchcodec_libraries # stray initialization of py::objects. The rest of the object code must # match. See: # https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes - if(NOT WIN32) # TODO unify WIN detection + if(NOT WIN32) target_compile_options( ${pybind_ops_library_name} PUBLIC From 9bfc4161b05a2ab9fc7ea198d1fb4af4476be2f8 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:44:48 +0100 Subject: [PATCH 39/57] Compact cmake defs --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 218 ++++++++++-------- 1 file changed, 122 insertions(+), 96 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 81f115ff4..cafc1b5e5 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -121,130 +121,156 @@ add_library(ffmpeg7 INTERFACE) # Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable +# Set platform-specific library directory and file naming if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + set(lib_dir "lib") set( - f4_library_paths - ${f4_SOURCE_DIR}/lib/libavutil.so.56 - ${f4_SOURCE_DIR}/lib/libavcodec.so.58 - ${f4_SOURCE_DIR}/lib/libavformat.so.58 - ${f4_SOURCE_DIR}/lib/libavdevice.so.58 - ${f4_SOURCE_DIR}/lib/libavfilter.so.7 - ${f4_SOURCE_DIR}/lib/libswscale.so.5 - ${f4_SOURCE_DIR}/lib/libswresample.so.3 + f4_library_file_names + libavutil.so.56 + libavcodec.so.58 + libavformat.so.58 + libavdevice.so.58 + libavfilter.so.7 + libswscale.so.5 + libswresample.so.3 ) set( - f5_library_paths - ${f5_SOURCE_DIR}/lib/libavutil.so.57 - ${f5_SOURCE_DIR}/lib/libavcodec.so.59 - ${f5_SOURCE_DIR}/lib/libavformat.so.59 - ${f5_SOURCE_DIR}/lib/libavdevice.so.59 - ${f5_SOURCE_DIR}/lib/libavfilter.so.8 - ${f5_SOURCE_DIR}/lib/libswscale.so.6 - ${f5_SOURCE_DIR}/lib/libswresample.so.4 + f5_library_file_names + libavutil.so.57 + libavcodec.so.59 + libavformat.so.59 + libavdevice.so.59 + libavfilter.so.8 + libswscale.so.6 + libswresample.so.4 ) set( - f6_library_paths - ${f6_SOURCE_DIR}/lib/libavutil.so.58 - ${f6_SOURCE_DIR}/lib/libavcodec.so.60 - ${f6_SOURCE_DIR}/lib/libavformat.so.60 - ${f6_SOURCE_DIR}/lib/libavdevice.so.60 - ${f6_SOURCE_DIR}/lib/libavfilter.so.9 - ${f6_SOURCE_DIR}/lib/libswscale.so.7 - ${f6_SOURCE_DIR}/lib/libswresample.so.4 + f6_library_file_names + libavutil.so.58 + libavcodec.so.60 + libavformat.so.60 + libavdevice.so.60 + libavfilter.so.9 + libswscale.so.7 + libswresample.so.4 ) set( - f7_library_paths - ${f7_SOURCE_DIR}/lib/libavutil.so.59 - ${f7_SOURCE_DIR}/lib/libavcodec.so.61 - ${f7_SOURCE_DIR}/lib/libavformat.so.61 - ${f7_SOURCE_DIR}/lib/libavdevice.so.61 - ${f7_SOURCE_DIR}/lib/libavfilter.so.10 - ${f7_SOURCE_DIR}/lib/libswscale.so.8 - ${f7_SOURCE_DIR}/lib/libswresample.so.5 + f7_library_file_names + libavutil.so.59 + libavcodec.so.61 + libavformat.so.61 + libavdevice.so.61 + libavfilter.so.10 + libswscale.so.8 + libswresample.so.5 ) elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + set(lib_dir "lib") set( - f4_library_paths - ${f4_SOURCE_DIR}/lib/libavutil.56.dylib - ${f4_SOURCE_DIR}/lib/libavcodec.58.dylib - ${f4_SOURCE_DIR}/lib/libavformat.58.dylib - ${f4_SOURCE_DIR}/lib/libavdevice.58.dylib - ${f4_SOURCE_DIR}/lib/libavfilter.7.dylib - ${f4_SOURCE_DIR}/lib/libswscale.5.dylib - ${f4_SOURCE_DIR}/lib/libswresample.3.dylib + f4_library_file_names + libavutil.56.dylib + libavcodec.58.dylib + libavformat.58.dylib + libavdevice.58.dylib + libavfilter.7.dylib + libswscale.5.dylib + libswresample.3.dylib ) set( - f5_library_paths - ${f5_SOURCE_DIR}/lib/libavutil.57.dylib - ${f5_SOURCE_DIR}/lib/libavcodec.59.dylib - ${f5_SOURCE_DIR}/lib/libavformat.59.dylib - ${f5_SOURCE_DIR}/lib/libavdevice.59.dylib - ${f5_SOURCE_DIR}/lib/libavfilter.8.dylib - ${f5_SOURCE_DIR}/lib/libswscale.6.dylib - ${f5_SOURCE_DIR}/lib/libswresample.4.dylib + f5_library_file_names + libavutil.57.dylib + libavcodec.59.dylib + libavformat.59.dylib + libavdevice.59.dylib + libavfilter.8.dylib + libswscale.6.dylib + libswresample.4.dylib ) set( - f6_library_paths - ${f6_SOURCE_DIR}/lib/libavutil.58.dylib - ${f6_SOURCE_DIR}/lib/libavcodec.60.dylib - ${f6_SOURCE_DIR}/lib/libavformat.60.dylib - ${f6_SOURCE_DIR}/lib/libavdevice.60.dylib - ${f6_SOURCE_DIR}/lib/libavfilter.9.dylib - ${f6_SOURCE_DIR}/lib/libswscale.7.dylib - ${f6_SOURCE_DIR}/lib/libswresample.4.dylib + f6_library_file_names + libavutil.58.dylib + libavcodec.60.dylib + libavformat.60.dylib + libavdevice.60.dylib + libavfilter.9.dylib + libswscale.7.dylib + libswresample.4.dylib ) set( - f7_library_paths - ${f7_SOURCE_DIR}/lib/libavutil.59.dylib - ${f7_SOURCE_DIR}/lib/libavcodec.61.dylib - ${f7_SOURCE_DIR}/lib/libavformat.61.dylib - ${f7_SOURCE_DIR}/lib/libavdevice.61.dylib - ${f7_SOURCE_DIR}/lib/libavfilter.10.dylib - ${f7_SOURCE_DIR}/lib/libswscale.8.dylib - ${f7_SOURCE_DIR}/lib/libswresample.5.dylib + f7_library_file_names + libavutil.59.dylib + libavcodec.61.dylib + libavformat.61.dylib + libavdevice.61.dylib + libavfilter.10.dylib + libswscale.8.dylib + libswresample.5.dylib ) elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set(lib_dir "bin") set( - f4_library_paths - ${f4_SOURCE_DIR}/bin/avutil.lib - ${f4_SOURCE_DIR}/bin/avcodec.lib - ${f4_SOURCE_DIR}/bin/avformat.lib - ${f4_SOURCE_DIR}/bin/avdevice.lib - ${f4_SOURCE_DIR}/bin/avfilter.lib - ${f4_SOURCE_DIR}/bin/swscale.lib - ${f4_SOURCE_DIR}/bin/swresample.lib + f4_library_file_names + avutil.lib + avcodec.lib + avformat.lib + avdevice.lib + avfilter.lib + swscale.lib + swresample.lib ) set( - f5_library_paths - ${f5_SOURCE_DIR}/bin/avutil.lib - ${f5_SOURCE_DIR}/bin/avcodec.lib - ${f5_SOURCE_DIR}/bin/avformat.lib - ${f5_SOURCE_DIR}/bin/avdevice.lib - ${f5_SOURCE_DIR}/bin/avfilter.lib - ${f5_SOURCE_DIR}/bin/swscale.lib - ${f5_SOURCE_DIR}/bin/swresample.lib + f5_library_file_names + avutil.lib + avcodec.lib + avformat.lib + avdevice.lib + avfilter.lib + swscale.lib + swresample.lib ) set( - f6_library_paths - ${f6_SOURCE_DIR}/bin/avutil.lib - ${f6_SOURCE_DIR}/bin/avcodec.lib - ${f6_SOURCE_DIR}/bin/avformat.lib - ${f6_SOURCE_DIR}/bin/avdevice.lib - ${f6_SOURCE_DIR}/bin/avfilter.lib - ${f6_SOURCE_DIR}/bin/swscale.lib - ${f6_SOURCE_DIR}/bin/swresample.lib + f6_library_file_names + avutil.lib + avcodec.lib + avformat.lib + avdevice.lib + avfilter.lib + swscale.lib + swresample.lib ) set( - f7_library_paths - ${f7_SOURCE_DIR}/bin/avutil.lib - ${f7_SOURCE_DIR}/bin/avcodec.lib - ${f7_SOURCE_DIR}/bin/avformat.lib - ${f7_SOURCE_DIR}/bin/avdevice.lib - ${f7_SOURCE_DIR}/bin/avfilter.lib - ${f7_SOURCE_DIR}/bin/swscale.lib - ${f7_SOURCE_DIR}/bin/swresample.lib + f7_library_file_names + avutil.lib + avcodec.lib + avformat.lib + avdevice.lib + avfilter.lib + swscale.lib + swresample.lib ) endif() + +# Generate full library paths using list transform +list( + TRANSFORM f4_library_file_names + PREPEND ${f4_SOURCE_DIR}/${lib_dir}/ + OUTPUT_VARIABLE f4_library_paths +) +list( + TRANSFORM f5_library_file_names + PREPEND ${f5_SOURCE_DIR}/${lib_dir}/ + OUTPUT_VARIABLE f5_library_paths +) +list( + TRANSFORM f6_library_file_names + PREPEND ${f6_SOURCE_DIR}/${lib_dir}/ + OUTPUT_VARIABLE f6_library_paths +) +list( + TRANSFORM f7_library_file_names + PREPEND ${f7_SOURCE_DIR}/${lib_dir}/ + OUTPUT_VARIABLE f7_library_paths +) target_include_directories(ffmpeg4 INTERFACE ${f4_SOURCE_DIR}/include) target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include) From d73d468965e7521067b9589580a5e4967875d1b3 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:47:06 +0100 Subject: [PATCH 40/57] cleaner platform detection --- .../_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index cafc1b5e5..0556159d2 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -9,6 +9,9 @@ endif() include(FetchContent) +# Define platform variables for cleaner conditionals +set(LINUX ${UNIX} AND NOT ${APPLE}) + set( base_url https://pytorch.s3.amazonaws.com/torchcodec/ffmpeg/2025-03-14 @@ -37,7 +40,7 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") f7_sha256 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 ) -elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") +elseif (APPLE) set( platform_url ${base_url}/macos_arm64 @@ -59,7 +62,7 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a ) -elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") +elseif (WIN32) set( platform_url ${base_url}/windows_x86_64 @@ -164,7 +167,7 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") libswscale.so.8 libswresample.so.5 ) -elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") +elseif (APPLE) set(lib_dir "lib") set( f4_library_file_names @@ -206,7 +209,7 @@ elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") libswscale.8.dylib libswresample.5.dylib ) -elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") +elseif (WIN32) set(lib_dir "bin") set( f4_library_file_names From 0e734c8d3855e8a006ea495d77dbb2e6ff4a7d21 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:48:14 +0100 Subject: [PATCH 41/57] move --- src/torchcodec/_core/CMakeLists.txt | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index fbb4890c6..913a917d6 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -17,8 +17,8 @@ endif() if (WIN32) # Avoid warnings about non-ASCII characters in source files. set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4819") - # Important for when we add Windows CUDA: exporting all symbols is aparently - # limited to 65535 symbols, which (aparently) will not work for CUDA. + # Important for when we add Windows CUDA: exporting all symbols is limited to + # 65535 symbols, which (apparently) will not work for CUDA. # https://github.com/pytorch/pytorch/pull/3650 set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif() @@ -178,16 +178,6 @@ function(make_torchcodec_libraries "-fvisibility=hidden" ) endif() - if(APPLE) # TODO unify APPLE detection - # If we don't make sure this flag is set, we run into segfauls at import - # time on Mac. See: - # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 - target_link_options( - ${pybind_ops_library_name} - PUBLIC - "LINKER:-undefined,dynamic_lookup" - ) - endif() # The value we use here must match the value we return from # _get_pybind_ops_module_name() on the Python side. If the values do not @@ -199,6 +189,17 @@ function(make_torchcodec_libraries PYBIND_OPS_MODULE_NAME=core_pybind_ops ) + if(APPLE) # TODO unify APPLE detection + # If we don't make sure this flag is set, we run into segfauls at import + # time on Mac. See: + # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 + target_link_options( + ${pybind_ops_library_name} + PUBLIC + "LINKER:-undefined,dynamic_lookup" + ) + endif() + # Install all libraries. set( all_libraries From 78ff70eeb0d9280d6af32b2c5de03d9d31f9e380 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:50:53 +0100 Subject: [PATCH 42/57] Nits --- .../_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 0556159d2..969ccab3c 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -9,7 +9,6 @@ endif() include(FetchContent) -# Define platform variables for cleaner conditionals set(LINUX ${UNIX} AND NOT ${APPLE}) set( @@ -17,8 +16,7 @@ set( https://pytorch.s3.amazonaws.com/torchcodec/ffmpeg/2025-03-14 ) - -if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") +if (LINUX) set( platform_url ${base_url}/linux_x86_64 @@ -125,7 +123,7 @@ add_library(ffmpeg7 INTERFACE) # Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable # Set platform-specific library directory and file naming -if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") +if (LINUX) set(lib_dir "lib") set( f4_library_file_names From 847bcc5709b886c1f252ff1a228674e32b336d44 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:54:04 +0100 Subject: [PATCH 43/57] refac --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 180 +++++++++--------- 1 file changed, 89 insertions(+), 91 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 969ccab3c..2ca8878ce 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -17,6 +17,8 @@ set( ) if (LINUX) + set(lib_dir "lib") + set( platform_url ${base_url}/linux_x86_64 @@ -38,93 +40,6 @@ if (LINUX) f7_sha256 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 ) -elseif (APPLE) - set( - platform_url - ${base_url}/macos_arm64 - ) - set( - f4_sha256 - f0335434529d9e19359eae0fe912dd9e747667534a1c92e662f5219a55dfad8c - ) - set( - f5_sha256 - cfc3449c9af6863731a431ce89e32c08c5f8ece94b306fb6b695828502a76166 - ) - set( - f6_sha256 - ec47b4783c342038e720e33b2fdfa55a9a490afb1cf37a26467733983688647e - ) - set( - f7_sha256 - 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a - ) - -elseif (WIN32) - set( - platform_url - ${base_url}/windows_x86_64 - ) - set( - f4_sha256 - 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb - ) - set( - f5_sha256 - b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 - ) - set( - f6_sha256 - 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a - ) - set( - f7_sha256 - ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 - ) -else() - message( - FATAL_ERROR - "Unsupported operating system: ${CMAKE_SYSTEM_NAME}" - ) -endif() - -FetchContent_Declare( - f4 - URL ${platform_url}/4.4.4.tar.gz - URL_HASH - SHA256=${f4_sha256} -) -FetchContent_Declare( - f5 - URL ${platform_url}/5.1.4.tar.gz - URL_HASH - SHA256=${f5_sha256} -) -FetchContent_Declare( - f6 - URL ${platform_url}/6.1.1.tar.gz - URL_HASH - SHA256=${f6_sha256} -) -FetchContent_Declare( - f7 - URL ${platform_url}/7.0.1.tar.gz - URL_HASH - SHA256=${f7_sha256} -) - -FetchContent_MakeAvailable(f4 f5 f6 f7) - -add_library(ffmpeg4 INTERFACE) -add_library(ffmpeg5 INTERFACE) -add_library(ffmpeg6 INTERFACE) -add_library(ffmpeg7 INTERFACE) - -# Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable - -# Set platform-specific library directory and file naming -if (LINUX) - set(lib_dir "lib") set( f4_library_file_names libavutil.so.56 @@ -167,6 +82,27 @@ if (LINUX) ) elseif (APPLE) set(lib_dir "lib") + set( + platform_url + ${base_url}/macos_arm64 + ) + set( + f4_sha256 + f0335434529d9e19359eae0fe912dd9e747667534a1c92e662f5219a55dfad8c + ) + set( + f5_sha256 + cfc3449c9af6863731a431ce89e32c08c5f8ece94b306fb6b695828502a76166 + ) + set( + f6_sha256 + ec47b4783c342038e720e33b2fdfa55a9a490afb1cf37a26467733983688647e + ) + set( + f7_sha256 + 48a4fc8ce098305cfd4a58f40889249c523ca3c285f66ba704b5bad0e3ada53a + ) + set( f4_library_file_names libavutil.56.dylib @@ -207,8 +143,30 @@ elseif (APPLE) libswscale.8.dylib libswresample.5.dylib ) + elseif (WIN32) set(lib_dir "bin") + set( + platform_url + ${base_url}/windows_x86_64 + ) + set( + f4_sha256 + 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb + ) + set( + f5_sha256 + b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 + ) + set( + f6_sha256 + 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a + ) + set( + f7_sha256 + ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 + ) + set( f4_library_file_names avutil.lib @@ -249,8 +207,52 @@ elseif (WIN32) swscale.lib swresample.lib ) +else() + message( + FATAL_ERROR + "Unsupported operating system: ${CMAKE_SYSTEM_NAME}" + ) endif() +FetchContent_Declare( + f4 + URL ${platform_url}/4.4.4.tar.gz + URL_HASH + SHA256=${f4_sha256} +) +FetchContent_Declare( + f5 + URL ${platform_url}/5.1.4.tar.gz + URL_HASH + SHA256=${f5_sha256} +) +FetchContent_Declare( + f6 + URL ${platform_url}/6.1.1.tar.gz + URL_HASH + SHA256=${f6_sha256} +) +FetchContent_Declare( + f7 + URL ${platform_url}/7.0.1.tar.gz + URL_HASH + SHA256=${f7_sha256} +) + +FetchContent_MakeAvailable(f4 f5 f6 f7) + +add_library(ffmpeg4 INTERFACE) +add_library(ffmpeg5 INTERFACE) +add_library(ffmpeg6 INTERFACE) +add_library(ffmpeg7 INTERFACE) + +# Note: the f?_SOURCE_DIR variables were set by FetchContent_MakeAvailable +target_include_directories(ffmpeg4 INTERFACE ${f4_SOURCE_DIR}/include) +target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) +target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include) +target_include_directories(ffmpeg7 INTERFACE ${f7_SOURCE_DIR}/include) + + # Generate full library paths using list transform list( TRANSFORM f4_library_file_names @@ -272,10 +274,6 @@ list( PREPEND ${f7_SOURCE_DIR}/${lib_dir}/ OUTPUT_VARIABLE f7_library_paths ) -target_include_directories(ffmpeg4 INTERFACE ${f4_SOURCE_DIR}/include) -target_include_directories(ffmpeg5 INTERFACE ${f5_SOURCE_DIR}/include) -target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include) -target_include_directories(ffmpeg7 INTERFACE ${f7_SOURCE_DIR}/include) target_link_libraries( From 72802764b4b045120dbe121bc5788d4b380ab3a1 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 13:59:33 +0100 Subject: [PATCH 44/57] reducxe diff --- setup.py | 4 +- src/torchcodec/_core/CMakeLists.txt | 2 +- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 130 +++++++++--------- src/torchcodec/_internally_replaced_utils.py | 4 +- 4 files changed, 67 insertions(+), 73 deletions(-) diff --git a/setup.py b/setup.py index 49e10b0a5..4eece1a03 100644 --- a/setup.py +++ b/setup.py @@ -158,9 +158,7 @@ def copy_extensions_to_source(self): elif sys.platform in ("win32", "cygwin"): extensions = ["dll"] else: - raise NotImplementedError( - f"Platform {sys.platform} is not supported" - ) + raise NotImplementedError(f"Platform {sys.platform} is not supported") for ext in extensions: for lib_file in self._install_prefix.glob(f"*.{ext}"): diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 913a917d6..5a9a14c61 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -65,7 +65,7 @@ function(make_torchcodec_sublibrary PUBLIC ${library_dependencies} ) - + endfunction() function(make_torchcodec_libraries diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 2ca8878ce..fff33fb72 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -41,44 +41,44 @@ if (LINUX) 1cb946d8b7c6393c2c3ebe1f900b8de7a2885fe614c45d4ec32c9833084f2f26 ) set( - f4_library_file_names - libavutil.so.56 - libavcodec.so.58 - libavformat.so.58 - libavdevice.so.58 - libavfilter.so.7 - libswscale.so.5 - libswresample.so.3 + f4_library_file_names + libavutil.so.56 + libavcodec.so.58 + libavformat.so.58 + libavdevice.so.58 + libavfilter.so.7 + libswscale.so.5 + libswresample.so.3 ) set( - f5_library_file_names - libavutil.so.57 - libavcodec.so.59 - libavformat.so.59 - libavdevice.so.59 - libavfilter.so.8 - libswscale.so.6 - libswresample.so.4 + f5_library_file_names + libavutil.so.57 + libavcodec.so.59 + libavformat.so.59 + libavdevice.so.59 + libavfilter.so.8 + libswscale.so.6 + libswresample.so.4 ) set( - f6_library_file_names - libavutil.so.58 - libavcodec.so.60 - libavformat.so.60 - libavdevice.so.60 - libavfilter.so.9 - libswscale.so.7 - libswresample.so.4 + f6_library_file_names + libavutil.so.58 + libavcodec.so.60 + libavformat.so.60 + libavdevice.so.60 + libavfilter.so.9 + libswscale.so.7 + libswresample.so.4 ) set( - f7_library_file_names - libavutil.so.59 - libavcodec.so.61 - libavformat.so.61 - libavdevice.so.61 - libavfilter.so.10 - libswscale.so.8 - libswresample.so.5 + f7_library_file_names + libavutil.so.59 + libavcodec.so.61 + libavformat.so.61 + libavdevice.so.61 + libavfilter.so.10 + libswscale.so.8 + libswresample.so.5 ) elseif (APPLE) set(lib_dir "lib") @@ -104,44 +104,44 @@ elseif (APPLE) ) set( - f4_library_file_names - libavutil.56.dylib - libavcodec.58.dylib - libavformat.58.dylib - libavdevice.58.dylib - libavfilter.7.dylib - libswscale.5.dylib - libswresample.3.dylib + f4_library_file_names + libavutil.56.dylib + libavcodec.58.dylib + libavformat.58.dylib + libavdevice.58.dylib + libavfilter.7.dylib + libswscale.5.dylib + libswresample.3.dylib ) set( - f5_library_file_names - libavutil.57.dylib - libavcodec.59.dylib - libavformat.59.dylib - libavdevice.59.dylib - libavfilter.8.dylib - libswscale.6.dylib - libswresample.4.dylib + f5_library_file_names + libavutil.57.dylib + libavcodec.59.dylib + libavformat.59.dylib + libavdevice.59.dylib + libavfilter.8.dylib + libswscale.6.dylib + libswresample.4.dylib ) set( - f6_library_file_names - libavutil.58.dylib - libavcodec.60.dylib - libavformat.60.dylib - libavdevice.60.dylib - libavfilter.9.dylib - libswscale.7.dylib - libswresample.4.dylib + f6_library_file_names + libavutil.58.dylib + libavcodec.60.dylib + libavformat.60.dylib + libavdevice.60.dylib + libavfilter.9.dylib + libswscale.7.dylib + libswresample.4.dylib ) set( - f7_library_file_names - libavutil.59.dylib - libavcodec.61.dylib - libavformat.61.dylib - libavdevice.61.dylib - libavfilter.10.dylib - libswscale.8.dylib - libswresample.5.dylib + f7_library_file_names + libavutil.59.dylib + libavcodec.61.dylib + libavformat.61.dylib + libavdevice.61.dylib + libavfilter.10.dylib + libswscale.8.dylib + libswresample.5.dylib ) elseif (WIN32) @@ -253,7 +253,6 @@ target_include_directories(ffmpeg6 INTERFACE ${f6_SOURCE_DIR}/include) target_include_directories(ffmpeg7 INTERFACE ${f7_SOURCE_DIR}/include) -# Generate full library paths using list transform list( TRANSFORM f4_library_file_names PREPEND ${f4_SOURCE_DIR}/${lib_dir}/ @@ -275,7 +274,6 @@ list( OUTPUT_VARIABLE f7_library_paths ) - target_link_libraries( ffmpeg4 INTERFACE diff --git a/src/torchcodec/_internally_replaced_utils.py b/src/torchcodec/_internally_replaced_utils.py index 261539194..e5b5847d1 100644 --- a/src/torchcodec/_internally_replaced_utils.py +++ b/src/torchcodec/_internally_replaced_utils.py @@ -21,9 +21,7 @@ def _get_extension_path(lib_name: str) -> str: elif sys.platform in ("win32", "cygwin"): extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES + [".dll"] else: - raise NotImplementedError( - f"{sys.platform = } is not not supported" - ) + raise NotImplementedError(f"{sys.platform = } is not not supported") loader_details = ( importlib.machinery.ExtensionFileLoader, extension_suffixes, From 9428b4256c743b5a1df36b0b2cfad29158c86716 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 14:00:26 +0100 Subject: [PATCH 45/57] nit --- src/torchcodec/_core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 5a9a14c61..3dcedad6a 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -189,7 +189,7 @@ function(make_torchcodec_libraries PYBIND_OPS_MODULE_NAME=core_pybind_ops ) - if(APPLE) # TODO unify APPLE detection + if(APPLE) # If we don't make sure this flag is set, we run into segfauls at import # time on Mac. See: # https://github.com/pybind/pybind11/issues/3907#issuecomment-1170412764 From eaae82379f625a8c4c3c8eee54d7341e345d2017 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 14:19:16 +0100 Subject: [PATCH 46/57] Fix LINUX --- .../_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index fff33fb72..22fbdff4c 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -9,7 +9,11 @@ endif() include(FetchContent) -set(LINUX ${UNIX} AND NOT ${APPLE}) +if (UNIX AND NOT APPLE) + set(LINUX TRUE) +else() + set(LINUX FALSE) +endif() set( base_url From 9664d8929a57aa3bad9f2dbfc90ab80f2c32976f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 14:32:32 +0100 Subject: [PATCH 47/57] Remove changes to setuppy --- setup.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 4eece1a03..59b5ef53c 100644 --- a/setup.py +++ b/setup.py @@ -174,14 +174,14 @@ def copy_extensions_to_source(self): build_against_all_ffmpeg_from_s3 = ( os.getenv(BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR) is not None ) -# if "bdist_wheel" in sys.argv and not ( -# build_against_all_ffmpeg_from_s3 or not_a_license_violation -# ): -# raise ValueError( -# "It looks like you're trying to build a wheel. " -# f"You probably want to set {BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR}. " -# f"If you have a good reason *not* to, then set {NOT_A_LICENSE_VIOLATION_VAR}." -# ) +if "bdist_wheel" in sys.argv and not ( + build_against_all_ffmpeg_from_s3 or not_a_license_violation +): + raise ValueError( + "It looks like you're trying to build a wheel. " + f"You probably want to set {BUILD_AGAINST_ALL_FFMPEG_FROM_S3_VAR}. " + f"If you have a good reason *not* to, then set {NOT_A_LICENSE_VIOLATION_VAR}." + ) # See `CMakeBuild.build_extension()`. fake_extension = Extension(name="FAKE_NAME", sources=[]) From 4bab88c4e6c88d4d209d42042ec877581a4bb708 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 14:37:22 +0100 Subject: [PATCH 48/57] Update compilation flags --- src/torchcodec/_core/CMakeLists.txt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 3dcedad6a..1e6d2ec80 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -11,7 +11,12 @@ find_package(Python3 ${PYTHON_VERSION} EXACT COMPONENTS Development) if(DEFINED TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR AND TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR) set(TORCHCODEC_WERROR_OPTION "") else() - set(TORCHCODEC_WERROR_OPTION "-Werror") + if (WIN32) + # TODO set warnings as errors on Windows as well. + # set(TORCHCODEC_WERROR_OPTION "/WX") + else() + set(TORCHCODEC_WERROR_OPTION "-Werror") + endif() endif() if (WIN32) @@ -23,9 +28,11 @@ if (WIN32) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif() -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") -# TODO put back previous line and set proper flags for windows. e.g. Wall is W4. -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") +if (WIN32) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic ${TORCHCODEC_WERROR_OPTION} ${TORCH_CXX_FLAGS}") +endif() function(make_torchcodec_sublibrary From af81e5f9c482f91744a02601889d3ffb067916c8 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 14:49:32 +0100 Subject: [PATCH 49/57] Set BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here --- .github/workflows/windows_wheel.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index c2723c436..3961feab4 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -42,8 +42,7 @@ jobs: strategy: fail-fast: false name: Build and Upload Windows wheel - # TODO - # We shouldn't use this, but this is needed for now. Note that this is where we are setting the BUILD_AGAINST_ALL_FFMPEG_FROM_S3 variable currently, we'll have to find a way around that. + # TODO: use @main uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows with: repository: pytorch/torchcodec @@ -52,12 +51,10 @@ jobs: test-infra-ref: build-platform-windows build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh - # post-script: packaging/post_build_script.sh # TODO enable post-build checks? + # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows env-script: packaging/vc_env_helper.bat smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - # build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" - # TODO see comment above - build-command: "python -m build --wheel -vvv --no-isolation" + build-command: "set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation" From 9f63e0ce8fc60f84ea2e3208dd44ccd3cffd9f32 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 14:56:22 +0100 Subject: [PATCH 50/57] Avoid splitting? --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 3961feab4..2f6058c8e 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -57,4 +57,4 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation" + build-command: "\"set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation\"" From bc08cde52ac2e439bc1861aabf91985ef685f8df Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 15:02:23 +0100 Subject: [PATCH 51/57] Try something else --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 2f6058c8e..159e911a6 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -57,4 +57,4 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "\"set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation\"" + build-command: "cmd /c \"set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation\"" From 80bafe6466774ad44485848bba8c26ddafb1a182 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 15:08:34 +0100 Subject: [PATCH 52/57] Try something else --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 159e911a6..189ae95b7 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -57,4 +57,4 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "cmd /c \"set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 && python -m build --wheel -vvv --no-isolation\"" + build-command: "set \"BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1\" & python -m build --wheel -vvv --no-isolation" From 09ec7f3f1cc61eb87e53d77affbe9285ccd40d50 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 15:14:44 +0100 Subject: [PATCH 53/57] Pass variable through env helper --- .github/workflows/windows_wheel.yaml | 2 +- packaging/vc_env_helper.bat | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 189ae95b7..8267f7abf 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -57,4 +57,4 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - build-command: "set \"BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1\" & python -m build --wheel -vvv --no-isolation" + build-command: "python -m build --wheel -vvv --no-isolation" diff --git a/packaging/vc_env_helper.bat b/packaging/vc_env_helper.bat index cccbb4c2d..59a156402 100644 --- a/packaging/vc_env_helper.bat +++ b/packaging/vc_env_helper.bat @@ -24,6 +24,7 @@ if "%VSDEVCMD_ARGS%" == "" ( if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" set DISTUTILS_USE_SDK=1 +set BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 set args=%1 shift From 2bb8ac9e7d4e6c6699d21ff40c5e5f8bf68486f6 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 15:31:28 +0100 Subject: [PATCH 54/57] Comments --- .github/workflows/cpp_tests.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/linux_cuda_wheel.yaml | 2 +- .github/workflows/linux_wheel.yaml | 2 +- .github/workflows/macos_wheel.yaml | 2 +- .github/workflows/windows_wheel.yaml | 2 ++ 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index 227e40e12..e08d90754 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -3,7 +3,7 @@ name: CPP tests on: push: branches: [ main ] - # pull_request: + pull_request: concurrency: group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e042ab75e..60bfbfa2e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,7 +3,7 @@ name: Docs on: push: branches: [ main ] - # pull_request: + pull_request: permissions: id-token: write diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index 100e95523..c57d8807d 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Linux CUDA wheels on: - # pull_request: + pull_request: push: branches: - nightly diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 0803d37cb..706e7c2b4 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Linux wheel on: - # pull_request: + pull_request: push: branches: - nightly diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index c1dc8ae88..b26a1d2d1 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test MacOS wheel on: - # pull_request: + pull_request: push: branches: - nightly diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 8267f7abf..709008bfe 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -57,4 +57,6 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" + # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is + # set in vc_env_helper.bat Couldn't find a way to set it from here. build-command: "python -m build --wheel -vvv --no-isolation" From 457b0ec81936ec35eae500191e61b68bdb04641e Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sun, 3 Aug 2025 15:35:49 +0100 Subject: [PATCH 55/57] put back lint job --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 456bc2126..c156a833c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -3,7 +3,7 @@ name: Lint on: push: branches: [ main ] - # pull_request: + pull_request: concurrency: group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} From f24caf6edde0576a4de04e205ab96f131228422f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 4 Aug 2025 15:12:40 +0100 Subject: [PATCH 56/57] Use official test-infra instead of mine --- .github/workflows/windows_wheel.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 709008bfe..a274118d6 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -42,8 +42,7 @@ jobs: strategy: fail-fast: false name: Build and Upload Windows wheel - # TODO: use @main - uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows + uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main with: repository: pytorch/torchcodec ref: "" From 62a3658817fcbcf3c43883e4e40bfefedceda273 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Mon, 4 Aug 2025 15:35:27 +0100 Subject: [PATCH 57/57] Revert "Use official test-infra instead of mine" This reverts commit f24caf6edde0576a4de04e205ab96f131228422f. --- .github/workflows/windows_wheel.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index a274118d6..709008bfe 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -42,7 +42,8 @@ jobs: strategy: fail-fast: false name: Build and Upload Windows wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main + # TODO: use @main + uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows with: repository: pytorch/torchcodec ref: ""