From d81c3e38d37259e062d1973eec5304dedc542208 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 08:50:49 +0100 Subject: [PATCH 01/58] Add test wheel job --- .github/workflows/docs.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/linux_cuda_wheel.yaml | 2 +- .github/workflows/linux_wheel.yaml | 2 +- .github/workflows/macos_wheel.yaml | 2 +- .github/workflows/windows_wheel.yaml | 81 +++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 60bfbfa2e..e042ab75e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,7 +3,7 @@ name: Docs on: push: branches: [ main ] - pull_request: + # pull_request: permissions: id-token: write diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c156a833c..456bc2126 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -3,7 +3,7 @@ name: Lint on: push: branches: [ main ] - pull_request: + # pull_request: concurrency: group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index c57d8807d..100e95523 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Linux CUDA wheels on: - pull_request: + # pull_request: push: branches: - nightly diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 706e7c2b4..0803d37cb 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Linux wheel on: - pull_request: + # pull_request: push: branches: - nightly diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index b26a1d2d1..c1dc8ae88 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test MacOS wheel on: - pull_request: + # pull_request: push: branches: - nightly diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 709008bfe..df263c812 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -60,3 +60,84 @@ jobs: # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is # set in vc_env_helper.bat Couldn't find a way to set it from here. build-command: "python -m build --wheel -vvv --no-isolation" + + install-and-test: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + needs: build + steps: + - uses: actions/download-artifact@v4 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 + path: pytorch/torchcodec/dist/ + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install PyTorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + source packaging/helpers.sh + assert_ffmpeg_not_installed + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version + # On Windows, ensure the conda Library/bin directory is in PATH + # This is needed for Windows DLL loading to find FFmpeg DLLs + if [[ "$RUNNER_OS" == "Windows" ]]; then + conda_env_path=$(conda info --base)/envs/test + library_bin_path="$conda_env_path/Library/bin" + echo "Adding conda Library/bin to PATH: $library_bin_path" + echo "$library_bin_path" >> $GITHUB_PATH + # Verify FFmpeg DLLs are accessible + echo "Checking if FFmpeg DLLs are in PATH:" + where avutil.dll || echo "avutil.dll not found in PATH" + where avcodec.dll || echo "avcodec.dll not found in PATH" + where avformat.dll || echo "avformat.dll not found in PATH" + fi + - name: Test torchcodec import after FFmpeg installation + run: | + echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." + python -c "import torchcodec; print('TorchCodec import successful!')" + - name: Install test dependencies + run: | + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Run Python tests + run: | + pytest test -vvv From 78ea520bcc0f58ff49d342b819b6c7a49675f629 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 09:18:17 +0100 Subject: [PATCH 02/58] Make pybind extension a pyd instead of dll --- setup.py | 2 +- src/torchcodec/_core/CMakeLists.txt | 5 +++++ src/torchcodec/_internally_replaced_utils.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 59b5ef53c..895b37985 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def copy_extensions_to_source(self): # https://stackoverflow.com/a/2339910 extensions = ["dylib", "so"] elif sys.platform in ("win32", "cygwin"): - extensions = ["dll"] + extensions = ["dll", "pyd"] else: raise NotImplementedError(f"Platform {sys.platform} is not supported") diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 1e6d2ec80..eda2fa987 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -174,6 +174,11 @@ function(make_torchcodec_libraries "${pybind_ops_sources}" "${pybind_ops_dependencies}" ) + + if(WIN32) + set_target_properties(${pybind_ops_library_name} PROPERTIES SUFFIX ".pyd") + endif() + # pybind11 limits the visibility of symbols in the shared library to prevent # stray initialization of py::objects. The rest of the object code must # match. See: diff --git a/src/torchcodec/_internally_replaced_utils.py b/src/torchcodec/_internally_replaced_utils.py index e5b5847d1..d0d1f59f7 100644 --- a/src/torchcodec/_internally_replaced_utils.py +++ b/src/torchcodec/_internally_replaced_utils.py @@ -19,7 +19,7 @@ def _get_extension_path(lib_name: str) -> str: elif sys.platform == "darwin": extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES + [".dylib"] elif sys.platform in ("win32", "cygwin"): - extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES + [".dll"] + extension_suffixes = importlib.machinery.EXTENSION_SUFFIXES + [".dll", ".pyd"] else: raise NotImplementedError(f"{sys.platform = } is not not supported") loader_details = ( From a0dd832e236f8cace0b0343a4883befa5fd8fc11 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 09:22:37 +0100 Subject: [PATCH 03/58] empty From 1b5f9262e36c5edb87e641d37049550ae92e32cf Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 09:57:51 +0100 Subject: [PATCH 04/58] libtorchcodec_custom_ops may not be able to find libtorchcodec_core. Set its RPATH to ORIGIN so it can find libtorchcodec_core --- src/torchcodec/_core/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index eda2fa987..cf90a2e14 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -148,6 +148,15 @@ function(make_torchcodec_libraries "${custom_ops_dependencies}" ) + if(WIN32) + # On Windows, ensure DLLs can find their dependencies in the same directory + set_target_properties(${custom_ops_library_name} PROPERTIES + INSTALL_RPATH "$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() + + # 3. Create libtorchcodec_pybind_opsN.so. set(pybind_ops_library_name "libtorchcodec_pybind_ops${ffmpeg_major_version}") set(pybind_ops_sources From 218ca2d14bb51b1bbd2308b808872fb73d12a2b0 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 10:02:52 +0100 Subject: [PATCH 05/58] Try to add more debugging output on Windows --- src/torchcodec/_core/ops.py | 41 ++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index 71b809d59..faf930e20 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -6,6 +6,8 @@ import io import json +import os +import sys import warnings from types import ModuleType from typing import List, Optional, Tuple, Union @@ -57,9 +59,42 @@ def load_torchcodec_shared_libraries(): ) return except Exception as e: - # TODO: recording and reporting exceptions this way is OK for now as it's just for debugging, - # but we should probably handle that via a proper logging mechanism. - exceptions.append((ffmpeg_major_version, e)) + # Enhanced error reporting with Windows-specific details + error_msg = str(e) + + # On Windows, try to get more detailed DLL loading error information + if sys.platform == "win32": + try: + import ctypes + # Get the last Windows error code + win_error = ctypes.get_last_error() + if win_error != 0: + win_error_msg = ctypes.FormatError(win_error) + error_msg += f" [Windows Error {win_error}: {win_error_msg}]" + except Exception: + pass + + # Check if the files actually exist + try: + core_path = _get_extension_path(decoder_library_name) + custom_ops_path = _get_extension_path(custom_ops_library_name) + pybind_ops_path = _get_extension_path(pybind_ops_library_name) + + paths_info = [] + for name, path in [ + ("core", core_path), + ("custom_ops", custom_ops_path), + ("pybind_ops", pybind_ops_path) + ]: + exists = os.path.exists(path) + size = os.path.getsize(path) if exists else 0 + paths_info.append(f"{name}: {path} (exists: {exists}, size: {size})") + + error_msg += f" [Library paths: {'; '.join(paths_info)}]" + except Exception as path_e: + error_msg += f" [Could not check library paths: {path_e}]" + + exceptions.append((ffmpeg_major_version, error_msg)) traceback = ( "\n[start of libtorchcodec loading traceback]\n" From 4cce232318d615c440ed56713c564c38078ead41 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 10:37:47 +0100 Subject: [PATCH 06/58] Print more debug --- src/torchcodec/_core/ops.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index faf930e20..0ccfdfa84 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -49,14 +49,20 @@ def load_torchcodec_shared_libraries(): custom_ops_library_name = f"libtorchcodec_custom_ops{ffmpeg_major_version}" pybind_ops_library_name = f"libtorchcodec_pybind_ops{ffmpeg_major_version}" try: + print("Loading {decoder_library_name}", flush=True) torch.ops.load_library(_get_extension_path(decoder_library_name)) + print("successfully loaded {decoder_library_name}", flush=True) + print("Loading {custom_ops_library_name}", flush=True) torch.ops.load_library(_get_extension_path(custom_ops_library_name)) + print("successfully loaded {custom_ops_library_name}", flush=True) + print("Loading {pybind_ops_library_name}", flush=True) pybind_ops_library_path = _get_extension_path(pybind_ops_library_name) global _pybind_ops _pybind_ops = _load_pybind11_module( pybind_ops_module_name, pybind_ops_library_path ) + print("successfully loaded {pybind_ops_library_name}", flush=True) return except Exception as e: # Enhanced error reporting with Windows-specific details From ab53ca957f2175da6e3ffc1c72dff19222ef76d0 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 10:42:06 +0100 Subject: [PATCH 07/58] More debug stuff --- src/torchcodec/_core/CMakeLists.txt | 8 -------- src/torchcodec/_core/ops.py | 26 +++++++++++++++++++------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index cf90a2e14..ba7495a32 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -148,14 +148,6 @@ function(make_torchcodec_libraries "${custom_ops_dependencies}" ) - if(WIN32) - # On Windows, ensure DLLs can find their dependencies in the same directory - set_target_properties(${custom_ops_library_name} PROPERTIES - INSTALL_RPATH "$ORIGIN" - BUILD_WITH_INSTALL_RPATH TRUE - ) - endif() - # 3. Create libtorchcodec_pybind_opsN.so. set(pybind_ops_library_name "libtorchcodec_pybind_ops${ffmpeg_major_version}") diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index 0ccfdfa84..821e96f2b 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -49,20 +49,32 @@ def load_torchcodec_shared_libraries(): custom_ops_library_name = f"libtorchcodec_custom_ops{ffmpeg_major_version}" pybind_ops_library_name = f"libtorchcodec_pybind_ops{ffmpeg_major_version}" try: - print("Loading {decoder_library_name}", flush=True) - torch.ops.load_library(_get_extension_path(decoder_library_name)) - print("successfully loaded {decoder_library_name}", flush=True) - print("Loading {custom_ops_library_name}", flush=True) + print(f"Loading {decoder_library_name}", flush=True) + core_path = _get_extension_path(decoder_library_name) + torch.ops.load_library(core_path) + print(f"successfully loaded {decoder_library_name}", flush=True) + + # On Windows, try to preload the core DLL to help resolve dependencies + if sys.platform == "win32": + try: + import ctypes + # Preload the core library so it's available for dependency resolution + ctypes.CDLL(core_path) + print(f"Preloaded core library via ctypes", flush=True) + except Exception as preload_e: + print(f"Could not preload core library: {preload_e}", flush=True) + + print(f"Loading {custom_ops_library_name}", flush=True) torch.ops.load_library(_get_extension_path(custom_ops_library_name)) - print("successfully loaded {custom_ops_library_name}", flush=True) + print(f"successfully loaded {custom_ops_library_name}", flush=True) - print("Loading {pybind_ops_library_name}", flush=True) + print(f"Loading {pybind_ops_library_name}", flush=True) pybind_ops_library_path = _get_extension_path(pybind_ops_library_name) global _pybind_ops _pybind_ops = _load_pybind11_module( pybind_ops_module_name, pybind_ops_library_path ) - print("successfully loaded {pybind_ops_library_name}", flush=True) + print(f"successfully loaded {pybind_ops_library_name}", flush=True) return except Exception as e: # Enhanced error reporting with Windows-specific details From 845029721a5b832af717519050be67f5acc7b15f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 11:05:34 +0100 Subject: [PATCH 08/58] Use MSVC toolchain when building FFmpeg on Windows --- packaging/build_ffmpeg.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packaging/build_ffmpeg.sh b/packaging/build_ffmpeg.sh index 9fef66814..c95587c88 100755 --- a/packaging/build_ffmpeg.sh +++ b/packaging/build_ffmpeg.sh @@ -20,6 +20,10 @@ set -eux prefix="${FFMPEG_ROOT}" +args="" +if [[ "$OSTYPE" == "msys" ]]; then + args="--toolchain=msvc" +fi archive="https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n${FFMPEG_VERSION}.tar.gz" build_dir=$(mktemp -d -t ffmpeg-build.XXXXXXXXXX) @@ -62,7 +66,7 @@ tar -xf ffmpeg.tar.gz --strip-components 1 --enable-avformat \ --enable-avutil \ --enable-swscale \ - --enable-swresample + --enable-swresample ${args} make -j install ls ${prefix}/* From 3b79ff566df2972c4bc7e9f1598c387bab64c337 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 11:25:22 +0100 Subject: [PATCH 09/58] Use msvc-built FFmpeg binaries when building the wheel --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 69 +++++++++++++------ 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 22fbdff4c..71aea7dc4 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -150,26 +150,55 @@ elseif (APPLE) elseif (WIN32) set(lib_dir "bin") - set( - platform_url - ${base_url}/windows_x86_64 - ) - set( - f4_sha256 - 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb - ) - set( - f5_sha256 - b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 - ) - set( - f6_sha256 - 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a - ) - set( - f7_sha256 - ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 - ) + + # Option to use MSVC-compatible FFmpeg libraries on Windows + option(USE_MSVC_FFMPEG "Use MSVC-compatible FFmpeg libraries on Windows" ON) + + if(USE_MSVC_FFMPEG) + message(STATUS "Using MSVC-compatible FFmpeg libraries for Windows") + set( + platform_url + ${base_url}/windows_x86_64_msvc + ) + set( + f4_sha256 + 407c0f4dea134050261b3e29ec75ca146afc50c5c1fbdd63b48507c70ccce6d4 + ) + set( + f5_sha256 + fa3850be150d64efb7fa3f91da4e488621ee03cd8041623b727e5ea3281a1574 + ) + set( + f6_sha256 + 3d65fca9597b9be7d9e3e11e1a4f4a9aa87c622c18e47abca752700a6ac92e8c + ) + set( + f7_sha256 + c1888939320b2c7de1d3ea687147f5b18aec42dd4797f59a818c840c5a4e5ee2 + ) + else() + message(STATUS "Using MinGW-compatible FFmpeg libraries for Windows") + set( + platform_url + ${base_url}/windows_x86_64 + ) + set( + f4_sha256 + 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb + ) + set( + f5_sha256 + b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 + ) + set( + f6_sha256 + 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a + ) + set( + f7_sha256 + ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 + ) + endif() set( f4_library_file_names From ac8116db23ca8be0b7e3e367dd0fd6d68f1e29ef Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 12:32:24 +0100 Subject: [PATCH 10/58] Try init_dll_path like in audio. But they only do it on 3.8 --- src/torchcodec/_core/ops.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index 821e96f2b..c657e5855 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -13,6 +13,29 @@ from typing import List, Optional, Tuple, Union import torch + + +def _init_dll_path(): + """Initialize DLL search paths for Windows + + On Windows Python-3.8+ has `os.add_dll_directory` call, + which is called to configure dll search path. + To find FFmpeg and other dependent dlls we need to make sure the + conda environment/bin path and other system paths are configured. + """ + if os.name == "nt": + for path in os.environ.get("PATH", "").split(";"): + if os.path.exists(path): + try: + os.add_dll_directory(path) + except Exception: + # Ignore paths that can't be added + pass + + +# Initialize DLL paths on Windows before attempting to load libraries +if os.name == "nt": + _init_dll_path() from torch.library import get_ctx, register_fake from torchcodec._internally_replaced_utils import ( # @manual=//pytorch/torchcodec/src:internally_replaced_utils From 9573fc21d247d62e92ab9090d8f8abeb63af390f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 13:07:56 +0100 Subject: [PATCH 11/58] revert previous. Now add Python3_FIND_ABI and explicitly set Python3 as dependency of custom_ops target --- src/torchcodec/_core/CMakeLists.txt | 7 +++++++ src/torchcodec/_core/ops.py | 23 ----------------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index ba7495a32..3e4e01ba2 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -6,6 +6,10 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(PYBIND11_FINDPYTHON ON) find_package(pybind11 REQUIRED) find_package(Torch REQUIRED) + +if (WIN32) + set(Python3_FIND_ABI "ANY" "ANY" "ANY" "ANY") +endif() find_package(Python3 ${PYTHON_VERSION} EXACT COMPONENTS Development) if(DEFINED TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR AND TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR) @@ -141,6 +145,9 @@ function(make_torchcodec_libraries ${core_library_name} ${Python3_LIBRARIES} ) + if (WIN32) + list(APPEND custom_ops_dependencies Python3::Python) + endif() make_torchcodec_sublibrary( "${custom_ops_library_name}" SHARED diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index c657e5855..821e96f2b 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -13,29 +13,6 @@ from typing import List, Optional, Tuple, Union import torch - - -def _init_dll_path(): - """Initialize DLL search paths for Windows - - On Windows Python-3.8+ has `os.add_dll_directory` call, - which is called to configure dll search path. - To find FFmpeg and other dependent dlls we need to make sure the - conda environment/bin path and other system paths are configured. - """ - if os.name == "nt": - for path in os.environ.get("PATH", "").split(";"): - if os.path.exists(path): - try: - os.add_dll_directory(path) - except Exception: - # Ignore paths that can't be added - pass - - -# Initialize DLL paths on Windows before attempting to load libraries -if os.name == "nt": - _init_dll_path() from torch.library import get_ctx, register_fake from torchcodec._internally_replaced_utils import ( # @manual=//pytorch/torchcodec/src:internally_replaced_utils From 963590ecd511668f3018fe792d603c25b5128ff9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 13:48:06 +0100 Subject: [PATCH 12/58] Set core lib dependencies as PRIVATE??? --- src/torchcodec/_core/CMakeLists.txt | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 3e4e01ba2..485349f2e 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -71,11 +71,21 @@ function(make_torchcodec_sublibrary ) endif() - target_link_libraries( - ${library_name} - PUBLIC - ${library_dependencies} - ) + # For the core library, use PRIVATE to prevent FFmpeg dependencies from leaking + # For other libraries, use PUBLIC so they can properly link to their dependencies + if(${library_name} MATCHES ".*_core[0-9]+$") + target_link_libraries( + ${library_name} + PRIVATE + ${library_dependencies} + ) + else() + target_link_libraries( + ${library_name} + PUBLIC + ${library_dependencies} + ) + endif() endfunction() From 2f2c91d7940b6c1a1a5bafc6650ccd2a72efa9c5 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 14:07:47 +0100 Subject: [PATCH 13/58] Same logic but try exposing the include directories as public --- src/torchcodec/_core/CMakeLists.txt | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 485349f2e..acc140922 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -71,14 +71,23 @@ function(make_torchcodec_sublibrary ) endif() - # For the core library, use PRIVATE to prevent FFmpeg dependencies from leaking - # For other libraries, use PUBLIC so they can properly link to their dependencies + # For the core library, make dependencies PRIVATE to prevent transitive linking issues + # but still expose include directories if(${library_name} MATCHES ".*_core[0-9]+$") target_link_libraries( ${library_name} PRIVATE ${library_dependencies} ) + # Manually expose FFmpeg include directories as PUBLIC for compilation + foreach(dep ${library_dependencies}) + if(TARGET ${dep}) + get_target_property(dep_include_dirs ${dep} INTERFACE_INCLUDE_DIRECTORIES) + if(dep_include_dirs) + target_include_directories(${library_name} PUBLIC ${dep_include_dirs}) + endif() + endif() + endforeach() else() target_link_libraries( ${library_name} From d8962119383d0356197af6426e75c825f512642a Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 5 Aug 2025 14:39:26 +0100 Subject: [PATCH 14/58] OK at this point I have no idea --- src/torchcodec/_core/CMakeLists.txt | 54 +++++++++++++---------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index acc140922..738feaca6 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -71,30 +71,11 @@ function(make_torchcodec_sublibrary ) endif() - # For the core library, make dependencies PRIVATE to prevent transitive linking issues - # but still expose include directories - if(${library_name} MATCHES ".*_core[0-9]+$") - target_link_libraries( - ${library_name} - PRIVATE - ${library_dependencies} - ) - # Manually expose FFmpeg include directories as PUBLIC for compilation - foreach(dep ${library_dependencies}) - if(TARGET ${dep}) - get_target_property(dep_include_dirs ${dep} INTERFACE_INCLUDE_DIRECTORIES) - if(dep_include_dirs) - target_include_directories(${library_name} PUBLIC ${dep_include_dirs}) - endif() - endif() - endforeach() - else() - target_link_libraries( - ${library_name} - PUBLIC - ${library_dependencies} - ) - endif() + target_link_libraries( + ${library_name} + PUBLIC + ${library_dependencies} + ) endfunction() @@ -160,12 +141,20 @@ function(make_torchcodec_libraries AVIOTensorContext.cpp custom_ops.cpp ) - set(custom_ops_dependencies - ${core_library_name} - ${Python3_LIBRARIES} - ) - if (WIN32) - list(APPEND custom_ops_dependencies Python3::Python) + # On Windows, custom_ops should not transitively link to FFmpeg to avoid DLL issues + if(WIN32) + set(custom_ops_dependencies + ${Python3_LIBRARIES} + Python3::Python + ) + # Link to core library but without transitive dependencies + set(custom_ops_core_dependency ${core_library_name}) + else() + set(custom_ops_dependencies + ${core_library_name} + ${Python3_LIBRARIES} + ) + set(custom_ops_core_dependency "") endif() make_torchcodec_sublibrary( "${custom_ops_library_name}" @@ -173,6 +162,11 @@ function(make_torchcodec_libraries "${custom_ops_sources}" "${custom_ops_dependencies}" ) + + # On Windows, link to core library privately to avoid transitive FFmpeg dependencies + if(WIN32 AND custom_ops_core_dependency) + target_link_libraries(${custom_ops_library_name} PRIVATE ${custom_ops_core_dependency}) + endif() # 3. Create libtorchcodec_pybind_opsN.so. From 01f9dc0b4737d6ad9dfc15cab02b8fdee99f19a2 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 08:48:23 +0100 Subject: [PATCH 15/58] Add test job on Windows. --- .github/workflows/windows_wheel.yaml | 216 +++++++++++++++++---------- 1 file changed, 140 insertions(+), 76 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index df263c812..c1e34c8da 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -61,83 +61,147 @@ jobs: # set in vc_env_helper.bat Couldn't find a way to set it from here. build-command: "python -m build --wheel -vvv --no-isolation" - install-and-test: - runs-on: windows-latest + # install-and-test: + # runs-on: windows-latest + # strategy: + # fail-fast: false + # matrix: + # python-version: ['3.9'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # needs: build + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 + # path: pytorch/torchcodec/dist/ + # - name: Setup conda env + # uses: conda-incubator/setup-miniconda@v2 + # with: + # auto-update-conda: true + # miniconda-version: "latest" + # activate-environment: test + # python-version: ${{ matrix.python-version }} + # - name: Update pip + # run: python -m pip install --upgrade pip + # - name: Install PyTorch + # run: | + # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + # echo Installing $wheel_path + # python -m pip install $wheel_path -vvv + # - name: Check out repo + # uses: actions/checkout@v3 + # - name: Install ffmpeg, post build + # run: | + # # Ideally we would have checked for that before installing the wheel, + # # but we need to checkout the repo to access this file, and we don't + # # want to checkout the repo before installing the wheel to avoid any + # # side-effect. It's OK. + # source packaging/helpers.sh + # assert_ffmpeg_not_installed + # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + # ffmpeg -version + # # On Windows, ensure the conda Library/bin directory is in PATH + # # This is needed for Windows DLL loading to find FFmpeg DLLs + # if [[ "$RUNNER_OS" == "Windows" ]]; then + # conda_env_path=$(conda info --base)/envs/test + # library_bin_path="$conda_env_path/Library/bin" + # echo "Adding conda Library/bin to PATH: $library_bin_path" + # echo "$library_bin_path" >> $GITHUB_PATH + # # Verify FFmpeg DLLs are accessible + # echo "Checking if FFmpeg DLLs are in PATH:" + # where avutil.dll || echo "avutil.dll not found in PATH" + # where avcodec.dll || echo "avcodec.dll not found in PATH" + # where avformat.dll || echo "avformat.dll not found in PATH" + # fi + # - name: Test torchcodec import after FFmpeg installation + # run: | + # echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." + # python -c "import torchcodec; print('TorchCodec import successful!')" + # - name: Install test dependencies + # run: | + # # Ideally we would find a way to get those dependencies from pyproject.toml + # python -m pip install numpy pytest pillow + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls + # - name: Run Python tests + # run: | + # pytest test -vvv + + install-and-test-on-test-infra: strategy: - fail-fast: false matrix: - python-version: ['3.9'] + python-version: + - "3.9" + # - "3.10" + # - "3.11" + # - "3.12" + runner: ["windows.4xlarge"] + gpu-arch-type: ["cpu"] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # TODO: put GPU testing back + # include: + # - python-version: "3.9" + # runner: windows.g5.4xlarge.nvidia.gpu + # gpu-arch-type: cuda + # gpu-arch-version: "11.8" + fail-fast: false needs: build - steps: - - uses: actions/download-artifact@v4 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 - path: pytorch/torchcodec/dist/ - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 - - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - source packaging/helpers.sh - assert_ffmpeg_not_installed - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version - # On Windows, ensure the conda Library/bin directory is in PATH - # This is needed for Windows DLL loading to find FFmpeg DLLs - if [[ "$RUNNER_OS" == "Windows" ]]; then - conda_env_path=$(conda info --base)/envs/test - library_bin_path="$conda_env_path/Library/bin" - echo "Adding conda Library/bin to PATH: $library_bin_path" - echo "$library_bin_path" >> $GITHUB_PATH - # Verify FFmpeg DLLs are accessible - echo "Checking if FFmpeg DLLs are in PATH:" - where avutil.dll || echo "avutil.dll not found in PATH" - where avcodec.dll || echo "avcodec.dll not found in PATH" - where avformat.dll || echo "avformat.dll not found in PATH" - fi - - name: Test torchcodec import after FFmpeg installation - run: | - echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." - python -c "import torchcodec; print('TorchCodec import successful!')" - - name: Install test dependencies - run: | - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Run Python tests - run: | - pytest test -vvv + uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + permissions: + id-token: write + contents: read + with: + repository: pytorch/torchcodec + runner: ${{ matrix.runner }} + gpu-arch-type: ${{ matrix.gpu-arch-type }} + gpu-arch-version: ${{ matrix.gpu-arch-version }} + timeout: 120 + test-infra-ref: main + script: | + set -euxo pipefail + + export PYTHON_VERSION=${{ matrix.python-version }} + export VC_YEAR=2022 + export VSDEVCMD_ARGS="" + export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} + export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} + export FFMPEG_VERSION_FOR_TESTS=${{ matrix.ffmpeg-version-for-tests }} + + # Download and install wheel + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + wheel_path=$(find pytorch/torchcodec/dist -type f -name "*${{ matrix.python-version }}*.whl" | head -1) + echo "Installing $wheel_path" + python -m pip install "$wheel_path" -vvv + + # Install FFmpeg and test dependencies + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + conda_env_path=$(conda info --base)/envs/test + library_bin_path="$conda_env_path/Library/bin" + echo "$library_bin_path" >> $GITHUB_PATH + + # Install test dependencies + python -m pip install numpy pytest pillow + + # Test torchcodec import + python -c "import torchcodec; print('TorchCodec import successful!')" + + # Remove source to ensure testing against wheel + rm -rf src/ + + # Run tests + pytest test -vvv From 60df9089740602daf908919ae426c387dc4bb442 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 08:48:58 +0100 Subject: [PATCH 16/58] Revert "OK at this point I have no idea" This reverts commit d8962119383d0356197af6426e75c825f512642a. --- src/torchcodec/_core/CMakeLists.txt | 54 ++++++++++++++++------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 738feaca6..acc140922 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -71,11 +71,30 @@ function(make_torchcodec_sublibrary ) endif() - target_link_libraries( - ${library_name} - PUBLIC - ${library_dependencies} - ) + # For the core library, make dependencies PRIVATE to prevent transitive linking issues + # but still expose include directories + if(${library_name} MATCHES ".*_core[0-9]+$") + target_link_libraries( + ${library_name} + PRIVATE + ${library_dependencies} + ) + # Manually expose FFmpeg include directories as PUBLIC for compilation + foreach(dep ${library_dependencies}) + if(TARGET ${dep}) + get_target_property(dep_include_dirs ${dep} INTERFACE_INCLUDE_DIRECTORIES) + if(dep_include_dirs) + target_include_directories(${library_name} PUBLIC ${dep_include_dirs}) + endif() + endif() + endforeach() + else() + target_link_libraries( + ${library_name} + PUBLIC + ${library_dependencies} + ) + endif() endfunction() @@ -141,20 +160,12 @@ function(make_torchcodec_libraries AVIOTensorContext.cpp custom_ops.cpp ) - # On Windows, custom_ops should not transitively link to FFmpeg to avoid DLL issues - if(WIN32) - set(custom_ops_dependencies - ${Python3_LIBRARIES} - Python3::Python - ) - # Link to core library but without transitive dependencies - set(custom_ops_core_dependency ${core_library_name}) - else() - set(custom_ops_dependencies - ${core_library_name} - ${Python3_LIBRARIES} - ) - set(custom_ops_core_dependency "") + set(custom_ops_dependencies + ${core_library_name} + ${Python3_LIBRARIES} + ) + if (WIN32) + list(APPEND custom_ops_dependencies Python3::Python) endif() make_torchcodec_sublibrary( "${custom_ops_library_name}" @@ -162,11 +173,6 @@ function(make_torchcodec_libraries "${custom_ops_sources}" "${custom_ops_dependencies}" ) - - # On Windows, link to core library privately to avoid transitive FFmpeg dependencies - if(WIN32 AND custom_ops_core_dependency) - target_link_libraries(${custom_ops_library_name} PRIVATE ${custom_ops_core_dependency}) - endif() # 3. Create libtorchcodec_pybind_opsN.so. From 09d077e44f3cfb3ce9989f2510fb21b829adb661 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 08:49:00 +0100 Subject: [PATCH 17/58] Revert "Same logic but try exposing the include directories as public" This reverts commit 2f2c91d7940b6c1a1a5bafc6650ccd2a72efa9c5. --- src/torchcodec/_core/CMakeLists.txt | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index acc140922..485349f2e 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -71,23 +71,14 @@ function(make_torchcodec_sublibrary ) endif() - # For the core library, make dependencies PRIVATE to prevent transitive linking issues - # but still expose include directories + # For the core library, use PRIVATE to prevent FFmpeg dependencies from leaking + # For other libraries, use PUBLIC so they can properly link to their dependencies if(${library_name} MATCHES ".*_core[0-9]+$") target_link_libraries( ${library_name} PRIVATE ${library_dependencies} ) - # Manually expose FFmpeg include directories as PUBLIC for compilation - foreach(dep ${library_dependencies}) - if(TARGET ${dep}) - get_target_property(dep_include_dirs ${dep} INTERFACE_INCLUDE_DIRECTORIES) - if(dep_include_dirs) - target_include_directories(${library_name} PUBLIC ${dep_include_dirs}) - endif() - endif() - endforeach() else() target_link_libraries( ${library_name} From 61ef07b7a88807369d16ca48e3f05c359ff34286 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 08:49:01 +0100 Subject: [PATCH 18/58] Revert "Set core lib dependencies as PRIVATE???" This reverts commit 963590ecd511668f3018fe792d603c25b5128ff9. --- src/torchcodec/_core/CMakeLists.txt | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 485349f2e..3e4e01ba2 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -71,21 +71,11 @@ function(make_torchcodec_sublibrary ) endif() - # For the core library, use PRIVATE to prevent FFmpeg dependencies from leaking - # For other libraries, use PUBLIC so they can properly link to their dependencies - if(${library_name} MATCHES ".*_core[0-9]+$") - target_link_libraries( - ${library_name} - PRIVATE - ${library_dependencies} - ) - else() - target_link_libraries( - ${library_name} - PUBLIC - ${library_dependencies} - ) - endif() + target_link_libraries( + ${library_name} + PUBLIC + ${library_dependencies} + ) endfunction() From 1ae09aac501381252e2afade759058c49c0be079 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 09:28:38 +0100 Subject: [PATCH 19/58] debug --- .github/workflows/windows_wheel.yaml | 62 +++++++++++++++++----------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index c1e34c8da..1f9d4860f 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -37,29 +37,29 @@ jobs: with-cuda: disable build-python-only: "disable" - build: - needs: generate-matrix - strategy: - fail-fast: false - name: Build and Upload Windows wheel - # TODO: use @main - uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows - with: - repository: pytorch/torchcodec - ref: "" - test-infra-repository: nicolashug/test-infra - test-infra-ref: build-platform-windows - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: packaging/pre_build_script.sh - # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec - trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is - # set in vc_env_helper.bat Couldn't find a way to set it from here. - build-command: "python -m build --wheel -vvv --no-isolation" + # build: + # needs: generate-matrix + # strategy: + # fail-fast: false + # name: Build and Upload Windows wheel + # # TODO: use @main + # uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows + # with: + # repository: pytorch/torchcodec + # ref: "" + # test-infra-repository: nicolashug/test-infra + # test-infra-ref: build-platform-windows + # build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + # pre-script: packaging/pre_build_script.sh + # # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows + # env-script: packaging/vc_env_helper.bat + # smoke-test-script: packaging/fake_smoke_test.py + # package-name: torchcodec + # trigger-event: ${{ github.event_name }} + # build-platform: "python-build-package" + # # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is + # # set in vc_env_helper.bat Couldn't find a way to set it from here. + # build-command: "python -m build --wheel -vvv --no-isolation" # install-and-test: # runs-on: windows-latest @@ -152,7 +152,8 @@ jobs: # - "3.12" runner: ["windows.4xlarge"] gpu-arch-type: ["cpu"] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + ffmpeg-version-for-tests: ['6.1.1'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] # TODO: put GPU testing back # include: # - python-version: "3.9" @@ -160,7 +161,8 @@ jobs: # gpu-arch-type: cuda # gpu-arch-version: "11.8" fail-fast: false - needs: build + # needs: build + needs: generate-matrix # TODO REMOVE uses: pytorch/test-infra/.github/workflows/windows_job.yml@main permissions: id-token: write @@ -182,6 +184,16 @@ jobs: export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} export FFMPEG_VERSION_FOR_TESTS=${{ matrix.ffmpeg-version-for-tests }} + # Prepare conda + set +x && eval "$($(which conda) shell.bash hook)" && set -x + + echo '::group::Create build environment' + conda create \ + --name ci \ + --quiet --yes \ + python="${PYTHON_VERSION}" pip \ + conda activate ci + # Download and install wheel python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu wheel_path=$(find pytorch/torchcodec/dist -type f -name "*${{ matrix.python-version }}*.whl" | head -1) From 6921d4c12f931fa04803b792196c18e6f1156c0c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 09:32:26 +0100 Subject: [PATCH 20/58] oops --- .github/workflows/build_ffmpeg.yaml | 2 +- .github/workflows/windows_wheel.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_ffmpeg.yaml b/.github/workflows/build_ffmpeg.yaml index 4d6a22869..e52a67864 100644 --- a/.github/workflows/build_ffmpeg.yaml +++ b/.github/workflows/build_ffmpeg.yaml @@ -11,7 +11,7 @@ name: Build non-GPL FFmpeg from source on: workflow_dispatch: - pull_request: + # pull_request: paths: - packaging/build_ffmpeg.sh - .github/workflows/build_ffmpeg.yaml # self reference diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 1f9d4860f..e094dac20 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -191,7 +191,8 @@ jobs: conda create \ --name ci \ --quiet --yes \ - python="${PYTHON_VERSION}" pip \ + python="${PYTHON_VERSION}" pip + conda activate ci # Download and install wheel From dab2d8ed812eacd23e1579bb82c71ad96f60990f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 09:37:00 +0100 Subject: [PATCH 21/58] Continuing --- .github/workflows/cpp_tests.yaml | 2 +- .github/workflows/windows_wheel.yaml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index e08d90754..227e40e12 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -3,7 +3,7 @@ name: CPP tests on: push: branches: [ main ] - pull_request: + # pull_request: concurrency: group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index e094dac20..5c3bd90c1 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -161,8 +161,7 @@ jobs: # gpu-arch-type: cuda # gpu-arch-version: "11.8" fail-fast: false - # needs: build - needs: generate-matrix # TODO REMOVE + needs: build uses: pytorch/test-infra/.github/workflows/windows_job.yml@main permissions: id-token: write From ff91aca1776a73cdacc41aef511a67caeddad24f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 09:37:34 +0100 Subject: [PATCH 22/58] oops --- .github/workflows/windows_wheel.yaml | 46 ++++++++++++++-------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 5c3bd90c1..57783f43e 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -37,29 +37,29 @@ jobs: with-cuda: disable build-python-only: "disable" - # build: - # needs: generate-matrix - # strategy: - # fail-fast: false - # name: Build and Upload Windows wheel - # # TODO: use @main - # uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows - # with: - # repository: pytorch/torchcodec - # ref: "" - # test-infra-repository: nicolashug/test-infra - # test-infra-ref: build-platform-windows - # build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - # pre-script: packaging/pre_build_script.sh - # # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows - # env-script: packaging/vc_env_helper.bat - # smoke-test-script: packaging/fake_smoke_test.py - # package-name: torchcodec - # trigger-event: ${{ github.event_name }} - # build-platform: "python-build-package" - # # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is - # # set in vc_env_helper.bat Couldn't find a way to set it from here. - # build-command: "python -m build --wheel -vvv --no-isolation" + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload Windows wheel + # TODO: use @main + uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: nicolashug/test-infra + test-infra-ref: build-platform-windows + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows + env-script: packaging/vc_env_helper.bat + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is + # set in vc_env_helper.bat Couldn't find a way to set it from here. + build-command: "python -m build --wheel -vvv --no-isolation" # install-and-test: # runs-on: windows-latest From bc919cc2018e67585a4602852a8b2d085f61601b Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 10:49:48 +0100 Subject: [PATCH 23/58] Try to build from source --- .github/workflows/windows_wheel.yaml | 79 ++++++++++++++-------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 57783f43e..f39ccb037 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -37,29 +37,29 @@ jobs: with-cuda: disable build-python-only: "disable" - build: - needs: generate-matrix - strategy: - fail-fast: false - name: Build and Upload Windows wheel - # TODO: use @main - uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows - with: - repository: pytorch/torchcodec - ref: "" - test-infra-repository: nicolashug/test-infra - test-infra-ref: build-platform-windows - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: packaging/pre_build_script.sh - # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows - env-script: packaging/vc_env_helper.bat - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec - trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is - # set in vc_env_helper.bat Couldn't find a way to set it from here. - build-command: "python -m build --wheel -vvv --no-isolation" + # build: + # needs: generate-matrix + # strategy: + # fail-fast: false + # name: Build and Upload Windows wheel + # # TODO: use @main + # uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows + # with: + # repository: pytorch/torchcodec + # ref: "" + # test-infra-repository: nicolashug/test-infra + # test-infra-ref: build-platform-windows + # build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + # pre-script: packaging/pre_build_script.sh + # # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows + # env-script: packaging/vc_env_helper.bat + # smoke-test-script: packaging/fake_smoke_test.py + # package-name: torchcodec + # trigger-event: ${{ github.event_name }} + # build-platform: "python-build-package" + # # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is + # # set in vc_env_helper.bat Couldn't find a way to set it from here. + # build-command: "python -m build --wheel -vvv --no-isolation" # install-and-test: # runs-on: windows-latest @@ -161,7 +161,8 @@ jobs: # gpu-arch-type: cuda # gpu-arch-version: "11.8" fail-fast: false - needs: build + # needs: build + needs: generate-matrix ## TODO REMOVE uses: pytorch/test-infra/.github/workflows/windows_job.yml@main permissions: id-token: write @@ -194,26 +195,26 @@ jobs: conda activate ci - # Download and install wheel + # Install PyTorch python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - wheel_path=$(find pytorch/torchcodec/dist -type f -name "*${{ matrix.python-version }}*.whl" | head -1) - echo "Installing $wheel_path" - python -m pip install "$wheel_path" -vvv - - # Install FFmpeg and test dependencies - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - conda_env_path=$(conda info --base)/envs/test + + # Install build dependencies + conda install cmake pkg-config pybind11 "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + + # Ensure FFmpeg DLLs are in PATH for Windows + conda_env_path=$(conda info --base)/envs/ci library_bin_path="$conda_env_path/Library/bin" + echo "Adding conda Library/bin to PATH: $library_bin_path" echo "$library_bin_path" >> $GITHUB_PATH - # Install test dependencies - python -m pip install numpy pytest pillow - + # Verify FFmpeg is available + ffmpeg -version + + # Install torchcodec from source in development mode + pip install -e ".[dev]" --no-build-isolation -vv + # Test torchcodec import python -c "import torchcodec; print('TorchCodec import successful!')" - - # Remove source to ensure testing against wheel - rm -rf src/ - + # Run tests pytest test -vvv From d9883931bb77d3a1d3cd723532ce2bfaa02f00eb Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 12:26:22 +0100 Subject: [PATCH 24/58] DEBUG stuff --- .github/workflows/windows_wheel.yaml | 29 ++++++++++++++++++++++++++-- src/torchcodec/_core/ops.py | 10 ++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index f39ccb037..8e835ffb8 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -183,6 +183,7 @@ jobs: export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} export FFMPEG_VERSION_FOR_TESTS=${{ matrix.ffmpeg-version-for-tests }} + export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 # Prepare conda set +x && eval "$($(which conda) shell.bash hook)" && set -x @@ -213,8 +214,32 @@ jobs: # Install torchcodec from source in development mode pip install -e ".[dev]" --no-build-isolation -vv - # Test torchcodec import - python -c "import torchcodec; print('TorchCodec import successful!')" + # Add torchcodec source directory to DLL search path for Windows + torchcodec_dll_path="$(pwd)/src/torchcodec" + echo "Adding torchcodec DLL path: $torchcodec_dll_path" + export PATH="$torchcodec_dll_path:$PATH" + + # Test torchcodec import with detailed error reporting + python -c " +import torch +print(f'PyTorch version: {torch.__version__}') + +try: + import torchcodec + print('TorchCodec import successful!') +except Exception as e: + print(f'TorchCodec import failed: {e}') + import traceback + traceback.print_exc() + + # Try to get more specific error info + try: + from torchcodec._core import ops + print('Core ops import successful') + except Exception as ops_e: + print(f'Core ops import failed: {ops_e}') + traceback.print_exc() +" # Run tests pytest test -vvv diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index 821e96f2b..7d91c36ea 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -58,6 +58,16 @@ def load_torchcodec_shared_libraries(): if sys.platform == "win32": try: import ctypes + + # Add the DLL directory to help with dependency resolution + dll_dir = os.path.dirname(core_path) + if hasattr(os, 'add_dll_directory'): + try: + os.add_dll_directory(dll_dir) + print(f"Added DLL directory: {dll_dir}", flush=True) + except Exception as add_dir_e: + print(f"Could not add DLL directory: {add_dir_e}", flush=True) + # Preload the core library so it's available for dependency resolution ctypes.CDLL(core_path) print(f"Preloaded core library via ctypes", flush=True) From 9da65393ba4d9beef4914a4ea57cc1b4a746bc6f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 6 Aug 2025 12:28:51 +0100 Subject: [PATCH 25/58] fix --- .github/workflows/windows_wheel.yaml | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 8e835ffb8..6af0ec1eb 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -220,26 +220,7 @@ jobs: export PATH="$torchcodec_dll_path:$PATH" # Test torchcodec import with detailed error reporting - python -c " -import torch -print(f'PyTorch version: {torch.__version__}') - -try: - import torchcodec - print('TorchCodec import successful!') -except Exception as e: - print(f'TorchCodec import failed: {e}') - import traceback - traceback.print_exc() - - # Try to get more specific error info - try: - from torchcodec._core import ops - print('Core ops import successful') - except Exception as ops_e: - print(f'Core ops import failed: {ops_e}') - traceback.print_exc() -" + python -c "import torch; import torchcodec" # Run tests pytest test -vvv From 99d0308c7f344fa91cd13605a7be7ec0245fd7d8 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 7 Aug 2025 12:58:43 +0100 Subject: [PATCH 26/58] empty From 7247adca8c6068deaf90ec700a3a39a7bd9c7cf6 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 7 Aug 2025 14:34:19 +0100 Subject: [PATCH 27/58] Make it a MODULE????? --- src/torchcodec/_core/CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 3e4e01ba2..22b262074 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -144,17 +144,23 @@ function(make_torchcodec_libraries set(custom_ops_dependencies ${core_library_name} ${Python3_LIBRARIES} + ${TORCH_LIBRARIES} # Add PyTorch libraries for custom ops registration ) if (WIN32) list(APPEND custom_ops_dependencies Python3::Python) endif() make_torchcodec_sublibrary( "${custom_ops_library_name}" - SHARED + MODULE # Use MODULE like pybind_ops instead of SHARED "${custom_ops_sources}" "${custom_ops_dependencies}" ) + # Apply same Windows handling as pybind_ops + if(WIN32) + set_target_properties(${custom_ops_library_name} PROPERTIES SUFFIX ".pyd") + endif() + # 3. Create libtorchcodec_pybind_opsN.so. set(pybind_ops_library_name "libtorchcodec_pybind_ops${ffmpeg_major_version}") From 11a7c5527cb6e83ab476f5e2d3af40b4fee5803d Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 9 Aug 2025 16:53:06 +0100 Subject: [PATCH 28/58] Put back wheel build and wheel test --- .github/workflows/windows_wheel.yaml | 211 ++++++++++++++------------- 1 file changed, 106 insertions(+), 105 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 6af0ec1eb..20b82a538 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -37,118 +37,119 @@ jobs: with-cuda: disable build-python-only: "disable" - # build: - # needs: generate-matrix - # strategy: - # fail-fast: false - # name: Build and Upload Windows wheel - # # TODO: use @main - # uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows - # with: - # repository: pytorch/torchcodec - # ref: "" - # test-infra-repository: nicolashug/test-infra - # test-infra-ref: build-platform-windows - # build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - # pre-script: packaging/pre_build_script.sh - # # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows - # env-script: packaging/vc_env_helper.bat - # smoke-test-script: packaging/fake_smoke_test.py - # package-name: torchcodec - # trigger-event: ${{ github.event_name }} - # build-platform: "python-build-package" - # # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is - # # set in vc_env_helper.bat Couldn't find a way to set it from here. - # build-command: "python -m build --wheel -vvv --no-isolation" + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload Windows wheel + # TODO: use @main + uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: nicolashug/test-infra + test-infra-ref: build-platform-windows + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows + env-script: packaging/vc_env_helper.bat + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is + # set in vc_env_helper.bat Couldn't find a way to set it from here. + build-command: "python -m build --wheel -vvv --no-isolation" - # install-and-test: - # runs-on: windows-latest - # strategy: - # fail-fast: false - # matrix: - # python-version: ['3.9'] - # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - # needs: build - # steps: - # - uses: actions/download-artifact@v4 - # with: - # name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 - # path: pytorch/torchcodec/dist/ - # - name: Setup conda env - # uses: conda-incubator/setup-miniconda@v2 - # with: - # auto-update-conda: true - # miniconda-version: "latest" - # activate-environment: test - # python-version: ${{ matrix.python-version }} - # - name: Update pip - # run: python -m pip install --upgrade pip - # - name: Install PyTorch - # run: | - # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - # - name: Install torchcodec from the wheel - # run: | - # wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - # echo Installing $wheel_path - # python -m pip install $wheel_path -vvv - # - name: Check out repo - # uses: actions/checkout@v3 - # - name: Install ffmpeg, post build - # run: | - # # Ideally we would have checked for that before installing the wheel, - # # but we need to checkout the repo to access this file, and we don't - # # want to checkout the repo before installing the wheel to avoid any - # # side-effect. It's OK. - # source packaging/helpers.sh - # assert_ffmpeg_not_installed - # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - # ffmpeg -version - # # On Windows, ensure the conda Library/bin directory is in PATH - # # This is needed for Windows DLL loading to find FFmpeg DLLs - # if [[ "$RUNNER_OS" == "Windows" ]]; then - # conda_env_path=$(conda info --base)/envs/test - # library_bin_path="$conda_env_path/Library/bin" - # echo "Adding conda Library/bin to PATH: $library_bin_path" - # echo "$library_bin_path" >> $GITHUB_PATH - # # Verify FFmpeg DLLs are accessible - # echo "Checking if FFmpeg DLLs are in PATH:" - # where avutil.dll || echo "avutil.dll not found in PATH" - # where avcodec.dll || echo "avcodec.dll not found in PATH" - # where avformat.dll || echo "avformat.dll not found in PATH" - # fi - # - name: Test torchcodec import after FFmpeg installation - # run: | - # echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." - # python -c "import torchcodec; print('TorchCodec import successful!')" - # - name: Install test dependencies - # run: | - # # Ideally we would find a way to get those dependencies from pyproject.toml - # python -m pip install numpy pytest pillow - # - name: Delete the src/ folder just for fun - # run: | - # # The only reason we checked-out the repo is to get access to the - # # tests. We don't care about the rest. Out of precaution, we delete - # # the src/ folder to be extra sure that we're running the code from - # # the installed wheel rather than from the source. - # # This is just to be extra cautious and very overkill because a) - # # there's no way the `torchcodec` package from src/ can be found from - # # the PythonPath: the main point of `src/` is precisely to protect - # # against that and b) if we ever were to execute code from - # # `src/torchcodec`, it would fail loudly because the built .so files - # # aren't present there. - # rm -r src/ - # ls - # - name: Run Python tests - # run: | - # pytest test -vvv + install-and-test: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + ffmpeg-version-for-tests: ['6.1.1'] + needs: build + steps: + - uses: actions/download-artifact@v4 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 + path: pytorch/torchcodec/dist/ + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install PyTorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + source packaging/helpers.sh + assert_ffmpeg_not_installed + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version + # On Windows, ensure the conda Library/bin directory is in PATH + # This is needed for Windows DLL loading to find FFmpeg DLLs + if [[ "$RUNNER_OS" == "Windows" ]]; then + conda_env_path=$(conda info --base)/envs/test + library_bin_path="$conda_env_path/Library/bin" + echo "Adding conda Library/bin to PATH: $library_bin_path" + echo "$library_bin_path" >> $GITHUB_PATH + # Verify FFmpeg DLLs are accessible + echo "Checking if FFmpeg DLLs are in PATH:" + where avutil.dll || echo "avutil.dll not found in PATH" + where avcodec.dll || echo "avcodec.dll not found in PATH" + where avformat.dll || echo "avformat.dll not found in PATH" + fi + - name: Test torchcodec import after FFmpeg installation + run: | + echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." + python -c "import torchcodec; print('TorchCodec import successful!')" + - name: Install test dependencies + run: | + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Run Python tests + run: | + pytest test -vvv install-and-test-on-test-infra: strategy: matrix: python-version: - - "3.9" + # - "3.9" # - "3.10" - # - "3.11" + - "3.11" # - "3.12" runner: ["windows.4xlarge"] gpu-arch-type: ["cpu"] From 2bdb64321a517db41110ed0b1fbdfe20161f2e17 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Sat, 9 Aug 2025 17:38:37 +0100 Subject: [PATCH 29/58] add more python versions --- .github/workflows/windows_wheel.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 20b82a538..6a0292fdb 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -66,7 +66,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9'] + python-version: ['3.9', '3.10', '3.11', '3.12'] # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] ffmpeg-version-for-tests: ['6.1.1'] needs: build @@ -147,8 +147,8 @@ jobs: strategy: matrix: python-version: - # - "3.9" - # - "3.10" + - "3.9" + - "3.10" - "3.11" # - "3.12" runner: ["windows.4xlarge"] From 7b12d7d9b86964654b81b64981d991f0301d8434 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Tue, 19 Aug 2025 12:54:28 +0200 Subject: [PATCH 30/58] Ensure that the CMake build type is respected alsof for multi-config generatros --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 895b37985..24f480c0c 100644 --- a/setup.py +++ b/setup.py @@ -135,9 +135,9 @@ def _build_all_extensions_with_cmake(self): ["cmake", str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp ) print("Calling cmake --build", flush=True) - subprocess.check_call(["cmake", "--build", "."], cwd=self.build_temp) + subprocess.check_call(["cmake", "--build", ".","--config",cmake_build_type], cwd=self.build_temp) print("Calling cmake --install", flush=True) - subprocess.check_call(["cmake", "--install", "."], cwd=self.build_temp) + subprocess.check_call(["cmake", "--install", ".","--config",cmake_build_type], cwd=self.build_temp) def copy_extensions_to_source(self): """Copy built extensions from temporary folder back into source tree. From cbff5159e6d6b4a285b0dc369e63aca649fa6782 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 14:44:58 +0100 Subject: [PATCH 31/58] try not using MSVC ffmpeg --- src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 71aea7dc4..1a7d511e6 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -152,7 +152,7 @@ elseif (WIN32) set(lib_dir "bin") # Option to use MSVC-compatible FFmpeg libraries on Windows - option(USE_MSVC_FFMPEG "Use MSVC-compatible FFmpeg libraries on Windows" ON) + option(USE_MSVC_FFMPEG "Use MSVC-compatible FFmpeg libraries on Windows" OFF) if(USE_MSVC_FFMPEG) message(STATUS "Using MSVC-compatible FFmpeg libraries for Windows") From f7c8bd5bb78dd57c3f47c005046d1ecca3662146 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 14:48:14 +0100 Subject: [PATCH 32/58] Remove windows-specific debug in ops --- src/torchcodec/_core/ops.py | 62 ++----------------------------------- 1 file changed, 3 insertions(+), 59 deletions(-) diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index 7d91c36ea..2bd02ef48 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -50,30 +50,9 @@ def load_torchcodec_shared_libraries(): pybind_ops_library_name = f"libtorchcodec_pybind_ops{ffmpeg_major_version}" try: print(f"Loading {decoder_library_name}", flush=True) - core_path = _get_extension_path(decoder_library_name) - torch.ops.load_library(core_path) + torch.ops.load_library(_get_extension_path(decoder_library_name)) print(f"successfully loaded {decoder_library_name}", flush=True) - # On Windows, try to preload the core DLL to help resolve dependencies - if sys.platform == "win32": - try: - import ctypes - - # Add the DLL directory to help with dependency resolution - dll_dir = os.path.dirname(core_path) - if hasattr(os, 'add_dll_directory'): - try: - os.add_dll_directory(dll_dir) - print(f"Added DLL directory: {dll_dir}", flush=True) - except Exception as add_dir_e: - print(f"Could not add DLL directory: {add_dir_e}", flush=True) - - # Preload the core library so it's available for dependency resolution - ctypes.CDLL(core_path) - print(f"Preloaded core library via ctypes", flush=True) - except Exception as preload_e: - print(f"Could not preload core library: {preload_e}", flush=True) - print(f"Loading {custom_ops_library_name}", flush=True) torch.ops.load_library(_get_extension_path(custom_ops_library_name)) print(f"successfully loaded {custom_ops_library_name}", flush=True) @@ -87,43 +66,8 @@ def load_torchcodec_shared_libraries(): print(f"successfully loaded {pybind_ops_library_name}", flush=True) return except Exception as e: - # Enhanced error reporting with Windows-specific details - error_msg = str(e) - - # On Windows, try to get more detailed DLL loading error information - if sys.platform == "win32": - try: - import ctypes - # Get the last Windows error code - win_error = ctypes.get_last_error() - if win_error != 0: - win_error_msg = ctypes.FormatError(win_error) - error_msg += f" [Windows Error {win_error}: {win_error_msg}]" - except Exception: - pass - - # Check if the files actually exist - try: - core_path = _get_extension_path(decoder_library_name) - custom_ops_path = _get_extension_path(custom_ops_library_name) - pybind_ops_path = _get_extension_path(pybind_ops_library_name) - - paths_info = [] - for name, path in [ - ("core", core_path), - ("custom_ops", custom_ops_path), - ("pybind_ops", pybind_ops_path) - ]: - exists = os.path.exists(path) - size = os.path.getsize(path) if exists else 0 - paths_info.append(f"{name}: {path} (exists: {exists}, size: {size})") - - error_msg += f" [Library paths: {'; '.join(paths_info)}]" - except Exception as path_e: - error_msg += f" [Could not check library paths: {path_e}]" - - exceptions.append((ffmpeg_major_version, error_msg)) - + exceptions.append((ffmpeg_major_version, e)) + traceback = ( "\n[start of libtorchcodec loading traceback]\n" + "\n".join(f"FFmpeg version {v}: {str(e)}" for v, e in exceptions) From 110f5b0b0e80ed0d6cc702a34ace6685a3af2fb1 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 15:37:07 +0100 Subject: [PATCH 33/58] remove USE_MSVC_FFMPEG --- ...fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 70 ++++++------------- 1 file changed, 21 insertions(+), 49 deletions(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 1a7d511e6..1a3ccdd37 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -150,55 +150,27 @@ elseif (APPLE) elseif (WIN32) set(lib_dir "bin") - - # Option to use MSVC-compatible FFmpeg libraries on Windows - option(USE_MSVC_FFMPEG "Use MSVC-compatible FFmpeg libraries on Windows" OFF) - - if(USE_MSVC_FFMPEG) - message(STATUS "Using MSVC-compatible FFmpeg libraries for Windows") - set( - platform_url - ${base_url}/windows_x86_64_msvc - ) - set( - f4_sha256 - 407c0f4dea134050261b3e29ec75ca146afc50c5c1fbdd63b48507c70ccce6d4 - ) - set( - f5_sha256 - fa3850be150d64efb7fa3f91da4e488621ee03cd8041623b727e5ea3281a1574 - ) - set( - f6_sha256 - 3d65fca9597b9be7d9e3e11e1a4f4a9aa87c622c18e47abca752700a6ac92e8c - ) - set( - f7_sha256 - c1888939320b2c7de1d3ea687147f5b18aec42dd4797f59a818c840c5a4e5ee2 - ) - else() - message(STATUS "Using MinGW-compatible FFmpeg libraries for Windows") - set( - platform_url - ${base_url}/windows_x86_64 - ) - set( - f4_sha256 - 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb - ) - set( - f5_sha256 - b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 - ) - set( - f6_sha256 - 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a - ) - set( - f7_sha256 - ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 - ) - endif() + message(STATUS "Using MinGW-compatible FFmpeg libraries for Windows") + set( + platform_url + ${base_url}/windows_x86_64 + ) + set( + f4_sha256 + 270a1aa8892225267e68a7eb87c417931da30dccbf08ee2bde8833e659cab5cb + ) + set( + f5_sha256 + b8b2a349a847e56a6da875b066dff1cae53cb8ee7cf5ba9321ec1243dea0cde0 + ) + set( + f6_sha256 + 5d9f8c76dc55f790fa31d825985e9270bf9e498b8bfec21a0ad3a1feb1fa053a + ) + set( + f7_sha256 + ae391ace382330e912793b70b68529ee7c91026d2869b4df7e7c3e7d3656bdd5 + ) set( f4_library_file_names From f0edf3286928652ca6d9a5d3beb1ef21362a8c14 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 15:38:01 +0100 Subject: [PATCH 34/58] remove changes to packaging/build_ffmpeg.sh --- packaging/build_ffmpeg.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packaging/build_ffmpeg.sh b/packaging/build_ffmpeg.sh index c95587c88..9fef66814 100755 --- a/packaging/build_ffmpeg.sh +++ b/packaging/build_ffmpeg.sh @@ -20,10 +20,6 @@ set -eux prefix="${FFMPEG_ROOT}" -args="" -if [[ "$OSTYPE" == "msys" ]]; then - args="--toolchain=msvc" -fi archive="https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n${FFMPEG_VERSION}.tar.gz" build_dir=$(mktemp -d -t ffmpeg-build.XXXXXXXXXX) @@ -66,7 +62,7 @@ tar -xf ffmpeg.tar.gz --strip-components 1 --enable-avformat \ --enable-avutil \ --enable-swscale \ - --enable-swresample ${args} + --enable-swresample make -j install ls ${prefix}/* From aacbfc6c945a36f444417b80ac7716fe3c7f3915 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 15:42:34 +0100 Subject: [PATCH 35/58] remove TORCH_LIBRARY from custom_ops_dependencies, should be transitive --- src/torchcodec/_core/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 22b262074..8ceba9bd1 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -144,7 +144,6 @@ function(make_torchcodec_libraries set(custom_ops_dependencies ${core_library_name} ${Python3_LIBRARIES} - ${TORCH_LIBRARIES} # Add PyTorch libraries for custom ops registration ) if (WIN32) list(APPEND custom_ops_dependencies Python3::Python) From fc17647ae43a09151735bb73f6ccb585a7bba2b2 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 15:44:47 +0100 Subject: [PATCH 36/58] Keep custom_ops SHARED instead of MODULE --- src/torchcodec/_core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 22b262074..817ef232f 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -151,7 +151,7 @@ function(make_torchcodec_libraries endif() make_torchcodec_sublibrary( "${custom_ops_library_name}" - MODULE # Use MODULE like pybind_ops instead of SHARED + SHARED "${custom_ops_sources}" "${custom_ops_dependencies}" ) From e08b5e7fb109065953bf3fee4695e70149d2b84a Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 17:10:08 +0100 Subject: [PATCH 37/58] Remove Python3::Python dep for custom_ops --- src/torchcodec/_core/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 143e30282..61939e910 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -145,9 +145,6 @@ function(make_torchcodec_libraries ${core_library_name} ${Python3_LIBRARIES} ) - if (WIN32) - list(APPEND custom_ops_dependencies Python3::Python) - endif() make_torchcodec_sublibrary( "${custom_ops_library_name}" SHARED From f073d1fd69f49f814e6c5fd5ec81e54768fe3350 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 17:11:31 +0100 Subject: [PATCH 38/58] let custom_ops be a dll instead of pyd --- src/torchcodec/_core/CMakeLists.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 143e30282..9611f685f 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -155,12 +155,6 @@ function(make_torchcodec_libraries "${custom_ops_dependencies}" ) - # Apply same Windows handling as pybind_ops - if(WIN32) - set_target_properties(${custom_ops_library_name} PROPERTIES SUFFIX ".pyd") - endif() - - # 3. Create libtorchcodec_pybind_opsN.so. set(pybind_ops_library_name "libtorchcodec_pybind_ops${ffmpeg_major_version}") set(pybind_ops_sources From ae8a2c6a3e4577db4656b7fb8fce68a623f97c65 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 17:41:39 +0100 Subject: [PATCH 39/58] Put comment back --- src/torchcodec/_core/ops.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index bed673092..8e5a8232f 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -66,6 +66,8 @@ def load_torchcodec_shared_libraries(): print(f"successfully loaded {pybind_ops_library_name}", flush=True) return except Exception as e: + # TODO: recording and reporting exceptions this way is OK for now as it's just for debugging, + # but we should probably handle that via a proper logging mechanism. exceptions.append((ffmpeg_major_version, e)) traceback = ( From 3b060004b8f1755968d845399a99bb4f0f5389cc Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 17:41:55 +0100 Subject: [PATCH 40/58] comment --- src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake index 1a3ccdd37..22fbdff4c 100644 --- a/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake +++ b/src/torchcodec/_core/fetch_and_expose_non_gpl_ffmpeg_libs.cmake @@ -150,7 +150,6 @@ elseif (APPLE) elseif (WIN32) set(lib_dir "bin") - message(STATUS "Using MinGW-compatible FFmpeg libraries for Windows") set( platform_url ${base_url}/windows_x86_64 From 06d0627115916e83c4ecb0192acad82aec086beb Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 19 Aug 2025 17:43:07 +0100 Subject: [PATCH 41/58] remove Python3_FIND_ABI --- src/torchcodec/_core/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 1c512c58b..fbb60db02 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -7,9 +7,9 @@ set(PYBIND11_FINDPYTHON ON) find_package(pybind11 REQUIRED) find_package(Torch REQUIRED) -if (WIN32) - set(Python3_FIND_ABI "ANY" "ANY" "ANY" "ANY") -endif() +# if (WIN32) +# set(Python3_FIND_ABI "ANY" "ANY" "ANY" "ANY") +# endif() find_package(Python3 ${PYTHON_VERSION} EXACT COMPONENTS Development) if(DEFINED TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR AND TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR) From 5903c07ac44eae35908dff53d5035df1ff081f58 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 20 Aug 2025 09:56:29 +0100 Subject: [PATCH 42/58] Comments --- src/torchcodec/_core/CMakeLists.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index fbb60db02..3614eb04f 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -88,16 +88,17 @@ function(make_torchcodec_libraries # # 1. libtorchcodec_coreN.{ext}: Base library which contains the # implementation of VideoDecoder and everything VideoDecoder needs. On - # Linux, {ext} is so. On Mac, it is dylib. + # Linux, {ext} is so. On Mac, it is dylib. On Windows it's dll. # # 2. libtorchcodec_custom_opsN.{ext}: Implementation of the PyTorch custom # ops. Depends on libtorchcodec_coreN.{ext}. On Linux, {ext} is so. - # On Mac, it is dylib. + # On Mac, it is dylib. On Windows it's dll. # # 3. libtorchcodec_pybind_opsN.{ext}: Implementation of the pybind11 ops. We # keep these separate from the PyTorch custom ops because we have to # load these libraries separately on the Python side. Depends on - # libtorchcodec_coreN.{ext}. On BOTH Linux and Mac {ext} is so. + # libtorchcodec_coreN.{ext}. On BOTH Linux and Mac {ext} is so. On + # Windows, it's pyd. # 1. Create libtorchcodec_coreN.{ext}. set(core_library_name "libtorchcodec_core${ffmpeg_major_version}") @@ -180,6 +181,9 @@ function(make_torchcodec_libraries ) if(WIN32) + # On Windows, we need to set the suffix to .pyd so that Python can + # import the shared library as a module. Just setting the MODULE type + # isn't enough. set_target_properties(${pybind_ops_library_name} PROPERTIES SUFFIX ".pyd") endif() @@ -232,7 +236,7 @@ function(make_torchcodec_libraries install( TARGETS ${all_libraries} LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX} - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows DLLs + RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX} # For Windows ) endfunction() From c7c6077a47067bee2266fcfa05c809a6912075a5 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 20 Aug 2025 09:57:44 +0100 Subject: [PATCH 43/58] lint --- .github/workflows/windows_wheel.yaml | 16 ++++++++-------- setup.py | 9 +++++++-- src/torchcodec/_core/ops.py | 11 +---------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 6a0292fdb..56072b3cb 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -188,7 +188,7 @@ jobs: # Prepare conda set +x && eval "$($(which conda) shell.bash hook)" && set -x - + echo '::group::Create build environment' conda create \ --name ci \ @@ -199,29 +199,29 @@ jobs: # Install PyTorch python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - + # Install build dependencies conda install cmake pkg-config pybind11 "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - + # Ensure FFmpeg DLLs are in PATH for Windows conda_env_path=$(conda info --base)/envs/ci library_bin_path="$conda_env_path/Library/bin" echo "Adding conda Library/bin to PATH: $library_bin_path" echo "$library_bin_path" >> $GITHUB_PATH - + # Verify FFmpeg is available ffmpeg -version - + # Install torchcodec from source in development mode pip install -e ".[dev]" --no-build-isolation -vv - + # Add torchcodec source directory to DLL search path for Windows torchcodec_dll_path="$(pwd)/src/torchcodec" echo "Adding torchcodec DLL path: $torchcodec_dll_path" export PATH="$torchcodec_dll_path:$PATH" - + # Test torchcodec import with detailed error reporting python -c "import torch; import torchcodec" - + # Run tests pytest test -vvv diff --git a/setup.py b/setup.py index 24f480c0c..974efda60 100644 --- a/setup.py +++ b/setup.py @@ -135,9 +135,14 @@ def _build_all_extensions_with_cmake(self): ["cmake", str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp ) print("Calling cmake --build", flush=True) - subprocess.check_call(["cmake", "--build", ".","--config",cmake_build_type], cwd=self.build_temp) + subprocess.check_call( + ["cmake", "--build", ".", "--config", cmake_build_type], cwd=self.build_temp + ) print("Calling cmake --install", flush=True) - subprocess.check_call(["cmake", "--install", ".","--config",cmake_build_type], cwd=self.build_temp) + subprocess.check_call( + ["cmake", "--install", ".", "--config", cmake_build_type], + cwd=self.build_temp, + ) def copy_extensions_to_source(self): """Copy built extensions from temporary folder back into source tree. diff --git a/src/torchcodec/_core/ops.py b/src/torchcodec/_core/ops.py index 8e5a8232f..807ed265d 100644 --- a/src/torchcodec/_core/ops.py +++ b/src/torchcodec/_core/ops.py @@ -6,8 +6,6 @@ import io import json -import os -import sys import warnings from types import ModuleType from typing import List, Optional, Tuple, Union @@ -49,27 +47,20 @@ def load_torchcodec_shared_libraries(): custom_ops_library_name = f"libtorchcodec_custom_ops{ffmpeg_major_version}" pybind_ops_library_name = f"libtorchcodec_pybind_ops{ffmpeg_major_version}" try: - print(f"Loading {decoder_library_name}", flush=True) torch.ops.load_library(_get_extension_path(decoder_library_name)) - print(f"successfully loaded {decoder_library_name}", flush=True) - - print(f"Loading {custom_ops_library_name}", flush=True) torch.ops.load_library(_get_extension_path(custom_ops_library_name)) - print(f"successfully loaded {custom_ops_library_name}", flush=True) - print(f"Loading {pybind_ops_library_name}", flush=True) pybind_ops_library_path = _get_extension_path(pybind_ops_library_name) global _pybind_ops _pybind_ops = _load_pybind11_module( pybind_ops_module_name, pybind_ops_library_path ) - print(f"successfully loaded {pybind_ops_library_name}", flush=True) return except Exception as e: # TODO: recording and reporting exceptions this way is OK for now as it's just for debugging, # but we should probably handle that via a proper logging mechanism. exceptions.append((ffmpeg_major_version, e)) - + traceback = ( "\n[start of libtorchcodec loading traceback]\n" + "\n".join(f"FFmpeg version {v}: {str(e)}" for v, e in exceptions) From 7822d73b0acb629ad57eb08d50875eddbdd6edb5 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 20 Aug 2025 11:30:49 +0100 Subject: [PATCH 44/58] try to fix encoder tests --- test/test_encoders.py | 47 +++++++++++++++++++++++++++++++++---------- test/utils.py | 1 + 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/test/test_encoders.py b/test/test_encoders.py index b32f961fe..92cc9dbc0 100644 --- a/test/test_encoders.py +++ b/test/test_encoders.py @@ -17,6 +17,7 @@ assert_tensor_close_on_at_least, get_ffmpeg_major_version, in_fbcode, + IS_WINDOWS, NASA_AUDIO_MP3, SINE_MONO_S32, TestContainerFile, @@ -151,15 +152,29 @@ def test_bad_input_parametrized(self, method, tmp_path): raise ValueError(f"Unknown method: {method}") decoder = AudioEncoder(self.decode(NASA_AUDIO_MP3).data, sample_rate=10) - with pytest.raises(RuntimeError, match="invalid sample rate=10"): + avcodec_open2_failed_msg = "avcodec_open2 failed: Invalid argument" + with pytest.raises( + RuntimeError, + match=avcodec_open2_failed_msg if IS_WINDOWS else "invalid sample rate=10", + ): getattr(decoder, method)(**valid_params) decoder = AudioEncoder( self.decode(NASA_AUDIO_MP3).data, sample_rate=NASA_AUDIO_MP3.sample_rate ) - with pytest.raises(RuntimeError, match="invalid sample rate=10"): + with pytest.raises( + RuntimeError, + match=avcodec_open2_failed_msg if IS_WINDOWS else "invalid sample rate=10", + ): getattr(decoder, method)(sample_rate=10, **valid_params) - with pytest.raises(RuntimeError, match="invalid sample rate=99999999"): + with pytest.raises( + RuntimeError, + match=( + avcodec_open2_failed_msg + if IS_WINDOWS + else "invalid sample rate=99999999" + ), + ): getattr(decoder, method)(sample_rate=99999999, **valid_params) with pytest.raises(RuntimeError, match="bit_rate=-1 must be >= 0"): getattr(decoder, method)(**valid_params, bit_rate=-1) @@ -175,12 +190,14 @@ def test_bad_input_parametrized(self, method, tmp_path): self.decode(NASA_AUDIO_MP3).data, sample_rate=NASA_AUDIO_MP3.sample_rate ) for num_channels in (0, 3): - with pytest.raises( - RuntimeError, - match=re.escape( + match = ( + avcodec_open2_failed_msg + if IS_WINDOWS + else re.escape( f"Desired number of channels ({num_channels}) is not supported" - ), - ): + ) + ) + with pytest.raises(RuntimeError, match=match): getattr(decoder, method)(**valid_params, num_channels=num_channels) @pytest.mark.parametrize("method", ("to_file", "to_tensor", "to_file_like")) @@ -295,8 +312,14 @@ def test_against_cli( rtol, atol = 0, 1e-3 else: rtol, atol = None, None + + if IS_WINDOWS and format == "mp3": + # We're getting a "Could not open input file" on Windows mp3 files when decoding. + return + samples_by_us = self.decode(encoded_by_us) samples_by_ffmpeg = self.decode(encoded_by_ffmpeg) + assert_close( samples_by_us.data, samples_by_ffmpeg.data, @@ -340,9 +363,11 @@ def test_against_to_file( else: raise ValueError(f"Unknown method: {method}") - torch.testing.assert_close( - self.decode(encoded_file).data, self.decode(encoded_output).data - ) + if not (IS_WINDOWS and format == "mp3"): + # We're getting a "Could not open input file" on Windows mp3 files when decoding. + torch.testing.assert_close( + self.decode(encoded_file).data, self.decode(encoded_output).data + ) def test_encode_to_tensor_long_output(self): # Check that we support re-allocating the output tensor when the encoded diff --git a/test/utils.py b/test/utils.py index ed611cfda..bb3556dae 100644 --- a/test/utils.py +++ b/test/utils.py @@ -15,6 +15,7 @@ from torchcodec._core import get_ffmpeg_library_versions +IS_WINDOWS = sys.platform in ("win32", "cygwin") # Decorator for skipping CUDA tests when CUDA isn't available. The tests are # effectively marked to be skipped in pytest_collection_modifyitems() of From 8a30f92a42ee4b1ccba786d40751012ca6e192ab Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 21 Aug 2025 13:24:56 +0100 Subject: [PATCH 45/58] remove windows-specific output dir --- src/torchcodec/_core/CMakeLists.txt | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 3614eb04f..4270e1d0d 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -57,20 +57,6 @@ function(make_torchcodec_sublibrary # Avoid adding the "lib" prefix which we already add explicitly. set_target_properties(${library_name} PROPERTIES PREFIX "") - if(WIN32) - # On Windows, the built artifacts are put in Release/Debug - # subdirectories by default. We want to avoid that, otherwise our - # install() step would not know where to find those. - set_target_properties(${library_name} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} - RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} - LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} - LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} - ARCHIVE_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} - ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} - ) - endif() - target_link_libraries( ${library_name} PUBLIC From b7b0ea9ffa9b6c63658c30c95d9b882a21ac8746 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 21 Aug 2025 13:26:48 +0100 Subject: [PATCH 46/58] Avoid 3.13 builds --- .github/workflows/windows_wheel.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 56072b3cb..5c165fcd4 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -36,6 +36,9 @@ jobs: with-rocm: disable with-cuda: disable build-python-only: "disable" + # Explicitly avoid 3.13 because 3.13t builds don't work. + # TODO remove eventually. + python-versions: ["3.9", "3.10", "3.11", "3.12"] build: needs: generate-matrix From f25a0a78207fa9114f295e41b2587390f9c256f6 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 21 Aug 2025 13:29:55 +0100 Subject: [PATCH 47/58] Use flac in test_num_channels --- test/test_encoders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_encoders.py b/test/test_encoders.py index 92cc9dbc0..49dfcedb2 100644 --- a/test/test_encoders.py +++ b/test/test_encoders.py @@ -439,7 +439,7 @@ def test_num_channels( sample_rate = 16_000 source_samples = torch.rand(num_channels_input, 1_000) - format = "mp3" + format = "flac" encoder = AudioEncoder(source_samples, sample_rate=sample_rate) params = dict(num_channels=num_channels_output) From 281cc38c815423aa43178c88691ea764b33b27d3 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 21 Aug 2025 13:48:53 +0100 Subject: [PATCH 48/58] Fix json list? --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 5c165fcd4..7f5185b21 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -38,7 +38,7 @@ jobs: build-python-only: "disable" # Explicitly avoid 3.13 because 3.13t builds don't work. # TODO remove eventually. - python-versions: ["3.9", "3.10", "3.11", "3.12"] + python-versions: '["3.9", "3.10", "3.11", "3.12"]' build: needs: generate-matrix From f2c34144f37ffa1f40195c5662123cba16effc4d Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 21 Aug 2025 16:25:09 +0100 Subject: [PATCH 49/58] Test on more ffmpeg versions --- .github/workflows/windows_wheel.yaml | 29 ++++------------------------ src/torchcodec/_core/CMakeLists.txt | 3 --- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 7f5185b21..1b6700a09 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -70,8 +70,8 @@ jobs: fail-fast: false matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - ffmpeg-version-for-tests: ['6.1.1'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # ffmpeg-version-for-tests: ['6.1.1'] needs: build steps: - uses: actions/download-artifact@v4 @@ -107,19 +107,6 @@ jobs: assert_ffmpeg_not_installed conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge ffmpeg -version - # On Windows, ensure the conda Library/bin directory is in PATH - # This is needed for Windows DLL loading to find FFmpeg DLLs - if [[ "$RUNNER_OS" == "Windows" ]]; then - conda_env_path=$(conda info --base)/envs/test - library_bin_path="$conda_env_path/Library/bin" - echo "Adding conda Library/bin to PATH: $library_bin_path" - echo "$library_bin_path" >> $GITHUB_PATH - # Verify FFmpeg DLLs are accessible - echo "Checking if FFmpeg DLLs are in PATH:" - where avutil.dll || echo "avutil.dll not found in PATH" - where avcodec.dll || echo "avcodec.dll not found in PATH" - where avformat.dll || echo "avformat.dll not found in PATH" - fi - name: Test torchcodec import after FFmpeg installation run: | echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." @@ -156,17 +143,9 @@ jobs: # - "3.12" runner: ["windows.4xlarge"] gpu-arch-type: ["cpu"] - ffmpeg-version-for-tests: ['6.1.1'] - # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - # TODO: put GPU testing back - # include: - # - python-version: "3.9" - # runner: windows.g5.4xlarge.nvidia.gpu - # gpu-arch-type: cuda - # gpu-arch-version: "11.8" + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] fail-fast: false - # needs: build - needs: generate-matrix ## TODO REMOVE + needs: generate-matrix uses: pytorch/test-infra/.github/workflows/windows_job.yml@main permissions: id-token: write diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 4270e1d0d..8e6bac54d 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -7,9 +7,6 @@ set(PYBIND11_FINDPYTHON ON) find_package(pybind11 REQUIRED) find_package(Torch REQUIRED) -# if (WIN32) -# set(Python3_FIND_ABI "ANY" "ANY" "ANY" "ANY") -# endif() find_package(Python3 ${PYTHON_VERSION} EXACT COMPONENTS Development) if(DEFINED TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR AND TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR) From 12df6c7bd699d87fbbe52668ccd8b14f306cad0b Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 21 Aug 2025 19:06:45 +0100 Subject: [PATCH 50/58] skip mp3 tests on windows when ffmpeg <= 5 --- test/test_encoders.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_encoders.py b/test/test_encoders.py index 49dfcedb2..1bb57c217 100644 --- a/test/test_encoders.py +++ b/test/test_encoders.py @@ -257,6 +257,8 @@ def test_against_cli( if get_ffmpeg_major_version() == 4 and format == "wav": pytest.skip("Swresample with FFmpeg 4 doesn't work on wav files") + if IS_WINDOWS and get_ffmpeg_major_version() <= 5 and format == "mp3": + pytest.skip("Encoding mp3 on Windows is weirdly buggy") encoded_by_ffmpeg = tmp_path / f"ffmpeg_output.{format}" subprocess.run( @@ -343,6 +345,8 @@ def test_against_to_file( ): if get_ffmpeg_major_version() == 4 and format == "wav": pytest.skip("Swresample with FFmpeg 4 doesn't work on wav files") + if IS_WINDOWS and get_ffmpeg_major_version() <= 5 and format == "mp3": + pytest.skip("Encoding mp3 on Windows is weirdly buggy") encoder = AudioEncoder(self.decode(asset).data, sample_rate=asset.sample_rate) From 9048f6cc5db3ef86098208247d95fb6952dc0703 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 21 Aug 2025 20:15:30 +0100 Subject: [PATCH 51/58] better close?? --- src/torchcodec/_core/Encoder.cpp | 8 ++++++-- test/utils.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/torchcodec/_core/Encoder.cpp b/src/torchcodec/_core/Encoder.cpp index 2b1e850ad..9e6b073ad 100644 --- a/src/torchcodec/_core/Encoder.cpp +++ b/src/torchcodec/_core/Encoder.cpp @@ -104,10 +104,14 @@ AudioEncoder::~AudioEncoder() { void AudioEncoder::close_avio() { if (avFormatContext_ && avFormatContext_->pb) { - avio_flush(avFormatContext_->pb); + if (avFormatContext_->pb->error == 0) { + avio_flush(avFormatContext_->pb); + } if (!avioContextHolder_) { - avio_close(avFormatContext_->pb); + if (avFormatContext_->pb->error == 0) { + avio_close(avFormatContext_->pb); + } // avoids closing again in destructor, which would segfault. avFormatContext_->pb = nullptr; } diff --git a/test/utils.py b/test/utils.py index bb3556dae..c1c3e0c31 100644 --- a/test/utils.py +++ b/test/utils.py @@ -17,6 +17,7 @@ IS_WINDOWS = sys.platform in ("win32", "cygwin") + # Decorator for skipping CUDA tests when CUDA isn't available. The tests are # effectively marked to be skipped in pytest_collection_modifyitems() of # conftest.py From c02ffc5d8bf1adfc9c68ef88468e0ad15e43ceb6 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Aug 2025 11:25:42 +0100 Subject: [PATCH 52/58] use 5.0.1 instead of 5.1.2 --- .github/workflows/windows_wheel.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 1b6700a09..e5edc4451 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -70,7 +70,8 @@ jobs: fail-fast: false matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + ffmpeg-version-for-tests: ['4.4.2', '5.0.1', '6.1.1', '7.0.1'] # ffmpeg-version-for-tests: ['6.1.1'] needs: build steps: @@ -143,7 +144,8 @@ jobs: # - "3.12" runner: ["windows.4xlarge"] gpu-arch-type: ["cpu"] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + ffmpeg-version-for-tests: ['4.4.2', '5.0.1', '6.1.1', '7.0.1'] fail-fast: false needs: generate-matrix uses: pytorch/test-infra/.github/workflows/windows_job.yml@main From 4e4f73933e4c8357ca3759c856e2fe63eb245f53 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Aug 2025 11:55:17 +0100 Subject: [PATCH 53/58] Use 5.1.1 --- .github/workflows/windows_wheel.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index e5edc4451..5dff18908 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -71,7 +71,7 @@ jobs: matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - ffmpeg-version-for-tests: ['4.4.2', '5.0.1', '6.1.1', '7.0.1'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.1', '6.1.1', '7.0.1'] # ffmpeg-version-for-tests: ['6.1.1'] needs: build steps: @@ -145,7 +145,7 @@ jobs: runner: ["windows.4xlarge"] gpu-arch-type: ["cpu"] # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - ffmpeg-version-for-tests: ['4.4.2', '5.0.1', '6.1.1', '7.0.1'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.1', '6.1.1', '7.0.1'] fail-fast: false needs: generate-matrix uses: pytorch/test-infra/.github/workflows/windows_job.yml@main From e40dfc2150ebeafa00b1b8324f37f5d8c184a913 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Fri, 22 Aug 2025 12:30:46 +0100 Subject: [PATCH 54/58] cleanup --- .github/workflows/build_ffmpeg.yaml | 2 +- .github/workflows/cpp_tests.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/linux_cuda_wheel.yaml | 2 +- .github/workflows/linux_wheel.yaml | 2 +- .github/workflows/macos_wheel.yaml | 2 +- .github/workflows/windows_wheel.yaml | 83 ++----------------------- src/torchcodec/_core/CMakeLists.txt | 1 - test/test_encoders.py | 4 ++ 10 files changed, 15 insertions(+), 87 deletions(-) diff --git a/.github/workflows/build_ffmpeg.yaml b/.github/workflows/build_ffmpeg.yaml index e52a67864..4d6a22869 100644 --- a/.github/workflows/build_ffmpeg.yaml +++ b/.github/workflows/build_ffmpeg.yaml @@ -11,7 +11,7 @@ name: Build non-GPL FFmpeg from source on: workflow_dispatch: - # pull_request: + pull_request: paths: - packaging/build_ffmpeg.sh - .github/workflows/build_ffmpeg.yaml # self reference diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index 227e40e12..e08d90754 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -3,7 +3,7 @@ name: CPP tests on: push: branches: [ main ] - # pull_request: + pull_request: concurrency: group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e042ab75e..60bfbfa2e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,7 +3,7 @@ name: Docs on: push: branches: [ main ] - # pull_request: + pull_request: permissions: id-token: write diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 456bc2126..c156a833c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -3,7 +3,7 @@ name: Lint on: push: branches: [ main ] - # pull_request: + pull_request: concurrency: group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index 100e95523..c57d8807d 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Linux CUDA wheels on: - # pull_request: + pull_request: push: branches: - nightly diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 0803d37cb..706e7c2b4 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test Linux wheel on: - # pull_request: + pull_request: push: branches: - nightly diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index c1dc8ae88..b26a1d2d1 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -1,7 +1,7 @@ name: Build and test MacOS wheel on: - # pull_request: + pull_request: push: branches: - nightly diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 5dff18908..42bc69b72 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -70,9 +70,10 @@ jobs: fail-fast: false matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.1', '6.1.1', '7.0.1'] - # ffmpeg-version-for-tests: ['6.1.1'] + # TODO: FFmpeg 5 on Windows segfaults in avcodec_open2() when passing + # bad parameters. + # See https://github.com/pytorch/torchcodec/pull/806 + ffmpeg-version-for-tests: ['4.4.2', '6.1.1', '7.0.1'] needs: build steps: - uses: actions/download-artifact@v4 @@ -133,79 +134,3 @@ jobs: - name: Run Python tests run: | pytest test -vvv - - install-and-test-on-test-infra: - strategy: - matrix: - python-version: - - "3.9" - - "3.10" - - "3.11" - # - "3.12" - runner: ["windows.4xlarge"] - gpu-arch-type: ["cpu"] - # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.1', '6.1.1', '7.0.1'] - fail-fast: false - needs: generate-matrix - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - permissions: - id-token: write - contents: read - with: - repository: pytorch/torchcodec - runner: ${{ matrix.runner }} - gpu-arch-type: ${{ matrix.gpu-arch-type }} - gpu-arch-version: ${{ matrix.gpu-arch-version }} - timeout: 120 - test-infra-ref: main - script: | - set -euxo pipefail - - export PYTHON_VERSION=${{ matrix.python-version }} - export VC_YEAR=2022 - export VSDEVCMD_ARGS="" - export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} - export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} - export FFMPEG_VERSION_FOR_TESTS=${{ matrix.ffmpeg-version-for-tests }} - export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 - - # Prepare conda - set +x && eval "$($(which conda) shell.bash hook)" && set -x - - echo '::group::Create build environment' - conda create \ - --name ci \ - --quiet --yes \ - python="${PYTHON_VERSION}" pip - - conda activate ci - - # Install PyTorch - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - # Install build dependencies - conda install cmake pkg-config pybind11 "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - - # Ensure FFmpeg DLLs are in PATH for Windows - conda_env_path=$(conda info --base)/envs/ci - library_bin_path="$conda_env_path/Library/bin" - echo "Adding conda Library/bin to PATH: $library_bin_path" - echo "$library_bin_path" >> $GITHUB_PATH - - # Verify FFmpeg is available - ffmpeg -version - - # Install torchcodec from source in development mode - pip install -e ".[dev]" --no-build-isolation -vv - - # Add torchcodec source directory to DLL search path for Windows - torchcodec_dll_path="$(pwd)/src/torchcodec" - echo "Adding torchcodec DLL path: $torchcodec_dll_path" - export PATH="$torchcodec_dll_path:$PATH" - - # Test torchcodec import with detailed error reporting - python -c "import torch; import torchcodec" - - # Run tests - pytest test -vvv diff --git a/src/torchcodec/_core/CMakeLists.txt b/src/torchcodec/_core/CMakeLists.txt index 8e6bac54d..1f09159e6 100644 --- a/src/torchcodec/_core/CMakeLists.txt +++ b/src/torchcodec/_core/CMakeLists.txt @@ -6,7 +6,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(PYBIND11_FINDPYTHON ON) find_package(pybind11 REQUIRED) find_package(Torch REQUIRED) - find_package(Python3 ${PYTHON_VERSION} EXACT COMPONENTS Development) if(DEFINED TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR AND TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR) diff --git a/test/test_encoders.py b/test/test_encoders.py index 1bb57c217..45b4359e4 100644 --- a/test/test_encoders.py +++ b/test/test_encoders.py @@ -258,6 +258,7 @@ def test_against_cli( if get_ffmpeg_major_version() == 4 and format == "wav": pytest.skip("Swresample with FFmpeg 4 doesn't work on wav files") if IS_WINDOWS and get_ffmpeg_major_version() <= 5 and format == "mp3": + # TODO: https://github.com/pytorch/torchcodec/issues/837 pytest.skip("Encoding mp3 on Windows is weirdly buggy") encoded_by_ffmpeg = tmp_path / f"ffmpeg_output.{format}" @@ -317,6 +318,7 @@ def test_against_cli( if IS_WINDOWS and format == "mp3": # We're getting a "Could not open input file" on Windows mp3 files when decoding. + # TODO: https://github.com/pytorch/torchcodec/issues/837 return samples_by_us = self.decode(encoded_by_us) @@ -346,6 +348,7 @@ def test_against_to_file( if get_ffmpeg_major_version() == 4 and format == "wav": pytest.skip("Swresample with FFmpeg 4 doesn't work on wav files") if IS_WINDOWS and get_ffmpeg_major_version() <= 5 and format == "mp3": + # TODO: https://github.com/pytorch/torchcodec/issues/837 pytest.skip("Encoding mp3 on Windows is weirdly buggy") encoder = AudioEncoder(self.decode(asset).data, sample_rate=asset.sample_rate) @@ -369,6 +372,7 @@ def test_against_to_file( if not (IS_WINDOWS and format == "mp3"): # We're getting a "Could not open input file" on Windows mp3 files when decoding. + # TODO: https://github.com/pytorch/torchcodec/issues/837 torch.testing.assert_close( self.decode(encoded_file).data, self.decode(encoded_output).data ) From ff6d5d107b99d92d30610865ddf44d8ec7df6e43 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 26 Aug 2025 14:26:45 +0100 Subject: [PATCH 55/58] Rely on defaults for python versions --- .github/workflows/windows_wheel.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 42bc69b72..002324f8a 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -36,9 +36,6 @@ jobs: with-rocm: disable with-cuda: disable build-python-only: "disable" - # Explicitly avoid 3.13 because 3.13t builds don't work. - # TODO remove eventually. - python-versions: '["3.9", "3.10", "3.11", "3.12"]' build: needs: generate-matrix @@ -69,7 +66,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9'] # TODO: FFmpeg 5 on Windows segfaults in avcodec_open2() when passing # bad parameters. # See https://github.com/pytorch/torchcodec/pull/806 From f83e9da0d9fefc2596a8c9a65f1fb8a2facb89c6 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 26 Aug 2025 14:56:35 +0100 Subject: [PATCH 56/58] Set 3.10 as minimum supported version --- .github/workflows/docs.yaml | 4 ++-- .github/workflows/linux_cuda_wheel.yaml | 4 ++-- .github/workflows/linux_wheel.yaml | 2 +- .github/workflows/macos_wheel.yaml | 2 +- .github/workflows/reference_resources.yaml | 2 +- README.md | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 60bfbfa2e..7d580cef3 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -51,10 +51,10 @@ jobs: strategy: fail-fast: false matrix: - # 3.9 corresponds to the minimum python version for which we build + # 3.10 corresponds to the minimum python version for which we build # the wheel unless the label cliflow/binaries/all is present in the # PR. - python-version: ['3.9'] + python-version: ['3.10'] cuda-version: ['12.6'] ffmpeg-version-for-tests: ['7'] container: diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index c57d8807d..89e1c0799 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -61,12 +61,12 @@ jobs: strategy: fail-fast: false matrix: - # 3.9 corresponds to the minimum python version for which we build + # 3.10 corresponds to the minimum python version for which we build # the wheel unless the label cliflow/binaries/all is present in the # PR. # For the actual release we should add that label and change this to # include more python versions. - python-version: ['3.9'] + python-version: ['3.10'] # We test against 12.6 and 12.9 to avoid having too big of a CI matrix, # but for releases we should add 12.8. cuda-version: ['12.6', '12.9'] diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 706e7c2b4..8f9acba0b 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -62,7 +62,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9'] + python-version: ['3.10'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] needs: build steps: diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index b26a1d2d1..5110ebe39 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -63,7 +63,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9'] + python-version: ['3.10'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] needs: build steps: diff --git a/.github/workflows/reference_resources.yaml b/.github/workflows/reference_resources.yaml index 7471134ae..d0c0fea26 100644 --- a/.github/workflows/reference_resources.yaml +++ b/.github/workflows/reference_resources.yaml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9'] + python-version: ['3.10'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] steps: - name: Setup conda env diff --git a/README.md b/README.md index 15c1b5f2f..7ce78e16a 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ The following table indicates the compatibility between versions of | `torchcodec` | `torch` | Python | | ------------------ | ------------------ | ------------------- | -| `main` / `nightly` | `main` / `nightly` | `>=3.9`, `<=3.13` | +| `main` / `nightly` | `main` / `nightly` | `>=3.10`, `<=3.13` | | `0.6` | `2.8` | `>=3.9`, `<=3.13` | | `0.5` | `2.7` | `>=3.9`, `<=3.13` | | `0.4` | `2.7` | `>=3.9`, `<=3.13` | From b531d4ba3369e86b461abd04a33f7dd5c86aecb0 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 26 Aug 2025 14:58:08 +0100 Subject: [PATCH 57/58] Set 3.10 for windows as well --- .github/workflows/windows_wheel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_wheel.yaml b/.github/workflows/windows_wheel.yaml index 002324f8a..de19292e9 100644 --- a/.github/workflows/windows_wheel.yaml +++ b/.github/workflows/windows_wheel.yaml @@ -66,7 +66,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9'] + python-version: ['3.10'] # TODO: FFmpeg 5 on Windows segfaults in avcodec_open2() when passing # bad parameters. # See https://github.com/pytorch/torchcodec/pull/806 From d89f81a5c300a7f6090fe9648156a7d4510062a0 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 26 Aug 2025 17:13:15 +0100 Subject: [PATCH 58/58] Shorter test name --- test/test_decoders.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_decoders.py b/test/test_decoders.py index 0c6acb190..2e837c316 100644 --- a/test/test_decoders.py +++ b/test/test_decoders.py @@ -1342,7 +1342,11 @@ def test_custom_frame_mappings_json_and_bytes( @pytest.mark.parametrize( "custom_frame_mappings,expected_match", [ - (NASA_VIDEO.generate_custom_frame_mappings(0), "seek_mode"), + pytest.param( + NASA_VIDEO.generate_custom_frame_mappings(0), + "seek_mode", + id="valid_content_approximate", + ), ("{}", "The input is empty or missing the required 'frames' key."), ( '{"valid": "json"}',