From fca4a768537e9fe2929ea56db2b668e735b3c376 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 10:00:18 -0700 Subject: [PATCH 01/67] Added new wheel file. --- .github/workflows/wheel2.yaml | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/wheel2.yaml diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml new file mode 100644 index 000000000..0e7751df6 --- /dev/null +++ b/.github/workflows/wheel2.yaml @@ -0,0 +1,52 @@ +name: Build Linux Wheels + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + # NOTE: Binary build pipelines should only get triggered on release candidate builds + # Release candidate tags look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-xpu: enable + build: + needs: generate-matrix + strategy: + fail-fast: false + matrix: + include: + - repository: pytorch/torchcodec + smoke-test-script: test/smoke_test/smoke_test_no_ffmpeg.py + package-name: torchcodec + name: ${{ matrix.repository }} + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: ${{ matrix.repository }} + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + smoke-test-script: ${{ matrix.smoke-test-script }} + package-name: ${{ matrix.package-name }} + trigger-event: ${{ github.event_name }} From c228184529f778cd3f92465afadb7405daac7622 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 10:30:46 -0700 Subject: [PATCH 02/67] . --- .github/workflows/wheel2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 0e7751df6..7dfb228c7 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -50,3 +50,4 @@ jobs: smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" From d283eecc74ed2b28eab4e5e3b3563458e62e048a Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 10:41:50 -0700 Subject: [PATCH 03/67] . --- .github/workflows/wheel2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 7dfb228c7..bc3c35248 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -37,6 +37,7 @@ jobs: matrix: include: - repository: pytorch/torchcodec + pre-script: packaging/pre_build_script.sh smoke-test-script: test/smoke_test/smoke_test_no_ffmpeg.py package-name: torchcodec name: ${{ matrix.repository }} From 8696a4a661dcec16cdbe7fa93c58382bf0124565 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 10:42:09 -0700 Subject: [PATCH 04/67] . --- packaging/pre_build_script.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 packaging/pre_build_script.sh diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh new file mode 100644 index 000000000..880fad763 --- /dev/null +++ b/packaging/pre_build_script.sh @@ -0,0 +1,9 @@ + +#!/bin/bash + +echo "Installing build pre-requisites from pre_build_script.sh" + +python -m pip install --upgrade pip +python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +python -m pip install auditwheel +python -m pip install --upgrade setuptools From 0b8d802a323ae235cf2681fb2f1f22b2e5621734 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 10:48:16 -0700 Subject: [PATCH 05/67] . --- .github/workflows/wheel2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index bc3c35248..6c6887b42 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -48,6 +48,7 @@ jobs: test-infra-repository: pytorch/test-infra test-infra-ref: main build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + pre-script: ${{ matrix.pre-script }} smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} trigger-event: ${{ github.event_name }} From 96e9cd708d6b14713882a3b249a0d184dbd5a43a Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 10:49:17 -0700 Subject: [PATCH 06/67] . --- packaging/pre_build_script.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 packaging/pre_build_script.sh diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh old mode 100644 new mode 100755 From b51813c0709e73c813948fa52d30ffdabd315131 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 11:07:05 -0700 Subject: [PATCH 07/67] . --- .github/workflows/wheel2.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 6c6887b42..4907df78f 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -23,7 +23,7 @@ permissions: jobs: generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + uses: ahmadsharif1/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: package-type: wheel os: linux @@ -53,3 +53,4 @@ jobs: package-name: ${{ matrix.package-name }} trigger-event: ${{ github.event_name }} build-platform: "python-build-package" + wheel-build-extra-args: "--no-build-isolation" From aea47d22c937708bb93d21b1a7d8862c1f241624 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 11:10:36 -0700 Subject: [PATCH 08/67] . --- .github/workflows/wheel2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 4907df78f..ae31a9800 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -23,7 +23,7 @@ permissions: jobs: generate-matrix: - uses: ahmadsharif1/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + uses: ahmadsharif1/test-infra/.github/workflows/generate_binary_build_matrix.yml@extra with: package-type: wheel os: linux @@ -41,7 +41,7 @@ jobs: smoke-test-script: test/smoke_test/smoke_test_no_ffmpeg.py package-name: torchcodec name: ${{ matrix.repository }} - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + uses: ahmadsharif1/test-infra/.github/workflows/build_wheels_linux.yml@extra with: repository: ${{ matrix.repository }} ref: "" From a70226ef78d2be2fa9c635dc4728543d20600ba8 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 11:14:25 -0700 Subject: [PATCH 09/67] . --- .github/workflows/wheel2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index ae31a9800..e1a4bd830 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -53,4 +53,5 @@ jobs: package-name: ${{ matrix.package-name }} trigger-event: ${{ github.event_name }} build-platform: "python-build-package" + # For torchcodec's build-time-dependency on libtorch. wheel-build-extra-args: "--no-build-isolation" From 62977aa75e75bc2d766ca591cf42c6a00ae78fb2 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 11:31:09 -0700 Subject: [PATCH 10/67] . --- .github/workflows/wheel2.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index e1a4bd830..8dd63a5a5 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -53,5 +53,6 @@ jobs: package-name: ${{ matrix.package-name }} trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - # For torchcodec's build-time-dependency on libtorch. + # For torchcodec's build-time-dependency on libtorch: + # https://github.com/pytorch/torchcodec/blob/main/setup.py#L7-L42 wheel-build-extra-args: "--no-build-isolation" From 60c2f48c91540370cd786eafc8c3c44c118db773 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 11:55:06 -0700 Subject: [PATCH 11/67] . --- .github/workflows/wheel2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 8dd63a5a5..3bb70d15b 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -23,7 +23,7 @@ permissions: jobs: generate-matrix: - uses: ahmadsharif1/test-infra/.github/workflows/generate_binary_build_matrix.yml@extra + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: package-type: wheel os: linux @@ -41,7 +41,7 @@ jobs: smoke-test-script: test/smoke_test/smoke_test_no_ffmpeg.py package-name: torchcodec name: ${{ matrix.repository }} - uses: ahmadsharif1/test-infra/.github/workflows/build_wheels_linux.yml@extra + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main with: repository: ${{ matrix.repository }} ref: "" From 74eada72a3d0dd3782f19a4b20f96f156e21a896 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 12:06:44 -0700 Subject: [PATCH 12/67] . --- .github/workflows/wheel2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 3bb70d15b..1aab9ee02 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -53,6 +53,7 @@ jobs: package-name: ${{ matrix.package-name }} trigger-event: ${{ github.event_name }} build-platform: "python-build-package" + # TODO: remove this comment: only added to trigger checks. # For torchcodec's build-time-dependency on libtorch: # https://github.com/pytorch/torchcodec/blob/main/setup.py#L7-L42 wheel-build-extra-args: "--no-build-isolation" From 9633dc6ae0420bc55a4d4a0c57710a9b89c54506 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 12:11:10 -0700 Subject: [PATCH 13/67] . --- .github/workflows/wheel2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 1aab9ee02..3edc99904 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -56,4 +56,4 @@ jobs: # TODO: remove this comment: only added to trigger checks. # For torchcodec's build-time-dependency on libtorch: # https://github.com/pytorch/torchcodec/blob/main/setup.py#L7-L42 - wheel-build-extra-args: "--no-build-isolation" + wheel-build-extra-args: "--no-isolation" From cb9d1a73df1618a3ec3c12ddb49d51ea666b7864 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 12:20:25 -0700 Subject: [PATCH 14/67] . --- .github/workflows/wheel2.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 3edc99904..e6551ef6f 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -38,6 +38,7 @@ jobs: include: - repository: pytorch/torchcodec pre-script: packaging/pre_build_script.sh + env-var-script: packaging/env_var_script.sh smoke-test-script: test/smoke_test/smoke_test_no_ffmpeg.py package-name: torchcodec name: ${{ matrix.repository }} @@ -49,6 +50,7 @@ jobs: test-infra-ref: main build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: ${{ matrix.pre-script }} + env-var-script: ${{ matrix.env-var-script }} smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} trigger-event: ${{ github.event_name }} From 3a0947cbff33ae3a9a57e1443be264e922bb7f54 Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 12:22:51 -0700 Subject: [PATCH 15/67] . --- packaging/env_var_script.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 packaging/env_var_script.sh diff --git a/packaging/env_var_script.sh b/packaging/env_var_script.sh new file mode 100644 index 000000000..655b052fb --- /dev/null +++ b/packaging/env_var_script.sh @@ -0,0 +1 @@ +BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 From dd7a7693d7c6f91f9c90e31fbb8c5756dc5473da Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 12:25:47 -0700 Subject: [PATCH 16/67] . --- packaging/env_var_script.sh | 2 +- packaging/pre_build_script.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/env_var_script.sh b/packaging/env_var_script.sh index 655b052fb..0ff9e074e 100644 --- a/packaging/env_var_script.sh +++ b/packaging/env_var_script.sh @@ -1 +1 @@ -BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 +export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh index 880fad763..f437225ec 100755 --- a/packaging/pre_build_script.sh +++ b/packaging/pre_build_script.sh @@ -1,4 +1,3 @@ - #!/bin/bash echo "Installing build pre-requisites from pre_build_script.sh" From 623ed35132f88bebd2e29c0f6d8f28920a80495b Mon Sep 17 00:00:00 2001 From: Ahmad Sharif Date: Wed, 31 Jul 2024 12:32:26 -0700 Subject: [PATCH 17/67] . --- .github/workflows/wheel2.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index e6551ef6f..ba4474d32 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -39,7 +39,6 @@ jobs: - repository: pytorch/torchcodec pre-script: packaging/pre_build_script.sh env-var-script: packaging/env_var_script.sh - smoke-test-script: test/smoke_test/smoke_test_no_ffmpeg.py package-name: torchcodec name: ${{ matrix.repository }} uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main From 9bb4181b0444e4db998d4699a587d04f86646471 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 08:04:50 +0100 Subject: [PATCH 18/67] Install ffmpeg in post build script --- .github/workflows/wheel2.yaml | 2 ++ packaging/post_build_script.sh | 3 +++ 2 files changed, 5 insertions(+) create mode 100755 packaging/post_build_script.sh diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index ba4474d32..b97b20ac7 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -38,6 +38,7 @@ jobs: include: - repository: pytorch/torchcodec pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh env-var-script: packaging/env_var_script.sh package-name: torchcodec name: ${{ matrix.repository }} @@ -49,6 +50,7 @@ jobs: test-infra-ref: main build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: ${{ matrix.pre-script }} + post-script: ${{ matrix.post-script }} env-var-script: ${{ matrix.env-var-script }} smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh new file mode 100755 index 000000000..9ab9c7517 --- /dev/null +++ b/packaging/post_build_script.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +conda install ffmpeg -c conda-forge From 6b619e3906d4f655bdb3370071ca59fa69f5b595 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 08:10:43 +0100 Subject: [PATCH 19/67] disable xpu, rocm and cuda --- .github/workflows/wheel2.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index b97b20ac7..dff4b8dbc 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -29,7 +29,9 @@ jobs: os: linux test-infra-repository: pytorch/test-infra test-infra-ref: main - with-xpu: enable + with-xpu: disable + with-rocm: disable + with-cuda: disable build: needs: generate-matrix strategy: From 327d8a173913be8c99f4f47c2ca4925d1f4250b2 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 08:41:39 +0100 Subject: [PATCH 20/67] Add relocation script --- packaging/post_build_script.sh | 2 + packaging/relocate.py | 263 +++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 packaging/relocate.py diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 9ab9c7517..9e539168d 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,3 +1,5 @@ #!/bin/bash conda install ffmpeg -c conda-forge + +LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/relocate.py diff --git a/packaging/relocate.py b/packaging/relocate.py new file mode 100644 index 000000000..b746d4131 --- /dev/null +++ b/packaging/relocate.py @@ -0,0 +1,263 @@ +"""Helper script to package wheels and relocate binaries.""" + +import glob +import hashlib + +# Standard library imports +import os +import os.path as osp +import platform +import shutil +import subprocess +import sys +import zipfile +from base64 import urlsafe_b64encode + +# Third party imports +if sys.platform == "linux": + from auditwheel.lddtree import lddtree + + +ALLOWLIST = { + "libgcc_s.so.1", + "libstdc++.so.6", + "libm.so.6", + "libdl.so.2", + "librt.so.1", + "libc.so.6", + "libnsl.so.1", + "libutil.so.1", + "libpthread.so.0", + "libresolv.so.2", + "libX11.so.6", + "libXext.so.6", + "libXrender.so.1", + "libICE.so.6", + "libSM.so.6", + "libGL.so.1", + "libgobject-2.0.so.0", + "libgthread-2.0.so.0", + "libglib-2.0.so.0", + "ld-linux-x86-64.so.2", + "ld-2.17.so", +} + + +HERE = osp.dirname(osp.abspath(__file__)) +PACKAGE_ROOT = osp.dirname(osp.dirname(HERE)) +PLATFORM_ARCH = platform.machine() +PYTHON_VERSION = sys.version_info + + +def rehash(path, blocksize=1 << 20): + """Return (hash, length) for path using hashlib.sha256()""" + h = hashlib.sha256() + length = 0 + with open(path, "rb") as f: + while block := f.read(blocksize): + length += len(block) + h.update(block) + digest = "sha256=" + urlsafe_b64encode(h.digest()).decode("latin1").rstrip("=") + # unicode/str python2 issues + return (digest, str(length)) # type: ignore + + +def unzip_file(file, dest): + """Decompress zip `file` into directory `dest`.""" + with zipfile.ZipFile(file, "r") as zip_ref: + zip_ref.extractall(dest) + + +def is_program_installed(basename): + """ + Return program absolute path if installed in PATH. + Otherwise, return None + On macOS systems, a .app is considered installed if + it exists. + """ + if sys.platform == "darwin" and basename.endswith(".app") and osp.exists(basename): + return basename + + for path in os.environ["PATH"].split(os.pathsep): + abspath = osp.join(path, basename) + if osp.isfile(abspath): + return abspath + + +def find_program(basename): + """ + Find program in PATH and return absolute path + Try adding .exe or .bat to basename on Windows platforms + (return None if not found) + """ + names = [basename] + if os.name == "nt": + # Windows platforms + extensions = (".exe", ".bat", ".cmd", ".dll") + if not basename.endswith(extensions): + names = [basename + ext for ext in extensions] + [basename] + for name in names: + path = is_program_installed(name) + if path: + return path + + +def patch_new_path(library_path, new_dir): + library = osp.basename(library_path) + name, *rest = library.split(".") + rest = ".".join(rest) + hash_id = hashlib.sha256(library_path.encode("utf-8")).hexdigest()[:8] + new_name = ".".join([name, hash_id, rest]) + return osp.join(new_dir, new_name) + + + + +def relocate_elf_library(patchelf, output_dir, output_library, binary): + """ + Relocate an ELF shared library to be packaged on a wheel. + + Given a shared library, find the transitive closure of its dependencies, + rename and copy them into the wheel while updating their respective rpaths. + """ + + print(f"Relocating {binary}") + binary_path = osp.join(output_library, binary) + + ld_tree = lddtree(binary_path) + tree_libs = ld_tree["libs"] + + binary_queue = [(n, binary) for n in ld_tree["needed"]] + binary_paths = {binary: binary_path} + binary_dependencies = {} + + while binary_queue != []: + library, parent = binary_queue.pop(0) + library_info = tree_libs[library] + print(library) + + if library_info["path"] is None: + print(f"Omitting {library}") + continue + + if library in ALLOWLIST: + # Omit glibc/gcc/system libraries + print(f"Omitting {library}") + continue + + parent_dependencies = binary_dependencies.get(parent, []) + parent_dependencies.append(library) + binary_dependencies[parent] = parent_dependencies + + if library in binary_paths: + continue + + binary_paths[library] = library_info["path"] + binary_queue += [(n, library) for n in library_info["needed"]] + + print("Copying dependencies to wheel directory") + new_libraries_path = osp.join(output_dir, "torchcodec.libs") + os.makedirs(new_libraries_path, exist_ok=True) + + new_names = {binary: binary_path} + + for library in binary_paths: + if library != binary: + library_path = binary_paths[library] + new_library_path = patch_new_path(library_path, new_libraries_path) + print(f"{library} -> {new_library_path}") + shutil.copyfile(library_path, new_library_path) + new_names[library] = new_library_path + + print("Updating dependency names by new files") + for library in binary_paths: + if library != binary: + if library not in binary_dependencies: + continue + library_dependencies = binary_dependencies[library] + new_library_name = new_names[library] + for dep in library_dependencies: + new_dep = osp.basename(new_names[dep]) + print(f"{library}: {dep} -> {new_dep}") + subprocess.check_output( + [patchelf, "--replace-needed", dep, new_dep, new_library_name], cwd=new_libraries_path + ) + + print("Updating library rpath") + subprocess.check_output([patchelf, "--set-rpath", "$ORIGIN", new_library_name], cwd=new_libraries_path) + + subprocess.check_output([patchelf, "--print-rpath", new_library_name], cwd=new_libraries_path) + + print("Update library dependencies") + library_dependencies = binary_dependencies[binary] + for dep in library_dependencies: + new_dep = osp.basename(new_names[dep]) + print(f"{binary}: {dep} -> {new_dep}") + subprocess.check_output([patchelf, "--replace-needed", dep, new_dep, binary], cwd=output_library) + + print("Update library rpath") + subprocess.check_output( + [patchelf, "--set-rpath", "$ORIGIN:$ORIGIN/../torchcodec.libs", binary_path], cwd=output_library + ) + + +def compress_wheel(output_dir, wheel, wheel_dir, wheel_name): + """Create RECORD file and compress wheel distribution.""" + print("Update RECORD file in wheel") + dist_info = glob.glob(osp.join(output_dir, "*.dist-info"))[0] + record_file = osp.join(dist_info, "RECORD") + + with open(record_file, "w") as f: + for root, _, files in os.walk(output_dir): + for this_file in files: + full_file = osp.join(root, this_file) + rel_file = osp.relpath(full_file, output_dir) + if full_file == record_file: + f.write(f"{rel_file},,\n") + else: + digest, size = rehash(full_file) + f.write(f"{rel_file},{digest},{size}\n") + + print("Compressing wheel") + base_wheel_name = osp.join(wheel_dir, wheel_name) + shutil.make_archive(base_wheel_name, "zip", output_dir) + os.remove(wheel) + shutil.move(f"{base_wheel_name}.zip", wheel) + shutil.rmtree(output_dir) + + +def patch_linux(): + # Get patchelf location + patchelf = find_program("patchelf") + if patchelf is None: + raise FileNotFoundError("Patchelf was not found in the system, please make sure that is available on the PATH.") + + # Find wheel + print("Finding wheels...") + wheels = glob.glob(osp.join(PACKAGE_ROOT, "dist", "*.whl")) + output_dir = osp.join(PACKAGE_ROOT, "dist", ".wheel-process") + + binaries = [f"libtorchcodec{n}.so" for n in (4,5,6,7)] + for wheel in wheels: + if osp.exists(output_dir): + shutil.rmtree(output_dir) + + os.makedirs(output_dir) + + print("Unzipping wheel...") + wheel_file = osp.basename(wheel) + wheel_dir = osp.dirname(wheel) + print(f"{wheel_file}") + wheel_name, _ = osp.splitext(wheel_file) + unzip_file(wheel, output_dir) + + print("Findingbinaries") + output_library = osp.join(output_dir, "torchcodec") + for binary in binaries: + if osp.exists(osp.join(output_library, binary)): + relocate_elf_library(patchelf, output_dir, output_library, binary) + + compress_wheel(output_dir, wheel, wheel_dir, wheel_name) + +if __name__ == "__main__": + patch_linux() From bde5e86125ce8db8a8cc8b7c4204616291d1908c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 08:53:25 +0100 Subject: [PATCH 21/67] moved to wheel/ --- packaging/post_build_script.sh | 2 +- packaging/{ => wheel}/relocate.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename packaging/{ => wheel}/relocate.py (100%) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 9e539168d..458f419b6 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -2,4 +2,4 @@ conda install ffmpeg -c conda-forge -LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/relocate.py +LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py diff --git a/packaging/relocate.py b/packaging/wheel/relocate.py similarity index 100% rename from packaging/relocate.py rename to packaging/wheel/relocate.py From e8231d692ba2fd4170400f668be6a4601f7e8c9e Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:05:25 +0100 Subject: [PATCH 22/67] debug --- packaging/wheel/relocate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/wheel/relocate.py b/packaging/wheel/relocate.py index b746d4131..a90dc661a 100644 --- a/packaging/wheel/relocate.py +++ b/packaging/wheel/relocate.py @@ -142,7 +142,7 @@ def relocate_elf_library(patchelf, output_dir, output_library, binary): if library in ALLOWLIST: # Omit glibc/gcc/system libraries - print(f"Omitting {library}") + print(f"Omitting {library} because in ALLOWLIST") continue parent_dependencies = binary_dependencies.get(parent, []) @@ -161,6 +161,7 @@ def relocate_elf_library(patchelf, output_dir, output_library, binary): new_names = {binary: binary_path} + print(f"{binary_paths = }") for library in binary_paths: if library != binary: library_path = binary_paths[library] @@ -189,6 +190,7 @@ def relocate_elf_library(patchelf, output_dir, output_library, binary): subprocess.check_output([patchelf, "--print-rpath", new_library_name], cwd=new_libraries_path) print("Update library dependencies") + print(f"{binary_dependencies = }") library_dependencies = binary_dependencies[binary] for dep in library_dependencies: new_dep = osp.basename(new_names[dep]) From 188b0970a3a4532786fe66caf158cafc95459c3c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:20:34 +0100 Subject: [PATCH 23/67] disable relocate --- packaging/post_build_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 458f419b6..275afb038 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -2,4 +2,4 @@ conda install ffmpeg -c conda-forge -LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py +# LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From 43c76c24c2041c1ef513a4e44e83f6b64cf87670 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:29:52 +0100 Subject: [PATCH 24/67] just install from normal conda --- packaging/post_build_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 275afb038..991f4916a 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,5 +1,5 @@ #!/bin/bash -conda install ffmpeg -c conda-forge +conda install ffmpeg # LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From 1ec79f85ac067866943983c5adb2a6e2352ba4dc Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:31:53 +0100 Subject: [PATCH 25/67] install 4.4.2 --- packaging/post_build_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 991f4916a..74bc0d9e3 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,5 +1,5 @@ #!/bin/bash -conda install ffmpeg +conda install "ffmpeg=4.4.2" -c conda-forge # LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From a99b5e1b0d9b7a85e6570549c036603df08bb533 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:40:06 +0100 Subject: [PATCH 26/67] . --- packaging/post_build_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 74bc0d9e3..feab2e752 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,5 +1,5 @@ #!/bin/bash -conda install "ffmpeg=4.4.2" -c conda-forge +conda install "ffmpeg<5" -c conda-forge # LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From 617151336fced588796eb72a01b1a8b8c4990232 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:40:27 +0100 Subject: [PATCH 27/67] . --- packaging/post_build_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 991f4916a..4e02e8d8b 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,5 +1,5 @@ #!/bin/bash -conda install ffmpeg +conda install "ffmpeg<6" -c conda-forge # LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From e27ff9ad658692b2434475963bfac2d3b920b2b3 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:53:11 +0100 Subject: [PATCH 28/67] use yum --- packaging/post_build_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 4e02e8d8b..2670b0df1 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,5 +1,5 @@ #!/bin/bash -conda install "ffmpeg<6" -c conda-forge +yum install ffmpeg # LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From cab67f360078ac45f059f498acba377a654839d3 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:59:22 +0100 Subject: [PATCH 29/67] try more stuff --- packaging/post_build_script.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 2670b0df1..5df566616 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,5 +1,7 @@ #!/bin/bash -yum install ffmpeg +sudo yum install epel-release +sudo yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm +sudo yum install ffmpeg # LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From 6f606aa289584dd3ccf36b0f3ab9abae7e0431b2 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 09:59:39 +0100 Subject: [PATCH 30/67] nosudo --- packaging/post_build_script.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index feab2e752..b8b693973 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,5 +1,7 @@ #!/bin/bash -conda install "ffmpeg<5" -c conda-forge +yum install epel-release +yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm +yum install ffmpeg # LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From 63d6790c0edbaefedfe59e29424639cc933513df Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 10:08:16 +0100 Subject: [PATCH 31/67] fake smoke test --- .github/workflows/wheel2.yaml | 1 + packaging/fake_smoke_test.py | 2 ++ packaging/post_build_script.sh | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 packaging/fake_smoke_test.py diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index dff4b8dbc..dea4f8e38 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -42,6 +42,7 @@ jobs: pre-script: packaging/pre_build_script.sh post-script: packaging/post_build_script.sh env-var-script: packaging/env_var_script.sh + smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec name: ${{ matrix.repository }} uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main diff --git a/packaging/fake_smoke_test.py b/packaging/fake_smoke_test.py new file mode 100644 index 000000000..efbfbb583 --- /dev/null +++ b/packaging/fake_smoke_test.py @@ -0,0 +1,2 @@ + +print("Success") diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index b8b693973..0069cb5c1 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,7 +1,7 @@ #!/bin/bash -yum install epel-release -yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm -yum install ffmpeg +# yum install epel-release +# yum localinstall --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm +# yum install ffmpeg # LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python packaging/wheel/relocate.py From aea589f10bbb71fe546e09c0038bab06247a5d7c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 10:55:03 +0100 Subject: [PATCH 32/67] try to run auditwheel? --- .github/workflows/wheel2.yaml | 2 ++ packaging/post_build_script.sh | 8 ++++++++ packaging/pre_build_script.sh | 3 +++ 3 files changed, 13 insertions(+) create mode 100755 packaging/post_build_script.sh create mode 100755 packaging/pre_build_script.sh diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index cb1d28779..dea4f8e38 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -39,6 +39,8 @@ jobs: matrix: include: - repository: pytorch/torchcodec + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh env-var-script: packaging/env_var_script.sh smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh new file mode 100755 index 000000000..343b41043 --- /dev/null +++ b/packaging/post_build_script.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +echo "LS" +ls +echo "LS dist" +ls dist +echo "auditwheel" +auditwheel repair --plat manylinux_2_17_x86_64 dist/* diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh new file mode 100755 index 000000000..f0c64b17c --- /dev/null +++ b/packaging/pre_build_script.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +pip install auditwheel \ No newline at end of file From 6d1e7952ac5a8c47e2f039473c8e9653710b5312 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 11:19:11 +0100 Subject: [PATCH 33/67] Add lots of exclude --- packaging/post_build_script.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 343b41043..30b7db526 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -5,4 +5,7 @@ ls echo "LS dist" ls dist echo "auditwheel" -auditwheel repair --plat manylinux_2_17_x86_64 dist/* +auditwheel repair --plat manylinux_2_17_x86_64 dist/* --exclude libtorch.so --exclude libtorch_cpu.so --exclude libc10.so --exclude libavutil.so.56 --exclude libavcodec.so.58 --exclude libavformat.so.58 --exclude libavdevice.so.58 --exclude libavfilter.so.7 --exclude libavutil.so.57 --exclude libavcodec.so.59 --exclude libavformat.so.59 --exclude libavdevice.so.59 --exclude libavfilter.so.8 --exclude libavutil.so.58 --exclude libavcodec.so.60 --exclude libavformat.so.60 --exclude libavdevice.so.60 --exclude libavfilter.so.9 --exclude libavutil.so.59 --exclude libavcodec.so.61 --exclude libavformat.so.61 --exclude libavdevice.so.61 --exclude libavfilter.so.10 + +echo "tree dist" +tree dist From 0bc9bb035ee062e821ec1446c3d6ab156e3c0801 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 11:26:23 +0100 Subject: [PATCH 34/67] tree -> find --- packaging/post_build_script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 30b7db526..58c578b1e 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -7,5 +7,5 @@ ls dist echo "auditwheel" auditwheel repair --plat manylinux_2_17_x86_64 dist/* --exclude libtorch.so --exclude libtorch_cpu.so --exclude libc10.so --exclude libavutil.so.56 --exclude libavcodec.so.58 --exclude libavformat.so.58 --exclude libavdevice.so.58 --exclude libavfilter.so.7 --exclude libavutil.so.57 --exclude libavcodec.so.59 --exclude libavformat.so.59 --exclude libavdevice.so.59 --exclude libavfilter.so.8 --exclude libavutil.so.58 --exclude libavcodec.so.60 --exclude libavformat.so.60 --exclude libavdevice.so.60 --exclude libavfilter.so.9 --exclude libavutil.so.59 --exclude libavcodec.so.61 --exclude libavformat.so.61 --exclude libavdevice.so.61 --exclude libavfilter.so.10 -echo "tree dist" -tree dist +echo "find dist" +find dist From b60d6ecf43fda1abc2a978128b4c5dbd3f679524 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 11:33:38 +0100 Subject: [PATCH 35/67] cp wheelhouse --- packaging/post_build_script.sh | 6 ++++++ src/torchcodec/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 58c578b1e..f60e3d1c7 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -7,5 +7,11 @@ ls dist echo "auditwheel" auditwheel repair --plat manylinux_2_17_x86_64 dist/* --exclude libtorch.so --exclude libtorch_cpu.so --exclude libc10.so --exclude libavutil.so.56 --exclude libavcodec.so.58 --exclude libavformat.so.58 --exclude libavdevice.so.58 --exclude libavfilter.so.7 --exclude libavutil.so.57 --exclude libavcodec.so.59 --exclude libavformat.so.59 --exclude libavdevice.so.59 --exclude libavfilter.so.8 --exclude libavutil.so.58 --exclude libavcodec.so.60 --exclude libavformat.so.60 --exclude libavdevice.so.60 --exclude libavfilter.so.9 --exclude libavutil.so.59 --exclude libavcodec.so.61 --exclude libavformat.so.61 --exclude libavdevice.so.61 --exclude libavfilter.so.10 +echo "LS" +ls + +echo "cp wheelhouse" +cp -r wheelhouse dist/ + echo "find dist" find dist diff --git a/src/torchcodec/__init__.py b/src/torchcodec/__init__.py index ff3f048d0..e3d1c285c 100644 --- a/src/torchcodec/__init__.py +++ b/src/torchcodec/__init__.py @@ -6,4 +6,4 @@ from . import decoders # noqa -__version__ = "0.0.1.dev" +__version__ = "0.0.1" From 89be61e922ca845e2d2b45cdf23c225ac01641ff Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 11:41:27 +0100 Subject: [PATCH 36/67] ugh --- packaging/post_build_script.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index f60e3d1c7..23c0464a6 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -10,8 +10,9 @@ auditwheel repair --plat manylinux_2_17_x86_64 dist/* --exclude libtorch.so --ex echo "LS" ls -echo "cp wheelhouse" -cp -r wheelhouse dist/ +echo "mv wheelhouse" +rm -r dist +mv wheelhouse dist echo "find dist" find dist From a2a56a5b9ec0222f4fb67477b18b4738948ef2f4 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Thu, 1 Aug 2024 12:22:16 +0100 Subject: [PATCH 37/67] lint --- packaging/fake_smoke_test.py | 1 - packaging/pre_build_script.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/fake_smoke_test.py b/packaging/fake_smoke_test.py index efbfbb583..e6d139906 100644 --- a/packaging/fake_smoke_test.py +++ b/packaging/fake_smoke_test.py @@ -1,2 +1 @@ - print("Success") diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh index f0c64b17c..a26bd5974 100755 --- a/packaging/pre_build_script.sh +++ b/packaging/pre_build_script.sh @@ -1,3 +1,3 @@ #!/bin/bash -pip install auditwheel \ No newline at end of file +pip install auditwheel From 0f110e4d7e65bfc724f090f556db98b8ad62286d Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 10:26:07 +0100 Subject: [PATCH 38/67] . --- .github/workflows/wheel2.yaml | 9 ++------- packaging/env_var_script.sh | 2 ++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index dea4f8e38..3e1eb4a19 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -8,8 +8,6 @@ on: - main - release/* tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: @@ -40,7 +38,6 @@ jobs: include: - repository: pytorch/torchcodec pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh env-var-script: packaging/env_var_script.sh smoke-test-script: packaging/fake_smoke_test.py package-name: torchcodec @@ -53,13 +50,11 @@ jobs: test-infra-ref: main build-matrix: ${{ needs.generate-matrix.outputs.matrix }} pre-script: ${{ matrix.pre-script }} - post-script: ${{ matrix.post-script }} + # post-script: ${{ matrix.post-script }} + post-script: packaging/post_build_script.sh env-var-script: ${{ matrix.env-var-script }} smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - # TODO: remove this comment: only added to trigger checks. - # For torchcodec's build-time-dependency on libtorch: - # https://github.com/pytorch/torchcodec/blob/main/setup.py#L7-L42 wheel-build-extra-args: "--no-isolation" diff --git a/packaging/env_var_script.sh b/packaging/env_var_script.sh index 0ff9e074e..f94c490a8 100644 --- a/packaging/env_var_script.sh +++ b/packaging/env_var_script.sh @@ -1 +1,3 @@ +#!/bin/bash + export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 From ad97d46b49fdb47eb42e5f0797f8910ddc238fbd Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 10:35:30 +0100 Subject: [PATCH 39/67] Cleanup matrix --- .github/workflows/wheel2.yaml | 19 +++++-------------- packaging/fake_smoke_test.py | 6 ++++++ packaging/pre_build_script.sh | 3 --- 3 files changed, 11 insertions(+), 17 deletions(-) delete mode 100755 packaging/pre_build_script.sh diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 3e1eb4a19..c6bae2a47 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -1,4 +1,4 @@ -name: Build Linux Wheels +name: Build and test Linux wheels on: pull_request: @@ -34,27 +34,18 @@ jobs: needs: generate-matrix strategy: fail-fast: false - matrix: - include: - - repository: pytorch/torchcodec - pre-script: packaging/pre_build_script.sh - env-var-script: packaging/env_var_script.sh - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec name: ${{ matrix.repository }} uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main with: - repository: ${{ matrix.repository }} + repository: pytorch/torchcodec ref: "" test-infra-repository: pytorch/test-infra test-infra-ref: main build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - # post-script: ${{ matrix.post-script }} post-script: packaging/post_build_script.sh - env-var-script: ${{ matrix.env-var-script }} - smoke-test-script: ${{ matrix.smoke-test-script }} - package-name: ${{ matrix.package-name }} + env-var-script: packaging/env_var_script.sh + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" wheel-build-extra-args: "--no-isolation" diff --git a/packaging/fake_smoke_test.py b/packaging/fake_smoke_test.py index e6d139906..ac1be55ef 100644 --- a/packaging/fake_smoke_test.py +++ b/packaging/fake_smoke_test.py @@ -1 +1,7 @@ +# This is a fake smoke test that runs on the test-infra instances after we build +# a wheel. We cannot run a real smoke test over there, because the machines are +# too old to even install a proper ffmpeg version - and without ffmpeg, +# importing torchcodec just fails. It's OK, we run our *entire* test suite on +# those wheels anyway (on other machines). + print("Success") diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh deleted file mode 100755 index a26bd5974..000000000 --- a/packaging/pre_build_script.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -pip install auditwheel From 888d28ff578e083c4627908bdfe6b08c9f916673 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 10:36:15 +0100 Subject: [PATCH 40/67] fix --- .github/workflows/wheel2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index c6bae2a47..73ffc9bb8 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -34,7 +34,7 @@ jobs: needs: generate-matrix strategy: fail-fast: false - name: ${{ matrix.repository }} + name: pytorch/torchcodec uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main with: repository: pytorch/torchcodec From 08ce172c45f46fe4063173b311a6229864e3c424 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 10:38:25 +0100 Subject: [PATCH 41/67] . --- packaging/post_build_script.sh | 12 ------------ src/torchcodec/__init__.py | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 23c0464a6..d35958ce9 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -4,15 +4,3 @@ echo "LS" ls echo "LS dist" ls dist -echo "auditwheel" -auditwheel repair --plat manylinux_2_17_x86_64 dist/* --exclude libtorch.so --exclude libtorch_cpu.so --exclude libc10.so --exclude libavutil.so.56 --exclude libavcodec.so.58 --exclude libavformat.so.58 --exclude libavdevice.so.58 --exclude libavfilter.so.7 --exclude libavutil.so.57 --exclude libavcodec.so.59 --exclude libavformat.so.59 --exclude libavdevice.so.59 --exclude libavfilter.so.8 --exclude libavutil.so.58 --exclude libavcodec.so.60 --exclude libavformat.so.60 --exclude libavdevice.so.60 --exclude libavfilter.so.9 --exclude libavutil.so.59 --exclude libavcodec.so.61 --exclude libavformat.so.61 --exclude libavdevice.so.61 --exclude libavfilter.so.10 - -echo "LS" -ls - -echo "mv wheelhouse" -rm -r dist -mv wheelhouse dist - -echo "find dist" -find dist diff --git a/src/torchcodec/__init__.py b/src/torchcodec/__init__.py index e3d1c285c..ff3f048d0 100644 --- a/src/torchcodec/__init__.py +++ b/src/torchcodec/__init__.py @@ -6,4 +6,4 @@ from . import decoders # noqa -__version__ = "0.0.1" +__version__ = "0.0.1.dev" From 26d486e7401a1c827294a4f1c9749f27144d0d65 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 10:39:06 +0100 Subject: [PATCH 42/67] Update current dev version to 0.0.2.dev --- src/torchcodec/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchcodec/__init__.py b/src/torchcodec/__init__.py index ff3f048d0..a97c32ce0 100644 --- a/src/torchcodec/__init__.py +++ b/src/torchcodec/__init__.py @@ -6,4 +6,4 @@ from . import decoders # noqa -__version__ = "0.0.1.dev" +__version__ = "0.0.2.dev" From 5f1a2065d4b23a782196b7f59191b95c26dcdf89 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 10:58:28 +0100 Subject: [PATCH 43/67] . --- .github/workflows/wheel.yaml | 284 +++++++++++++++++----------------- .github/workflows/wheel2.yaml | 72 +++++++++ 2 files changed, 214 insertions(+), 142 deletions(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index f91659730..4d9485968 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -1,156 +1,156 @@ -name: Build wheel +# name: Build wheel -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -concurrency: - group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} - cancel-in-progress: true +# concurrency: +# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} +# cancel-in-progress: true -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - build: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.12'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install Pytorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Check GCC version - run: | - # We want to build wheels on gcc 9 to be consistent with the torch - # binaries, which is why we're using ubuntu-20.04 and not - # ubuntu-latest. More details below about the GLIBCXX checks. - if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" - then - echo "Wrong GCC version, exiting!" - gcc --version - exit 1 - fi - - name: Build the torchcodec wheel - run: | - # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. - .github/scripts/assert_ffmpeg_not_installed.sh +# jobs: +# build: +# runs-on: ubuntu-20.04 +# strategy: +# fail-fast: false +# matrix: +# python-version: ['3.8', '3.12'] +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v2 +# with: +# auto-update-conda: true +# miniconda-version: "latest" +# activate-environment: test +# python-version: ${{ matrix.python-version }} +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install Pytorch +# run: | +# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +# - name: Check GCC version +# run: | +# # We want to build wheels on gcc 9 to be consistent with the torch +# # binaries, which is why we're using ubuntu-20.04 and not +# # ubuntu-latest. More details below about the GLIBCXX checks. +# if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" +# then +# echo "Wrong GCC version, exiting!" +# gcc --version +# exit 1 +# fi +# - name: Build the torchcodec wheel +# run: | +# # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. +# .github/scripts/assert_ffmpeg_not_installed.sh - python -m pip install build - BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation - - name: Validate wheel content - run: | - source .github/scripts/helpers.sh - wheel_path=$(pwd)/$(find dist -type f -name "*.whl") - echo "Wheel content:" - unzip -l $wheel_path +# python -m pip install build +# BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation +# - name: Validate wheel content +# run: | +# source .github/scripts/helpers.sh +# wheel_path=$(pwd)/$(find dist -type f -name "*.whl") +# echo "Wheel content:" +# unzip -l $wheel_path - for ffmpeg_major_version in 4 5 6 7; do - assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so - done - assert_not_in_wheel $wheel_path libtorchcodec.so +# for ffmpeg_major_version in 4 5 6 7; do +# assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so +# done +# assert_not_in_wheel $wheel_path libtorchcodec.so - for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do - assert_not_in_wheel $wheel_path $ffmpeg_so - done +# for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do +# assert_not_in_wheel $wheel_path $ffmpeg_so +# done - assert_not_in_wheel $wheel_path "^test" - assert_not_in_wheel $wheel_path "^doc" - assert_not_in_wheel $wheel_path "^benchmarks" +# assert_not_in_wheel $wheel_path "^test" +# assert_not_in_wheel $wheel_path "^doc" +# assert_not_in_wheel $wheel_path "^benchmarks" - # See invoked python script below for details about this check. - extracted_wheel_dir=$(mktemp -d) - unzip -q $wheel_path -d $extracted_wheel_dir - symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) - python .github/scripts/check_glibcxx.py "$symbols_matches" - - uses: actions/upload-artifact@v4 - with: - name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - path: dist/* +# # See invoked python script below for details about this check. +# extracted_wheel_dir=$(mktemp -d) +# unzip -q $wheel_path -d $extracted_wheel_dir +# symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) +# python .github/scripts/check_glibcxx.py "$symbols_matches" +# - uses: actions/upload-artifact@v4 +# with: +# name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} +# path: dist/* - install-and-test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.12'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - if: ${{ always() }} - needs: build - steps: - - uses: actions/download-artifact@v4 - with: - name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - path: dist/ - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install torchcodec from the wheel - run: | - wheel_path=`find dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv +# install-and-test: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# python-version: ['3.8', '3.12'] +# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] +# if: ${{ always() }} +# needs: build +# steps: +# - uses: actions/download-artifact@v4 +# with: +# name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} +# path: dist/ +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v2 +# with: +# auto-update-conda: true +# miniconda-version: "latest" +# activate-environment: test +# python-version: ${{ matrix.python-version }} +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install PyTorch +# run: | +# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +# - name: Install torchcodec from the wheel +# run: | +# wheel_path=`find dist -type f -name "*.whl"` +# echo Installing $wheel_path +# python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 - - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - .github/scripts/assert_ffmpeg_not_installed.sh +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Install ffmpeg, post build +# run: | +# # Ideally we would have checked for that before installing the wheel, +# # but we need to checkout the repo to access this file, and we don't +# # want to checkout the repo before installing the wheel to avoid any +# # side-effect. It's OK. +# .github/scripts/assert_ffmpeg_not_installed.sh - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version +# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge +# ffmpeg -version - - name: Install test dependencies - run: | - python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow +# - name: Install test dependencies +# run: | +# python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu +# # Ideally we would find a way to get those dependencies from pyproject.toml +# python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Smoke test - run: | - python test/decoders/manual_smoke_test.py - - name: Run Python tests - run: | - pytest test +# - name: Delete the src/ folder just for fun +# run: | +# # The only reason we checked-out the repo is to get access to the +# # tests. We don't care about the rest. Out of precaution, we delete +# # the src/ folder to be extra sure that we're running the code from +# # the installed wheel rather than from the source. +# # This is just to be extra cautious and very overkill because a) +# # there's no way the `torchcodec` package from src/ can be found from +# # the PythonPath: the main point of `src/` is precisely to protect +# # against that and b) if we ever were to execute code from +# # `src/torchcodec`, it would fail loudly because the built .so files +# # aren't present there. +# rm -r src/ +# ls +# - name: Smoke test +# run: | +# python test/decoders/manual_smoke_test.py +# - name: Run Python tests +# run: | +# pytest test diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 73ffc9bb8..7d89d3a77 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -49,3 +49,75 @@ jobs: trigger-event: ${{ github.event_name }} build-platform: "python-build-package" wheel-build-extra-args: "--no-isolation" + + install-and-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.9'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + if: ${{ always() }} + needs: build + steps: + - uses: actions/download-artifact@v4 + with: + name: pytorch_torchcodec__3.9_cpu_x86_64 + path: dist/ + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install PyTorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv + + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + .github/scripts/assert_ffmpeg_not_installed.sh + + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version + + - name: Install test dependencies + run: | + python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow + + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Smoke test + run: | + python test/decoders/manual_smoke_test.py + - name: Run Python tests + run: | + pytest test From 0ecb50ca427574cc4a9860c1788860f976385e71 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 10:59:27 +0100 Subject: [PATCH 44/67] temporary disable cpp and docs job --- .github/workflows/cpp_tests.yaml | 128 +++++++++++++++---------------- .github/workflows/docs.yaml | 90 +++++++++++----------- 2 files changed, 109 insertions(+), 109 deletions(-) diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index 2a948b0df..a03ed90bf 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -1,69 +1,69 @@ -name: CPP tests +# name: CPP tests -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -concurrency: - group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} - cancel-in-progress: true +# concurrency: +# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} +# cancel-in-progress: true -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - Cpp-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: '3.12' - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install dependencies - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install ffmpeg and pkg-config - run: | - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config -c conda-forge - ffmpeg -version - - name: Build and run C++ tests - run: | - # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so - # we're building libtorchcodec against the installed FFmpeg version - # (from conda-forge) instead of building against our pre-built non-GPL - # FFmpeg libraries. - # The reason we need this is because the C++ tests decode x264 files. - # x264 support is not LGPL, os it is not supported by our - # pre-built non-GPL FFmpeg libraries. And if we were to build against - # those, this is also what the tests would be loading at run time, - # then failing when we try to decode x264. - # To remediate that, we build against the FFmpeg that we installed - # from conda-forge (which is able to decode x264), and that's also - # what gets loaded at run time. - # The Python tests are also decoding x264 files, and are built against - # our non-GPL FFmpeg. And yet they pass. This is because in Python - # we're able to distinguish between build-time (non-GPL FFmpeg) and - # run time (conda-forge FFmpeg). +# jobs: +# Cpp-tests: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v2 +# with: +# auto-update-conda: true +# miniconda-version: "latest" +# activate-environment: test +# python-version: '3.12' +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install dependencies +# run: | +# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +# - name: Install ffmpeg and pkg-config +# run: | +# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config -c conda-forge +# ffmpeg -version +# - name: Build and run C++ tests +# run: | +# # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so +# # we're building libtorchcodec against the installed FFmpeg version +# # (from conda-forge) instead of building against our pre-built non-GPL +# # FFmpeg libraries. +# # The reason we need this is because the C++ tests decode x264 files. +# # x264 support is not LGPL, os it is not supported by our +# # pre-built non-GPL FFmpeg libraries. And if we were to build against +# # those, this is also what the tests would be loading at run time, +# # then failing when we try to decode x264. +# # To remediate that, we build against the FFmpeg that we installed +# # from conda-forge (which is able to decode x264), and that's also +# # what gets loaded at run time. +# # The Python tests are also decoding x264 files, and are built against +# # our non-GPL FFmpeg. And yet they pass. This is because in Python +# # we're able to distinguish between build-time (non-GPL FFmpeg) and +# # run time (conda-forge FFmpeg). - build_tests_dir="${PWD}/build_tests" - mkdir $build_tests_dir - pushd $build_tests_dir - TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") - Torch_DIR="${TORCH_PATH}/share/cmake/Torch" - cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON - cmake --build . - ctest - popd +# build_tests_dir="${PWD}/build_tests" +# mkdir $build_tests_dir +# pushd $build_tests_dir +# TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") +# Torch_DIR="${TORCH_PATH}/share/cmake/Torch" +# cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON +# cmake --build . +# ctest +# popd diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 8ed149a87..c93741e2b 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,48 +1,48 @@ -name: Docs +# name: Docs -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} -jobs: - build: - runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: '3.12' - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install dependencies and FFmpeg - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - conda install "ffmpeg=7.0.1" pkg-config -c conda-forge - ffmpeg -version - - name: Build and install torchcodec - run: | - python -m pip install -e ".[dev]" --no-build-isolation -vvv - - name: Install doc dependencies - run: | - cd docs - python -m pip install -r requirements.txt - - name: Build docs - run: | - cd docs - make html - - uses: actions/upload-artifact@v3 - with: - name: Built-Docs - path: docs/build/html/ +# jobs: +# build: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# steps: +# - name: Check out repo +# uses: actions/checkout@v3 +# - name: Setup conda env +# uses: conda-incubator/setup-miniconda@v2 +# with: +# auto-update-conda: true +# miniconda-version: "latest" +# activate-environment: test +# python-version: '3.12' +# - name: Update pip +# run: python -m pip install --upgrade pip +# - name: Install dependencies and FFmpeg +# run: | +# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu +# conda install "ffmpeg=7.0.1" pkg-config -c conda-forge +# ffmpeg -version +# - name: Build and install torchcodec +# run: | +# python -m pip install -e ".[dev]" --no-build-isolation -vvv +# - name: Install doc dependencies +# run: | +# cd docs +# python -m pip install -r requirements.txt +# - name: Build docs +# run: | +# cd docs +# make html +# - uses: actions/upload-artifact@v3 +# with: +# name: Built-Docs +# path: docs/build/html/ From d00fe321f8ec6b7573e7b3ff44593c7e4d9829f2 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 11:05:08 +0100 Subject: [PATCH 45/67] Fix name? --- .github/workflows/wheel2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 7d89d3a77..285dd300a 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -62,8 +62,8 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - name: pytorch_torchcodec__3.9_cpu_x86_64 - path: dist/ + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 + path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v2 with: From bd2b7e425a0fad269df0d29c957135ba07f81afa Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 11:12:56 +0100 Subject: [PATCH 46/67] Fallback to v3?? --- .github/workflows/wheel2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 285dd300a..1289df2d6 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -60,7 +60,7 @@ jobs: if: ${{ always() }} needs: build steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 with: name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 path: pytorch/torchcodec/dist/ From c2a327a933e81f9170eb47db6d9629f3261b1f12 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 11:22:04 +0100 Subject: [PATCH 47/67] Fix wheel path --- .github/workflows/wheel2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 1289df2d6..b18cb6088 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -34,7 +34,7 @@ jobs: needs: generate-matrix strategy: fail-fast: false - name: pytorch/torchcodec + name: Build and Upload wheel uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main with: repository: pytorch/torchcodec @@ -78,7 +78,7 @@ jobs: python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - name: Install torchcodec from the wheel run: | - wheel_path=`find dist -type f -name "*.whl"` + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` echo Installing $wheel_path python -m pip install $wheel_path -vvv From 540ee89ed641bcd4f8cf5dbc97eb3a945fe0d6db Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 11:37:36 +0100 Subject: [PATCH 48/67] rename wheel --- packaging/post_build_script.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index d35958ce9..20a8139e3 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -4,3 +4,10 @@ echo "LS" ls echo "LS dist" ls dist + + +echo "MVVVV" +mv dist/*linux*.whl $(echo dist/*linux*.whl | sed 's/linux/manylinux2014/') + +echo "LS dist" +ls dist \ No newline at end of file From 699e4149acde7b559b3d6b9b2195e7c44f88bc16 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 11:48:56 +0100 Subject: [PATCH 49/67] debuv --- .github/workflows/wheel.yaml | 284 ++++++++++++++++----------------- packaging/post_build_script.sh | 4 +- 2 files changed, 144 insertions(+), 144 deletions(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index 4d9485968..f91659730 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -1,156 +1,156 @@ -# name: Build wheel +name: Build wheel -# on: -# push: -# branches: [ main ] -# pull_request: +on: + push: + branches: [ main ] + pull_request: -# concurrency: -# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} -# cancel-in-progress: true +concurrency: + group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} + cancel-in-progress: true -# defaults: -# run: -# shell: bash -l -eo pipefail {0} +defaults: + run: + shell: bash -l -eo pipefail {0} -# jobs: -# build: -# runs-on: ubuntu-20.04 -# strategy: -# fail-fast: false -# matrix: -# python-version: ['3.8', '3.12'] -# steps: -# - name: Check out repo -# uses: actions/checkout@v3 -# - name: Setup conda env -# uses: conda-incubator/setup-miniconda@v2 -# with: -# auto-update-conda: true -# miniconda-version: "latest" -# activate-environment: test -# python-version: ${{ matrix.python-version }} -# - name: Update pip -# run: python -m pip install --upgrade pip -# - name: Install Pytorch -# run: | -# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -# - name: Check GCC version -# run: | -# # We want to build wheels on gcc 9 to be consistent with the torch -# # binaries, which is why we're using ubuntu-20.04 and not -# # ubuntu-latest. More details below about the GLIBCXX checks. -# if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" -# then -# echo "Wrong GCC version, exiting!" -# gcc --version -# exit 1 -# fi -# - name: Build the torchcodec wheel -# run: | -# # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. -# .github/scripts/assert_ffmpeg_not_installed.sh +jobs: + build: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.12'] + steps: + - name: Check out repo + uses: actions/checkout@v3 + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install Pytorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Check GCC version + run: | + # We want to build wheels on gcc 9 to be consistent with the torch + # binaries, which is why we're using ubuntu-20.04 and not + # ubuntu-latest. More details below about the GLIBCXX checks. + if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" + then + echo "Wrong GCC version, exiting!" + gcc --version + exit 1 + fi + - name: Build the torchcodec wheel + run: | + # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. + .github/scripts/assert_ffmpeg_not_installed.sh -# python -m pip install build -# BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation -# - name: Validate wheel content -# run: | -# source .github/scripts/helpers.sh -# wheel_path=$(pwd)/$(find dist -type f -name "*.whl") -# echo "Wheel content:" -# unzip -l $wheel_path + python -m pip install build + BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation + - name: Validate wheel content + run: | + source .github/scripts/helpers.sh + wheel_path=$(pwd)/$(find dist -type f -name "*.whl") + echo "Wheel content:" + unzip -l $wheel_path -# for ffmpeg_major_version in 4 5 6 7; do -# assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so -# done -# assert_not_in_wheel $wheel_path libtorchcodec.so + for ffmpeg_major_version in 4 5 6 7; do + assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so + done + assert_not_in_wheel $wheel_path libtorchcodec.so -# for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do -# assert_not_in_wheel $wheel_path $ffmpeg_so -# done + for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do + assert_not_in_wheel $wheel_path $ffmpeg_so + done -# assert_not_in_wheel $wheel_path "^test" -# assert_not_in_wheel $wheel_path "^doc" -# assert_not_in_wheel $wheel_path "^benchmarks" + assert_not_in_wheel $wheel_path "^test" + assert_not_in_wheel $wheel_path "^doc" + assert_not_in_wheel $wheel_path "^benchmarks" -# # See invoked python script below for details about this check. -# extracted_wheel_dir=$(mktemp -d) -# unzip -q $wheel_path -d $extracted_wheel_dir -# symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) -# python .github/scripts/check_glibcxx.py "$symbols_matches" -# - uses: actions/upload-artifact@v4 -# with: -# name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} -# path: dist/* + # See invoked python script below for details about this check. + extracted_wheel_dir=$(mktemp -d) + unzip -q $wheel_path -d $extracted_wheel_dir + symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) + python .github/scripts/check_glibcxx.py "$symbols_matches" + - uses: actions/upload-artifact@v4 + with: + name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} + path: dist/* -# install-and-test: -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# python-version: ['3.8', '3.12'] -# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] -# if: ${{ always() }} -# needs: build -# steps: -# - uses: actions/download-artifact@v4 -# with: -# name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} -# path: dist/ -# - name: Setup conda env -# uses: conda-incubator/setup-miniconda@v2 -# with: -# auto-update-conda: true -# miniconda-version: "latest" -# activate-environment: test -# python-version: ${{ matrix.python-version }} -# - name: Update pip -# run: python -m pip install --upgrade pip -# - name: Install PyTorch -# run: | -# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -# - name: Install torchcodec from the wheel -# run: | -# wheel_path=`find dist -type f -name "*.whl"` -# echo Installing $wheel_path -# python -m pip install $wheel_path -vvv + install-and-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.12'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + if: ${{ always() }} + needs: build + steps: + - uses: actions/download-artifact@v4 + with: + name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} + path: dist/ + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install PyTorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv -# - name: Check out repo -# uses: actions/checkout@v3 -# - name: Install ffmpeg, post build -# run: | -# # Ideally we would have checked for that before installing the wheel, -# # but we need to checkout the repo to access this file, and we don't -# # want to checkout the repo before installing the wheel to avoid any -# # side-effect. It's OK. -# .github/scripts/assert_ffmpeg_not_installed.sh + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + .github/scripts/assert_ffmpeg_not_installed.sh -# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge -# ffmpeg -version + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version -# - name: Install test dependencies -# run: | -# python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -# # Ideally we would find a way to get those dependencies from pyproject.toml -# python -m pip install numpy pytest pillow + - name: Install test dependencies + run: | + python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow -# - name: Delete the src/ folder just for fun -# run: | -# # The only reason we checked-out the repo is to get access to the -# # tests. We don't care about the rest. Out of precaution, we delete -# # the src/ folder to be extra sure that we're running the code from -# # the installed wheel rather than from the source. -# # This is just to be extra cautious and very overkill because a) -# # there's no way the `torchcodec` package from src/ can be found from -# # the PythonPath: the main point of `src/` is precisely to protect -# # against that and b) if we ever were to execute code from -# # `src/torchcodec`, it would fail loudly because the built .so files -# # aren't present there. -# rm -r src/ -# ls -# - name: Smoke test -# run: | -# python test/decoders/manual_smoke_test.py -# - name: Run Python tests -# run: | -# pytest test + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Smoke test + run: | + python test/decoders/manual_smoke_test.py + - name: Run Python tests + run: | + pytest test diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 20a8139e3..a8aa6c462 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -7,7 +7,7 @@ ls dist echo "MVVVV" -mv dist/*linux*.whl $(echo dist/*linux*.whl | sed 's/linux/manylinux2014/') +mv dist/*linux_x86_64*.whl $(echo dist/*linux_x86_64*.whl | sed 's/linux_x86_64/manylinux_2_17_x86_64.manylinux2014_x86_64/') echo "LS dist" -ls dist \ No newline at end of file +ls dist From 0e7db2dbc56e291f519ea09746215500e8e0f005 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 11:56:36 +0100 Subject: [PATCH 50/67] Use old ubuntu??? --- .github/workflows/wheel2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index b18cb6088..bbaa15a63 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -51,7 +51,7 @@ jobs: wheel-build-extra-args: "--no-isolation" install-and-test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: From 51f7b764bcc08c5ed44151bda3dd0f35d6681004 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:03:36 +0100 Subject: [PATCH 51/67] change permissions --- .github/workflows/wheel2.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index bbaa15a63..8254a5eab 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -15,10 +15,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} cancel-in-progress: true -permissions: - id-token: write - contents: read - jobs: generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main @@ -51,7 +47,7 @@ jobs: wheel-build-extra-args: "--no-isolation" install-and-test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: From e0e2cf87af30f7bd2e26413654635c9eb6a31622 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:06:35 +0100 Subject: [PATCH 52/67] again --- .github/workflows/wheel2.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 8254a5eab..eb7a1fc24 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -15,6 +15,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} cancel-in-progress: true +permissions: + id-token: write + contents: write + jobs: generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main From 5fa125894358bf9b8b6db3de7be99a55ce8c847c Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:15:57 +0100 Subject: [PATCH 53/67] wtfff --- .github/workflows/wheel2.yaml | 96 ++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index eb7a1fc24..05a7a9fa9 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -58,12 +58,12 @@ jobs: python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] if: ${{ always() }} - needs: build + # needs: build steps: - - uses: actions/download-artifact@v3 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 - path: pytorch/torchcodec/dist/ + # - uses: actions/download-artifact@v3 + # with: + # name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 + # path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v2 with: @@ -72,52 +72,54 @@ jobs: activate-environment: test python-version: ${{ matrix.python-version }} - name: Update pip - run: python -m pip install --upgrade pip + run: | + python --version + python -m pip install --upgrade pip - name: Install PyTorch run: | python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + # echo Installing $wheel_path + # python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 - - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - .github/scripts/assert_ffmpeg_not_installed.sh + # - name: Check out repo + # uses: actions/checkout@v3 + # - name: Install ffmpeg, post build + # run: | + # # Ideally we would have checked for that before installing the wheel, + # # but we need to checkout the repo to access this file, and we don't + # # want to checkout the repo before installing the wheel to avoid any + # # side-effect. It's OK. + # .github/scripts/assert_ffmpeg_not_installed.sh - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version + # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + # ffmpeg -version - - name: Install test dependencies - run: | - python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow + # - name: Install test dependencies + # run: | + # python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # # Ideally we would find a way to get those dependencies from pyproject.toml + # python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Smoke test - run: | - python test/decoders/manual_smoke_test.py - - name: Run Python tests - run: | - pytest test + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls + # - name: Smoke test + # run: | + # python test/decoders/manual_smoke_test.py + # - name: Run Python tests + # run: | + # pytest test From ed379064289e68932114e804114beb64fceae6f9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:16:56 +0100 Subject: [PATCH 54/67] uhghusrghusrgh --- .github/workflows/wheel2.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 05a7a9fa9..b443ae873 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -15,9 +15,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} cancel-in-progress: true -permissions: - id-token: write - contents: write +# permissions: +# id-token: write +# contents: write jobs: generate-matrix: From 8935a41a658c57aa8cb859e50748ef322be65a0a Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:21:50 +0100 Subject: [PATCH 55/67] sigh --- .github/workflows/wheel2.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index b443ae873..05a7a9fa9 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -15,9 +15,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} cancel-in-progress: true -# permissions: -# id-token: write -# contents: write +permissions: + id-token: write + contents: write jobs: generate-matrix: From 133480925efb56b0d736c828a594ed22e12fb0cc Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:23:33 +0100 Subject: [PATCH 56/67] activate env?? --- .github/workflows/wheel2.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 05a7a9fa9..4788e5c1a 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -73,6 +73,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update pip run: | + conda activate test python --version python -m pip install --upgrade pip - name: Install PyTorch From f2a8fa0a5fac499d39dc8b845725159b39d1ff69 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:27:40 +0100 Subject: [PATCH 57/67] . --- .github/workflows/wheel.yaml | 227 +++++++++++++++++----------------- .github/workflows/wheel2.yaml | 60 ++++----- 2 files changed, 145 insertions(+), 142 deletions(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index f91659730..b0534f51a 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -31,126 +31,129 @@ jobs: activate-environment: test python-version: ${{ matrix.python-version }} - name: Update pip - run: python -m pip install --upgrade pip - - name: Install Pytorch run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Check GCC version - run: | - # We want to build wheels on gcc 9 to be consistent with the torch - # binaries, which is why we're using ubuntu-20.04 and not - # ubuntu-latest. More details below about the GLIBCXX checks. - if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" - then - echo "Wrong GCC version, exiting!" - gcc --version - exit 1 - fi - - name: Build the torchcodec wheel - run: | - # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. - .github/scripts/assert_ffmpeg_not_installed.sh + echo $CONDA_PREFIX + python --version + python -m pip install --upgrade pip + # - name: Install Pytorch + # run: | + # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + # - name: Check GCC version + # run: | + # # We want to build wheels on gcc 9 to be consistent with the torch + # # binaries, which is why we're using ubuntu-20.04 and not + # # ubuntu-latest. More details below about the GLIBCXX checks. + # if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" + # then + # echo "Wrong GCC version, exiting!" + # gcc --version + # exit 1 + # fi + # - name: Build the torchcodec wheel + # run: | + # # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. + # .github/scripts/assert_ffmpeg_not_installed.sh - python -m pip install build - BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation - - name: Validate wheel content - run: | - source .github/scripts/helpers.sh - wheel_path=$(pwd)/$(find dist -type f -name "*.whl") - echo "Wheel content:" - unzip -l $wheel_path + # python -m pip install build + # BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation + # - name: Validate wheel content + # run: | + # source .github/scripts/helpers.sh + # wheel_path=$(pwd)/$(find dist -type f -name "*.whl") + # echo "Wheel content:" + # unzip -l $wheel_path - for ffmpeg_major_version in 4 5 6 7; do - assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so - done - assert_not_in_wheel $wheel_path libtorchcodec.so + # for ffmpeg_major_version in 4 5 6 7; do + # assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so + # done + # assert_not_in_wheel $wheel_path libtorchcodec.so - for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do - assert_not_in_wheel $wheel_path $ffmpeg_so - done + # for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do + # assert_not_in_wheel $wheel_path $ffmpeg_so + # done - assert_not_in_wheel $wheel_path "^test" - assert_not_in_wheel $wheel_path "^doc" - assert_not_in_wheel $wheel_path "^benchmarks" + # assert_not_in_wheel $wheel_path "^test" + # assert_not_in_wheel $wheel_path "^doc" + # assert_not_in_wheel $wheel_path "^benchmarks" - # See invoked python script below for details about this check. - extracted_wheel_dir=$(mktemp -d) - unzip -q $wheel_path -d $extracted_wheel_dir - symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) - python .github/scripts/check_glibcxx.py "$symbols_matches" - - uses: actions/upload-artifact@v4 - with: - name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - path: dist/* + # # See invoked python script below for details about this check. + # extracted_wheel_dir=$(mktemp -d) + # unzip -q $wheel_path -d $extracted_wheel_dir + # symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) + # python .github/scripts/check_glibcxx.py "$symbols_matches" + # - uses: actions/upload-artifact@v4 + # with: + # name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} + # path: dist/* - install-and-test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.12'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - if: ${{ always() }} - needs: build - steps: - - uses: actions/download-artifact@v4 - with: - name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - path: dist/ - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: python -m pip install --upgrade pip - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install torchcodec from the wheel - run: | - wheel_path=`find dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv + # install-and-test: + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # matrix: + # python-version: ['3.8', '3.12'] + # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + # if: ${{ always() }} + # needs: build + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} + # path: dist/ + # - name: Setup conda env + # uses: conda-incubator/setup-miniconda@v2 + # with: + # auto-update-conda: true + # miniconda-version: "latest" + # activate-environment: test + # python-version: ${{ matrix.python-version }} + # - name: Update pip + # run: python -m pip install --upgrade pip + # - name: Install PyTorch + # run: | + # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + # - name: Install torchcodec from the wheel + # run: | + # wheel_path=`find dist -type f -name "*.whl"` + # echo Installing $wheel_path + # python -m pip install $wheel_path -vvv - - name: Check out repo - uses: actions/checkout@v3 - - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - .github/scripts/assert_ffmpeg_not_installed.sh + # - name: Check out repo + # uses: actions/checkout@v3 + # - name: Install ffmpeg, post build + # run: | + # # Ideally we would have checked for that before installing the wheel, + # # but we need to checkout the repo to access this file, and we don't + # # want to checkout the repo before installing the wheel to avoid any + # # side-effect. It's OK. + # .github/scripts/assert_ffmpeg_not_installed.sh - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version + # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + # ffmpeg -version - - name: Install test dependencies - run: | - python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow + # - name: Install test dependencies + # run: | + # python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # # Ideally we would find a way to get those dependencies from pyproject.toml + # python -m pip install numpy pytest pillow - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Smoke test - run: | - python test/decoders/manual_smoke_test.py - - name: Run Python tests - run: | - pytest test + # - name: Delete the src/ folder just for fun + # run: | + # # The only reason we checked-out the repo is to get access to the + # # tests. We don't care about the rest. Out of precaution, we delete + # # the src/ folder to be extra sure that we're running the code from + # # the installed wheel rather than from the source. + # # This is just to be extra cautious and very overkill because a) + # # there's no way the `torchcodec` package from src/ can be found from + # # the PythonPath: the main point of `src/` is precisely to protect + # # against that and b) if we ever were to execute code from + # # `src/torchcodec`, it would fail loudly because the built .so files + # # aren't present there. + # rm -r src/ + # ls + # - name: Smoke test + # run: | + # python test/decoders/manual_smoke_test.py + # - name: Run Python tests + # run: | + # pytest test diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 4788e5c1a..18585ca27 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -20,35 +20,35 @@ permissions: contents: write jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-xpu: disable - with-rocm: disable - with-cuda: disable - build: - needs: generate-matrix - strategy: - fail-fast: false - name: Build and Upload wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: pytorch/torchcodec - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - post-script: packaging/post_build_script.sh - env-var-script: packaging/env_var_script.sh - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec - trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - wheel-build-extra-args: "--no-isolation" + # generate-matrix: + # uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + # with: + # package-type: wheel + # os: linux + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # with-xpu: disable + # with-rocm: disable + # with-cuda: disable + # build: + # needs: generate-matrix + # strategy: + # fail-fast: false + # name: Build and Upload wheel + # uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + # with: + # repository: pytorch/torchcodec + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + # post-script: packaging/post_build_script.sh + # env-var-script: packaging/env_var_script.sh + # smoke-test-script: packaging/fake_smoke_test.py + # package-name: torchcodec + # trigger-event: ${{ github.event_name }} + # build-platform: "python-build-package" + # wheel-build-extra-args: "--no-isolation" install-and-test: runs-on: ubuntu-latest @@ -73,7 +73,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update pip run: | - conda activate test + echo $CONDA_PREFIX python --version python -m pip install --upgrade pip - name: Install PyTorch From 7687d6ad174298f32a18acad7e40d2d9e2d02140 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:31:18 +0100 Subject: [PATCH 58/67] . --- .github/workflows/wheel2.yaml | 39 ++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 18585ca27..7a596f681 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -1,23 +1,38 @@ name: Build and test Linux wheels +# on: +# pull_request: +# push: +# branches: +# - nightly +# - main +# - release/* +# tags: +# - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ +# workflow_dispatch: + +# concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} +# cancel-in-progress: true + +# permissions: +# id-token: write +# contents: write + + on: - pull_request: push: - branches: - - nightly - - main - - release/* - tags: - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: + branches: [ main ] + pull_request: concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} cancel-in-progress: true -permissions: - id-token: write - contents: write +defaults: + run: + shell: bash -l -eo pipefail {0} + jobs: # generate-matrix: From 5520f1fc3f1ef4dde3c210212f9c7ac7666e5388 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:35:34 +0100 Subject: [PATCH 59/67] . --- .github/workflows/wheel2.yaml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 7a596f681..5fecdd629 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -1,15 +1,15 @@ name: Build and test Linux wheels -# on: -# pull_request: -# push: -# branches: -# - nightly -# - main -# - release/* -# tags: -# - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ -# workflow_dispatch: +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: # concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} @@ -20,14 +20,14 @@ name: Build and test Linux wheels # contents: write -on: - push: - branches: [ main ] - pull_request: +# on: +# push: +# branches: [ main ] +# pull_request: -concurrency: - group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} - cancel-in-progress: true +# concurrency: +# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} +# cancel-in-progress: true defaults: run: From 286a089cd3b994e765d7d7acd2bc6d20fe0ed5b7 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:35:47 +0100 Subject: [PATCH 60/67] . --- .github/workflows/wheel2.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 5fecdd629..b289a502a 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -11,9 +11,9 @@ on: - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: -# concurrency: -# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} -# cancel-in-progress: true +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true # permissions: # id-token: write From dcb3cba5a362d6dbf27997cec075e7afc2a50b37 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:36:43 +0100 Subject: [PATCH 61/67] . --- .github/workflows/wheel2.yaml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index b289a502a..3c05a0672 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -20,18 +20,9 @@ concurrency: # contents: write -# on: -# push: -# branches: [ main ] -# pull_request: - -# concurrency: -# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} -# cancel-in-progress: true - -defaults: - run: - shell: bash -l -eo pipefail {0} +# defaults: +# run: +# shell: bash -l -eo pipefail {0} jobs: From 6782c0a84db9757538225c806e804bf269bbfe64 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:37:48 +0100 Subject: [PATCH 62/67] . --- .github/workflows/wheel2.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 3c05a0672..b9169c168 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -15,14 +15,14 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} cancel-in-progress: true -# permissions: -# id-token: write -# contents: write +permissions: + id-token: write + contents: write -# defaults: -# run: -# shell: bash -l -eo pipefail {0} +defaults: + run: + shell: bash -l -eo pipefail {0} jobs: From 794104a529f73f61a821b6927678b851acdf7827 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 12:38:46 +0100 Subject: [PATCH 63/67] . --- .github/workflows/wheel2.yaml | 150 +++++++++++++++++----------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index b9169c168..9ee6ab114 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -26,35 +26,35 @@ defaults: jobs: - # generate-matrix: - # uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - # with: - # package-type: wheel - # os: linux - # test-infra-repository: pytorch/test-infra - # test-infra-ref: main - # with-xpu: disable - # with-rocm: disable - # with-cuda: disable - # build: - # needs: generate-matrix - # strategy: - # fail-fast: false - # name: Build and Upload wheel - # uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - # with: - # repository: pytorch/torchcodec - # ref: "" - # test-infra-repository: pytorch/test-infra - # test-infra-ref: main - # build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - # post-script: packaging/post_build_script.sh - # env-var-script: packaging/env_var_script.sh - # smoke-test-script: packaging/fake_smoke_test.py - # package-name: torchcodec - # trigger-event: ${{ github.event_name }} - # build-platform: "python-build-package" - # wheel-build-extra-args: "--no-isolation" + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-xpu: disable + with-rocm: disable + with-cuda: disable + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + post-script: packaging/post_build_script.sh + env-var-script: packaging/env_var_script.sh + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + wheel-build-extra-args: "--no-isolation" install-and-test: runs-on: ubuntu-latest @@ -64,12 +64,12 @@ jobs: python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] if: ${{ always() }} - # needs: build + needs: build steps: - # - uses: actions/download-artifact@v3 - # with: - # name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 - # path: pytorch/torchcodec/dist/ + - uses: actions/download-artifact@v3 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 + path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v2 with: @@ -85,48 +85,48 @@ jobs: - name: Install PyTorch run: | python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - # - name: Install torchcodec from the wheel - # run: | - # wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - # echo Installing $wheel_path - # python -m pip install $wheel_path -vvv + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv - # - name: Check out repo - # uses: actions/checkout@v3 - # - name: Install ffmpeg, post build - # run: | - # # Ideally we would have checked for that before installing the wheel, - # # but we need to checkout the repo to access this file, and we don't - # # want to checkout the repo before installing the wheel to avoid any - # # side-effect. It's OK. - # .github/scripts/assert_ffmpeg_not_installed.sh + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + .github/scripts/assert_ffmpeg_not_installed.sh - # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - # ffmpeg -version + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version - # - name: Install test dependencies - # run: | - # python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # # Ideally we would find a way to get those dependencies from pyproject.toml - # python -m pip install numpy pytest pillow + - name: Install test dependencies + run: | + python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow - # - name: Delete the src/ folder just for fun - # run: | - # # The only reason we checked-out the repo is to get access to the - # # tests. We don't care about the rest. Out of precaution, we delete - # # the src/ folder to be extra sure that we're running the code from - # # the installed wheel rather than from the source. - # # This is just to be extra cautious and very overkill because a) - # # there's no way the `torchcodec` package from src/ can be found from - # # the PythonPath: the main point of `src/` is precisely to protect - # # against that and b) if we ever were to execute code from - # # `src/torchcodec`, it would fail loudly because the built .so files - # # aren't present there. - # rm -r src/ - # ls - # - name: Smoke test - # run: | - # python test/decoders/manual_smoke_test.py - # - name: Run Python tests - # run: | - # pytest test + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Smoke test + run: | + python test/decoders/manual_smoke_test.py + - name: Run Python tests + run: | + pytest test From ddd32b1f354e73d5b4431d6ea96525da5232ebc4 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 13:10:58 +0100 Subject: [PATCH 64/67] Add sanity checks --- .github/workflows/wheel.yaml | 225 +++++++++++++++++---------------- packaging/post_build_script.sh | 32 +++-- 2 files changed, 137 insertions(+), 120 deletions(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index b0534f51a..49beaa23c 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -35,125 +35,126 @@ jobs: echo $CONDA_PREFIX python --version python -m pip install --upgrade pip - # - name: Install Pytorch - # run: | - # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - # - name: Check GCC version - # run: | - # # We want to build wheels on gcc 9 to be consistent with the torch - # # binaries, which is why we're using ubuntu-20.04 and not - # # ubuntu-latest. More details below about the GLIBCXX checks. - # if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" - # then - # echo "Wrong GCC version, exiting!" - # gcc --version - # exit 1 - # fi - # - name: Build the torchcodec wheel - # run: | - # # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. - # .github/scripts/assert_ffmpeg_not_installed.sh + - name: Install Pytorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Check GCC version + run: | + # We want to build wheels on gcc 9 to be consistent with the torch + # binaries, which is why we're using ubuntu-20.04 and not + # ubuntu-latest. More details below about the GLIBCXX checks. + if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" + then + echo "Wrong GCC version, exiting!" + gcc --version + exit 1 + fi + - name: Build the torchcodec wheel + run: | + # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. + .github/scripts/assert_ffmpeg_not_installed.sh - # python -m pip install build - # BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation - # - name: Validate wheel content - # run: | - # source .github/scripts/helpers.sh - # wheel_path=$(pwd)/$(find dist -type f -name "*.whl") - # echo "Wheel content:" - # unzip -l $wheel_path + python -m pip install build + BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation + - name: Validate wheel content + run: | + source .github/scripts/helpers.sh + wheel_path=$(pwd)/$(find dist -type f -name "*.whl") + echo "Wheel content:" + unzip -l $wheel_path - # for ffmpeg_major_version in 4 5 6 7; do - # assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so - # done - # assert_not_in_wheel $wheel_path libtorchcodec.so + for ffmpeg_major_version in 4 5 6 7; do + assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so + done + assert_not_in_wheel $wheel_path libtorchcodec.so - # for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do - # assert_not_in_wheel $wheel_path $ffmpeg_so - # done + for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do + assert_not_in_wheel $wheel_path $ffmpeg_so + done - # assert_not_in_wheel $wheel_path "^test" - # assert_not_in_wheel $wheel_path "^doc" - # assert_not_in_wheel $wheel_path "^benchmarks" + assert_not_in_wheel $wheel_path "^test" + assert_not_in_wheel $wheel_path "^doc" + assert_not_in_wheel $wheel_path "^benchmarks" + assert_not_in_wheel $wheel_path "^packaging" - # # See invoked python script below for details about this check. - # extracted_wheel_dir=$(mktemp -d) - # unzip -q $wheel_path -d $extracted_wheel_dir - # symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) - # python .github/scripts/check_glibcxx.py "$symbols_matches" - # - uses: actions/upload-artifact@v4 - # with: - # name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - # path: dist/* + # See invoked python script below for details about this check. + extracted_wheel_dir=$(mktemp -d) + unzip -q $wheel_path -d $extracted_wheel_dir + symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) + python .github/scripts/check_glibcxx.py "$symbols_matches" + - uses: actions/upload-artifact@v4 + with: + name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} + path: dist/* - # install-and-test: - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # python-version: ['3.8', '3.12'] - # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - # if: ${{ always() }} - # needs: build - # steps: - # - uses: actions/download-artifact@v4 - # with: - # name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - # path: dist/ - # - name: Setup conda env - # uses: conda-incubator/setup-miniconda@v2 - # with: - # auto-update-conda: true - # miniconda-version: "latest" - # activate-environment: test - # python-version: ${{ matrix.python-version }} - # - name: Update pip - # run: python -m pip install --upgrade pip - # - name: Install PyTorch - # run: | - # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - # - name: Install torchcodec from the wheel - # run: | - # wheel_path=`find dist -type f -name "*.whl"` - # echo Installing $wheel_path - # python -m pip install $wheel_path -vvv + install-and-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.12'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + if: ${{ always() }} + needs: build + steps: + - uses: actions/download-artifact@v4 + with: + name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} + path: dist/ + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install PyTorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv - # - name: Check out repo - # uses: actions/checkout@v3 - # - name: Install ffmpeg, post build - # run: | - # # Ideally we would have checked for that before installing the wheel, - # # but we need to checkout the repo to access this file, and we don't - # # want to checkout the repo before installing the wheel to avoid any - # # side-effect. It's OK. - # .github/scripts/assert_ffmpeg_not_installed.sh + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + .github/scripts/assert_ffmpeg_not_installed.sh - # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - # ffmpeg -version + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version - # - name: Install test dependencies - # run: | - # python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # # Ideally we would find a way to get those dependencies from pyproject.toml - # python -m pip install numpy pytest pillow + - name: Install test dependencies + run: | + python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow - # - name: Delete the src/ folder just for fun - # run: | - # # The only reason we checked-out the repo is to get access to the - # # tests. We don't care about the rest. Out of precaution, we delete - # # the src/ folder to be extra sure that we're running the code from - # # the installed wheel rather than from the source. - # # This is just to be extra cautious and very overkill because a) - # # there's no way the `torchcodec` package from src/ can be found from - # # the PythonPath: the main point of `src/` is precisely to protect - # # against that and b) if we ever were to execute code from - # # `src/torchcodec`, it would fail loudly because the built .so files - # # aren't present there. - # rm -r src/ - # ls - # - name: Smoke test - # run: | - # python test/decoders/manual_smoke_test.py - # - name: Run Python tests - # run: | - # pytest test + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Smoke test + run: | + python test/decoders/manual_smoke_test.py + - name: Run Python tests + run: | + pytest test diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index a8aa6c462..f256b50ce 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,13 +1,29 @@ #!/bin/bash -echo "LS" -ls -echo "LS dist" -ls dist +source .github/scripts/helpers.sh +wheel_path=$(pwd)/$(find dist -type f -name "*.whl") +echo "Wheel content:" +unzip -l $wheel_path -echo "MVVVV" -mv dist/*linux_x86_64*.whl $(echo dist/*linux_x86_64*.whl | sed 's/linux_x86_64/manylinux_2_17_x86_64.manylinux2014_x86_64/') +for ffmpeg_major_version in 4 5 6 7; do + assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so +done +assert_not_in_wheel $wheel_path libtorchcodec.so + +for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do + assert_not_in_wheel $wheel_path $ffmpeg_so +done + +assert_not_in_wheel $wheel_path "^test" +assert_not_in_wheel $wheel_path "^doc" +assert_not_in_wheel $wheel_path "^benchmarks" +assert_not_in_wheel $wheel_path "^packaging" -echo "LS dist" -ls dist +# See invoked python script below for details about this check. +extracted_wheel_dir=$(mktemp -d) +unzip -q $wheel_path -d $extracted_wheel_dir +symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) +python .github/scripts/check_glibcxx.py "$symbols_matches" + +mv dist/*linux_x86_64*.whl $(echo dist/*linux_x86_64*.whl | sed 's/linux_x86_64/manylinux_2_17_x86_64.manylinux2014_x86_64/') From 7dbc1ecdd6edaff105b29c2c7d005e334be76875 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 13:28:22 +0100 Subject: [PATCH 65/67] cleanup --- .github/workflows/cpp_tests.yaml | 128 +++++++++++++++---------------- .github/workflows/docs.yaml | 90 +++++++++++----------- .github/workflows/wheel2.yaml | 4 +- packaging/post_build_script.sh | 11 ++- 4 files changed, 120 insertions(+), 113 deletions(-) diff --git a/.github/workflows/cpp_tests.yaml b/.github/workflows/cpp_tests.yaml index a03ed90bf..2a948b0df 100644 --- a/.github/workflows/cpp_tests.yaml +++ b/.github/workflows/cpp_tests.yaml @@ -1,69 +1,69 @@ -# name: CPP tests +name: CPP tests -# on: -# push: -# branches: [ main ] -# pull_request: +on: + push: + branches: [ main ] + pull_request: -# concurrency: -# group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} -# cancel-in-progress: true +concurrency: + group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} + cancel-in-progress: true -# defaults: -# run: -# shell: bash -l -eo pipefail {0} +defaults: + run: + shell: bash -l -eo pipefail {0} -# jobs: -# Cpp-tests: -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] -# steps: -# - name: Check out repo -# uses: actions/checkout@v3 -# - name: Setup conda env -# uses: conda-incubator/setup-miniconda@v2 -# with: -# auto-update-conda: true -# miniconda-version: "latest" -# activate-environment: test -# python-version: '3.12' -# - name: Update pip -# run: python -m pip install --upgrade pip -# - name: Install dependencies -# run: | -# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -# - name: Install ffmpeg and pkg-config -# run: | -# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config -c conda-forge -# ffmpeg -version -# - name: Build and run C++ tests -# run: | -# # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so -# # we're building libtorchcodec against the installed FFmpeg version -# # (from conda-forge) instead of building against our pre-built non-GPL -# # FFmpeg libraries. -# # The reason we need this is because the C++ tests decode x264 files. -# # x264 support is not LGPL, os it is not supported by our -# # pre-built non-GPL FFmpeg libraries. And if we were to build against -# # those, this is also what the tests would be loading at run time, -# # then failing when we try to decode x264. -# # To remediate that, we build against the FFmpeg that we installed -# # from conda-forge (which is able to decode x264), and that's also -# # what gets loaded at run time. -# # The Python tests are also decoding x264 files, and are built against -# # our non-GPL FFmpeg. And yet they pass. This is because in Python -# # we're able to distinguish between build-time (non-GPL FFmpeg) and -# # run time (conda-forge FFmpeg). +jobs: + Cpp-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + steps: + - name: Check out repo + uses: actions/checkout@v3 + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: '3.12' + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install dependencies + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install ffmpeg and pkg-config + run: | + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" pkg-config -c conda-forge + ffmpeg -version + - name: Build and run C++ tests + run: | + # Note: we're not setting BUILD_AGAINST_ALL_FFMPEG_FROM_S3 here, so + # we're building libtorchcodec against the installed FFmpeg version + # (from conda-forge) instead of building against our pre-built non-GPL + # FFmpeg libraries. + # The reason we need this is because the C++ tests decode x264 files. + # x264 support is not LGPL, os it is not supported by our + # pre-built non-GPL FFmpeg libraries. And if we were to build against + # those, this is also what the tests would be loading at run time, + # then failing when we try to decode x264. + # To remediate that, we build against the FFmpeg that we installed + # from conda-forge (which is able to decode x264), and that's also + # what gets loaded at run time. + # The Python tests are also decoding x264 files, and are built against + # our non-GPL FFmpeg. And yet they pass. This is because in Python + # we're able to distinguish between build-time (non-GPL FFmpeg) and + # run time (conda-forge FFmpeg). -# build_tests_dir="${PWD}/build_tests" -# mkdir $build_tests_dir -# pushd $build_tests_dir -# TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") -# Torch_DIR="${TORCH_PATH}/share/cmake/Torch" -# cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON -# cmake --build . -# ctest -# popd + build_tests_dir="${PWD}/build_tests" + mkdir $build_tests_dir + pushd $build_tests_dir + TORCH_PATH=$(python -c "import pathlib, torch; print(pathlib.Path(torch.__path__[0]))") + Torch_DIR="${TORCH_PATH}/share/cmake/Torch" + cmake .. -DTorch_DIR=$Torch_DIR -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON + cmake --build . + ctest + popd diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c93741e2b..8ed149a87 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,48 +1,48 @@ -# name: Docs +name: Docs -# on: -# push: -# branches: [ main ] -# pull_request: +on: + push: + branches: [ main ] + pull_request: -# defaults: -# run: -# shell: bash -l -eo pipefail {0} +defaults: + run: + shell: bash -l -eo pipefail {0} -# jobs: -# build: -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# steps: -# - name: Check out repo -# uses: actions/checkout@v3 -# - name: Setup conda env -# uses: conda-incubator/setup-miniconda@v2 -# with: -# auto-update-conda: true -# miniconda-version: "latest" -# activate-environment: test -# python-version: '3.12' -# - name: Update pip -# run: python -m pip install --upgrade pip -# - name: Install dependencies and FFmpeg -# run: | -# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -# conda install "ffmpeg=7.0.1" pkg-config -c conda-forge -# ffmpeg -version -# - name: Build and install torchcodec -# run: | -# python -m pip install -e ".[dev]" --no-build-isolation -vvv -# - name: Install doc dependencies -# run: | -# cd docs -# python -m pip install -r requirements.txt -# - name: Build docs -# run: | -# cd docs -# make html -# - uses: actions/upload-artifact@v3 -# with: -# name: Built-Docs -# path: docs/build/html/ +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Check out repo + uses: actions/checkout@v3 + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: '3.12' + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install dependencies and FFmpeg + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + conda install "ffmpeg=7.0.1" pkg-config -c conda-forge + ffmpeg -version + - name: Build and install torchcodec + run: | + python -m pip install -e ".[dev]" --no-build-isolation -vvv + - name: Install doc dependencies + run: | + cd docs + python -m pip install -r requirements.txt + - name: Build docs + run: | + cd docs + make html + - uses: actions/upload-artifact@v3 + with: + name: Built-Docs + path: docs/build/html/ diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml index 9ee6ab114..e65fc9100 100644 --- a/.github/workflows/wheel2.yaml +++ b/.github/workflows/wheel2.yaml @@ -19,12 +19,10 @@ permissions: id-token: write contents: write - defaults: run: shell: bash -l -eo pipefail {0} - jobs: generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main @@ -54,7 +52,7 @@ jobs: package-name: torchcodec trigger-event: ${{ github.event_name }} build-platform: "python-build-package" - wheel-build-extra-args: "--no-isolation" + wheel-build-extra-args: "-vvv --no-isolation" install-and-test: runs-on: ubuntu-latest diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index f256b50ce..688d88771 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -26,4 +26,13 @@ unzip -q $wheel_path -d $extracted_wheel_dir symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) python .github/scripts/check_glibcxx.py "$symbols_matches" -mv dist/*linux_x86_64*.whl $(echo dist/*linux_x86_64*.whl | sed 's/linux_x86_64/manylinux_2_17_x86_64.manylinux2014_x86_64/') +echo "ls dist" +ls dist + +old="linux_x86_64" +new="manylinux_2_17_x86_64.manylinux2014_x86_64" +echo "Replacing ${old} with ${new} in wheel name" +mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/") + +echo "ls dist" +ls dist From 55044915b12652910eca971af87b5dd2ed0fd989 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 13:29:48 +0100 Subject: [PATCH 66/67] rename file --- .github/workflows/wheel.yaml | 128 +++++++++++++-------------------- .github/workflows/wheel2.yaml | 130 ---------------------------------- 2 files changed, 49 insertions(+), 209 deletions(-) delete mode 100644 .github/workflows/wheel2.yaml diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index 49beaa23c..e65fc9100 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -1,106 +1,73 @@ -name: Build wheel +name: Build and test Linux wheels on: - push: - branches: [ main ] pull_request: + push: + branches: + - nightly + - main + - release/* + tags: + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + workflow_dispatch: concurrency: - group: unit-test${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} cancel-in-progress: true +permissions: + id-token: write + contents: write + defaults: run: shell: bash -l -eo pipefail {0} jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-xpu: disable + with-rocm: disable + with-cuda: disable build: - runs-on: ubuntu-20.04 + needs: generate-matrix strategy: fail-fast: false - matrix: - python-version: ['3.8', '3.12'] - steps: - - name: Check out repo - uses: actions/checkout@v3 - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: | - echo $CONDA_PREFIX - python --version - python -m pip install --upgrade pip - - name: Install Pytorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Check GCC version - run: | - # We want to build wheels on gcc 9 to be consistent with the torch - # binaries, which is why we're using ubuntu-20.04 and not - # ubuntu-latest. More details below about the GLIBCXX checks. - if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" - then - echo "Wrong GCC version, exiting!" - gcc --version - exit 1 - fi - - name: Build the torchcodec wheel - run: | - # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. - .github/scripts/assert_ffmpeg_not_installed.sh - - python -m pip install build - BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation - - name: Validate wheel content - run: | - source .github/scripts/helpers.sh - wheel_path=$(pwd)/$(find dist -type f -name "*.whl") - echo "Wheel content:" - unzip -l $wheel_path - - for ffmpeg_major_version in 4 5 6 7; do - assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so - done - assert_not_in_wheel $wheel_path libtorchcodec.so - - for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do - assert_not_in_wheel $wheel_path $ffmpeg_so - done - - assert_not_in_wheel $wheel_path "^test" - assert_not_in_wheel $wheel_path "^doc" - assert_not_in_wheel $wheel_path "^benchmarks" - assert_not_in_wheel $wheel_path "^packaging" - - # See invoked python script below for details about this check. - extracted_wheel_dir=$(mktemp -d) - unzip -q $wheel_path -d $extracted_wheel_dir - symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) - python .github/scripts/check_glibcxx.py "$symbols_matches" - - uses: actions/upload-artifact@v4 - with: - name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - path: dist/* + name: Build and Upload wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + post-script: packaging/post_build_script.sh + env-var-script: packaging/env_var_script.sh + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + wheel-build-extra-args: "-vvv --no-isolation" install-and-test: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] if: ${{ always() }} needs: build steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v3 with: - name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - path: dist/ + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 + path: pytorch/torchcodec/dist/ - name: Setup conda env uses: conda-incubator/setup-miniconda@v2 with: @@ -109,13 +76,16 @@ jobs: activate-environment: test python-version: ${{ matrix.python-version }} - name: Update pip - run: python -m pip install --upgrade pip + run: | + echo $CONDA_PREFIX + python --version + python -m pip install --upgrade pip - name: Install PyTorch run: | python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - name: Install torchcodec from the wheel run: | - wheel_path=`find dist -type f -name "*.whl"` + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` echo Installing $wheel_path python -m pip install $wheel_path -vvv diff --git a/.github/workflows/wheel2.yaml b/.github/workflows/wheel2.yaml deleted file mode 100644 index e65fc9100..000000000 --- a/.github/workflows/wheel2.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: Build and test Linux wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true - -permissions: - id-token: write - contents: write - -defaults: - run: - shell: bash -l -eo pipefail {0} - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - with-xpu: disable - with-rocm: disable - with-cuda: disable - build: - needs: generate-matrix - strategy: - fail-fast: false - name: Build and Upload wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: pytorch/torchcodec - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - post-script: packaging/post_build_script.sh - env-var-script: packaging/env_var_script.sh - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec - trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - wheel-build-extra-args: "-vvv --no-isolation" - - install-and-test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ['3.9'] - ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - if: ${{ always() }} - needs: build - steps: - - uses: actions/download-artifact@v3 - with: - name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x86_64 - path: pytorch/torchcodec/dist/ - - name: Setup conda env - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - miniconda-version: "latest" - activate-environment: test - python-version: ${{ matrix.python-version }} - - name: Update pip - run: | - echo $CONDA_PREFIX - python --version - python -m pip install --upgrade pip - - name: Install PyTorch - run: | - python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Install torchcodec from the wheel - run: | - wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` - echo Installing $wheel_path - python -m pip install $wheel_path -vvv - - - name: Check out repo - uses: actions/checkout@v3 - - name: Install ffmpeg, post build - run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - .github/scripts/assert_ffmpeg_not_installed.sh - - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version - - - name: Install test dependencies - run: | - python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # Ideally we would find a way to get those dependencies from pyproject.toml - python -m pip install numpy pytest pillow - - - name: Delete the src/ folder just for fun - run: | - # The only reason we checked-out the repo is to get access to the - # tests. We don't care about the rest. Out of precaution, we delete - # the src/ folder to be extra sure that we're running the code from - # the installed wheel rather than from the source. - # This is just to be extra cautious and very overkill because a) - # there's no way the `torchcodec` package from src/ can be found from - # the PythonPath: the main point of `src/` is precisely to protect - # against that and b) if we ever were to execute code from - # `src/torchcodec`, it would fail loudly because the built .so files - # aren't present there. - rm -r src/ - ls - - name: Smoke test - run: | - python test/decoders/manual_smoke_test.py - - name: Run Python tests - run: | - pytest test From dfcd305b93fb2b696d471101795fae4b50c36dc9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Wed, 7 Aug 2024 13:39:34 +0100 Subject: [PATCH 67/67] last cleanup --- .github/workflows/wheel.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index e65fc9100..69dc66a83 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -76,10 +76,7 @@ jobs: activate-environment: test python-version: ${{ matrix.python-version }} - name: Update pip - run: | - echo $CONDA_PREFIX - python --version - python -m pip install --upgrade pip + run: python -m pip install --upgrade pip - name: Install PyTorch run: | python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu