From db7b0ecc12445cba5bb5d6b632659001009a9fac Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 20 Sep 2024 07:49:40 +0530 Subject: [PATCH 1/6] allow faster downloads from the Hub in CI. --- docker/diffusers-flax-cpu/Dockerfile | 5 ++++- docker/diffusers-flax-tpu/Dockerfile | 5 ++++- docker/diffusers-onnxruntime-cpu/Dockerfile | 5 ++++- docker/diffusers-onnxruntime-cuda/Dockerfile | 5 ++++- docker/diffusers-pytorch-compile-cuda/Dockerfile | 5 ++++- docker/diffusers-pytorch-cpu/Dockerfile | 5 ++++- docker/diffusers-pytorch-cuda/Dockerfile | 5 ++++- docker/diffusers-pytorch-xformers-cuda/Dockerfile | 5 ++++- 8 files changed, 32 insertions(+), 8 deletions(-) diff --git a/docker/diffusers-flax-cpu/Dockerfile b/docker/diffusers-flax-cpu/Dockerfile index 86a49171d290..93ed4b757510 100644 --- a/docker/diffusers-flax-cpu/Dockerfile +++ b/docker/diffusers-flax-cpu/Dockerfile @@ -43,6 +43,9 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ numpy==1.26.4 \ scipy \ tensorboard \ - transformers + transformers \ + hf_transfer + +ENV HF_HUB_ENABLE_HF_TRANSFER=1 CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/diffusers-flax-tpu/Dockerfile b/docker/diffusers-flax-tpu/Dockerfile index b40cd55a1c16..8a0c5503c4d1 100644 --- a/docker/diffusers-flax-tpu/Dockerfile +++ b/docker/diffusers-flax-tpu/Dockerfile @@ -45,6 +45,9 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ numpy==1.26.4 \ scipy \ tensorboard \ - transformers + transformers \ + hf_transfer + +ENV HF_HUB_ENABLE_HF_TRANSFER=1 CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/diffusers-onnxruntime-cpu/Dockerfile b/docker/diffusers-onnxruntime-cpu/Dockerfile index a5a6e98605cb..e6f5b9b755b2 100644 --- a/docker/diffusers-onnxruntime-cpu/Dockerfile +++ b/docker/diffusers-onnxruntime-cpu/Dockerfile @@ -43,6 +43,9 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ numpy==1.26.4 \ scipy \ tensorboard \ - transformers + transformers \ + hf_transfer + +ENV HF_HUB_ENABLE_HF_TRANSFER=1 CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/diffusers-onnxruntime-cuda/Dockerfile b/docker/diffusers-onnxruntime-cuda/Dockerfile index 3364698fe945..8c7f4346723e 100644 --- a/docker/diffusers-onnxruntime-cuda/Dockerfile +++ b/docker/diffusers-onnxruntime-cuda/Dockerfile @@ -44,6 +44,9 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ numpy==1.26.4 \ scipy \ tensorboard \ - transformers + transformers \ + hf_transfer + +ENV HF_HUB_ENABLE_HF_TRANSFER=1 CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/diffusers-pytorch-compile-cuda/Dockerfile b/docker/diffusers-pytorch-compile-cuda/Dockerfile index a5454328b851..d6fa8e6f87c7 100644 --- a/docker/diffusers-pytorch-compile-cuda/Dockerfile +++ b/docker/diffusers-pytorch-compile-cuda/Dockerfile @@ -44,6 +44,9 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ numpy==1.26.4 \ scipy \ tensorboard \ - transformers + transformers \ + hf_transfer + +ENV HF_HUB_ENABLE_HF_TRANSFER=1 CMD ["/bin/bash"] diff --git a/docker/diffusers-pytorch-cpu/Dockerfile b/docker/diffusers-pytorch-cpu/Dockerfile index 910765bb0b9c..7975b1a962b3 100644 --- a/docker/diffusers-pytorch-cpu/Dockerfile +++ b/docker/diffusers-pytorch-cpu/Dockerfile @@ -44,6 +44,9 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ numpy==1.26.4 \ scipy \ tensorboard \ - transformers matplotlib + transformers matplotlib \ + hf_transfer + +ENV HF_HUB_ENABLE_HF_TRANSFER=1 CMD ["/bin/bash"] diff --git a/docker/diffusers-pytorch-cuda/Dockerfile b/docker/diffusers-pytorch-cuda/Dockerfile index 8b5439ffb6c6..4fc1a4184376 100644 --- a/docker/diffusers-pytorch-cuda/Dockerfile +++ b/docker/diffusers-pytorch-cuda/Dockerfile @@ -45,6 +45,9 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ scipy \ tensorboard \ transformers \ - pytorch-lightning + pytorch-lightning \ + hf_transfer + +ENV HF_HUB_ENABLE_HF_TRANSFER=1 CMD ["/bin/bash"] diff --git a/docker/diffusers-pytorch-xformers-cuda/Dockerfile b/docker/diffusers-pytorch-xformers-cuda/Dockerfile index 7a3408c48624..e54a5b7a915b 100644 --- a/docker/diffusers-pytorch-xformers-cuda/Dockerfile +++ b/docker/diffusers-pytorch-xformers-cuda/Dockerfile @@ -45,6 +45,9 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ scipy \ tensorboard \ transformers \ - xformers + xformers \ + hf_transfer + +ENV HF_HUB_ENABLE_HF_TRANSFER=1 CMD ["/bin/bash"] From 22aa170a71b73c37ce95d5c6216ff9a7aa1bdc30 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 23 Sep 2024 20:21:53 +0530 Subject: [PATCH 2/6] HF_HUB_ENABLE_HF_TRANSFER: 1 --- .github/workflows/benchmark.yml | 1 + .github/workflows/pr_tests.yml | 1 + .github/workflows/push_tests.yml | 1 + .github/workflows/push_tests_fast.yml | 1 + .github/workflows/push_tests_mps.yml | 1 + 5 files changed, 5 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2e7e82f056ed..d311c1c73f11 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -7,6 +7,7 @@ on: env: DIFFUSERS_IS_CI: yes + HF_HUB_ENABLE_HF_TRANSFER: 1 HF_HOME: /mnt/cache OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index d40270ab46fd..6bb67976b170 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -22,6 +22,7 @@ concurrency: env: DIFFUSERS_IS_CI: yes + HF_HUB_ENABLE_HF_TRANSFER: 1 OMP_NUM_THREADS: 4 MKL_NUM_THREADS: 4 PYTEST_TIMEOUT: 60 diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index b8214da328ff..f07e6cda0d59 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -14,6 +14,7 @@ env: DIFFUSERS_IS_CI: yes OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 + HF_HUB_ENABLE_HF_TRANSFER: 1 PYTEST_TIMEOUT: 600 PIPELINE_USAGE_CUTOFF: 50000 diff --git a/.github/workflows/push_tests_fast.yml b/.github/workflows/push_tests_fast.yml index 4e3a01fdd97f..e8a73446de73 100644 --- a/.github/workflows/push_tests_fast.yml +++ b/.github/workflows/push_tests_fast.yml @@ -18,6 +18,7 @@ env: HF_HOME: /mnt/cache OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 + HF_HUB_ENABLE_HF_TRANSFER: 1 PYTEST_TIMEOUT: 600 RUN_SLOW: no diff --git a/.github/workflows/push_tests_mps.yml b/.github/workflows/push_tests_mps.yml index f261b6c00d1c..8d521074a08f 100644 --- a/.github/workflows/push_tests_mps.yml +++ b/.github/workflows/push_tests_mps.yml @@ -13,6 +13,7 @@ env: HF_HOME: /mnt/cache OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 + HF_HUB_ENABLE_HF_TRANSFER: 1 PYTEST_TIMEOUT: 600 RUN_SLOW: no From 0dfe5b8d4980ac4a7f3abfe063ec630c5ff74b67 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 23 Sep 2024 20:30:47 +0530 Subject: [PATCH 3/6] empty From 7bf6f1530db20412040574b237cca9f3dff77c9d Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 23 Sep 2024 21:18:08 +0530 Subject: [PATCH 4/6] empty From 4cdafa207dfa44ea6e8615fde7ca00e97da09565 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 23 Sep 2024 22:28:52 +0530 Subject: [PATCH 5/6] remove ENV HF_HUB_ENABLE_HF_TRANSFER=1. --- docker/diffusers-flax-cpu/Dockerfile | 2 -- docker/diffusers-flax-tpu/Dockerfile | 2 -- docker/diffusers-onnxruntime-cpu/Dockerfile | 2 -- docker/diffusers-onnxruntime-cuda/Dockerfile | 2 -- docker/diffusers-pytorch-compile-cuda/Dockerfile | 2 -- docker/diffusers-pytorch-cpu/Dockerfile | 2 -- docker/diffusers-pytorch-cuda/Dockerfile | 2 -- docker/diffusers-pytorch-xformers-cuda/Dockerfile | 2 -- 8 files changed, 16 deletions(-) diff --git a/docker/diffusers-flax-cpu/Dockerfile b/docker/diffusers-flax-cpu/Dockerfile index 93ed4b757510..051008aa9a2e 100644 --- a/docker/diffusers-flax-cpu/Dockerfile +++ b/docker/diffusers-flax-cpu/Dockerfile @@ -46,6 +46,4 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ transformers \ hf_transfer -ENV HF_HUB_ENABLE_HF_TRANSFER=1 - CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/diffusers-flax-tpu/Dockerfile b/docker/diffusers-flax-tpu/Dockerfile index 8a0c5503c4d1..405f068923b7 100644 --- a/docker/diffusers-flax-tpu/Dockerfile +++ b/docker/diffusers-flax-tpu/Dockerfile @@ -48,6 +48,4 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ transformers \ hf_transfer -ENV HF_HUB_ENABLE_HF_TRANSFER=1 - CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/diffusers-onnxruntime-cpu/Dockerfile b/docker/diffusers-onnxruntime-cpu/Dockerfile index e6f5b9b755b2..6f4b13e8a9ba 100644 --- a/docker/diffusers-onnxruntime-cpu/Dockerfile +++ b/docker/diffusers-onnxruntime-cpu/Dockerfile @@ -46,6 +46,4 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ transformers \ hf_transfer -ENV HF_HUB_ENABLE_HF_TRANSFER=1 - CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/diffusers-onnxruntime-cuda/Dockerfile b/docker/diffusers-onnxruntime-cuda/Dockerfile index 8c7f4346723e..bd1d871033c9 100644 --- a/docker/diffusers-onnxruntime-cuda/Dockerfile +++ b/docker/diffusers-onnxruntime-cuda/Dockerfile @@ -47,6 +47,4 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ transformers \ hf_transfer -ENV HF_HUB_ENABLE_HF_TRANSFER=1 - CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/diffusers-pytorch-compile-cuda/Dockerfile b/docker/diffusers-pytorch-compile-cuda/Dockerfile index d6fa8e6f87c7..cb4a9c0f9896 100644 --- a/docker/diffusers-pytorch-compile-cuda/Dockerfile +++ b/docker/diffusers-pytorch-compile-cuda/Dockerfile @@ -47,6 +47,4 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ transformers \ hf_transfer -ENV HF_HUB_ENABLE_HF_TRANSFER=1 - CMD ["/bin/bash"] diff --git a/docker/diffusers-pytorch-cpu/Dockerfile b/docker/diffusers-pytorch-cpu/Dockerfile index 7975b1a962b3..8d98c52598d2 100644 --- a/docker/diffusers-pytorch-cpu/Dockerfile +++ b/docker/diffusers-pytorch-cpu/Dockerfile @@ -47,6 +47,4 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ transformers matplotlib \ hf_transfer -ENV HF_HUB_ENABLE_HF_TRANSFER=1 - CMD ["/bin/bash"] diff --git a/docker/diffusers-pytorch-cuda/Dockerfile b/docker/diffusers-pytorch-cuda/Dockerfile index 4fc1a4184376..695f5ed08dc5 100644 --- a/docker/diffusers-pytorch-cuda/Dockerfile +++ b/docker/diffusers-pytorch-cuda/Dockerfile @@ -48,6 +48,4 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ pytorch-lightning \ hf_transfer -ENV HF_HUB_ENABLE_HF_TRANSFER=1 - CMD ["/bin/bash"] diff --git a/docker/diffusers-pytorch-xformers-cuda/Dockerfile b/docker/diffusers-pytorch-xformers-cuda/Dockerfile index e54a5b7a915b..1693eb293024 100644 --- a/docker/diffusers-pytorch-xformers-cuda/Dockerfile +++ b/docker/diffusers-pytorch-xformers-cuda/Dockerfile @@ -48,6 +48,4 @@ RUN python3.10 -m pip install --no-cache-dir --upgrade pip uv==0.1.11 && \ xformers \ hf_transfer -ENV HF_HUB_ENABLE_HF_TRANSFER=1 - CMD ["/bin/bash"] From 9db1f8135a890034b28081a6864fc61f5796666f Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 24 Sep 2024 09:27:05 +0530 Subject: [PATCH 6/6] empty