[do not merge] Test/adapt huggingface_hub v1.0.0

.github/workflows/benchmark.yml

@@ -40,6 +40,8 @@ jobs:
           apt install -y libpq-dev postgresql-client
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
           python -m uv pip install -e [quality,test]
+          python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+          python -m uv pip install compel==0.1.8
           python -m uv pip install -r benchmarks/requirements.txt
       - name: Environment
         run: |

.github/workflows/mirror_community_pipeline.yml

@@ -76,6 +76,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install --upgrade huggingface_hub
+          pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
 
       # Check secret is set
       - name: whoami

.github/workflows/nightly_tests.yml

@@ -75,6 +75,7 @@ jobs:
           python -m uv pip install -e [quality,test]
           pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           python -m uv pip install pytest-reportlog
+          python -m uv pip install compel==0.1.8
       - name: Environment
         run: |
           python utils/print_env.py
@@ -129,6 +130,7 @@ jobs:
         python -m uv pip install peft@git+https://github.com/huggingface/peft.git
         pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
         python -m uv pip install pytest-reportlog
+        python -m uv pip install compel==0.1.8
     - name: Environment
       run: python utils/print_env.py
 
@@ -237,6 +239,7 @@ jobs:
           python -m uv pip install peft@git+https://github.com/huggingface/peft.git
           pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           python -m uv pip install pytest-reportlog
+          python -m uv pip install compel==0.1.8
       - name: Environment
         run: |
           python utils/print_env.py
@@ -286,6 +289,7 @@ jobs:
           python -m uv pip install -e [quality,test]
           python -m uv pip install peft@git+https://github.com/huggingface/peft.git
           pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
+          python -m uv pip install compel==0.1.8
 
       - name: Environment
         run: |

.github/workflows/pr_dependency_test.yml

@@ -22,13 +22,15 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
           python -m pip install --upgrade pip uv
-          python -m uv pip install -e .
+          python -m uv pip install --prerelease=allow -e .
+          python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
           python -m uv pip install pytest
+          python -m uv pip install compel==0.1.8
       - name: Check for soft dependencies
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"

.github/workflows/pr_modular_tests.yml

@@ -44,6 +44,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install .[quality]
+          pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
       - name: Check quality
         run: make quality
       - name: Check if failure
@@ -64,6 +65,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install .[quality]
+          pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
       - name: Check repo consistency
         run: |
           python utils/check_copies.py
@@ -109,9 +111,10 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test]
+        python -m uv pip install --prerelease=allow -e [quality,test]
         pip uninstall transformers -y && python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
         pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |

.github/workflows/pr_test_fetcher.yml

@@ -34,7 +34,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test]
+        python -m uv pip install --prerelease=allow -e [quality,test]
+        python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+        python -m uv pip install compel==0.1.8
     - name: Environment
       run: |
         python utils/print_env.py
@@ -92,7 +94,9 @@ jobs:
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
         python -m pip install -e [quality,test]
+        python -m pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
         python -m pip install accelerate
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |
@@ -149,7 +153,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
+        python -m pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
         python -m pip install -e [quality,test]
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |

.github/workflows/pr_tests.yml

@@ -35,11 +35,12 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install .[quality]
+          pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
       - name: Check quality
         run: make quality
       - name: Check if failure
@@ -55,11 +56,12 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install .[quality]
+          pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
       - name: Check repo consistency
         run: |
           python utils/check_copies.py
@@ -115,9 +117,10 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test]
+        python -m uv pip install --prerelease=allow -e [quality,test]
         pip uninstall transformers -y && python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
         pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |
@@ -196,7 +199,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test]
+        python -m uv pip install --prerelease=allow -e [quality,test]
+        python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |
@@ -250,12 +255,13 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test]
+        python -m uv pip install --prerelease=allow -e [quality,test]
         # TODO (sayakpaul, DN6): revisit `--no-deps`
         python -m pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps
         python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
         python -m uv pip install -U tokenizers
         pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |

.github/workflows/pr_tests_gpu.yml

@@ -36,11 +36,12 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install .[quality]
+          pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
       - name: Check quality
         run: make quality
       - name: Check if failure
@@ -56,11 +57,12 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install .[quality]
+          pip install git+https://github.com/huggingface/transformers.git # temporary (only for tests)
       - name: Check repo consistency
         run: |
           python utils/check_copies.py
@@ -89,7 +91,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-          python -m uv pip install -e [quality,test]
+          python -m uv pip install --prerelease=allow -e [quality,test]
+          python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+          python -m uv pip install compel==0.1.8
       - name: Environment
         run: |
           python utils/print_env.py
@@ -131,9 +135,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-          python -m uv pip install -e [quality,test]
+          python -m uv pip install --prerelease=allow -e [quality,test]
           pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
           pip uninstall transformers -y && python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
+          python -m uv pip install compel==0.1.8
 
       - name: Environment
         run: |
@@ -201,10 +206,11 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test]
+        python -m uv pip install --prerelease=allow -e [quality,test]
         python -m uv pip install peft@git+https://github.com/huggingface/peft.git
         pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
         pip uninstall transformers -y && python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |
@@ -267,7 +273,8 @@ jobs:
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
         pip uninstall transformers -y && python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
-        python -m uv pip install -e [quality,test,training]
+        python -m uv pip install --prerelease=allow -e [quality,test,training]
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |

.github/workflows/pr_torch_dependency_test.yml

@@ -22,14 +22,16 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
           python -m pip install --upgrade pip uv
-          python -m uv pip install -e .
+          python -m uv pip install --prerelease=allow -e .
           python -m uv pip install torch torchvision torchaudio
           python -m uv pip install pytest
+          python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+          python -m uv pip install compel==0.1.8
       - name: Check for soft dependencies
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"

.github/workflows/push_tests.yml

@@ -35,7 +35,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-          python -m uv pip install -e [quality,test]
+          python -m uv pip install --prerelease=allow -e [quality,test]
+          python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+          python -m uv pip install compel==0.1.8
       - name: Environment
         run: |
           python utils/print_env.py
@@ -76,8 +78,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-          python -m uv pip install -e [quality,test]
+          python -m uv pip install --prerelease=allow -e [quality,test]
+          python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
           pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
+          python -m uv pip install compel==0.1.8
       - name: Environment
         run: |
           python utils/print_env.py
@@ -127,9 +131,10 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test]
+        python -m uv pip install --prerelease=allow -e [quality,test]
         python -m uv pip install peft@git+https://github.com/huggingface/peft.git
         pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |
@@ -181,7 +186,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test,training]
+        python -m uv pip install --prerelease=allow -e [quality,test,training]
+        python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+        python -m uv pip install compel==0.1.8
     - name: Environment
       run: |
         python utils/print_env.py
@@ -224,7 +231,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test,training]
+        python -m uv pip install --prerelease=allow -e [quality,test,training]
+        python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+        python -m uv pip install compel==0.1.8
     - name: Environment
       run: |
         python utils/print_env.py
@@ -265,7 +274,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test,training]
+        python -m uv pip install --prerelease=allow -e [quality,test,training]
+        python -m uv pip install --prerelease=allow "transformers @ git+https://github.com/huggingface/transformers.git" # temporary (only for tests)
+        python -m uv pip install compel==0.1.8
 
     - name: Environment
       run: |

.github/workflows/push_tests_fast.yml

@@ -61,7 +61,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m uv pip install -e [quality,test]
+        python -m uv pip install --prerelease=allow -e [quality,test]
 
     - name: Environment
       run: |

.github/workflows/push_tests_mps.yml

@@ -41,10 +41,11 @@ jobs:
       shell: arch -arch arm64 bash {0}
       run: |
         ${CONDA_RUN} python -m pip install --upgrade pip uv
-        ${CONDA_RUN} python -m uv pip install -e ".[quality,test]"
+        ${CONDA_RUN} python -m uv pip install --prerelease=allow -e ".[quality,test]"
         ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio
         ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git
         ${CONDA_RUN} python -m uv pip install transformers --upgrade
+        ${CONDA_RUN} python -m uv pip install compel==0.1.8
 
     - name: Environment
       shell: arch -arch arm64 bash {0}