diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cc97e043c139..939cce9ffb00 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -38,9 +38,8 @@ jobs: run: | apt update apt install -y libpq-dev postgresql-client - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install -r benchmarks/requirements.txt + uv pip install -e ".[quality]" + uv pip install -r benchmarks/requirements.txt - name: Environment run: | python utils/print_env.py diff --git a/.github/workflows/mirror_community_pipeline.yml b/.github/workflows/mirror_community_pipeline.yml index 9cf573312b34..ab4ded973047 100644 --- a/.github/workflows/mirror_community_pipeline.yml +++ b/.github/workflows/mirror_community_pipeline.yml @@ -74,7 +74,7 @@ jobs: python-version: "3.10" - name: Install dependencies run: | - python -m pip install --upgrade pip + pip install --upgrade pip pip install --upgrade huggingface_hub # Check secret is set diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index 479e5503eed2..c8fa3a7ad93a 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -71,10 +71,9 @@ jobs: run: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - python -m uv pip install pytest-reportlog + uv pip install -e ".[quality]" + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install pytest-reportlog - name: Environment run: | python utils/print_env.py @@ -84,7 +83,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \ --report-log=tests_pipeline_${{ matrix.module }}_cuda.log \ @@ -124,11 +123,10 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install peft@git+https://github.com/huggingface/peft.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - python -m uv pip install pytest-reportlog + uv pip install -e ".[quality]" + uv pip install peft@git+https://github.com/huggingface/peft.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install pytest-reportlog - name: Environment run: python utils/print_env.py @@ -139,7 +137,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_torch_${{ matrix.module }}_cuda \ --report-log=tests_torch_${{ matrix.module }}_cuda.log \ @@ -152,7 +150,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v --make-reports=examples_torch_cuda \ --report-log=examples_torch_cuda.log \ examples/ @@ -191,8 +189,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test,training] + uv pip install -e ".[quality,training]" - name: Environment run: | python utils/print_env.py @@ -201,7 +198,7 @@ jobs: HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} RUN_COMPILE: yes run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ + pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ - name: Failure short reports if: ${{ failure() }} run: cat reports/tests_torch_compile_cuda_failures_short.txt @@ -232,11 +229,10 @@ jobs: run: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install peft@git+https://github.com/huggingface/peft.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - python -m uv pip install pytest-reportlog + uv pip install -e ".[quality]" + uv pip install peft@git+https://github.com/huggingface/peft.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install pytest-reportlog - name: Environment run: | python utils/print_env.py @@ -247,7 +243,7 @@ jobs: CUBLAS_WORKSPACE_CONFIG: :16:8 BIG_GPU_MEMORY: 40 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -m "big_accelerator" \ --make-reports=tests_big_gpu_torch_cuda \ --report-log=tests_big_gpu_torch_cuda.log \ @@ -282,10 +278,9 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install peft@git+https://github.com/huggingface/peft.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install -e ".[quality]" + uv pip install peft@git+https://github.com/huggingface/peft.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | @@ -297,7 +292,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_torch_minimum_version_cuda \ tests/models/test_modeling_common.py \ @@ -357,13 +352,12 @@ jobs: run: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install -U ${{ matrix.config.backend }} + uv pip install -e ".[quality]" + uv pip install -U ${{ matrix.config.backend }} if [ "${{ join(matrix.config.additional_deps, ' ') }}" != "" ]; then - python -m uv pip install ${{ join(matrix.config.additional_deps, ' ') }} + uv pip install ${{ join(matrix.config.additional_deps, ' ') }} fi - python -m uv pip install pytest-reportlog + uv pip install pytest-reportlog - name: Environment run: | python utils/print_env.py @@ -374,7 +368,7 @@ jobs: CUBLAS_WORKSPACE_CONFIG: :16:8 BIG_GPU_MEMORY: 40 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ --make-reports=tests_${{ matrix.config.backend }}_torch_cuda \ --report-log=tests_${{ matrix.config.backend }}_torch_cuda.log \ tests/quantization/${{ matrix.config.test_location }} @@ -409,10 +403,9 @@ jobs: run: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install -U bitsandbytes optimum_quanto - python -m uv pip install pytest-reportlog + uv pip install -e ".[quality]" + uv pip install -U bitsandbytes optimum_quanto + uv pip install pytest-reportlog - name: Environment run: | python utils/print_env.py @@ -423,7 +416,7 @@ jobs: CUBLAS_WORKSPACE_CONFIG: :16:8 BIG_GPU_MEMORY: 40 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ --make-reports=tests_pipeline_level_quant_torch_cuda \ --report-log=tests_pipeline_level_quant_torch_cuda.log \ tests/quantization/test_pipeline_level_quantization.py @@ -523,11 +516,11 @@ jobs: # - name: Install dependencies # shell: arch -arch arm64 bash {0} # run: | -# ${CONDA_RUN} python -m pip install --upgrade pip uv -# ${CONDA_RUN} python -m uv pip install -e [quality,test] -# ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu -# ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate -# ${CONDA_RUN} python -m uv pip install pytest-reportlog +# ${CONDA_RUN} pip install --upgrade pip uv +# ${CONDA_RUN} uv pip install -e ".[quality]" +# ${CONDA_RUN} uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu +# ${CONDA_RUN} uv pip install accelerate@git+https://github.com/huggingface/accelerate +# ${CONDA_RUN} uv pip install pytest-reportlog # - name: Environment # shell: arch -arch arm64 bash {0} # run: | @@ -538,7 +531,7 @@ jobs: # HF_HOME: /System/Volumes/Data/mnt/cache # HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} # run: | -# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \ +# ${CONDA_RUN} pytest -n 1 -s -v --make-reports=tests_torch_mps \ # --report-log=tests_torch_mps.log \ # tests/ # - name: Failure short reports @@ -579,11 +572,11 @@ jobs: # - name: Install dependencies # shell: arch -arch arm64 bash {0} # run: | -# ${CONDA_RUN} python -m pip install --upgrade pip uv -# ${CONDA_RUN} python -m uv pip install -e [quality,test] -# ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu -# ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate -# ${CONDA_RUN} python -m uv pip install pytest-reportlog +# ${CONDA_RUN} pip install --upgrade pip uv +# ${CONDA_RUN} uv pip install -e ".[quality]" +# ${CONDA_RUN} uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu +# ${CONDA_RUN} uv pip install accelerate@git+https://github.com/huggingface/accelerate +# ${CONDA_RUN} uv pip install pytest-reportlog # - name: Environment # shell: arch -arch arm64 bash {0} # run: | @@ -594,7 +587,7 @@ jobs: # HF_HOME: /System/Volumes/Data/mnt/cache # HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} # run: | -# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \ +# ${CONDA_RUN} pytest -n 1 -s -v --make-reports=tests_torch_mps \ # --report-log=tests_torch_mps.log \ # tests/ # - name: Failure short reports diff --git a/.github/workflows/pr_dependency_test.yml b/.github/workflows/pr_dependency_test.yml index d9350c09ac42..b914d1076190 100644 --- a/.github/workflows/pr_dependency_test.yml +++ b/.github/workflows/pr_dependency_test.yml @@ -25,11 +25,8 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pip install --upgrade pip uv - python -m uv pip install -e . - python -m uv pip install pytest + pip install -e . + pip install pytest - name: Check for soft dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - pytest tests/others/test_dependencies.py + pytest tests/others/test_dependencies.py diff --git a/.github/workflows/pr_modular_tests.yml b/.github/workflows/pr_modular_tests.yml index 75258771e4dc..83c84fbab219 100644 --- a/.github/workflows/pr_modular_tests.yml +++ b/.github/workflows/pr_modular_tests.yml @@ -42,7 +42,7 @@ jobs: python-version: "3.10" - name: Install dependencies run: | - python -m pip install --upgrade pip + pip install --upgrade pip pip install .[quality] - name: Check quality run: make quality @@ -62,7 +62,7 @@ jobs: python-version: "3.10" - name: Install dependencies run: | - python -m pip install --upgrade pip + pip install --upgrade pip pip install .[quality] - name: Check repo consistency run: | @@ -108,21 +108,18 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps + uv pip install -e ".[quality]" + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run fast PyTorch Pipeline CPU tests if: ${{ matrix.config.framework == 'pytorch_pipelines' }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile \ + pytest -n 8 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ tests/modular_pipelines diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index b032bb842786..83b2ab4edbf6 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -33,8 +33,7 @@ jobs: fetch-depth: 0 - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] + uv pip install -e ".[quality]" - name: Environment run: | python utils/print_env.py @@ -90,19 +89,16 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pip install -e [quality,test] - python -m pip install accelerate + uv pip install -e ".[quality]" + uv pip install accelerate - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run all selected tests on CPU run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }} + pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }} - name: Failure short reports if: ${{ failure() }} @@ -148,19 +144,16 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pip install -e [quality,test] + pip install -e [quality] - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run Hub tests for models, schedulers, and pipelines on a staging env if: ${{ matrix.config.framework == 'hub_tests_pytorch' }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - HUGGINGFACE_CO_STAGING=true python -m pytest \ + HUGGINGFACE_CO_STAGING=true pytest \ -m "is_staging_test" \ --make-reports=tests_${{ matrix.config.report }} \ tests diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 005e89f4056a..03205fcec6ef 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -38,7 +38,7 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m pip install --upgrade pip + pip install --upgrade pip pip install .[quality] - name: Check quality run: make quality @@ -58,7 +58,7 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m pip install --upgrade pip + pip install --upgrade pip pip install .[quality] - name: Check repo consistency run: | @@ -114,21 +114,18 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps + uv pip install -e ".[quality]" + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run fast PyTorch Pipeline CPU tests if: ${{ matrix.config.framework == 'pytorch_pipelines' }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile \ + pytest -n 8 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ tests/pipelines @@ -136,8 +133,7 @@ jobs: - name: Run fast PyTorch Model Scheduler CPU tests if: ${{ matrix.config.framework == 'pytorch_models' }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \ + pytest -n 4 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx and not Dependency" \ --make-reports=tests_${{ matrix.config.report }} \ tests/models tests/schedulers tests/others @@ -145,9 +141,8 @@ jobs: - name: Run example PyTorch CPU tests if: ${{ matrix.config.framework == 'pytorch_examples' }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install peft timm - python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \ + uv pip install ".[training]" + pytest -n 4 --max-worker-restart=0 --dist=loadfile \ --make-reports=tests_${{ matrix.config.report }} \ examples @@ -195,19 +190,16 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] + uv pip install -e ".[quality]" - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run Hub tests for models, schedulers, and pipelines on a staging env if: ${{ matrix.config.framework == 'hub_tests_pytorch' }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - HUGGINGFACE_CO_STAGING=true python -m pytest \ + HUGGINGFACE_CO_STAGING=true pytest \ -m "is_staging_test" \ --make-reports=tests_${{ matrix.config.report }} \ tests @@ -249,27 +241,24 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] + uv pip install -e ".[quality]" # TODO (sayakpaul, DN6): revisit `--no-deps` - python -m pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps - python -m uv pip install -U tokenizers - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps + uv pip install -U tokenizers + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run fast PyTorch LoRA tests with PEFT run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \ + pytest -n 4 --max-worker-restart=0 --dist=loadfile \ -s -v \ --make-reports=tests_peft_main \ tests/lora/ - python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \ + pytest -n 4 --max-worker-restart=0 --dist=loadfile \ -s -v \ --make-reports=tests_models_lora_peft_main \ tests/models/ -k "lora" @@ -286,3 +275,4 @@ jobs: with: name: pr_main_test_reports path: reports + diff --git a/.github/workflows/pr_tests_gpu.yml b/.github/workflows/pr_tests_gpu.yml index 89b6abe20d1e..900a53da9484 100644 --- a/.github/workflows/pr_tests_gpu.yml +++ b/.github/workflows/pr_tests_gpu.yml @@ -39,7 +39,7 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m pip install --upgrade pip + pip install --upgrade pip pip install .[quality] - name: Check quality run: make quality @@ -59,7 +59,7 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m pip install --upgrade pip + pip install --upgrade pip pip install .[quality] - name: Check repo consistency run: | @@ -88,8 +88,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] + uv pip install -e ".[quality]" - name: Environment run: | python utils/print_env.py @@ -130,10 +129,9 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip install -e ".[quality]" + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - name: Environment run: | @@ -152,13 +150,13 @@ jobs: CUBLAS_WORKSPACE_CONFIG: :16:8 run: | if [ "${{ matrix.module }}" = "ip_adapters" ]; then - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \ tests/pipelines/${{ matrix.module }} else pattern=$(cat ${{ steps.extract_tests.outputs.pattern_file }}) - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx and $pattern" \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \ tests/pipelines/${{ matrix.module }} @@ -200,11 +198,10 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install peft@git+https://github.com/huggingface/peft.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip install -e ".[quality]" + uv pip install peft@git+https://github.com/huggingface/peft.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - name: Environment run: | @@ -225,10 +222,10 @@ jobs: run: | pattern=$(cat ${{ steps.extract_tests.outputs.pattern_file }}) if [ -z "$pattern" ]; then - python -m pytest -n 1 -sv --max-worker-restart=0 --dist=loadfile -k "not Flax and not Onnx" tests/${{ matrix.module }} \ + pytest -n 1 -sv --max-worker-restart=0 --dist=loadfile -k "not Flax and not Onnx" tests/${{ matrix.module }} \ --make-reports=tests_torch_cuda_${{ matrix.module }} else - python -m pytest -n 1 -sv --max-worker-restart=0 --dist=loadfile -k "not Flax and not Onnx and $pattern" tests/${{ matrix.module }} \ + pytest -n 1 -sv --max-worker-restart=0 --dist=loadfile -k "not Flax and not Onnx and $pattern" tests/${{ matrix.module }} \ --make-reports=tests_torch_cuda_${{ matrix.module }} fi @@ -265,22 +262,19 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - python -m uv pip install -e [quality,test,training] + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip install -e ".[quality,training]" - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run example tests on GPU env: HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install timm - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/ + uv pip install ".[training]" + pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/ - name: Failure short reports if: ${{ failure() }} diff --git a/.github/workflows/pr_torch_dependency_test.yml b/.github/workflows/pr_torch_dependency_test.yml index c39d5eca2d9a..4b6160ff71e2 100644 --- a/.github/workflows/pr_torch_dependency_test.yml +++ b/.github/workflows/pr_torch_dependency_test.yml @@ -25,12 +25,8 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pip install --upgrade pip uv - python -m uv pip install -e . - python -m uv pip install torch torchvision torchaudio - python -m uv pip install pytest + pip install -e . + pip install torch torchvision torchaudio pytest - name: Check for soft dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - pytest tests/others/test_dependencies.py + pytest tests/others/test_dependencies.py diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 6896e0145cbb..a1f5e121537d 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -34,8 +34,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] + uv pip install -e ".[quality]" - name: Environment run: | python utils/print_env.py @@ -75,9 +74,8 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install -e ".[quality]" + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | python utils/print_env.py @@ -87,7 +85,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \ tests/pipelines/${{ matrix.module }} @@ -126,10 +124,9 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install peft@git+https://github.com/huggingface/peft.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install -e ".[quality]" + uv pip install peft@git+https://github.com/huggingface/peft.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | @@ -141,7 +138,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_torch_cuda_${{ matrix.module }} \ tests/${{ matrix.module }} @@ -180,8 +177,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test,training] + uv pip install -e ".[quality,training]" - name: Environment run: | python utils/print_env.py @@ -190,7 +186,7 @@ jobs: HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} RUN_COMPILE: yes run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ + pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ - name: Failure short reports if: ${{ failure() }} run: cat reports/tests_torch_compile_cuda_failures_short.txt @@ -223,8 +219,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test,training] + uv pip install -e ".[quality,training]" - name: Environment run: | python utils/print_env.py @@ -232,7 +227,7 @@ jobs: env: HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/ + pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/ - name: Failure short reports if: ${{ failure() }} run: cat reports/tests_torch_xformers_cuda_failures_short.txt @@ -264,21 +259,18 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test,training] + uv pip install -e ".[quality,training]" - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run example tests on GPU env: HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install timm - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/ + uv pip install ".[training]" + pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/ - name: Failure short reports if: ${{ failure() }} diff --git a/.github/workflows/push_tests_fast.yml b/.github/workflows/push_tests_fast.yml index e274cb021892..6b1dd0b2d006 100644 --- a/.github/workflows/push_tests_fast.yml +++ b/.github/workflows/push_tests_fast.yml @@ -60,19 +60,16 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] + uv pip install -e ".[quality]" - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run fast PyTorch CPU tests if: ${{ matrix.config.framework == 'pytorch' }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \ + pytest -n 4 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_${{ matrix.config.report }} \ tests/ @@ -80,9 +77,8 @@ jobs: - name: Run example PyTorch CPU tests if: ${{ matrix.config.framework == 'pytorch_examples' }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install peft timm - python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \ + uv pip install ".[training]" + pytest -n 4 --max-worker-restart=0 --dist=loadfile \ --make-reports=tests_${{ matrix.config.report }} \ examples diff --git a/.github/workflows/release_tests_fast.yml b/.github/workflows/release_tests_fast.yml index 81a34f7a464d..808818beada3 100644 --- a/.github/workflows/release_tests_fast.yml +++ b/.github/workflows/release_tests_fast.yml @@ -32,8 +32,7 @@ jobs: fetch-depth: 2 - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] + uv pip install -e ".[quality]" - name: Environment run: | python utils/print_env.py @@ -73,9 +72,8 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install -e ".[quality]" + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | python utils/print_env.py @@ -85,7 +83,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_pipeline_${{ matrix.module }}_cuda \ tests/pipelines/${{ matrix.module }} @@ -124,10 +122,9 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install peft@git+https://github.com/huggingface/peft.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install -e ".[quality]" + uv pip install peft@git+https://github.com/huggingface/peft.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | @@ -139,7 +136,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_torch_${{ matrix.module }}_cuda \ tests/${{ matrix.module }} @@ -175,10 +172,9 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install peft@git+https://github.com/huggingface/peft.git - pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git + uv pip install -e ".[quality]" + uv pip install peft@git+https://github.com/huggingface/peft.git + uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - name: Environment run: | @@ -190,7 +186,7 @@ jobs: # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms CUBLAS_WORKSPACE_CONFIG: :16:8 run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + pytest -n 1 --max-worker-restart=0 --dist=loadfile \ -s -v -k "not Flax and not Onnx" \ --make-reports=tests_torch_minimum_cuda \ tests/models/test_modeling_common.py \ @@ -235,8 +231,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test,training] + uv pip install -e ".[quality,training]" - name: Environment run: | python utils/print_env.py @@ -245,7 +240,7 @@ jobs: HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} RUN_COMPILE: yes run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ + pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/ - name: Failure short reports if: ${{ failure() }} run: cat reports/tests_torch_compile_cuda_failures_short.txt @@ -278,8 +273,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test,training] + uv pip install -e ".[quality,training]" - name: Environment run: | python utils/print_env.py @@ -287,7 +281,7 @@ jobs: env: HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} run: | - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/ + pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "xformers" --make-reports=tests_torch_xformers_cuda tests/ - name: Failure short reports if: ${{ failure() }} run: cat reports/tests_torch_xformers_cuda_failures_short.txt @@ -321,21 +315,18 @@ jobs: - name: Install dependencies run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test,training] + uv pip install -e ".[quality,training]" - name: Environment run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python utils/print_env.py - name: Run example tests on GPU env: HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }} run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install timm - python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/ + uv pip install ".[training]" + pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/ - name: Failure short reports if: ${{ failure() }} diff --git a/.github/workflows/run_tests_from_a_pr.yml b/.github/workflows/run_tests_from_a_pr.yml index c8eee8dbbc33..fa8c579dd768 100644 --- a/.github/workflows/run_tests_from_a_pr.yml +++ b/.github/workflows/run_tests_from_a_pr.yml @@ -63,9 +63,8 @@ jobs: - name: Install pytest run: | - python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" - python -m uv pip install -e [quality,test] - python -m uv pip install peft + uv pip install -e ".[quality]" + uv pip install peft - name: Run tests env: diff --git a/docker/diffusers-doc-builder/Dockerfile b/docker/diffusers-doc-builder/Dockerfile index 313eb8b8f85f..4ea282930933 100644 --- a/docker/diffusers-doc-builder/Dockerfile +++ b/docker/diffusers-doc-builder/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get -y update && apt-get install -y bash \ git-lfs \ curl \ ca-certificates \ + libglib2.0-0 \ libsndfile1-dev \ libgl1 diff --git a/docker/diffusers-pytorch-cpu/Dockerfile b/docker/diffusers-pytorch-cpu/Dockerfile index 3fc16d57e8a7..dc5d0fc71cc7 100644 --- a/docker/diffusers-pytorch-cpu/Dockerfile +++ b/docker/diffusers-pytorch-cpu/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get -y update && apt-get install -y bash \ git-lfs \ curl \ ca-certificates \ + libglib2.0-0 \ libsndfile1-dev \ libgl1 diff --git a/docker/diffusers-pytorch-cuda/Dockerfile b/docker/diffusers-pytorch-cuda/Dockerfile index 1cfcda643a5a..2bdfd409b4b4 100644 --- a/docker/diffusers-pytorch-cuda/Dockerfile +++ b/docker/diffusers-pytorch-cuda/Dockerfile @@ -16,6 +16,7 @@ RUN apt install -y bash \ git-lfs \ curl \ ca-certificates \ + libglib2.0-0 \ libsndfile1-dev \ libgl1 \ python3 \ diff --git a/docker/diffusers-pytorch-minimum-cuda/Dockerfile b/docker/diffusers-pytorch-minimum-cuda/Dockerfile index 8ca6a157da6b..a2ce193f6858 100644 --- a/docker/diffusers-pytorch-minimum-cuda/Dockerfile +++ b/docker/diffusers-pytorch-minimum-cuda/Dockerfile @@ -19,6 +19,7 @@ RUN apt install -y bash \ git-lfs \ curl \ ca-certificates \ + libglib2.0-0 \ libsndfile1-dev \ libgl1 \ python3 \ diff --git a/docker/diffusers-pytorch-xformers-cuda/Dockerfile b/docker/diffusers-pytorch-xformers-cuda/Dockerfile index fe1f475504fa..1ea258bdb710 100644 --- a/docker/diffusers-pytorch-xformers-cuda/Dockerfile +++ b/docker/diffusers-pytorch-xformers-cuda/Dockerfile @@ -16,6 +16,7 @@ RUN apt install -y bash \ git-lfs \ curl \ ca-certificates \ + libglib2.0-0 \ libsndfile1-dev \ libgl1 \ python3 \ diff --git a/setup.py b/setup.py index 372a5685957e..8d346ddfecca 100644 --- a/setup.py +++ b/setup.py @@ -145,6 +145,7 @@ "black", "phonemizer", "opencv-python", + "timm", ] # this is a lookup table with items like: @@ -218,7 +219,7 @@ def run(self): extras = {} extras["quality"] = deps_list("urllib3", "isort", "ruff", "hf-doc-builder") extras["docs"] = deps_list("hf-doc-builder") -extras["training"] = deps_list("accelerate", "datasets", "protobuf", "tensorboard", "Jinja2", "peft") +extras["training"] = deps_list("accelerate", "datasets", "protobuf", "tensorboard", "Jinja2", "peft", "timm") extras["test"] = deps_list( "compel", "GitPython", diff --git a/src/diffusers/dependency_versions_table.py b/src/diffusers/dependency_versions_table.py index bfc4e9818ba3..6e5ac630ab08 100644 --- a/src/diffusers/dependency_versions_table.py +++ b/src/diffusers/dependency_versions_table.py @@ -52,4 +52,5 @@ "black": "black", "phonemizer": "phonemizer", "opencv-python": "opencv-python", + "timm": "timm", }