From 1c5913a7e19684d2709eba43cfec23778a787b34 Mon Sep 17 00:00:00 2001 From: Logan Adams Date: Thu, 15 May 2025 17:53:24 -0700 Subject: [PATCH 1/5] Temporarily skip AIO tests due to an issue with runners Signed-off-by: Logan Adams --- .github/workflows/nv-torch-latest-v100.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml index ab6da53acf23..57319fb080e0 100644 --- a/.github/workflows/nv-torch-latest-v100.yml +++ b/.github/workflows/nv-torch-latest-v100.yml @@ -55,5 +55,5 @@ jobs: run: | unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch cd tests - pytest $PYTEST_OPTS --forked -n 8 unit/ --torch_ver="2.6" --cuda_ver="12.4" + pytest $PYTEST_OPTS --forked -n 8 unit/ --torch_ver="2.6" --cuda_ver="12.4" --ignore=unit/ops/aio/ pytest $PYTEST_OPTS --forked -m 'sequential' unit/ --torch_ver="2.6" --cuda_ver="12.4" From 9c1c85a7048c3ed5ee2e451e16044aa23ed635b6 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Fri, 16 May 2025 15:00:36 +0000 Subject: [PATCH 2/5] Disable NVMe tests Signed-off-by: Olatunji Ruwase --- .github/workflows/setup-venv/action.yml | 3 ++- .../unit/inference/quantization/test_intX_quantization.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/setup-venv/action.yml b/.github/workflows/setup-venv/action.yml index 9a88e0651860..0830f45ba579 100644 --- a/.github/workflows/setup-venv/action.yml +++ b/.github/workflows/setup-venv/action.yml @@ -6,7 +6,8 @@ runs: - id: update-env run: | sudo apt-get update - sudo apt-get install -y libaio-dev + # Temporary disable nvme UTs + # sudo apt-get install -y libaio-dev python -m pip install --user --upgrade pip python -m pip install --user --upgrade virtualenv shell: bash diff --git a/tests/unit/inference/quantization/test_intX_quantization.py b/tests/unit/inference/quantization/test_intX_quantization.py index 77b51fcd5814..8169912ae487 100644 --- a/tests/unit/inference/quantization/test_intX_quantization.py +++ b/tests/unit/inference/quantization/test_intX_quantization.py @@ -17,6 +17,7 @@ import pytest from collections import OrderedDict from typing import Dict +from deepspeed.ops.aio import AsyncIOBuilder device = get_accelerator().device_name() if get_accelerator().is_available() else 'cpu' @@ -57,6 +58,9 @@ def zero3_post_init_quantization_test_helper(cpu_offload: bool, nvme_offload: bo import deepspeed from transformers.integrations.deepspeed import HfDeepSpeedConfig + if nvme_offload and not deepspeed.ops.__compatible_ops__[AsyncIOBuilder.NAME]: + pytest.skip('Skip tests since async-io is not compatible') + def get_zero3_ds_config(hf_config: OPTConfig, cpu_offload: bool, nvme_offload: bool, bits: int) -> Dict: GB = 1 << 30 @@ -174,6 +178,9 @@ def zero3_quantized_initialization_test_helper(cpu_offload: bool, nvme_offload: import deepspeed from transformers.integrations.deepspeed import HfDeepSpeedConfig + if nvme_offload and not deepspeed.ops.__compatible_ops__[AsyncIOBuilder.NAME]: + pytest.skip('Skip tests since async-io is not compatible') + def get_zero3_ds_config(hf_config: OPTConfig, cpu_offload: bool, nvme_offload: bool, bits: int) -> Dict: GB = 1 << 30 From 1504a4004fcbf50f8aa03788013f3ae99245622e Mon Sep 17 00:00:00 2001 From: Logan Adams Date: Fri, 16 May 2025 19:00:44 -0700 Subject: [PATCH 3/5] Rememove ignore in tests Signed-off-by: Logan Adams --- .github/workflows/nv-torch-latest-v100.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml index 57319fb080e0..ab6da53acf23 100644 --- a/.github/workflows/nv-torch-latest-v100.yml +++ b/.github/workflows/nv-torch-latest-v100.yml @@ -55,5 +55,5 @@ jobs: run: | unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch cd tests - pytest $PYTEST_OPTS --forked -n 8 unit/ --torch_ver="2.6" --cuda_ver="12.4" --ignore=unit/ops/aio/ + pytest $PYTEST_OPTS --forked -n 8 unit/ --torch_ver="2.6" --cuda_ver="12.4" pytest $PYTEST_OPTS --forked -m 'sequential' unit/ --torch_ver="2.6" --cuda_ver="12.4" From 84bd900900df92772d9fe40558f23483a584948d Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Sun, 18 May 2025 11:53:12 +0000 Subject: [PATCH 4/5] Remove libaio-dev Signed-off-by: Olatunji Ruwase --- .github/workflows/setup-venv/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/setup-venv/action.yml b/.github/workflows/setup-venv/action.yml index 0830f45ba579..af7913290b71 100644 --- a/.github/workflows/setup-venv/action.yml +++ b/.github/workflows/setup-venv/action.yml @@ -8,6 +8,7 @@ runs: sudo apt-get update # Temporary disable nvme UTs # sudo apt-get install -y libaio-dev + sudo apt remove -y libaio-dev python -m pip install --user --upgrade pip python -m pip install --user --upgrade virtualenv shell: bash From 48dad0e51c109bd71f90ef8799ba1e0a1ab9a26d Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Sun, 18 May 2025 21:53:45 +0000 Subject: [PATCH 5/5] Skip nvme UTs Signed-off-by: Olatunji Ruwase --- tests/unit/runtime/compile/test_compile_zero.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/runtime/compile/test_compile_zero.py b/tests/unit/runtime/compile/test_compile_zero.py index 5f6cd6c6c23a..ca5b7e74b64c 100644 --- a/tests/unit/runtime/compile/test_compile_zero.py +++ b/tests/unit/runtime/compile/test_compile_zero.py @@ -13,6 +13,8 @@ from unit.runtime.compile.util import compare_loss from unit.common import DistributedTest from unit.util import bf16_required_version_check, skip_on_arch +import deepspeed +from deepspeed.ops.aio import AsyncIOBuilder pytestmark = pytest.mark.skipif(not required_torch_version(min_version=2.1), reason="Compile tests requires Pytorch version 2.1 or above") @@ -36,6 +38,8 @@ def test_compile_zero(self, tmpdir, zero_stage, dtype, offload_device): pytest.skip("CPU does not support this test yet") if offload_device == OffloadDeviceEnum.nvme: + if not deepspeed.ops.__compatible_ops__[AsyncIOBuilder.NAME]: + pytest.skip('Skip tests since async-io is not compatible') if zero_stage != 3: pytest.skip(f"Nvme offload not supported for zero stage {zero_stage}")