From 4abe2334119c80d3d21775b750b61da7ba27096b Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 22 Mar 2023 14:20:55 +0100 Subject: [PATCH 01/11] integrations --- .github/workflows/ci-tests-pytorch.yml | 8 +++++++- requirements/pytorch/integrations.txt | 5 +++++ requirements/pytorch/strategies.txt | 2 -- src/lightning/__setup__.py | 2 +- src/pytorch_lightning/__setup__.py | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 requirements/pytorch/integrations.txt diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 316710ffe1677..9a3fb65f3047d 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -115,7 +115,10 @@ jobs: - name: Install package & dependencies run: | - pip install -e .[extra,test] "pytest-timeout" -U -f ${TORCH_URL} ${TORCH_PREINSTALL} -f ${PYPI_CACHE} --prefer-binary + pip install -q pip -U + pip install .[extra,test] -U \ + "pytest-timeout" -r requirements/pytorch/integrations.txt \ + -f ${TORCH_URL} ${TORCH_PREINSTALL} -f ${PYPI_CACHE} --prefer-binary pip list - name: Dump handy wheels if: github.event_name == 'push' && github.ref == 'refs/heads/master' @@ -145,6 +148,9 @@ jobs: --source_import="lightning.fabric,lightning.pytorch" \ --target_import="lightning_fabric,pytorch_lightning" + - name: Prevent using raw source + run: rm -rf src/ + - name: Testing Warnings working-directory: tests/tests_pytorch # needs to run outside of `pytest` diff --git a/requirements/pytorch/integrations.txt b/requirements/pytorch/integrations.txt new file mode 100644 index 0000000000000..57c57aa4305d1 --- /dev/null +++ b/requirements/pytorch/integrations.txt @@ -0,0 +1,5 @@ +lightning-colossalai>=0.1.0 +lightning-bagua>=0.1.0 + +# validation HPU connectors +lightning-habana>=0.1.0rc0 diff --git a/requirements/pytorch/strategies.txt b/requirements/pytorch/strategies.txt index 3588468672809..c8a5c9531fe3d 100644 --- a/requirements/pytorch/strategies.txt +++ b/requirements/pytorch/strategies.txt @@ -2,5 +2,3 @@ # in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment deepspeed>=0.6.0, <0.8.0 # TODO: Include 0.8.x after https://github.com/microsoft/DeepSpeed/commit/b587c7e85470329ac25df7c7c2521ff9b2833db7 gets released -lightning-colossalai>=0.1.0dev1 -lightning-bagua>=0.1.0rc2 diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index ceef349a96d6e..aa80fff886a81 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -36,7 +36,7 @@ def _prepare_extras() -> Dict[str, Any]: extras = { f"{p.parent.name}-{p.stem}": _ASSISTANT.load_requirements(file_name=p.name, path_dir=p.parent, **common_args) for p in req_files - if p.name not in ("docs.txt", "devel.txt", "base.txt") + if p.name not in ("docs.txt", "devel.txt", "base.txt", "integrations.txt") } for extra in list(extras): name = "-".join(extra.split("-")[1:]) diff --git a/src/pytorch_lightning/__setup__.py b/src/pytorch_lightning/__setup__.py index 8db8ef1cbe8fd..c40ccbce0ba13 100644 --- a/src/pytorch_lightning/__setup__.py +++ b/src/pytorch_lightning/__setup__.py @@ -40,7 +40,7 @@ def _prepare_extras() -> Dict[str, Any]: extras = { p.stem: assistant.load_requirements(file_name=p.name, **common_args) for p in req_files - if p.name not in ("docs.txt", "devel.txt", "base.txt") + if p.name not in ("docs.txt", "devel.txt", "base.txt", "integrations.txt") } for req in parse_requirements(extras["strategies"]): extras[req.key] = [str(req)] From 6963d8b45e6636ea0f70538a8b69046408bd7bdf Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Wed, 22 Mar 2023 16:50:59 +0100 Subject: [PATCH 02/11] Rename --- requirements/pytorch/integration_strategies.txt | 2 ++ requirements/pytorch/integrations.txt | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 requirements/pytorch/integration_strategies.txt delete mode 100644 requirements/pytorch/integrations.txt diff --git a/requirements/pytorch/integration_strategies.txt b/requirements/pytorch/integration_strategies.txt new file mode 100644 index 0000000000000..c3c291295f7d5 --- /dev/null +++ b/requirements/pytorch/integration_strategies.txt @@ -0,0 +1,2 @@ +lightning-colossalai>=0.1.0 +lightning-bagua>=0.1.0 diff --git a/requirements/pytorch/integrations.txt b/requirements/pytorch/integrations.txt deleted file mode 100644 index 57c57aa4305d1..0000000000000 --- a/requirements/pytorch/integrations.txt +++ /dev/null @@ -1,5 +0,0 @@ -lightning-colossalai>=0.1.0 -lightning-bagua>=0.1.0 - -# validation HPU connectors -lightning-habana>=0.1.0rc0 From ec5709265336ce291e2fa65745723d02ed880d50 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Wed, 22 Mar 2023 16:57:39 +0100 Subject: [PATCH 03/11] Apply suggestions from code review --- src/lightning/__setup__.py | 2 +- src/pytorch_lightning/__setup__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index aa80fff886a81..d2a7a0ab56f13 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -36,7 +36,7 @@ def _prepare_extras() -> Dict[str, Any]: extras = { f"{p.parent.name}-{p.stem}": _ASSISTANT.load_requirements(file_name=p.name, path_dir=p.parent, **common_args) for p in req_files - if p.name not in ("docs.txt", "devel.txt", "base.txt", "integrations.txt") + if p.name not in ("docs.txt", "devel.txt", "base.txt") and not p.name.startswith("integration") } for extra in list(extras): name = "-".join(extra.split("-")[1:]) diff --git a/src/pytorch_lightning/__setup__.py b/src/pytorch_lightning/__setup__.py index c40ccbce0ba13..e121ba41aadf3 100644 --- a/src/pytorch_lightning/__setup__.py +++ b/src/pytorch_lightning/__setup__.py @@ -40,7 +40,7 @@ def _prepare_extras() -> Dict[str, Any]: extras = { p.stem: assistant.load_requirements(file_name=p.name, **common_args) for p in req_files - if p.name not in ("docs.txt", "devel.txt", "base.txt", "integrations.txt") + if p.name not in ("docs.txt", "devel.txt", "base.txt") and not p.name.startswith("integration") } for req in parse_requirements(extras["strategies"]): extras[req.key] = [str(req)] From e7ec8ef056f80b34880528232cd08cadff4398f2 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 22 Mar 2023 21:16:46 +0100 Subject: [PATCH 04/11] cut out --- .azure/gpu-tests-pytorch.yml | 2 +- .github/workflows/ci-tests-pytorch.yml | 3 +-- .../strategies.txt} | 0 src/lightning/__setup__.py | 2 +- src/pytorch_lightning/__setup__.py | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) rename requirements/{pytorch/integration_strategies.txt => _integrations/strategies.txt} (100%) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 02b628e9c3c43..4f19394a48678 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -117,7 +117,7 @@ jobs: - bash: | set -e - pip install -r requirements/pytorch/strategies.txt --find-links ${TORCH_URL} + pip install -r requirements/pytorch/strategies.txt -r requirements/_integrations/strategies.txt --find-links ${TORCH_URL} python requirements/pytorch/check-avail-strategies.py condition: eq(variables['scope'], 'strategies') displayName: 'Install strategies' diff --git a/.github/workflows/ci-tests-pytorch.yml b/.github/workflows/ci-tests-pytorch.yml index 9a3fb65f3047d..666afd72e7818 100644 --- a/.github/workflows/ci-tests-pytorch.yml +++ b/.github/workflows/ci-tests-pytorch.yml @@ -116,8 +116,7 @@ jobs: - name: Install package & dependencies run: | pip install -q pip -U - pip install .[extra,test] -U \ - "pytest-timeout" -r requirements/pytorch/integrations.txt \ + pip install .[extra,test] -U "pytest-timeout" \ -f ${TORCH_URL} ${TORCH_PREINSTALL} -f ${PYPI_CACHE} --prefer-binary pip list - name: Dump handy wheels diff --git a/requirements/pytorch/integration_strategies.txt b/requirements/_integrations/strategies.txt similarity index 100% rename from requirements/pytorch/integration_strategies.txt rename to requirements/_integrations/strategies.txt diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index d2a7a0ab56f13..ceef349a96d6e 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -36,7 +36,7 @@ def _prepare_extras() -> Dict[str, Any]: extras = { f"{p.parent.name}-{p.stem}": _ASSISTANT.load_requirements(file_name=p.name, path_dir=p.parent, **common_args) for p in req_files - if p.name not in ("docs.txt", "devel.txt", "base.txt") and not p.name.startswith("integration") + if p.name not in ("docs.txt", "devel.txt", "base.txt") } for extra in list(extras): name = "-".join(extra.split("-")[1:]) diff --git a/src/pytorch_lightning/__setup__.py b/src/pytorch_lightning/__setup__.py index e121ba41aadf3..8db8ef1cbe8fd 100644 --- a/src/pytorch_lightning/__setup__.py +++ b/src/pytorch_lightning/__setup__.py @@ -40,7 +40,7 @@ def _prepare_extras() -> Dict[str, Any]: extras = { p.stem: assistant.load_requirements(file_name=p.name, **common_args) for p in req_files - if p.name not in ("docs.txt", "devel.txt", "base.txt") and not p.name.startswith("integration") + if p.name not in ("docs.txt", "devel.txt", "base.txt") } for req in parse_requirements(extras["strategies"]): extras[req.key] = [str(req)] From a7feb5ea5a80f7dbf4a7cbb63c01a0fe85dd46e1 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 22 Mar 2023 21:21:28 +0100 Subject: [PATCH 05/11] skip _ --- .actions/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index ca30b13decbe3..cd7a3984e44ed 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -240,7 +240,7 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme load_requirements(d, unfreeze="none" if freeze_requirements else "major") for d in glob.glob(os.path.join(req_dir, "*")) # skip empty folder as git artefacts, and resolving Will's special issue - if os.path.isdir(d) and len(glob.glob(os.path.join(d, "*"))) > 0 and "__pycache__" not in d + if os.path.isdir(d) and len(glob.glob(os.path.join(d, "*"))) > 0 and not d.startswith("_") ] if not requires: return From b1991f2ae6376161e0e3009779d11bb99410eeb9 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 22 Mar 2023 21:37:28 +0100 Subject: [PATCH 06/11] fix --- .actions/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index cd7a3984e44ed..d791793af9490 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -240,7 +240,7 @@ def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requireme load_requirements(d, unfreeze="none" if freeze_requirements else "major") for d in glob.glob(os.path.join(req_dir, "*")) # skip empty folder as git artefacts, and resolving Will's special issue - if os.path.isdir(d) and len(glob.glob(os.path.join(d, "*"))) > 0 and not d.startswith("_") + if os.path.isdir(d) and len(glob.glob(os.path.join(d, "*"))) > 0 and not os.path.basename(d).startswith("_") ] if not requires: return From b06d4d3058e3cd3a6b572beeeccda33cf10ffa1c Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 22 Mar 2023 21:54:31 +0100 Subject: [PATCH 07/11] setup --- .azure/gpu-tests-pytorch.yml | 4 +++- requirements/_integrations/strategies.txt | 4 ++-- src/lightning/__setup__.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index 4f19394a48678..c13493790bb53 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -117,7 +117,9 @@ jobs: - bash: | set -e - pip install -r requirements/pytorch/strategies.txt -r requirements/_integrations/strategies.txt --find-links ${TORCH_URL} + pip install -r requirements/pytorch/strategies.txt \ + -r requirements/_integrations/strategies.txt \ + --find-links ${TORCH_URL} python requirements/pytorch/check-avail-strategies.py condition: eq(variables['scope'], 'strategies') displayName: 'Install strategies' diff --git a/requirements/_integrations/strategies.txt b/requirements/_integrations/strategies.txt index c3c291295f7d5..3366bbebb4664 100644 --- a/requirements/_integrations/strategies.txt +++ b/requirements/_integrations/strategies.txt @@ -1,2 +1,2 @@ -lightning-colossalai>=0.1.0 -lightning-bagua>=0.1.0 +lightning-colossalai >=0.1.0 +lightning-bagua >=0.1.0 diff --git a/src/lightning/__setup__.py b/src/lightning/__setup__.py index ceef349a96d6e..15d4a478082dc 100644 --- a/src/lightning/__setup__.py +++ b/src/lightning/__setup__.py @@ -36,7 +36,7 @@ def _prepare_extras() -> Dict[str, Any]: extras = { f"{p.parent.name}-{p.stem}": _ASSISTANT.load_requirements(file_name=p.name, path_dir=p.parent, **common_args) for p in req_files - if p.name not in ("docs.txt", "devel.txt", "base.txt") + if p.name not in ("docs.txt", "devel.txt", "base.txt") and not p.parts[-2].startswith("_") } for extra in list(extras): name = "-".join(extra.split("-")[1:]) From 283fd64be5d1a848c3634f2c7a7110b14993f457 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 22 Mar 2023 22:03:44 +0100 Subject: [PATCH 08/11] 1e-2 --- tests/tests_fabric/parity/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_fabric/parity/utils.py b/tests/tests_fabric/parity/utils.py index 0248c036f76f0..de9202f01c969 100644 --- a/tests/tests_fabric/parity/utils.py +++ b/tests/tests_fabric/parity/utils.py @@ -24,7 +24,7 @@ def is_state_dict_equal(state0, state1): return all(eq_fn(w0.cpu(), w1.cpu()) for w0, w1 in zip(state0.values(), state1.values())) -def is_timing_close(timings_torch, timings_fabric, rtol=1e-3, atol=1e-3): +def is_timing_close(timings_torch, timings_fabric, rtol=1e-2, atol=1e-2): # Drop measurements of the first iterations, as they may be slower than others # The median is more robust to outliers than the mean # Given relative and absolute tolerances, we want to satisfy: |torch – fabric| < RTOL * torch + ATOL From 2b48a739f861b2c5a35bfcd9230628ec0e4b06d5 Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 22 Mar 2023 22:26:10 +0100 Subject: [PATCH 09/11] uninstall --- .azure/gpu-tests-pytorch.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure/gpu-tests-pytorch.yml b/.azure/gpu-tests-pytorch.yml index c13493790bb53..ceffccf8b6228 100644 --- a/.azure/gpu-tests-pytorch.yml +++ b/.azure/gpu-tests-pytorch.yml @@ -111,7 +111,9 @@ jobs: - bash: pip install -e .[extra,test,examples] -U --find-links ${TORCH_URL} displayName: 'Install package & dependencies' - - bash: pip uninstall -y -r requirements/pytorch/strategies.txt + - bash: | + pip uninstall -y -r requirements/pytorch/strategies.txt \ + -r requirements/_integrations/strategies.txt condition: ne(variables['scope'], 'strategies') displayName: 'Uninstall strategies' From 600c91f87cad19df43baa35797ff776d3cd628c4 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:22:24 +0100 Subject: [PATCH 10/11] revert --- tests/tests_fabric/parity/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_fabric/parity/utils.py b/tests/tests_fabric/parity/utils.py index de9202f01c969..0248c036f76f0 100644 --- a/tests/tests_fabric/parity/utils.py +++ b/tests/tests_fabric/parity/utils.py @@ -24,7 +24,7 @@ def is_state_dict_equal(state0, state1): return all(eq_fn(w0.cpu(), w1.cpu()) for w0, w1 in zip(state0.values(), state1.values())) -def is_timing_close(timings_torch, timings_fabric, rtol=1e-2, atol=1e-2): +def is_timing_close(timings_torch, timings_fabric, rtol=1e-3, atol=1e-3): # Drop measurements of the first iterations, as they may be slower than others # The median is more robust to outliers than the mean # Given relative and absolute tolerances, we want to satisfy: |torch – fabric| < RTOL * torch + ATOL From 670145e489ccd1dd60aef64860ca84242213a264 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:24:10 +0100 Subject: [PATCH 11/11] note --- requirements/_integrations/strategies.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements/_integrations/strategies.txt b/requirements/_integrations/strategies.txt index 3366bbebb4664..3227549ab722b 100644 --- a/requirements/_integrations/strategies.txt +++ b/requirements/_integrations/strategies.txt @@ -1,2 +1,5 @@ +# NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package +# in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment + lightning-colossalai >=0.1.0 lightning-bagua >=0.1.0