From 3e6e88fd59bd7795e528f8fce1d51cb7f79f5fbf Mon Sep 17 00:00:00 2001 From: Jirka Date: Wed, 4 May 2022 18:56:38 +0200 Subject: [PATCH 1/4] parse strategies as own extras --- requirements/devel.txt | 15 +-------------- setup.py | 4 ++++ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/requirements/devel.txt b/requirements/devel.txt index 09c2699ccd1dd..739bc34c7bdfd 100644 --- a/requirements/devel.txt +++ b/requirements/devel.txt @@ -1,17 +1,4 @@ -# install all mandatory dependencies --r ../requirements.txt - -# install all extra dependencies for full package testing --r ./extra.txt - -# install all loggers for full package testing --r ./loggers.txt +-r ./devel-base.txt # install all strategies for full package testing -r ./strategies.txt - -# extended list of dependencies for development and run lint and tests --r ./test.txt - -# install all extra dependencies for running examples --r ./examples.txt diff --git a/setup.py b/setup.py index e5be7cd37df7f..86fbb83137397 100755 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ import os from importlib.util import module_from_spec, spec_from_file_location +from pkg_resources import parse_requirements from setuptools import find_packages, setup # https://packaging.python.org/guides/single-sourcing-package-version/ @@ -46,6 +47,9 @@ def _load_py_module(fname, pkg="pytorch_lightning"): "strategies": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="strategies.txt"), "test": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="test.txt"), } + +for req in parse_requirements(os.path.join(_PATH_REQUIRE, "strategies.txt")): + extras[req.key] = str(req) extras["dev"] = extras["extra"] + extras["loggers"] + extras["strategies"] + extras["test"] extras["all"] = extras["dev"] + extras["examples"] # + extras['docs'] From a923a7c4563ba81c7045059f929b459d914dee5d Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 5 May 2022 18:59:24 +0200 Subject: [PATCH 2/4] prune devel --- .azure-pipelines/gpu-tests.yml | 5 +++-- .azure-pipelines/ipu-tests.yml | 2 +- .github/workflows/ci_test-conda.yml | 1 + .github/workflows/docs-checks.yml | 2 +- Makefile | 3 ++- dockers/base-cuda/Dockerfile | 2 +- dockers/base-xla/Dockerfile | 2 +- dockers/tpu-tests/Dockerfile | 2 +- pytorch_lightning/setup_tools.py | 22 ------------------- requirements/devel-base.txt | 14 ------------ requirements/devel.txt | 16 +++++++++++--- setup.py | 34 ++++++++++++++--------------- 12 files changed, 40 insertions(+), 65 deletions(-) delete mode 100644 requirements/devel-base.txt diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml index ee7e952e69830..c45613dbb7b0d 100644 --- a/.azure-pipelines/gpu-tests.yml +++ b/.azure-pipelines/gpu-tests.yml @@ -52,10 +52,11 @@ jobs: - bash: | python -c "fname = 'requirements/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)" CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))") - pip install "bagua-cuda$CUDA_VERSION_MM>=0.9.0" - pip install . --requirement requirements/devel.txt # TODO: Prepare a docker image with 1.8.2 (LTS) installed and remove manual installation. pip install torch==1.8.2+cu102 torchvision==0.9.2+cu102 torchtext==0.9.2 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html + pip install "bagua-cuda$CUDA_VERSION_MM>=0.9.0" + pip install . --requirement requirements/devel.txt + pip install . --requirement requirements/strategies.txt pip list displayName: 'Install dependencies' diff --git a/.azure-pipelines/ipu-tests.yml b/.azure-pipelines/ipu-tests.yml index 0a60cf7867725..80d49f9da3e78 100644 --- a/.azure-pipelines/ipu-tests.yml +++ b/.azure-pipelines/ipu-tests.yml @@ -55,7 +55,7 @@ jobs: export GIT_TERMINAL_PROMPT=1 python ./requirements/adjust-versions.py requirements/extra.txt python ./requirements/adjust-versions.py requirements/examples.txt - pip install . --requirement ./requirements/devel-base.txt + pip install . --requirement ./requirements/devel.txt pip list displayName: 'Install dependencies' diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index 31bd11429a050..c907e13b4ac23 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -48,6 +48,7 @@ jobs: python ./requirements/adjust-versions.py requirements/extra.txt python ./requirements/adjust-versions.py requirements/examples.txt pip install -r requirements/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html + pip install -r requirements/strategies.txt # set a per-test timeout of 2.5 minutes to fail sooner; this aids with hanging tests pip install pytest-timeout pip list diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 7c9069e53dd67..dd7ac9314ca70 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -43,7 +43,7 @@ jobs: pip install -q fire # python -m pip install --upgrade --user pip pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet - pip install --requirement requirements/devel-base.txt + pip install --requirement requirements/devel.txt pip install --requirement requirements/docs.txt pip list shell: bash diff --git a/Makefile b/Makefile index e6cc00207e832..de837a22ae39d 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,9 @@ clean: rm -rf ./docs/source/api test: clean - # Review the CONTRIBUTING docmentation for other ways to test. + # Review the CONTRIBUTING documentation for other ways to test. pip install -r requirements/devel.txt + pip install -r requirements/strategies.txt # install APEX, see https://github.com/NVIDIA/apex#linux # run tests with coverage diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile index c39670c1b8045..2938a570aa21e 100644 --- a/dockers/base-cuda/Dockerfile +++ b/dockers/base-cuda/Dockerfile @@ -90,7 +90,7 @@ RUN \ python ./requirements/adjust-versions.py requirements/extra.txt ${PYTORCH_VERSION} && \ python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \ # Install all requirements \ - pip install -r requirements/devel-base.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \ + pip install -r requirements/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \ rm -rf requirements.* && \ rm assistant.py diff --git a/dockers/base-xla/Dockerfile b/dockers/base-xla/Dockerfile index 2fb5b99905121..d933aac67d422 100644 --- a/dockers/base-xla/Dockerfile +++ b/dockers/base-xla/Dockerfile @@ -97,7 +97,7 @@ RUN \ # drop unnecessary packages python ./requirements/adjust-versions.py ./requirements/extra.txt && \ # install PL dependencies - pip install --requirement ./requirements/devel-base.txt --no-cache-dir && \ + pip install --requirement ./requirements/devel.txt --no-cache-dir && \ cd .. && \ rm -rf pytorch-lightning && \ rm -rf /root/.cache diff --git a/dockers/tpu-tests/Dockerfile b/dockers/tpu-tests/Dockerfile index 9a7528cc22e21..602ef1684b859 100644 --- a/dockers/tpu-tests/Dockerfile +++ b/dockers/tpu-tests/Dockerfile @@ -30,7 +30,7 @@ RUN cd pytorch-lightning && \ RUN \ pip install -q fire && \ # drop unnecessary packages - pip install -r pytorch-lightning/requirements/devel-base.txt --no-cache-dir + pip install -r pytorch-lightning/requirements/devel.txt --no-cache-dir COPY ./dockers/tpu-tests/docker-entrypoint.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/docker-entrypoint.sh diff --git a/pytorch_lightning/setup_tools.py b/pytorch_lightning/setup_tools.py index da16d9475a41b..47a3bc03e7cd3 100644 --- a/pytorch_lightning/setup_tools.py +++ b/pytorch_lightning/setup_tools.py @@ -14,32 +14,10 @@ # limitations under the License. import os import re -from typing import List _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__)) -def _load_requirements(path_dir: str, file_name: str = "requirements.txt", comment_char: str = "#") -> List[str]: - """Load requirements from a file. - - >>> _load_requirements(_PROJECT_ROOT) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - ['numpy...', 'torch...', ...] - """ - with open(os.path.join(path_dir, file_name)) as file: - lines = [ln.strip() for ln in file.readlines()] - reqs = [] - for ln in lines: - # filer all comments - if comment_char in ln: - ln = ln[: ln.index(comment_char)].strip() - # skip directly installed dependencies - if ln.startswith("http") or "@http" in ln: - continue - if ln: # if requirement is not empty - reqs.append(ln) - return reqs - - def _load_readme_description(path_dir: str, homepage: str, version: str) -> str: """Load readme as decribtion. diff --git a/requirements/devel-base.txt b/requirements/devel-base.txt deleted file mode 100644 index 3252ebf39f431..0000000000000 --- a/requirements/devel-base.txt +++ /dev/null @@ -1,14 +0,0 @@ -# install all mandatory dependencies --r ../requirements.txt - -# install all extra dependencies for full package testing --r ./extra.txt - -# install all loggers for full package testing --r ./loggers.txt - -# extended list of dependencies for development and run lint and tests --r ./test.txt - -# install all extra dependencies for running examples --r ./examples.txt diff --git a/requirements/devel.txt b/requirements/devel.txt index 739bc34c7bdfd..3252ebf39f431 100644 --- a/requirements/devel.txt +++ b/requirements/devel.txt @@ -1,4 +1,14 @@ --r ./devel-base.txt +# install all mandatory dependencies +-r ../requirements.txt -# install all strategies for full package testing --r ./strategies.txt +# install all extra dependencies for full package testing +-r ./extra.txt + +# install all loggers for full package testing +-r ./loggers.txt + +# extended list of dependencies for development and run lint and tests +-r ./test.txt + +# install all extra dependencies for running examples +-r ./examples.txt diff --git a/setup.py b/setup.py index 86fbb83137397..9df83ef7dcea4 100755 --- a/setup.py +++ b/setup.py @@ -35,31 +35,29 @@ def _load_py_module(fname, pkg="pytorch_lightning"): about = _load_py_module("__about__.py") setup_tools = _load_py_module("setup_tools.py") + +def _load_requirements(path_dir: str, file_name: str = "requirements.txt") -> list: + reqs = parse_requirements(open(os.path.join(path_dir, file_name)).readlines()) + return list(map(str, reqs)) + + # https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras # Define package extras. These are only installed if you specify them. # From remote, use like `pip install pytorch-lightning[dev, docs]` # From local copy of repo, use like `pip install ".[dev, docs]"` extras = { # 'docs': load_requirements(file_name='docs.txt'), - "examples": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="examples.txt"), - "loggers": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="loggers.txt"), - "extra": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="extra.txt"), - "strategies": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="strategies.txt"), - "test": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="test.txt"), + "examples": _load_requirements(path_dir=_PATH_REQUIRE, file_name="examples.txt"), + "loggers": _load_requirements(path_dir=_PATH_REQUIRE, file_name="loggers.txt"), + "extra": _load_requirements(path_dir=_PATH_REQUIRE, file_name="extra.txt"), + "strategies": _load_requirements(path_dir=_PATH_REQUIRE, file_name="strategies.txt"), + "test": _load_requirements(path_dir=_PATH_REQUIRE, file_name="test.txt"), } -for req in parse_requirements(os.path.join(_PATH_REQUIRE, "strategies.txt")): - extras[req.key] = str(req) -extras["dev"] = extras["extra"] + extras["loggers"] + extras["strategies"] + extras["test"] -extras["all"] = extras["dev"] + extras["examples"] # + extras['docs'] - -# These packages shall be installed only on GPU machines -PACKAGES_GPU_ONLY = ["horovod"] -# create a version for CPU machines -for ex in ("cpu", "cpu-extra"): - kw = ex.split("-")[1] if "-" in ex else "all" - # filter cpu only packages - extras[ex] = [pkg for pkg in extras[kw] if not any(pgpu.lower() in pkg.lower() for pgpu in PACKAGES_GPU_ONLY)] +for req in parse_requirements(extras["strategies"]): + extras[req.key] = [str(req)] +extras["dev"] = extras["extra"] + extras["loggers"] + extras["test"] +extras["all"] = extras["dev"] + extras["examples"] + extras["strategies"] # + extras['docs'] long_description = setup_tools._load_readme_description( _PATH_ROOT, homepage=about.__homepage__, version=about.__version__ @@ -87,7 +85,7 @@ def _load_py_module(fname, pkg="pytorch_lightning"): keywords=["deep learning", "pytorch", "AI"], python_requires=">=3.7", setup_requires=[], - install_requires=setup_tools._load_requirements(_PATH_ROOT), + install_requires=_load_requirements(_PATH_ROOT), extras_require=extras, project_urls={ "Bug Tracker": "https://github.com/PyTorchLightning/pytorch-lightning/issues", From 13386ebdf3b762cf5ec58e2bf3fbe5106232559d Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 6 May 2022 04:22:50 +0200 Subject: [PATCH 3/4] Update Makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Carlos Mocholí --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index de837a22ae39d..efd2a7f3c5496 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,6 @@ test: clean # Review the CONTRIBUTING documentation for other ways to test. pip install -r requirements/devel.txt pip install -r requirements/strategies.txt - # install APEX, see https://github.com/NVIDIA/apex#linux - # run tests with coverage python -m coverage run --source pytorch_lightning -m pytest pytorch_lightning tests pl_examples -v python -m coverage report From b865d891702e32ae991d11bda61d1461600870c6 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 6 May 2022 05:15:52 +0200 Subject: [PATCH 4/4] revert parse_requirements --- pytorch_lightning/setup_tools.py | 22 ++++++++++++++++++++++ setup.py | 18 ++++++------------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/pytorch_lightning/setup_tools.py b/pytorch_lightning/setup_tools.py index 47a3bc03e7cd3..da16d9475a41b 100644 --- a/pytorch_lightning/setup_tools.py +++ b/pytorch_lightning/setup_tools.py @@ -14,10 +14,32 @@ # limitations under the License. import os import re +from typing import List _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__)) +def _load_requirements(path_dir: str, file_name: str = "requirements.txt", comment_char: str = "#") -> List[str]: + """Load requirements from a file. + + >>> _load_requirements(_PROJECT_ROOT) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + ['numpy...', 'torch...', ...] + """ + with open(os.path.join(path_dir, file_name)) as file: + lines = [ln.strip() for ln in file.readlines()] + reqs = [] + for ln in lines: + # filer all comments + if comment_char in ln: + ln = ln[: ln.index(comment_char)].strip() + # skip directly installed dependencies + if ln.startswith("http") or "@http" in ln: + continue + if ln: # if requirement is not empty + reqs.append(ln) + return reqs + + def _load_readme_description(path_dir: str, homepage: str, version: str) -> str: """Load readme as decribtion. diff --git a/setup.py b/setup.py index 9df83ef7dcea4..d3cc4a0602e3e 100755 --- a/setup.py +++ b/setup.py @@ -35,23 +35,17 @@ def _load_py_module(fname, pkg="pytorch_lightning"): about = _load_py_module("__about__.py") setup_tools = _load_py_module("setup_tools.py") - -def _load_requirements(path_dir: str, file_name: str = "requirements.txt") -> list: - reqs = parse_requirements(open(os.path.join(path_dir, file_name)).readlines()) - return list(map(str, reqs)) - - # https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras # Define package extras. These are only installed if you specify them. # From remote, use like `pip install pytorch-lightning[dev, docs]` # From local copy of repo, use like `pip install ".[dev, docs]"` extras = { # 'docs': load_requirements(file_name='docs.txt'), - "examples": _load_requirements(path_dir=_PATH_REQUIRE, file_name="examples.txt"), - "loggers": _load_requirements(path_dir=_PATH_REQUIRE, file_name="loggers.txt"), - "extra": _load_requirements(path_dir=_PATH_REQUIRE, file_name="extra.txt"), - "strategies": _load_requirements(path_dir=_PATH_REQUIRE, file_name="strategies.txt"), - "test": _load_requirements(path_dir=_PATH_REQUIRE, file_name="test.txt"), + "examples": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="examples.txt"), + "loggers": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="loggers.txt"), + "extra": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="extra.txt"), + "strategies": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="strategies.txt"), + "test": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="test.txt"), } for req in parse_requirements(extras["strategies"]): @@ -85,7 +79,7 @@ def _load_requirements(path_dir: str, file_name: str = "requirements.txt") -> li keywords=["deep learning", "pytorch", "AI"], python_requires=">=3.7", setup_requires=[], - install_requires=_load_requirements(_PATH_ROOT), + install_requires=setup_tools._load_requirements(_PATH_ROOT), extras_require=extras, project_urls={ "Bug Tracker": "https://github.com/PyTorchLightning/pytorch-lightning/issues",