From 6983c3a2d3c295ba4451a05a558be915e957cb9d Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 6 May 2022 05:26:54 +0200 Subject: [PATCH 01/12] freeze versions --- requirements.txt | 21 +++++++++++---------- requirements/examples.txt | 6 +++--- requirements/extra.txt | 16 +++++++++------- requirements/loggers.txt | 10 ++++++---- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/requirements.txt b/requirements.txt index 39f0d586ba18f..6761cca0967e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,13 @@ # the default package dependencies -numpy>=1.17.2 -torch>=1.8.* -tqdm>=4.57.0 -PyYAML>=5.4 -fsspec[http]>=2021.05.0, !=2021.06.0 -tensorboard>=2.2.0 -torchmetrics>=0.4.1 -pyDeprecate>=0.3.1, <0.4.0 -packaging>=17.0 -typing-extensions>=4.0.0 + +numpy>=1.17.2, <=1.22.3 +torch>=1.8.*, <=1.11.0 +tqdm>=4.57.0, <=4.63.0 +PyYAML>=5.4, <=6.0 +fsspec[http]>=2021.05.0, !=2021.06.0, <=2022.2.0 +tensorboard>=2.2.0, <=2.8.0 +torchmetrics>=0.4.1, <=0.7.2 +pyDeprecate>=0.3.1, <0.4.0, <=0.3.2 +packaging>=17.0, <=21.3 +typing-extensions>=4.0.0, <=4.1.1 diff --git a/requirements/examples.txt b/requirements/examples.txt index 501d413de90ce..993087f2e1c88 100644 --- a/requirements/examples.txt +++ b/requirements/examples.txt @@ -1,3 +1,3 @@ -torchvision>=0.9.* -gym[classic_control]>=0.17.0 -ipython[all] +torchvision>=0.9.*, <=0.12.0 +gym[classic_control]>=0.17.0, <=0.23.1 +ipython[all] <=8.1.1 diff --git a/requirements/extra.txt b/requirements/extra.txt index 880fb47de9759..f319e6bdcea96 100644 --- a/requirements/extra.txt +++ b/requirements/extra.txt @@ -1,9 +1,11 @@ # extended list of package dependencies to reach full functionality -matplotlib>3.1 -torchtext>=0.9.* -omegaconf>=2.0.5 -hydra-core>=1.0.5 -jsonargparse[signatures]>=4.7.1 -gcsfs>=2021.5.0 -rich>=10.2.2,!=10.15.* +matplotlib>3.1, <=3.5.1 +# no need to install with [pytorch] as pytorch is already installed +horovod>=0.21.2,!=0.24.0, <=0.24.2 +torchtext>=0.9.*, <=0.12.0 +omegaconf>=2.0.5, <=2.2.0.dev1 +hydra-core>=1.0.5, <=1.2.0.dev2 +jsonargparse[signatures]>=4.7.1, <=4.7.1 +gcsfs>=2021.5.0, <=2022.2.0 +rich>=10.2.2,!=10.15.*, <=12.0.0 diff --git a/requirements/loggers.txt b/requirements/loggers.txt index 3b02aac28f6c0..9e64170e39f85 100644 --- a/requirements/loggers.txt +++ b/requirements/loggers.txt @@ -1,5 +1,7 @@ # all supported loggers -neptune-client>=0.10.0 -comet-ml>=3.1.12 -mlflow>=1.0.0 -wandb>=0.8.21 + +neptune-client>=0.10.0, <=0.15.2 +comet-ml>=3.1.12, <=3.28.2 +mlflow>=1.0.0, <=1.24.0 +test_tube>=0.7.5, <=0.7.5 +wandb>=0.8.21, <=0.12.11 From 049a9449e56682bc4532626c82360a3f0d6373f7 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 6 May 2022 06:18:42 +0200 Subject: [PATCH 02/12] unfreeze --- pytorch_lightning/setup_tools.py | 17 ++++++++++++----- requirements/extra.txt | 4 ++-- requirements/strategies.txt | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pytorch_lightning/setup_tools.py b/pytorch_lightning/setup_tools.py index da16d9475a41b..ebf540dadf3dc 100644 --- a/pytorch_lightning/setup_tools.py +++ b/pytorch_lightning/setup_tools.py @@ -19,7 +19,9 @@ _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__)) -def _load_requirements(path_dir: str, file_name: str = "requirements.txt", comment_char: str = "#") -> List[str]: +def _load_requirements( + path_dir: str, file_name: str = "requirements.txt", comment_char: str = "#", unfreeze: bool = True +) -> List[str]: """Load requirements from a file. >>> _load_requirements(_PROJECT_ROOT) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE @@ -31,12 +33,17 @@ def _load_requirements(path_dir: str, file_name: str = "requirements.txt", comme for ln in lines: # filer all comments if comment_char in ln: - ln = ln[: ln.index(comment_char)].strip() + ln = ln[: ln.index(comment_char)] + comment = ln[ln.index(comment_char) :] if comment_char in ln else "" + req = ln.strip() # skip directly installed dependencies - if ln.startswith("http") or "@http" in ln: + if not req or req.startswith("http") or "@http" in req: continue - if ln: # if requirement is not empty - reqs.append(ln) + # remove version restrictions unless they are strict + if unfreeze and "<" in req and "strict" not in comment: + req = re.sub(r"<= *[\d\.\*]+", "", req).strip() + req = req[:-1] if req[-1] == "," else req + reqs.append(req) return reqs diff --git a/requirements/extra.txt b/requirements/extra.txt index f319e6bdcea96..d02416ed30486 100644 --- a/requirements/extra.txt +++ b/requirements/extra.txt @@ -4,8 +4,8 @@ matplotlib>3.1, <=3.5.1 # no need to install with [pytorch] as pytorch is already installed horovod>=0.21.2,!=0.24.0, <=0.24.2 torchtext>=0.9.*, <=0.12.0 -omegaconf>=2.0.5, <=2.2.0.dev1 -hydra-core>=1.0.5, <=1.2.0.dev2 +omegaconf>=2.0.5, <=2.1.* +hydra-core>=1.0.5, <=1.1.* jsonargparse[signatures]>=4.7.1, <=4.7.1 gcsfs>=2021.5.0, <=2022.2.0 rich>=10.2.2,!=10.15.*, <=12.0.0 diff --git a/requirements/strategies.txt b/requirements/strategies.txt index 7846a297e30bc..1200fe4ada2fb 100644 --- a/requirements/strategies.txt +++ b/requirements/strategies.txt @@ -1,4 +1,4 @@ -fairscale>=0.4.5 +fairscale>=0.4.5, <=0.4.6 deepspeed<0.6.0 horovod>=0.21.2,!=0.24.0 # no need to install with [pytorch] as pytorch is already installed hivemind>=1.0.1; sys_platform == 'linux' From d1beae079844dad2e1558a8109549438952e2643 Mon Sep 17 00:00:00 2001 From: Jirka Date: Fri, 6 May 2022 06:28:47 +0200 Subject: [PATCH 03/12] dependabot --- .github/dependabot.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000000..060c9d253886e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,25 @@ +# Basic dependabot.yml file with +# minimum configuration for two package managers + +version: 2 +updates: + # Enable version updates for python + - package-ecosystem: "pip" + # Look for a `requirements` in the `root` directory + directory: "/" + # Check for updates once a week + schedule: + interval: "weekly" + # Labels on pull requests for version updates only + labels: + - "dependencies" + pull-request-branch-name: + # Separate sections of the branch name with a hyphen + # for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1` + separator: "-" + # Allow up to 10 open pull requests for pip dependencies + open-pull-requests-limit: 10 + reviewers: + - "octocat" + - "carmocca" + - "PyTorchLightning/teams/core-lightning" From 0b0a3349d6319c845a7d74495fc800e55bd4a462 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 6 May 2022 15:38:15 +0200 Subject: [PATCH 04/12] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Carlos Mocholí --- .github/dependabot.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 060c9d253886e..9b859a3e1b621 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,7 +12,7 @@ updates: interval: "weekly" # Labels on pull requests for version updates only labels: - - "dependencies" + - "ci" pull-request-branch-name: # Separate sections of the branch name with a hyphen # for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1` @@ -20,6 +20,5 @@ updates: # Allow up to 10 open pull requests for pip dependencies open-pull-requests-limit: 10 reviewers: - - "octocat" - "carmocca" - "PyTorchLightning/teams/core-lightning" From 2a48c45a30f67317fff83353b164269fee7f496d Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 9 May 2022 08:43:28 +0200 Subject: [PATCH 05/12] fix all req --- pytorch_lightning/setup_tools.py | 2 +- requirements/strategies.txt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/setup_tools.py b/pytorch_lightning/setup_tools.py index ebf540dadf3dc..65379131f6bb5 100644 --- a/pytorch_lightning/setup_tools.py +++ b/pytorch_lightning/setup_tools.py @@ -41,7 +41,7 @@ def _load_requirements( continue # remove version restrictions unless they are strict if unfreeze and "<" in req and "strict" not in comment: - req = re.sub(r"<= *[\d\.\*]+", "", req).strip() + req = re.sub(r",? *<= *[\d\.\*]+", "", req).strip() req = req[:-1] if req[-1] == "," else req reqs.append(req) return reqs diff --git a/requirements/strategies.txt b/requirements/strategies.txt index 1200fe4ada2fb..78268a9fbb6f1 100644 --- a/requirements/strategies.txt +++ b/requirements/strategies.txt @@ -1,4 +1,5 @@ fairscale>=0.4.5, <=0.4.6 deepspeed<0.6.0 -horovod>=0.21.2,!=0.24.0 # no need to install with [pytorch] as pytorch is already installed -hivemind>=1.0.1; sys_platform == 'linux' +# no need to install with [pytorch] as pytorch is already installed +horovod>=0.21.2,!=0.24.0, <=0.24.3 +hivemind>=1.0.1, <=1.0.1; sys_platform == 'linux' From 39e2745f0aa5d92009517ac8132e44733e84dbf9 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 9 May 2022 08:50:32 +0200 Subject: [PATCH 06/12] ... --- requirements/extra.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements/extra.txt b/requirements/extra.txt index d02416ed30486..271ef8c0b833d 100644 --- a/requirements/extra.txt +++ b/requirements/extra.txt @@ -1,8 +1,6 @@ # extended list of package dependencies to reach full functionality matplotlib>3.1, <=3.5.1 -# no need to install with [pytorch] as pytorch is already installed -horovod>=0.21.2,!=0.24.0, <=0.24.2 torchtext>=0.9.*, <=0.12.0 omegaconf>=2.0.5, <=2.1.* hydra-core>=1.0.5, <=1.1.* From 52f5f4a5964dc43b4c168b862d5abf1909e0914e Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 9 May 2022 09:12:48 +0200 Subject: [PATCH 07/12] use base --- .actions/assistant.py | 2 +- .github/dependabot.yml | 2 +- .github/workflows/ci_dockers.yml | 2 +- .github/workflows/ci_test-base.yml | 6 +++--- .github/workflows/ci_test-full.yml | 8 ++++---- .github/workflows/ci_test-slow.yml | 6 +++--- .github/workflows/code-checks.yml | 2 +- .github/workflows/docs-checks.yml | 10 +++++----- .github/workflows/release-pypi.yml | 2 +- dockers/base-conda/Dockerfile | 31 ++++++++++-------------------- dockers/base-cuda/Dockerfile | 3 +-- docs/source/conf.py | 7 ++++--- pytorch_lightning/setup_tools.py | 2 +- requirements.txt | 13 +------------ requirements/base.txt | 10 ++++++++++ requirements/devel-base.txt | 2 +- requirements/devel.txt | 2 +- setup.py | 2 +- 18 files changed, 50 insertions(+), 62 deletions(-) create mode 100644 requirements/base.txt diff --git a/.actions/assistant.py b/.actions/assistant.py index e8088599ae186..08a23b4a6675d 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -7,7 +7,7 @@ import fire REQUIREMENT_FILES = ( - "requirements.txt", + "requirements/base.txt", "requirements/extra.txt", "requirements/loggers.txt", "requirements/strategies.txt", diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9b859a3e1b621..b753ac35b4d2f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ updates: # Enable version updates for python - package-ecosystem: "pip" # Look for a `requirements` in the `root` directory - directory: "/" + directory: "/requirements" # Check for updates once a week schedule: interval: "weekly" diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml index 37d8df484c2d5..94d5be6901c5a 100644 --- a/.github/workflows/ci_dockers.yml +++ b/.github/workflows/ci_dockers.yml @@ -11,8 +11,8 @@ on: # Trigger the workflow on push or pull request, but only for the master bran - "dockers/**" - "!dockers/README.md" - "requirements/*" - - "environment.yml" - "requirements.txt" + - "environment.yml" - ".github/workflows/*docker*.yml" - ".github/workflows/events-nightly.yml" - "setup.py" diff --git a/.github/workflows/ci_test-base.yml b/.github/workflows/ci_test-base.yml index bcc37e6870003..221f92d26ad5e 100644 --- a/.github/workflows/ci_test-base.yml +++ b/.github/workflows/ci_test-base.yml @@ -45,7 +45,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements/base.txt') }} restore-keys: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.requires }}- - name: Install dependencies @@ -53,8 +53,8 @@ jobs: python --version python -m pip install --upgrade --user pip pip --version - pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade - pip install --requirement requirements/test.txt + pip install -r requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade + pip install -r requirements/test.txt pip list shell: bash diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 84ca60fd30f2e..f260c67069912 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -72,7 +72,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements/extra.txt') }} + key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-${{ hashFiles('requirements/*.txt') }} restore-keys: | ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}- @@ -89,11 +89,11 @@ jobs: run: | flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1) url=$(python -c "print('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html')" 2>&1) - pip install --requirement requirements.txt --upgrade $flag --find-links "https://download.pytorch.org/whl/${url}" + pip install -r requirements.txt --upgrade $flag --find-links "https://download.pytorch.org/whl/${url}" # adjust versions according installed Torch version python ./requirements/adjust-versions.py requirements/examples.txt - pip install --requirement requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade - pip install --requirement requirements/test.txt --upgrade + pip install -r requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade + pip install -r requirements/test.txt --upgrade pip list shell: bash diff --git a/.github/workflows/ci_test-slow.yml b/.github/workflows/ci_test-slow.yml index c3e1a41ae50d0..e2b5ca20b2f95 100644 --- a/.github/workflows/ci_test-slow.yml +++ b/.github/workflows/ci_test-slow.yml @@ -41,7 +41,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ hashFiles('requirements/base.txt') }} restore-keys: | ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}- @@ -49,8 +49,8 @@ jobs: run: | # adjust versions according installed Torch version python ./requirements/adjust-versions.py requirements.txt ${{ matrix.pytorch-version }} - pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade - pip install --requirement requirements/test.txt + pip install -r requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade + pip install -r requirements/test.txt pip list shell: bash diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index df34d2c47208d..8f4aa133372fb 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -25,7 +25,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.txt') }} restore-keys: | ${{ runner.os }}-pip- diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 7c9069e53dd67..80fd26209c194 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -31,7 +31,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements/base.txt') }} restore-keys: | ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}- @@ -42,9 +42,9 @@ jobs: pip --version pip install -q fire # python -m pip install --upgrade --user pip - pip install --requirement requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet - pip install --requirement requirements/devel-base.txt - pip install --requirement requirements/docs.txt + pip install -r requirements.txt --upgrade-strategy only-if-needed --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet + pip install -r requirements/devel-base.txt + pip install -r requirements/docs.txt pip list shell: bash @@ -74,7 +74,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.txt') }} restore-keys: | ${{ runner.os }}-pip- diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 4feefb750cef9..d7398e8e142e2 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -102,7 +102,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-${{ hashFiles('requirements/base.txt') }} restore-keys: ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}- - name: Install dependencies diff --git a/dockers/base-conda/Dockerfile b/dockers/base-conda/Dockerfile index d778cc3f9e8b7..eb388b88dfa7d 100644 --- a/dockers/base-conda/Dockerfile +++ b/dockers/base-conda/Dockerfile @@ -88,25 +88,19 @@ ENV \ PATH=/root/miniconda3/envs/${CONDA_ENV}/bin:$PATH \ LD_LIBRARY_PATH="/root/miniconda3/envs/${CONDA_ENV}/lib:$LD_LIBRARY_PATH" -COPY ./requirements.txt requirements.txt -COPY ./requirements/extra.txt requirements-extra.txt -COPY ./requirements/examples.txt requirements-examples.txt -COPY ./requirements/strategies.txt requirements-strategies.txt -COPY ./requirements/adjust-versions.py requirements_adjust_versions.py +COPY ./requirements/ ./requirements/ COPY ./.actions/assistant.py assistant.py RUN \ pip list | grep torch && \ python -c "import torch; print(torch.__version__)" && \ - python requirements_adjust_versions.py requirements-extra.txt && \ - python -c "print(' '.join([ln for ln in open('requirements-extra.txt').readlines() if 'horovod' in ln]))" > requirements_horovod.txt && \ pip install -q fire && \ - python assistant.py requirements_prune_pkgs "horovod" --req_files requirements-extra.txt && \ - python requirements_adjust_versions.py requirements-examples.txt && \ + python requirements/adjust-versions.py requirements/extra.txt && \ + python requirements/adjust-versions.py requirements/examples.txt && \ # Install remaining requirements - pip install -r requirements.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \ - pip install -r requirements-extra.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \ - pip install -r requirements-examples.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \ + pip install -r requirements/base.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \ + pip install -r requirements/extra.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \ + pip install -r requirements/examples.txt --no-cache-dir --find-links https://download.pytorch.org/whl/test/torch_test.html && \ rm assistant.py ENV \ @@ -121,11 +115,9 @@ ENV \ HOROVOD_WITH_MPI=1 RUN \ - HOROVOD_BUILD_CUDA_CC_LIST=${TORCH_CUDA_ARCH_LIST//";"/","} && \ - export HOROVOD_BUILD_CUDA_CC_LIST=${HOROVOD_BUILD_CUDA_CC_LIST//"."/""} && \ - cat requirements_horovod.txt && \ - pip install --no-cache-dir -r requirements-strategies.txt && \ - rm requirements* + HOROVOD_BUILD_CUDA_CC_LIST=${TORCH_CUDA_ARCH_LIST//";"/","} && \ + export HOROVOD_BUILD_CUDA_CC_LIST=${HOROVOD_BUILD_CUDA_CC_LIST//"."/""} && \ + pip install --no-cache-dir -r requirements/strategies.txt RUN \ CUDA_VERSION_MAJOR=$(python -c "import torch ; print(torch.version.cuda.split('.')[0])") && \ @@ -149,9 +141,6 @@ RUN \ python -c "import bagua_core; bagua_core.install_deps()" && \ python -c "import bagua; print(bagua.__version__)" -COPY requirements/check-avail-extras.py check-avail-extras.py -COPY requirements/check-avail-strategies.py check-avail-strategies.py - RUN \ # Show what we have pip --version && \ @@ -161,4 +150,4 @@ RUN \ python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__" && \ python check-avail-extras.py && \ python check-avail-strategies.py && \ - rm check-avail-*.py + rm -rf requirements/ diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile index c39670c1b8045..f38716b707287 100644 --- a/dockers/base-cuda/Dockerfile +++ b/dockers/base-cuda/Dockerfile @@ -70,7 +70,6 @@ RUN \ rm -rf /root/.cache && \ rm -rf /var/lib/apt/lists/* -COPY ./requirements.txt requirements.txt COPY ./requirements/ ./requirements/ COPY ./.actions/assistant.py assistant.py @@ -86,7 +85,7 @@ RUN \ CUDA_VERSION_MM=$(python -c "print(''.join('$CUDA_VERSION'.split('.')[:2]))") && \ pip config set global.cache-dir false && \ # set particular PyTorch version - python ./requirements/adjust-versions.py requirements.txt ${PYTORCH_VERSION} && \ + python ./requirements/adjust-versions.py requirements/base.txt ${PYTORCH_VERSION} && \ python ./requirements/adjust-versions.py requirements/extra.txt ${PYTORCH_VERSION} && \ python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \ # Install all requirements \ diff --git a/docs/source/conf.py b/docs/source/conf.py index f1118ccc74e07..1263b848d154a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -345,10 +345,11 @@ def package_list_from_file(file): MOCK_PACKAGES = [] if SPHINX_MOCK_REQUIREMENTS: MOCK_PACKAGES += ["fairscale"] + _path_require = lambda fname: os.path.join(PATH_ROOT, "requirements", fname) # mock also base packages when we are on RTD since we don't install them there - MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, "requirements.txt")) - MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, "requirements", "extra.txt")) - MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, "requirements", "loggers.txt")) + MOCK_PACKAGES += package_list_from_file(_path_require("base.txt")) + MOCK_PACKAGES += package_list_from_file(_path_require("extra.txt")) + MOCK_PACKAGES += package_list_from_file(_path_require("loggers.txt")) MOCK_PACKAGES = [PACKAGE_MAPPING.get(pkg, pkg) for pkg in MOCK_PACKAGES] autodoc_mock_imports = MOCK_PACKAGES diff --git a/pytorch_lightning/setup_tools.py b/pytorch_lightning/setup_tools.py index 65379131f6bb5..50b1091ee218e 100644 --- a/pytorch_lightning/setup_tools.py +++ b/pytorch_lightning/setup_tools.py @@ -20,7 +20,7 @@ def _load_requirements( - path_dir: str, file_name: str = "requirements.txt", comment_char: str = "#", unfreeze: bool = True + path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: bool = True ) -> List[str]: """Load requirements from a file. diff --git a/requirements.txt b/requirements.txt index 6761cca0967e8..7ea27fc77e63e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,2 @@ # the default package dependencies - - -numpy>=1.17.2, <=1.22.3 -torch>=1.8.*, <=1.11.0 -tqdm>=4.57.0, <=4.63.0 -PyYAML>=5.4, <=6.0 -fsspec[http]>=2021.05.0, !=2021.06.0, <=2022.2.0 -tensorboard>=2.2.0, <=2.8.0 -torchmetrics>=0.4.1, <=0.7.2 -pyDeprecate>=0.3.1, <0.4.0, <=0.3.2 -packaging>=17.0, <=21.3 -typing-extensions>=4.0.0, <=4.1.1 +-r ./requirements/base.txt diff --git a/requirements/base.txt b/requirements/base.txt new file mode 100644 index 0000000000000..4fe1fae302387 --- /dev/null +++ b/requirements/base.txt @@ -0,0 +1,10 @@ +numpy>=1.17.2, <=1.22.3 +torch>=1.8.*, <=1.11.0 +tqdm>=4.57.0, <=4.63.0 +PyYAML>=5.4, <=6.0 +fsspec[http]>=2021.05.0, !=2021.06.0, <=2022.2.0 +tensorboard>=2.2.0, <=2.8.0 +torchmetrics>=0.4.1, <=0.7.2 +pyDeprecate>=0.3.1, <0.4.0, <=0.3.2 +packaging>=17.0, <=21.3 +typing-extensions>=4.0.0, <=4.1.1 \ No newline at end of file diff --git a/requirements/devel-base.txt b/requirements/devel-base.txt index 3252ebf39f431..1169a78e387ca 100644 --- a/requirements/devel-base.txt +++ b/requirements/devel-base.txt @@ -1,5 +1,5 @@ # install all mandatory dependencies --r ../requirements.txt +-r ./base.txt # install all extra dependencies for full package testing -r ./extra.txt diff --git a/requirements/devel.txt b/requirements/devel.txt index 09c2699ccd1dd..ee8992c5ec18c 100644 --- a/requirements/devel.txt +++ b/requirements/devel.txt @@ -1,5 +1,5 @@ # install all mandatory dependencies --r ../requirements.txt +-r ./base.txt # install all extra dependencies for full package testing -r ./extra.txt diff --git a/setup.py b/setup.py index e5be7cd37df7f..11fc20b764037 100755 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def _load_py_module(fname, pkg="pytorch_lightning"): keywords=["deep learning", "pytorch", "AI"], python_requires=">=3.7", setup_requires=[], - install_requires=setup_tools._load_requirements(_PATH_ROOT), + install_requires=setup_tools._load_requirements(_PATH_REQUIRE), extras_require=extras, project_urls={ "Bug Tracker": "https://github.com/PyTorchLightning/pytorch-lightning/issues", From 03073e42fc106086ccc9d477d2c51b37b7391ac1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 07:14:09 +0000 Subject: [PATCH 08/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 4fe1fae302387..c4afd1c9c0273 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,4 +7,4 @@ tensorboard>=2.2.0, <=2.8.0 torchmetrics>=0.4.1, <=0.7.2 pyDeprecate>=0.3.1, <0.4.0, <=0.3.2 packaging>=17.0, <=21.3 -typing-extensions>=4.0.0, <=4.1.1 \ No newline at end of file +typing-extensions>=4.0.0, <=4.1.1 From 638170064bdb5a38f7efcb46eb8bea1506650ad9 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 9 May 2022 10:08:52 +0200 Subject: [PATCH 09/12] fix refs --- pytorch_lightning/setup_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/setup_tools.py b/pytorch_lightning/setup_tools.py index 50b1091ee218e..36cbb52fde9d8 100644 --- a/pytorch_lightning/setup_tools.py +++ b/pytorch_lightning/setup_tools.py @@ -24,7 +24,7 @@ def _load_requirements( ) -> List[str]: """Load requirements from a file. - >>> _load_requirements(_PROJECT_ROOT) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + >>> _load_requirements(os.path.join(_PROJECT_ROOT, "requirements")) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ['numpy...', 'torch...', ...] """ with open(os.path.join(path_dir, file_name)) as file: From 93857184bc5547a2f51e8d60e531378ccab348b7 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 9 May 2022 12:57:42 +0200 Subject: [PATCH 10/12] Apply suggestions from code review Co-authored-by: Akihiro Nitta --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index c4afd1c9c0273..1dddef6602ce7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,6 +5,6 @@ PyYAML>=5.4, <=6.0 fsspec[http]>=2021.05.0, !=2021.06.0, <=2022.2.0 tensorboard>=2.2.0, <=2.8.0 torchmetrics>=0.4.1, <=0.7.2 -pyDeprecate>=0.3.1, <0.4.0, <=0.3.2 +pyDeprecate>=0.3.1, <=0.3.2 packaging>=17.0, <=21.3 typing-extensions>=4.0.0, <=4.1.1 From fd935be0167b544b2068e886e45fc771562b404c Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 11 May 2022 16:29:47 +0200 Subject: [PATCH 11/12] Apply suggestions from code review --- pytorch_lightning/setup_tools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/setup_tools.py b/pytorch_lightning/setup_tools.py index 36cbb52fde9d8..94cef50579d5e 100644 --- a/pytorch_lightning/setup_tools.py +++ b/pytorch_lightning/setup_tools.py @@ -40,9 +40,8 @@ def _load_requirements( if not req or req.startswith("http") or "@http" in req: continue # remove version restrictions unless they are strict - if unfreeze and "<" in req and "strict" not in comment: + if unfreeze and "<=" in req and "strict" not in comment: req = re.sub(r",? *<= *[\d\.\*]+", "", req).strip() - req = req[:-1] if req[-1] == "," else req reqs.append(req) return reqs From 9e81eccef41310ef714c2cc83ca8dc3fdffec334 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 12 May 2022 13:42:52 +0200 Subject: [PATCH 12/12] dockers --- dockers/base-conda/Dockerfile | 4 ++-- dockers/base-cuda/Dockerfile | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/dockers/base-conda/Dockerfile b/dockers/base-conda/Dockerfile index eb388b88dfa7d..05227707b31fa 100644 --- a/dockers/base-conda/Dockerfile +++ b/dockers/base-conda/Dockerfile @@ -148,6 +148,6 @@ RUN \ pip list && \ python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \ python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__" && \ - python check-avail-extras.py && \ - python check-avail-strategies.py && \ + python requirements/check-avail-extras.py && \ + python requirements/check-avail-strategies.py && \ rm -rf requirements/ diff --git a/dockers/base-cuda/Dockerfile b/dockers/base-cuda/Dockerfile index 1136d1a955e46..321c02d8444ae 100644 --- a/dockers/base-cuda/Dockerfile +++ b/dockers/base-cuda/Dockerfile @@ -90,7 +90,6 @@ RUN \ python ./requirements/adjust-versions.py requirements/examples.txt ${PYTORCH_VERSION} && \ # Install all requirements \ pip install -r requirements/devel.txt --no-cache-dir --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html && \ - rm -rf requirements.* && \ rm assistant.py RUN \ @@ -120,8 +119,7 @@ RUN \ echo $HOROVOD_BUILD_CUDA_CC_LIST && \ cmake --version && \ pip install --no-cache-dir -r ./requirements/strategies.txt && \ - horovodrun --check-build && \ - rm -rf requirements/ + horovodrun --check-build RUN \ CUDA_VERSION_MAJOR=$(python -c "import torch; print(torch.version.cuda.split('.')[0])") && \ @@ -156,6 +154,6 @@ RUN \ pip list && \ python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \ python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__" && \ - python check-avail-extras.py && \ - python check-avail-strategies.py && \ - rm check-avail-*.py + python requirements/check-avail-extras.py && \ + python requirements/check-avail-strategies.py && \ + rm -rf requirements/