From 5b03572cf078537a5007d2b9a6b2b4c5d2880447 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 29 Sep 2021 08:26:43 +0200 Subject: [PATCH 1/8] add torchdata as prototype dependency --- .circleci/config.yml | 1 + .circleci/config.yml.in | 1 + .circleci/unittest/linux/scripts/environment.yml | 1 + .circleci/unittest/windows/scripts/environment.yml | 1 + torchvision/prototype/datasets/__init__.py | 10 ++++++++++ 5 files changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c91a6973d16..f1dabc9d7a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -200,6 +200,7 @@ jobs: pip install --user --progress-bar off mypy pip install --user --progress-bar off types-requests pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip install --user --progress-bar off git+https://github.com/pytorch/data.git pip install --user --progress-bar off --editable . mypy --config-file mypy.ini diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 77b8a1468a3..c8dd8a5cbc7 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -200,6 +200,7 @@ jobs: pip install --user --progress-bar off mypy pip install --user --progress-bar off types-requests pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip install --user --progress-bar off git+https://github.com/pytorch/data.git pip install --user --progress-bar off --editable . mypy --config-file mypy.ini diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index ce3fe3be6e6..28474e29ba7 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -14,3 +14,4 @@ dependencies: - pillow >=5.3.0, !=8.3.0 - scipy - av + - git+https://github.com/pytorch/data.git diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 5b0d3f5462d..ab2e1ecb1ce 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -15,3 +15,4 @@ dependencies: - scipy - av - dataclasses + - git+https://github.com/pytorch/data.git diff --git a/torchvision/prototype/datasets/__init__.py b/torchvision/prototype/datasets/__init__.py index 18217b4bd3d..4170bb546f7 100644 --- a/torchvision/prototype/datasets/__init__.py +++ b/torchvision/prototype/datasets/__init__.py @@ -1,3 +1,13 @@ +try: + import torchdata +except (ModuleNotFoundError, TypeError) as error: + raise ModuleNotFoundError( + "`torchvision.prototype.datasets` depends on PyTorch's `torchdata` (https://github.com/pytorch/data). " + "You can install it with `pip install git+https://github.com/pytorch/data.git`. " + "Note that you cannot install it with `pip install torchdata`, since this is another package." + ) from error + + from ._home import home from . import decoder, utils From 37671cbabe4372d20bc8931bbf2fc0baeced8528 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 29 Sep 2021 08:34:36 +0200 Subject: [PATCH 2/8] ignore torchdata for mypy --- mypy.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy.ini b/mypy.ini index de04d6d173e..dac60e11ce0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -67,3 +67,7 @@ ignore_missing_imports = True [mypy-defusedxml.*] ignore_missing_imports = True + +[mypy-torchdata.*] + +ignore_missing_imports = True From 512f01874b52d9eca908f434f1ada8bbf1656527 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 29 Sep 2021 13:31:24 +0200 Subject: [PATCH 3/8] move git dependency out of conda environment file --- .circleci/unittest/linux/scripts/environment.yml | 1 - .circleci/unittest/linux/scripts/setup_env.sh | 1 + .circleci/unittest/windows/scripts/environment.yml | 1 - .circleci/unittest/windows/scripts/setup_env.sh | 1 + 4 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index 28474e29ba7..ce3fe3be6e6 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -14,4 +14,3 @@ dependencies: - pillow >=5.3.0, !=8.3.0 - scipy - av - - git+https://github.com/pytorch/data.git diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 773bd78f202..8ec7d32aed2 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -43,3 +43,4 @@ fi conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" conda env update --file "${this_dir}/environment.yml" --prune +pip install git+https://github.com/pytorch/data.git diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index ab2e1ecb1ce..5b0d3f5462d 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -15,4 +15,3 @@ dependencies: - scipy - av - dataclasses - - git+https://github.com/pytorch/data.git diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh index b0b70631112..7629c5ad14e 100644 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -37,3 +37,4 @@ conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" conda env update --file "${this_dir}/environment.yml" --prune +pip install git+https://github.com/pytorch/data.git From ceb8648fbe9159ee0e26b5066b1f288b2992b56b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 29 Sep 2021 13:45:57 +0200 Subject: [PATCH 4/8] try unsetting PYTORCH_VERSION env var --- .circleci/unittest/linux/scripts/setup_env.sh | 4 ++++ .circleci/unittest/windows/scripts/setup_env.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 8ec7d32aed2..5cc520842e3 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -43,4 +43,8 @@ fi conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" conda env update --file "${this_dir}/environment.yml" --prune + +PYTORCH_VERSION_TMP=PYTORCH_VERSION +unset PYTORCH_VERSION pip install git+https://github.com/pytorch/data.git +PYTORCH_VERSION=PYTORCH_VERSION_TMP diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh index 7629c5ad14e..7d83dd73ad8 100644 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -37,4 +37,8 @@ conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" conda env update --file "${this_dir}/environment.yml" --prune + +PYTORCH_VERSION_TMP=PYTORCH_VERSION +unset PYTORCH_VERSION pip install git+https://github.com/pytorch/data.git +PYTORCH_VERSION=PYTORCH_VERSION_TMP From e208a7fc2aa434d1e904be15fef5316a91f60b5b Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 29 Sep 2021 17:23:05 +0200 Subject: [PATCH 5/8] cleanup --- .circleci/unittest/linux/scripts/setup_env.sh | 5 +---- .circleci/unittest/windows/scripts/setup_env.sh | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index 5cc520842e3..eed5c585d56 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -44,7 +44,4 @@ fi conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" conda env update --file "${this_dir}/environment.yml" --prune -PYTORCH_VERSION_TMP=PYTORCH_VERSION -unset PYTORCH_VERSION -pip install git+https://github.com/pytorch/data.git -PYTORCH_VERSION=PYTORCH_VERSION_TMP +pip install git+https://github.com/pytorch/data.git@main diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh index 7d83dd73ad8..446b5aaef1b 100644 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -38,7 +38,4 @@ conda activate "${env_dir}" printf "* Installing dependencies (except PyTorch)\n" conda env update --file "${this_dir}/environment.yml" --prune -PYTORCH_VERSION_TMP=PYTORCH_VERSION -unset PYTORCH_VERSION -pip install git+https://github.com/pytorch/data.git -PYTORCH_VERSION=PYTORCH_VERSION_TMP +pip install git+https://github.com/pytorch/data.git@main From be625c98c92181ce3670171dd411f45529f67ae9 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 30 Sep 2021 10:27:47 +0200 Subject: [PATCH 6/8] move installation of torchdata after installtion of PyTorch --- .circleci/config.yml.in | 2 +- .circleci/unittest/linux/scripts/install.sh | 4 ++++ .circleci/unittest/linux/scripts/setup_env.sh | 2 -- .circleci/unittest/windows/scripts/install.sh | 4 ++++ .circleci/unittest/windows/scripts/setup_env.sh | 2 -- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index c8dd8a5cbc7..250de766cc1 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -200,7 +200,7 @@ jobs: pip install --user --progress-bar off mypy pip install --user --progress-bar off types-requests pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip install --user --progress-bar off git+https://github.com/pytorch/data.git + pip install --user --progress-bar off git+https://github.com/pytorch/data.git@main pip install --user --progress-bar off --editable . mypy --config-file mypy.ini diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index e9b9e80c077..50607b8f0a2 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -26,6 +26,10 @@ fi printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest +printf "Installing torchdata from source" +pip install git+https://github.com/pytorch/data.git@main + + if [ $PYTHON_VERSION == "3.6" ]; then printf "Installing minimal PILLOW version\n" # Install the minimal PILLOW version. Otherwise, let setup.py install the latest diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh index eed5c585d56..773bd78f202 100755 --- a/.circleci/unittest/linux/scripts/setup_env.sh +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -43,5 +43,3 @@ fi conda install -y -c pytorch "ffmpeg${FFMPEG_PIN}" conda env update --file "${this_dir}/environment.yml" --prune - -pip install git+https://github.com/pytorch/data.git@main diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index e4756179371..f2d44cd9c79 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -29,6 +29,10 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}" # conda-forge channel is required for cudatoolkit 11.1 on Windows, see https://github.com/pytorch/vision/issues/4458 conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest +printf "Installing torchdata from source" +pip install git+https://github.com/pytorch/data.git@main + + if [ $PYTHON_VERSION == "3.6" ]; then printf "Installing minimal PILLOW version\n" # Install the minimal PILLOW version. Otherwise, let setup.py install the latest diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh index 446b5aaef1b..b0b70631112 100644 --- a/.circleci/unittest/windows/scripts/setup_env.sh +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -37,5 +37,3 @@ conda activate "${env_dir}" # 3. Install Conda dependencies printf "* Installing dependencies (except PyTorch)\n" conda env update --file "${this_dir}/environment.yml" --prune - -pip install git+https://github.com/pytorch/data.git@main From 9921f77fe252c0524e38a5eb260c1d5c7428138d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 30 Sep 2021 10:30:45 +0200 Subject: [PATCH 7/8] regenerate circlci config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f1dabc9d7a9..0a8c25b23d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -200,7 +200,7 @@ jobs: pip install --user --progress-bar off mypy pip install --user --progress-bar off types-requests pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip install --user --progress-bar off git+https://github.com/pytorch/data.git + pip install --user --progress-bar off git+https://github.com/pytorch/data.git@main pip install --user --progress-bar off --editable . mypy --config-file mypy.ini From 30aaac8385200e602b9515be5f1204bdc81bc222 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 30 Sep 2021 10:42:21 +0200 Subject: [PATCH 8/8] remove branch from torchdata download link --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- .circleci/unittest/linux/scripts/install.sh | 2 +- .circleci/unittest/windows/scripts/install.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a8c25b23d7..f1dabc9d7a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -200,7 +200,7 @@ jobs: pip install --user --progress-bar off mypy pip install --user --progress-bar off types-requests pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip install --user --progress-bar off git+https://github.com/pytorch/data.git@main + pip install --user --progress-bar off git+https://github.com/pytorch/data.git pip install --user --progress-bar off --editable . mypy --config-file mypy.ini diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 250de766cc1..c8dd8a5cbc7 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -200,7 +200,7 @@ jobs: pip install --user --progress-bar off mypy pip install --user --progress-bar off types-requests pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip install --user --progress-bar off git+https://github.com/pytorch/data.git@main + pip install --user --progress-bar off git+https://github.com/pytorch/data.git pip install --user --progress-bar off --editable . mypy --config-file mypy.ini diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 50607b8f0a2..2bf3684ead1 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -27,7 +27,7 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install -y -c "pytorch-${UPLOAD_CHANNEL}" "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest printf "Installing torchdata from source" -pip install git+https://github.com/pytorch/data.git@main +pip install git+https://github.com/pytorch/data.git if [ $PYTHON_VERSION == "3.6" ]; then diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index f2d44cd9c79..49ef8224b59 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -30,7 +30,7 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}" conda install -y -c "pytorch-${UPLOAD_CHANNEL}" -c conda-forge "pytorch-${UPLOAD_CHANNEL}"::pytorch "${cudatoolkit}" pytest printf "Installing torchdata from source" -pip install git+https://github.com/pytorch/data.git@main +pip install git+https://github.com/pytorch/data.git if [ $PYTHON_VERSION == "3.6" ]; then