Add Circle-CI support (#38)

.circleci/config.yml

@@ -0,0 +1,131 @@
+# Python CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+version: 2
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - py3.7-pillow
+      - py3.7-pillow-simd
+      - py3.6-pillow
+
+common_config: &common_config
+  docker:
+    - image: continuumio/miniconda3
+  environment:
+    PYTHON_VERSION: 3.7
+    IMAGE_BACKEND: Pillow
+  working_directory: ~/repo
+  steps:
+    - checkout
+    - run:
+        name: Set python version
+        command: |
+          [[ -z "$PYTHON_VERSION" ]] && { echo "Python version not specified"; exit 1; }
+          sed -i  "s/python=[0-9\.]\+/python=${PYTHON_VERSION}/g" environment.yml
+
+    - restore_cache:
+        keys:
+          - dependencies-v2-{{ .Environment.IMAGE_BACKEND }}-{{ .Branch}}-{{ checksum "environment.yml" }}
+
+    - run:
+        name: Setup conda
+        command: |
+          conda config --set always_yes yes --set changeps1 no
+          conda update -q conda
+          conda info -a
+          echo 'source /opt/conda/etc/profile.d/conda.sh' >> $BASH_ENV
+
+    - run:
+        name: Install system deps
+        command: |
+          apt-get update
+          # Needed for building Pillow
+          apt-get install -y gcc g++
+          # These are OpenCV runtime dependencies
+          # See https://github.com/ray-project/ray/issues/923 for details.
+          apt-get install -y libsm6 libxext6 libxrender-dev
+          # Needed for running tests
+          apt-get install -y make
+
+    - run:
+        name: Make conda environment
+        command: |
+          cat environment.yml
+          if [[ ! -d /opt/conda/envs/torchvideo ]]; then
+            export CFLAGS="-mavx2"
+            conda env create -n torchvideo -f environment.yml
+            conda activate torchvideo
+            pip install mypy flake8 jupyter codecov
+            if [[ "$IMAGE_BACKEND" == "Pillow-SIMD" ]]; then
+              pip uninstall -y pillow && CC="${CC} -march=native" pip install --force-reinstall pillow-simd;
+            fi
+          fi
+
+    - save_cache:
+        key: dependencies-v2-{{ .Branch}}-{{ checksum "environment.yml" }}
+        paths:
+          - "/opt/conda/envs"
+
+    - run:
+        name: Lint torchvideo
+        command: |
+          conda activate torchvideo
+          mypy src/torchvideo
+
+    - run:
+        name: Install torchvideo
+        command: |
+          conda activate torchvideo
+          python setup.py install
+
+    - restore_cache:
+        keys:
+          - test-data-{{ checksum "tests/data/media/md5sums.txt" }}
+
+    - run:
+        name: Download data for system tests
+        command: |
+          conda activate torchvideo
+          if [[ -f /tmp/cache/big_buck_bunny_360p_5mb.mp4 ]]; then
+            cp /tmp/cache/big_buck_bunny_360p_5mb.mp4 tests/data/media;
+          fi
+          pushd tests/data/media; ./gen_test_media.sh; popd
+          if [[ ! -f /tmp/cache/big_buck_bunny_360p_5mb.mp4 ]]; then
+            mkdir -p /tmp/cache;
+            cp tests/data/media/big_buck_bunny_360p_5mb.mp4 /tmp/cache;
+          fi
+
+    - save_cache:
+        key: test-data-{{ checksum "tests/data/media/md5sums.txt" }}
+        paths:
+          - "/tmp/cache/"
+
+    - run:
+        name: Run tests
+        command: |
+          conda activate torchvideo
+          make test
+
+    - run:
+        name: Run notebooks
+        command: |
+          conda activate torchvideo
+          ./examples/run-notebooks.sh
+jobs:
+  py3.7-pillow:
+    <<: *common_config
+    environment:
+      PYTHON_VERSION: 3.7
+      IMAGE_BACKEND: Pillow
+  py3.7-pillow-simd:
+    <<: *common_config
+    environment:
+      PYTHON_VERSION: 3.7
+      IMAGE_BACKEND: Pillow-SIMD
+  py3.6-pillow:
+    <<: *common_config
+    environment:
+      PYTHON_VERSION: 3.6
+      IMAGE_BACKEND: Pillow

Makefile

@@ -9,15 +9,15 @@ test: unit_test functional_test
 
 .PHONY: unit_test
 unit_test:
-	COVERAGE_FILE=.coverage-unit PYTHONPATH=$(PYTHONPATH) pytest tests/unit
+	COVERAGE_FILE=.coverage-unit PYTHONPATH=$(PYTHONPATH) pytest tests/unit --junitxml=test-results-unit.xml
 
 .PHONY: functional_test
 functional_test:
-	COVERAGE_FILE=.coverage-functional PYTHONPATH=$(PYTHONPATH) pytest tests/functional
+	COVERAGE_FILE=.coverage-functional PYTHONPATH=$(PYTHONPATH) pytest tests/functional --junitxml=test-results-unit.xml
 
 .PHONY: doctest
 doctest:
-	COVERAGE_FILE=.coverage-doctest PYTHONPATH=$(PYTHONPATH) pytest --doctest-modules src
+	COVERAGE_FILE=.coverage-doctest PYTHONPATH=$(PYTHONPATH) pytest --doctest-modules src --junitxml=test-results-doctest.xml
 
 
 .PHONY: docs

dev-requirements.txt

@@ -1,7 +1,7 @@
-hypothesis==3.84.3
-mypy==0.650
-pyfakefs==3.5.5
-pytest==4.0.2
-pytest-cov==2.6.0
-pytest-xdist=1.24.0
-scipy==1.2.0
+hypothesis==4.32.3
+mypy==0.720
+pyfakefs==3.6
+pytest==5.0.1
+pytest-cov==2.7.1
+pytest-xdist==1.29.0
+scipy==1.3.1

environment.yml

@@ -11,7 +11,7 @@ dependencies:
   - pandas==0.25.0
   - pytorch=1.2.0
   - python=3.7.3
-  - pip=19.2.1
+  - pip
   - torchvision=0.4.0
   - pre_commit==1.18.0
   - pip:

examples/run-notebooks.sh

@@ -3,5 +3,7 @@ NOTEBOOK_DIR="$(dirname $(readlink -f $0))"
 set -ex
 
 for f in "$NOTEBOOK_DIR"/*.ipynb; do
-    jupyter nbconvert --to notebook --execute --inplace "$f"
+    # Some cells can take a long time to execute, by default jupyter
+    # only has a 30s timeout, so we increase this
+    jupyter nbconvert --ExecutePreprocessor.timeout=120 --to notebook --execute --inplace "$f"
 done

src/torchvideo/datasets/video_folder_dataset.py

@@ -1,15 +1,15 @@
-import torch
 from pathlib import Path
-from typing import Union, Optional, Callable, Tuple, List, Iterator
+from typing import Any, Callable, Iterator, List, Optional, Tuple, Union
 
 from PIL.Image import Image
 
 from torchvideo.internal.readers import _get_videofile_frame_count, _is_video_file
 from torchvideo.samplers import FrameSampler, _default_sampler
 from torchvideo.transforms import PILVideoToTensor
-from .types import Label, empty_label, PILVideoTransform
-from .label_sets import LabelSet
+
 from .helpers import invoke_transform
+from .label_sets import LabelSet
+from .types import Label, PILVideoTransform, empty_label
 from .video_dataset import VideoDataset
 
 
@@ -65,9 +65,7 @@ def video_ids(self):
 
     # TODO: This is very similar to ImageFolderVideoDataset consider merging into
     #  VideoDataset
-    def __getitem__(
-        self, index: int
-    ) -> Union[torch.Tensor, Tuple[torch.Tensor, Label]]:
+    def __getitem__(self, index: int) -> Union[Any, Tuple[Any, Label]]:
         video_file = self._video_paths[index]
         video_length = self.video_lengths[index]
         frames_idx = self.sampler.sample(video_length)

src/torchvideo/transforms/functional/normalize.py

@@ -38,7 +38,7 @@ def normalize(
         tensor = tensor.clone()
 
     statistic_shape = tuple([-1] + [1] * ((tensor.dim() - 1)))
-    mean = torch.tensor(mean, dtype=torch.float32).view(*statistic_shape)
-    std = torch.tensor(std, dtype=torch.float32).view(*statistic_shape)
-    tensor.sub_(mean).div_(std)
+    mean_: torch.Tensor = torch.tensor(mean, dtype=torch.float32).view(*statistic_shape)
+    std_: torch.Tensor = torch.tensor(std, dtype=torch.float32).view(*statistic_shape)
+    tensor.sub_(mean_).div_(std_)  # type: ignore
     return tensor

tests/data/media/md5sums.txt

@@ -0,0 +1 @@
+e68758bdd9a96d2b2c276477bc4f2fbc  big_buck_bunny_360p_5mb.mp4