Switch to pytest in CI and add unstable dependency environment

.codebeatignore

@@ -1,2 +1 @@
 satpy/version.py
-versioneer.py

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,7 +2,7 @@
 <!-- For works in progress choose "Create draft pull request" from the drop-down green button. -->
 
  - [ ] Closes #xxxx <!-- remove if there is no corresponding issue, which should only be the case for minor changes -->
- - [ ] Tests added and test suite added to parent suite <!-- for all bug fixes or enhancements -->
+ - [ ] Tests added <!-- for all bug fixes or enhancements -->
  - [ ] Tests passed <!-- for all non-documentation changes -->
  - [ ] Passes ``flake8 satpy`` <!-- remove if you did not edit any Python files -->
  - [ ] Fully documented <!-- remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later -->

.travis.yml

@@ -3,15 +3,18 @@ env:
     global:
         # Set defaults to avoid repeating in most cases
         - PYTHON_VERSION=$TRAVIS_PYTHON_VERSION
-        - NUMPY_VERSION=1.17
+        - NUMPY_VERSION=stable
         - MAIN_CMD='python setup.py'
-        - CONDA_DEPENDENCIES='xarray dask distributed toolz Cython sphinx cartopy pillow matplotlib scipy pyyaml pyproj=2.4.1 pyresample coveralls coverage codecov behave netcdf4 h5py h5netcdf gdal rasterio imageio pyhdf mock libtiff geoviews zarr six python-eccodes'
+        - CONDA_DEPENDENCIES='xarray dask distributed toolz Cython sphinx cartopy pillow matplotlib scipy pyyaml
+          pyproj=2.4.1 pyresample coveralls coverage codecov behave netcdf4 h5py h5netcdf gdal rasterio imageio pyhdf
+          mock libtiff geoviews zarr python-eccodes geoviews pytest pytest-cov'
         - PIP_DEPENDENCIES='trollsift trollimage pyspectral pyorbital libtiff'
         - SETUP_XVFB=False
         - EVENT_TYPE='push pull_request'
         - SETUP_CMD='test'
         - CONDA_CHANNELS='conda-forge'
         - CONDA_CHANNEL_PRIORITY='strict'
+        - UNSTABLE_DEPS=False
 matrix:
   include:
   - env: PYTHON_VERSION=3.8
@@ -24,11 +27,41 @@ matrix:
   - env: PYTHON_VERSION=3.7
     os: osx
     language: generic
+  # allowed to fail:
+  - os: linux
+    env:
+      - PYTHON_VERSION=3.8
+      - UNSTABLE_DEPS=True
+
+  allow_failures:
+  - os: linux
+    env:
+      - PYTHON_VERSION=3.8
+      - UNSTABLE_DEPS=True
 install:
     - git clone --depth 1 git://github.com/astropy/ci-helpers.git
     - source ci-helpers/travis/setup_conda.sh
+    - if [ "$UNSTABLE_DEPS" == "True" ]; then
+        python -m pip install
+        -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com
+        --no-deps --pre --upgrade
+        matplotlib
+        numpy
+        pandas
+        scipy;
+        python -m pip install
+        --no-deps --upgrade
+        git+https://github.com/dask/dask
+        git+https://github.com/dask/distributed
+        git+https://github.com/zarr-developers/zarr
+        git+https://github.com/Unidata/cftime
+        git+https://github.com/mapbox/rasterio
+        git+https://github.com/pydata/bottleneck
+        git+https://github.com/pydata/xarray;
+      fi
+    - pip install --no-deps -e .
 script:
-- coverage run --source=satpy setup.py test
+- pytest --cov=satpy satpy/tests
 - coverage run -a --source=satpy -m behave satpy/tests/features --tags=-download
 after_success:
 - if [[ $PYTHON_VERSION == 3.8 ]]; then coveralls; codecov; fi

appveyor.yml

@@ -3,7 +3,7 @@ environment:
     PYTHON: "C:\\conda"
     MINICONDA_VERSION: "latest"
     CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd"
-    CONDA_DEPENDENCIES: "xarray dask distributed toolz Cython sphinx cartopy pillow matplotlib scipy pyyaml pyproj pyresample coverage netcdf4 h5py h5netcdf gdal rasterio imageio pyhdf mock libtiff zarr six"
+    CONDA_DEPENDENCIES: "xarray dask distributed toolz Cython sphinx cartopy pillow matplotlib scipy pyyaml pyproj pyresample coverage netcdf4 h5py h5netcdf gdal rasterio imageio pyhdf libtiff geoviews zarr pytest pytest-cov"
     PIP_DEPENDENCIES: "trollsift trollimage pyspectral pyorbital libtiff"
     CONDA_CHANNELS: "conda-forge"
 
@@ -21,12 +21,13 @@ install:
     - "git clone --depth 1 git://github.com/astropy/ci-helpers.git"
     - "powershell ci-helpers/appveyor/install-miniconda.ps1"
     - "conda activate test"
+    - "pip install --no-deps -e ."
 
 build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
   # Build the compiled extension and run the project tests
-  - "%CMD_IN_ENV% python setup.py test"
+  - "%CMD_IN_ENV% pytest --cov=satpy satpy/tests"
 
 after_test:
   # If tests are successful, create a whl package for the project.

doc/source/conf.py

@@ -260,6 +260,7 @@ def __getattr__(cls, name):
     'pyorbital': ('https://pyorbital.readthedocs.io/en/stable', None),
     'pyproj': ('https://pyproj4.github.io/pyproj/dev', None),
     'pyresample': ('https://pyresample.readthedocs.io/en/stable', None),
+    'pytest': ('https://docs.pytest.org/en/stable/', None),
     'python': ('https://docs.python.org/3', None),
     'scipy': ('https://docs.scipy.org/doc/scipy/reference', None),
     'trollimage': ('https://trollimage.readthedocs.io/en/stable', None),

doc/source/dev_guide/index.rst

@@ -44,10 +44,11 @@ clone your fork. The package can then be installed in development by doing::
 Running tests
 =============
 
-Satpy tests are written using the python :mod:`unittest` module and the tests
-can be executed by running::
+Satpy tests are written using the python :mod:`unittest` module and the
+third-party :doc:`pytest <pytest:index>` package. Satpy tests can be executed by
+running::
 
-    python setup.py test
+    pytest satpy/tests
 
 Documentation
 =============

satpy/composites/__init__.py

@@ -25,7 +25,6 @@
 
 import dask.array as da
 import numpy as np
-import six
 import xarray as xr
 import yaml
 
@@ -818,7 +817,7 @@ def _get_sensors(self, projectables):
         for projectable in projectables:
             current_sensor = projectable.attrs.get("sensor", None)
             if current_sensor:
-                if isinstance(current_sensor, (str, bytes, six.text_type)):
+                if isinstance(current_sensor, (str, bytes)):
                     sensor.add(current_sensor)
                 else:
                     sensor |= current_sensor

satpy/config.py

@@ -20,11 +20,11 @@
 import glob
 import logging
 import os
+import configparser
 from collections.abc import Mapping
 from collections import OrderedDict
 
 import yaml
-from six.moves import configparser
 
 try:
     from yaml import UnsafeLoader

satpy/dataset.py

@@ -55,16 +55,7 @@ def average_datetimes(dt_list):
     Returns: Average datetime as a datetime object
 
     """
-    if sys.version_info < (3, 3):
-        # timestamp added in python 3.3
-        import time
-
-        def timestamp_func(dt):
-            return time.mktime(dt.timetuple())
-    else:
-        timestamp_func = datetime.timestamp
-
-    total = [timestamp_func(dt) for dt in dt_list]
+    total = [datetime.timestamp(dt) for dt in dt_list]
     return datetime.fromtimestamp(sum(total) / len(total))
 
 

satpy/readers/__init__.py

@@ -22,7 +22,6 @@
 import os
 from datetime import datetime, timedelta
 
-import six
 import yaml
 
 try:
@@ -35,11 +34,6 @@
 from satpy.dataset import DATASET_KEYS, DatasetID
 from satpy import CALIBRATION_ORDER
 
-try:
-    import configparser  # noqa
-except ImportError:
-    from six.moves import configparser  # noqa
-
 LOG = logging.getLogger(__name__)
 
 
@@ -209,7 +203,7 @@ def get_key(key, key_container, num_results=1, best=True,
         # we want this ID to act as a query so we set modifiers to None
         # meaning "we don't care how many modifiers it has".
         key = DatasetID(wavelength=key, modifiers=None)
-    elif isinstance(key, (str, six.text_type)):
+    elif isinstance(key, str):
         # ID should act as a query (see wl comment above)
         key = DatasetID(name=key, modifiers=None)
     elif not isinstance(key, DatasetID):
@@ -331,7 +325,7 @@ def __setitem__(self, key, value):
             try:
                 key = self.get_key(key)
             except KeyError:
-                if isinstance(old_key, (str, six.text_type)):
+                if isinstance(old_key, str):
                     new_name = old_key
                 else:
                     new_name = d.get("name")

satpy/readers/file_handlers.py

@@ -20,14 +20,13 @@
 from abc import ABCMeta
 
 import numpy as np
-import six
 from pathlib import PurePath
 
 from pyresample.geometry import SwathDefinition
 from satpy.dataset import combine_metadata
 
 
-class BaseFileHandler(six.with_metaclass(ABCMeta, object)):
+class BaseFileHandler(metaclass=ABCMeta):
     """Base file handler."""
 
     def __init__(self, filename, filename_info, filetype_info):

satpy/readers/hdf4_utils.py

@@ -23,7 +23,6 @@
 import dask.array as da
 import xarray as xr
 import numpy as np
-import six
 
 from satpy import CHUNK_SIZE
 from satpy.readers.file_handlers import BaseFileHandler
@@ -67,7 +66,7 @@ def __init__(self, filename, filename_info, filetype_info):
         del file_handle
 
     def _collect_attrs(self, name, attrs):
-        for key, value in six.iteritems(attrs):
+        for key, value in attrs.items():
             value = np.squeeze(value)
             if issubclass(value.dtype.type, (np.string_, np.unicode_)) and not value.shape:
                 value = value.item()  # convert to scalar

satpy/readers/hdf5_utils.py

@@ -20,7 +20,6 @@
 import logging
 import h5py
 import numpy as np
-import six
 import xarray as xr
 import dask.array as da
 
@@ -52,7 +51,7 @@ def __init__(self, filename, filename_info, filetype_info):
         file_handle.close()
 
     def _collect_attrs(self, name, attrs):
-        for key, value in six.iteritems(attrs):
+        for key, value in attrs.items():
             value = np.squeeze(value)
             fc_key = "{}/attr/{}".format(name, key)
             try:

satpy/readers/hrit_base.py

@@ -32,7 +32,7 @@
 from datetime import timedelta
 from tempfile import gettempdir
 import os
-from six import BytesIO
+from io import BytesIO
 from subprocess import Popen, PIPE
 
 import numpy as np

satpy/readers/utils.py

@@ -22,15 +22,13 @@
 from contextlib import closing
 import tempfile
 import bz2
-import sys
 import os
 import shutil
 import numpy as np
 import pyproj
-from six import BytesIO
+from io import BytesIO
 from subprocess import Popen, PIPE
 from pyresample.geometry import AreaDefinition
-from pyresample.boundary import AreaDefBoundary, Boundary
 
 from satpy import CHUNK_SIZE
 
@@ -157,33 +155,6 @@ def get_geostationary_bounding_box(geos_area, nb_points=50):
     return _lonlat_from_geos_angle(x, y, geos_area)
 
 
-def get_area_slices(data_area, area_to_cover):
-    """Compute the slice to read from an *area* based on an *area_to_cover*."""
-    if data_area.proj_dict['proj'] != 'geos':
-        raise NotImplementedError('Only geos supported')
-
-    # Intersection only required for two different projections
-    if area_to_cover.proj_dict['proj'] == data_area.proj_dict['proj']:
-        LOGGER.debug('Projections for data and slice areas are'
-                     ' identical: {}'.format(area_to_cover.proj_dict['proj']))
-        # Get xy coordinates
-        llx, lly, urx, ury = area_to_cover.area_extent
-        x, y = data_area.get_xy_from_proj_coords([llx, urx], [lly, ury])
-
-        return slice(x[0], x[1] + 1), slice(y[1], y[0] + 1)
-
-    data_boundary = Boundary(*get_geostationary_bounding_box(data_area))
-
-    area_boundary = AreaDefBoundary(area_to_cover, 100)
-    intersection = data_boundary.contour_poly.intersection(
-        area_boundary.contour_poly)
-
-    x, y = data_area.get_xy_from_lonlat(np.rad2deg(intersection.lon),
-                                        np.rad2deg(intersection.lat))
-
-    return slice(min(x), max(x) + 1), slice(min(y), max(y) + 1)
-
-
 def get_sub_area(area, xslice, yslice):
     """Apply slices to the area_extent and size of the area."""
     new_area_extent = ((area.pixel_upper_left[0] +
@@ -207,39 +178,39 @@ def unzip_file(filename):
     if filename.endswith('bz2'):
         fdn, tmpfilepath = tempfile.mkstemp()
         LOGGER.info("Using temp file for BZ2 decompression: %s", tmpfilepath)
-        # If in python 3, try pbzip2
-        if sys.version_info.major >= 3:
-            pbzip = which('pbzip2')
-            # Run external pbzip2
-            if pbzip is not None:
-                n_thr = os.environ.get('OMP_NUM_THREADS')
-                if n_thr:
-                    runner = [pbzip,
-                              '-dc',
-                              '-p'+str(n_thr),
-                              filename]
-                else:
-                    runner = [pbzip,
-                              '-dc',
-                              filename]
-                p = Popen(runner, stdout=PIPE, stderr=PIPE)
-                stdout = BytesIO(p.communicate()[0])
-                status = p.returncode
-                if status != 0:
-                    raise IOError("pbzip2 error '%s', failed, status=%d"
-                                  % (filename, status))
-                with closing(os.fdopen(fdn, 'wb')) as ofpt:
-                    try:
-                        stdout.seek(0)
-                        shutil.copyfileobj(stdout, ofpt)
-                    except IOError:
-                        import traceback
-                        traceback.print_exc()
-                        LOGGER.info("Failed to read bzipped file %s",
-                                    str(filename))
-                        os.remove(tmpfilepath)
-                        raise
-                return tmpfilepath
+        # try pbzip2
+        pbzip = which('pbzip2')
+        # Run external pbzip2
+        if pbzip is not None:
+            n_thr = os.environ.get('OMP_NUM_THREADS')
+            if n_thr:
+                runner = [pbzip,
+                          '-dc',
+                          '-p'+str(n_thr),
+                          filename]
+            else:
+                runner = [pbzip,
+                          '-dc',
+                          filename]
+            p = Popen(runner, stdout=PIPE, stderr=PIPE)
+            stdout = BytesIO(p.communicate()[0])
+            status = p.returncode
+            if status != 0:
+                raise IOError("pbzip2 error '%s', failed, status=%d"
+                              % (filename, status))
+            with closing(os.fdopen(fdn, 'wb')) as ofpt:
+                try:
+                    stdout.seek(0)
+                    shutil.copyfileobj(stdout, ofpt)
+                except IOError:
+                    import traceback
+                    traceback.print_exc()
+                    LOGGER.info("Failed to read bzipped file %s",
+                                str(filename))
+                    os.remove(tmpfilepath)
+                    raise
+            return tmpfilepath
+
         # Otherwise, fall back to the original method
         bz2file = bz2.BZ2File(filename)
         with closing(os.fdopen(fdn, 'wb')) as ofpt: