Merge pull request #2 from pytroll/master

Update local to latest remote

.codebeatignore

@@ -1,2 +1 @@
 satpy/version.py
-versioneer.py

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,7 +2,7 @@
 <!-- For works in progress choose "Create draft pull request" from the drop-down green button. -->
 
  - [ ] Closes #xxxx <!-- remove if there is no corresponding issue, which should only be the case for minor changes -->
- - [ ] Tests added and test suite added to parent suite <!-- for all bug fixes or enhancements -->
+ - [ ] Tests added <!-- for all bug fixes or enhancements -->
  - [ ] Tests passed <!-- for all non-documentation changes -->
  - [ ] Passes ``flake8 satpy`` <!-- remove if you did not edit any Python files -->
  - [ ] Fully documented <!-- remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later -->

.travis.yml

@@ -3,15 +3,18 @@ env:
     global:
         # Set defaults to avoid repeating in most cases
         - PYTHON_VERSION=$TRAVIS_PYTHON_VERSION
-        - NUMPY_VERSION=1.17
+        - NUMPY_VERSION=stable
         - MAIN_CMD='python setup.py'
-        - CONDA_DEPENDENCIES='xarray dask distributed toolz Cython sphinx cartopy pillow matplotlib scipy pyyaml pyproj=2.4.1 pyresample coveralls coverage codecov behave netcdf4 h5py h5netcdf gdal rasterio imageio pyhdf mock libtiff geoviews zarr six python-eccodes'
+        - CONDA_DEPENDENCIES='xarray dask distributed toolz Cython sphinx cartopy pillow matplotlib scipy pyyaml
+          pyproj pyresample coveralls coverage codecov behave netcdf4 h5py h5netcdf gdal rasterio imageio pyhdf
+          mock libtiff geoviews zarr python-eccodes geoviews pytest pytest-cov fsspec'
         - PIP_DEPENDENCIES='trollsift trollimage pyspectral pyorbital libtiff'
         - SETUP_XVFB=False
         - EVENT_TYPE='push pull_request'
         - SETUP_CMD='test'
         - CONDA_CHANNELS='conda-forge'
         - CONDA_CHANNEL_PRIORITY='strict'
+        - UNSTABLE_DEPS=False
 matrix:
   include:
   - env: PYTHON_VERSION=3.8
@@ -24,11 +27,43 @@ matrix:
   - env: PYTHON_VERSION=3.7
     os: osx
     language: generic
+  # allowed to fail:
+  - os: linux
+    env:
+      - PYTHON_VERSION=3.8
+      - UNSTABLE_DEPS=True
+
+  allow_failures:
+  - os: linux
+    env:
+      - PYTHON_VERSION=3.8
+      - UNSTABLE_DEPS=True
 install:
     - git clone --depth 1 git://github.com/astropy/ci-helpers.git
     - source ci-helpers/travis/setup_conda.sh
+    # See https://github.com/travis-ci/travis-ci/issues/8920
+    - python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)"
+    - if [ "$UNSTABLE_DEPS" == "True" ]; then
+        python -m pip install
+        -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com
+        --no-deps --pre --upgrade
+        matplotlib
+        numpy
+        pandas
+        scipy;
+        python -m pip install
+        --no-deps --upgrade
+        git+https://github.com/dask/dask
+        git+https://github.com/dask/distributed
+        git+https://github.com/zarr-developers/zarr
+        git+https://github.com/Unidata/cftime
+        git+https://github.com/mapbox/rasterio
+        git+https://github.com/pydata/bottleneck
+        git+https://github.com/pydata/xarray;
+      fi
+    - pip install --no-deps -e .
 script:
-- coverage run --source=satpy setup.py test
+- pytest --cov=satpy satpy/tests
 - coverage run -a --source=satpy -m behave satpy/tests/features --tags=-download
 after_success:
 - if [[ $PYTHON_VERSION == 3.8 ]]; then coveralls; codecov; fi

AUTHORS.md

@@ -56,6 +56,8 @@ The following people have made contributions to this project:
 - [Eysteinn Sigurðsson (eysteinn)](https://github.com/eysteinn)
 - [Dario Stelitano (bornagain1981)](https://github.com/bornagain1981)
 - [Matias Takala (elfsprite)](https://github.com/elfsprite)
+- [Taiga Tsukada (tsukada-cs)](https://github.com/tsukada-cs)
+- [Helga Weber (helgaweb)](https://github.com/helgaweb)
 - [hazbottles (hazbottles)](https://github.com/hazbottles)
 - [oananicola (oananicola)](https://github.com/oananicola)
 - [praerien (praerien)](https://github.com/praerien)

CHANGELOG.md

@@ -1,4 +1,61 @@
-## Version v0.20.0 (2020/02/25)
+## Version 0.21.0 (2020/04/06)
+
+### Issues Closed
+
+* [Issue 1124](https://github.com/pytroll/satpy/issues/1124) - Crop scene of visual spectrum of the sentinel 2 satellite ([PR 1125](https://github.com/pytroll/satpy/pull/1125))
+* [Issue 1112](https://github.com/pytroll/satpy/issues/1112) - Loading both abi and nwcsaf-geo confuses satpy into sometimes trying the wrong composite ([PR 1113](https://github.com/pytroll/satpy/pull/1113))
+* [Issue 1096](https://github.com/pytroll/satpy/issues/1096) - Saving an image with NinjoTIFFWriter is broken in satpy v.0.20.0 ([PR 1098](https://github.com/pytroll/satpy/pull/1098))
+* [Issue 1092](https://github.com/pytroll/satpy/issues/1092) - Avhrr l1b eps reader changes values of angles after reading ([PR 1101](https://github.com/pytroll/satpy/pull/1101))
+* [Issue 1087](https://github.com/pytroll/satpy/issues/1087) - Saving each scene in a separate image file
+* [Issue 1075](https://github.com/pytroll/satpy/issues/1075) - SEVIRI L1b netCDF reader not dask-compliant ([PR 1109](https://github.com/pytroll/satpy/pull/1109))
+* [Issue 1059](https://github.com/pytroll/satpy/issues/1059) - test against xarray master ([PR 1095](https://github.com/pytroll/satpy/pull/1095))
+* [Issue 1013](https://github.com/pytroll/satpy/issues/1013) - Fails to load solar_zenith_angle from SLSTR l1b data
+* [Issue 883](https://github.com/pytroll/satpy/issues/883) - satpy resample call -> numby.ndarray deepcopy error ([PR 1126](https://github.com/pytroll/satpy/pull/1126))
+* [Issue 840](https://github.com/pytroll/satpy/issues/840) - MTG-FCI-FDHSI reader has wrong projection ([PR 845](https://github.com/pytroll/satpy/pull/845))
+* [Issue 630](https://github.com/pytroll/satpy/issues/630) - Converting hdf5 attributes to string containing h5py.Reference of size 1 causes a AttributeError ([PR 1126](https://github.com/pytroll/satpy/pull/1126))
+
+In this release 11 issues were closed.
+
+### Pull Requests Merged
+
+#### Bugs fixed
+
+* [PR 1131](https://github.com/pytroll/satpy/pull/1131) - Fix geostationary utilities assuming a/b radii are always available
+* [PR 1129](https://github.com/pytroll/satpy/pull/1129) - Make the viirs_sdr reader return float32s
+* [PR 1125](https://github.com/pytroll/satpy/pull/1125) - Fix Scene.crop using PROJ definition to create target area definition ([1124](https://github.com/pytroll/satpy/issues/1124))
+* [PR 1118](https://github.com/pytroll/satpy/pull/1118) - Fix supported Python version in devguide
+* [PR 1116](https://github.com/pytroll/satpy/pull/1116) - Make an alias for the snow composite in viirs
+* [PR 1115](https://github.com/pytroll/satpy/pull/1115) - Fix mitiff writer to support sensors as a set
+* [PR 1113](https://github.com/pytroll/satpy/pull/1113) - Add sensor-name property to NWCSAF readers ([1112](https://github.com/pytroll/satpy/issues/1112), [1111](https://github.com/pytroll/satpy/issues/1111))
+* [PR 1107](https://github.com/pytroll/satpy/pull/1107) - Raise an error if data and angle shapes don't match in NIRReflectance
+* [PR 1106](https://github.com/pytroll/satpy/pull/1106) - Scale valid range if available.
+* [PR 1101](https://github.com/pytroll/satpy/pull/1101) - Fix eps l1b angles computation returning non deterministic results ([1092](https://github.com/pytroll/satpy/issues/1092))
+* [PR 1098](https://github.com/pytroll/satpy/pull/1098) - Fix ninjotiff writer tests failing when pyninjotiff is installed ([1096](https://github.com/pytroll/satpy/issues/1096))
+* [PR 1089](https://github.com/pytroll/satpy/pull/1089) - Make sunz correction use available sunz dataset
+* [PR 1038](https://github.com/pytroll/satpy/pull/1038) - Switch to pyproj for projection to CF NetCDF grid mapping ([1029](https://github.com/pytroll/satpy/issues/1029), [1029](https://github.com/pytroll/satpy/issues/1029))
+
+#### Features added
+
+* [PR 1128](https://github.com/pytroll/satpy/pull/1128) - Add tm5_constant_a and tm5_constant_b for tropomi_l2
+* [PR 1126](https://github.com/pytroll/satpy/pull/1126) - Update omps edr reader and hdf5_utils to handle OMPS SO2 data from FMI ([883](https://github.com/pytroll/satpy/issues/883), [630](https://github.com/pytroll/satpy/issues/630))
+* [PR 1121](https://github.com/pytroll/satpy/pull/1121) - HY-2B scatterometer l2b hdf5 reader
+* [PR 1117](https://github.com/pytroll/satpy/pull/1117) - Add support for satpy.composites entry points
+* [PR 1113](https://github.com/pytroll/satpy/pull/1113) - Add sensor-name property to NWCSAF readers ([1112](https://github.com/pytroll/satpy/issues/1112), [1111](https://github.com/pytroll/satpy/issues/1111))
+* [PR 1109](https://github.com/pytroll/satpy/pull/1109) - Fix dask and attribute issue in seviri_l1b_nc reader ([1075](https://github.com/pytroll/satpy/issues/1075))
+* [PR 1095](https://github.com/pytroll/satpy/pull/1095) - Switch to pytest in CI and add unstable dependency environment ([1059](https://github.com/pytroll/satpy/issues/1059))
+* [PR 1091](https://github.com/pytroll/satpy/pull/1091) - Add assembled_lat_bounds, assembled_lon_bounds and time variables
+* [PR 1071](https://github.com/pytroll/satpy/pull/1071) - Add SEVIRI L2 GRIB reader
+* [PR 1044](https://github.com/pytroll/satpy/pull/1044) - Set travis and appveyor numpy version back to 'stable'
+* [PR 845](https://github.com/pytroll/satpy/pull/845) - MTG: get projection and extent information from file ([840](https://github.com/pytroll/satpy/issues/840), [840](https://github.com/pytroll/satpy/issues/840))
+* [PR 606](https://github.com/pytroll/satpy/pull/606) - Add enhanced (more natural) version of natural colors composite
+
+#### Documentation changes
+
+* [PR 1130](https://github.com/pytroll/satpy/pull/1130) - Add note about datatype in custom reader documentation
+* [PR 1118](https://github.com/pytroll/satpy/pull/1118) - Fix supported Python version in devguide
+
+
+## Version 0.20.0 (2020/02/25)
 
 ### Issues Closed
 

appveyor.yml

@@ -3,9 +3,10 @@ environment:
     PYTHON: "C:\\conda"
     MINICONDA_VERSION: "latest"
     CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd"
-    CONDA_DEPENDENCIES: "xarray dask distributed toolz Cython sphinx cartopy pillow matplotlib scipy pyyaml pyproj pyresample coverage netcdf4 h5py h5netcdf gdal rasterio imageio pyhdf mock libtiff zarr six"
+    CONDA_DEPENDENCIES: "xarray dask distributed toolz Cython sphinx cartopy pillow matplotlib scipy pyyaml pyproj pyresample coverage netcdf4 h5py h5netcdf gdal rasterio imageio pyhdf libtiff geoviews zarr pytest pytest-cov"
     PIP_DEPENDENCIES: "trollsift trollimage pyspectral pyorbital libtiff"
     CONDA_CHANNELS: "conda-forge"
+    CONDA_CHANNEL_PRIORITY: "strict"
 
   matrix:
     - PYTHON: "C:\\Python38_64"
@@ -15,18 +16,19 @@ environment:
     - PYTHON: "C:\\Python38_64"
       PYTHON_VERSION: "3.7"
       PYTHON_ARCH: "64"
-      NUMPY_VERSION: "1.16"
+      NUMPY_VERSION: "stable"
 
 install:
     - "git clone --depth 1 git://github.com/astropy/ci-helpers.git"
     - "powershell ci-helpers/appveyor/install-miniconda.ps1"
     - "conda activate test"
+    - "pip install --no-deps -e ."
 
 build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
   # Build the compiled extension and run the project tests
-  - "%CMD_IN_ENV% python setup.py test"
+  - "%CMD_IN_ENV% pytest --cov=satpy satpy/tests"
 
 after_test:
   # If tests are successful, create a whl package for the project.

doc/source/conf.py

@@ -260,9 +260,11 @@ def __getattr__(cls, name):
     'pyorbital': ('https://pyorbital.readthedocs.io/en/stable', None),
     'pyproj': ('https://pyproj4.github.io/pyproj/dev', None),
     'pyresample': ('https://pyresample.readthedocs.io/en/stable', None),
+    'pytest': ('https://docs.pytest.org/en/stable/', None),
     'python': ('https://docs.python.org/3', None),
     'scipy': ('https://docs.scipy.org/doc/scipy/reference', None),
     'trollimage': ('https://trollimage.readthedocs.io/en/stable', None),
     'trollsift': ('https://trollsift.readthedocs.io/en/stable', None),
-    'xarray': ('https://xarray.pydata.org/en/stable', None)
+    'xarray': ('https://xarray.pydata.org/en/stable', None),
+    'rasterio': ('https://rasterio.readthedocs.io/en/latest', None),
 }

doc/source/dev_guide/custom_reader.rst

@@ -506,6 +506,25 @@ a convenience and are not required to read these formats. In many cases using
 the :func:`xarray.open_dataset` function in a custom file handler is a much
 better idea.
 
+.. note::
+   Be careful about the data types of the datasets your reader is returning.
+   It is easy to let the data be coerced into double precision floats (`np.float64`). At the
+   moment, satellite instruments are rarely measuring in a resolution greater
+   than what can be encoded in 16 bits. As such, to preserve processing power,
+   please consider carefully what data type you should scale or calibrate your
+   data to.
+
+   Single precision floats (`np.float32`) is a good compromise, as it has 23
+   significant bits (mantissa) and can thus represent 16 bit integers exactly,
+   as well as keeping the memory footprint half of a double precision float.
+
+   One commonly used method in readers is :meth:`xarray.DataArray.where` (to
+   mask invalid data) which can be coercing the data to `np.float64`. To ensure
+   for example that integer data is coerced to `np.float32` when
+   :meth:`xarray.DataArray.where` is used, you can do::
+
+     my_float_dataarray = my_int_dataarray.where(some_condition, np.float32(np.nan))
+
 One way of implementing a file handler is shown below:
 
 .. code:: python

doc/source/dev_guide/index.rst

@@ -14,17 +14,19 @@ at the pages listed below.
     CONTRIBUTING
     xarray_migration
     custom_reader
+    plugins
 
 Coding guidelines
 =================
 
-Satpy is part of `PyTroll <http://pytroll.github.io/>`_,
+Satpy is part of `Pytroll <http://pytroll.github.io/>`_,
 and all code should follow the
-`PyTroll coding guidelines and best
+`Pytroll coding guidelines and best
 practices <http://pytroll.github.io/guidelines.html>`_.
 
-Satpy currently supports Python 2.7 and 3.4+. All code should be written to
-be compatible with these versions.
+Satpy is now Python 3 only and it is no longer needed to support Python 2.
+Check ``setup.py`` for the current Python versions any new code needs
+to support.
 
 Development installation
 ========================
@@ -44,10 +46,11 @@ clone your fork. The package can then be installed in development by doing::
 Running tests
 =============
 
-Satpy tests are written using the python :mod:`unittest` module and the tests
-can be executed by running::
+Satpy tests are written using the python :mod:`unittest` module and the
+third-party :doc:`pytest <pytest:index>` package. Satpy tests can be executed by
+running::
 
-    python setup.py test
+    pytest satpy/tests
 
 Documentation
 =============

doc/source/dev_guide/plugins.rst

@@ -0,0 +1,34 @@
+================================================
+ Adding new functionality to Satpy via plugins
+================================================
+
+.. warning::
+    This feature is experimental and being modified without warnings.
+    For now, it should not be used for anything else than toy examples and
+    should not be relied on.
+
+Satpy has the capability of using plugins. At the moment, new composites can be
+added to satpy through external plugins. Plugins for reader and writers may be
+added at a later date (PRs are welcome!).
+
+Here is an
+`example <https://github.com/mraspaud/satpy-composites-plugin-example>`_ of a
+composites plugin.
+
+The key is to use the same configuration directory structure as satpy and add
+a `satpy.composites` entry point in the setup.py file of the plugin:
+
+.. code: python
+
+    from setuptools import setup
+    import os
+
+    setup(
+        name='satpy_cpe',
+        entry_points={
+            'satpy.composites': [
+                'example_composites = satpy_cpe',
+            ],
+        },
+        package_data={'satpy_cpe': [os.path.join('etc', 'composites/*.yaml')]},
+    )

doc/source/examples/fci_l1c_natural_color.rst

@@ -0,0 +1,41 @@
+MTG FCI - Natural Color Example
+===============================
+
+Satpy includes a reader for the Meteosat Third Generation (MTG) FCI Level 1c
+data. The following Python code snippet shows an example on how to use Satpy
+to generate a Natural Color RGB composite over the European area.
+
+.. warning::
+
+    This example is currently a work in progress. Some of the below code may
+    not work with the currently released version of Satpy. Additional updates
+    to this example will be coming soon.
+
+.. code-block:: python
+
+    from satpy.scene import Scene
+    from satpy import find_files_and_readers
+
+    # define path to FCI test data folder
+    path_to_data = 'your/path/to/FCI/data/folder/'
+
+    # find files and assign the FCI reader
+    files = find_files_and_readers(base_dir=path_to_data, reader='fci_l1c_fdhsi')
+
+    # create an FCI scene from the selected files
+    scn = Scene(filenames=files)
+
+    # print available dataset names for this scene (e.g. 'vis_04', 'vis_05','ir_38',...)
+    print(scn.available_dataset_names())
+
+    # print available composite names for this scene (e.g. 'natural_color', 'airmass', 'convection',...)
+    print(scn.available_composite_names())
+
+    # load the datasets/composites of interest
+    scn.load(['natural_color','vis_04'])
+
+    # resample the scene to a specified area (e.g. "eurol1" for Europe in 1km resolution)
+    scn_resampled = scn.resample("eurol", resampler='nearest', radius_of_influence=5000)
+
+    # save the resampled dataset/composite to disk
+    scn_resampled.save_dataset("natural_color", filename='./fci_natural_color_resampled.png')

doc/source/examples.rst → doc/source/examples/index.rst

@@ -3,11 +3,18 @@ Examples
 
 Satpy examples are available as Jupyter Notebooks on the
 `pytroll-examples <https://github.com/pytroll/pytroll-examples/tree/master/satpy>`_
-git repository. They include python code, PNG images, and descriptions of
+git repository. Some examples are described in further detail as separate pages
+in this documentation. They include python code, PNG images, and descriptions of
 what the example is doing. Below is a list of some of the examples and a brief
-summary. Additional example can be found at the repository mentioned above or
+summary. Additional examples can be found at the repository mentioned above or
 as explanations in the various sections of this documentation.
 
+.. toctree::
+    :hidden:
+    :maxdepth: 1
+
+    fci_l1c_natural_color
+
 .. list-table::
     :header-rows: 1
 
@@ -33,3 +40,5 @@ as explanations in the various sections of this documentation.
       - Reading Level 2 EARS-NWC cloud products
     * - `Level 2 MAIA cloud products <https://github.com/pytroll/pytroll-examples/blob/master/satpy/polar_maia.ipynb>`_
       - Reading Level 2 MAIA cloud products
+    * - :doc:`Meteosat Third Generation FCI Natural Color RGB <fci_l1c_natural_color>`
+      - Generate Natural Color RGB from Meteosat Third Generation (MTG) FCI Level 1c data