Merge remote-tracking branch 'origin/develop' into add-wave-spectra

ecmwf · Feb 3, 2019 · 86f8e8f · 86f8e8f
2 parents 60267fe + 6366104
commit 86f8e8f
Show file tree

Hide file tree

Showing 30 changed files with 495 additions and 377 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -2,10 +2,19 @@
 Changelog for cfgrib
 ====================
 
-0.9.5.5 (unreleased)
+0.9.6 (unreleased)
+------------------
+
+- Do not set explicit timezone in ``units`` to avoid crashing some versions of xarray.
+  See: `#44 <https://github.com/ecmwf/cfgrib/issues/44>`_.
+
+
+0.9.5.5 (2019-02-02)
 --------------------
 
-- Nothing changed yet.
+- Enable ecCodes implicit MULTI-FIELD support by default, needed for NAM Products by NCEP.
+  See: `#45 <https://github.com/ecmwf/cfgrib/issues/45>`_.
+- Added support for ``depthBelowLand`` coordinate.
 
 
 0.9.5.4 (2019-01-25)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -8,6 +8,8 @@ Contributing
 Contributions are welcome, and they are greatly appreciated! Every
 little bit helps, and credit will always be given.
 
+Please note, that we have hooked a CLA assiatant to this GitHub Repo. Please accept the contributors license agreement to allow us to keep a legal track of contributions and keep this package open source for the future.  
+
 You can contribute in many ways:
 
 Types of Contributions
@@ -126,6 +128,6 @@ Then you can run::
     $ pytest -vv tests/cds_test_*.py
 
 
-.. cfgrib: https://github.com/ecmwk/cfgrib
+.. cfgrib: https://github.com/ecmwf/cfgrib
 .. virtualenv: https://virtualenv.pypa.io/en/stable/installation
-.. git: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git
+.. git: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git
diff --git a/README.rst b/README.rst
@@ -370,6 +370,7 @@ Data variables:
     r              (isobaricInhPa, y, x) float32 ...
     w              (isobaricInhPa, y, x) float32 ...
     u              (isobaricInhPa, y, x) float32 ...
+    v              (isobaricInhPa, y, x) float32 ...
 Attributes:
     GRIB_edition:            2
     GRIB_centre:             kwbc
@@ -432,6 +433,7 @@ Data variables:
     pres        (y, x) float32 ...
     gh          (y, x) float32 ...
     u           (y, x) float32 ...
+    v           (y, x) float32 ...
 Attributes:
     GRIB_edition:            2
     GRIB_centre:             kwbc
@@ -601,7 +603,7 @@ See also the list of `contributors <https://github.com/ecmwf/cfgrib/contributors
 License
 =======
 
-Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

diff --git a/cf2cdm/__init__.py b/cf2cdm/__init__.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/cf2cdm/cfcoords.py b/cf2cdm/cfcoords.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -145,6 +145,16 @@ def is_valid_time(coord):
 )
 
 
+def is_depth(coord):
+    # type: (xr.Coordinate) -> bool
+    return coord.attrs.get('standard_name') == 'depth'
+
+
+COORD_TRANSLATORS['depthBelowLand'] = functools.partial(
+    coord_translator, 'depthBelowLand', 'm', 'decreasing', is_depth,
+)
+
+
 def is_isobaric(coord):
     # type: (xr.Coordinate) -> bool
     return cfunits.are_convertible(coord.attrs.get('units', ''), 'Pa')

diff --git a/cf2cdm/cfunits.py b/cf2cdm/cfunits.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,6 +30,12 @@
     ('atmosphere', 'atmospheres', 'atm'): 101325.,
 }  # type: T.Dict[T.Tuple, float]
 
+LENGTH_CONVERSION_RULES = {
+    ('m', 'meter', 'meters'): 1.,
+    ('cm', 'centimeter', 'centimeters'): 0.01,
+    ('km', 'kilometer', 'kilometers'): 1000.,
+}
+
 
 class ConversionError(Exception):
     pass
@@ -55,7 +61,7 @@ def convert_units(data, target_units, source_units):
     # type: (T.Any, str, str) -> T.Any
     if target_units == source_units:
         return data
-    for rules in [PRESSURE_CONVERSION_RULES]:
+    for rules in [PRESSURE_CONVERSION_RULES, LENGTH_CONVERSION_RULES]:
         try:
             return data * simple_conversion_factor(target_units, source_units, rules)
         except ConversionError:

diff --git a/cf2cdm/datamodels.py b/cf2cdm/datamodels.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
 
 
 CDS = {
+    # geography
     'latitude': {
         'out_name': 'lat',
         'stored_direction': 'increasing',
@@ -29,20 +30,23 @@
         'out_name': 'lon',
         'stored_direction': 'increasing',
     },
-    'isobaricInhPa': {
-        'out_name': 'plev',
-        'units': 'Pa',
-        'stored_direction': 'decreasing',
+    # vertical
+    'depthBelowLand': {
+        'out_name': 'depth',
+        'units': 'm',
+        'stored_direction': 'increasing',
     },
-    'isobaricInPa': {
+    'isobaricInhPa': {
         'out_name': 'plev',
         'units': 'Pa',
         'stored_direction': 'decreasing',
     },
+    # ensemble
     'number': {
         'out_name': 'realization',
         'stored_direction': 'increasing',
     },
+    # time
     'time': {
         'out_name': 'forecast_reference_time',
         'stored_direction': 'increasing',
@@ -62,6 +66,11 @@
 
 
 ECMWF = {
+    'depthBelowLand': {
+        'out_name': 'level',
+        'units': 'm',
+        'stored_direction': 'increasing',
+    },
     'isobaricInhPa': {
         'out_name': 'level',
         'units': 'hPa',

diff --git a/cfgrib/__init__.py b/cfgrib/__init__.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-__version__ = '0.9.5.5.dev0'
+__version__ = '0.9.6.dev0'
 
 # cfgrib core API depends on the ECMWF ecCodes C-library only
 from .cfmessage import CfMessage

diff --git a/cfgrib/__main__.py b/cfgrib/__main__.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,9 +31,9 @@ def cfgrib_cli():
 
 @cfgrib_cli.command('selfcheck')
 def selfcheck():
-    from . import eccodes
+    from . import bindings
 
-    print("Found: ecCodes v%s." % eccodes.codes_get_api_version())
+    print("Found: ecCodes v%s." % bindings.codes_get_api_version())
     print("Your system is ready.")
 
 

diff --git a/cfgrib/eccodes.py → cfgrib/bindings.py b/cfgrib/eccodes.py → cfgrib/bindings.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -77,6 +77,12 @@ def __getattr__(self, attr):
 CODES_PRODUCT_TAF = 5
 """ TAF product kind """
 
+# Constants for 'missing'
+GRIB_MISSING_DOUBLE = -1e+100
+GRIB_MISSING_LONG = 2147483647
+
+CODES_MISSING_DOUBLE = GRIB_MISSING_DOUBLE
+CODES_MISSING_LONG = GRIB_MISSING_LONG
 
 #
 # Helper values to discriminate key types
@@ -153,9 +159,11 @@ def codes_index_new_from_file(path, keys):
     return check_last(lib.codes_index_new_from_file)(ffi.NULL, path, keys_enc)
 
 
-def codes_handle_new_from_file(fileobj, product_kind=CODES_PRODUCT_GRIB):
+def codes_handle_new_from_file(fileobj, product_kind=CODES_PRODUCT_GRIB, context=None):
+    if context is None:
+        context = ffi.NULL
     try:
-        retval = check_last(lib.codes_handle_new_from_file)(ffi.NULL, fileobj, product_kind)
+        retval = check_last(lib.codes_handle_new_from_file)(context, fileobj, product_kind)
         if retval == ffi.NULL:
             raise EOFError("End of file: %r" % fileobj)
         else:
@@ -664,6 +672,18 @@ def codes_set_array(handle, key, values):
         raise ValueError("Cannot set an empty list.")
 
 
+def codes_grib_multi_support_on(context=None):
+    if context is None:
+        context = ffi.NULL
+    lib.codes_grib_multi_support_on(context)
+
+
+def codes_grib_multi_support_off(context=None):
+    if context is None:
+        context = ffi.NULL
+    lib.codes_grib_multi_support_off(context)
+
+
 def codes_write(handle, outfile):
     # type: (cffi.FFI.CData, T.BinaryIO) -> None
     """

diff --git a/cfgrib/cfmessage.py b/cfgrib/cfmessage.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/cfgrib/dataset.py b/cfgrib/dataset.py
@@ -1,5 +1,5 @@
 #
-# Copyright 2017-2018 European Centre for Medium-Range Weather Forecasts (ECMWF).
+# Copyright 2017-2019 European Centre for Medium-Range Weather Forecasts (ECMWF).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,8 +31,8 @@
 import numpy as np
 
 from . import __version__
+from . import bindings
 from . import cfmessage
-from . import eccodes
 from . import messages
 
 LOG = logging.getLogger(__name__)
@@ -109,18 +109,7 @@
 ALL_KEYS = GLOBAL_ATTRIBUTES_KEYS + DATA_ATTRIBUTES_KEYS + GRID_TYPE_KEYS + ALL_HEADER_DIMS
 
 COORD_ATTRS = {
-    'time': {
-        'units': 'seconds since 1970-01-01T00:00:00+00:00', 'calendar': 'proleptic_gregorian',
-        'standard_name': 'forecast_reference_time', 'long_name': 'initial time of forecast',
-    },
-    'step': {
-        'units': 'hours',
-        'standard_name': 'forecast_period', 'long_name': 'time since forecast_reference_time',
-    },
-    'valid_time': {
-        'units': 'seconds since 1970-01-01T00:00:00+00:00', 'calendar': 'proleptic_gregorian',
-        'standard_name': 'time', 'long_name': 'time',
-    },
+    # geography
     'latitude': {
         'units': 'degrees_north',
         'standard_name': 'latitude', 'long_name': 'latitude',
@@ -129,9 +118,14 @@
         'units': 'degrees_east',
         'standard_name': 'longitude', 'long_name': 'longitude',
     },
-    'isobaricInhPa': {
-        'units': 'hPa', 'positive': 'down', 'stored_direction': 'decreasing',
-        'standard_name': 'air_pressure', 'long_name': 'pressure',
+    # vertical
+    'depthBelowLand': {
+        'units': 'm', 'positive': 'down', 'long_name': 'soil depth',
+        'standard_name': 'depth',
+    },
+    'depthBelowLandLayer': {
+        'units': 'm', 'positive': 'down', 'long_name': 'soil depth',
+        'standard_name': 'depth',
     },
     'hybrid': {
         'units': '1', 'positive': 'down', 'long_name': 'hybrid level',
@@ -141,10 +135,36 @@
         'units': 'm', 'positive': 'up', 'long_name': 'height above the surface',
         'standard_name': 'height',
     },
+    'isobaricInhPa': {
+        'units': 'hPa', 'positive': 'down', 'stored_direction': 'decreasing',
+        'standard_name': 'air_pressure', 'long_name': 'pressure',
+    },
+    'isobaricInPa': {
+        'units': 'Pa', 'positive': 'down', 'stored_direction': 'decreasing',
+        'standard_name': 'air_pressure', 'long_name': 'pressure',
+    },
+    'isobaricLayer': {
+        'units': 'Pa', 'positive': 'down',
+        'standard_name': 'air_pressure', 'long_name': 'pressure',
+    },
+    # ensemble
     'number': {
         'units': '1',
         'standard_name': 'realization', 'long_name': 'ensemble member numerical id',
-    }
+    },
+    # time
+    'step': {
+        'units': 'hours',
+        'standard_name': 'forecast_period', 'long_name': 'time since forecast_reference_time',
+    },
+    'time': {
+        'units': 'seconds since 1970-01-01T00:00:00', 'calendar': 'proleptic_gregorian',
+        'standard_name': 'forecast_reference_time', 'long_name': 'initial time of forecast',
+    },
+    'valid_time': {
+        'units': 'seconds since 1970-01-01T00:00:00', 'calendar': 'proleptic_gregorian',
+        'standard_name': 'time', 'long_name': 'time',
+    },
 }
 
 
@@ -218,7 +238,7 @@ def build_array(self):
             for header_indexes, offset in self.offsets.items():
                 # NOTE: fill a single field as found in the message
                 message = self.stream.message_from_file(file, offset=offset[0])
-                values = message.message_get('values', eccodes.CODES_TYPE_DOUBLE)
+                values = message.message_get('values', bindings.CODES_TYPE_DOUBLE)
                 array.__getitem__(header_indexes).flat[:] = values
         array[array == self.missing_value] = np.nan
         return array
@@ -240,7 +260,7 @@ def __getitem__(self, item):
                     continue
                 # NOTE: fill a single field as found in the message
                 message = self.stream.message_from_file(file, offset=offset[0])
-                values = message.message_get('values', eccodes.CODES_TYPE_DOUBLE)
+                values = message.message_get('values', bindings.CODES_TYPE_DOUBLE)
                 array_field.__getitem__(tuple(array_field_indexes)).flat[:] = values
 
         array = array_field[(Ellipsis,) + item[-self.geo_ndim:]]
@@ -448,7 +468,7 @@ def build_dataset_components(
     attributes_namespace = {
         'cfgrib_version': __version__,
         'cfgrib_open_kwargs': json.dumps(encoding),
-        'eccodes_version': eccodes.codes_get_api_version(),
+        'eccodes_version': bindings.codes_get_api_version(),
         'timestamp': timestamp or datetime.datetime.now().isoformat().partition('.')[0]
     }
     history_in = '{timestamp} GRIB to CDM+CF via ' \