From bf3c56af035bd296d423dedde3bfabf770f1084b Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Sun, 24 Feb 2019 01:25:55 +0100 Subject: [PATCH] Raise all non-unique values in the file not just the one for the variable. #54 --- README.rst | 108 +++++++++++++++++++++++++++++++++++++++++ cfgrib/dataset.py | 23 ++++++--- cfgrib/xarray_store.py | 2 +- 3 files changed, 126 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 8b946af8..b700b560 100644 --- a/README.rst +++ b/README.rst @@ -351,6 +351,49 @@ and it is not part of the stable API. In the future it may change or be removed >>> from cfgrib import xarray_store >>> xarray_store.open_datasets('nam.t00z.awp21100.tm00.grib2', backend_kwargs={'errors': 'ignore'}) [ +Dimensions: (x: 93, y: 65) +Coordinates: + time datetime64[ns] ... + step timedelta64[ns] ... + meanSea int64 ... + latitude (y, x) float64 ... + longitude (y, x) float64 ... + valid_time datetime64[ns] ... +Dimensions without coordinates: x, y +Data variables: + prmsl (y, x) float32 ... + mslet (y, x) float32 ... +Attributes: + GRIB_edition: 2 + GRIB_centre: kwbc + GRIB_centreDescription: US National Weather Service - NCEP... + GRIB_subCentre: 0 + Conventions: CF-1.7 + institution: US National Weather Service - NCEP... + history: ..., +Dimensions: (x: 93, y: 65) +Coordinates: + time datetime64[ns] ... + step timedelta64[ns] ... + surface int64 ... + latitude (y, x) float64 ... + longitude (y, x) float64 ... + valid_time datetime64[ns] ... +Dimensions without coordinates: x, y +Data variables: + gust (y, x) float32 ... + sp (y, x) float32 ... + orog (y, x) float32 ... + tp (y, x) float32 ... + acpcp (y, x) float32 ... +Attributes: + GRIB_edition: 2 + GRIB_centre: kwbc + GRIB_centreDescription: US National Weather Service - NCEP... + GRIB_subCentre: 0 + Conventions: CF-1.7 + institution: US National Weather Service - NCEP... + history: ..., Dimensions: (isobaricInhPa: 19, x: 93, y: 65) Coordinates: time datetime64[ns] ... @@ -375,6 +418,47 @@ Attributes: Conventions: CF-1.7 institution: US National Weather Service - NCEP... history: ..., +Dimensions: (x: 93, y: 65) +Coordinates: + time datetime64[ns] ... + step timedelta64[ns] ... + heightAboveGround int64 ... + latitude (y, x) float64 ... + longitude (y, x) float64 ... + valid_time datetime64[ns] ... +Dimensions without coordinates: x, y +Data variables: + t2m (y, x) float32 ... + r2 (y, x) float32 ... + u10 (y, x) float32 ... + v10 (y, x) float32 ... +Attributes: + GRIB_edition: 2 + GRIB_centre: kwbc + GRIB_centreDescription: US National Weather Service - NCEP... + GRIB_subCentre: 0 + Conventions: CF-1.7 + institution: US National Weather Service - NCEP... + history: ..., +Dimensions: (x: 93, y: 65) +Coordinates: + time datetime64[ns] ... + step timedelta64[ns] ... + level int64 ... + latitude (y, x) float64 ... + longitude (y, x) float64 ... + valid_time datetime64[ns] ... +Dimensions without coordinates: x, y +Data variables: + pwat (y, x) float32 ... +Attributes: + GRIB_edition: 2 + GRIB_centre: kwbc + GRIB_centreDescription: US National Weather Service - NCEP... + GRIB_subCentre: 0 + Conventions: CF-1.7 + institution: US National Weather Service - NCEP... + history: ..., Dimensions: (x: 93, y: 65) Coordinates: time datetime64[ns] ... @@ -450,6 +534,30 @@ Dimensions without coordinates: x, y Data variables: gh (y, x) float32 ... r (y, x) float32 ... +Attributes: + GRIB_edition: 2 + GRIB_centre: kwbc + GRIB_centreDescription: US National Weather Service - NCEP... + GRIB_subCentre: 0 + Conventions: CF-1.7 + institution: US National Weather Service - NCEP... + history: ..., +Dimensions: (pressureFromGroundLayer: 5, x: 93, y: 65) +Coordinates: + time datetime64[ns] ... + step timedelta64[ns] ... + * pressureFromGroundLayer (pressureFromGroundLayer) int64 3000 6000 ... 15000 + latitude (y, x) float64 ... + longitude (y, x) float64 ... + valid_time datetime64[ns] ... +Dimensions without coordinates: x, y +Data variables: + t (pressureFromGroundLayer, y, x) float32 ... + r (pressureFromGroundLayer, y, x) float32 ... + u (pressureFromGroundLayer, y, x) float32 ... + v (pressureFromGroundLayer, y, x) float32 ... + pli (y, x) float32 ... + 4lftx (y, x) float32 ... Attributes: GRIB_edition: 2 GRIB_centre: kwbc diff --git a/cfgrib/dataset.py b/cfgrib/dataset.py index b989ac22..6fe6b0c5 100644 --- a/cfgrib/dataset.py +++ b/cfgrib/dataset.py @@ -178,14 +178,12 @@ def enforce_unique_attributes(index, attributes_keys, filter_by_keys={}): for key in attributes_keys: values = index[key] if len(values) > 1: - error_message = "multiple values for unique key, try re-open the file with one of:" fbks = [] for value in values: fbk = {key: value} fbk.update(filter_by_keys) fbks.append(fbk) - error_message += "\n filter_by_keys=%r" % fbk - raise DatasetBuildError(error_message, fbks) + raise DatasetBuildError("multiple values for key %r" % key, key, fbks) if values and values[0] not in ('undef', 'unknown'): attributes['GRIB_' + key] = values[0] return attributes @@ -443,9 +441,22 @@ def build_dataset_components( variables = collections.OrderedDict() for param_id, short_name, var_name in zip(param_ids, index['shortName'], index['cfVarName']): var_index = index.subindex(paramId=param_id) - dims, data_var, coord_vars = build_variable_components( - var_index, encode_cf, filter_by_keys, errors=errors, - ) + try: + dims, data_var, coord_vars = build_variable_components( + var_index, encode_cf, filter_by_keys, errors=errors, + ) + except DatasetBuildError as ex: + # NOTE: When a variable has more than one value for an attribute we need to raise all + # the values in the file, not just the ones associated with that variable. See #54. + key = ex.args[1] + error_message = "multiple values for unique key, try re-open the file with one of:" + fbks = [] + for value in index[key]: + fbk = {key: value} + fbk.update(filter_by_keys) + fbks.append(fbk) + error_message += "\n filter_by_keys=%r" % fbk + raise DatasetBuildError(error_message, key, fbks) if 'parameter' in encode_cf and var_name not in ('undef', 'unknown'): short_name = var_name vars = collections.OrderedDict([(short_name, data_var)]) diff --git a/cfgrib/xarray_store.py b/cfgrib/xarray_store.py index 7daf4d2d..428e847e 100644 --- a/cfgrib/xarray_store.py +++ b/cfgrib/xarray_store.py @@ -54,7 +54,7 @@ def open_datasets(path, backend_kwargs={}, no_warn=False, **kwargs): try: datasets.append(open_dataset(path, backend_kwargs=backend_kwargs, **kwargs)) except DatasetBuildError as ex: - fbks.extend(ex.args[1]) + fbks.extend(ex.args[2]) # NOTE: the recursive call needs to stay out of the exception handler to avoid showing # to the user a confusing error message due to exception chaining for fbk in fbks: