Skip to content

Commit

Permalink
Raise all non-unique values in the file not just the one for the vari…
Browse files Browse the repository at this point in the history 
…able. #54
  • Loading branch information
@alexamici
alexamici committed Feb 24, 2019
1 parent 8c6d7c2 commit bf3c56a
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 7 deletions.
108 changes: 108 additions & 0 deletions README.rst
View file
Expand Up @@ -351,6 +351,49 @@ and it is not part of the stable API. In the future it may change or be removed
>>> from cfgrib import xarray_store
>>> xarray_store.open_datasets('nam.t00z.awp21100.tm00.grib2', backend_kwargs={'errors': 'ignore'})
[<xarray.Dataset>
Dimensions: (x: 93, y: 65)
Coordinates:
time datetime64[ns] ...
step timedelta64[ns] ...
meanSea int64 ...
latitude (y, x) float64 ...
longitude (y, x) float64 ...
valid_time datetime64[ns] ...
Dimensions without coordinates: x, y
Data variables:
prmsl (y, x) float32 ...
mslet (y, x) float32 ...
Attributes:
GRIB_edition: 2
GRIB_centre: kwbc
GRIB_centreDescription: US National Weather Service - NCEP...
GRIB_subCentre: 0
Conventions: CF-1.7
institution: US National Weather Service - NCEP...
history: ..., <xarray.Dataset>
Dimensions: (x: 93, y: 65)
Coordinates:
time datetime64[ns] ...
step timedelta64[ns] ...
surface int64 ...
latitude (y, x) float64 ...
longitude (y, x) float64 ...
valid_time datetime64[ns] ...
Dimensions without coordinates: x, y
Data variables:
gust (y, x) float32 ...
sp (y, x) float32 ...
orog (y, x) float32 ...
tp (y, x) float32 ...
acpcp (y, x) float32 ...
Attributes:
GRIB_edition: 2
GRIB_centre: kwbc
GRIB_centreDescription: US National Weather Service - NCEP...
GRIB_subCentre: 0
Conventions: CF-1.7
institution: US National Weather Service - NCEP...
history: ..., <xarray.Dataset>
Dimensions: (isobaricInhPa: 19, x: 93, y: 65)
Coordinates:
time datetime64[ns] ...
Expand All @@ -375,6 +418,47 @@ Attributes:
Conventions: CF-1.7
institution: US National Weather Service - NCEP...
history: ..., <xarray.Dataset>
Dimensions: (x: 93, y: 65)
Coordinates:
time datetime64[ns] ...
step timedelta64[ns] ...
heightAboveGround int64 ...
latitude (y, x) float64 ...
longitude (y, x) float64 ...
valid_time datetime64[ns] ...
Dimensions without coordinates: x, y
Data variables:
t2m (y, x) float32 ...
r2 (y, x) float32 ...
u10 (y, x) float32 ...
v10 (y, x) float32 ...
Attributes:
GRIB_edition: 2
GRIB_centre: kwbc
GRIB_centreDescription: US National Weather Service - NCEP...
GRIB_subCentre: 0
Conventions: CF-1.7
institution: US National Weather Service - NCEP...
history: ..., <xarray.Dataset>
Dimensions: (x: 93, y: 65)
Coordinates:
time datetime64[ns] ...
step timedelta64[ns] ...
level int64 ...
latitude (y, x) float64 ...
longitude (y, x) float64 ...
valid_time datetime64[ns] ...
Dimensions without coordinates: x, y
Data variables:
pwat (y, x) float32 ...
Attributes:
GRIB_edition: 2
GRIB_centre: kwbc
GRIB_centreDescription: US National Weather Service - NCEP...
GRIB_subCentre: 0
Conventions: CF-1.7
institution: US National Weather Service - NCEP...
history: ..., <xarray.Dataset>
Dimensions: (x: 93, y: 65)
Coordinates:
time datetime64[ns] ...
Expand Down Expand Up @@ -450,6 +534,30 @@ Dimensions without coordinates: x, y
Data variables:
gh (y, x) float32 ...
r (y, x) float32 ...
Attributes:
GRIB_edition: 2
GRIB_centre: kwbc
GRIB_centreDescription: US National Weather Service - NCEP...
GRIB_subCentre: 0
Conventions: CF-1.7
institution: US National Weather Service - NCEP...
history: ..., <xarray.Dataset>
Dimensions: (pressureFromGroundLayer: 5, x: 93, y: 65)
Coordinates:
time datetime64[ns] ...
step timedelta64[ns] ...
* pressureFromGroundLayer (pressureFromGroundLayer) int64 3000 6000 ... 15000
latitude (y, x) float64 ...
longitude (y, x) float64 ...
valid_time datetime64[ns] ...
Dimensions without coordinates: x, y
Data variables:
t (pressureFromGroundLayer, y, x) float32 ...
r (pressureFromGroundLayer, y, x) float32 ...
u (pressureFromGroundLayer, y, x) float32 ...
v (pressureFromGroundLayer, y, x) float32 ...
pli (y, x) float32 ...
4lftx (y, x) float32 ...
Attributes:
GRIB_edition: 2
GRIB_centre: kwbc
Expand Down
23 changes: 17 additions & 6 deletions cfgrib/dataset.py
View file
Expand Up @@ -178,14 +178,12 @@ def enforce_unique_attributes(index, attributes_keys, filter_by_keys={}):
for key in attributes_keys:
values = index[key]
if len(values) > 1:
error_message = "multiple values for unique key, try re-open the file with one of:"
fbks = []
for value in values:
fbk = {key: value}
fbk.update(filter_by_keys)
fbks.append(fbk)
error_message += "\n filter_by_keys=%r" % fbk
raise DatasetBuildError(error_message, fbks)
raise DatasetBuildError("multiple values for key %r" % key, key, fbks)
if values and values[0] not in ('undef', 'unknown'):
attributes['GRIB_' + key] = values[0]
return attributes
Expand Down Expand Up @@ -443,9 +441,22 @@ def build_dataset_components(
variables = collections.OrderedDict()
for param_id, short_name, var_name in zip(param_ids, index['shortName'], index['cfVarName']):
var_index = index.subindex(paramId=param_id)
dims, data_var, coord_vars = build_variable_components(
var_index, encode_cf, filter_by_keys, errors=errors,
)
try:
dims, data_var, coord_vars = build_variable_components(
var_index, encode_cf, filter_by_keys, errors=errors,
)
except DatasetBuildError as ex:
# NOTE: When a variable has more than one value for an attribute we need to raise all
# the values in the file, not just the ones associated with that variable. See #54.
key = ex.args[1]
error_message = "multiple values for unique key, try re-open the file with one of:"
fbks = []
for value in index[key]:
fbk = {key: value}
fbk.update(filter_by_keys)
fbks.append(fbk)
error_message += "\n filter_by_keys=%r" % fbk
raise DatasetBuildError(error_message, key, fbks)
if 'parameter' in encode_cf and var_name not in ('undef', 'unknown'):
short_name = var_name
vars = collections.OrderedDict([(short_name, data_var)])
Expand Down
2 changes: 1 addition & 1 deletion cfgrib/xarray_store.py
View file
Expand Up @@ -54,7 +54,7 @@ def open_datasets(path, backend_kwargs={}, no_warn=False, **kwargs):
try:
datasets.append(open_dataset(path, backend_kwargs=backend_kwargs, **kwargs))
except DatasetBuildError as ex:
fbks.extend(ex.args[1])
fbks.extend(ex.args[2])
# NOTE: the recursive call needs to stay out of the exception handler to avoid showing
# to the user a confusing error message due to exception chaining
for fbk in fbks:
Expand Down

0 comments on commit bf3c56a

Please sign in to comment.