Skip to content

Commit

Permalink
Fixed #3046 - Failures converting data to VDC format (#3051)
Browse files Browse the repository at this point in the history 
* Changed NetCDFCpp::DefVar to not return an error if the same variable is
defined twice, but with the same definition.

* Fixed #3046 - Failures converting data to VDC format

VDC library wasn't supporting variables that not compressed, but had
missing values.

* clang-format pre-push hook

* Removed newly added, third overloaded version of VDC::DefineDataVar()
The original two were sufficient with some minor logic changes.

* clang-format pre-push hook
  • Loading branch information
@clyne
clyne committed Feb 24, 2022
1 parent 524915a commit bbf18a5
Show file tree
Hide file tree
Showing 5 changed files with 67 additions and 14 deletions.
18 changes: 15 additions & 3 deletions apps/cfvdccreate/cfvdccreate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,14 @@ int main(int argc, char **argv)
ok = dccf.GetVarCoordVars(datanames[i], false, coordvars);
VAssert(ok);

if (!dvar.GetHasMissing() || !compress) {
rc = vdc.DefineDataVar(dvar.GetName(), dimnames, coordvars, dvar.GetUnits(), dvar.GetXType(), compress);
} else {
// Don't compress the variable if it is also a coordinate variable
// Compression errors in coordinate variables can lead to
// non-conformant meshes
//
bool doCompress = compress;
if (find(coordnames.begin(), coordnames.end(), datanames[i]) != coordnames.end()) { doCompress = false; }

if (dvar.GetHasMissing() && doCompress) {
vector<string> sdimnames;
bool ok = dccf.GetVarDimNames(datanames[i], true, sdimnames);
VAssert(ok);
Expand All @@ -294,6 +299,13 @@ int main(int argc, char **argv)
maskvar(sdimnames, maskvar_name);

rc = vdc.DefineDataVar(dvar.GetName(), dimnames, coordvars, dvar.GetUnits(), dvar.GetXType(), dvar.GetMissingValue(), maskvar_name);

} else if (dvar.GetHasMissing() && !doCompress) {
rc = vdc.DefineDataVar(dvar.GetName(), dimnames, coordvars, dvar.GetUnits(), dvar.GetXType(), dvar.GetMissingValue(), "");


} else {
rc = vdc.DefineDataVar(dvar.GetName(), dimnames, coordvars, dvar.GetUnits(), dvar.GetXType(), doCompress);
}

if (rc < 0) { return (1); }
Expand Down
9 changes: 5 additions & 4 deletions include/vapor/VDC.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ class VDF_API VDC : public VAPoR::DC {
int DefineDataVar(string varname, std::vector<string> dimnames, std::vector<string> coordvars, string units, XType type, bool compressed);

//!
//! Define a compressed data variable with missing data
//! Define a compressed or non-compressed data variable with missing data
//!
//! \copydoc VDC::DefineDataVar(
//! string varname, std::vector <string> dimnames,
Expand All @@ -571,7 +571,8 @@ class VDF_API VDC : public VAPoR::DC {
//! variable may be less than or equal to that of \p varname. The dimensions
//! of \p maskvar must match the fastest varying dimensions of \p varname.
//! The \p maskvar variable must have been previously defined with
//! DefineDataVar().
//! DefineDataVar(). If \p maskvar is empty, the variable will
//! not be compressed.
//!
//! \sa DefineDimension(), DefineCoordVar(), SetCompressionBlock()
//!
Expand Down Expand Up @@ -914,7 +915,7 @@ class VDF_API VDC : public VAPoR::DC {

bool _valid_mask_var(string varname, vector<DC::Dimension> dimensions, vector<size_t> bs, bool compressed, string maskvar) const;

bool _ValidDefineDataVar(string varname, vector<string> dimnames, vector<string> coordnames, string units, XType type, bool compressed, string maskvar) const;
bool _ValidDefineDataVar(string varname, vector<string> dimnames, vector<string> coordnames, string units, XType type, bool compressed, bool has_missing, string maskvar) const;

bool _ValidCompressionBlock(vector<size_t> bs, string wname, vector<size_t> cratios) const;

Expand All @@ -925,7 +926,7 @@ class VDF_API VDC : public VAPoR::DC {

void _DefineMesh(string meshname, vector<string> dim_names, vector<string> coord_vars);

int _DefineDataVar(string varname, std::vector<string> dimnames, std::vector<string> coordvars, string units, XType type, bool compressed, double mv, string maskvar);
int _DefineDataVar(string varname, std::vector<string> dimnames, std::vector<string> coordvars, string units, XType type, bool compressed, bool has_missing, double mv, string maskvar);

vector<string> _GetCoordVarDimNames(const CoordVar &var, bool &time_varying) const;

Expand Down
18 changes: 13 additions & 5 deletions lib/vdc/VDC.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,7 @@ void VDC::_DefineMesh(string meshname, vector<string> dim_names, vector<string>
_meshes[meshname] = DC::Mesh(meshname, dim_names, coord_vars);
}

int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed, double mv, string maskvar)
int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed, bool has_missing, double mv, string maskvar)
{
if (!_defineMode) {
SetErrMsg("Not in define mode");
Expand All @@ -458,7 +458,7 @@ int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string>
int rc = _DefineImplicitCoordVars(dim_names, coord_vars, coord_vars);
if (rc < 0) return (-1);

if (!_ValidDefineDataVar(varname, dim_names, coord_vars, units, type, compressed, maskvar)) { return (-1); }
if (!_ValidDefineDataVar(varname, dim_names, coord_vars, units, type, compressed, has_missing, maskvar)) { return (-1); }

//
// Dimensions must have previosly been defined
Expand Down Expand Up @@ -499,6 +499,8 @@ int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string>

if (!maskvar.empty()) {
_dataVars[varname] = DataVar(varname, units, type, wname, cratios, periodic, meshname, time_coord_var, DC::Mesh::NODE, mv, maskvar);
} else if (has_missing) {
_dataVars[varname] = DataVar(varname, units, type, wname, cratios, periodic, meshname, time_coord_var, DC::Mesh::NODE, mv);
} else {
_dataVars[varname] = DataVar(varname, units, type, wname, cratios, periodic, meshname, time_coord_var, DC::Mesh::NODE);
}
Expand All @@ -508,14 +510,15 @@ int VDC::_DefineDataVar(string varname, vector<string> dim_names, vector<string>

int VDC::DefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed)
{
return (VDC::_DefineDataVar(varname, dim_names, coord_vars, units, type, compressed, 0.0, ""));
return (VDC::_DefineDataVar(varname, dim_names, coord_vars, units, type, compressed, false, 0.0, ""));
}

int VDC::DefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, double mv, string maskvar

)
{
return (VDC::_DefineDataVar(varname, dim_names, coord_vars, units, type, true, mv, maskvar));
bool compressed = !maskvar.empty();
return (VDC::_DefineDataVar(varname, dim_names, coord_vars, units, type, compressed, true, mv, maskvar));
}

bool VDC::getDataVarInfo(string varname, DC::DataVar &datavar) const
Expand Down Expand Up @@ -1089,8 +1092,13 @@ bool VDC::_valid_mask_var(string varname, vector<DC::Dimension> dimensions, vect
return (true);
}

bool VDC::_ValidDefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed, string maskvar) const
bool VDC::_ValidDefineDataVar(string varname, vector<string> dim_names, vector<string> coord_vars, string units, XType type, bool compressed, bool has_missing, string maskvar) const
{
if (compressed && has_missing && maskvar.empty()) {
SetErrMsg("Mask variable required for compressed data with missing values");
return (false);
}

if (dim_names.size() > 4) {
SetErrMsg("Invalid number of dimensions");
return (false);
Expand Down
2 changes: 1 addition & 1 deletion lib/vdc/VDC_c.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ int VDC_DefineDataVar(VDC *p, const char *varname, const char **dimnames, size_t
for (int i = 0; i < coordvarsCount; i++) coordvars_v.push_back(string(coordvars[i]));
VDC_DEBUG_printff_nRun(": calling VDC::DefineDataVar(\"%s\", %s, %s, \"%s\", %s, %s);\n", varname, _stringVectorToString(dimnames_v).c_str(), _stringVectorToString(coordvars_v).c_str(), units,
_XTypeToString(_IntToXType(xtype)), _boolToStr(compressed));
int ret = p->DefineDataVar(string(varname), dimnames_v, coordvars_v, string(units), _IntToXType(xtype), compressed);
int ret = p->DefineDataVar(string(varname), dimnames_v, coordvars_v, string(units), _IntToXType(xtype), (bool)compressed);
VDC_DEBUG_printff_nRun(": return (%i);\n", ret);
if (ret < 0) VDC_DEBUG_printff_error(": Error message = \"%s\"\n", Wasp::MyBase::GetErrMsg());
VDC_DEBUG_printff_nRun(": Current coordvars = %s\n", _stringVectorToString(p->GetCoordVarNames()).c_str());
Expand Down
34 changes: 33 additions & 1 deletion lib/wasp/NetCDFCpp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,27 @@ using namespace VAPoR;
return (rc); \
}

namespace {

bool var_defs_match(int ncid, int varid, int dimids[], int ndimids, nc_type xtype)
{
int mydimids[NC_MAX_DIMS];
int myndims;
(void)nc_inq_varndims(ncid, varid, &myndims);
(void)nc_inq_vardimid(ncid, varid, mydimids);
for (int i = 0; i < ndimids; i++) {
if (mydimids[i] != dimids[i]) return (false);
}
nc_type myxtype;
(void)nc_inq_vartype(ncid, varid, &myxtype);
if (myxtype != xtype) return (false);

return (true);
}


}; // namespace

NetCDFCpp::NetCDFCpp()
{
_ncid = -1;
Expand Down Expand Up @@ -108,8 +129,19 @@ int NetCDFCpp::DefVar(string name, nc_type xtype, vector<string> dimnames)
MY_NC_ERR(rc, _path, "nc_inq_dimid(" + dimnames[i] + ")");
}

// Don't error out if variable already exists and has same definition
//
int varid;
int rc = nc_def_var(_ncid, name.c_str(), xtype, dimnames.size(), dimids, &varid);
int rc = nc_inq_varid(_ncid, name.c_str(), &varid);
if (rc == NC_NOERR) {
if (!var_defs_match(_ncid, varid, dimids, dimnames.size(), xtype)) {
MY_NC_ERR(NC_ENAMEINUSE, _path, "nc_def_var(" + name + ")");
return (NC_ENAMEINUSE);
}
return (NC_NOERR);
}

rc = nc_def_var(_ncid, name.c_str(), xtype, dimnames.size(), dimids, &varid);
MY_NC_ERR(rc, _path, "nc_def_var(" + name + ")");
return (NC_NOERR);
}
Expand Down

0 comments on commit bbf18a5

Please sign in to comment.