From 9929e7acf9e2ee50f077cebcdc42cd64f5f4da76 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 7 Nov 2018 11:33:02 -0700 Subject: [PATCH 01/46] moving att HDF5 stuff to libhdf5 --- include/hdf5internal.h | 6 ++++++ include/nc4internal.h | 1 + 2 files changed, 7 insertions(+) diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 1f8066e631..bf672e614d 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -56,6 +56,12 @@ typedef struct NC_HDF5_FILE_INFO hid_t hdfid; } NC_HDF5_FILE_INFO_T; +/** Strut to hold HDF5-specific info for attributes. */ +typedef struct NC_HDF5_ATT_INFO +{ + hid_t native_hdf_typeid; /* Native HDF5 datatype for attribute's data */ +} NC_HDF5_ATT_INFO_T; + /* These functions do HDF5 things. */ int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); diff --git a/include/nc4internal.h b/include/nc4internal.h index ae432525ad..56b33c2a75 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -131,6 +131,7 @@ typedef struct NC_ATT_INFO nc_bool_t created; /* True if attribute already created */ nc_type nc_typeid; /* netCDF type of attribute's data */ hid_t native_hdf_typeid; /* Native HDF5 datatype for attribute's data */ + void *format_dim_info; void *data; nc_vlen_t *vldata; /* only used for vlen */ char **stdata; /* only for string type. */ From 11d725faccb404d71f157f5df1dc63fbfcf738db Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 7 Nov 2018 13:45:51 -0700 Subject: [PATCH 02/46] allocating and freeing memory for hdf5-specific attribute info --- include/nc4internal.h | 2 +- libhdf5/hdf5attr.c | 6 +++++- libhdf5/hdf5internal.c | 4 +++- libhdf5/hdf5open.c | 4 ++++ libsrc4/nc4internal.c | 4 ++++ 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index 56b33c2a75..39a7674db9 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -131,7 +131,7 @@ typedef struct NC_ATT_INFO nc_bool_t created; /* True if attribute already created */ nc_type nc_typeid; /* netCDF type of attribute's data */ hid_t native_hdf_typeid; /* Native HDF5 datatype for attribute's data */ - void *format_dim_info; + void *format_att_info; void *data; nc_vlen_t *vldata; /* only used for vlen */ char **stdata; /* only for string type. */ diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 2779783290..87c3be1982 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -451,7 +451,11 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, { LOG((3, "adding attribute %s to the list...", norm_name)); if ((ret = nc4_att_list_add(attlist, norm_name, &att))) - BAIL (ret); + BAIL(ret); + + /* Allocate storage for the HDF5 specific att info. */ + if (!(att->format_att_info = calloc(1, sizeof(NC_HDF5_ATT_INFO_T)))) + BAIL(NC_ENOMEM); } /* Now fill in the metadata. */ diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 49e00a4af4..55e3c6126a 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -507,9 +507,11 @@ nc4_rec_grp_HDF5_del(NC_GRP_INFO_T *grp) i)))) return retval; - /* Close HDF5 resources associated with attributes. */ + /* Close HDF5 resources associated with global attributes. */ for (a = 0; a < ncindexsize(grp->att); a++) { + NC_HDF5_ATT_INFO_T hdf5_att; + att = (NC_ATT_INFO_T *)ncindexith(grp->att, a); assert(att); diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index de011a6868..40bb694634 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1593,6 +1593,10 @@ att_read_callbk(hid_t loc_id, const char *att_name, const H5A_info_t *ainfo, if ((retval = nc4_att_list_add(list, att_name, &att))) BAIL(-1); + /* Allocate storage for the HDF5 specific att info. */ + if (!(att->format_att_info = calloc(1, sizeof(NC_HDF5_ATT_INFO_T)))) + BAIL(-1); + /* Open the att by name. */ if ((attid = H5Aopen(loc_id, att_name, H5P_DEFAULT)) < 0) BAIL(-1); diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 61c1d12a7c..0c10cf8764 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1143,6 +1143,10 @@ att_free(NC_ATT_INFO_T *att) free(att->vldata); } + /* Free any format-sepecific info. */ + if (att->format_att_info) + free(att->format_att_info); + free(att); return NC_NOERR; } From 6f4b4ac80db6241f8f7c511fbbd62ba0f5b03679 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 7 Nov 2018 14:21:57 -0700 Subject: [PATCH 03/46] moving attribute HDF5 stuff to libhdf5 --- include/nc4internal.h | 2 +- libhdf5/hdf5file.c | 6 +++--- libhdf5/hdf5internal.c | 15 ++++++++++----- libhdf5/hdf5open.c | 26 +++++++++++++++----------- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index 39a7674db9..0f0e98d408 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -130,7 +130,7 @@ typedef struct NC_ATT_INFO nc_bool_t dirty; /* True if attribute modified */ nc_bool_t created; /* True if attribute already created */ nc_type nc_typeid; /* netCDF type of attribute's data */ - hid_t native_hdf_typeid; /* Native HDF5 datatype for attribute's data */ + /* hid_t native_hdf_typeid; /\* Native HDF5 datatype for attribute's data *\/ */ void *format_att_info; void *data; nc_vlen_t *vldata; /* only used for vlen */ diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index 21cfae9a1b..d07b553229 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -258,14 +258,14 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio) int nc4_close_hdf5_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio) { - NC_HDF5_FILE_INFO_T *hdf5_info; + /* NC_HDF5_FILE_INFO_T *hdf5_info; */ int retval; assert(h5 && h5->root_grp && h5->format_file_info); LOG((3, "%s: h5->path %s abort %d", __func__, h5->controller->path, abort)); - /* Get HDF5 specific info. */ - hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info; + /* /\* Get HDF5 specific info. *\/ */ + /* hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info; */ /* According to the docs, always end define mode on close. */ if (h5->flags & NC_INDEF) diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 55e3c6126a..5820fabfa9 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -510,13 +510,15 @@ nc4_rec_grp_HDF5_del(NC_GRP_INFO_T *grp) /* Close HDF5 resources associated with global attributes. */ for (a = 0; a < ncindexsize(grp->att); a++) { - NC_HDF5_ATT_INFO_T hdf5_att; + NC_HDF5_ATT_INFO_T *hdf5_att; att = (NC_ATT_INFO_T *)ncindexith(grp->att, a); - assert(att); + assert(att && att->format_att_info); + hdf5_att = (NC_HDF5_ATT_INFO_T *)att->format_att_info; /* Close the HDF5 typeid. */ - if (att->native_hdf_typeid && H5Tclose(att->native_hdf_typeid) < 0) + if (hdf5_att->native_hdf_typeid && + H5Tclose(hdf5_att->native_hdf_typeid) < 0) return NC_EHDFERR; } @@ -536,11 +538,14 @@ nc4_rec_grp_HDF5_del(NC_GRP_INFO_T *grp) for (a = 0; a < ncindexsize(var->att); a++) { + NC_HDF5_ATT_INFO_T *hdf5_att; att = (NC_ATT_INFO_T *)ncindexith(var->att, a); - assert(att); + assert(att && att->format_att_info); + hdf5_att = (NC_HDF5_ATT_INFO_T *)att->format_att_info; /* Close the HDF5 typeid if one is open. */ - if (att->native_hdf_typeid && H5Tclose(att->native_hdf_typeid) < 0) + if (hdf5_att->native_hdf_typeid && + H5Tclose(hdf5_att->native_hdf_typeid) < 0) return NC_EHDFERR; } } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 40bb694634..1bb5a9df8e 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1033,36 +1033,40 @@ get_netcdf_type(NC_FILE_INFO_T *h5, hid_t native_typeid, static int read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) { + NC_HDF5_ATT_INFO_T *hdf5_att; hid_t spaceid = 0, file_typeid = 0; hsize_t dims[1] = {0}; /* netcdf attributes always 1-D. */ - int retval = NC_NOERR; size_t type_size; int att_ndims; hssize_t att_npoints; H5T_class_t att_class; int fixed_len_string = 0; size_t fixed_size = 0; + int retval = NC_NOERR; - assert(att->hdr.name); + assert(att && att->hdr.name && att->format_att_info); LOG((5, "%s: att->hdr.id %d att->hdr.name %s att->nc_typeid %d att->len %d", __func__, att->hdr.id, att->hdr.name, (int)att->nc_typeid, att->len)); + /* Get HDF5-sepecific info stuct for this attribute. */ + hdf5_att = (NC_HDF5_ATT_INFO_T *)att->format_att_info; + /* Get type of attribute in file. */ if ((file_typeid = H5Aget_type(attid)) < 0) return NC_EATTMETA; - if ((att->native_hdf_typeid = H5Tget_native_type(file_typeid, + if ((hdf5_att->native_hdf_typeid = H5Tget_native_type(file_typeid, H5T_DIR_DEFAULT)) < 0) BAIL(NC_EHDFERR); - if ((att_class = H5Tget_class(att->native_hdf_typeid)) < 0) + if ((att_class = H5Tget_class(hdf5_att->native_hdf_typeid)) < 0) BAIL(NC_EATTMETA); if (att_class == H5T_STRING && - !H5Tis_variable_str(att->native_hdf_typeid)) + !H5Tis_variable_str(hdf5_att->native_hdf_typeid)) { fixed_len_string++; - if (!(fixed_size = H5Tget_size(att->native_hdf_typeid))) + if (!(fixed_size = H5Tget_size(hdf5_att->native_hdf_typeid))) BAIL(NC_EATTMETA); } - if ((retval = get_netcdf_type(grp->nc4_info, att->native_hdf_typeid, + if ((retval = get_netcdf_type(grp->nc4_info, hdf5_att->native_hdf_typeid, &(att->nc_typeid)))) BAIL(retval); @@ -1139,7 +1143,7 @@ read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) { if (!(att->vldata = malloc((unsigned int)(att->len * sizeof(hvl_t))))) BAIL(NC_ENOMEM); - if (H5Aread(attid, att->native_hdf_typeid, att->vldata) < 0) + if (H5Aread(attid, hdf5_att->native_hdf_typeid, att->vldata) < 0) BAIL(NC_EATTMETA); } else if (att->nc_typeid == NC_STRING) @@ -1167,7 +1171,7 @@ read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) BAIL(NC_ENOMEM); /* Read the fixed-len strings as one big block. */ - if (H5Aread(attid, att->native_hdf_typeid, contig_buf) < 0) { + if (H5Aread(attid, hdf5_att->native_hdf_typeid, contig_buf) < 0) { free(contig_buf); BAIL(NC_EATTMETA); } @@ -1192,7 +1196,7 @@ read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) else { /* Read variable-length string atts. */ - if (H5Aread(attid, att->native_hdf_typeid, att->stdata) < 0) + if (H5Aread(attid, hdf5_att->native_hdf_typeid, att->stdata) < 0) BAIL(NC_EATTMETA); } } @@ -1200,7 +1204,7 @@ read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) { if (!(att->data = malloc((unsigned int)(att->len * type_size)))) BAIL(NC_ENOMEM); - if (H5Aread(attid, att->native_hdf_typeid, att->data) < 0) + if (H5Aread(attid, hdf5_att->native_hdf_typeid, att->data) < 0) BAIL(NC_EATTMETA); } } From 8d31f5b806bec5a19f6b424fa77d86e2bd729bdd Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 7 Nov 2018 14:23:55 -0700 Subject: [PATCH 04/46] clean up --- include/nc4internal.h | 1 - libhdf5/hdf5file.c | 4 ---- 2 files changed, 5 deletions(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index 0f0e98d408..36ea92a5f3 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -130,7 +130,6 @@ typedef struct NC_ATT_INFO nc_bool_t dirty; /* True if attribute modified */ nc_bool_t created; /* True if attribute already created */ nc_type nc_typeid; /* netCDF type of attribute's data */ - /* hid_t native_hdf_typeid; /\* Native HDF5 datatype for attribute's data *\/ */ void *format_att_info; void *data; nc_vlen_t *vldata; /* only used for vlen */ diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index d07b553229..730417ec67 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -258,15 +258,11 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio) int nc4_close_hdf5_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio) { - /* NC_HDF5_FILE_INFO_T *hdf5_info; */ int retval; assert(h5 && h5->root_grp && h5->format_file_info); LOG((3, "%s: h5->path %s abort %d", __func__, h5->controller->path, abort)); - /* /\* Get HDF5 specific info. *\/ */ - /* hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info; */ - /* According to the docs, always end define mode on close. */ if (h5->flags & NC_INDEF) h5->flags ^= NC_INDEF; From 7e4138a11c47081d1fabf8c93d773a89f0de5eb1 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 7 Nov 2018 14:26:08 -0700 Subject: [PATCH 05/46] comment --- libsrc4/nc4internal.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 0c10cf8764..8d48c27467 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1143,7 +1143,8 @@ att_free(NC_ATT_INFO_T *att) free(att->vldata); } - /* Free any format-sepecific info. */ + /* Free any format-sepecific info. Some formats use this (ex. HDF5) + * and some don't (ex. HDF4). So it may be NULL. */ if (att->format_att_info) free(att->format_att_info); From 8aabd2020c9afa9cc7f16089afdec45265f336c7 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 07:09:11 -0700 Subject: [PATCH 06/46] moving HDF5 dim fields to hdf5internal.h from nc4internal.h --- include/hdf5internal.h | 7 +++++++ include/nc4internal.h | 5 +++-- libsrc4/nc4internal.c | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/include/hdf5internal.h b/include/hdf5internal.h index bf672e614d..e60991555c 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -56,6 +56,13 @@ typedef struct NC_HDF5_FILE_INFO hid_t hdfid; } NC_HDF5_FILE_INFO_T; +/* This is a struct to handle the dim metadata. */ +typedef struct NC_HDF5_DIM_INFO +{ + hid_t hdf_dimscaleid; /* Non-zero if a DIM_WITHOUT_VARIABLE dataset is in use (no coord var). */ + HDF5_OBJID_T hdf5_objid; +} NC_HDF5_DIM_INFO_T; + /** Strut to hold HDF5-specific info for attributes. */ typedef struct NC_HDF5_ATT_INFO { diff --git a/include/nc4internal.h b/include/nc4internal.h index 36ea92a5f3..5387cd639d 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -112,11 +112,12 @@ typedef struct NC_OBJ { typedef struct NC_DIM_INFO { NC_OBJ hdr; - struct NC_GRP_INFO* container; /* containing group */ + struct NC_GRP_INFO *container; /* containing group */ size_t len; nc_bool_t unlimited; /* True if the dimension is unlimited */ nc_bool_t extended; /* True if the dimension needs to be extended */ nc_bool_t too_long; /* True if len is too big to fit in local size_t. */ + void *format_dim_info; /* Pointer to format-specific dim info. */ hid_t hdf_dimscaleid; /* Non-zero if a DIM_WITHOUT_VARIABLE dataset is in use (no coord var). */ HDF5_OBJID_T hdf5_objid; struct NC_VAR_INFO *coord_var; /* The coord var, if it exists. */ @@ -125,7 +126,7 @@ typedef struct NC_DIM_INFO typedef struct NC_ATT_INFO { NC_OBJ hdr; - struct NC_OBJ* container; /* containing group|var */ + struct NC_OBJ *container; /* containing group|var */ int len; nc_bool_t dirty; /* True if attribute modified */ nc_bool_t created; /* True if attribute already created */ diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 8d48c27467..e82d2a2092 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1277,6 +1277,10 @@ dim_free(NC_DIM_INFO_T *dim) if (dim->hdr.name) free(dim->hdr.name); + /* Release any format-specific information. */ + if (dim->format_dim_info) + free(dim->format_dim_info); + free(dim); return NC_NOERR; } From 9373989931f0b6b172b6768b1039f4e32207e66a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 08:51:46 -0700 Subject: [PATCH 07/46] allocating and freeing space for HDF5-specific dim info --- libhdf5/hdf5dim.c | 4 ++++ libhdf5/hdf5open.c | 4 ++++ libhdf5/nc4hdf.c | 7 ++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index fe8d959119..ed21edd95c 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -104,6 +104,10 @@ NC4_def_dim(int ncid, const char *name, size_t len, int *idp) if ((retval = nc4_dim_list_add(grp, norm_name, len, -1, &dim))) return retval; + /* Create struct for HDF5-specific dim info. */ + if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T)))) + return NC_ENOMEM; + /* Pass back the dimid. */ if (idp) *idp = dim->hdr.id; diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 1bb5a9df8e..1bc66bb590 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1738,6 +1738,10 @@ read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, if ((retval = nc4_dim_list_add(grp, obj_name, len, assigned_id, &new_dim))) BAIL(retval); + /* Create struct for HDF5-specific dim info. */ + if (!(new_dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T)))) + BAIL(NC_ENOMEM); + new_dim->too_long = too_long; dimscale_created++; diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 111f0e2dbf..dc12e6fb2c 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -2891,6 +2891,7 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type, * * @returns NC_NOERR No error. * @returns NC_EHDFERR HDF5 returned an error. + * @returns NC_ENOMEM Out of memory. * @author Ed Hartnett */ int @@ -3049,11 +3050,15 @@ nc4_rec_match_dimscales(NC_GRP_INFO_T *grp) char phony_dim_name[NC_MAX_NAME + 1]; sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid); LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name)); - if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim))) { + if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim))) + { free(h5dimlenmax); free(h5dimlen); return retval; } + /* Create struct for HDF5-specific dim info. */ + if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T)))) + return NC_ENOMEM; if (h5dimlenmax[d] == H5S_UNLIMITED) dim->unlimited = NC_TRUE; } From 008d4ee796ac70f2d73eece93fc2f309d0d1b3e0 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 08:57:44 -0700 Subject: [PATCH 08/46] starting to find HDF5 specific dim info --- libhdf5/hdf5internal.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 5820fabfa9..bfda8f277f 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -341,11 +341,13 @@ nc4_break_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *coord_var, NC_DIM_INFO_T int delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *dim) { + NC_HDF5_DIM_INFO_T *hdf5_dim; int retval; - assert(grp && dim); + assert(grp && dim && dim->format_dim_info); LOG((2, "%s: deleting dimscale dataset %s dimid %d", __func__, dim->hdr.name, dimid)); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; /* Detach dimscale from any variables using it */ if ((retval = rec_detach_scales(grp, dimid, dim->hdf_dimscaleid)) < 0) @@ -553,8 +555,11 @@ nc4_rec_grp_HDF5_del(NC_GRP_INFO_T *grp) /* Close HDF5 resources associated with dims. */ for (i = 0; i < ncindexsize(grp->dim); i++) { + NC_HDF5_DIM_INFO_T *hdf5_dim; + dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, i); - assert(dim); + assert(dim && dim->format_dim_info); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; /* If this is a dim without a coordinate variable, then close * the HDF5 DIM_WITHOUT_VARIABLE dataset associated with this From 3e8072352580f3414a9194b17baf6fee3833c5be Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 09:20:01 -0700 Subject: [PATCH 09/46] continuing to find HDF5 specific dim info --- libhdf5/hdf5open.c | 7 +++++-- libhdf5/nc4hdf.c | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 1bc66bb590..703b9f55db 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1699,6 +1699,7 @@ read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, NC_DIM_INFO_T **dim) { NC_DIM_INFO_T *new_dim; /* Dimension added to group */ + NC_HDF5_DIM_INFO_T *new_hdf5_dim; /* HDF5-specific dim info. */ char dimscale_name_att[NC_MAX_NAME + 1] = ""; /* Dimscale name, for checking if dim without var */ htri_t attr_exists = -1; /* Flag indicating hidden attribute exists */ hid_t attid = -1; /* ID of hidden attribute (to store dim ID) */ @@ -1741,6 +1742,7 @@ read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, /* Create struct for HDF5-specific dim info. */ if (!(new_dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T)))) BAIL(NC_ENOMEM); + new_hdf5_dim = (NC_HDF5_DIM_INFO_T *)new_dim->format_dim_info; new_dim->too_long = too_long; @@ -1775,8 +1777,9 @@ read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, /* Hold open the dataset, since the dimension doesn't have a * coordinate variable */ + new_hdf5_dim->hdf_dimscaleid = datasetid; new_dim->hdf_dimscaleid = datasetid; - H5Iinc_ref(new_dim->hdf_dimscaleid); /* Increment number of objects using ID */ + H5Iinc_ref(new_hdf5_dim->hdf_dimscaleid); /* Increment number of objects using ID */ } } @@ -1854,7 +1857,7 @@ read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, /* Add a var to the linked list, and get its metadata, * unless this is one of those funny dimscales that are a * dimension in netCDF but not a variable. (Spooky!) */ - if (NULL == dim || (dim && !dim->hdf_dimscaleid)) + if (!dim || (dim && !dim->hdf_dimscaleid)) if ((retval = read_var(grp, datasetid, obj_name, ndims, dim))) BAIL(retval); diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index dc12e6fb2c..11cd7c1894 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1399,9 +1399,11 @@ attach_dimscales(NC_GRP_INFO_T *grp) { if (!var->dimscale_attached[d]) { + NC_HDF5_DIM_INFO_T *hdf5_dim; hid_t dim_datasetid; /* Dataset ID for dimension */ dim1 = var->dim[d]; - assert(dim1 && dim1->hdr.id == var->dimids[d]); + assert(dim1 && dim1->hdr.id == var->dimids[d] && dim1->format_dim_info); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim1->format_dim_info; LOG((2, "%s: attaching scale for dimid %d to var %s", __func__, var->dimids[d], var->hdr.name)); @@ -1585,20 +1587,27 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) } } - /* Check dims if the variable will be replaced, so that the dimensions - * will be de-attached and re-attached correctly. */ - /* (Note: There's a temptation to merge this loop over the dimensions with - * the prior loop over dimensions, but that blurs the line over the - * purpose of them, so they are currently separate. If performance - * becomes an issue here, it would be possible to merge them. -QAK) + /* Check dims if the variable will be replaced, so that the + * dimensions will be de-attached and re-attached correctly. (Note: + * There's a temptation to merge this loop over the dimensions with + * the prior loop over dimensions, but that blurs the line over the + * purpose of them, so they are currently separate. If performance + * becomes an issue here, it would be possible to merge them. -QAK) */ if (replace_existing_var) { - NC_DIM_INFO_T *d1; int i; - for(i=0;idim);i++) { - if((d1 = (NC_DIM_INFO_T*)ncindexith(grp->dim,i)) == NULL) continue; + for (i = 0; i < ncindexsize(grp->dim); i++) + { + NC_DIM_INFO_T *d1; + NC_HDF5_DIM_INFO_T *hdf5_d1; + + /* Get info about the dim, including HDF5-specific info. */ + d1 = (NC_DIM_INFO_T *)ncindexith(grp->dim, i); + assert(d1 && d1->format_dim_info && d1->hdr.name); + hdf5_d1 = (NC_HDF5_DIM_INFO_T *)d1->format_dim_info; + if (!strcmp(d1->hdr.name, var->hdr.name)) { nc_bool_t exists; @@ -1647,11 +1656,14 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) /* If this is a regular var, detach all its dim scales. */ for (d = 0; d < var->ndims; d++) + { if (var->dimscale_attached[d]) { hid_t dim_datasetid; /* Dataset ID for dimension */ NC_DIM_INFO_T *dim1 = var->dim[d]; - assert(dim1 && dim1->hdr.id == var->dimids[d]); + NC_HDF5_DIM_INFO_T *hdf5_dim1; + assert(dim1 && dim1->hdr.id == var->dimids[d] && dim1->format_dim_info); + hdf5_dim1 = (NC_HDF5_DIM_INFO_T *)dim1->format_dim_info; /* Find dataset ID for dimension */ if (dim1->coord_var) @@ -1664,6 +1676,7 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) BAIL(NC_EHDFERR); var->dimscale_attached[d] = NC_FALSE; } + } } } @@ -1745,8 +1758,12 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) static int write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) { + NC_HDF5_DIM_INFO_T *hdf5_dim; int retval; + assert(dim && dim->format_dim_info); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; + /* If there's no dimscale dataset for this dim, create one, * and mark that it should be hidden from netCDF as a * variable. (That is, it should appear as a dimension From e89ad03b249ffba425a95635fc462b11fb45a6e8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:02:17 -0700 Subject: [PATCH 10/46] more looking up HDF5-specific dim info --- libhdf5/hdf5var.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 8dc028cd29..ad822f1224 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -404,10 +404,13 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, for (d = 0; d < ndims; d++) { NC_GRP_INFO_T *dim_grp; + NC_HDF5_DIM_INFO_T *hdf5_dim; /* Look up each dimension */ if ((retval = nc4_find_dim(grp, dimidsp[d], &dim, &dim_grp))) BAIL(retval); + assert(dim && dim->format_dim_info); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; /* Check for dim index 0 having the same name, in the same group */ if (d == 0 && dim_grp == grp && strcmp(dim->hdr.name, norm_name) == 0) @@ -1053,16 +1056,22 @@ NC4_rename_var(int ncid, int varid, const char *name) return NC_ENOTINDEFINE; /* Change the HDF5 file, if this var has already been created - there. Should we check here to ensure there is not already a - dimscale dataset of name name??? */ + there. */ if (var->created) { - /* Is there an existing dimscale-only dataset of this name? If - * so, it must be deleted. */ - if (var->ndims && var->dim[0]->hdf_dimscaleid) + if (var->ndims) { - if ((retval = delete_existing_dimscale_dataset(grp, var->dim[0]->hdr.id, var->dim[0]))) - return retval; + NC_HDF5_DIM_INFO_T *hdf5_d0; + hdf5_d0 = (NC_HDF5_DIM_INFO_T *)var->dim[0]->format_dim_info; + + /* Is there an existing dimscale-only dataset of this name? If + * so, it must be deleted. */ + if (var->dim[0]->hdf_dimscaleid) + { + if ((retval = delete_existing_dimscale_dataset(grp, var->dim[0]->hdr.id, + var->dim[0]))) + return retval; + } } LOG((3, "Moving dataset %s to %s", var->hdr.name, name)); From b85eec1bb84673acf5c1f38aa0cde503609bd616 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:07:59 -0700 Subject: [PATCH 11/46] starting to set hdf5-specific dim info struct --- libhdf5/nc4hdf.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 11cd7c1894..1a25821acb 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1808,9 +1808,10 @@ write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) /* Create the dataset that will be the dimension scale. */ LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__, dim->hdr.name)); - if ((dim->hdf_dimscaleid = H5Dcreate1(grp->hdf_grpid, dim->hdr.name, H5T_IEEE_F32BE, - spaceid, create_propid)) < 0) + if ((hdf5_dim->hdf_dimscaleid = H5Dcreate1(grp->hdf_grpid, dim->hdr.name, H5T_IEEE_F32BE, + spaceid, create_propid)) < 0) BAIL(NC_EHDFERR); + dim->hdf_dimscaleid = hdf5_dim->hdf_dimscaleid; /* Close the spaceid and create_propid. */ if (H5Sclose(spaceid) < 0) From 92dc85b8026c7152aaa9a6e9a6f0af14d934dfa3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:11:46 -0700 Subject: [PATCH 12/46] continuing to set hdf5-specific dim info struct --- libhdf5/hdf5internal.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index bfda8f277f..2bb00d281e 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -356,6 +356,7 @@ delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *d /* Close the HDF5 dataset */ if (H5Dclose(dim->hdf_dimscaleid) < 0) return NC_EHDFERR; + hdf5_dim->hdf_dimscaleid = 0; dim->hdf_dimscaleid = 0; /* Now delete the dataset. */ @@ -404,9 +405,10 @@ nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) for (g = grp; g && !finished; g = g->parent) { NC_DIM_INFO_T *dim1; - for(k=0;kdim);k++) + for (k = 0; k < ncindexsize(g->dim); k++) { - if((dim1 = (NC_DIM_INFO_T*)ncindexith(g->dim,k)) == NULL) continue; + dim1 = (NC_DIM_INFO_T *)ncindexith(g->dim, k); + assert(dim1); if (var->dimids[d] == dim1->hdr.id) { hid_t dim_datasetid; /* Dataset ID for dimension */ From dab52a97ee27b222add58e958ef945e97ba50895 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:16:55 -0700 Subject: [PATCH 13/46] more looking up of HDF5 dim info --- libhdf5/hdf5internal.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 2bb00d281e..b69f8daea4 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -380,14 +380,17 @@ delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *d int nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) { + NC_HDF5_DIM_INFO_T *hdf5_dim; int need_to_reattach_scales = 0; int retval = NC_NOERR; - assert(grp && var && dim); - LOG((3, "%s: dim->hdr.name %s var->hdr.name %s", __func__, dim->hdr.name, var->hdr.name)); + assert(grp && var && dim && dim->format_dim_info); + LOG((3, "%s: dim->hdr.name %s var->hdr.name %s", __func__, dim->hdr.name, + var->hdr.name)); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; /* Detach dimscales from the [new] coordinate variable */ - if(var->dimscale_attached) + if (var->dimscale_attached) { int dims_detached = 0; int finished = 0; @@ -397,7 +400,7 @@ nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) for (d = 0; d < var->ndims && !finished; d++) { /* Is there a dimscale attached to this axis? */ - if(var->dimscale_attached[d]) + if (var->dimscale_attached[d]) { NC_GRP_INFO_T *g; int k; @@ -405,10 +408,14 @@ nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) for (g = grp; g && !finished; g = g->parent) { NC_DIM_INFO_T *dim1; + NC_HDF5_DIM_INFO_T *hdf5_dim1; + for (k = 0; k < ncindexsize(g->dim); k++) { dim1 = (NC_DIM_INFO_T *)ncindexith(g->dim, k); - assert(dim1); + assert(dim1 && dim1->format_dim_info); + hdf5_dim1 = (NC_HDF5_DIM_INFO_T *)dim1->format_dim_info; + if (var->dimids[d] == dim1->hdr.id) { hid_t dim_datasetid; /* Dataset ID for dimension */ From bff06f2c4d837652e266066b7a75c6dc1f5ecd7a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:22:37 -0700 Subject: [PATCH 14/46] starting to use HDF5-specific dim info --- libhdf5/hdf5internal.c | 1 + libhdf5/hdf5var.c | 1 + libhdf5/nc4hdf.c | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index b69f8daea4..777ff0c8f7 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -458,6 +458,7 @@ nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) if (H5Dclose(dim->hdf_dimscaleid) < 0) BAIL(NC_EHDFERR); dim->hdf_dimscaleid = 0; + hdf5_dim->hdf_dimscaleid = 0; /* Now delete the dimscale's dataset (it will be recreated later, if necessary) */ diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index ad822f1224..e5e9dcb31f 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -431,6 +431,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, if (H5Dclose(dim->hdf_dimscaleid) < 0) BAIL(NC_EHDFERR); dim->hdf_dimscaleid = 0; + hdf5_dim->hdf_dimscaleid = 0; /* Now delete the DIM_WITHOUT_VARIABLE dataset (it will be * recreated later, if necessary). */ diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 1a25821acb..a6c2363b4e 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1823,7 +1823,7 @@ write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) * be shown to the user as a variable. It is hidden. It is * a DIM WITHOUT A VARIABLE! */ sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len); - if (H5DSset_scale(dim->hdf_dimscaleid, dimscale_wo_var) < 0) + if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0) BAIL(NC_EHDFERR); } @@ -1864,7 +1864,7 @@ write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) * creation order. This can be necessary when dims and their * coordinate variables were created in different order. */ if (write_dimid && dim->hdf_dimscaleid) - if ((retval = write_netcdf4_dimid(dim->hdf_dimscaleid, dim->hdr.id))) + if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id))) BAIL(retval); return NC_NOERR; From 0ef97a84f12e8c1b27e080580558f17ce2eba36c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:27:34 -0700 Subject: [PATCH 15/46] cleanup --- libhdf5/hdf5dim.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index ed21edd95c..6847968057 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -208,7 +208,8 @@ NC4_rename_dim(int ncid, int dimid, const char *name) { NC *nc; NC_GRP_INFO_T *grp; - NC_DIM_INFO_T *dim, *tmpdim; + NC_DIM_INFO_T *dim; + NC_HDF5_DIM_INFO_T *hdf5_dim; NC_FILE_INFO_T *h5; char norm_name[NC_MAX_NAME + 1]; int retval; @@ -234,15 +235,14 @@ NC4_rename_dim(int ncid, int dimid, const char *name) return retval; /* Get the original dim */ - if((retval=nc4_find_dim(grp,dimid,&dim,NULL)) != NC_NOERR) - return retval; - if(dim == NULL) /* No such dim */ - return NC_EBADDIM; + if ((retval = nc4_find_dim(grp, dimid, &dim, NULL))) + return retval; + if (!dim) /* No such dim */ + return NC_EBADDIM; /* Check if new name is in use */ - tmpdim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,norm_name); - if(tmpdim != NULL) - return NC_ENAMEINUSE; + if (ncindexlookup(grp->dim, norm_name)) + return NC_ENAMEINUSE; /* Check for renaming dimension w/o variable */ if (dim->hdf_dimscaleid) From 30da28edff1fa0ee0ac8c43994667763630fd210 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:37:32 -0700 Subject: [PATCH 16/46] using hdf5-specific dim info --- libhdf5/hdf5dim.c | 21 ++++++++++----------- libhdf5/nc4hdf.c | 8 ++++---- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index 6847968057..f9ec06ed99 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -234,20 +234,19 @@ NC4_rename_dim(int ncid, int dimid, const char *name) if ((retval = nc4_check_name(name, norm_name))) return retval; - /* Get the original dim */ + /* Get the original dim. */ if ((retval = nc4_find_dim(grp, dimid, &dim, NULL))) return retval; - if (!dim) /* No such dim */ - return NC_EBADDIM; + assert(dim && dim->format_dim_info); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; - /* Check if new name is in use */ + /* Check if new name is in use. */ if (ncindexlookup(grp->dim, norm_name)) return NC_ENAMEINUSE; - /* Check for renaming dimension w/o variable */ - if (dim->hdf_dimscaleid) + /* Check for renaming dimension w/o variable. */ + if (hdf5_dim->hdf_dimscaleid) { - /* Sanity check */ assert(!dim->coord_var); LOG((3, "dim %s is a dim without variable", dim->hdr.name)); @@ -265,14 +264,14 @@ NC4_rename_dim(int ncid, int dimid, const char *name) LOG((3, "dim is now named %s", dim->hdr.name)); dim->hdr.hashkey = NC_hashmapkey(dim->hdr.name,strlen(dim->hdr.name)); /* Fix hash key */ - if(!ncindexrebuild(grp->dim)) - return NC_EINTERNAL; + if (!ncindexrebuild(grp->dim)) + return NC_EINTERNAL; /* Check if dimension was a coordinate variable, but names are - * different now */ + * different now. */ if (dim->coord_var && strcmp(dim->hdr.name, dim->coord_var->hdr.name)) { - /* Break up the coordinate variable */ + /* Break up the coordinate variable. */ if ((retval = nc4_break_coord_var(grp, dim->coord_var, dim))) return retval; } diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index a6c2363b4e..0a619b041b 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1399,11 +1399,11 @@ attach_dimscales(NC_GRP_INFO_T *grp) { if (!var->dimscale_attached[d]) { - NC_HDF5_DIM_INFO_T *hdf5_dim; + NC_HDF5_DIM_INFO_T *hdf5_dim1; hid_t dim_datasetid; /* Dataset ID for dimension */ dim1 = var->dim[d]; assert(dim1 && dim1->hdr.id == var->dimids[d] && dim1->format_dim_info); - hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim1->format_dim_info; + hdf5_dim1 = (NC_HDF5_DIM_INFO_T *)dim1->format_dim_info; LOG((2, "%s: attaching scale for dimid %d to var %s", __func__, var->dimids[d], var->hdr.name)); @@ -1412,7 +1412,7 @@ attach_dimscales(NC_GRP_INFO_T *grp) if (dim1->coord_var) dim_datasetid = dim1->coord_var->hdf_datasetid; else - dim_datasetid = dim1->hdf_dimscaleid; + dim_datasetid = hdf5_dim1->hdf_dimscaleid; if(!(dim_datasetid > 0)) assert(dim_datasetid > 0); if (H5DSattach_scale(var->hdf_datasetid, dim_datasetid, d) < 0) @@ -1622,7 +1622,7 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) if (d1->coord_var) dim_datasetid = d1->coord_var->hdf_datasetid; else - dim_datasetid = d1->hdf_dimscaleid; + dim_datasetid = hdf5_d1->hdf_dimscaleid; assert(dim_datasetid > 0); /* If we're replacing an existing dimscale dataset, go to From 7ab71a69bbb51572922f46bd1a35c2073228977c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:40:39 -0700 Subject: [PATCH 17/46] more use of HDF5-specific dim info --- libhdf5/nc4hdf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 0a619b041b..3314964966 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1669,7 +1669,7 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) if (dim1->coord_var) dim_datasetid = dim1->coord_var->hdf_datasetid; else - dim_datasetid = dim1->hdf_dimscaleid; + dim_datasetid = hdf5_dim1->hdf_dimscaleid; assert(dim_datasetid > 0); if (H5DSdetach_scale(var->hdf_datasetid, dim_datasetid, d) < 0) @@ -1768,7 +1768,7 @@ write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) * and mark that it should be hidden from netCDF as a * variable. (That is, it should appear as a dimension * without an associated variable.) */ - if (!dim->hdf_dimscaleid) + if (!hdf5_dim->hdf_dimscaleid) { hid_t spaceid, create_propid; hsize_t dims[1], max_dims[1], chunk_dims[1] = {1}; @@ -1863,7 +1863,7 @@ write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) * the dimid that the dimension would otherwise receive based on * creation order. This can be necessary when dims and their * coordinate variables were created in different order. */ - if (write_dimid && dim->hdf_dimscaleid) + if (write_dimid && hdf5_dim->hdf_dimscaleid) if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id))) BAIL(retval); From 1dd9004a11bd745303bcb1973a0fe34e8b4f3fa2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:43:36 -0700 Subject: [PATCH 18/46] more use of HDF5-specific dim info --- libhdf5/hdf5var.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index e5e9dcb31f..f2e21c5e60 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -421,14 +421,15 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, /* Use variable's dataset ID for the dimscale ID. So delete * the HDF5 DIM_WITHOUT_VARIABLE dataset that was created for * this dim. */ - if (dim->hdf_dimscaleid) + if (hdf5_dim->hdf_dimscaleid) { /* Detach dimscale from any variables using it */ - if ((retval = rec_detach_scales(grp, dimidsp[d], dim->hdf_dimscaleid)) < 0) + if ((retval = rec_detach_scales(grp, dimidsp[d], + hdf5_dim->hdf_dimscaleid)) < 0) BAIL(retval); /* Close the HDF5 DIM_WITHOUT_VARIABLE dataset. */ - if (H5Dclose(dim->hdf_dimscaleid) < 0) + if (H5Dclose(hdf5_dim->hdf_dimscaleid) < 0) BAIL(NC_EHDFERR); dim->hdf_dimscaleid = 0; hdf5_dim->hdf_dimscaleid = 0; @@ -1067,7 +1068,7 @@ NC4_rename_var(int ncid, int varid, const char *name) /* Is there an existing dimscale-only dataset of this name? If * so, it must be deleted. */ - if (var->dim[0]->hdf_dimscaleid) + if (hdf5_d0->hdf_dimscaleid) { if ((retval = delete_existing_dimscale_dataset(grp, var->dim[0]->hdr.id, var->dim[0]))) From cc02ec3aa34203beddd3d85b89ced9de60fe709e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:48:02 -0700 Subject: [PATCH 19/46] more use of HDF5-specific dim info --- libhdf5/hdf5open.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 703b9f55db..d8906013fe 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1825,6 +1825,7 @@ read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, const H5G_stat_t *statbuf) { NC_DIM_INFO_T *dim = NULL; /* Dimension created for scales */ + NC_HDF5_DIM_INFO_T *hdf5_dim; hid_t spaceid = 0; int ndims; htri_t is_scale; @@ -1852,12 +1853,13 @@ read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, if ((retval = read_scale(grp, datasetid, obj_name, statbuf, dims[0], max_dims[0], &dim))) BAIL(retval); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; } /* Add a var to the linked list, and get its metadata, * unless this is one of those funny dimscales that are a * dimension in netCDF but not a variable. (Spooky!) */ - if (!dim || (dim && !dim->hdf_dimscaleid)) + if (!dim || (dim && !hdf5_dim->hdf_dimscaleid)) if ((retval = read_var(grp, datasetid, obj_name, ndims, dim))) BAIL(retval); From 856e4ead03188a7f51140b15b0ef00ec24940698 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 10:55:21 -0700 Subject: [PATCH 20/46] moved hdf_dimscaleid to hdf5-specific dim info --- include/nc4internal.h | 2 +- libhdf5/hdf5internal.c | 16 +++++++--------- libhdf5/hdf5open.c | 1 - libhdf5/hdf5var.c | 1 - libhdf5/nc4hdf.c | 1 - 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index 5387cd639d..0fc81fa214 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -118,7 +118,7 @@ typedef struct NC_DIM_INFO nc_bool_t extended; /* True if the dimension needs to be extended */ nc_bool_t too_long; /* True if len is too big to fit in local size_t. */ void *format_dim_info; /* Pointer to format-specific dim info. */ - hid_t hdf_dimscaleid; /* Non-zero if a DIM_WITHOUT_VARIABLE dataset is in use (no coord var). */ + /* hid_t hdf_dimscaleid; /\* Non-zero if a DIM_WITHOUT_VARIABLE dataset is in use (no coord var). *\/ */ HDF5_OBJID_T hdf5_objid; struct NC_VAR_INFO *coord_var; /* The coord var, if it exists. */ } NC_DIM_INFO_T; diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 777ff0c8f7..6efd4bfe33 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -279,7 +279,7 @@ nc4_break_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *coord_var, NC_DIM_INFO_T /* Sanity checks */ assert(grp && coord_var && dim && dim->coord_var == coord_var && coord_var->dim[0] == dim && coord_var->dimids[0] == dim->hdr.id && - !dim->hdf_dimscaleid); + !((NC_HDF5_DIM_INFO_T *)(dim->format_dim_info))->hdf_dimscaleid); LOG((3, "%s dim %s was associated with var %s, but now has different name", __func__, dim->hdr.name, coord_var->hdr.name)); @@ -350,14 +350,13 @@ delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *d hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; /* Detach dimscale from any variables using it */ - if ((retval = rec_detach_scales(grp, dimid, dim->hdf_dimscaleid)) < 0) + if ((retval = rec_detach_scales(grp, dimid, hdf5_dim->hdf_dimscaleid)) < 0) return retval; /* Close the HDF5 dataset */ - if (H5Dclose(dim->hdf_dimscaleid) < 0) + if (H5Dclose(hdf5_dim->hdf_dimscaleid) < 0) return NC_EHDFERR; hdf5_dim->hdf_dimscaleid = 0; - dim->hdf_dimscaleid = 0; /* Now delete the dataset. */ if (H5Gunlink(grp->hdf_grpid, dim->hdr.name) < 0) @@ -424,7 +423,7 @@ nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) if (dim1->coord_var) dim_datasetid = dim1->coord_var->hdf_datasetid; else - dim_datasetid = dim1->hdf_dimscaleid; + dim_datasetid = hdf5_dim1->hdf_dimscaleid; /* dim_datasetid may be 0 in some cases when * renames of dims and vars are happening. In @@ -452,12 +451,11 @@ nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) } /* Use variable's dataset ID for the dimscale ID. */ - if (dim->hdf_dimscaleid && grp != NULL) + if (hdf5_dim->hdf_dimscaleid && grp != NULL) { LOG((3, "closing and unlinking dimscale dataset %s", dim->hdr.name)); - if (H5Dclose(dim->hdf_dimscaleid) < 0) + if (H5Dclose(hdf5_dim->hdf_dimscaleid) < 0) BAIL(NC_EHDFERR); - dim->hdf_dimscaleid = 0; hdf5_dim->hdf_dimscaleid = 0; /* Now delete the dimscale's dataset @@ -574,7 +572,7 @@ nc4_rec_grp_HDF5_del(NC_GRP_INFO_T *grp) /* If this is a dim without a coordinate variable, then close * the HDF5 DIM_WITHOUT_VARIABLE dataset associated with this * dim. */ - if (dim->hdf_dimscaleid && H5Dclose(dim->hdf_dimscaleid) < 0) + if (hdf5_dim->hdf_dimscaleid && H5Dclose(hdf5_dim->hdf_dimscaleid) < 0) return NC_EHDFERR; } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index d8906013fe..a127ea5eba 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1778,7 +1778,6 @@ read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, /* Hold open the dataset, since the dimension doesn't have a * coordinate variable */ new_hdf5_dim->hdf_dimscaleid = datasetid; - new_dim->hdf_dimscaleid = datasetid; H5Iinc_ref(new_hdf5_dim->hdf_dimscaleid); /* Increment number of objects using ID */ } } diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index f2e21c5e60..d81f660a2b 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -431,7 +431,6 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, /* Close the HDF5 DIM_WITHOUT_VARIABLE dataset. */ if (H5Dclose(hdf5_dim->hdf_dimscaleid) < 0) BAIL(NC_EHDFERR); - dim->hdf_dimscaleid = 0; hdf5_dim->hdf_dimscaleid = 0; /* Now delete the DIM_WITHOUT_VARIABLE dataset (it will be diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 3314964966..1223f377be 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1811,7 +1811,6 @@ write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) if ((hdf5_dim->hdf_dimscaleid = H5Dcreate1(grp->hdf_grpid, dim->hdr.name, H5T_IEEE_F32BE, spaceid, create_propid)) < 0) BAIL(NC_EHDFERR); - dim->hdf_dimscaleid = hdf5_dim->hdf_dimscaleid; /* Close the spaceid and create_propid. */ if (H5Sclose(spaceid) < 0) From e88b98daaa08b73aa100cf4e850e1659454f0bee Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 11:05:20 -0700 Subject: [PATCH 21/46] moving hdf5_objid field to hdf5-specific dim info --- libhdf5/hdf5open.c | 4 ++++ libhdf5/nc4hdf.c | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index a127ea5eba..b5dc3aed08 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1748,6 +1748,10 @@ read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, dimscale_created++; + new_hdf5_dim->hdf5_objid.fileno[0] = statbuf->fileno[0]; + new_hdf5_dim->hdf5_objid.fileno[1] = statbuf->fileno[1]; + new_hdf5_dim->hdf5_objid.objno[0] = statbuf->objno[0]; + new_hdf5_dim->hdf5_objid.objno[1] = statbuf->objno[1]; new_dim->hdf5_objid.fileno[0] = statbuf->fileno[0]; new_dim->hdf5_objid.fileno[1] = statbuf->fileno[1]; new_dim->hdf5_objid.objno[0] = statbuf->objno[0]; diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 1223f377be..5be87398d1 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -2978,9 +2978,15 @@ nc4_rec_match_dimscales(NC_GRP_INFO_T *grp) * match. */ for (g = grp; g && !finished; g = g->parent) { - for(j=0;jdim);j++) + for (j = 0; j < ncindexsize(g->dim); j++) { - if((dim = (NC_DIM_INFO_T*)ncindexith(g->dim,j)) == NULL) continue; + NC_HDF5_DIM_INFO_T *hdf5_dim; + dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j); + assert(dim && dim->format_dim_info); + hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; + + /* Check for exact match to find identical + * objects in HDF5 file. */ if (var->dimscale_hdf5_objids[d].fileno[0] == dim->hdf5_objid.fileno[0] && var->dimscale_hdf5_objids[d].objno[0] == dim->hdf5_objid.objno[0] && var->dimscale_hdf5_objids[d].fileno[1] == dim->hdf5_objid.fileno[1] && From 7534cb24fea555caff4cd33bf39da06e24815480 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 8 Nov 2018 11:12:38 -0700 Subject: [PATCH 22/46] cleanup --- include/nc4internal.h | 2 -- libhdf5/hdf5open.c | 4 ---- libhdf5/nc4hdf.c | 26 ++++++++++++++------------ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index 0fc81fa214..afcffa3d31 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -118,8 +118,6 @@ typedef struct NC_DIM_INFO nc_bool_t extended; /* True if the dimension needs to be extended */ nc_bool_t too_long; /* True if len is too big to fit in local size_t. */ void *format_dim_info; /* Pointer to format-specific dim info. */ - /* hid_t hdf_dimscaleid; /\* Non-zero if a DIM_WITHOUT_VARIABLE dataset is in use (no coord var). *\/ */ - HDF5_OBJID_T hdf5_objid; struct NC_VAR_INFO *coord_var; /* The coord var, if it exists. */ } NC_DIM_INFO_T; diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index b5dc3aed08..7ba05a850b 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1752,10 +1752,6 @@ read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, new_hdf5_dim->hdf5_objid.fileno[1] = statbuf->fileno[1]; new_hdf5_dim->hdf5_objid.objno[0] = statbuf->objno[0]; new_hdf5_dim->hdf5_objid.objno[1] = statbuf->objno[1]; - new_dim->hdf5_objid.fileno[0] = statbuf->fileno[0]; - new_dim->hdf5_objid.fileno[1] = statbuf->fileno[1]; - new_dim->hdf5_objid.objno[0] = statbuf->objno[0]; - new_dim->hdf5_objid.objno[1] = statbuf->objno[1]; /* If the dimscale has an unlimited dimension, then this dimension * is unlimited. */ diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 5be87398d1..f0e5ba0326 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -2972,28 +2972,29 @@ nc4_rec_match_dimscales(NC_GRP_INFO_T *grp) for (d = 0; d < var->ndims; d++) { nc_bool_t finished = NC_FALSE; - LOG((5, "%s: var %s has dimscale info...", __func__, var->hdr.name)); - /* Look at all the dims in this group to see if they - * match. */ + + /* Check this and parent groups. */ for (g = grp; g && !finished; g = g->parent) { + /* Check all dims in this group. */ for (j = 0; j < ncindexsize(g->dim); j++) { + /* Get the HDF5 specific dim info. */ NC_HDF5_DIM_INFO_T *hdf5_dim; dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j); assert(dim && dim->format_dim_info); hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; - /* Check for exact match to find identical - * objects in HDF5 file. */ - if (var->dimscale_hdf5_objids[d].fileno[0] == dim->hdf5_objid.fileno[0] && - var->dimscale_hdf5_objids[d].objno[0] == dim->hdf5_objid.objno[0] && - var->dimscale_hdf5_objids[d].fileno[1] == dim->hdf5_objid.fileno[1] && - var->dimscale_hdf5_objids[d].objno[1] == dim->hdf5_objid.objno[1]) + /* Check for exact match of fileno/objid arrays + * to find identical objects in HDF5 file. */ + if (var->dimscale_hdf5_objids[d].fileno[0] == hdf5_dim->hdf5_objid.fileno[0] && + var->dimscale_hdf5_objids[d].objno[0] == hdf5_dim->hdf5_objid.objno[0] && + var->dimscale_hdf5_objids[d].fileno[1] == hdf5_dim->hdf5_objid.fileno[1] && + var->dimscale_hdf5_objids[d].objno[1] == hdf5_dim->hdf5_objid.objno[1]) { - LOG((4, "%s: for dimension %d, found dim %s", - __func__, d, dim->hdr.name)); + LOG((4, "%s: for dimension %d, found dim %s", __func__, + d, dim->hdr.name)); var->dimids[d] = dim->hdr.id; var->dim[d] = dim; finished = NC_TRUE; @@ -3001,7 +3002,8 @@ nc4_rec_match_dimscales(NC_GRP_INFO_T *grp) } } /* next dim */ } /* next grp */ - LOG((5, "%s: dimid for this dimscale is %d", __func__, var->type_info->hdr.id)); + LOG((5, "%s: dimid for this dimscale is %d", __func__, + var->type_info->hdr.id)); } /* next var->dim */ } /* No dimscales for this var! Invent phony dimensions. */ From 8b75fbb6d944709f592d5a25530dfe103860811c Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Thu, 8 Nov 2018 12:37:03 -0700 Subject: [PATCH 23/46] Corrected VS2010 compilation issue in support of https://github.com/Unidata/netcdf-c/issues/1184 . Solution provided by Mark Rivers on the NetCDF mailing list. --- libdispatch/drc.c | 9 ++++--- libdispatch/nchashmap.c | 56 ++++++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 391be4b9bc..2f7c5bf25b 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -367,9 +367,12 @@ rclocate(const char* key, const char* hostport) if(hostport == NULL) hostport = ""; for(found=0,i=0;ihost == NULL ? 0 : strlen(triple->host)); - int t; + int t; + size_t hplen; + triple = (NCTriple*)nclistget(rc,i); + + hplen = (triple->host == NULL ? 0 : strlen(triple->host)); + if(strcmp(key,triple->key) != 0) continue; /* keys do not match */ /* If the triple entry has no url, then use it (because we have checked all other cases)*/ diff --git a/libdispatch/nchashmap.c b/libdispatch/nchashmap.c index 169fabd86d..ff45dcfb40 100644 --- a/libdispatch/nchashmap.c +++ b/libdispatch/nchashmap.c @@ -108,7 +108,7 @@ rehash(NC_hashmap* hm) /* Locate where given object is or should be placed in indexp. if fail to find spot return 0 else 1. If deletok then a deleted slot is ok to return; - return invariant: return == 0 || *indexp is defined + return invariant: return == 0 || *indexp is defined */ static int locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize, size_t* indexp, int deletedok) @@ -118,15 +118,15 @@ locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize, size_t step = 1; /* simple linear probe */ int deletefound = 0; size_t deletedindex = 0; /* first deleted entry encountered */ - + NC_hentry* entry; TRACE("locate"); /* Compute starting point */ index = (size_t)(hashkey % hash->alloc); /* Search table using linear probing */ for (i = 0; i < hash->alloc; i++) { - NC_hentry* entry = &hash->table[index]; - if(entry->flags & ACTIVE) { + entry = &hash->table[index]; + if(entry->flags & ACTIVE) { if(indexp) *indexp = index; /* assume a match */ if(entry->hashkey == hashkey && entry->keysize == keysize) { /* Check content */ @@ -148,7 +148,7 @@ locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize, index = (index + step) % hash->alloc; } if(deletedok && deletefound) { - if(indexp) *indexp = deletedindex; + if(indexp) *indexp = deletedindex; return 1; } return 0; @@ -187,12 +187,13 @@ NC_hashmapnew(size_t startsize) int NC_hashmapadd(NC_hashmap* hash, uintptr_t data, const char* key, size_t keysize) { - unsigned int hashkey; + NC_hentry* entry; + unsigned int hashkey; TRACE("NC_hashmapadd"); if(key == NULL || keysize == 0) - return 0; + return 0; hashkey = NC_crc32(0,(unsigned char*)key,(unsigned int)keysize); if(hash->alloc*3/4 <= hash->active) @@ -203,7 +204,7 @@ NC_hashmapadd(NC_hashmap* hash, uintptr_t data, const char* key, size_t keysize) rehash(hash); continue; /* try on larger table */ } - NC_hentry* entry = &hash->table[index]; + entry = &hash->table[index]; if(entry->flags & ACTIVE) { /* key already exists in table => overwrite data */ entry->data = data; @@ -264,13 +265,13 @@ NC_hashmapget(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* data return 0; hashkey = NC_crc32(0,(unsigned char*)key,(unsigned int)keysize); if(hash->active) { - size_t index; - NC_hentry* h; - if(!locate(hash,hashkey,key,keysize,&index,0)) + size_t index; + NC_hentry* h; + if(!locate(hash,hashkey,key,keysize,&index,0)) return 0; /* not present */ - h = &hash->table[index]; + h = &hash->table[index]; if(h->flags & ACTIVE) { - if(datap) *datap = h->data; + if(datap) *datap = h->data; return 1; } else /* Not found */ return 0; @@ -300,7 +301,7 @@ NC_hashmapsetdata(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t n h = &hash->table[index]; assert((h->flags & ACTIVE) == ACTIVE); h->data = newdata; - return 1; + return 1; } size_t @@ -339,7 +340,7 @@ Allows a hack by ncindex. int NC_hashmapdeactivate(NC_hashmap* map, uintptr_t data) { - size_t i; + size_t i; NC_hentry* h; for(h=map->table,i=0;ialloc;i++,h++) { if((h->flags & ACTIVE) && h->data == data) { @@ -370,21 +371,23 @@ findPrimeGreaterThan(size_t val) int L = 1; /* skip leading flag number */ int R = (n - 2); /* skip trailing flag */ unsigned int v = 0; + int m; if(val >= 0xFFFFFFFF) - return 0; /* Too big */ + return 0; /* Too big */ v = (unsigned int)val; for(;;) { - if(L >= R) break; - int m = (L + R) / 2; - /* is this an acceptable prime? */ - if(NC_primes[m-1] < v && NC_primes[m] >= v) - return NC_primes[m]; /* acceptable*/ - else if(NC_primes[m-1] >= v) - R = m; - else if(NC_primes[m] < v) - L = m; + if(L >= R) break; + + m = (L + R) / 2; + /* is this an acceptable prime? */ + if(NC_primes[m-1] < v && NC_primes[m] >= v) + return NC_primes[m]; /* acceptable*/ + else if(NC_primes[m-1] >= v) + R = m; + else if(NC_primes[m] < v) + L = m; } return 0; } @@ -2046,6 +2049,7 @@ static unsigned int NC_nprimes = (sizeof(NC_primes) / sizeof(unsigned int)); void printhashmapstats(NC_hashmap* hm) { + size_t n,i; size_t step = 1; size_t maxchain = 0; @@ -2061,7 +2065,7 @@ printhashmapstats(NC_hashmap* hm) default: /* empty slot, stop walking */ if(chainlen > maxchain) maxchain = chainlen; goto next; - } + } /* linear probe */ index = (index + step) % hm->alloc; } From e3f2c6811f1c57f45e040979f61d6d2364506721 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 9 Nov 2018 07:39:12 -0700 Subject: [PATCH 24/46] fixed dap configure issue --- configure.ac | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/configure.ac b/configure.ac index e528abbba4..bdf51fe266 100644 --- a/configure.ac +++ b/configure.ac @@ -349,34 +349,15 @@ AC_ARG_ENABLE([dap], test "x$enable_dap" = xno || enable_dap=yes AC_MSG_RESULT($enable_dap) -# --enable-dap => enable-dap4 -enable_dap4=$enable_dap - -# Curl support is required if and only if any of these flags are set: -# 1. --enable-dap - -if test "x$enable_dap" = "xyes" ; then -require_curl=yes -else -require_curl=no -fi - -# See if the user provided us with a curl library -# Do an initial lib test for curl, but suppress the default action +# We need curl for DAP. AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no]) -# If curl is required but there is no curl, then complain -if test $require_curl = yes ; then - if test $found_curl = no ; then - AC_MSG_NOTICE([libcurl not found; disabling remote protocol(s) support]) - enable_dap=no - enable_dap4=no - elif test $found_curl = yes ; then - # Redo the check lib to actually add -lcurl - #AC_CHECK_LIB([curl], [curl_easy_setopt]) - AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll], [], []) - fi +if test "x$enable_dap" = "xyes" ; then + AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll], [], + [AC_MSG_ERROR([curl required for remote access. Install curl or build with --disable-dap.])]) fi +# --enable-dap => enable-dap4 +enable_dap4=$enable_dap # Default is now to always do the short remote tests AC_MSG_CHECKING([whether dap remote testing should be enabled (default on)]) AC_ARG_ENABLE([dap-remote-tests], @@ -1032,7 +1013,7 @@ if test "x$enable_netcdf_4" = xyes; then AC_SEARCH_LIBS([H5DSis_scale], [hdf5_hldll hdf5_hl], [], [AC_MSG_ERROR([Can't find or link to the hdf5 high-level. Use --disable-netcdf-4, or see config.log for errors.])]) - AC_CHECK_HEADERS([hdf5.h], [], [AC_MSG_ERROR([Compiling a test with HDF5 failed. Either hdf5.h cannot be found, or config.log should be checked for other reason.])]) + AC_CHECK_HEADERS([hdf5.h], [], [AC_MSG_ERROR([Compiling a test with HDF5 failed. Either hdf5.h cannot be found, or config.log should be checked for other reason.])]. [sys/types.h]) AC_CHECK_FUNCS([H5Z_SZIP]) hdf5_parallel=no From 50bda41b6ba4c02aa55363d74cf434ec9433e286 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 9 Nov 2018 08:45:27 -0700 Subject: [PATCH 25/46] fixed dap configure issue --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bdf51fe266..c47e48c19b 100644 --- a/configure.ac +++ b/configure.ac @@ -1013,7 +1013,7 @@ if test "x$enable_netcdf_4" = xyes; then AC_SEARCH_LIBS([H5DSis_scale], [hdf5_hldll hdf5_hl], [], [AC_MSG_ERROR([Can't find or link to the hdf5 high-level. Use --disable-netcdf-4, or see config.log for errors.])]) - AC_CHECK_HEADERS([hdf5.h], [], [AC_MSG_ERROR([Compiling a test with HDF5 failed. Either hdf5.h cannot be found, or config.log should be checked for other reason.])]. [sys/types.h]) + AC_CHECK_HEADERS([hdf5.h], [], [AC_MSG_ERROR([Compiling a test with HDF5 failed. Either hdf5.h cannot be found, or config.log should be checked for other reason.])]) AC_CHECK_FUNCS([H5Z_SZIP]) hdf5_parallel=no From aff2aa9341bb42e320b1c2e113e4f4c37f5d2771 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 9 Nov 2018 09:01:30 -0700 Subject: [PATCH 26/46] added release notes --- RELEASE_NOTES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 21a07d9b18..7fc28d0b25 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,9 @@ This file contains a high-level description of this package's evolution. Release ## 4.6.2 - TBD +* [Enhancement] Lazy att read - only read atts when user requests one of them. See [GitHub #857](https://github.com/Unidata/netcdf-c/issues/857). +* [Enhancement] Fast global att read - when global atts are read, they are read much more quickly. See [GitHub #857](https://github.com/Unidata/netcdf-c/issues/857). + ## 4.6.2-rc2 November 1, 2018 From aac0d6c4674b8c8df93058c22381b27721859285 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 9 Nov 2018 09:35:15 -0700 Subject: [PATCH 27/46] minor doc change --- docs/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.md b/docs/install.md index d666a33d7b..ef0e278a64 100644 --- a/docs/install.md +++ b/docs/install.md @@ -150,7 +150,7 @@ Indicate where you want to install netCDF in another shell variable, for example ~~~~{.py} $ # Build and install netCDF-4 $ NCDIR=/usr/local - $ CPPFLAGS=-I${H5DIR}/include LDFLAGS=-L${H5DIR}/lib ./configure --prefix=${NCDIR} + $ CPPFLAGS='-I${H5DIR}/include -I${ZDIR{/include' LDFLAGS='-L${H5DIR}/lib -L${ZDIR}/lib' ./configure --prefix=${NCDIR} $ make check $ make install # or sudo make install ~~~~ From a33655d91676b44071019d878cdf9890f463811e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 9 Nov 2018 11:04:47 -0700 Subject: [PATCH 28/46] fixed typo --- docs/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.md b/docs/install.md index ef0e278a64..36c8fa30c8 100644 --- a/docs/install.md +++ b/docs/install.md @@ -150,7 +150,7 @@ Indicate where you want to install netCDF in another shell variable, for example ~~~~{.py} $ # Build and install netCDF-4 $ NCDIR=/usr/local - $ CPPFLAGS='-I${H5DIR}/include -I${ZDIR{/include' LDFLAGS='-L${H5DIR}/lib -L${ZDIR}/lib' ./configure --prefix=${NCDIR} + $ CPPFLAGS='-I${H5DIR}/include -I${ZDIR}/include' LDFLAGS='-L${H5DIR}/lib -L${ZDIR}/lib' ./configure --prefix=${NCDIR} $ make check $ make install # or sudo make install ~~~~ From c35aa3ccb92d046d49479e19c26f29f169989331 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 07:40:15 -0700 Subject: [PATCH 29/46] changed header files to separate HDF5-specific grp info --- include/hdf5internal.h | 6 ++++++ include/nc4internal.h | 1 + 2 files changed, 7 insertions(+) diff --git a/include/hdf5internal.h b/include/hdf5internal.h index e60991555c..8cd16b7718 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -69,6 +69,12 @@ typedef struct NC_HDF5_ATT_INFO hid_t native_hdf_typeid; /* Native HDF5 datatype for attribute's data */ } NC_HDF5_ATT_INFO_T; +/* Struct to hold HDF5-specific info for a group. */ +typedef struct NC_HDF5_GRP_INFO +{ + hid_t hdf_grpid; +} NC_HDF5_GRP_INFO_T; + /* These functions do HDF5 things. */ int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); diff --git a/include/nc4internal.h b/include/nc4internal.h index afcffa3d31..c21af56bc0 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -250,6 +250,7 @@ typedef struct NC_GRP_INFO { NC_OBJ hdr; hid_t hdf_grpid; + void *format_grp_info; struct NC_FILE_INFO *nc4_info; struct NC_GRP_INFO *parent; int atts_not_read; From 9364157a9b8e004a241be9210cdf04629b8dc493 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 08:11:06 -0700 Subject: [PATCH 30/46] allocating and freeing memory for HDF5-specific group info --- libhdf5/hdf5grp.c | 2 ++ libhdf5/hdf5open.c | 4 ++++ libsrc4/nc4internal.c | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/libhdf5/hdf5grp.c b/libhdf5/hdf5grp.c index 1ddbcf773a..3e3e78ad64 100644 --- a/libhdf5/hdf5grp.c +++ b/libhdf5/hdf5grp.c @@ -65,6 +65,8 @@ NC4_def_grp(int parent_ncid, const char *name, int *new_ncid) * sync. */ if ((retval = nc4_grp_list_add(h5, grp, norm_name, &g))) return retval; + if (!(g->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T)))) + return NC_ENOMEM; if (new_ncid) *new_ncid = grp->nc4_info->controller->ext_ncid | g->hdr.id; diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 7ba05a850b..05aebe0c27 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -2086,6 +2086,10 @@ nc4_rec_read_metadata(NC_GRP_INFO_T *grp) &child_grp))) BAIL(retval); + /* Allocate storage for HDF5-specific group info. */ + if (!(child_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T)))) + return NC_ENOMEM; + /* Recursively read the child group's metadata */ if ((retval = nc4_rec_read_metadata(child_grp))) BAIL(retval); diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index e82d2a2092..a5817b6dd1 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1358,6 +1358,10 @@ nc4_rec_grp_del(NC_GRP_INFO_T *grp) /* Free the name. */ free(grp->hdr.name); + /* Release any format-specific information about this group. */ + if (grp->format_grp_info) + free(grp->format_grp_info); + /* Free up this group */ free(grp); From 6b75bb483b021fdc1e7f4244e8d1c3e8a8a4982f Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 09:02:46 -0700 Subject: [PATCH 31/46] allocating HDF5-specific group info struct for root group --- libhdf5/hdf5create.c | 4 ++++ libhdf5/hdf5open.c | 4 ++++ libhdf5/nc4hdf.c | 3 ++- libsrc4/nc4internal.c | 7 ++++++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index dbbb5fe81f..11caf08b9e 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -71,6 +71,10 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, BAIL(NC_ENOMEM); hdf5_info = (NC_HDF5_FILE_INFO_T *)nc4_info->format_file_info; + /* Add struct to hold HDF5-specific group info. */ + if (!(nc4_info->root_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T)))) + return NC_ENOMEM; + nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY; nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS; nc4_info->mem.persist = (cmode & NC_PERSIST) == NC_PERSIST; diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 05aebe0c27..73caa9ba22 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -388,6 +388,10 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if (!(nc4_info->format_file_info = calloc(1, sizeof(NC_HDF5_FILE_INFO_T)))) BAIL(NC_ENOMEM); + /* Add struct to hold HDF5-specific group info. */ + if (!(nc4_info->root_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T)))) + return NC_ENOMEM; + nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); nc4_info->mem.persist = ((mode & NC_PERSIST) == NC_PERSIST); diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index f0e5ba0326..dd97c27d69 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1324,7 +1324,8 @@ create_group(NC_GRP_INFO_T *grp) hid_t gcpl_id = -1; int retval = NC_NOERR;; - assert(grp && grp->parent && grp->parent->hdf_grpid); + assert(grp && grp->format_grp_info && grp->parent && + grp->parent->format_grp_info && grp->parent->hdf_grpid); /* Create group, with link_creation_order set in the group * creation property list. */ diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index a5817b6dd1..c74ce8b732 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -92,12 +92,14 @@ nc4_check_name(const char *name, char *norm_name) * @param mode The mode flag. * * @return ::NC_NOERR No error. + * @return ::NC_ENOMEM Out of memory. * @author Ed Hartnett, Dennis Heimbigner */ int nc4_nc4f_list_add(NC *nc, const char *path, int mode) { NC_FILE_INFO_T *h5; + int retval; assert(nc && !NC4_DATA(nc) && path); @@ -123,7 +125,10 @@ nc4_nc4f_list_add(NC *nc, const char *path, int mode) /* There's always at least one open group - the root * group. Allocate space for one group's worth of information. Set * its hdf id, name, and a pointer to it's file structure. */ - return nc4_grp_list_add(h5, NULL, NC_GROUP_NAME, &h5->root_grp); + if ((retval = nc4_grp_list_add(h5, NULL, NC_GROUP_NAME, &h5->root_grp))) + return retval; + + return NC_NOERR; } /** From 17552b7eb37e41790b450fbf901bbe2e147ac7c3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 10:37:06 -0700 Subject: [PATCH 32/46] HDF5-specific group changes --- libhdf5/hdf5create.c | 6 ++++-- libhdf5/nc4hdf.c | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 11caf08b9e..7d6b8af427 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -48,6 +48,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, int retval = NC_NOERR; NC_FILE_INFO_T* nc4_info = NULL; NC_HDF5_FILE_INFO_T *hdf5_info; + NC_HDF5_GRP_INFO_T *hdf5_grp; #ifdef USE_PARALLEL4 NC_MPI_INFO *mpiinfo = NULL; @@ -74,6 +75,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, /* Add struct to hold HDF5-specific group info. */ if (!(nc4_info->root_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T)))) return NC_ENOMEM; + hdf5_grp = (NC_HDF5_GRP_INFO_T *)nc4_info->root_grp->format_grp_info; nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY; nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS; @@ -220,9 +222,9 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, } /* Open the root group. */ - if ((nc4_info->root_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/", - H5P_DEFAULT)) < 0) + if ((hdf5_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/", H5P_DEFAULT)) < 0) BAIL(NC_EFILEMETA); + nc4_info->root_grp->hdf_grpid = hdf5_grp->hdf_grpid; /* Release the property lists. */ if (H5Pclose(fapl_id) < 0 || H5Pclose(fcpl_id) < 0) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index dd97c27d69..cf58b681bd 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1321,12 +1321,17 @@ write_nc3_strict_att(hid_t hdf_grpid) static int create_group(NC_GRP_INFO_T *grp) { + NC_HDF5_GRP_INFO_T *hdf5_grp, *parent_hdf5_grp; hid_t gcpl_id = -1; int retval = NC_NOERR;; assert(grp && grp->format_grp_info && grp->parent && grp->parent->format_grp_info && grp->parent->hdf_grpid); + /* Get HDF5 specific group info for group and parent. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; + parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info; + /* Create group, with link_creation_order set in the group * creation property list. */ if ((gcpl_id = H5Pcreate(H5P_GROUP_CREATE)) < 0) @@ -1345,9 +1350,10 @@ create_group(NC_GRP_INFO_T *grp) BAIL(NC_EHDFERR); /* Create the group. */ - if ((grp->hdf_grpid = H5Gcreate2(grp->parent->hdf_grpid, grp->hdr.name, - H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0) + if ((hdf5_grp->hdf_grpid = H5Gcreate2(grp->parent->hdf_grpid, grp->hdr.name, + H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0) BAIL(NC_EHDFERR); + grp->hdf_grpid = hdf5_grp->hdf_grpid; exit: if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0) From d6c5e1f12736f600689ebfe3b5dc42e5c9a8122f Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 10:48:24 -0700 Subject: [PATCH 33/46] HDF5-specific group changes --- libhdf5/hdf5grp.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libhdf5/hdf5grp.c b/libhdf5/hdf5grp.c index 3e3e78ad64..b8c0d273a5 100644 --- a/libhdf5/hdf5grp.c +++ b/libhdf5/hdf5grp.c @@ -92,6 +92,7 @@ int NC4_rename_grp(int grpid, const char *name) { NC_GRP_INFO_T *grp, *parent; + NC_HDF5_GRP_INFO_T *hdf5_grp; NC_FILE_INFO_T *h5; char norm_name[NC_MAX_NAME + 1]; int retval; @@ -101,7 +102,10 @@ NC4_rename_grp(int grpid, const char *name) /* Find info for this file and group, and set pointer to each. */ if ((retval = nc4_find_grp_h5(grpid, &grp, &h5))) return retval; - assert(h5); + assert(h5 && grp && grp->format_grp_info); + + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; if (h5->no_write) return NC_EPERM; /* attempt to write to a read-only file */ From 65bde4878ba1eafc506dddda599c7626b8f6a0f0 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 11:03:00 -0700 Subject: [PATCH 34/46] HDF5-specific group changes --- libhdf5/hdf5grp.c | 7 ++++++- libhdf5/hdf5open.c | 38 ++++++++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/libhdf5/hdf5grp.c b/libhdf5/hdf5grp.c index b8c0d273a5..08d53fb0b6 100644 --- a/libhdf5/hdf5grp.c +++ b/libhdf5/hdf5grp.c @@ -132,9 +132,13 @@ NC4_rename_grp(int grpid, const char *name) /* Rename the group, if it exists in the file */ if (grp->hdf_grpid) { + NC_HDF5_GRP_INFO_T *parent_hdf5_grp; + parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info; + /* Close the group */ if (H5Gclose(grp->hdf_grpid) < 0) return NC_EHDFERR; + hdf5_grp->hdf_grpid = 0; grp->hdf_grpid = 0; /* Attempt to rename & re-open the group, if the parent group is open */ @@ -145,8 +149,9 @@ NC4_rename_grp(int grpid, const char *name) return NC_EHDFERR; /* Reopen the group, with the new name */ - if ((grp->hdf_grpid = H5Gopen2(parent->hdf_grpid, name, H5P_DEFAULT)) < 0) + if ((hdf5_grp->hdf_grpid = H5Gopen2(parent->hdf_grpid, name, H5P_DEFAULT)) < 0) return NC_EHDFERR; + grp->hdf_grpid = hdf5_grp->hdf_grpid; } } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 73caa9ba22..dd87a63b1f 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -2011,17 +2011,21 @@ nc4_rec_read_metadata_cb(hid_t grpid, const char *name, const H5L_info_t *info, static int nc4_rec_read_metadata(NC_GRP_INFO_T *grp) { + NC_HDF5_GRP_INFO_T *hdf5_grp; NC4_rec_read_metadata_ud_t udata; /* User data for iteration */ NC4_rec_read_metadata_obj_info_t *oinfo; /* Pointer to info for object */ - hsize_t idx=0; + hsize_t idx = 0; hid_t pid = 0; unsigned crt_order_flags = 0; H5_index_t iter_index; int i, retval = NC_NOERR; /* everything worked! */ - assert(grp && grp->hdr.name); + assert(grp && grp->hdr.name && grp->format_grp_info); LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name)); + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; + /* Portably initialize user data for later */ memset(&udata, 0, sizeof(udata)); @@ -2031,24 +2035,29 @@ nc4_rec_read_metadata(NC_GRP_INFO_T *grp) { if (grp->parent) { - if ((grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid, - grp->hdr.name, H5P_DEFAULT)) < 0) + NC_HDF5_GRP_INFO_T *parent_hdf5_grp; + parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info; + + if ((hdf5_grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid, + grp->hdr.name, H5P_DEFAULT)) < 0) BAIL(NC_EHDFERR); + grp->hdf_grpid = hdf5_grp->hdf_grpid; } else { NC_HDF5_FILE_INFO_T *hdf5_info; hdf5_info = (NC_HDF5_FILE_INFO_T *)grp->nc4_info->format_file_info; - if ((grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/", - H5P_DEFAULT)) < 0) + if ((hdf5_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/", + H5P_DEFAULT)) < 0) BAIL(NC_EHDFERR); + grp->hdf_grpid = hdf5_grp->hdf_grpid; } } - assert(grp->hdf_grpid > 0); + assert(hdf5_grp->hdf_grpid > 0); /* Get the group creation flags, to check for creation ordering */ - pid = H5Gget_create_plist(grp->hdf_grpid); + pid = H5Gget_create_plist(hdf5_grp->hdf_grpid); H5Pget_link_creation_order(pid, &crt_order_flags); if (H5Pclose(pid) < 0) BAIL(NC_EHDFERR); @@ -2073,17 +2082,17 @@ nc4_rec_read_metadata(NC_GRP_INFO_T *grp) /* Iterate over links in this group, building lists for the types, * datasets and groups encountered. */ - if (H5Literate(grp->hdf_grpid, iter_index, H5_ITER_INC, &idx, + if (H5Literate(hdf5_grp->hdf_grpid, iter_index, H5_ITER_INC, &idx, nc4_rec_read_metadata_cb, (void *)&udata) < 0) BAIL(NC_EHDFERR); /* Process the child groups found. (Deferred until now, so that the * types in the current group get processed and are available for * vars in the child group(s).) */ - for(i=0;inc4_info, grp, oinfo->oname, @@ -2107,15 +2116,16 @@ nc4_rec_read_metadata(NC_GRP_INFO_T *grp) grp->atts_not_read = 1; /* when exiting define mode, mark all variable written */ - for (i=0; ivars); i++) { + for (i = 0; i < ncindexsize(grp->vars); i++) + { NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i); - if(var == NULL) continue; + assert(var); var->written_to = NC_TRUE; } exit: /* Clean up local information, if anything remains */ - for(i=0;i Date: Mon, 12 Nov 2018 12:12:56 -0700 Subject: [PATCH 35/46] HDF5-specific group changes --- libhdf5/nc4hdf.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index cf58b681bd..a9b6a242db 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -180,6 +180,8 @@ nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset) { NC_VAR_INFO_T *var; + assert(grp && grp->format_grp_info && dataset); + /* Find the requested varid. */ var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); if (!var) return NC_ENOTVAR; @@ -187,9 +189,14 @@ nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset) /* Open this dataset if necessary. */ if (!var->hdf_datasetid) - if ((var->hdf_datasetid = H5Dopen2(grp->hdf_grpid, var->hdr.name, - H5P_DEFAULT)) < 0) + { + NC_HDF5_GRP_INFO_T *hdf5_grp; + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; + + if ((var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid, + var->hdr.name, H5P_DEFAULT)) < 0) return NC_ENOTVAR; + } *dataset = var->hdf_datasetid; @@ -542,6 +549,7 @@ nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype, static int put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) { + NC_HDF5_GRP_INFO_T *hdf5_grp; hid_t datasetid = 0, locid; hid_t attid = 0, spaceid = 0, file_typeid = 0; hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0; @@ -552,11 +560,14 @@ put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) int phoney_data = 99; int retval = NC_NOERR; - assert(att->hdr.name); + assert(att->hdr.name && grp && grp->format_grp_info); LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d " "att->len %d", __func__, varid, att->hdr.id, att->hdr.name, att->nc_typeid, att->len)); + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; + /* If the file is read-only, return an error. */ if (grp->nc4_info->no_write) BAIL(NC_EPERM); From e74009c7083f0ae6a97bc5d52996f9dedbb32795 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 12:32:21 -0700 Subject: [PATCH 36/46] HDF5-specific group changes --- libhdf5/nc4hdf.c | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index a9b6a242db..0b91197fcb 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -574,7 +574,7 @@ put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) /* Get the hid to attach the attribute to, or read it from. */ if (varid == NC_GLOBAL) - locid = grp->hdf_grpid; + locid = hdf5_grp->hdf_grpid; else { if ((retval = nc4_open_var_grp2(grp, varid, &datasetid))) @@ -838,6 +838,7 @@ write_netcdf4_dimid(hid_t datasetid, int dimid) static int var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid) { + NC_HDF5_GRP_INFO_T *hdf5_grp; hid_t plistid = 0, access_plistid = 0, typeid = 0, spaceid = 0; hsize_t chunksize[H5S_MAX_RANK], dimsize[H5S_MAX_RANK], maxdimsize[H5S_MAX_RANK]; int d; @@ -846,8 +847,13 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid char *name_to_use; int retval; + assert(grp && grp->format_grp_info && var); + LOG((3, "%s:: name %s", __func__, var->hdr.name)); + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; + /* Scalar or not, we need a creation property list. */ if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0) BAIL(NC_EHDFERR); @@ -1035,7 +1041,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid name_to_use = var->hdf5_name ? var->hdf5_name : var->hdr.name; LOG((4, "%s: about to H5Dcreate2 dataset %s of type 0x%x", __func__, name_to_use, typeid)); - if ((var->hdf_datasetid = H5Dcreate2(grp->hdf_grpid, name_to_use, typeid, + if ((var->hdf_datasetid = H5Dcreate2(hdf5_grp->hdf_grpid, name_to_use, typeid, spaceid, H5P_DEFAULT, plistid, access_plistid)) < 0) BAIL(NC_EHDFERR); var->created = NC_TRUE; @@ -1153,9 +1159,13 @@ nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) static int commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type) { + NC_HDF5_GRP_INFO_T *hdf5_grp; int retval; - assert(grp && type); + assert(grp && grp->format_grp_info && type); + + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; /* Did we already record this type? */ if (type->committed) @@ -1258,7 +1268,7 @@ commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type) } /* Commit the type. */ - if (H5Tcommit(grp->hdf_grpid, type->hdr.name, type->hdf_typeid) < 0) + if (H5Tcommit(hdf5_grp->hdf_grpid, type->hdr.name, type->hdf_typeid) < 0) return NC_EHDFERR; type->committed = NC_TRUE; LOG((4, "just committed type %s, HDF typeid: 0x%x", type->hdr.name, @@ -1370,7 +1380,7 @@ create_group(NC_GRP_INFO_T *grp) if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0) BAIL2(NC_EHDFERR); if (retval) - if (grp->hdf_grpid > 0 && H5Gclose(grp->hdf_grpid) < 0) + if (hdf5_grp->hdf_grpid > 0 && H5Gclose(hdf5_grp->hdf_grpid) < 0) BAIL2(NC_EHDFERR); return retval; } @@ -1555,11 +1565,17 @@ remove_coord_atts(hid_t hdf_datasetid) static int write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) { + NC_HDF5_GRP_INFO_T *hdf5_grp; nc_bool_t replace_existing_var = NC_FALSE; int retval; + assert(var && grp && grp->format_grp_info); + LOG((4, "%s: writing var %s", __func__, var->hdr.name)); + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; + /* If the variable has already been created & the fill value changed, * indicate that the existing variable should be replaced. */ if (var->created && var->fill_val_changed) @@ -1586,8 +1602,10 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) NC_DIM_INFO_T *d1; int i; - for(i=0;idim);i++) { - if((d1 = (NC_DIM_INFO_T*)ncindexith(grp->dim,i)) == NULL) continue; + for (i = 0; i < ncindexsize(grp->dim); i++) + { + d1 = (NC_DIM_INFO_T*)ncindexith(grp->dim,i); + assert(d1); if (!strcmp(d1->hdr.name, var->hdr.name)) { nc_bool_t exists; @@ -1707,7 +1725,7 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) var->hdf_datasetid = 0; /* Now delete the variable. */ - if (H5Gunlink(grp->hdf_grpid, var->hdr.name) < 0) + if (H5Gunlink(hdf5_grp->hdf_grpid, var->hdr.name) < 0) return NC_EDIMMETA; } @@ -1776,11 +1794,15 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) static int write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) { + NC_HDF5_GRP_INFO_T *hdf5_grp; NC_HDF5_DIM_INFO_T *hdf5_dim; int retval; - assert(dim && dim->format_dim_info); + assert(dim && dim->format_dim_info && grp && grp->format_grp_info); + + /* Get HDF5-specific dim and group info. */ hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; /* If there's no dimscale dataset for this dim, create one, * and mark that it should be hidden from netCDF as a @@ -1826,7 +1848,7 @@ write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) /* Create the dataset that will be the dimension scale. */ LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__, dim->hdr.name)); - if ((hdf5_dim->hdf_dimscaleid = H5Dcreate1(grp->hdf_grpid, dim->hdr.name, H5T_IEEE_F32BE, + if ((hdf5_dim->hdf_dimscaleid = H5Dcreate1(hdf5_grp->hdf_grpid, dim->hdr.name, H5T_IEEE_F32BE, spaceid, create_propid)) < 0) BAIL(NC_EHDFERR); From 5c7fc1fb555c2d2a9e8f4c0699a4a04667e19272 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 12:52:24 -0700 Subject: [PATCH 37/46] HDF5-specific group changes --- libhdf5/nc4hdf.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 0b91197fcb..8695ee887f 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1347,11 +1347,12 @@ create_group(NC_GRP_INFO_T *grp) int retval = NC_NOERR;; assert(grp && grp->format_grp_info && grp->parent && - grp->parent->format_grp_info && grp->parent->hdf_grpid); + grp->parent->format_grp_info); /* Get HDF5 specific group info for group and parent. */ hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info; + assert(parent_hdf5_grp->hdf_grpid); /* Create group, with link_creation_order set in the group * creation property list. */ @@ -1610,7 +1611,7 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) { nc_bool_t exists; - if ((retval = var_exists(grp->hdf_grpid, var->hdr.name, &exists))) + if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists))) return retval; if (exists) { @@ -1648,7 +1649,8 @@ write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid) { nc_bool_t exists; - if ((retval = var_exists(grp->hdf_grpid, var->hdr.name, &exists))) + if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, + &exists))) return retval; if (exists) { @@ -2014,7 +2016,8 @@ nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order) int retval; int i; - assert(grp && grp->hdr.name && grp->hdf_grpid); + assert(grp && grp->hdr.name && + ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid); LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name, bad_coord_order)); @@ -2090,22 +2093,26 @@ int nc4_rec_write_groups_types(NC_GRP_INFO_T *grp) { NC_GRP_INFO_T *child_grp; + NC_HDF5_GRP_INFO_T *hdf5_grp; NC_TYPE_INFO_T *type; int retval; int i; - assert(grp && grp->hdr.name); + assert(grp && grp->hdr.name && grp->format_grp_info); LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name)); + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; + /* Create the group in the HDF5 file if it doesn't exist. */ - if (!grp->hdf_grpid) + if (!hdf5_grp->hdf_grpid) if ((retval = create_group(grp))) return retval; /* If this is the root group of a file with strict NC3 rules, write * an attribute. But don't leave the attribute open. */ if (!grp->parent && (grp->nc4_info->cmode & NC_CLASSIC_MODEL)) - if ((retval = write_nc3_strict_att(grp->hdf_grpid))) + if ((retval = write_nc3_strict_att(hdf5_grp->hdf_grpid))) return retval; /* If there are any user-defined types, write them now. */ @@ -3517,7 +3524,8 @@ NC4_isnetcdf4(struct NC_FILE_INFO* h5) /* attribute did not exist */ /* => last resort: walk the HDF5 file looking for markers */ count = 0; - stat = NC4_walk(h5->root_grp->hdf_grpid, &count); + stat = NC4_walk(((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid, + &count); if(stat != NC_NOERR) isnc4 = 0; else /* Threshold is at least two matches */ @@ -3536,15 +3544,16 @@ NC4_isnetcdf4(struct NC_FILE_INFO* h5) * @author Dennis Heimbigner. */ static int -NC4_get_strict_att(NC_FILE_INFO_T* h5) +NC4_get_strict_att(NC_FILE_INFO_T *h5) { - hid_t grp = -1; + hid_t grpid = -1; hid_t attid = -1; - /* Get root group */ - grp = h5->root_grp->hdf_grpid; /* get root group */ + /* Get root group ID. */ + grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid; + /* Try to extract the NC3_STRICT_ATT_NAME attribute */ - attid = H5Aopen_name(grp, NC3_STRICT_ATT_NAME); + attid = H5Aopen_name(grpid, NC3_STRICT_ATT_NAME); H5Aclose(attid); return attid; } From 3ad02fe94f80820788aa5ee537b559a217257f9c Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 12 Nov 2018 12:59:56 -0700 Subject: [PATCH 38/46] Fix failure in the nccopy -c option re: https://github.com/Unidata/netcdf-c/issues/1183 Changes: 1. fix typo in nccopy.1 man page 2. fix chunkspec parsing for -c so that "-c var:" (with no chunkspec arguments) disables chunking on the specified variable 3. Add test case to ncdump/tst_nccopy5.sh --- ncdump/chunkspec.c | 9 +++++++++ ncdump/nccopy.1 | 3 ++- ncdump/tst_nccopy5.sh | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/ncdump/chunkspec.c b/ncdump/chunkspec.c index 62dd0506dd..7554c6de38 100644 --- a/ncdump/chunkspec.c +++ b/ncdump/chunkspec.c @@ -242,10 +242,19 @@ varchunkspec_parse(int igrp, const char *spec0) if(p == NULL) {ret = NC_EINVAL; goto done;} *p++ = '\0'; + /* Lookup the variable by name */ ret = nc_inq_varid2(igrp, spec, &chunkspec->ivarid, &chunkspec->igrpid); if(ret != NC_NOERR) goto done; + if(*p == '\0') {/* we have -c var: => do not chunk var */ + chunkspec->omit = 1; + /* add the chunkspec to our list */ + listpush(varchunkspecs,chunkspec); + chunkspec = NULL; + goto done; + } + /* Iterate over dimension sizes */ while(*p) { unsigned long dimsize; diff --git a/ncdump/nccopy.1 b/ncdump/nccopy.1 index 7517d4e689..af67dd9243 100644 --- a/ncdump/nccopy.1 +++ b/ncdump/nccopy.1 @@ -179,7 +179,8 @@ syntax: \fI var:n1,n2,...,nn \fP. This assumes that the variable named variable is specified by the values of n1 through nn. This second form of chunking specification can be repeated multiple times to specify the exact chunking for different variables. -If the variable is specified but no chunk sizes are specified (i.e. \fI -d var: \fP) +If the variable is specified but no chunk sizes are specified +(i.e. \fI -c var: \fP) then chunking is disabled for that variable. If the same variable is specified more than once, the second and later specifications are ignored. diff --git a/ncdump/tst_nccopy5.sh b/ncdump/tst_nccopy5.sh index bc92c24f24..e3311c327f 100755 --- a/ncdump/tst_nccopy5.sh +++ b/ncdump/tst_nccopy5.sh @@ -11,6 +11,7 @@ T1=1 T2=1 T3=1 T4=1 +T5=1 # For a netCDF-4 build, test nccopy chunking rules @@ -38,6 +39,16 @@ checkfvar() { fi } +# usage: checkivar +checkivar() { + # Make sure that ivar was not chunked + C5IVAR=`sed -e '/ivar:_ChunkSizes/p' -e d <$1` + if test "x$C5IVAR" != x ; then + echo "***Fail: ivar was chunked" + exit 1 + fi +} + # usage: verifychunkline line1 line2 verifychunkline() { # trim leading whitespace @@ -53,12 +64,14 @@ verifychunkline() { cleanup() { rm -f tmp_nc5.nc tmp_nc5a.nc rm -f tmp_nc5.cdl tmp_nc5a.cdl tmp_nc5b.cdl + rm -f tmp_nc5_omit.nc tmp_nc5_omit.cdl } # remove all created files reset() { cleanup rm -fr tst_nc5.nc tst_nc5.cdl + rm -f tst_nc5_omit.nc tst_nc5_omit.cdl } reset @@ -159,6 +172,29 @@ checkfvar tmp_nc5.cdl fi # T4 +if test "x$T5" = x1 ; then + +echo "*** Test nccopy -c fvar: to suppress chunking; classic ->enhanced" +reset +./tst_chunking tst_nc5_omit.nc +${NCDUMP} -n tst_nc5_omit tst_nc5_omit.nc > tst_nc5_omit.cdl +${NCCOPY} -c ivar:7,1,2,1,5,1,9 -c fvar: tst_nc5_omit.nc tmp_nc5_omit.nc +${NCDUMP} -n tst_nc5_omit tmp_nc5_omit.nc > tmp_nc5_omit.cdl +diff tst_nc5_omit.cdl tmp_nc5_omit.cdl + +# Verify chunking of ivar +${NCDUMP} -hs -n tst_nc5_omit tmp_nc5_omit.nc > tmp_nc5_omit.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d Date: Mon, 12 Nov 2018 13:01:54 -0700 Subject: [PATCH 39/46] hdf5-specific group stuff in hdf5var.c --- libhdf5/hdf5var.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index d81f660a2b..c9de6e0be4 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -30,9 +30,16 @@ int nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) { hid_t access_pid; + hid_t grpid; + + assert(var && grp && grp->format_grp_info); if (var->hdf_datasetid) { + /* Get the HDF5 group id. */ + grpid = ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid; + + if ((access_pid = H5Pcreate(H5P_DATASET_ACCESS)) < 0) return NC_EHDFERR; if (H5Pset_chunk_cache(access_pid, var->chunk_cache_nelems, @@ -41,8 +48,7 @@ nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) return NC_EHDFERR; if (H5Dclose(var->hdf_datasetid) < 0) return NC_EHDFERR; - if ((var->hdf_datasetid = H5Dopen2(grp->hdf_grpid, var->hdr.name, - access_pid)) < 0) + if ((var->hdf_datasetid = H5Dopen2(grpid, var->hdr.name, access_pid)) < 0) return NC_EHDFERR; if (H5Pclose(access_pid) < 0) return NC_EHDFERR; @@ -250,6 +256,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, NC_DIM_INFO_T *dim; NC_FILE_INFO_T *h5; NC_TYPE_INFO_T *type_info = NULL; + NC_HDF5_GRP_INFO_T *hdf5_grp; char norm_name[NC_MAX_NAME + 1]; int d; int retval; @@ -257,7 +264,10 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, /* Find info for this file and group, and set pointer to each. */ if ((retval = nc4_find_grp_h5(ncid, &grp, &h5))) BAIL(retval); - assert(grp && h5); + assert(grp && grp->format_grp_info && h5); + + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; /* HDF5 allows maximum of 32 dimensions. */ if (ndims > H5S_MAX_RANK) @@ -435,7 +445,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, /* Now delete the DIM_WITHOUT_VARIABLE dataset (it will be * recreated later, if necessary). */ - if (H5Gunlink(grp->hdf_grpid, dim->hdr.name) < 0) + if (H5Gunlink(hdf5_grp->hdf_grpid, dim->hdr.name) < 0) BAIL(NC_EDIMMETA); } } @@ -1011,6 +1021,7 @@ NC4_rename_var(int ncid, int varid, const char *name) { NC *nc; NC_GRP_INFO_T *grp; + NC_HDF5_GRP_INFO_T *hdf5_grp; NC_FILE_INFO_T *h5; NC_VAR_INFO_T *var, *tmpvar; int retval = NC_NOERR; @@ -1024,7 +1035,10 @@ NC4_rename_var(int ncid, int varid, const char *name) /* Find info for this file and group, and set pointer to each. */ if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) return retval; - assert(h5 && grp && h5); + assert(h5 && grp && grp->format_grp_info && h5); + + /* Get HDF5-specific group info. */ + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; /* Is the new name too long? */ if (strlen(name) > NC_MAX_NAME) @@ -1076,7 +1090,7 @@ NC4_rename_var(int ncid, int varid, const char *name) } LOG((3, "Moving dataset %s to %s", var->hdr.name, name)); - if (H5Gmove(grp->hdf_grpid, var->hdr.name, name) < 0) + if (H5Gmove(hdf5_grp->hdf_grpid, var->hdr.name, name) < 0) BAIL(NC_EHDFERR); } From 261c7fb8d67f0a24a1437af60206b8c2b9c1fd07 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 13:06:15 -0700 Subject: [PATCH 40/46] hdf5-specific group info for nc4info.c --- libhdf5/nc4info.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libhdf5/nc4info.c b/libhdf5/nc4info.c index 066f92165b..70b921cf38 100644 --- a/libhdf5/nc4info.c +++ b/libhdf5/nc4info.c @@ -307,7 +307,7 @@ NC4_put_ncproperties(NC_FILE_INFO_T* file) char* text = NULL; /* Get root group */ - grp = h5->root_grp->hdf_grpid; /* get root group */ + grp = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid; /* See if the NCPROPS attribute exists */ if(H5Aexists(grp,NCPROPS) <= 0) { /* Does not exist */ ncstat = NC4_buildpropinfo(&h5->fileinfo->propattr,&text); @@ -514,7 +514,7 @@ NC4_read_ncproperties(NC_FILE_INFO_T* h5) H5T_class_t t_class; hsize_t size; - hdf5grpid = h5->root_grp->hdf_grpid; + hdf5grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid; if(H5Aexists(hdf5grpid,NCPROPS) <= 0) { /* Does not exist */ /* File did not contain a _NCProperties attribute */ @@ -578,7 +578,7 @@ NC4_write_ncproperties(NC_FILE_INFO_T* h5) if (h5->no_write) {retval = NC_EPERM; goto done;} - hdf5grpid = h5->root_grp->hdf_grpid; + hdf5grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid; if(H5Aexists(hdf5grpid,NCPROPS) > 0) /* Already exists, no overwrite */ goto done; From 902ee64ff0bda24428ff5d69bb906542893d2ae5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 13:38:11 -0700 Subject: [PATCH 41/46] hdf5-specific group stuff for hdf5attr.c --- libhdf5/hdf5attr.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 87c3be1982..82df14e779 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -143,7 +143,8 @@ NC4_rename_att(int ncid, int varid, const char *name, const char *newname) { if (varid == NC_GLOBAL) { - if (H5Adelete(grp->hdf_grpid, att->hdr.name) < 0) + if (H5Adelete(((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid, + att->hdr.name) < 0) return NC_EHDFERR; } else @@ -237,7 +238,7 @@ NC4_del_att(int ncid, int varid, const char *name) /* Determine the location id in the HDF5 file. */ if (varid == NC_GLOBAL) - locid = grp->hdf_grpid; + locid = ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid; else if (var->created) locid = var->hdf_datasetid; From 766f1d0195c9b9b493287b8350fd3c1f8b616b26 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 13:43:43 -0700 Subject: [PATCH 42/46] hdf5-specific changes for hdf5grp.c --- libhdf5/hdf5grp.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libhdf5/hdf5grp.c b/libhdf5/hdf5grp.c index 08d53fb0b6..a924f7a6b8 100644 --- a/libhdf5/hdf5grp.c +++ b/libhdf5/hdf5grp.c @@ -91,7 +91,7 @@ NC4_def_grp(int parent_ncid, const char *name, int *new_ncid) int NC4_rename_grp(int grpid, const char *name) { - NC_GRP_INFO_T *grp, *parent; + NC_GRP_INFO_T *grp; NC_HDF5_GRP_INFO_T *hdf5_grp; NC_FILE_INFO_T *h5; char norm_name[NC_MAX_NAME + 1]; @@ -113,7 +113,6 @@ NC4_rename_grp(int grpid, const char *name) /* Do not allow renaming the root group */ if (grp->parent == NULL) return NC_EBADGRPID; - parent = grp->parent; /* Check and normalize the name. */ if ((retval = nc4_check_name(name, norm_name))) @@ -121,7 +120,7 @@ NC4_rename_grp(int grpid, const char *name) /* Check that this name is not in use as a var, grp, or type in the * parent group (i.e. the group that grp is in). */ - if ((retval = nc4_check_dup_name(parent, norm_name))) + if ((retval = nc4_check_dup_name(grp->parent, norm_name))) return retval; /* If it's not in define mode, switch to define mode. */ @@ -130,26 +129,27 @@ NC4_rename_grp(int grpid, const char *name) return retval; /* Rename the group, if it exists in the file */ - if (grp->hdf_grpid) + if (hdf5_grp->hdf_grpid) { NC_HDF5_GRP_INFO_T *parent_hdf5_grp; parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info; /* Close the group */ - if (H5Gclose(grp->hdf_grpid) < 0) + if (H5Gclose(hdf5_grp->hdf_grpid) < 0) return NC_EHDFERR; hdf5_grp->hdf_grpid = 0; grp->hdf_grpid = 0; /* Attempt to rename & re-open the group, if the parent group is open */ - if (grp->parent->hdf_grpid) + if (parent_hdf5_grp->hdf_grpid) { /* Rename the group */ - if (H5Gmove(parent->hdf_grpid, grp->hdr.name, name) < 0) + if (H5Gmove(parent_hdf5_grp->hdf_grpid, grp->hdr.name, name) < 0) return NC_EHDFERR; /* Reopen the group, with the new name */ - if ((hdf5_grp->hdf_grpid = H5Gopen2(parent->hdf_grpid, name, H5P_DEFAULT)) < 0) + if ((hdf5_grp->hdf_grpid = H5Gopen2(parent_hdf5_grp->hdf_grpid, name, + H5P_DEFAULT)) < 0) return NC_EHDFERR; grp->hdf_grpid = hdf5_grp->hdf_grpid; } @@ -162,7 +162,7 @@ NC4_rename_grp(int grpid, const char *name) return NC_ENOMEM; grp->hdr.hashkey = NC_hashmapkey(grp->hdr.name,strlen(grp->hdr.name)); /* Fix hash key */ - if(!ncindexrebuild(parent->children)) + if(!ncindexrebuild(grp->parent->children)) return NC_EINTERNAL; return NC_NOERR; From 971b16cdc85c893070cc64e1bd2a0268f7d243c7 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 13:51:08 -0700 Subject: [PATCH 43/46] hdf5 specific changes for hdf5open.c --- libhdf5/hdf5open.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index dd87a63b1f..8fb4fe718a 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -332,13 +332,18 @@ static int check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic) { htri_t attr_exists = -1; + hid_t grpid; /* Check inputs. */ - assert(!root_grp->parent && is_classic); + assert(root_grp && root_grp->format_grp_info && !root_grp->parent + && is_classic); + + /* Get the HDF5 group id. */ + grpid = ((NC_HDF5_GRP_INFO_T *)(root_grp->format_grp_info))->hdf_grpid; /* If this attribute exists in the root group, then classic model * is in effect. */ - if ((attr_exists = H5Aexists(root_grp->hdf_grpid, NC3_STRICT_ATT_NAME)) < 0) + if ((attr_exists = H5Aexists(grpid, NC3_STRICT_ATT_NAME)) < 0) return NC_EHDFERR; *is_classic = attr_exists ? 1 : 0; @@ -1661,7 +1666,8 @@ nc4_read_atts(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) att_info.grp = grp; /* Determine where to read from in the HDF5 file. */ - locid = var ? var->hdf_datasetid : grp->hdf_grpid; + locid = var ? var->hdf_datasetid : + ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid; /* Now read all the attributes at this location, ignoring special * netCDF hidden attributes. */ @@ -2031,14 +2037,14 @@ nc4_rec_read_metadata(NC_GRP_INFO_T *grp) /* Open this HDF5 group and retain its grpid. It will remain open * with HDF5 until this file is nc_closed. */ - if (!grp->hdf_grpid) + if (!hdf5_grp->hdf_grpid) { if (grp->parent) { NC_HDF5_GRP_INFO_T *parent_hdf5_grp; parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info; - if ((hdf5_grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid, + if ((hdf5_grp->hdf_grpid = H5Gopen2(parent_hdf5_grp->hdf_grpid, grp->hdr.name, H5P_DEFAULT)) < 0) BAIL(NC_EHDFERR); grp->hdf_grpid = hdf5_grp->hdf_grpid; From 79c2f843d85a15287f023d1d2a7e98e99c8c5d86 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 13:57:04 -0700 Subject: [PATCH 44/46] rest of hdf5 specific group changes --- libhdf5/hdf5internal.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 6efd4bfe33..5c1753c2f3 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -380,13 +380,17 @@ int nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) { NC_HDF5_DIM_INFO_T *hdf5_dim; + NC_HDF5_GRP_INFO_T *hdf5_grp; int need_to_reattach_scales = 0; int retval = NC_NOERR; - assert(grp && var && dim && dim->format_dim_info); + assert(grp && grp->format_grp_info && var && dim && dim->format_dim_info); LOG((3, "%s: dim->hdr.name %s var->hdr.name %s", __func__, dim->hdr.name, var->hdr.name)); + + /* Get HDF5-specific dim and group info. */ hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; /* Detach dimscales from the [new] coordinate variable */ if (var->dimscale_attached) @@ -460,7 +464,7 @@ nc4_reform_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, NC_DIM_INFO_T *dim) /* Now delete the dimscale's dataset (it will be recreated later, if necessary) */ - if (H5Gunlink(grp->hdf_grpid, dim->hdr.name) < 0) + if (H5Gunlink(hdf5_grp->hdf_grpid, dim->hdr.name) < 0) return NC_EDIMMETA; } @@ -503,13 +507,16 @@ nc4_rec_grp_HDF5_del(NC_GRP_INFO_T *grp) NC_VAR_INFO_T *var; NC_DIM_INFO_T *dim; NC_ATT_INFO_T *att; + NC_HDF5_GRP_INFO_T *hdf5_grp; int a; int i; int retval; - assert(grp); + assert(grp && grp->format_grp_info); LOG((3, "%s: grp->name %s", __func__, grp->hdr.name)); + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; + /* Recursively call this function for each child, if any, stopping * if there is an error. */ for (i = 0; i < ncindexsize(grp->children); i++) @@ -615,7 +622,7 @@ nc4_rec_grp_HDF5_del(NC_GRP_INFO_T *grp) /* Close the HDF5 group. */ LOG((4, "%s: closing group %s", __func__, grp->hdr.name)); - if (grp->hdf_grpid && H5Gclose(grp->hdf_grpid) < 0) + if (hdf5_grp->hdf_grpid && H5Gclose(hdf5_grp->hdf_grpid) < 0) return NC_EHDFERR; return NC_NOERR; From 825047b8f623d32671040b36866f5de289cdd350 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 12 Nov 2018 14:04:17 -0700 Subject: [PATCH 45/46] rest of moving HDF5 specific group info to libhdf5 --- include/nc4internal.h | 1 - libhdf5/hdf5create.c | 1 - libhdf5/hdf5grp.c | 2 -- libhdf5/hdf5internal.c | 8 ++++++-- libhdf5/hdf5open.c | 2 -- libhdf5/nc4hdf.c | 3 +-- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index c21af56bc0..fdde906c83 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -249,7 +249,6 @@ typedef struct NC_TYPE_INFO typedef struct NC_GRP_INFO { NC_OBJ hdr; - hid_t hdf_grpid; void *format_grp_info; struct NC_FILE_INFO *nc4_info; struct NC_GRP_INFO *parent; diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 7d6b8af427..b09f26fccf 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -224,7 +224,6 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, /* Open the root group. */ if ((hdf5_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/", H5P_DEFAULT)) < 0) BAIL(NC_EFILEMETA); - nc4_info->root_grp->hdf_grpid = hdf5_grp->hdf_grpid; /* Release the property lists. */ if (H5Pclose(fapl_id) < 0 || H5Pclose(fcpl_id) < 0) diff --git a/libhdf5/hdf5grp.c b/libhdf5/hdf5grp.c index a924f7a6b8..058dab6f84 100644 --- a/libhdf5/hdf5grp.c +++ b/libhdf5/hdf5grp.c @@ -138,7 +138,6 @@ NC4_rename_grp(int grpid, const char *name) if (H5Gclose(hdf5_grp->hdf_grpid) < 0) return NC_EHDFERR; hdf5_grp->hdf_grpid = 0; - grp->hdf_grpid = 0; /* Attempt to rename & re-open the group, if the parent group is open */ if (parent_hdf5_grp->hdf_grpid) @@ -151,7 +150,6 @@ NC4_rename_grp(int grpid, const char *name) if ((hdf5_grp->hdf_grpid = H5Gopen2(parent_hdf5_grp->hdf_grpid, name, H5P_DEFAULT)) < 0) return NC_EHDFERR; - grp->hdf_grpid = hdf5_grp->hdf_grpid; } } diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 5c1753c2f3..c28dfd0f71 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -342,12 +342,16 @@ int delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *dim) { NC_HDF5_DIM_INFO_T *hdf5_dim; + NC_HDF5_GRP_INFO_T *hdf5_grp; int retval; - assert(grp && dim && dim->format_dim_info); + assert(grp && grp->format_grp_info && dim && dim->format_dim_info); LOG((2, "%s: deleting dimscale dataset %s dimid %d", __func__, dim->hdr.name, dimid)); + + /* Get HDF5 specific grp and dim info. */ hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info; + hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info; /* Detach dimscale from any variables using it */ if ((retval = rec_detach_scales(grp, dimid, hdf5_dim->hdf_dimscaleid)) < 0) @@ -359,7 +363,7 @@ delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *d hdf5_dim->hdf_dimscaleid = 0; /* Now delete the dataset. */ - if (H5Gunlink(grp->hdf_grpid, dim->hdr.name) < 0) + if (H5Gunlink(hdf5_grp->hdf_grpid, dim->hdr.name) < 0) return NC_EHDFERR; return NC_NOERR; diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 8fb4fe718a..c43d267d22 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -2047,7 +2047,6 @@ nc4_rec_read_metadata(NC_GRP_INFO_T *grp) if ((hdf5_grp->hdf_grpid = H5Gopen2(parent_hdf5_grp->hdf_grpid, grp->hdr.name, H5P_DEFAULT)) < 0) BAIL(NC_EHDFERR); - grp->hdf_grpid = hdf5_grp->hdf_grpid; } else { @@ -2057,7 +2056,6 @@ nc4_rec_read_metadata(NC_GRP_INFO_T *grp) if ((hdf5_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/", H5P_DEFAULT)) < 0) BAIL(NC_EHDFERR); - grp->hdf_grpid = hdf5_grp->hdf_grpid; } } assert(hdf5_grp->hdf_grpid > 0); diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 8695ee887f..4d6c750913 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -1372,10 +1372,9 @@ create_group(NC_GRP_INFO_T *grp) BAIL(NC_EHDFERR); /* Create the group. */ - if ((hdf5_grp->hdf_grpid = H5Gcreate2(grp->parent->hdf_grpid, grp->hdr.name, + if ((hdf5_grp->hdf_grpid = H5Gcreate2(parent_hdf5_grp->hdf_grpid, grp->hdr.name, H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0) BAIL(NC_EHDFERR); - grp->hdf_grpid = hdf5_grp->hdf_grpid; exit: if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0) From ff7a46178995d56dcf1ef2cdbe97a7e6e9598658 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Mon, 12 Nov 2018 16:06:50 -0700 Subject: [PATCH 46/46] Triggering re-run of TravisCI --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 7fc28d0b25..cfc1b94ba6 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,7 +2,7 @@ Release Notes {#RELEASE_NOTES} ============= \brief Release notes file for the netcdf-c package. - + This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries. ## 4.6.2 - TBD