Merge pull request #1191 from Unidata/v4.6.2-release-branch.wif

V4.6.2 release branch.wif
Unidata · Nov 12, 2018 · 8cb325c · 8cb325c
2 parents 086801a + ff7a461
commit 8cb325c
Show file tree

Hide file tree

Showing 21 changed files with 504 additions and 242 deletions.
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -2,11 +2,14 @@ Release Notes       {#RELEASE_NOTES}
 =============
 
 \brief Release notes file for the netcdf-c package.
-
+ 
 This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries.
 
 ## 4.6.2 - TBD
 
+* [Enhancement] Lazy att read - only read atts when user requests one of them.  See [GitHub #857](https://github.com/Unidata/netcdf-c/issues/857).
+* [Enhancement] Fast global att read - when global atts are read, they are read much more quickly.  See [GitHub #857](https://github.com/Unidata/netcdf-c/issues/857).
+
 ## 4.6.2-rc2 November 1, 2018
 
 

diff --git a/configure.ac b/configure.ac
@@ -349,34 +349,15 @@ AC_ARG_ENABLE([dap],
 test "x$enable_dap" = xno || enable_dap=yes
 AC_MSG_RESULT($enable_dap)
 
-# --enable-dap => enable-dap4
-enable_dap4=$enable_dap
-
-# Curl support is required if and only if any of these flags are set:
-# 1. --enable-dap
-
-if test "x$enable_dap" = "xyes" ; then
-require_curl=yes
-else
-require_curl=no
-fi
-
-# See if the user provided us with a curl library
-# Do an initial lib test for curl, but suppress the default action
+# We need curl for DAP.
 AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
-# If curl is required but there is no curl, then complain
-if test $require_curl = yes ; then
-  if test $found_curl = no ; then
-    AC_MSG_NOTICE([libcurl not found; disabling remote protocol(s) support])
-    enable_dap=no
-    enable_dap4=no
-  elif test $found_curl = yes ; then
-    # Redo the check lib to actually add -lcurl
-    #AC_CHECK_LIB([curl], [curl_easy_setopt])
-    AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll], [], [])
-  fi
+if test "x$enable_dap" = "xyes" ; then
+   AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll], [],
+      [AC_MSG_ERROR([curl required for remote access. Install curl or build with --disable-dap.])])
 fi
 
+# --enable-dap => enable-dap4
+enable_dap4=$enable_dap
 # Default is now to always do the short remote tests
 AC_MSG_CHECKING([whether dap remote testing should be enabled (default on)])
 AC_ARG_ENABLE([dap-remote-tests],

diff --git a/docs/install.md b/docs/install.md
@@ -150,7 +150,7 @@ Indicate where you want to install netCDF in another shell variable, for example
 ~~~~{.py}
     $ # Build and install netCDF-4
     $ NCDIR=/usr/local
-    $ CPPFLAGS=-I${H5DIR}/include LDFLAGS=-L${H5DIR}/lib ./configure --prefix=${NCDIR}
+    $ CPPFLAGS='-I${H5DIR}/include -I${ZDIR}/include' LDFLAGS='-L${H5DIR}/lib -L${ZDIR}/lib' ./configure --prefix=${NCDIR}
     $ make check
     $ make install  # or sudo make install
 ~~~~

diff --git a/include/hdf5internal.h b/include/hdf5internal.h
@@ -56,6 +56,25 @@ typedef struct  NC_HDF5_FILE_INFO
    hid_t hdfid;
 } NC_HDF5_FILE_INFO_T;
 
+/* This is a struct to handle the dim metadata. */
+typedef struct NC_HDF5_DIM_INFO
+{
+   hid_t hdf_dimscaleid;        /* Non-zero if a DIM_WITHOUT_VARIABLE dataset is in use (no coord var). */
+   HDF5_OBJID_T hdf5_objid;
+} NC_HDF5_DIM_INFO_T;
+
+/** Strut to hold HDF5-specific info for attributes. */
+typedef struct  NC_HDF5_ATT_INFO
+{
+   hid_t native_hdf_typeid;     /* Native HDF5 datatype for attribute's data */
+} NC_HDF5_ATT_INFO_T;
+
+/* Struct to hold HDF5-specific info for a group. */
+typedef struct NC_HDF5_GRP_INFO
+{
+   hid_t hdf_grpid;
+} NC_HDF5_GRP_INFO_T;
+
 /* These functions do HDF5 things. */
 int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
 int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);

diff --git a/include/nc4internal.h b/include/nc4internal.h
@@ -112,25 +112,24 @@ typedef struct NC_OBJ {
 typedef struct NC_DIM_INFO
 {
    NC_OBJ hdr;
-   struct NC_GRP_INFO* container;  /* containing group */
+   struct NC_GRP_INFO *container;  /* containing group */
    size_t len;
    nc_bool_t unlimited;         /* True if the dimension is unlimited */
    nc_bool_t extended;          /* True if the dimension needs to be extended */
    nc_bool_t too_long;          /* True if len is too big to fit in local size_t. */
-   hid_t hdf_dimscaleid;        /* Non-zero if a DIM_WITHOUT_VARIABLE dataset is in use (no coord var). */
-   HDF5_OBJID_T hdf5_objid;
+   void *format_dim_info;       /* Pointer to format-specific dim info. */
    struct NC_VAR_INFO *coord_var; /* The coord var, if it exists. */
 } NC_DIM_INFO_T;
 
 typedef struct NC_ATT_INFO
 {
    NC_OBJ hdr;
-   struct NC_OBJ* container;    /* containing group|var */
+   struct NC_OBJ *container;    /* containing group|var */
    int len;
    nc_bool_t dirty;             /* True if attribute modified */
    nc_bool_t created;           /* True if attribute already created */
    nc_type nc_typeid;           /* netCDF type of attribute's data */
-   hid_t native_hdf_typeid;     /* Native HDF5 datatype for attribute's data */
+   void *format_att_info;
    void *data;
    nc_vlen_t *vldata; /* only used for vlen */
    char **stdata; /* only for string type. */
@@ -250,7 +249,7 @@ typedef struct NC_TYPE_INFO
 typedef struct NC_GRP_INFO
 {
    NC_OBJ hdr;
-   hid_t hdf_grpid;
+   void *format_grp_info;
    struct NC_FILE_INFO *nc4_info;
    struct NC_GRP_INFO *parent;
    int atts_not_read;

diff --git a/libdispatch/drc.c b/libdispatch/drc.c
@@ -367,9 +367,12 @@ rclocate(const char* key, const char* hostport)
     if(hostport == NULL) hostport = "";
 
     for(found=0,i=0;i<nclistlength(rc);i++) {
-	triple = (NCTriple*)nclistget(rc,i);
-        size_t hplen = (triple->host == NULL ? 0 : strlen(triple->host));
-        int t;
+      int t;
+      size_t hplen;
+      triple = (NCTriple*)nclistget(rc,i);
+
+      hplen = (triple->host == NULL ? 0 : strlen(triple->host));
+
         if(strcmp(key,triple->key) != 0) continue; /* keys do not match */
         /* If the triple entry has no url, then use it
            (because we have checked all other cases)*/

diff --git a/libdispatch/nchashmap.c b/libdispatch/nchashmap.c
@@ -108,7 +108,7 @@ rehash(NC_hashmap* hm)
 /* Locate where given object is or should be placed in indexp.
    if fail to find spot return 0 else 1.
    If deletok then a deleted slot is ok to return;
-   return invariant: return == 0 || *indexp is defined 
+   return invariant: return == 0 || *indexp is defined
  */
 static int
 locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize, size_t* indexp, int deletedok)
@@ -118,15 +118,15 @@ locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize,
     size_t step = 1; /* simple linear probe */
     int deletefound = 0;
     size_t deletedindex = 0; /* first deleted entry encountered */
-
+    NC_hentry* entry;
     TRACE("locate");
     /* Compute starting point */
     index = (size_t)(hashkey % hash->alloc);
 
     /* Search table using linear probing */
     for (i = 0; i < hash->alloc; i++) {
-        NC_hentry* entry = &hash->table[index];
-        if(entry->flags & ACTIVE) {
+      entry = &hash->table[index];
+      if(entry->flags & ACTIVE) {
 	    if(indexp) *indexp = index; /* assume a match */
             if(entry->hashkey == hashkey && entry->keysize == keysize) {
 		/* Check content */
@@ -148,7 +148,7 @@ locate(NC_hashmap* hash, unsigned int hashkey, const char* key, size_t keysize,
 	index = (index + step) % hash->alloc;
     }
     if(deletedok && deletefound) {
-	if(indexp) *indexp = deletedindex;	
+	if(indexp) *indexp = deletedindex;
 	return 1;
     }
     return 0;
@@ -187,12 +187,13 @@ NC_hashmapnew(size_t startsize)
 int
 NC_hashmapadd(NC_hashmap* hash, uintptr_t data, const char* key, size_t keysize)
 {
-    unsigned int hashkey;
+  NC_hentry* entry;
+  unsigned int hashkey;
 
     TRACE("NC_hashmapadd");
 
     if(key == NULL || keysize == 0)
-	return 0;
+      return 0;
     hashkey = NC_crc32(0,(unsigned char*)key,(unsigned int)keysize);
 
     if(hash->alloc*3/4 <= hash->active)
@@ -203,7 +204,7 @@ NC_hashmapadd(NC_hashmap* hash, uintptr_t data, const char* key, size_t keysize)
 	    rehash(hash);
 	    continue; /* try on larger table */
 	}
-        NC_hentry* entry = &hash->table[index];
+    entry = &hash->table[index];
 	if(entry->flags & ACTIVE) {
 	    /* key already exists in table => overwrite data */
 	    entry->data = data;
@@ -264,13 +265,13 @@ NC_hashmapget(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t* data
 	return 0;
     hashkey = NC_crc32(0,(unsigned char*)key,(unsigned int)keysize);
     if(hash->active) {
-        size_t index;
-        NC_hentry* h;
-        if(!locate(hash,hashkey,key,keysize,&index,0))
+      size_t index;
+      NC_hentry* h;
+      if(!locate(hash,hashkey,key,keysize,&index,0))
 	    return 0; /* not present */
-	h = &hash->table[index];
+      h = &hash->table[index];
 	if(h->flags & ACTIVE) {
-	    if(datap) *datap = h->data;
+      if(datap) *datap = h->data;
 	    return 1;
         } else /* Not found */
 	    return 0;
@@ -300,7 +301,7 @@ NC_hashmapsetdata(NC_hashmap* hash, const char* key, size_t keysize, uintptr_t n
     h = &hash->table[index];
     assert((h->flags & ACTIVE) == ACTIVE);
     h->data = newdata;
-    return 1;    
+    return 1;
 }
 
 size_t
@@ -339,7 +340,7 @@ Allows a hack by ncindex.
 int
 NC_hashmapdeactivate(NC_hashmap* map, uintptr_t data)
 {
-    size_t i;        
+    size_t i;
     NC_hentry* h;
     for(h=map->table,i=0;i<map->alloc;i++,h++) {
 	if((h->flags & ACTIVE) && h->data == data) {
@@ -370,21 +371,23 @@ findPrimeGreaterThan(size_t val)
       int L = 1; /* skip leading flag number */
       int R = (n - 2); /* skip trailing flag */
       unsigned int v = 0;
+      int m;
 
       if(val >= 0xFFFFFFFF)
-	return 0; /* Too big */
+        return 0; /* Too big */
       v = (unsigned int)val;
 
       for(;;) {
-	if(L >= R) break;
-            int m = (L + R) / 2;
-	/* is this an acceptable prime? */
-            if(NC_primes[m-1] < v && NC_primes[m] >= v)
-	    return NC_primes[m]; /* acceptable*/
-            else if(NC_primes[m-1] >= v)
-	        R = m;
-            else if(NC_primes[m] < v)
-	        L = m;
+        if(L >= R) break;
+
+        m = (L + R) / 2;
+        /* is this an acceptable prime? */
+        if(NC_primes[m-1] < v && NC_primes[m] >= v)
+          return NC_primes[m]; /* acceptable*/
+        else if(NC_primes[m-1] >= v)
+          R = m;
+        else if(NC_primes[m] < v)
+          L = m;
       }
       return 0;
 }
@@ -2046,6 +2049,7 @@ static unsigned int NC_nprimes = (sizeof(NC_primes) / sizeof(unsigned int));
 void
 printhashmapstats(NC_hashmap* hm)
 {
+
     size_t n,i;
     size_t step = 1;
     size_t maxchain = 0;
@@ -2061,7 +2065,7 @@ printhashmapstats(NC_hashmap* hm)
 	    default: /* empty slot, stop walking */
 		if(chainlen > maxchain) maxchain = chainlen;
 		goto next;
-	    }		
+	    }
             /* linear probe */
 	    index = (index + step) % hm->alloc;
 	}

diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c
@@ -143,7 +143,8 @@ NC4_rename_att(int ncid, int varid, const char *name, const char *newname)
    {
       if (varid == NC_GLOBAL)
       {
-         if (H5Adelete(grp->hdf_grpid, att->hdr.name) < 0)
+         if (H5Adelete(((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid,
+                       att->hdr.name) < 0)
             return NC_EHDFERR;
       }
       else
@@ -237,7 +238,7 @@ NC4_del_att(int ncid, int varid, const char *name)
 
    /* Determine the location id in the HDF5 file. */
    if (varid == NC_GLOBAL)
-      locid = grp->hdf_grpid;
+      locid = ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid;
    else if (var->created)
       locid = var->hdf_datasetid;
 
@@ -451,7 +452,11 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
    {
       LOG((3, "adding attribute %s to the list...", norm_name));
       if ((ret = nc4_att_list_add(attlist, norm_name, &att)))
-         BAIL (ret);
+         BAIL(ret);
+
+      /* Allocate storage for the HDF5 specific att info. */
+      if (!(att->format_att_info = calloc(1, sizeof(NC_HDF5_ATT_INFO_T))))
+         BAIL(NC_ENOMEM);
    }
 
    /* Now fill in the metadata. */

diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c
@@ -48,6 +48,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
    int retval = NC_NOERR;
    NC_FILE_INFO_T* nc4_info = NULL;
    NC_HDF5_FILE_INFO_T *hdf5_info;
+   NC_HDF5_GRP_INFO_T *hdf5_grp;
 
 #ifdef USE_PARALLEL4
    NC_MPI_INFO *mpiinfo = NULL;
@@ -71,6 +72,11 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
       BAIL(NC_ENOMEM);
    hdf5_info = (NC_HDF5_FILE_INFO_T *)nc4_info->format_file_info;
 
+   /* Add struct to hold HDF5-specific group info. */
+   if (!(nc4_info->root_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T))))
+      return NC_ENOMEM;
+   hdf5_grp = (NC_HDF5_GRP_INFO_T *)nc4_info->root_grp->format_grp_info;
+
    nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY;
    nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS;
    nc4_info->mem.persist =  (cmode & NC_PERSIST) == NC_PERSIST;
@@ -216,8 +222,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
    }
 
    /* Open the root group. */
-   if ((nc4_info->root_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/",
-                                                 H5P_DEFAULT)) < 0)
+   if ((hdf5_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/", H5P_DEFAULT)) < 0)
       BAIL(NC_EFILEMETA);
 
    /* Release the property lists. */