From 2ea1cf5f1bc2a7352e3f66721f5181e26e556011 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 25 Aug 2018 21:44:41 -0600 Subject: [PATCH 1/2] There was a request to extend the provenance information stored in the _NCProperties attribute to allow two things: 1. capture of additional library dependencies (over and above hdf5) 2. Recognition of non-netcdf libraries that create netcdf-4 format files. To this end, the _NCProperties format has been extended to be and arbitrary set of key=value pairs separated by commas. This new format has version = 2, and uses commas as the pair separator. Thus the general form is: _NCProperties = "version=2,key1=value,key2=value2..." ; This new version is accompanied by a new ./configure option of the form --with-ncproperties="key1=value1,key2=value2..." that specifies pairs to add to the _NCProperties attribute for all files created with that netcdf library. At this point, what is missing is some programmatic way to specify either all the pairs or additional pairs to the _NCProperties attribute. Not sure of the best way to do this. Builders using non-netcdf libraries can specify whatever they want in the key value pairs (as long as the version=2 is specified first). By convention, the primary library is expected to be the the first pair after the leading version=2 pair, but this is convention only and is neither required nor enforced. Related changes: 1. Fixed the tests that check _NCProperties to properly operate with version=2. 2. When reading a version 1 _NCProperties attribute, convert it to look like a version 2 attribute. 2. Added some version 2 tests to ncdump/tst_fileinfo.c and ncdump/tst_fileinfo.sh Misc Changes: 1. Fix minor problem in ncdap_test/testurl.sh where a parameter to buildurl needed to be quoted. 2. Minor fix to ncgen to swap switches -H and -h to be consistent with other utilities. 3. Document the -M flag in nccopy usage() and the nccopy man page. 4. Modify a test case to use the nccopy -M flag. --- CMakeLists.txt | 1 + RELEASE_NOTES.md | 1 + cf | 18 +- cf.cmake | 7 +- config.h.cmake.in | 3 + configure.ac | 12 + docs/attribute_conventions.md | 28 +- include/hdf5internal.h | 3 + include/nc4internal.h | 77 +++- libhdf5/hdf5attr.c | 62 +++- libhdf5/hdf5create.c | 9 +- libhdf5/hdf5file.c | 22 +- libhdf5/hdf5open.c | 23 +- libhdf5/nc4info.c | 626 ++++++++++++++++++++++++-------- liblib/nc_initialize.c | 2 +- libsrc4/nc4attr.c | 6 +- nc_test/Make0 | 2 +- ncdap_test/testurl.sh | 8 +- ncdump/Make0 | 4 +- ncdump/Makefile.am | 3 +- ncdump/nccopy.1 | 6 + ncdump/nccopy.c | 5 +- ncdump/ref_provenance_v1.nc | Bin 0 -> 1469 bytes ncdump/ref_tst_ncf213.cdl | 2 +- ncdump/ref_tst_special_atts.cdl | 2 +- ncdump/tst_fileinfo.c | 3 - ncdump/tst_fileinfo.sh | 41 ++- ncdump/tst_nccopy3.sh | 1 + ncdump/tst_nccopy4.sh | 24 +- ncdump/tst_netcdf4.sh | 24 +- ncdump/tst_netcdf4_4.sh | 34 +- ncgen/main.c | 5 +- 32 files changed, 787 insertions(+), 277 deletions(-) create mode 100644 ncdump/ref_provenance_v1.nc diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d1a327cc3..68fe53c94b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -315,6 +315,7 @@ SET(CHUNK_CACHE_PREEMPTION 0.75 CACHE STRING "Default file chunk cache preemptio SET(MAX_DEFAULT_CACHE_SIZE 67108864 CACHE STRING "Default maximum cache size.") SET(NETCDF_LIB_NAME "" CACHE STRING "Default name of the netcdf library.") SET(TEMP_LARGE "." CACHE STRING "Where to put large temp files if large file tests are run.") +SET(NCPROPERTIES_EXTRA "" CACHE STRNG "Specify extra pairs for _NCProperties.") IF(NOT NETCDF_LIB_NAME STREQUAL "") SET(MOD_NETCDF_NAME ON) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 0360b16368..8cef0a6f1e 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.7.0 - TBD +* [Enhancement] Create a new version of _NCProperties provenance attribute. This version (version 2) supports arbitrary key-value pairs. It is the default when new files are created. Version 1 continues to be accepted. * [Enhancement] Support DAP4 remote tests using a new remote test server locatedon the Unidata JetStream project. * [Enhancement] Improved the performance of the nc_get/put_vars operations by using the equivalent slab capabilities of hdf5. Result is a significant speedup of these operations. See [GitHub #1001](https://github.com/Unidata/netcdf-c/pull/1001) for more information. * [Enhancement] Expanded the capabilities of `NC_INMEMORY` to support writing and accessing the final modified memory. See [GitHub #879](https://github.com/Unidata/netcdf-c/pull/879) for more information. diff --git a/cf b/cf index 48edb83e3e..0660dfd13b 100644 --- a/cf +++ b/cf @@ -3,7 +3,7 @@ DB=1 #X=-x -#FAST=1 +FAST=1 #PROF=1 HDF5=1 @@ -111,7 +111,7 @@ FLAGS="$FLAGS --enable-extreme-numbers" #FLAGS="$FLAGS --enable-doxygen" #FLAGS="$FLAGS --enable-internal-docs" FLAGS="$FLAGS --enable-logging" -FLAGS="$FLAGS --disable-diskless" +#FLAGS="$FLAGS --disable-diskless" #FLAGS="$FLAGS --enable-mmap" #FLAGS="$FLAGS --with-udunits" #FLAGS="$FLAGS --with-libcf" @@ -120,6 +120,7 @@ FLAGS="$FLAGS --disable-diskless" #FLAGS="$FLAGS --disable-silent-rules" #FLAGS="$FLAGS --disable-filter-testing" #FLAGS="$FLAGS --enable-metadata-perf" +#FLAGS="$FLAGS --with-ncproperties-extra=key1=value1,key2=value2" if test "x$TESTSERVERS" != x ; then FLAGS="$FLAGS --with-testservers=$TESTSERVERS" @@ -184,9 +185,6 @@ CFLAGS="${CFLAGS} -pg" LDFLAGS="${LDFLAGS} -pg" fi - -#FLAGS="${FLAGS} --enable-stdio" - export PATH export CC export CPPFLAGS @@ -199,17 +197,13 @@ export CXXFLAGS DISTCHECK_CONFIGURE_FLAGS="$FLAGS" export DISTCHECK_CONFIGURE_FLAGS -if test "x$NB" != x -o "x$FAST" = x ; then -${MAKE} maintainer-clean >/dev/null 2>&1 +if test "x$NB" != x1 && test "x$FAST" != x1 ; then +${MAKE} distclean >/dev/null 2>&1 fi -if test -z "$NB" ; then +if test "x$NB" != x1 ; then if autoreconf -i --force ; then ok=1; else exit ; fi fi -if test -z "$FAST" ; then - if test -f Makefile ; then ${MAKE} distclean >/dev/null 2>&1 ; fi -fi - sh $X ./configure ${FLAGS} for c in $cmds; do printenv LD_LIBRARY_PATH diff --git a/cf.cmake b/cf.cmake index 1dd4ad581e..a2b31fc13c 100644 --- a/cf.cmake +++ b/cf.cmake @@ -6,6 +6,8 @@ DAP=1 #CDF5=1 #HDF4=1 +export SETX=1 + for arg in "$@" ; do case "$arg" in vs|VS) VS=1 ;; @@ -60,7 +62,7 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true" FLAGS="$FLAGS -DENABLE_LOGGING=true" #FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true" #FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true" -FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" +#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" # Disables FLAGS="$FLAGS -DENABLE_EXAMPLES=false" @@ -68,6 +70,9 @@ FLAGS="$FLAGS -DENABLE_CONVERSION_WARNINGS=false" #FLAGS="$FLAGS -DENABLE_TESTS=false" #FLAGS="$FLAGS -DENABLE_DISKLESS=false" +# Withs +FLAGS="$FLAGS -DNCPROPERTIES_EXTRA=\"key1=value1|key2=value2\"" + rm -fr build mkdir build cd build diff --git a/config.h.cmake.in b/config.h.cmake.in index 143474b626..8eef200e1a 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -375,6 +375,9 @@ are set when opening a binary file on Windows. */ /* min blocksize for posixio. */ #cmakedefine NCIO_MINBLOCKSIZE ${NCIO_MINBLOCKSIZE} +/* Add extra properties to _NCProperties attribute */ +#cmakedefine NCPROPERTIES_EXTRA ${NCPROPERTIES_EXTRA} + /* no IEEE float on this platform */ #cmakedefine NO_IEEE_FLOAT 1 diff --git a/configure.ac b/configure.ac index 48f88a944f..8ade146874 100644 --- a/configure.ac +++ b/configure.ac @@ -592,6 +592,18 @@ AC_MSG_RESULT($TEMP_LARGE) #AC_SUBST(TEMP_LARGE) AC_DEFINE_UNQUOTED([TEMP_LARGE], ["$TEMP_LARGE"], [Place to put very large netCDF test files.]) +# Specify extra values to add to _NCProperties attribute +# --with-ncproperties-extra="=|...". +# Note: need to figure out a way to do this programmatically also +AC_MSG_CHECKING([Extra values for _NCProperties]) +AC_ARG_WITH([ncproperties-extra], + [AS_HELP_STRING([--with-ncproperties-extra="=,...], + [specify extra pairs for _NCProperties])], + [NCPROPERTIES_EXTRA=$with_ncproperties_extra], + [NCPROPERTIES_EXTRA=""]) +AC_MSG_RESULT([$NCPROPERTIES_EXTRA]) +AC_DEFINE_UNQUOTED([NCPROPERTIES_EXTRA], ["$NCPROPERTIES_EXTRA"], [Extra pairs for _NCProperties]) + # Did the user specify a user-defined format 0? AC_MSG_CHECKING([whether user-defined format 0 was specified]) AC_ARG_WITH([udf0], diff --git a/docs/attribute_conventions.md b/docs/attribute_conventions.md index 17046fe665..884f89f3ce 100644 --- a/docs/attribute_conventions.md +++ b/docs/attribute_conventions.md @@ -161,18 +161,24 @@ Using the following API calls will fail. `_NCProperties` -> This attribute is persistent in the file, but hidden. It is inserted in the file at creation time and is never modified after that point. It specifies the following. -> - The version for the netcdf library used at creation time. -> - The version for the HDF5 library used at creation time. -> - The type of this attribute is NC_CHAR. - -> Its format is: `name=value|name=value ...`
-> Occurrences of '|' in the name or value are disallowed. +> This attribute is persistent in the file, but hidden. It is inserted in the file at creation time and is never modified after that point. The type of this attribute is currently NC_CHAR. There two versions of this property, but both have the general form +>> version=n,key=value,key=value...,key=value +> where the version number n is either 1 or 2. -> The current set of known names is as follows. -> - version=... The current format version for the _NCProperties file, currently 1. -> - netcdflibversion=... The version of the netcdf library used to create the file. The value is, for example, 4.4.1-rc1-development or 4.4.1. -> - hdf5libversion=... The version of the HDF5 library used to create the file. The value is, for example, 1.8.16 or 1.10.0. +> Version 1 has two (key,value) pairs (after than the initial version pair) +> - netcdfversion={netcdfversion} where the version number is the version for the netcdf library used at creation time. +> - hdf5version={hdf5fversion} where the version number is the version for the hdf5 library used at creation time. +> - The version for the HDF5 library used at creation time. +> - Note that for version 1, the separator character is '|' instead of ',' +> - Occurrences of '=' or '|' in the name or value are disallowed. + +> Version 2 has an unlimited set of (key,value) pairs (after the initial version pair). By convention (but unenforced), the first pair is the name and version of the primary library used to create this file. For netcdf, it has the form _netcdf={version}_. The remaining fields are by convention as follows. +> - If the primary build library is netcdf, then the second pair is of the form _hdf5={version}. The remaining pairs consist of a combination of the name and version of important supporting libraries -- the libcurl version, for example -- plus an arbitrary set of pairs as specified by the _--with-ncproperties_ option to the ./configure command. Note that the argument to --with-ncproperties should be wrapped with double quotes, like this. +>> _./configure ... --with-ncproperties="key1=value,key2=value"_ +> +> Note the following for version 2. +> - The pair separator character was changed from '|' to ',' because of problems with bash, which did not like '|' in the --with-ncproperties value. +> - It is possible to include escaped characters using the standard '\' escape convention. `_SuperblockVersion` diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 14ffdb6402..05e3b58d3d 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -60,6 +60,9 @@ typedef struct NC_HDF5_FILE_INFO int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); +/* Used by NC4_set_provenance */ +int nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, + size_t len, const void *data, nc_type mem_type, int force); /* These functions are internal to the libhdf5 directory. */ int nc4_detect_preserve_dimids(NC_GRP_INFO_T *grp, nc_bool_t *bad_coord_orderp); int hdf5_set_log_level(); diff --git a/include/nc4internal.h b/include/nc4internal.h index f49eac9d49..7036e67c6e 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -85,7 +85,7 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT; typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t; /*Forward*/ -struct NCFILEINFO; +struct NCPROVENANCE; struct NC_GRP_INFO; struct NC_TYPE_INFO; @@ -299,7 +299,7 @@ typedef struct NC_FILE_INFO NClist* alltypes; NClist* allgroups; /* including root group */ void *format_file_info; - struct NCFILEINFO* fileinfo; + struct NCPROVENANCE* provenance; struct NC4_Memio { NC_memio memio; int locked; /* do not copy and do not release */ @@ -439,8 +439,10 @@ typedef struct NC_reservedatt { #define DIMSCALEFLAG 1 /* Readonly global attributes; readable, but immutable thru the API */ #define READONLYFLAG 2 -/* Subset of readonly flags; readable by name only thru the API*/ +/* Subset of readonly flags; readable by name only thru the API */ #define NAMEONLYFLAG 4 +/* Subset of readonly flags; Value is actually in file */ +#define MATERIALIZEDFLAG 8 /* Binary searcher for reserved attributes */ extern const NC_reservedatt* NC_findreserved(const char* name); @@ -463,38 +465,73 @@ For netcdf4 files, capture state information about the following: 5. Per file: _NCProperties attribute */ -#define NCPROPS "_NCProperties" -#define NCPROPS_VERSION (1) -#define NCPROPSSEP '|' +/* Most of this needs to be moved to hdf5internal.h */ -/* Currently used properties */ +#define NCPROPS "_NCProperties" #define NCPVERSION "version" /* Of the properties format */ -#define NCPHDF5LIBVERSION "hdf5libversion" -#define NCPNCLIBVERSION "netcdflibversion" +#define NCPHDF5LIB1 "hdf5libversion" +#define NCPNCLIB1 "netcdflibversion" +#define NCPHDF5LIB2 "hdf5" +#define NCPNCLIB2 "netcdf" +#define NCPROPS_VERSION (2) +/* Version 2 changes this because '|' was causing bash problems */ +#define NCPROPSSEP1 '|' +#define NCPROPSSEP2 ',' + /* Other hidden attributes */ #define ISNETCDF4ATT "_IsNetcdf4" #define SUPERBLOCKATT "_SuperblockVersion" -struct NCFILEINFO { +struct NCPROVENANCE { int superblockversion; - /* Following is filled from NCPROPS attribute or from global version */ - struct NCPROPINFO { - int version; /* 0 => not defined */ - char hdf5ver[NC_MAX_NAME+1]; - char netcdfver[NC_MAX_NAME+1]; - } propattr; + struct NCPROPINFO { + int version; /* 0 => not defined */ + /* Following is filled from NCPROPS attribute or from global version */ + /* Version 1 format is: + "netcdflibversion=" + Version 2 format is: + "==...|=..." + */ + /* The _NCProperties values are stored as an arbitrary + set of (key,value) pairs */ + /* It is assumed that the first entry is the primary library + used to build the file, and it is followed by other libraries + used in the build, and finally an arbitrary list of other + (key,value) pairs. */ + NClist* properties; + } propattr; }; +/* Provenance Initialization */ extern struct NCPROPINFO globalpropinfo; -extern int NC4_fileinfo_init(void); /*libsrc4/ncinfo.c*/ -extern int NC4_get_fileinfo(struct NC_FILE_INFO* info, struct NCPROPINFO*); /*libsrc4/ncinfo.c*/ -extern int NC4_put_propattr(struct NC_FILE_INFO* info); /*libsrc4/ncinfo.c*/ -extern int NC4_buildpropinfo(struct NCPROPINFO* info,char** propdatap); +/* Initialize the fileinfo global state */ +extern int NC4_provenance_init(); + +/* Write the properties attribute to file. */ +extern int NC4_put_ncproperties(NC_FILE_INFO_T* file); + +/* Extract the provenance from a file, using dfalt as default */ +extern int NC4_get_provenance(NC_FILE_INFO_T* file, const char* propstring, const struct NCPROPINFO* dfalt); + +/* Set the provenance for a created file using dfalt as default */ +extern int NC4_set_provenance(NC_FILE_INFO_T* file, const struct NCPROPINFO* dfalt); + +/* Recover memory of an NCPROVENANCE object */ +extern int NC4_free_provenance(struct NCPROVENANCE* prov); extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/ extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/ extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/ +/* Convert a NCPROPINFO instance to a single string. */ +extern int NC4_buildpropinfo(struct NCPROPINFO* info, char** propdatap); + +/* Use HDF5 API to read the _NCProperties attribute */ +extern int NC4_read_ncproperties(NC_FILE_INFO_T*); + +/* Use HDF5 API to write the _NCProperties attribute */ +extern int NC4_write_ncproperties(NC_FILE_INFO_T*); + #endif /* _NC4INTERNAL_ */ diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 4da1a65008..a7514d9ddd 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -309,7 +309,8 @@ nc4typelen(nc_type type) } /** - * @internal Write an attribute to a netCDF-4/HDF5 file, converting + * @internal + * Write an attribute to a netCDF-4/HDF5 file, converting * data type if necessary. * * @param ncid File and group ID. @@ -319,6 +320,7 @@ nc4typelen(nc_type type) * @param len Number of elements in attribute array. * @param data Attribute data. * @param mem_type Type of data in memory. + * @param force write even if the attribute is special * * @return ::NC_NOERR No error. * @return ::NC_EINVAL Invalid parameters. @@ -329,11 +331,10 @@ nc4typelen(nc_type type) * @author Ed Hartnett, Dennis Heimbigner */ int -NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, - size_t len, const void *data, nc_type mem_type) +nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, + size_t len, const void *data, nc_type mem_type, int force) { - NC *nc; - NC_GRP_INFO_T *grp; + NC* nc; NC_FILE_INFO_T *h5; NC_VAR_INFO_T *var = NULL; NCindex* attlist = NULL; @@ -344,12 +345,14 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, size_t type_size; int i; int ret; + int ncid; - /* Find info for this file, group, and h5 info. */ - if ((ret = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) - return ret; + h5 = grp->nc4_info; + nc = h5->controller; assert(nc && grp && h5); + ncid = nc->ext_ncid | grp->hdr.id; + /* Find att, if it exists. (Must check varid first or nc_test will * break.) */ if ((ret = getattlist(grp, varid, &var, &attlist))) @@ -365,7 +368,7 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, return NC_EBADNAME; LOG((1, "%s: ncid 0x%x varid %d name %s file_type %d mem_type %d len %d", - __func__, ncid, varid, name, file_type, mem_type, len)); + __func__,ncid, varid, name, file_type, mem_type, len)); /* If len is not zero, then there must be some data. */ if (len && !data) @@ -381,7 +384,7 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, /* Check that a reserved att name is not being used improperly */ const NC_reservedatt* ra = NC_findreserved(name); - if(ra != NULL) { + if(ra != NULL && !force) { /* case 1: grp=root, varid==NC_GLOBAL, flags & READONLYFLAG */ if (nc->ext_ncid == ncid && varid == NC_GLOBAL && grp->parent == NULL && (ra->flags & READONLYFLAG)) @@ -656,3 +659,42 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, return NC_ERANGE; return NC_NOERR; } + +/** + * @internal + * Write an attribute to a netCDF-4/HDF5 file, converting + * data type if necessary. + * Wrapper around nc4_put_att + * + * @param ncid File and group ID. + * @param varid Variable ID. + * @param name Name of attribute. + * @param file_type Type of the attribute data in file. + * @param len Number of elements in attribute array. + * @param data Attribute data. + * @param mem_type Type of data in memory. + * + * @return ::NC_NOERR No error. + * @return ::NC_EINVAL Invalid parameters. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_ENOTVAR Variable not found. + * @return ::NC_EBADNAME Name contains illegal characters. + * @return ::NC_ENAMEINUSE Name already in use. + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, + size_t len, const void *data, nc_type mem_type) +{ + int ret = NC_NOERR; + NC *nc; + NC_FILE_INFO_T *h5; + NC_GRP_INFO_T *grp; + + /* Find info for this file, group, and h5 info. */ + if ((ret = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) + return ret; + assert(nc && grp && h5); + + return nc4_put_att(grp,varid,name,file_type,len,data,mem_type,0); +} diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index beaaa20949..2a9bee7d3a 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -228,13 +228,8 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, /* Define mode gets turned on automatically on create. */ nc4_info->flags |= NC_INDEF; - /* Get the HDF5 superblock and read and parse the special - * _NCProperties attribute. */ - if ((retval = NC4_get_fileinfo(nc4_info, &globalpropinfo))) - BAIL(retval); - - /* Write the _NCProperties attribute. */ - if ((retval = NC4_put_propattr(nc4_info))) + /* Save the HDF5 superblock number and set the _NCProperties attribute. */ + if ((retval = NC4_set_provenance(nc4_info, &globalpropinfo))) BAIL(retval); return NC_NOERR; diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index b68934150a..b935701b04 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -35,13 +35,13 @@ static const NC_reservedatt NC_reserved[NRESERVED] = { {NC_ATT_DIMENSION_LIST, READONLYFLAG|DIMSCALEFLAG}, /*DIMENSION_LIST*/ {NC_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*NAME*/ {NC_ATT_REFERENCE_LIST, READONLYFLAG|DIMSCALEFLAG}, /*REFERENCE_LIST*/ - {NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/ - {ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/ - {NCPROPS, READONLYFLAG|NAMEONLYFLAG}, /*_NCProperties*/ - {NC_ATT_COORDINATES, READONLYFLAG|DIMSCALEFLAG}, /*_Netcdf4Coordinates*/ - {NC_DIMID_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*_Netcdf4Dimid*/ + {NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/ + {ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/ + {NCPROPS, READONLYFLAG|NAMEONLYFLAG|MATERIALIZEDFLAG},/*_NCProperties*/ + {NC_ATT_COORDINATES, READONLYFLAG|DIMSCALEFLAG|MATERIALIZEDFLAG},/*_Netcdf4Coordinates*/ + {NC_DIMID_ATT_NAME, READONLYFLAG|DIMSCALEFLAG|MATERIALIZEDFLAG},/*_Netcdf4Dimid*/ {SUPERBLOCKATT, READONLYFLAG|NAMEONLYFLAG},/*_SuperblockVersion*/ - {NC3_STRICT_ATT_NAME, READONLYFLAG}, /*_nc3_strict*/ + {NC3_STRICT_ATT_NAME, READONLYFLAG|MATERIALIZEDFLAG}, /*_nc3_strict*/ }; extern void reportopenobjects(int log, hid_t); @@ -116,7 +116,7 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5) #endif /* Write any metadata that has changed. */ - if (!(h5->cmode & NC_NOWRITE)) + if (!h5->no_write) { nc_bool_t bad_coord_order = NC_FALSE; @@ -133,6 +133,10 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5) /* Write all the metadata. */ if ((retval = nc4_rec_write_metadata(h5->root_grp, bad_coord_order))) return retval; + + /* Write out _NCProperties */ + if((retval = NC4_write_ncproperties(h5))) + return retval; } /* Tell HDF5 to flush all changes to the file. */ @@ -199,8 +203,8 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) /* Free the fileinfo struct, which holds info from the fileinfo * hidden attribute. */ - if (h5->fileinfo) - free(h5->fileinfo); + if (h5->provenance) + free(h5->provenance); /* Check to see if this is an in-memory file and we want to get its final content. */ diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 6a0095ab60..4e9966df29 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -400,7 +400,7 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) /* Need this access plist to control how HDF5 handles open objects * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to - * fail if there are any open objects in the file. */ + * fail if there are any open objects in the file). */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) BAIL(NC_EHDFERR); @@ -502,6 +502,22 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if (is_classic) nc4_info->cmode |= NC_CLASSIC_MODEL; + /* See if this file contained _NCPROPERTIES, + * and if yes, process it, if no, then fake it. + */ + if(nc4_info->root_grp != NULL) { + /* Since _NCProperties does not exist as an NC_ATT_INFO object, + * we need to check using the HDF5 API + */ + if((retval = NC4_read_ncproperties(nc4_info))) + BAIL(retval); + } + + if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic))) + BAIL(retval); + if (is_classic) + nc4_info->cmode |= NC_CLASSIC_MODEL; + /* Now figure out which netCDF dims are indicated by the dimscale * information. */ if ((retval = nc4_rec_match_dimscales(nc4_info->root_grp))) @@ -517,11 +533,6 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if (H5Pclose(fapl_id) < 0) BAIL(NC_EHDFERR); - /* Get the HDF5 superblock and read and parse the special - * _NCProperties attribute. */ - if ((retval = NC4_get_fileinfo(nc4_info, NULL))) - BAIL(retval); - return NC_NOERR; exit: diff --git a/libhdf5/nc4info.c b/libhdf5/nc4info.c index 866fe450d8..b04a39274d 100644 --- a/libhdf5/nc4info.c +++ b/libhdf5/nc4info.c @@ -8,158 +8,274 @@ */ #include "config.h" +#include "nc4internal.h" #include "hdf5internal.h" +#include "nclist.h" +#include "ncbytes.h" +/* Various Constants */ +#define NCPROPS_MAX_NAME 1024 /* max key name size */ +#define NCPROPS_MAX_VALUE 1024 /* max value size */ #define HDF5_MAX_NAME 1024 /**< HDF5 max name. */ +#define ESCAPECHARS "\\=|," + /** @internal Check NetCDF return code. */ #define NCHECK(expr) {if((expr)!=NC_NOERR) {goto done;}} /** @internal Check HDF5 return code. */ #define HCHECK(expr) {if((expr)<0) {ncstat = NC_EHDFERR; goto done;}} +static int globalpropinitialized = 0; struct NCPROPINFO globalpropinfo; /**< Global property info. */ +/* Forward */ +static int properties_parse(const char* text0, NClist* pairs); + /** - * @internal Initialize file info. + * @internal Initialize default provenance info + * This will only be used for newly created files + * or for opened files that do not contain an _NCProperties + * attribute. * * @return ::NC_NOERR No error. * @author Dennis Heimbigner */ int -NC4_fileinfo_init(void) +NC4_provenance_init(void) { int stat = NC_NOERR; + int i; + NClist* other = NULL; + char* name = NULL; + char* value = NULL; unsigned major,minor,release; - /* Build nc properties */ + if(globalpropinitialized) + return stat; + + /* Build _NCProperties info */ + + /* Initialize globalpropinfo */ memset((void*)&globalpropinfo,0,sizeof(globalpropinfo)); globalpropinfo.version = NCPROPS_VERSION; - globalpropinfo.netcdfver[0] = '\0'; - globalpropinfo.hdf5ver[0] = '\0'; + globalpropinfo.properties = nclistnew(); + if(globalpropinfo.properties == NULL) + {stat = NC_ENOMEM; goto done;} + /* Insert primary library version as first entry */ + if((value = strdup(PACKAGE_VERSION)) == NULL) + {stat = NC_ENOMEM; goto done;} + + if((name = strdup(NCPNCLIB2)) == NULL) + {stat = NC_ENOMEM; goto done;} + nclistpush(globalpropinfo.properties,name); + name = NULL; + nclistpush(globalpropinfo.properties,value); + value = NULL; + + /* Insert the HDF5 as underlying storage format library */ stat = NC4_hdf5get_libversion(&major,&minor,&release); if(stat) goto done; - snprintf(globalpropinfo.hdf5ver,sizeof(globalpropinfo.hdf5ver), - "%1u.%1u.%1u",major,minor,release); - strncpy(globalpropinfo.netcdfver,PACKAGE_VERSION,sizeof(globalpropinfo.netcdfver)); + { + char sversion[64]; + snprintf(sversion,sizeof(sversion),"%1u.%1u.%1u",major,minor,release); + if((value = strdup(sversion)) == NULL) + {stat = NC_ENOMEM; goto done;} + } + if((name = strdup(NCPHDF5LIB2)) == NULL) + {stat = NC_ENOMEM; goto done;} + nclistpush(globalpropinfo.properties,name); + name = NULL; + nclistpush(globalpropinfo.properties,value); + value = NULL; + + /* Add any extra fields */ + /*Parse them into an NClist */ + other = nclistnew(); + if(other == NULL) {stat = NC_ENOMEM; goto done;} +#ifdef NCPROPERTIES + stat = properties_parse(NCPROPERTIES_EXTRA,other); + if(stat) goto done; +#endif + /* merge into the properties list */ + for(i=0;iversion = 0; - ncprops->hdf5ver[0] = '\0'; - ncprops->netcdfver[0] = '\0'; + text = strdup(text0); + if(text == NULL) return NC_ENOMEM; - len = strlen(text); - if(len == 0) return NC_NOERR; - propdata = (char*)malloc(len+1); - if(propdata == NULL) return NC_ENOMEM; - memcpy(propdata,text,len+1); - propdata[len] = '\0'; /* guarantee */ + /* For back compatibility with version 1, translate '|' -> ',' */ + for(p=text;*p;p++) { + if(*p == NCPROPSSEP1) + *p = NCPROPSSEP2; + } /* Walk and fill in ncinfo */ - p = propdata; + p = text; while(*p) { - char* name = p; - char* value = NULL; - char* q = strchr(p,'='); - if(q == NULL) - {ret = NC_EINVAL; goto done;} - *q++ = '\0'; - value = p = q; - q = strchr(p,NCPROPSSEP); - if(q == NULL) q = (p+strlen(p)); else* q++ = '\0'; - p = q; - if(value != NULL) { - if(strcmp(name,NCPVERSION) == 0) { - int v = atoi(value); - if(v < 0) v = 0; - ncprops->version = v; - } else if(strcmp(name,NCPNCLIBVERSION) == 0) - strncpy(ncprops->netcdfver,value,sizeof(ncprops->netcdfver)-1); - else if(strcmp(name,NCPHDF5LIBVERSION) == 0) - strncpy(ncprops->hdf5ver,value,sizeof(ncprops->hdf5ver)-1); - /* else ignore */ - } + char* name = p; + char* value = NULL; + char* next = NULL; + + /* Delimit whole (key,value) pair */ + q = locate(p,NCPROPSSEP2); + if(*q != '\0') /* Never go beyond the final nul term */ + *q++ = '\0'; + next = q; + /* split key and value */ + q = locate(p,'='); + name = p; + *q++ = '\0'; + value = q; + /* Set up p for next iteration */ + p = next; + nclistpush(pairs,strdup(name)); + nclistpush(pairs,strdup(value)); } - /* Guarantee null term */ - ncprops->netcdfver[sizeof(ncprops->netcdfver)-1] = '\0'; - ncprops->hdf5ver[sizeof(ncprops->hdf5ver)-1] = '\0'; done: - if(propdata != NULL) free(propdata); + if(text) free(text); return ret; } + +/* Utility to transfer a string to a buffer with escaping */ +static void +escapify(NCbytes* buffer, const char* s) +{ + const char* p; + for(p=s;*p;p++) { + if(strchr(ESCAPECHARS,*p) != NULL) + ncbytesappend(buffer,'\\'); + ncbytesappend(buffer,*p); + } +} + +/* Utility to copy contents of the dfalt into an NCPROPINFO object */ +static int +propinfo_default(struct NCPROPINFO* dst, const struct NCPROPINFO* dfalt) +{ + int i; + if(dst->properties == NULL) { + dst->properties = nclistnew(); + if(dst->properties == NULL) return NC_ENOMEM; + } + dst->version = dfalt->version; + for(i=0;iproperties);i++) { + char* s = nclistget(dfalt->properties,i); + s = strdup(s); + if(s == NULL) return NC_ENOMEM; + nclistpush(dst->properties,s); + } + return NC_NOERR; +} + /** - * @internal Get properties attribure. + * @internal Build _NCProperties attribute value. * - * @param h5 Pointer to HDF5 file info struct. + * Convert a NCPROPINFO instance to a single string. + * + * @param info Properties info. + * @param propdatap Pointer that gets properties string. * * @return ::NC_NOERR No error. + * @return ::NC_EINVAL failed. * @author Dennis Heimbigner */ -static int -NC4_get_propattr(NC_FILE_INFO_T* h5) +int +NC4_buildpropinfo(struct NCPROPINFO* info, char** propdatap) { - int ncstat = NC_NOERR; - size_t size; - H5T_class_t t_class; - hid_t grp = -1; - hid_t attid = -1; - hid_t aspace = -1; - hid_t atype = -1; - hid_t ntype = -1; - char* text = NULL; + int stat = NC_NOERR; + int i; + NCbytes* buffer = NULL; + char sversion[64]; - /* Get root group */ - grp = h5->root_grp->hdf_grpid; /* get root group */ - /* Try to extract the NCPROPS attribute */ - if(H5Aexists(grp,NCPROPS) > 0) { /* Does exist */ - attid = H5Aopen_name(grp, NCPROPS); - aspace = H5Aget_space(attid); /* dimensions of attribute data */ - atype = H5Aget_type(attid); - /* Verify that atype and size */ - t_class = H5Tget_class(atype); - if(t_class != H5T_STRING) {ncstat = NC_EATTMETA; goto done;} - size = H5Tget_size(atype); - if(size == 0) goto done; - text = (char*)malloc(size+1); - if(text == NULL) - {ncstat = NC_ENOMEM; goto done;} - HCHECK((ntype = H5Tget_native_type(atype, H5T_DIR_ASCEND))); - HCHECK((H5Aread(attid, ntype, text))); - /* Make sure its null terminated */ - text[size] = '\0'; - /* Try to parse text */ - ncstat = NC4_properties_parse(&h5->fileinfo->propattr,text); + if(info == NULL || info->version == 0 || propdatap == NULL) + {stat = NC_EINVAL; goto done;} + + *propdatap = NULL; + + buffer = ncbytesnew(); + if(!buffer) {stat = NC_ENOMEM; goto done;} + + /* start with version */ + ncbytescat(buffer,NCPVERSION); + ncbytesappend(buffer,'='); + snprintf(sversion,sizeof(sversion),"%d",info->version); + ncbytescat(buffer,sversion); + + for(i=0;iproperties);i+=2) { + char* value, *name; + name = nclistget(info->properties,i); + if(name == NULL) continue; + value = nclistget(info->properties,i+1); + ncbytesappend(buffer,NCPROPSSEP2); /* terminate last entry */ + escapify(buffer,name); + ncbytesappend(buffer,'='); + escapify(buffer,value); } + /* Force null termination */ + ncbytesnull(buffer); + *propdatap = ncbytesextract(buffer); + done: - if(attid >= 0) HCHECK((H5Aclose(attid))); - if(aspace >= 0) HCHECK((H5Sclose(aspace))); - if(ntype >= 0) HCHECK((H5Tclose(ntype))); - if(atype >= 0) HCHECK((H5Tclose(atype))); - if(text != NULL) free(text); - return ncstat; + if(buffer != NULL) ncbytesfree(buffer); + return stat; } +#if 0 /** * @internal Write the properties attribute to file. * @@ -169,19 +285,15 @@ NC4_get_propattr(NC_FILE_INFO_T* h5) * @author Dennis Heimbigner */ int -NC4_put_propattr(NC_FILE_INFO_T* h5) +NC4_put_ncproperties(NC_FILE_INFO_T* file) { int ncstat = NC_NOERR; - hid_t grp = -1; - hid_t attid = -1; - hid_t aspace = -1; - hid_t atype = -1; char* text = NULL; /* Get root group */ grp = h5->root_grp->hdf_grpid; /* get root group */ /* See if the NCPROPS attribute exists */ - if(H5Aexists(grp,NCPROPS) == 0) { /* Does not exist */ + if(H5Aexists(grp,NCPROPS) <= 0) { /* Does not exist */ ncstat = NC4_buildpropinfo(&h5->fileinfo->propattr,&text); if(text == NULL || ncstat != NC_NOERR) { goto done; @@ -205,85 +317,315 @@ NC4_put_propattr(NC_FILE_INFO_T* h5) if(atype >= 0) HCHECK((H5Tclose(atype))); return ncstat; } +#endif /** * @internal * - * @param h5 Pointer to HDF5 file info struct. - * @param init Initialization. + * Construct the provenance information for a newly created file + * using dfalt as the default. + * Note that creation of the _NCProperties attribute is deferred + * to the sync_netcdf4_file function. + * + * @param file Pointer to file object. + * @param dfalt * * @return ::NC_NOERR No error. + * [Note: other errors are reported via LOG()] * @author Dennis Heimbigner */ int -NC4_get_fileinfo(NC_FILE_INFO_T* h5, struct NCPROPINFO* init) +NC4_set_provenance(NC_FILE_INFO_T* file, const struct NCPROPINFO* dfalt) { int ncstat = NC_NOERR; + struct NCPROVENANCE* provenance = NULL; + int superblock = -1; + + assert(file->provenance == NULL); + provenance = calloc(1,sizeof(struct NCPROVENANCE)); + if(provenance == NULL) {ncstat = NC_ENOMEM; goto done;} - /* Allocate the fileinfo in h5 */ - h5->fileinfo = (struct NCFILEINFO*)calloc(1,sizeof(struct NCFILEINFO)); - if(h5->fileinfo == NULL) + /* Initialize from the default */ + provenance->propattr.version = globalpropinfo.version; + provenance->propattr.properties = nclistnew(); + if(provenance->propattr.properties == NULL) {ncstat = NC_ENOMEM; goto done;} + + /* Get the superblock number */ + if((ncstat = NC4_hdf5get_superblock(file,&superblock))) + goto done; + provenance->superblockversion = superblock; - /* Get superblock version */ - NCHECK((ncstat = NC4_hdf5get_superblock(h5,&h5->fileinfo->superblockversion))); - /* Get properties attribute if not already defined */ - if(init == NULL) { - NCHECK((ncstat = NC4_get_propattr(h5))); - } else { /*init != NULL*/ - h5->fileinfo->propattr = *init; /* Initialize */ + /* add in the dfalt values */ + if(dfalt != NULL) { + int i; + for(i=0;iproperties);i++) { + nclistpush(provenance->propattr.properties,nclistget(dfalt->properties,i)); + } } + done: + if(ncstat) { + LOG((0,"Could not create _NCProperties attribute")); + (void)NC4_free_provenance(provenance); + } else + file->provenance = provenance; + return NC_NOERR; +} + +/** + * @internal + * + * Construct the provenance information for a newly opened file + * Using the specified _NCProperties value. If NULL, then + * initialize using dfalt. + * + * @param file Pointer to file object. + * @param propstring The contents of _NCProperties + * @param dfalt + * + * @return ::NC_NOERR No error. + * @return ::NC_ENOMEM + * @return ::NC_EINVAL + * @author Dennis Heimbigner + */ +int +NC4_get_provenance(NC_FILE_INFO_T* file, const char* propstring, const struct NCPROPINFO* dfalt) +{ + int ncstat = NC_NOERR; + struct NCPROVENANCE* provenance; + char *name = NULL; + char *value = NULL; + int v = 0; + int superblock = -1; + + assert(file->provenance == NULL); + if((file->provenance = calloc(1,sizeof(struct NCPROVENANCE))) == NULL) + {ncstat = NC_ENOMEM; goto done;} + provenance = file->provenance; + if((provenance->propattr.properties = nclistnew()) == NULL) + {ncstat = NC_ENOMEM; goto done;} + + /* Set the superblock */ + if((ncstat = NC4_hdf5get_superblock(file,&superblock))) + goto done; + provenance->superblockversion = superblock; + + if(propstring == NULL) { + /* Use dfalt */ + if((ncstat=propinfo_default(&provenance->propattr,dfalt))) + goto done; + } else { + NClist* list = provenance->propattr.properties; + if((ncstat=properties_parse(propstring,list))) + goto done; + /* Check the version and remove from properties list*/ + if(nclistlength(list) < 2) + {ncstat = NC_EINVAL; goto done;} /* bad _NCProperties attribute */ + /* Extract the purported version=... */ + name = nclistremove(list,0); + value = nclistremove(list,0); + if(strcmp(name,NCPVERSION) == 0) { + if(sscanf(value,"%d",&v) != 1) + {ncstat = NC_EINVAL; goto done;} /* illegal version */ + if(v <= 0 || v > NCPROPS_VERSION) + {ncstat = NC_EINVAL; goto done;} /* unknown version */ + provenance->propattr.version = v; + } else + {ncstat = NC_EINVAL; goto done;} /* bad _NCProperties attribute */ +#if 0 + /* Now, rebuild from version 1 to version 2 if necessary */ + if(provenance->propattr.version == 1) { + int i; + for(i=0;ipropattr.properties) + nclistfreeall(prov->propattr.properties); + free(prov); + return NC_NOERR; +} - if(info == NULL || info->version == 0) return NC_EINVAL; - if(propdatap == NULL) - return NC_NOERR; - *propdatap = NULL; +/* HDF5 Specific attribute read/write of _NCProperties */ +int +NC4_read_ncproperties(NC_FILE_INFO_T* h5) +{ + int retval = NC_NOERR; + hid_t hdf5grpid = -1; + hid_t attid = -1; + hid_t aspace = -1; + hid_t atype = -1; + hid_t ntype = -1; + char* text = NULL; + H5T_class_t t_class; + hsize_t size; + + hdf5grpid = h5->root_grp->hdf_grpid; - /* compute attribute length */ - total = 0; - total += strlen(NCPVERSION); - total += strlen("=00000000"); - if(strlen(info->netcdfver) > 0) { - total += 1; /*|NCPROPSEP|*/ - total += strlen(NCPNCLIBVERSION); - total += strlen("="); - total += strlen(info->netcdfver); + if(H5Aexists(hdf5grpid,NCPROPS) <= 0) { /* Does not exist */ + /* File did not contain a _NCProperties attribute */ + retval=NC4_get_provenance(h5,NULL,&globalpropinfo); + goto done; } - if(strlen(info->hdf5ver) > 0) { - total += 1; /*|NCPROPSEP|*/ - total += strlen(NCPHDF5LIBVERSION); - total += strlen("="); - total += strlen(info->hdf5ver); + + /* NCPROPS Attribute exists, make sure it is legitimate */ + attid = H5Aopen_name(hdf5grpid, NCPROPS); + assert(attid > 0); + aspace = H5Aget_space(attid); + atype = H5Aget_type(attid); + /* Verify atype and size */ + t_class = H5Tget_class(atype); + if(t_class != H5T_STRING) + {retval = NC_EINVAL; goto done;} + size = H5Tget_size(atype); + if(size == 0) + {retval = NC_EINVAL; goto done;} + text = (char*)malloc(1+(size_t)size); + if(text == NULL) + {retval = NC_ENOMEM; goto done;} + if((ntype = H5Tget_native_type(atype, H5T_DIR_DEFAULT)) < 0) + {retval = NC_EHDFERR; goto done;} + if((H5Aread(attid, ntype, text)) < 0) + {retval = NC_EHDFERR; goto done;} + /* Make sure its null terminated */ + text[(size_t)size] = '\0'; + /* Process the _NCProperties value */ + if((retval = NC4_get_provenance(h5, text, &globalpropinfo))) + goto done; + +done: + if(text != NULL) free(text); + /* Close out the HDF5 objects */ + if(attid > 0 && H5Aclose(attid) < 0) retval = NC_EHDFERR; + if(aspace > 0 && H5Sclose(aspace) < 0) retval = NC_EHDFERR; + if(atype > 0 && H5Tclose(atype) < 0) retval = NC_EHDFERR; + if(ntype > 0 && H5Tclose(ntype) < 0) retval = NC_EHDFERR; + + /* For certain errors, actually fail, else log that attribute was invalid and ignore */ + if(retval != NC_ENOMEM && retval != NC_EHDFERR) { + LOG((0,"Invalid _NCProperties attribute")); + retval = NC_NOERR; } - propdata = (char*)malloc(total+1); - if(propdata == NULL) - return NC_ENOMEM; - snprintf(propdata,total+1, - "%s=%d|%s=%s|%s=%s", - NCPVERSION,info->version, - NCPNCLIBVERSION,info->netcdfver, - NCPHDF5LIBVERSION,info->hdf5ver); - /* Force null termination */ - propdata[total] = '\0'; - *propdatap = propdata; + return retval; +} - return NC_NOERR; +int +NC4_write_ncproperties(NC_FILE_INFO_T* h5) +{ + int retval = NC_NOERR; + hid_t hdf5grpid = -1; + hid_t attid = -1; + hid_t aspace = -1; + hid_t atype = -1; + char* text = NULL; + size_t len = 0; + + /* If the file is read-only, return an error. */ + if (h5->no_write) + {retval = NC_EPERM; goto done;} + + hdf5grpid = h5->root_grp->hdf_grpid; + + if(H5Aexists(hdf5grpid,NCPROPS) > 0) /* Already exists, no overwrite */ + goto done; + + /* Build the attribute string */ + if((retval = NC4_buildpropinfo(&h5->provenance->propattr,&text))) + goto done; + + /* Build the HDF5 string type */ + if ((atype = H5Tcopy(H5T_C_S1)) < 0) + {retval = NC_EHDFERR; goto done;} + if (H5Tset_strpad(atype, H5T_STR_NULLTERM) < 0) + {retval = NC_EHDFERR; goto done;} + if(H5Tset_cset(atype, H5T_CSET_ASCII) < 0) + {retval = NC_EHDFERR; goto done;} + + /* Create NCPROPS attribute */ + + len = strlen(text); + if(H5Tset_size(atype, len) < 0) + {retval = NC_EFILEMETA; goto done;} + if((aspace = H5Screate(H5S_SCALAR)) < 0) + {retval = NC_EFILEMETA; goto done;} + if ((attid = H5Acreate(hdf5grpid, NCPROPS, atype, aspace, H5P_DEFAULT)) < 0) + {retval = NC_EFILEMETA; goto done;} + if (H5Awrite(attid, atype, text) < 0) + {retval = NC_EFILEMETA; goto done;} + +done: + if(text != NULL) free(text); + /* Close out the HDF5 objects */ + if(attid > 0 && H5Aclose(attid) < 0) retval = NC_EHDFERR; + if(aspace > 0 && H5Sclose(aspace) < 0) retval = NC_EHDFERR; + if(atype > 0 && H5Tclose(atype) < 0) retval = NC_EHDFERR; + + /* For certain errors, actually fail, else log that attribute was invalid and ignore */ + switch (retval) { + case NC_ENOMEM: + case NC_EHDFERR: + case NC_EPERM: + case NC_EFILEMETA: + case NC_NOERR: + break; + default: + LOG((0,"Invalid _NCProperties attribute")); + retval = NC_NOERR; + break; + } + return retval; } + +/* Debugging */ + +void +ncprintpropinfo(struct NCPROPINFO* info) +{ + int i; + fprintf(stderr,"[%p] version=%d\n",info,info->version); + for(i=0;iproperties);i+=2) { + char* name = nclistget(info->properties,i); + char* value = nclistget(info->properties,i+1); + fprintf(stderr,"\t[%d] name=|%s| value=|%s|\n",i,name,value); + } +} + +void +ncprintprovenance(struct NCPROVENANCE* prov) +{ + fprintf(stderr,"[%p] superblockversion=%d\n",prov,prov->superblockversion); + ncprintpropinfo(&prov->propattr); +} + diff --git a/liblib/nc_initialize.c b/liblib/nc_initialize.c index abcf65258f..28f1ca55bd 100644 --- a/liblib/nc_initialize.c +++ b/liblib/nc_initialize.c @@ -94,7 +94,7 @@ nc_initialize() #endif #ifdef USE_NETCDF4 if((stat = NC4_initialize())) goto done; - stat = NC4_fileinfo_init(); + stat = NC4_provenance_init(); #endif /* USE_NETCDF4 */ done: diff --git a/libsrc4/nc4attr.c b/libsrc4/nc4attr.c index b348b3eb3c..abf8aba5c9 100644 --- a/libsrc4/nc4attr.c +++ b/libsrc4/nc4attr.c @@ -52,13 +52,13 @@ nc4_get_att_special(NC_FILE_INFO_T* h5, const char* name, char* propdata = NULL; int stat = NC_NOERR; int len; - if(h5->fileinfo->propattr.version == 0) + if(h5->provenance->propattr.version == 0) return NC_ENOTATT; if(mem_type == NC_NAT) mem_type = NC_CHAR; if(mem_type != NC_CHAR) return NC_ECHAR; if(filetypep) *filetypep = NC_CHAR; - stat = NC4_buildpropinfo(&h5->fileinfo->propattr, &propdata); + stat = NC4_buildpropinfo(&h5->provenance->propattr, &propdata); if(stat != NC_NOERR) return stat; len = strlen(propdata); if(lenp) *lenp = len; @@ -70,7 +70,7 @@ nc4_get_att_special(NC_FILE_INFO_T* h5, const char* name, if(filetypep) *filetypep = NC_INT; if(lenp) *lenp = 1; if(strcmp(name,SUPERBLOCKATT)==0) - iv = (unsigned long long)h5->fileinfo->superblockversion; + iv = (unsigned long long)h5->provenance->superblockversion; else /* strcmp(name,ISNETCDF4ATT)==0 */ iv = NC4_isnetcdf4(h5); if(mem_type == NC_NAT) mem_type = NC_INT; diff --git a/nc_test/Make0 b/nc_test/Make0 index 04c1b940b5..3c20cd94d5 100644 --- a/nc_test/Make0 +++ b/nc_test/Make0 @@ -1,5 +1,5 @@ # Test c output -T=tst_open_mem +T=nc_test #H58=8 H510=10 diff --git a/ncdap_test/testurl.sh b/ncdap_test/testurl.sh index 95ba9eb30f..b851e8c79a 100755 --- a/ncdap_test/testurl.sh +++ b/ncdap_test/testurl.sh @@ -56,7 +56,7 @@ locreset if test "x$NOP" != x1 ; then echo "***Testing url prefix parameters" -buildurl $PREFIX "" +buildurl "$PREFIX" "" # Invoke ncdump to extract the URL echo "command: ${NCDUMP} -h $url" @@ -66,7 +66,7 @@ if test "x${SHOW}" = x1 ; then cat ./tmp ; fi # Test that maxstrlen works as alias for stringlength echo "***Testing maxstrlen=stringlength alias" -buildurl $STRLEN "" +buildurl "$STRLEN" "" # Invoke ncdump to extract the URL echo "command: ${NCDUMP} -h $url" ${NCDUMP} "$url" >./tmp_testurl 2> ./errtmp_testurl @@ -82,7 +82,7 @@ fi locreset if test "x$NOS" != x1 ; then echo "***Testing url suffix parameters" -buildurl "" $SUFFIX +buildurl "" "$SUFFIX" # Invoke ncdump to extract the URL ${NCDUMP} -h "$url" >./tmp_testurl 2> ./errtmp_testurl if test "x${SHOW}" = x1 ; then cat ./tmp_testurl ; fi @@ -92,7 +92,7 @@ locreset if test "x$NOB" != x1 ; then echo "***Testing url prefix+suffix parameters" -buildurl $BOTHP $BOTHS +buildurl "$BOTHP" "$BOTHS" # Invoke ncdump to extract the URL ${NCDUMP} -h "$url" >./tmp_testurl 2> ./errtmp_testurl if test "x${SHOW}" = x1 ; then cat ./tmp_testurl ; fi diff --git a/ncdump/Make0 b/ncdump/Make0 index 1d02f46ce8..c18b783306 100644 --- a/ncdump/Make0 +++ b/ncdump/Make0 @@ -1,12 +1,12 @@ # Test c output -T=tst_chunking +T=tst_fileinfo #ARGS=./x.nc #TF=test_atomic_array.cdl #CMD=valgrind --leak-check=full -#CMD=gdb --args +CMD=gdb --args #HDF4=1 #PAR=1 diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 1df11844aa..4bf673d9e0 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -7,6 +7,7 @@ #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT = export SETX=1; # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am @@ -130,7 +131,7 @@ tst_hdf5_offset.sh run_ncgen_nc4_tests.sh tst_nccopy3_subset.sh \ ref_nccopy3_subset.nc ref_test_corrupt_magic.nc tst_ncgen_shared.sh \ tst_ncgen4.sh tst_ncgen4_classic.sh tst_ncgen4_diff.sh \ tst_ncgen4_cycle.sh tst_null_byte_padding.sh \ -ref_null_byte_padding_test.nc ref_tst_irish_rover.nc +ref_null_byte_padding_test.nc ref_tst_irish_rover.nc ref_provenance_v1.nc # The L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. diff --git a/ncdump/nccopy.1 b/ncdump/nccopy.1 index 1eda799713..e80dbbd5c3 100644 --- a/ncdump/nccopy.1 +++ b/ncdump/nccopy.1 @@ -21,6 +21,8 @@ nccopy \%[\-e \fI cache_elems \fP] \%[\-r] \%[\-F \fI filterspec \fP] +\%[\-L \fI n \fP] +\%[\-M \fI n \fP] \%\fI infile \fP \%\fI outfile \fP .hy @@ -252,6 +254,10 @@ Read netCDF classic or 64-bit offset input file into a diskless netCDF file in memory before copying. Requires that input file be small enough to fit into memory. For \fBnccopy\fP, this doesn't seem to provide any significant speedup, so may not be a useful option. +.IP "\fB \-L \fP \fIn\fP" +Set the log level; only usable if nccopy supports netCDF-4 (enhanced). +.IP "\fB \-M \fP \fIn\fP" +Set the minimum chunk size; only usable if nccopy supports netCDF-4 (enhanced). .IP "\fB \-F \fP \fIfilterspec\fP" For netCDF-4 output, including netCDF-4 classic model, specify a filter to apply to an specified variable in the output. As a rule, the filter diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index ec9e0d9d3f..db3716127d 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -1978,13 +1978,14 @@ usage(void) [-r] read whole input file into diskless file on open (classic or 64-bit offset or cdf5 formats only)\n\ [-F filterspec] specify the compression algorithm to apply to an output variable.\n\ [-Ln] set log level to n (>= 0); ignored if logging isn't enabled.\n\ + [-Mn] set minimum chunk size to n bytes (n >= 0)\n\ infile name of netCDF input file\n\ outfile name for netCDF output file\n" /* Don't document this flaky option until it works better */ /* [-x] use experimental computed estimates for variable-specific chunk caches\n\ */ - error("%s [-k kind] [-[3|4|6|7]] [-d n] [-s] [-c chunkspec] [-u] [-w] [-[v|V] varlist] [-[g|G] grplist] [-m n] [-h n] [-e n] [-r] infile outfile\n%s\nnetCDF library version %s", + error("%s [-k kind] [-[3|4|6|7]] [-d n] [-s] [-c chunkspec] [-u] [-w] [-[v|V] varlist] [-[g|G] grplist] [-m n] [-h n] [-e n] [-r] [-F filterspec] [-Ln] [-Mn] infile outfile\n%s\nnetCDF library version %s", progname, USAGE, nc_inq_libvers()); } @@ -2160,7 +2161,7 @@ main(int argc, char**argv) case 'M': /* set min chunk size */ #ifdef USE_NETCDF4 if(optarg == NULL) - option_min_chunk_bytes = -1; + option_min_chunk_bytes = 0; else option_min_chunk_bytes = atol(optarg); if(option_min_chunk_bytes < 0) diff --git a/ncdump/ref_provenance_v1.nc b/ncdump/ref_provenance_v1.nc new file mode 100644 index 0000000000000000000000000000000000000000..2ccec37935928e8dbe43761992fd1fab75efa677 GIT binary patch literal 1469 zcmcf>O-~b1aNcfP)@C6UV}aavt4@PK*GV8 z7)ksQCLBC^G@iKn8@zz=Pl#{ky;0bt2aqsL-|T#UyqWi|>OC9JPG#r|yCj!~grOe|g0T0bu+#J7_0Gmtu*R&SQ*s_Y==ei_E9`guUcB?V zv+?MZLcuA_I|ay5V@if{h(QP_UD(?XokXduzzHXDvQ#m;R1bX#xCS;%LC}lat8pB$ z|C|NXD}Y&I387L_C)z2vj%Yy>|AF$wkN2BywXXW{Mc-(RJ?c4;q%G4Hmy-6kJDdB| z-Ej~?miw>1Dz=H5qRRR*K(rXtCWHP~Kz6@}aVy1sCAKUrO7g=2BO@buowufI*5B4>JXScUQVo2+6tb?jvw zn_Fy-p7eRj12VMK?qa#sk_=pCTs~bGc(tWUqg87*+AX)dSOGhaG689_YLu6#ShT_f z*eVXXJ^*cj ./tmp_tst_fileinfo ; then +rm -f ./tst_fileinfo.tmp +if $NCDUMP -s $NF | fgrep '_IsNetcdf4 = 0' > ./tst_fileinfo.tmp ; then echo "Pass: False negative for file: $NF" else echo "FAIL: False negative for file: $NF" + EXIT=1 fi -rm -f ./tmp_tst_fileinfo -if ${execdir}/tst_fileinfo > /dev/null ; then +rm -f ./tst_fileinfo.tmp +if test -e $NCF ; then # look at the _IsNetcdf4 flag N_IS=`${NCDUMP} -s $NCF | fgrep '_IsNetcdf4' | tr -d ' ;'` N_IS=`echo $N_IS | cut -d= -f2` @@ -30,16 +36,31 @@ if ${execdir}/tst_fileinfo > /dev/null ; then H_IS=`echo $H_IS | cut -d= -f2` if test "x$N_IS" = 'x0' ;then echo "FAIL: $NCF is marked as not netcdf-4" + EXIT=1 fi if test "x$H_IS" = 'x1' ;then echo "FAIL: $HDF is marked as netcdf-4" + EXIT=1 fi else -echo "FAIL: tst_fileinfo" -EXIT=1 + echo "FAIL: tst_fileinfo: $NCF does not exist" + EXIT=1 +fi + +# Test what happens when we read a file that used provenance version 1 +rm -f ./tst_fileinfo.tmp ./tst_fileinfo2.tmp +$NCDUMP -hs $NPV1 >tst_fileinfo2.tmp +fgrep '_NCProperties' ./tst_fileinfo.tmp +if ! fgrep 'version=1' tst_fileinfo.tmp ; then + echo "FAIL: $NPV1 is not marked as version=1" + EXIT=1 fi rm -f $NCF rm -f $HDF +rm -f *.tmp +if test "x$EXIT" = x0 ; then +echo "*** Pass all tests" +fi exit $EXIT diff --git a/ncdump/tst_nccopy3.sh b/ncdump/tst_nccopy3.sh index 449e896865..c657868970 100755 --- a/ncdump/tst_nccopy3.sh +++ b/ncdump/tst_nccopy3.sh @@ -34,6 +34,7 @@ fi echo "*** Testing netCDF-3 features of nccopy on ncdump/*.nc files" for i in $TESTFILES ; do echo "*** Testing nccopy $i.nc nccopy3_copy_of_$i.nc ..." +ls -l $i.nc ${NCCOPY} $i.nc nccopy3_copy_of_$i.nc ${NCDUMP} -n nccopy3_copy_of_$i $i.nc > tmp_tst_nccopy3.cdl ${NCDUMP} nccopy3_copy_of_$i.nc > nccopy3_copy_of_$i.cdl diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index 80bc6ac3e5..9b914a25bb 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -23,7 +23,7 @@ ${execdir}/tst_compress echo "*** Testing netCDF-4 features of nccopy on ncdump/*.nc files" for i in $TESTFILES ; do echo "*** Test nccopy $i.nc copy_of_$i.nc ..." - $NCCOPY $i.nc copy_of_$i.nc + ${NCCOPY} $i.nc copy_of_$i.nc ${NCDUMP} -n copy_of_$i $i.nc > tmp.cdl ${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl # echo "*** compare " with copy_of_$i.cdl @@ -33,27 +33,27 @@ done # echo "*** Testing compression of deflatable files ..." ./tst_compress echo "*** Test nccopy -d1 can compress a classic format file ..." -$NCCOPY -d1 tst_inflated.nc tst_deflated.nc +${NCCOPY} -d1 tst_inflated.nc tst_deflated.nc if test `wc -c < tst_deflated.nc` -ge `wc -c < tst_inflated.nc`; then exit 1 fi echo "*** Test nccopy -d1 can compress a netCDF-4 format file ..." -$NCCOPY -d1 tst_inflated4.nc tst_deflated.nc +${NCCOPY} -d1 tst_inflated4.nc tst_deflated.nc if test `wc -c < tst_deflated.nc` -ge `wc -c < tst_inflated4.nc`; then exit 1 fi echo "*** Test nccopy -d1 -s can compress a classic model netCDF-4 file even more ..." -$NCCOPY -d1 -s tst_inflated.nc tmp.nc +${NCCOPY} -d1 -s tst_inflated.nc tmp.nc if test `wc -c < tmp.nc` -ge `wc -c < tst_inflated.nc`; then exit 1 fi echo "*** Test nccopy -d1 -s can compress a netCDF-4 file even more ..." -$NCCOPY -d1 -s tst_inflated4.nc tmp.nc +${NCCOPY} -d1 -s tst_inflated4.nc tmp.nc if test `wc -c < tmp.nc` -ge `wc -c < tst_inflated4.nc`; then exit 1 fi echo "*** Test nccopy -d0 turns off compression, shuffling of compressed, shuffled file ..." -$NCCOPY -d0 tst_inflated4.nc tmp.nc +${NCCOPY} -d0 tst_inflated4.nc tmp.nc ${NCDUMP} -sh tmp.nc > tmp.cdl if fgrep '_DeflateLevel' < tmp.cdl ; then exit 1 @@ -66,7 +66,7 @@ rm tst_deflated.nc tst_inflated.nc tst_inflated4.nc tmp.nc tmp.cdl echo "*** Testing nccopy -d1 -s on ncdump/*.nc files" for i in $TESTFILES ; do echo "*** Test nccopy -d1 -s $i.nc copy_of_$i.nc ..." - $NCCOPY -d1 -s $i.nc copy_of_$i.nc + ${NCCOPY} -d1 -s $i.nc copy_of_$i.nc ${NCDUMP} -n copy_of_$i $i.nc > tmp.cdl ${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl # echo "*** compare " with copy_of_$i.cdl @@ -75,20 +75,20 @@ ${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl done ./tst_chunking echo "*** Test that nccopy -c can chunk and unchunk files" -$NCCOPY tst_chunking.nc tmp.nc +${NCCOPY} -M0 tst_chunking.nc tmp.nc ${NCDUMP} tmp.nc > tmp.cdl -$NCCOPY -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ tst_chunking.nc tmp-chunked.nc +${NCCOPY} -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ tst_chunking.nc tmp-chunked.nc ${NCDUMP} -n tmp tmp-chunked.nc > tmp-chunked.cdl diff tmp.cdl tmp-chunked.cdl -$NCCOPY -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ tmp-chunked.nc tmp-unchunked.nc +${NCCOPY} -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ tmp-chunked.nc tmp-unchunked.nc ${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl diff tmp.cdl tmp-unchunked.cdl -$NCCOPY -c // tmp-chunked.nc tmp-unchunked.nc +${NCCOPY} -c // tmp-chunked.nc tmp-unchunked.nc ${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl diff tmp.cdl tmp-unchunked.cdl echo "*** Test that nccopy -c works as intended for record dimension default (1)" ${NCGEN} -b -o tst_bug321.nc $srcdir/tst_bug321.cdl -$NCCOPY -k nc7 -c"lat/2,lon/2" tst_bug321.nc tmp.nc +${NCCOPY} -k nc7 -c"lat/2,lon/2" tst_bug321.nc tmp.nc ${NCDUMP} -n tst_bug321 tmp.nc > tmp.cdl diff -b $srcdir/tst_bug321.cdl tmp.cdl # echo "*** Test that nccopy compression with chunking can improve compression" diff --git a/ncdump/tst_netcdf4.sh b/ncdump/tst_netcdf4.sh index 15c41721d3..76f6e490f8 100755 --- a/ncdump/tst_netcdf4.sh +++ b/ncdump/tst_netcdf4.sh @@ -7,6 +7,18 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +# Remove the version information from _NCProperties +cleanncprops() { + src="$1" + dst="$2" + rm -f $dst + cat $src \ + | sed -e 's/_SuperblockVersion = 1/_SuperblockVersion = 0/' \ + | sed -e 's/\(netcdflibversion\|netcdf\)=.*|/\1=NNNN|/' \ + | sed -e 's/\(hdf5libversion\|hdf5\)=.*"/\1=HHHH"/' \ + | cat >$dst +} + ERR() { RES=$? if [ $RES -ne 0 ]; then @@ -84,14 +96,11 @@ fi echo "*** Running tst_special_atts.c to create test files." ${execdir}/tst_special_atts ; ERR -${NCDUMP} -c -s tst_special_atts.nc \ - | sed 's/e+0/e+/g' \ - | sed -e 's/netcdflibversion=.*[|]/netcdflibversion=0.0.0|/' \ - | sed -e 's/hdf5libversion=.*"/hdf5libversion=0.0.0"/' \ - | sed -e 's|_SuperblockVersion = [0-9]|_SuperblockVersion = 0|' \ - | cat > tst_special_atts.cdl ; ERR +${NCDUMP} -c -s tst_special_atts.nc > tst_special_atts.cdl ; ERR +cleanncprops tst_special_atts.cdl tst_special_atts.tmp +cleanncprops $srcdir/ref_tst_special_atts.cdl ref_tst_special_atts.tmp echo "*** comparing tst_special_atts.cdl with ref_tst_special_atts.cdl..." -diff -b tst_special_atts.cdl $srcdir/ref_tst_special_atts.cdl ; ERR +diff -b tst_special_atts.tmp ref_tst_special_atts.tmp ; ERR #echo "" #echo "*** Testing ncdump on file with corrupted header " @@ -107,5 +116,6 @@ diff -b tst_special_atts.cdl $srcdir/ref_tst_special_atts.cdl ; ERR #echo "*** creating tst_output_irish_rover.cdl from ref_tst_irish_rover.nc..." #${NCDUMP} ref_tst_irish_rover.nc > tst_output_irish_rover.cdl +rm -f *.tmp echo "*** All ncgen and ncdump test output for netCDF-4 format passed!" exit 0 diff --git a/ncdump/tst_netcdf4_4.sh b/ncdump/tst_netcdf4_4.sh index 458cc1bd63..24a76962d2 100755 --- a/ncdump/tst_netcdf4_4.sh +++ b/ncdump/tst_netcdf4_4.sh @@ -7,6 +7,18 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +# Remove the version information from _NCProperties +cleanncprops() { + src="$1" + dst="$2" + rm -f $dst + cat $src \ + | sed -e 's/_SuperblockVersion = 1/_SuperblockVersion = 0/' \ + | sed -e 's/\(netcdflibversion\|netcdf\)=.*|/\1=NNNN|/' \ + | sed -e 's/\(hdf5libversion\|hdf5\)=.*"/\1=HHHH"/' \ + | cat >$dst +} + echo "" echo "*** Running extra netcdf-4 tests." @@ -30,8 +42,10 @@ ${execdir}/tst_string_data echo "*** dumping tst_string_data.nc to tst_string_data.cdl..." ${NCDUMP} tst_string_data.nc > tst_string_data.cdl +cleanncprops tst_string_data.cdl tst_string_data.tmp +cleanncprops ${srcdir}/ref_tst_string_data.cdl ref_tst_string_data.tmp echo "*** comparing tst_string_data.cdl with ref_tst_string_data.cdl..." -diff -b tst_string_data.cdl ${top_srcdir}/ncdump/ref_tst_string_data.cdl +diff -b tst_string_data.tmp ref_tst_string_data.tmp #echo '*** testing non-coordinate variable of same name as dimension...' #${NCGEN} -v4 -b -o tst_noncoord.nc ${top_srcdir}/ncdump/ref_tst_noncoord.cdl @@ -50,17 +64,19 @@ diff -b tst_compounds4.cdl ${top_srcdir}/ncdump/ref_tst_compounds4.cdl # Exercise Jira NCF-213 bug fix # rm -f tst_ncf213.cdl tst_ncf213.nc -${NCGEN} -b -o tst_ncf213.nc ${top_srcdir}/ncdump/ref_tst_ncf213.cdl -${NCDUMP} -s -h tst_ncf213.nc \ - | sed -e 's/netcdflibversion=.*[|]/netcdflibversion=0.0.0|/' \ - | sed -e 's/hdf5libversion=.*"/hdf5libversion=0.0.0"/' \ - | sed -e 's|_SuperblockVersion = [0-9]|_SuperblockVersion = 0|' \ - | cat >tst_ncf213.cdl +# Remove specific _NCProperties values +${NCGEN} -b -o tst_ncf213.nc $srcdir/ref_tst_ncf213.cdl +${NCDUMP} -s -h tst_ncf213.nc > tst_ncf213.cdl +cleanncprops tst_ncf213.cdl tst_ncf213.tmp +cleanncprops ${srcdir}/ref_tst_ncf213.cdl ref_tst_ncf213.tmp # Now compare ok=1; -if diff -b ${top_srcdir}/ncdump/ref_tst_ncf213.cdl tst_ncf213.cdl ; then ok=1; else ok=0; fi +if diff -b tst_ncf213.tmp ref_tst_ncf213.tmp ; then ok=1; else ok=0; fi + # cleanup -# rm -f tst_ncf213.cdl tst_ncf213.nc +#rm -f tst_ncf213.cdl tst_ncf213.nc +rm -f *.tmp + if test $ok = 0 ; then echo "*** FAIL: NCF-213 Bug Fix test" exit 1 diff --git a/ncgen/main.c b/ncgen/main.c index 8af6909cd3..90ebbaf17e 100644 --- a/ncgen/main.c +++ b/ncgen/main.c @@ -212,6 +212,7 @@ usage(void) " [-x]" " [-N datasetname]" " [-L loglevel]" +" [-H]" " [file ... ]", progname); derror("netcdf library version %s", nc_inq_libvers()); @@ -293,10 +294,10 @@ main( } l_flag = L_BINARY; break; - case 'h': + case 'H': header_only = 1; break; - case 'H': + case 'h': usage(); goto done; case 'l': /* specify language, instead of using -c or -f or -b */ From 6efa25e7e40a9ece2d6de428aabb5c549051ab7e Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Thu, 6 Sep 2018 15:53:25 -0600 Subject: [PATCH 2/2] Updated and removed a check for _NCProperties during file comparison, as this test fails if the environment is different from the environment the reference file was built on. --- ncdump/tst_netcdf4.sh | 1 + ncdump/tst_netcdf4_4.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ncdump/tst_netcdf4.sh b/ncdump/tst_netcdf4.sh index 76f6e490f8..a98e19bb79 100755 --- a/ncdump/tst_netcdf4.sh +++ b/ncdump/tst_netcdf4.sh @@ -16,6 +16,7 @@ cleanncprops() { | sed -e 's/_SuperblockVersion = 1/_SuperblockVersion = 0/' \ | sed -e 's/\(netcdflibversion\|netcdf\)=.*|/\1=NNNN|/' \ | sed -e 's/\(hdf5libversion\|hdf5\)=.*"/\1=HHHH"/' \ + | grep -v '_NCProperties' \ | cat >$dst } diff --git a/ncdump/tst_netcdf4_4.sh b/ncdump/tst_netcdf4_4.sh index 24a76962d2..44d2c01156 100755 --- a/ncdump/tst_netcdf4_4.sh +++ b/ncdump/tst_netcdf4_4.sh @@ -2,7 +2,7 @@ # This shell script runs extra tests ncdump for netcdf-4 # Dennis Heimbigner, Ward Fisher -if test "x$srcdir" = x ; then srcdir=`pwd`; fi +if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh set -e @@ -16,6 +16,7 @@ cleanncprops() { | sed -e 's/_SuperblockVersion = 1/_SuperblockVersion = 0/' \ | sed -e 's/\(netcdflibversion\|netcdf\)=.*|/\1=NNNN|/' \ | sed -e 's/\(hdf5libversion\|hdf5\)=.*"/\1=HHHH"/' \ + | grep -v '_NCProperties' \ | cat >$dst } @@ -29,7 +30,7 @@ echo "*** Running extra netcdf-4 tests." # 'ncdump ./ref_tst_compounds2'. This causes the test to fail. # But, 'srcdir' is necessary for make distcheck. # -# Short term solution, use sed when on windows/MSYS to +# Short term solution, use sed when on windows/MSYS to # remove the './','../../ncdump'. # # I am undoing this because libdispatch/dwinpath.c @@ -84,4 +85,3 @@ fi echo "*** All ncgen and ncdump extra test output for netCDF-4 format passed!" exit 0 -