diff --git a/CMakeLists.txt b/CMakeLists.txt index 5418757220..a00ad5d4d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -315,6 +315,8 @@ SET(CHUNK_CACHE_PREEMPTION 0.75 CACHE STRING "Default file chunk cache preemptio SET(MAX_DEFAULT_CACHE_SIZE 67108864 CACHE STRING "Default maximum cache size.") SET(NETCDF_LIB_NAME "" CACHE STRING "Default name of the netcdf library.") SET(TEMP_LARGE "." CACHE STRING "Where to put large temp files if large file tests are run.") +SET(NCPROPERTIES_EXTRA "" CACHE STRNG "Specify extra pairs for _NCProperties.") +SET(MPIEXEC "mpiexec" CACHE STRING "Command to run MPI programs if parallel tests are run.") IF(NOT NETCDF_LIB_NAME STREQUAL "") SET(MOD_NETCDF_NAME ON) @@ -735,6 +737,7 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4) ENDIF() CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Pset_libver_bounds "" HDF5_HAS_LIBVER_BOUNDS) + CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5free_memory "" HDF5_HAS_H5FREE) IF(HDF5_PARALLEL) SET(HDF5_CC h5pcc) @@ -986,7 +989,7 @@ IF(NOT ENABLE_NETCDF_4 AND ENABLE_EXAMPLE_TESTS) SET(ENABLE_EXAMPLE_TESTS OFF) ENDIF() -# Enable Parallel (different than pnetcdf). +# Enable Parallel IO with netCDF-4/HDF5 files using HDF5 parallel I/O. SET(STATUS_PARALLEL "OFF") OPTION(ENABLE_PARALLEL4 "Build netCDF-4 with parallel IO" "${HDF5_PARALLEL}") IF(ENABLE_PARALLEL4 AND ENABLE_NETCDF_4) @@ -998,10 +1001,16 @@ IF(ENABLE_PARALLEL4 AND ENABLE_NETCDF_4) SET(USE_PARALLEL ON CACHE BOOL "") SET(USE_PARALLEL4 ON CACHE BOOL "") SET(STATUS_PARALLEL "ON") + configure_file("${netCDF_SOURCE_DIR}/nc_test4/run_par_test.sh.in" + "${netCDF_BINARY_DIR}/tmp/run_par_test.sh" @ONLY NEWLINE_STYLE LF) + FILE(COPY "${netCDF_BINARY_DIR}/tmp/run_par_test.sh" + DESTINATION ${netCDF_BINARY_DIR}/nc_test4 + FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + ENDIF() ENDIF() -# Options to enable parallel IO, tests. +# Options to enable parallel IO for classic formats with parallel-netcdf library. SET(STATUS_PNETCDF "OFF") OPTION(ENABLE_PNETCDF "Build with parallel I/O for classic and 64-bit offset files using parallel-netcdf." OFF) @@ -1984,6 +1993,12 @@ IF(ENABLE_EXAMPLES) configure_file(${CMAKE_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF) ENDIF() +##### +# Build ncdap_test|dap4_test/findtestserver[4].c +##### +configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF) +configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF) + #### # Export files #### diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e6058f8671..95f10a35da 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.7.0 - TBD - +* [Enhancement] Create a new version of _NCProperties provenance attribute. This version (version 2) supports arbitrary key-value pairs. It is the default when new files are created. Version 1 continues to be accepted. * [Enhancement] Allow user to set http read buffersize for DAP2 and DAP4 using the tag HTTP.READ.BUFFERSIZE in the .daprc file. * [Enhancement] Allow user to set http keepalive for DAP2 and DAP4 using the tag HTTP.KEEPALIVE in the .daprc file (see the OPeNDAP documentation for details). * [Enhancement] Support DAP4 remote tests using a new remote test server locatedon the Unidata JetStream project. diff --git a/cf b/cf index 0b24d06cb5..28edfb5207 100644 --- a/cf +++ b/cf @@ -3,7 +3,7 @@ DB=1 #X=-x -#FAST=1 +FAST=1 #PROF=1 HDF5=1 @@ -108,10 +108,10 @@ FLAGS="$FLAGS --enable-extreme-numbers" #FLAGS="$FLAGS --disable-testsets" #FLAGS="$FLAGS --disable-dap-remote-tests" #FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server -#FLAGS="$FLAGS --enable-doxygen" -#FLAGS="$FLAGS --enable-internal-docs" +FLAGS="$FLAGS --enable-doxygen" +FLAGS="$FLAGS --enable-internal-docs" FLAGS="$FLAGS --enable-logging" -FLAGS="$FLAGS --disable-diskless" +#FLAGS="$FLAGS --disable-diskless" #FLAGS="$FLAGS --enable-mmap" #FLAGS="$FLAGS --with-udunits" #FLAGS="$FLAGS --with-libcf" @@ -120,6 +120,7 @@ FLAGS="$FLAGS --disable-diskless" #FLAGS="$FLAGS --disable-silent-rules" #FLAGS="$FLAGS --disable-filter-testing" #FLAGS="$FLAGS --enable-metadata-perf" +#FLAGS="$FLAGS --with-ncproperties-extra=key1=value1,key2=value2" if test "x$TESTSERVERS" != x ; then FLAGS="$FLAGS --with-testservers=$TESTSERVERS" @@ -184,9 +185,6 @@ CFLAGS="${CFLAGS} -pg" LDFLAGS="${LDFLAGS} -pg" fi - -#FLAGS="${FLAGS} --enable-stdio" - export PATH export CC export CPPFLAGS @@ -199,17 +197,13 @@ export CXXFLAGS DISTCHECK_CONFIGURE_FLAGS="$FLAGS" export DISTCHECK_CONFIGURE_FLAGS -if test "x$NB" != x -o "x$FAST" = x ; then +if test "x$NB" != x1 && test "x$FAST" != x1 ; then ${MAKE} distclean >/dev/null 2>&1 fi -if test -z "$NB" ; then +if test "x$NB" != x1 ; then if autoreconf -i --force ; then ok=1; else exit ; fi fi -if test -z "$FAST" ; then - if test -f Makefile ; then ${MAKE} distclean >/dev/null 2>&1 ; fi -fi - sh $X ./configure ${FLAGS} for c in $cmds; do printenv LD_LIBRARY_PATH diff --git a/cf.cmake b/cf.cmake index 1dd4ad581e..a2b31fc13c 100644 --- a/cf.cmake +++ b/cf.cmake @@ -6,6 +6,8 @@ DAP=1 #CDF5=1 #HDF4=1 +export SETX=1 + for arg in "$@" ; do case "$arg" in vs|VS) VS=1 ;; @@ -60,7 +62,7 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true" FLAGS="$FLAGS -DENABLE_LOGGING=true" #FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true" #FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true" -FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" +#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" # Disables FLAGS="$FLAGS -DENABLE_EXAMPLES=false" @@ -68,6 +70,9 @@ FLAGS="$FLAGS -DENABLE_CONVERSION_WARNINGS=false" #FLAGS="$FLAGS -DENABLE_TESTS=false" #FLAGS="$FLAGS -DENABLE_DISKLESS=false" +# Withs +FLAGS="$FLAGS -DNCPROPERTIES_EXTRA=\"key1=value1|key2=value2\"" + rm -fr build mkdir build cd build diff --git a/config.h.cmake.in b/config.h.cmake.in index dd3c77f7f3..9c247a9c32 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -381,6 +381,9 @@ are set when opening a binary file on Windows. */ /* min blocksize for posixio. */ #cmakedefine NCIO_MINBLOCKSIZE ${NCIO_MINBLOCKSIZE} +/* Add extra properties to _NCProperties attribute */ +#cmakedefine NCPROPERTIES_EXTRA ${NCPROPERTIES_EXTRA} + /* no IEEE float on this platform */ #cmakedefine NO_IEEE_FLOAT 1 diff --git a/configure.ac b/configure.ac index e9c0a7fac5..6d9b804123 100644 --- a/configure.ac +++ b/configure.ac @@ -1,16 +1,13 @@ # -*- Autoconf -*- ## Process this file with autoconf to produce a configure script. -# This is part of Unidata's netCDF package. Copyright 2005-2012, see +# This is part of Unidata's netCDF package. Copyright 2005-2018, see # the COPYRIGHT file for more information. # Ed Hartnett, Ward Fisher, Dennis Heimbigner # Recall that ${VAR-exp} expands to $VAR if var is set (even to null), # and to exp otherwise. -## This puts the cvs ID tag in the output configure script. -AC_REVISION([$Id: configure.ac,v 1.450 2010/05/28 19:42:47 dmh Exp $]) - # Running autoconf on this file will trigger a warning if # autoconf is not at least the specified version. AC_PREREQ([2.59]) @@ -172,8 +169,12 @@ AC_DEFINE([USE_FSYNC], [1], [if true, include experimental fsync code]) fi # Temporary until JNA bug is fixed (which is probably never). -# See Jira NCF-298 -AC_MSG_CHECKING([if jna bug workaround is enabledd]) +# The problem being solved is this: +# > On Windows using the microsoft runtime, it is an error +# > for one library to free memory allocated by a different library. +# This is probably only an issue when using the netcdf-c library +# via JNA under Java. +AC_MSG_CHECKING([if jna bug workaround is enabled]) AC_ARG_ENABLE([jna], [AS_HELP_STRING([--enable-jna], [enable jna bug workaround])], @@ -255,6 +256,15 @@ AC_ARG_ENABLE([parallel-tests], test "x$enable_parallel_tests" = xyes || enable_parallel_tests=no AC_MSG_RESULT($enable_parallel_tests) +# Did the user specify an MPI launcher other than mpiexec? +AC_MSG_CHECKING([whether a user specified program to run mpi programs]) +AC_ARG_WITH([mpiexec], + [AS_HELP_STRING([--with-mpiexec=], + [Specify command to launch MPI parallel tests.])], + [MPIEXEC=$with_mpiexec], [MPIEXEC=mpiexec]) +AC_MSG_RESULT([$MPIEXEC]) +AC_SUBST([MPIEXEC], [$MPIEXEC]) + # Did the user specify a default chunk size? AC_MSG_CHECKING([whether a default chunk size in bytes was specified]) AC_ARG_WITH([default-chunk-size], @@ -592,6 +602,18 @@ AC_MSG_RESULT($TEMP_LARGE) #AC_SUBST(TEMP_LARGE) AC_DEFINE_UNQUOTED([TEMP_LARGE], ["$TEMP_LARGE"], [Place to put very large netCDF test files.]) +# Specify extra values to add to _NCProperties attribute +# --with-ncproperties-extra="=|...". +# Note: need to figure out a way to do this programmatically also +AC_MSG_CHECKING([Extra values for _NCProperties]) +AC_ARG_WITH([ncproperties-extra], + [AS_HELP_STRING([--with-ncproperties-extra="=,...], + [specify extra pairs for _NCProperties])], + [NCPROPERTIES_EXTRA=$with_ncproperties_extra], + [NCPROPERTIES_EXTRA=""]) +AC_MSG_RESULT([$NCPROPERTIES_EXTRA]) +AC_DEFINE_UNQUOTED([NCPROPERTIES_EXTRA], ["$NCPROPERTIES_EXTRA"], [Extra pairs for _NCProperties]) + # Did the user specify a user-defined format 0? AC_MSG_CHECKING([whether user-defined format 0 was specified]) AC_ARG_WITH([udf0], @@ -1505,13 +1527,18 @@ AC_SUBST([MSVC], []) AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in) +# DAP 2/4 findtestserver[4].c setup +AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in) +AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in) + ##### # End netcdf_meta.h definitions. ##### AC_MSG_NOTICE([generating header files and makefiles]) +AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh]) +AC_CONFIG_FILES([nc-config], [chmod 755 nc-config]) AC_CONFIG_FILES([Makefile - nc-config netcdf.pc libnetcdf.settings postinstall.sh @@ -1548,11 +1575,7 @@ AC_CONFIG_FILES([Makefile ncdap_test/expectremote3/Makefile dap4_test/Makefile plugins/Makefile - ], - [test -f nc-config && chmod 755 nc-config]) - + ]) AC_OUTPUT() - #mv -f ${abs_top_srcdir}/test_common.sh ${abs_top_builddir}/test_common.sh - cat libnetcdf.settings diff --git a/dap4_test/Makefile.am b/dap4_test/Makefile.am index 3718b9d543..6c61be8908 100644 --- a/dap4_test/Makefile.am +++ b/dap4_test/Makefile.am @@ -8,8 +8,11 @@ include $(top_srcdir)/lib_flags.am #TEST_EXTENSIONS = .sh +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose - +#TESTS_ENVIRONMENT = export SETX=1; + LDADD = ${top_builddir}/liblib/libnetcdf.la AM_CPPFLAGS += -I$(top_srcdir)/libdap4 @@ -62,6 +65,8 @@ CLEANFILES = *.exe # This should only be left behind if using parallel io CLEANFILES += tmp_* +DISTCLEANFILES = findtestserver4.c + # One last thing BUILT_SOURCES = .daprc diff --git a/dap4_test/d4test_common.sh b/dap4_test/d4test_common.sh index 5be59e4472..f9f7405ce7 100755 --- a/dap4_test/d4test_common.sh +++ b/dap4_test/d4test_common.sh @@ -24,8 +24,8 @@ cd ${srcdir}/baselineremote; BASELINEREM=`pwd` ; cd ${WD} BASELINEH=${BASELINEREM} setresultdir() { -rm -fr ./$1 -mkdir -p ./$1 +rm -fr ${builddir}/$1 +mkdir -p ${builddir}/$1 } FAILURES= diff --git a/dap4_test/test_raw.sh b/dap4_test/test_raw.sh index f294e937c4..ef46a1a6d3 100755 --- a/dap4_test/test_raw.sh +++ b/dap4_test/test_raw.sh @@ -42,29 +42,29 @@ resultclean() { } setresultdir results_test_raw + if test "x${RESET}" = x1 ; then rm -fr ${BASELINERAW}/*.dmp ; fi for f in $F ; do echo "testing: $f" - URL="[dap4]file://${DAPTESTFILES}/${f}" - if ! ${VG} ${NCDUMP} "${URL}" > ./results_test_raw/${f}.dmp; then + URL="[log][dap4]file://${DAPTESTFILES}/${f}" + if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_raw/${f}.dmp; then failure "${URL}" fi if test "x${TEST}" = x1 ; then - if ! diff -wBb ${BASELINERAW}/${f}.dmp ./results_test_raw/${f}.dmp ; then + if ! diff -wBb ${BASELINERAW}/${f}.dmp ${builddir}/results_test_raw/${f}.dmp ; then failure "diff ${f}.dmp" fi elif test "x${RESET}" = x1 ; then echo "${f}:" - cp ./results_test_raw/${f}.dmp ${BASELINERAW}/${f}.dmp + cp ${builddir}/results_test_raw/${f}.dmp ${BASELINERAW}/${f}.dmp elif test "x${DIFF}" = x1 ; then echo "hdrtest: ${f}" - rm -f ./tr1 ./tr2 ./tb1 ./tb2 baseclean - if ! diff -wBb ./${BASELINERAW}/${f}.dmp ./${BASELINE}/${f}.ncdump ; then - failure diff -wBb ./${BASELINERAW}/${f}.dmp ./${BASELINE}/${f}.ncdump + if ! diff -wBb ${BASELINERAW}/${f}.dmp ${BASELINE}/${f}.ncdump ; then + failure diff -wBb ${BASELINERAW}/${f}.dmp ${BASELINE}/${f}.ncdump fi fi done -rm -rf ./results_test_raw +rm -rf ${builddir}/results_test_raw finish diff --git a/dap4_test/test_remote.sh b/dap4_test/test_remote.sh index 5e9b6809a6..5582ab237e 100755 --- a/dap4_test/test_remote.sh +++ b/dap4_test/test_remote.sh @@ -13,47 +13,47 @@ echo "test_remote.sh:" #NOCSUM=1 F="\ -test_atomic_array.nc -test_atomic_types.nc -test_enum.nc -test_enum_2.nc -test_enum_array.nc -test_fill.nc -test_groups1.nc -test_misc1.nc -test_one_var.nc -test_one_vararray.nc -test_opaque.nc -test_opaque_array.nc -test_struct1.nc -test_struct_array.nc -test_struct_nested.nc -test_struct_nested3.nc -test_struct_type.nc -test_utf8.nc -test_vlen1.nc -test_vlen2.nc -test_vlen3.nc -test_vlen4.nc -test_vlen5.nc -test_vlen6.nc -test_vlen7.nc -test_vlen8.nc -test_vlen9.nc -test_vlen10.nc -test_vlen11.nc -tst_fills.nc -test_struct_nested.hdf5 -test_struct_nested3.hdf5 -test_vlen3.hdf5 -test_vlen4.hdf5 -test_vlen5.hdf5 -test_anon_dim.syn -test_atomic_array.syn -test_atomic_types.syn -test_sequence_1.syn -test_sequence_2.syn -test_struct_array.syn +test_atomic_array.nc \ +test_atomic_types.nc \ +test_enum.nc \ +test_enum_2.nc \ +test_enum_array.nc \ +test_fill.nc \ +test_groups1.nc \ +test_misc1.nc \ +test_one_var.nc \ +test_one_vararray.nc \ +test_opaque.nc \ +test_opaque_array.nc \ +test_struct1.nc \ +test_struct_array.nc \ +test_struct_nested.nc \ +test_struct_nested3.nc \ +test_struct_type.nc \ +test_utf8.nc \ +test_vlen1.nc \ +test_vlen2.nc \ +test_vlen3.nc \ +test_vlen4.nc \ +test_vlen5.nc \ +test_vlen6.nc \ +test_vlen7.nc \ +test_vlen8.nc \ +test_vlen9.nc \ +test_vlen10.nc \ +test_vlen11.nc \ +tst_fills.nc \ +test_struct_nested.hdf5 \ +test_struct_nested3.hdf5 \ +test_vlen3.hdf5 \ +test_vlen4.hdf5 \ +test_vlen5.hdf5 \ +test_anon_dim.syn \ +test_atomic_array.syn \ +test_atomic_types.syn \ +test_sequence_1.syn \ +test_sequence_2.syn \ +test_struct_array.syn \ " setresultdir results_test_remote @@ -66,27 +66,27 @@ fi if test "x${RESET}" = x1 ; then rm -fr ${BASELINER}/*.dmp ; fi for f in $F ; do - URL="[log][dap4]${TESTSERVER}/d4ts/testfiles/${f}" + URL="[log][show=fetch][dap4]${TESTSERVER}/testfiles/${f}" if test "x$BIG" = x1; then URL="[ucar.littleendian=0]${URL}" fi if test "x$NOCSUM" = x1; then URL="[ucar.checksummode=none]${URL}" fi - if ! ${VG} ${NCDUMP} "${URL}" > ./results_test_remote/${f}.dmp; then + if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_remote/${f}.dmp; then failure "${URL}" fi if test "x${TEST}" = x1 ; then - if ! diff -wBb ${BASELINEREM}/${f}.dmp ./results_test_remote/${f}.dmp ; then + if ! diff -wBb "${BASELINEREM}/${f}.dmp" "${builddir}/results_test_remote/${f}.dmp" ; then failure "diff ${f}.dmp" fi elif test "x${RESET}" = x1 ; then echo "${f}:" - cp ./results_test_remote/${f}.dmp ${BASELINEREM}/${f}.dmp + cp "${builddir}/results_test_remote/${f}.dmp" "${BASELINEREM}/${f}.dmp" fi done -rm -fr ./results_test_remote +rm -fr ${builddir}/results_test_remote finish diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index e335c78d04..020a28cd78 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -768,6 +768,7 @@ INPUT = \ @abs_top_srcdir@/docs/bestpractices.md \ @abs_top_srcdir@/docs/tutorial.dox \ @abs_top_srcdir@/docs/internal.dox \ + @abs_top_srcdir@/docs/inmeminternal.dox \ @abs_top_srcdir@/docs/indexing.dox \ @abs_top_srcdir@/docs/testserver.dox \ @abs_top_srcdir@/include/netcdf.h \ diff --git a/docs/Makefile.am b/docs/Makefile.am index 5c644435a6..43016cd366 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -11,7 +11,7 @@ install-fortran.md all-error-codes.md credits.md auth.md \ obsolete/fan_utils.html bestpractices.md filters.md indexing.dox \ inmemory.md DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md \ file_format_specifications.md known_problems.md COPYRIGHT.dox \ -user_defined_formats.md +user_defined_formats.md inmeminternal.dox # Turn off parallel builds in this directory. .NOTPARALLEL: diff --git a/docs/attribute_conventions.md b/docs/attribute_conventions.md index 17046fe665..884f89f3ce 100644 --- a/docs/attribute_conventions.md +++ b/docs/attribute_conventions.md @@ -161,18 +161,24 @@ Using the following API calls will fail. `_NCProperties` -> This attribute is persistent in the file, but hidden. It is inserted in the file at creation time and is never modified after that point. It specifies the following. -> - The version for the netcdf library used at creation time. -> - The version for the HDF5 library used at creation time. -> - The type of this attribute is NC_CHAR. - -> Its format is: `name=value|name=value ...`
-> Occurrences of '|' in the name or value are disallowed. +> This attribute is persistent in the file, but hidden. It is inserted in the file at creation time and is never modified after that point. The type of this attribute is currently NC_CHAR. There two versions of this property, but both have the general form +>> version=n,key=value,key=value...,key=value +> where the version number n is either 1 or 2. -> The current set of known names is as follows. -> - version=... The current format version for the _NCProperties file, currently 1. -> - netcdflibversion=... The version of the netcdf library used to create the file. The value is, for example, 4.4.1-rc1-development or 4.4.1. -> - hdf5libversion=... The version of the HDF5 library used to create the file. The value is, for example, 1.8.16 or 1.10.0. +> Version 1 has two (key,value) pairs (after than the initial version pair) +> - netcdfversion={netcdfversion} where the version number is the version for the netcdf library used at creation time. +> - hdf5version={hdf5fversion} where the version number is the version for the hdf5 library used at creation time. +> - The version for the HDF5 library used at creation time. +> - Note that for version 1, the separator character is '|' instead of ',' +> - Occurrences of '=' or '|' in the name or value are disallowed. + +> Version 2 has an unlimited set of (key,value) pairs (after the initial version pair). By convention (but unenforced), the first pair is the name and version of the primary library used to create this file. For netcdf, it has the form _netcdf={version}_. The remaining fields are by convention as follows. +> - If the primary build library is netcdf, then the second pair is of the form _hdf5={version}. The remaining pairs consist of a combination of the name and version of important supporting libraries -- the libcurl version, for example -- plus an arbitrary set of pairs as specified by the _--with-ncproperties_ option to the ./configure command. Note that the argument to --with-ncproperties should be wrapped with double quotes, like this. +>> _./configure ... --with-ncproperties="key1=value,key2=value"_ +> +> Note the following for version 2. +> - The pair separator character was changed from '|' to ',' because of problems with bash, which did not like '|' in the --with-ncproperties value. +> - It is possible to include escaped characters using the standard '\' escape convention. `_SuperblockVersion` diff --git a/docs/inmeminternal.dox b/docs/inmeminternal.dox new file mode 100644 index 0000000000..ba22ed595a --- /dev/null +++ b/docs/inmeminternal.dox @@ -0,0 +1,401 @@ +/** +@if INTERNAL + +@page inmemintern Internal Architecture for NC_INMEMORY Support + +\tableofcontents + + + + +# Introduction {#inmemintern_intro} + +This document describes the internal workings +of the inmemory features of the netcdf-c library. +The companion document to this -- inmemory.md -- +describes the "external" operation of the inmemory features. + +This document describes how the in-memory operation +is implemented both for netcdf-3 files and for netcdf-4 files. + +# Generic Capabilities {#inmemintern_general} + +Both the netcdf-3 and netcdf-4 implementations assume that +they are initially given a (pointer,size) pair representing +a chunk of allocated memory of specified size. + +If a file is being created instead of opened, then only the size +is needed and the netcdf-c library will internally allocate the +corresponding memory chunk. + +If NC_DISKLESS is being used, then a chunk of memory is allocated +whose size is the same as the length of the file, and the contents +of the file is then read into that chunk of memory. + +This information is in general represented by the following struct +(see include/netcdf_mem.h). +```` +typedef struct NC_memio { + size_t size; + void* memory; + int flags; +} NC_memio; + +```` +The flags field describes properties and constraints to be applied +to the given memory. At the moment, only this one flag is defined. +```` +#define NC_MEMIO_LOCKED 1 +```` +If this flag is set, then the netcdf library will ensure that +the original allocated memory is ```locked```, which means +that it will never be realloc'd nor free'd. +Note that this flag is ignored when creating a memory file: it is only +relevant when opening a pre-allocated chunk of memory via the +_nc_open_mem_ function. + +Note that this flag does not prevent the memory from being modified. +If there is room, then the memory may be modified in place. If the size +of the memory needs to be increased and the this flag is set, then +the operation will fail. + +When the _nc_close_memio_ function is called instead of +_nc_close_, then the currently allocated memory (and its size) +is returned. If the _NC_MEMIO_LOCKED_ flag is set, then it +should be the case that the chunk of memory returned is the same +as originally provided. However, the size may be different +because it represents the amount of memory that contains +meaningful data; this value may be less than the original provided size. +The actual allocated size for the memory chunk is the same as originally +provided, so it that value is needed, then the caller must save it somewhere. + +Note also that ownership of the memory chunk is given to the +caller, and it is the caller's responsibility to _free_ the memory. + +# NetCDF-4 Implementation {#inmemintern_nc4} + +The implementation of in-memory support for netcdf-4 files +is quite complicated. + +The netCDF-4 implemention relies on the HDF5 library. In order +to implement in-memory storage of data, the HDF5 core driver is +used to manage underlying storage of the netcdf-c file. + +An HDF5 driver is an abstract interface that allows different +underlying storage implementations. So there is a standard file +driver as well as a core driver, which uses memory as the +underlying storage. + +Generically, the memory is referred to as a file image [1]. + +## libhdf5/nc4mem + +The primary API for in-memory operations is in the file +libhdf5/nc4mem.c and the defined functions are described in the next sections + +### nc4mem.NC4_open_image_file + +The signature is: +```` +int NC4_open_image_file(NC_FILE_INFO_T* h5) +```` +Basically, this function sets up the necessary state information +to use the HDF5 core driver. +It obtains the memory chunk and size from the _h5->mem.memio_ field. + +Specifically, this function converts the +_NC_MEMIO_LOCKED_ flag into using the HDF5 image specific flags: +_H5LT_FILE_IMAGE_DONT_COPY_ and _H5LT_FILE_IMAGE_DONT_RELEASE_. +It then invokes the function _libhdf5/nc4memcb/NC4_image_init_ +function to do the necessary HDF5 specific setup. + +### nc4mem.NC4_create_image_file + +The signature is: +```` +int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t initialsize) +```` + +This function sets up the necessary state information +to use the HDF5 core driver, but for a newly created file. +It initializes the memory chunk and size in the _h5->mem.memio_ field +from the _initialsize_ argument and it leaves the memory chunk pointer NULL. +It ignores the _NC_MEMIO_LOCKED_ flag. +It then invokes the function _libhdf5/nc4memcb/NC4_image_init_ +function to do the necessary HDF5 specific setup. + +### libhdf5/hdf5file.c/nc4_close-netcdf4_file + +When a file is closed, this function is invoked. As part of its operation, +and if the file is an in-memory file, it does one of two things. + +1. If the user provided an _NC_memio_ instance, then return the final image +in that instance; the user is then responsible for freeing it. +2. If no _NC_memio_ instance was provided, then just discard the final image. + +## libhdf5/nc4memcb + +The HDF5 core driver uses an abstract interface for managing the +allocation and free'ing of memory. This interface is defined +as a set of callback functions [2] that implement the functions +of this struct. + +```` +typedef struct { + void *(*_malloc)(size_t size, H5_file_image_op_t op, void *udata); + void *(*_memcpy)(void *dest, const void *src, size_t size, + H5_file_image_op_t op, void *udata); + void *(*_realloc)(void *ptr, size_t size, + H5_file_image_op_t op, void *udata); + herr_t (*_free)(void *ptr, H5_file_image_op_t op, void *udata); + void *(*udata_copy)(void *udata); + herr_t (*udata_free)(void *udata); + void *udata; +} H5_file_image_callbacks_t; +```` +The _udata_ field at the end defines any extra state needed by the functions. +Each function is passed the udata as its last argument. The structure of the +udata is arbitrary, and is passed as _void*_ to the functions. + +The _udata_ structure and callback functions used by the netcdf-c library +are defined in the file _libhdf5/nc4memcb.c_. Setup is defined by the +function _NC4_image_init_ in that same file. + +The _udata_ structure used by netcdf is as follows. +```` +typedef struct { + void *app_image_ptr; /* Pointer to application buffer */ + size_t app_image_size; /* Size of application buffer */ + void *fapl_image_ptr; /* Pointer to FAPL buffer */ + size_t fapl_image_size; /* Size of FAPL buffer */ + int fapl_ref_count; /* Reference counter for FAPL buffer */ + void *vfd_image_ptr; /* Pointer to VFD buffer (Note: VFD => used by core driver) */ + size_t vfd_image_size; /* Size of VFD buffer */ + int vfd_ref_count; /* Reference counter for VFD buffer */ + unsigned flags; /* Flags indicate how the file image will be opened */ + int ref_count; /* Reference counter on udata struct */ + NC_FILE_INFO_T* h5; /* Pointer to the netcdf parent structure */ +} H5LT_file_image_ud_t; +```` + +It is necessary to understand one more point about the callback functions. +The first four take an argument of type _H5_file_image_op_t_ -- the operator. +This is an enumeration that indicates additional context about the purpose for +which the callback is being invoked. For the purposes of the netcdf-4 +implementation, only the following operators are used. + +- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET +- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY +- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET +- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE +- H5FD_FILE_IMAGE_OP_FILE_OPEN +- H5FD_FILE_IMAGE_OP_FILE_RESIZE +- H5FD_FILE_IMAGE_OP_FILE_CLOSE + +As can be seen, basically the operators indicate if the operation is with respect to +an HDF5 property list, or with respect to a file (i.e. a core image in this case). +For each callback described below, the per-operator actions will be described. +Not all operators are used with all callbacks. + +Internally, the HDF5 core driver thinks it is doing the following: + +1. Allocate memory and copy the incoming memory chunk into that newly allocated memory + (call image_malloc followed by image_memcpy). +2. Periodically reallocate the memory to increase its size + (call image_realloc). +3. Free up the memory as no longer needed + (call image_free). + +It turns out that for propertly lists, realloc is never called. +However the HDF5 core driver follows all of the above steps. + +The following sections describe the callback function operation. + +### libhdf5/nc4memcb/local_image_malloc + +This function is called to allocated an internal chunk of memory so +the original provided memory is no longer needed. In order to implement +the netcdf-c semantics, we modify this behavior. + +#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET +We assume that the property list image info will never need to be modified, +so we just copy the incoming buffer info (the app_image fields) into the fapl_image fields. + +#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY +Basically just return the fapl_image_ptr field, so no actual copying. + +#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY and H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET +Basically just return the fapl_image_ptr field, so no actual copying or malloc needed. + +#### Operator H5FD_FILE_IMAGE_OP_FILE_OPEN +Since we always start by using the original incoming image buffer, we just +need to store that pointer and size into the vfd_image fields (remember, vfd is that +used by the core driver). + +### libhdf5/nc4memcb/local_image_memcpy +This function is supposed to be used to copy the incoming buffer into an internally +malloc'd buffer. Since we use the original buffer, no memcpy is actually needed. +As a safety check, we do actually do a memcpy if, for some reason, the _src_ and _dest_ +arguments are different. In practice, this never happens. + +### libhdf5/nc4memcb/local_image_realloc +Since the property list image is never realloc'd this is only called with +_H5FD_FILE_IMAGE_OP_FILE_RESIZE_. + +If the memory is not locked (i.e. the _NC_MEMIO_LOCKED_ flag was not used), +then we are free to realloc the vfd_ptr. But if the memory is locked, +then we cannot realloc and we must fake it as follows: + +1. If the chunk is big enough, then pretend to do a realloc by + changing the vfd_image_size. +2. If the chunk is not big enough to accomodate the requested new size, + then fail. + +There is one important complication. It turns out that the image_realloc +callback is sometimes called with a ptr argument value of NULL. This assumes +that if realloc is called with a NULL buffer pointer, then it acts like _malloc_. +Since we have found that some systems to do not implement this, we implement it +in our local_image_realloc code and do a _malloc_ instead of _realloc_. + +### libhdf5/nc4memcb/local_image_free + +This function is, of course, invoked to deallocate memory. +It is only invoked with the +H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE +and H5FD_FILE_IMAGE_OP_FILE_CLOSE +operators. + +#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE +For the way the netcdf library uses it, it should still be the case that +the fapl pointer is same as original incoming app_ptr, so we do not need +to do anything for this operator. + +#### Operator H5FD_FILE_IMAGE_OP_FILE_CLOSE +Since in our implementation, we maintain control of the memory, this case +will never free any memory, but may save a pointer to the current vfd memory +so it can be returned to the original caller, if they want it. +Specifically the vfd_image_ptr and vfd_image_size are always +copied to the _udata->h5->mem.memio_ field so they can +be referenced by higher level code. + +### libhdf5/nc4memcb/local_udata_copy +Our version of this function only manipulates the reference count. + +### libhdf5/nc4memcb/local_udata_free +Our version of this function only manipulates the reference count. + +# NetCDF-3 Implementation {#inmemintern_nc3} + +The netcdf-3 code -- in libsrc -- has its own, internal storage +management API as defined in the file _libsrc/ncio.h_. It implements +the API in the form of a set of function pointers as defined in the +structure _struct_ _ncio_. These function have the following signatures +and semantics. + +- int ncio_relfunc(ncio*, off_t offset, int rflags) -- + Indicate that you are done with the region which begins at offset. +- int ncio_getfunc(ncio*, off_t offset, size_t extent, int rflags, void **const vpp) -- + Request that the region (offset, extent) be made available through *vpp. +- int ncio_movefunc(ncio*, off_t to, off_t from, size_t nbytes, int rflags) -- + Like memmove(), safely move possibly overlapping data. +- int ncio_syncfunc(ncio*) -- + Write out any dirty buffers to disk and ensure that next read will get data from disk. +- int ncio_pad_lengthfunc(ncio*, off_t length) -- + Sync any changes to disk, then truncate or extend file so its size is length. +- int ncio_closefunc(ncio*, int doUnlink) + -- Write out any dirty buffers and ensure that next read will not get cached data. + Then sync any changes, and then close the open file. + +The _NC_INMEMORY_ semantics are implemented by creating an implementation of the above functions +specific for handling in-memory support. This is implemented in the file _libsrc/memio.c_. + +## Open/Create/Close + +Open and close related functions exist in _memio.c_ that are not specifically part of the API. +These functions are defined in the following sections. + +### memio_create +Signature: +```` +int memio_create(const char* path, int ioflags, size_t initialsz, off_t igeto, size_t igetsz, size_t* sizehintp, void* parameters /*ignored*/, ncio* *nciopp, void** const mempp) +```` +Create a new file. Invoke _memio_new_ to create the _ncio_ +instance. If it is intended that the resulting file be +persisted to the file system, then verify that writing such a +file is possible. Also create an initial in-memory buffer to +hold the file data. Otherwise act like e.g. _posixio_create_. + +### memio_open +Signature: +```` +int memio_open(const char* path, int ioflags, off_t igeto, size_t igetsz, size_t* sizehintp, void* parameters, ncio* *nciopp, void** const mempp) +```` +Open an existing file. Invoke _memio_new_ to create the _ncio_ +instance. If it is intended that the resulting file be +persisted to the file system, then verify that writing such a +file is possible. Also create an initial in-memory buffer to +hold the file data. Read the contents of the existing file into +the allocated memory. +Otherwise act like e.g. _posixio_open_. + +### memio_extract +Signature: +```` +int memio_extract(ncio* const nciop, size_t* sizep, void** memoryp) +```` +This function is called as part of the NC3_close function in the event that +the user wants the final in-memory chunk returned to them via _nc_close_mem_. +It captures the existing in-memory chunk and returns it. At this point, +memio will no longer have access to that memory. + +## API Semantics + +The semantic interaction of the above API and NC_INMEMORY are described in the following sections. + +### ncio_relfunc +Just unlock the in-memory chunk. + +### ncio_getfunc +First guarantee that the requested region exists, and if necessary, +realloc to make it exist. If realloc is needed, and the +file is locked, then fail. + +### ncio_movefunc +First guarantee that the requested destination region exists, and if necessary, +realloc to make it exist. If realloc is needed, and the +file is locked, then fail. + +### ncio_syncfunc +This is a no-op as far as memio is concerned. + +### ncio_pad_lengthfunc +This may realloc the allocated in-memory buffer to achieve padding +rounded up to the pagesize. + +### ncio_filesizefunc +This just returns the used size of the in-memory chunk. +Note that the allocated size might be larger. + +### ncio_closefunc +If the usere wants the contents persisted, then write out the used portion +of the in-memory chunk to the target file. +Then, if the in-memory chunk is not locked, or for some reason has +been modified, go ahead and free that memory. + +# References {#inmemintern_bib} + +1. https://support.hdfgroup.org/HDF5/doc1.8/Advanced/FileImageOperations/HDF5FileImageOperations.pdf +2. https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetFileImageCallbacks + +# Point of Contact {#inmemintern_poc} + +__Author__: Dennis Heimbigner
+__Email__: dmh at ucar dot edu
+__Initial Version__: 8/28/2018
+__Last Revised__: 8/28/2018 + + + +@endif + +*/ diff --git a/docs/inmemory.md b/docs/inmemory.md index b3bc88ba0a..db22b9ffc3 100644 --- a/docs/inmemory.md +++ b/docs/inmemory.md @@ -129,16 +129,19 @@ Note that it is still possible to modify the in-memory file if the NC_WRITE mode flag was set. However, failures can occur if an operation cannot complete because the memory needs to be expanded. 2. If the *NC_MEMIO_LOCKED* flag is not set, then -the netcdf library will take control of the incoming memory -and will feel free to reallocate the provided +the netcdf library will take control of the incoming memory. +This means that the user should not make any attempt to free +or even read the incoming memory block in this case. +The newcdf library is free to reallocate the incomming memory block to obtain a larger block when an attempt to modify the in-memory file requires more space. Note that implicit in this is that the old block -- the one originally provided -- may be free'd as a side effect of re-allocating the memory using the *realloc()* function. -If the caller invokes the *nc_close_memio()* function to retrieve the -final memory block, the returned block must always be freed -by the caller and that the original block should not be freed. +The caller may invoke the *nc_close_memio()* function to retrieve the +final memory block, which may not be the same as the originally block +provided by the caller. In any case, the returned block must always be freed +by the caller and the original block should not be freed. ### The **nc_create_mem** Function diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 83957d37cf..a4ad40f932 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -60,6 +60,15 @@ typedef struct NC_HDF5_FILE_INFO int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); + +/* Used by NC4_set_provenance */ +int nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, + size_t len, const void *data, nc_type mem_type, int force); + +/* In-memory functions */ +extern hid_t NC4_image_init(NC_FILE_INFO_T* h5); +extern void NC4_image_finalize(void*); + /* These functions are internal to the libhdf5 directory. */ int nc4_detect_preserve_dimids(NC_GRP_INFO_T *grp, nc_bool_t *bad_coord_orderp); int hdf5_set_log_level(); diff --git a/include/nc4internal.h b/include/nc4internal.h index 5bf4df5224..7046fb53cf 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -85,7 +85,7 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT; typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t; /*Forward*/ -struct NCFILEINFO; +struct NCPROVENANCE; struct NC_GRP_INFO; struct NC_TYPE_INFO; @@ -299,17 +299,17 @@ typedef struct NC_FILE_INFO NClist* alltypes; NClist* allgroups; /* including root group */ void *format_file_info; - struct NCFILEINFO* fileinfo; + struct NCPROVENANCE* provenance; struct NC4_Memio { - NC_memio memio; - int locked; /* do not copy and do not release */ + NC_memio memio; /* What we sent to image_init and what comes back*/ + int locked; /* do not copy and do not free */ int persist; /* Should file be persisted out on close? */ - int inmemory; - int diskless; - unsigned int flags; /* for H5LTopen_file_image */ - int fapl; - size_t initialsize; + int inmemory; /* NC_INMEMORY flag was set */ + int diskless; /* NC_DISKLESS flag was set => inmemory */ int created; /* 1 => create, 0 => open */ + unsigned int imageflags; /* for H5LTopen_file_image */ + size_t initialsize; + void* udata; /* extra memory allocated in NC4_image_init */ } mem; } NC_FILE_INFO_T; @@ -414,7 +414,7 @@ int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name); int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info, void *fill_value); /* Close the file. */ -int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem); +int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio*); /* HDF5 initialization */ extern int nc4_hdf5_initialized; @@ -441,8 +441,10 @@ typedef struct NC_reservedatt { #define DIMSCALEFLAG 1 /* Readonly global attributes; readable, but immutable thru the API */ #define READONLYFLAG 2 -/* Subset of readonly flags; readable by name only thru the API*/ +/* Subset of readonly flags; readable by name only thru the API */ #define NAMEONLYFLAG 4 +/* Subset of readonly flags; Value is actually in file */ +#define MATERIALIZEDFLAG 8 /* Binary searcher for reserved attributes */ extern const NC_reservedatt* NC_findreserved(const char* name); @@ -465,38 +467,73 @@ For netcdf4 files, capture state information about the following: 5. Per file: _NCProperties attribute */ -#define NCPROPS "_NCProperties" -#define NCPROPS_VERSION (1) -#define NCPROPSSEP '|' +/* Most of this needs to be moved to hdf5internal.h */ -/* Currently used properties */ +#define NCPROPS "_NCProperties" #define NCPVERSION "version" /* Of the properties format */ -#define NCPHDF5LIBVERSION "hdf5libversion" -#define NCPNCLIBVERSION "netcdflibversion" +#define NCPHDF5LIB1 "hdf5libversion" +#define NCPNCLIB1 "netcdflibversion" +#define NCPHDF5LIB2 "hdf5" +#define NCPNCLIB2 "netcdf" +#define NCPROPS_VERSION (2) +/* Version 2 changes this because '|' was causing bash problems */ +#define NCPROPSSEP1 '|' +#define NCPROPSSEP2 ',' + /* Other hidden attributes */ #define ISNETCDF4ATT "_IsNetcdf4" #define SUPERBLOCKATT "_SuperblockVersion" -struct NCFILEINFO { +struct NCPROVENANCE { int superblockversion; - /* Following is filled from NCPROPS attribute or from global version */ - struct NCPROPINFO { - int version; /* 0 => not defined */ - char hdf5ver[NC_MAX_NAME+1]; - char netcdfver[NC_MAX_NAME+1]; - } propattr; + struct NCPROPINFO { + int version; /* 0 => not defined */ + /* Following is filled from NCPROPS attribute or from global version */ + /* Version 1 format is: + "netcdflibversion=" + Version 2 format is: + "==...|=..." + */ + /* The _NCProperties values are stored as an arbitrary + set of (key,value) pairs */ + /* It is assumed that the first entry is the primary library + used to build the file, and it is followed by other libraries + used in the build, and finally an arbitrary list of other + (key,value) pairs. */ + NClist* properties; + } propattr; }; +/* Provenance Initialization */ extern struct NCPROPINFO globalpropinfo; -extern int NC4_fileinfo_init(void); /*libsrc4/ncinfo.c*/ -extern int NC4_get_fileinfo(struct NC_FILE_INFO* info, struct NCPROPINFO*); /*libsrc4/ncinfo.c*/ -extern int NC4_put_propattr(struct NC_FILE_INFO* info); /*libsrc4/ncinfo.c*/ -extern int NC4_buildpropinfo(struct NCPROPINFO* info,char** propdatap); +/* Initialize the fileinfo global state */ +extern int NC4_provenance_init(); + +/* Write the properties attribute to file. */ +extern int NC4_put_ncproperties(NC_FILE_INFO_T* file); + +/* Extract the provenance from a file, using dfalt as default */ +extern int NC4_get_provenance(NC_FILE_INFO_T* file, const char* propstring, const struct NCPROPINFO* dfalt); + +/* Set the provenance for a created file using dfalt as default */ +extern int NC4_set_provenance(NC_FILE_INFO_T* file, const struct NCPROPINFO* dfalt); + +/* Recover memory of an NCPROVENANCE object */ +extern int NC4_free_provenance(struct NCPROVENANCE* prov); extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/ extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/ extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/ +/* Convert a NCPROPINFO instance to a single string. */ +extern int NC4_buildpropinfo(struct NCPROPINFO* info, char** propdatap); + +/* Use HDF5 API to read the _NCProperties attribute */ +extern int NC4_read_ncproperties(NC_FILE_INFO_T*); + +/* Use HDF5 API to write the _NCProperties attribute */ +extern int NC4_write_ncproperties(NC_FILE_INFO_T*); + #endif /* _NC4INTERNAL_ */ diff --git a/include/ncdap.h b/include/ncdap.h index 50ccb4492b..e30d36395d 100644 --- a/include/ncdap.h +++ b/include/ncdap.h @@ -17,8 +17,8 @@ affect the operation of the system. */ typedef unsigned int NCFLAGS; -# define SETFLAG(controls,flag) ((controls.flags) |= (flag)) -# define CLRFLAG(controls,flag) ((controls.flags) &= ~(flag)) +# define SETFLAG(controls,flag) (((controls).flags) |= (flag)) +# define CLRFLAG(controls,flag) (((controls).flags) &= ~(flag)) # define FLAGSET(controls,flag) (((controls.flags) & (flag)) != 0) /* Defined flags */ diff --git a/include/nctestserver.h b/include/nctestserver.h index 6c603602c4..637b9ac93c 100644 --- a/include/nctestserver.h +++ b/include/nctestserver.h @@ -55,7 +55,7 @@ This indicates that the server is up and running. Return the complete url for the server plus the path. */ -char* +static char* nc_findtestserver(const char* path, int isdap4, const char* serverlist) { char** svc; diff --git a/include/ncwinpath.h b/include/ncwinpath.h index 28e9058a44..e62368d971 100644 --- a/include/ncwinpath.h +++ b/include/ncwinpath.h @@ -8,6 +8,9 @@ #include "config.h" #include #include +#ifdef HAVE_UNISTD_H +#include +#endif #include "ncexternl.h" #ifndef WINPATH @@ -19,6 +22,19 @@ #endif #endif +/* Define wrapper constants for use with NCaccess */ +#ifdef _MSC_VER +#define ACCESS_MODE_EXISTS 0 +#define ACCESS_MODE_R 4 +#define ACCESS_MODE_W 2 +#define ACCESS_MODE_RW 6 +#else +#define ACCESS_MODE_EXISTS (F_OK) +#define ACCESS_MODE_R (R_OK) +#define ACCESS_MODE_W (W_OK) +#define ACCESS_MODE_RW (R_OK|W_OK) +#endif + /* Path Converter */ EXTERNL char* NCpathcvt(const char* path); @@ -27,10 +43,18 @@ EXTERNL char* NCpathcvt(const char* path); EXTERNL FILE* NCfopen(const char* path, const char* flags); EXTERNL int NCopen3(const char* path, int flags, int mode); EXTERNL int NCopen2(const char* path, int flags); +EXTERNL int NCaccess(const char* path, int mode); +EXTERNL int NCremove(const char* path); #else /*!WINPATH*/ #define NCfopen(path,flags) fopen((path),(flags)) #define NCopen3(path,flags,mode) open((path),(flags),(mode)) #define NCopen2(path,flags) open((path),(flags)) +#define NCremove(path) remove(path) +#ifdef _MSC_VER +#define NCaccess(path,mode) _access(path,mode) +#else +#define NCaccess(path,mode) access(path,mode) +#endif #endif /*WINPATH*/ #endif /* _NCWINIO_H_ */ diff --git a/include/netcdf.h b/include/netcdf.h index dc59e538d3..bfc3d4f227 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -66,7 +66,7 @@ extern "C" { #define NC_FILL_BYTE ((signed char)-127) #define NC_FILL_CHAR ((char)0) #define NC_FILL_SHORT ((short)-32767) -#define NC_FILL_INT (-2147483647L) +#define NC_FILL_INT (-2147483647) #define NC_FILL_FLOAT (9.9692099683868690e+36f) /* near 15 * 2^119 */ #define NC_FILL_DOUBLE (9.9692099683868690e+36) #define NC_FILL_UBYTE (255) diff --git a/include/netcdf_mem.h b/include/netcdf_mem.h index c8079896ca..2592907b20 100644 --- a/include/netcdf_mem.h +++ b/include/netcdf_mem.h @@ -17,8 +17,8 @@ typedef struct NC_memio { size_t size; void* memory; -#define NC_MEMIO_LOCKED 1 /* Do not try to realloc or free provided memory */ int flags; +#define NC_MEMIO_LOCKED 1 /* Do not try to realloc or free provided memory */ } NC_memio; #if defined(__cplusplus) diff --git a/libdap2/test_vara.c b/libdap2/test_vara.c index c15cdbf657..a2c9e87151 100644 --- a/libdap2/test_vara.c +++ b/libdap2/test_vara.c @@ -2,9 +2,8 @@ #include #include #include -#include - -extern char* nc_findtestserver(const char* path, int isdap4); +#include "netcdf.h" +#include "nctestserver.h" #define DTSTEST "/dts/test.06" diff --git a/libdap4/d4file.c b/libdap4/d4file.c index c1d04a4604..4ca44aeb9a 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -306,7 +306,7 @@ freeInfo(NCD4INFO* d4info) for some reason, so we delete it ourselves. */ if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); +// unlink(d4info->substrate.filename); } } nullfree(d4info->substrate.filename); /* always reclaim */ diff --git a/libdap4/d4read.c b/libdap4/d4read.c index b3d87b5a10..434055faac 100644 --- a/libdap4/d4read.c +++ b/libdap4/d4read.c @@ -9,8 +9,24 @@ /*Forward*/ static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, long*); -static int readfile(const NCURI*, const char* suffix, NCbytes* packet); -static int readfiletofile(const NCURI*, const char* suffix, FILE* stream, d4size_t*); +static int readfile(NCD4INFO* state, const NCURI*, const char* suffix, NCbytes* packet); +static int readfiletofile(NCD4INFO* state, const NCURI*, const char* suffix, FILE* stream, d4size_t*); + +#ifdef HAVE_GETTIMEOFDAY +static struct timeval time0; +static struct timeval time1; + +static double +deltatime() +{ + double t0, t1; + t0 = ((double)time0.tv_sec); + t0 += ((double)time0.tv_usec) / 1000000.0; + t1 = ((double)time1.tv_sec); + t1 += ((double)time1.tv_usec) / 1000000.0; + return (t1 - t0); +} +#endif int NCD4_readDMR(NCD4INFO* state) @@ -38,7 +54,7 @@ NCD4_readDAP(NCD4INFO* state, int flags) NCURI* url = state->uri; int fileprotocol = (strcmp(url->protocol,"file")==0); if(fileprotocol) { - stat = readfiletofile(url, ".dap", state->data.ondiskfile, &state->data.datasize); + stat = readfiletofile(state, url, ".dap", state->data.ondiskfile, &state->data.datasize); } else { char* readurl = NULL; int flags = 0; @@ -51,15 +67,11 @@ NCD4_readDAP(NCD4INFO* state, int flags) readurl = ncuribuild(url,NULL,".dods",NCURISVC); if(readurl == NULL) return THROW(NC_ENOMEM); - if (state->debug > 0) - {fprintf(stderr, "fetch url=%s\n", readurl);fflush(stderr);} stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, &state->data.datasize, &lastmod); nullfree(readurl); if(stat == NC_NOERR) state->data.daplastmodified = lastmod; - if (state->debug > 0) - {fprintf(stderr,"fetch complete\n"); fflush(stderr);} } } return THROW(stat); @@ -89,7 +101,7 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, long* las if(fileprotocol) { /* Short circuit file://... urls*/ /* We do this because the test code always needs to read files*/ - stat = readfile(url,suffix,packet); + stat = readfile(state, url,suffix,packet); } else { char* fetchurl = NULL; int flags = NCURIBASE; @@ -97,13 +109,23 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, long* las flags |= NCURIENCODE; fetchurl = ncuribuild(url,NULL,suffix,flags); MEMCHECK(fetchurl); - if(state->debug > 0) - {fprintf(stderr,"fetch url=%s\n",fetchurl); fflush(stderr);} + if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { + nclog(NCLOGDBG,"fetch url=%s",fetchurl); +#ifdef HAVE_GETTIMEOFDAY + gettimeofday(&time0,NULL); +#endif + } stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified); nullfree(fetchurl); if(stat) goto fail; - if(state->debug > 0) - {fprintf(stderr,"fetch complete\n"); fflush(stderr);} + if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { + double secs = 0; +#ifdef HAVE_GETTIMEOFDAY + gettimeofday(&time1,NULL); + secs = deltatime(); +#endif + nclog(NCLOGDBG,"fetch complete: %0.3f",secs); + } } #ifdef D4DEBUG { @@ -116,12 +138,12 @@ fprintf(stderr,"readpacket: packet.size=%lu\n", } static int -readfiletofile(const NCURI* uri, const char* suffix, FILE* stream, d4size_t* sizep) +readfiletofile(NCD4INFO* state, const NCURI* uri, const char* suffix, FILE* stream, d4size_t* sizep) { int stat = NC_NOERR; NCbytes* packet = ncbytesnew(); size_t len; - stat = readfile(uri,suffix,packet); + stat = readfile(state, uri,suffix,packet); #ifdef D4DEBUG fprintf(stderr,"readfiletofile: packet.size=%lu\n", (unsigned long)ncbyteslength(packet)); @@ -147,7 +169,7 @@ fprintf(stderr,"readfiletofile: written!=length: %lu :: %lu\n", } static int -readfile(const NCURI* uri, const char* suffix, NCbytes* packet) +readfile(NCD4INFO* state, const NCURI* uri, const char* suffix, NCbytes* packet) { int stat = NC_NOERR; NCbytes* tmp = ncbytesnew(); @@ -159,6 +181,24 @@ readfile(const NCURI* uri, const char* suffix, NCbytes* packet) filename = ncbytesextract(tmp); ncbytesfree(tmp); + state->fileproto.filename = strdup(filename); + + if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { + char* surl = NULL; +#ifdef HAVE_GETTIMEOFDAY + gettimeofday(&time0,NULL); +#endif + surl = ncuribuild((NCURI*)uri,NULL,NULL,NCURIALL); + nclog(NCLOGDBG,"fetch uri=%s file=%s",surl,filename); + } stat = NC_readfile(filename,packet); + if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { + double secs; +#ifdef HAVE_GETTIMEOFDAY + gettimeofday(&time1,NULL); + secs = deltatime(); +#endif + nclog(NCLOGDBG,"fetch complete: %0.3f",secs); + } return THROW(stat); } diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index 7379f13f7d..5c92fadca2 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -286,7 +286,6 @@ struct NCD4curl { struct NCD4INFO { NC* controller; /* Parent instance of NCD4INFO */ - int debug; char* rawurltext; /* as given to ncd4_open */ char* urltext; /* as modified by ncd4_open */ NCURI* uri; /* parse of rawuritext */ @@ -313,6 +312,9 @@ struct NCD4INFO { char substratename[NC_MAX_NAME]; } controls; NCauth auth; + struct { + char* filename; + } fileproto; }; #endif /*D4TYPES_H*/ diff --git a/libdispatch/dauth.c b/libdispatch/dauth.c index e03b91baf9..914ad6ab40 100644 --- a/libdispatch/dauth.c +++ b/libdispatch/dauth.c @@ -30,7 +30,7 @@ See LICENSE.txt for license information. /* Define the curl flag defaults in envv style */ static const char* AUTHDEFAULTS[] = { -"HTTP.TIMEOUT","10", /*seconds */ +"HTTP.TIMEOUT","1800", /*seconds */ /* Long but not infinite */ NULL }; diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 25914d3898..c877130658 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -45,7 +45,8 @@ struct MagicFile { long long filelen; int use_parallel; int inmemory; - void* parameters; + int diskless; + void* parameters; /* !NULL if inmemory && !diskless */ FILE* fp; #ifdef USE_PARALLEL MPI_File fh; @@ -292,9 +293,10 @@ NC_check_file_type(const char *path, int flags, void *parameters, { char magic[MAGIC_NUMBER_LEN]; int status = NC_NOERR; - int diskless = ((flags & NC_DISKLESS) == NC_DISKLESS); - int inmemory = (!diskless && ((flags & NC_INMEMORY) == NC_INMEMORY)); + int inmemory = ((flags & NC_INMEMORY) == NC_INMEMORY); + int mmap = ((flags & NC_MMAP) == NC_MMAP); + #ifdef USE_PARALLEL int use_parallel = ((flags & NC_MPIIO) == NC_MPIIO); #endif /* USE_PARALLEL */ @@ -303,25 +305,20 @@ NC_check_file_type(const char *path, int flags, void *parameters, *model = 0; *version = 0; + assert(inmemory ? !mmap : 1); /* inmemory => !mmap */ + assert((diskless && inmemory) ? !mmap : 1);/*diskless & inmemory => !mmap*/ + memset((void*)&file,0,sizeof(file)); file.path = path; /* do not free */ file.parameters = parameters; - if(inmemory && parameters == NULL) - {status = NC_EINMEMORY; goto done;} - if(inmemory) { - file.inmemory = inmemory; - goto next; - } - /* presumably a real file */ + file.inmemory = inmemory; + file.diskless = diskless; #ifdef USE_PARALLEL + /* presumably a real file */ /* for parallel, use the MPI functions instead (why?) */ - if (use_parallel) { - file.use_parallel = use_parallel; - goto next; - } + file.use_parallel = use_parallel; #endif /* USE_PARALLEL */ -next: status = openmagic(&file); if(status != NC_NOERR) {goto done;} /* Verify we have a large enough file */ @@ -1954,6 +1951,8 @@ static int check_create_mode(int mode) { int mode_format; + int mmap = 0; + int inmemory = 0; /* This is a clever check to see if more than one format bit is * set. */ @@ -1967,9 +1966,14 @@ check_create_mode(int mode) if (mode & NC_MPIIO && mode & NC_MPIPOSIX) return NC_EINVAL; - /* Can't use both parallel and diskless. */ - if ((mode & NC_MPIIO && mode & NC_DISKLESS) || - (mode & NC_MPIPOSIX && mode & NC_DISKLESS)) + mmap = ((mode & NC_MMAP) == NC_MMAP); + inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); + if(mmap && inmemory) /* cannot have both */ + return NC_EINMEMORY; + + /* Can't use both parallel and diskless|inmemory. */ + if ((mode & NC_MPIIO && mode & (NC_DISKLESS|NC_INMEMORY)) + || (mode & NC_MPIPOSIX && mode & (NC_DISKLESS|NC_INMEMORY))) return NC_EINVAL; #ifndef USE_NETCDF4 @@ -2029,11 +2033,19 @@ NC_create(const char *path0, int cmode, size_t initialsz, int model = NC_FORMATX_UNDEFINED; /* one of the NC_FORMATX values */ int isurl = 0; /* dap or cdmremote or neither */ char* path = NULL; + int mmap = 0; + int diskless = 0; TRACE(nc_create); if(path0 == NULL) return NC_EINVAL; + /* Fix the inmemory related flags */ + mmap = ((cmode & NC_MMAP) == NC_MMAP); + diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS); + /* diskless && !mmap => inmemory */ + if(diskless && !mmap) cmode |= NC_INMEMORY; + /* Check mode flag for sanity. */ if ((stat = check_create_mode(cmode))) return stat; @@ -2197,6 +2209,7 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp, NC_Dispatch* dispatcher = NULL; int inmemory = 0; int diskless = 0; + int mmap = 0; /* Need pieces of information for now to decide model*/ int model = 0; int isurl = 0; @@ -2210,14 +2223,23 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp, if(stat) return stat; } + /* Fix the inmemory related flags */ + mmap = ((cmode & NC_MMAP) == NC_MMAP); + diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS); + + /* diskless && !mmap => inmemory */ + if(diskless && !mmap) cmode |= NC_INMEMORY; + + inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY); + + if(mmap && inmemory) /* cannot have both */ + return NC_EINMEMORY; + /* Attempt to do file path conversion: note that this will do nothing if path is a 'file:...' url, so it will need to be repeated in protocol code: libdap2 and libdap4 */ - inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY); - diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS); - #ifdef WINPATH path = NCpathcvt(path0); #else @@ -2272,6 +2294,7 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp, if(useparallel) flags |= NC_MPIIO; if(inmemory) flags |= NC_INMEMORY; if(diskless) flags |= NC_DISKLESS; + if(mmap) flags |= NC_MMAP; stat = NC_check_file_type(path,flags,parameters,&model,&version); if(stat == NC_NOERR) { if(model == 0) { @@ -2476,7 +2499,8 @@ static int openmagic(struct MagicFile* file) { int status = NC_NOERR; - if(file->inmemory) { + assert((!file->diskless && file->inmemory) ? file->parameters != NULL : 1); + if(file->inmemory && !file->diskless) { /* Get its length */ NC_memio* meminfo = (NC_memio*)file->parameters; file->filelen = (long long)meminfo->size; @@ -2538,7 +2562,7 @@ readmagic(struct MagicFile* file, long pos, char* magic) { int status = NC_NOERR; memset(magic,0,MAGIC_NUMBER_LEN); - if(file->inmemory) { + if(file->inmemory && !file->diskless) { char* mempos; NC_memio* meminfo = (NC_memio*)file->parameters; if((pos + MAGIC_NUMBER_LEN) > meminfo->size) diff --git a/libdispatch/dparallel.c b/libdispatch/dparallel.c index d420281689..578cfa9203 100644 --- a/libdispatch/dparallel.c +++ b/libdispatch/dparallel.c @@ -1,10 +1,9 @@ -/** \file - -This file has the parallel I/O functions which correspond to the -serial I/O functions. - -Copyright 2010 University Corporation for Atmospheric -Research/Unidata. See COPYRIGHT file for more info. +/* Copyright 2010 University Corporation for Atmospheric + Research/Unidata. See COPYRIGHT file for more info. */ +/** + * @file + * @internal This file has the parallel I/O functions which correspond + * to the serial I/O functions. */ #include "config.h" #include "ncdispatch.h" diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c index 8701204e0c..2dcc7289e4 100644 --- a/libdispatch/dvar.c +++ b/libdispatch/dvar.c @@ -149,7 +149,8 @@ nc_inq_ncid(). @param ndims Number of dimensions for the variable. For example, 2 specifies a matrix, 1 specifies a vector, and 0 means the variable is a scalar with no dimensions. Must not be negative or greater than the -predefined constant ::NC_MAX_VAR_DIMS. +predefined constant ::NC_MAX_VAR_DIMS. In netCDF-4/HDF5 files, may not +exceed the HDF5 maximum number of dimensions (32). @param dimidsp Vector of ndims dimension IDs corresponding to the variable dimensions. For classic model netCDF files, if the ID of the @@ -647,7 +648,7 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) * @return ::NC_EBADID Bad ncid. * @return ::NC_ENOTVAR Variable not found. * @return ::NC_ENOMEM Out of memory. - * @return ::NC_EINVALCOORS Missing start array. + * @return ::NC_EINVALCOORDS Missing start array. * @author Ed Hartnett */ int diff --git a/libdispatch/dwinpath.c b/libdispatch/dwinpath.c index acd6205576..6f5e551c86 100644 --- a/libdispatch/dwinpath.c +++ b/libdispatch/dwinpath.c @@ -202,4 +202,37 @@ NCopen2(const char *path, int flags) return NCopen3(path,flags,0); } +/* +Provide wrappers for other file system functions +*/ + +/* Return access applied to path+mode */ +EXTERNL +int +NCaccess(const char* path, int mode) +{ + int status = 0; + char* cvtname = NCpathcvt(path); + if(cvtname == NULL) return -1; +#ifdef _MSC_VER + status = _access(cvtname,mode); +#else + status = access(cvtname,mode); +#endif + free(cvtname); + return status; +} + +EXTERNL +int +NCremove(const char* path) +{ + int status = 0; + char* cvtname = NCpathcvt(path); + if(cvtname == NULL) return ENOENT; + status = remove(cvtname); + free(cvtname); + return status; +} + #endif /*WINPATH*/ diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 4da1a65008..2779783290 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -309,7 +309,8 @@ nc4typelen(nc_type type) } /** - * @internal Write an attribute to a netCDF-4/HDF5 file, converting + * @internal + * Write an attribute to a netCDF-4/HDF5 file, converting * data type if necessary. * * @param ncid File and group ID. @@ -319,6 +320,7 @@ nc4typelen(nc_type type) * @param len Number of elements in attribute array. * @param data Attribute data. * @param mem_type Type of data in memory. + * @param force write even if the attribute is special * * @return ::NC_NOERR No error. * @return ::NC_EINVAL Invalid parameters. @@ -329,11 +331,10 @@ nc4typelen(nc_type type) * @author Ed Hartnett, Dennis Heimbigner */ int -NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, - size_t len, const void *data, nc_type mem_type) +nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, + size_t len, const void *data, nc_type mem_type, int force) { - NC *nc; - NC_GRP_INFO_T *grp; + NC* nc; NC_FILE_INFO_T *h5; NC_VAR_INFO_T *var = NULL; NCindex* attlist = NULL; @@ -344,12 +345,14 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, size_t type_size; int i; int ret; + int ncid; - /* Find info for this file, group, and h5 info. */ - if ((ret = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) - return ret; + h5 = grp->nc4_info; + nc = h5->controller; assert(nc && grp && h5); + ncid = nc->ext_ncid | grp->hdr.id; + /* Find att, if it exists. (Must check varid first or nc_test will * break.) */ if ((ret = getattlist(grp, varid, &var, &attlist))) @@ -365,7 +368,7 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, return NC_EBADNAME; LOG((1, "%s: ncid 0x%x varid %d name %s file_type %d mem_type %d len %d", - __func__, ncid, varid, name, file_type, mem_type, len)); + __func__,ncid, varid, name, file_type, mem_type, len)); /* If len is not zero, then there must be some data. */ if (len && !data) @@ -381,7 +384,7 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, /* Check that a reserved att name is not being used improperly */ const NC_reservedatt* ra = NC_findreserved(name); - if(ra != NULL) { + if(ra != NULL && !force) { /* case 1: grp=root, varid==NC_GLOBAL, flags & READONLYFLAG */ if (nc->ext_ncid == ncid && varid == NC_GLOBAL && grp->parent == NULL && (ra->flags & READONLYFLAG)) @@ -403,7 +406,7 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, if (!(h5->flags & NC_INDEF)) { if (h5->cmode & NC_CLASSIC_MODEL) - return NC_EINDEFINE; + return NC_ENOTINDEFINE; if ((retval = NC4_redef(ncid))) BAIL(retval); } @@ -417,7 +420,7 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, len * nc4typelen(file_type) > (size_t)att->len * nc4typelen(att->nc_typeid)) { if (h5->cmode & NC_CLASSIC_MODEL) - return NC_EINDEFINE; + return NC_ENOTINDEFINE; if ((retval = NC4_redef(ncid))) BAIL(retval); } @@ -656,3 +659,42 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, return NC_ERANGE; return NC_NOERR; } + +/** + * @internal + * Write an attribute to a netCDF-4/HDF5 file, converting + * data type if necessary. + * Wrapper around nc4_put_att + * + * @param ncid File and group ID. + * @param varid Variable ID. + * @param name Name of attribute. + * @param file_type Type of the attribute data in file. + * @param len Number of elements in attribute array. + * @param data Attribute data. + * @param mem_type Type of data in memory. + * + * @return ::NC_NOERR No error. + * @return ::NC_EINVAL Invalid parameters. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_ENOTVAR Variable not found. + * @return ::NC_EBADNAME Name contains illegal characters. + * @return ::NC_ENAMEINUSE Name already in use. + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NC4_put_att(int ncid, int varid, const char *name, nc_type file_type, + size_t len, const void *data, nc_type mem_type) +{ + int ret = NC_NOERR; + NC *nc; + NC_FILE_INFO_T *h5; + NC_GRP_INFO_T *grp; + + /* Find info for this file, group, and h5 info. */ + if ((ret = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) + return ret; + assert(nc && grp && h5); + + return nc4_put_att(grp,varid,name,file_type,len,data,mem_type,0); +} diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index cc5194f178..1e3dd7f673 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -155,10 +155,6 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, } } #else /* only set cache for non-parallel... */ - if(cmode & NC_DISKLESS) { - if (H5Pset_fapl_core(fapl_id, 4096, nc4_info->mem.persist)) - BAIL(NC_EDISKLESS); - } if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, nc4_chunk_cache_preemption) < 0) BAIL(NC_EHDFERR); @@ -228,13 +224,8 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, /* Define mode gets turned on automatically on create. */ nc4_info->flags |= NC_INDEF; - /* Get the HDF5 superblock and read and parse the special - * _NCProperties attribute. */ - if ((retval = NC4_get_fileinfo(nc4_info, &globalpropinfo))) - BAIL(retval); - - /* Write the _NCProperties attribute. */ - if ((retval = NC4_put_propattr(nc4_info))) + /* Save the HDF5 superblock number and set the _NCProperties attribute. */ + if ((retval = NC4_set_provenance(nc4_info, &globalpropinfo))) BAIL(retval); return NC_NOERR; @@ -244,11 +235,9 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, if (comm_duped) MPI_Comm_free(&nc4_info->comm); if (info_duped) MPI_Info_free(&nc4_info->info); #endif - if (fapl_id != H5P_DEFAULT) - H5Pclose(fapl_id); - if (!nc4_info) - return retval; - nc4_close_netcdf4_file(nc4_info, 1, 0); /* treat like abort */ + if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); + if(!nc4_info) return retval; + nc4_close_netcdf4_file(nc4_info,1,NULL); /* treat like abort */ return retval; } diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index b68934150a..b7881b208a 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -16,9 +16,6 @@ extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); -/* From nc4mem.c */ -extern int NC4_extract_file_image(NC_FILE_INFO_T* h5); - /** @internal When we have open objects at file close, should we log them or print to stdout. Default is to log. */ #define LOGOPEN 1 @@ -35,13 +32,13 @@ static const NC_reservedatt NC_reserved[NRESERVED] = { {NC_ATT_DIMENSION_LIST, READONLYFLAG|DIMSCALEFLAG}, /*DIMENSION_LIST*/ {NC_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*NAME*/ {NC_ATT_REFERENCE_LIST, READONLYFLAG|DIMSCALEFLAG}, /*REFERENCE_LIST*/ - {NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/ - {ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/ - {NCPROPS, READONLYFLAG|NAMEONLYFLAG}, /*_NCProperties*/ - {NC_ATT_COORDINATES, READONLYFLAG|DIMSCALEFLAG}, /*_Netcdf4Coordinates*/ - {NC_DIMID_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*_Netcdf4Dimid*/ + {NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/ + {ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/ + {NCPROPS, READONLYFLAG|NAMEONLYFLAG|MATERIALIZEDFLAG},/*_NCProperties*/ + {NC_ATT_COORDINATES, READONLYFLAG|DIMSCALEFLAG|MATERIALIZEDFLAG},/*_Netcdf4Coordinates*/ + {NC_DIMID_ATT_NAME, READONLYFLAG|DIMSCALEFLAG|MATERIALIZEDFLAG},/*_Netcdf4Dimid*/ {SUPERBLOCKATT, READONLYFLAG|NAMEONLYFLAG},/*_SuperblockVersion*/ - {NC3_STRICT_ATT_NAME, READONLYFLAG}, /*_nc3_strict*/ + {NC3_STRICT_ATT_NAME, READONLYFLAG|MATERIALIZEDFLAG}, /*_nc3_strict*/ }; extern void reportopenobjects(int log, hid_t); @@ -116,7 +113,7 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5) #endif /* Write any metadata that has changed. */ - if (!(h5->cmode & NC_NOWRITE)) + if (!h5->no_write) { nc_bool_t bad_coord_order = NC_FALSE; @@ -133,6 +130,10 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5) /* Write all the metadata. */ if ((retval = nc4_rec_write_metadata(h5->root_grp, bad_coord_order))) return retval; + + /* Write out _NCProperties */ + if((retval = NC4_write_ncproperties(h5))) + return retval; } /* Tell HDF5 to flush all changes to the file. */ @@ -146,17 +147,18 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5) /** * @internal This function will free all allocated metadata memory, * and close the HDF5 file. The group that is passed in must be the - * root group of the file. + * root group of the file. If inmemory is used, then save + * the final memory in mem.memio. * * @param h5 Pointer to HDF5 file info struct. * @param abort True if this is an abort. - * @param extractmem True if we need to extract and save final inmemory + * @param memio the place to return a core image if not NULL * * @return ::NC_NOERR No error. * @author Ed Hartnett */ int -nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) +nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio* memio) { NC_HDF5_FILE_INFO_T *hdf5_info; int retval = NC_NOERR; @@ -199,26 +201,36 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) /* Free the fileinfo struct, which holds info from the fileinfo * hidden attribute. */ - if (h5->fileinfo) - free(h5->fileinfo); - - /* Check to see if this is an in-memory file and we want to get its - final content. */ - if(extractmem) { - /* File must be read/write */ - if(!h5->no_write) { - retval = NC4_extract_file_image(h5); - } - } + if (h5->provenance) + free(h5->provenance); /* Close hdf file. It may not be open, since this function is also * called by NC_create() when a file opening is aborted. */ - if (hdf5_info->hdfid && H5Fclose(hdf5_info->hdfid) < 0) + if (hdf5_info->hdfid > 0 && H5Fclose(hdf5_info->hdfid) < 0) { dumpopenobjects(h5); BAIL(NC_EHDFERR); } + /* If inmemory is used and user wants the final memory block, + then capture and return the final memory block else free it */ + if(h5->mem.inmemory) { + if(!abort && memio != NULL) { + *memio = h5->mem.memio; /* capture it */ + h5->mem.memio.memory = NULL; /* avoid duplicate free */ + } + /* If needed, reclaim extraneous memory */ + if(h5->mem.memio.memory != NULL) { + /* If the original block of memory is not resizeable, then + it belongs to the caller and we should not free it. */ + if(!h5->mem.locked) + free(h5->mem.memio.memory); + } + h5->mem.memio.memory = NULL; + h5->mem.memio.size = 0; + NC4_image_finalize(h5->mem.udata); + } + /* Free the HDF5-specific info. */ if (h5->format_file_info) free(h5->format_file_info); @@ -242,6 +254,9 @@ dumpopenobjects(NC_FILE_INFO_T* h5) assert(h5 && h5->format_file_info); hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info; + if(hdf5_info->hdfid <= 0) + return; /* File was never opened */ + nobjs = H5Fget_obj_count(hdf5_info->hdfid, H5F_OBJ_ALL); /* Apparently we can get an error even when nobjs == 0 */ @@ -275,7 +290,6 @@ size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE; /**< Default chunk ca size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */ float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */ - /** * Set chunk cache size. Only affects files opened/created *after* it * is called. @@ -589,7 +603,7 @@ NC4_abort(int ncid) /* Free any resources the netcdf-4 library has for this file's * metadata. */ - if ((retval = nc4_close_netcdf4_file(nc4_info, 1, 0))) + if ((retval = nc4_close_netcdf4_file(nc4_info, 1, NULL))) return retval; /* Delete the file, if we should. */ @@ -617,6 +631,7 @@ NC4_close(int ncid, void* params) NC_FILE_INFO_T *h5; int retval; int inmemory; + NC_memio* memio = NULL; LOG((1, "%s: ncid 0x%x", __func__, ncid)); @@ -632,14 +647,14 @@ NC4_close(int ncid, void* params) inmemory = ((h5->cmode & NC_INMEMORY) == NC_INMEMORY); - /* Call the nc4 close. */ - if ((retval = nc4_close_netcdf4_file(grp->nc4_info, 0, (inmemory?1:0)))) - return retval; if(inmemory && params != NULL) { - NC_memio* memio = (NC_memio*)params; - *memio = h5->mem.memio; + memio = (NC_memio*)params; } + /* Call the nc4 close. */ + if ((retval = nc4_close_netcdf4_file(grp->nc4_info, 0, memio))) + return retval; + return NC_NOERR; } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 6a0095ab60..0dcfde24d2 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -230,8 +230,8 @@ get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid, } /** - * @internal This function reads the hacked in coordinates attribute I - * use for multi-dimensional coordinates. + * @internal This function reads the coordinates attribute used for + * multi-dimensional coordinates. * * @param grp Group info pointer. * @param var Var info pointer. @@ -244,37 +244,46 @@ read_coord_dimids(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) { hid_t coord_att_typeid = -1, coord_attid = -1, spaceid = -1; hssize_t npoints; - int ret = 0; + int retval = NC_NOERR; int d; /* There is a hidden attribute telling us the ids of the * dimensions that apply to this multi-dimensional coordinate * variable. Read it. */ - if ((coord_attid = H5Aopen_name(var->hdf_datasetid, COORDINATES)) < 0) ret++; - if (!ret && (coord_att_typeid = H5Aget_type(coord_attid)) < 0) ret++; + if ((coord_attid = H5Aopen_name(var->hdf_datasetid, COORDINATES)) < 0) + BAIL(NC_EATTMETA); + + if ((coord_att_typeid = H5Aget_type(coord_attid)) < 0) + BAIL(NC_EATTMETA); /* How many dimensions are there? */ - if (!ret && (spaceid = H5Aget_space(coord_attid)) < 0) ret++; - if (!ret && (npoints = H5Sget_simple_extent_npoints(spaceid)) < 0) ret++; + if ((spaceid = H5Aget_space(coord_attid)) < 0) + BAIL(NC_EATTMETA); + if ((npoints = H5Sget_simple_extent_npoints(spaceid)) < 0) + BAIL(NC_EATTMETA); - /* Check that the number of points is the same as the number of dimensions - * for the variable */ - if (!ret && npoints != var->ndims) ret++; + /* Check that the number of points is the same as the number of + * dimensions for the variable. */ + if (npoints != var->ndims) + BAIL(NC_EATTMETA); - if (!ret && H5Aread(coord_attid, coord_att_typeid, var->dimids) < 0) ret++; + if (H5Aread(coord_attid, coord_att_typeid, var->dimids) < 0) + BAIL(NC_EATTMETA); LOG((4, "dimscale %s is multidimensional and has coords", var->hdr.name)); - /* Update var->dim field based on the var->dimids */ - for (d = 0; d < var->ndims; d++) { - /* Ok if does not find a dim at this time, but if found set it */ + /* Update var->dim field based on the var->dimids. Ok if does not + * find a dim at this time, but if found set it. */ + for (d = 0; d < var->ndims; d++) nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL); - } - /* Set my HDF5 IDs free! */ - if (spaceid >= 0 && H5Sclose(spaceid) < 0) ret++; - if (coord_att_typeid >= 0 && H5Tclose(coord_att_typeid) < 0) ret++; - if (coord_attid >= 0 && H5Aclose(coord_attid) < 0) ret++; - return ret ? NC_EATTMETA : NC_NOERR; +exit: + if (spaceid >= 0 && H5Sclose(spaceid) < 0) + BAIL2(NC_EHDFERR); + if (coord_att_typeid >= 0 && H5Tclose(coord_att_typeid) < 0) + BAIL2(NC_EHDFERR); + if (coord_attid >= 0 && H5Aclose(coord_attid) < 0) + BAIL2(NC_EHDFERR); + return retval; } /** @@ -380,31 +389,18 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); - if(nc4_info->mem.inmemory) { - NC_memio* memparams = NULL; - if(parameters == NULL) - BAIL(NC_EINMEMORY); - memparams = (NC_memio*)parameters; - nc4_info->mem.memio = *memparams; /* keep local copy */ - /* As a safeguard, if !locked and NC_WRITE is set, - then we must take control of the incoming memory */ - nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED; - if(!nc4_info->mem.locked && ((mode & NC_WRITE) == NC_WRITE)) { - memparams->memory = NULL; - } + #ifdef USE_PARALLEL4 - } else { - mpiinfo = (NC_MPI_INFO*)parameters; + mpiinfo = (NC_MPI_INFO*)parameters; /* assume, may be changed if inmemory is true */ #endif /* !USE_PARALLEL4 */ - } /* Need this access plist to control how HDF5 handles open objects * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to - * fail if there are any open objects in the file. */ + * fail if there are any open objects in the file). */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) BAIL(NC_EHDFERR); - if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) + if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI) < 0) BAIL(NC_EHDFERR); #ifdef USE_PARALLEL4 @@ -427,10 +423,9 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) BAIL(NC_EPARINIT); } #else /* USE_PARALLEL_POSIX */ - /* Should not happen! Code in NC4_create/NC4_open should alias the - * NC_MPIPOSIX flag to NC_MPIIO, if the MPI-POSIX VFD is not - * available in HDF5. -QAK - */ + /* Should not happen! Code in NC4_create/NC4_open should alias + * the NC_MPIPOSIX flag to NC_MPIIO, if the MPI-POSIX VFD is not + * available in HDF5. */ else /* MPI/POSIX */ BAIL(NC_EPARINIT); #endif /* USE_PARALLEL_POSIX */ @@ -451,6 +446,12 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) nc4_info->info = mpiinfo->info; } } + +#ifdef HDF5_HAS_COLL_METADATA_OPS + if (H5Pset_all_coll_metadata_ops(fapl_id, 1) < 0) + BAIL(NC_EPARINIT); +#endif + #else /* only set cache for non-parallel. */ if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, nc4_chunk_cache_preemption) < 0) @@ -460,21 +461,33 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) nc4_chunk_cache_preemption)); #endif /* USE_PARALLEL4 */ - /* The NetCDF-3.x prototype contains an mode option NC_SHARE for - multiple processes accessing the dataset concurrently. As there - is no HDF5 equivalent, NC_SHARE is treated as NC_NOWRITE. */ -#ifdef HDF5_HAS_COLL_METADATA_OPS - H5Pset_all_coll_metadata_ops(fapl_id, 1 ); -#endif - /* Does the mode specify that this file is read-only? */ if ((mode & NC_WRITE) == 0) nc4_info->no_write = NC_TRUE; + /* Now process if NC_INMEMORY is set (recall NC_DISKLESS => NC_INMEMORY) */ if(nc4_info->mem.inmemory) { + NC_memio* memio; + /* validate */ - if(nc4_info->mem.memio.size == 0 || nc4_info->mem.memio.memory == NULL) - BAIL(NC_INMEMORY); + if(parameters == NULL) + BAIL(NC_EINMEMORY); + memio = (NC_memio*)parameters; + if(memio->memory == NULL || memio->size == 0) + BAIL(NC_EINMEMORY); + + /* initialize h5->mem */ + nc4_info->mem.memio = *memio; + + /* Is the incoming memory locked? */ + nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED; + + /* As a safeguard, if !locked and not read-only, + then we must take control of the incoming memory */ + if(!nc4_info->mem.locked && !nc4_info->no_write) { + memio->memory = NULL; /* take control */ + memio->size = 0; + } retval = NC4_open_image_file(nc4_info); if(retval) BAIL(NC_EHDFERR); @@ -502,6 +515,22 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if (is_classic) nc4_info->cmode |= NC_CLASSIC_MODEL; + /* See if this file contained _NCPROPERTIES, + * and if yes, process it, if no, then fake it. + */ + if(nc4_info->root_grp != NULL) { + /* Since _NCProperties does not exist as an NC_ATT_INFO object, + * we need to check using the HDF5 API + */ + if((retval = NC4_read_ncproperties(nc4_info))) + BAIL(retval); + } + + if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic))) + BAIL(retval); + if (is_classic) + nc4_info->cmode |= NC_CLASSIC_MODEL; + /* Now figure out which netCDF dims are indicated by the dimscale * information. */ if ((retval = nc4_rec_match_dimscales(nc4_info->root_grp))) @@ -517,11 +546,6 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if (H5Pclose(fapl_id) < 0) BAIL(NC_EHDFERR); - /* Get the HDF5 superblock and read and parse the special - * _NCProperties attribute. */ - if ((retval = NC4_get_fileinfo(nc4_info, NULL))) - BAIL(retval); - return NC_NOERR; exit: @@ -529,9 +553,11 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if (comm_duped) MPI_Comm_free(&nc4_info->comm); if (info_duped) MPI_Info_free(&nc4_info->info); #endif - if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); - if (!nc4_info) return retval; - nc4_close_netcdf4_file(nc4_info,1,0); /* treat like abort*/ + + if (fapl_id > 0 && fapl_id != H5P_DEFAULT) + H5Pclose(fapl_id); + if (nc4_info) + nc4_close_netcdf4_file(nc4_info, 1, 0); /* treat like abort*/ return retval; } @@ -1869,13 +1895,11 @@ static int nc4_rec_read_metadata_cb(hid_t grpid, const char *name, const H5L_info_t *info, void *_op_data) { - NC4_rec_read_metadata_ud_t *udata = (NC4_rec_read_metadata_ud_t *)_op_data; /* Pointer to user data for callback */ + /* Pointer to user data for callback */ + NC4_rec_read_metadata_ud_t *udata = (NC4_rec_read_metadata_ud_t *)_op_data; NC4_rec_read_metadata_obj_info_t oinfo; /* Pointer to info for object */ int retval = H5_ITER_CONT; - /* Reset the memory for the object's info */ - memset(&oinfo, 0, sizeof(oinfo)); - /* Open this critter. */ if ((oinfo.oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0) BAIL(H5_ITER_ERROR); @@ -1892,10 +1916,9 @@ nc4_rec_read_metadata_cb(hid_t grpid, const char *name, const H5L_info_t *info, case H5G_GROUP: LOG((3, "found group %s", oinfo.oname)); - /* Defer descending into child group immediately, so that the types - * in the current group can be processed and be ready for use by - * vars in the child group(s). - */ + /* Defer descending into child group immediately, so that the + * types in the current group can be processed and be ready for + * use by vars in the child group(s). */ if (nc4_rec_read_metadata_cb_list_add(udata, &oinfo)) BAIL(H5_ITER_ERROR); break; @@ -1909,11 +1932,10 @@ nc4_rec_read_metadata_cb(hid_t grpid, const char *name, const H5L_info_t *info, &oinfo.statbuf))) { /* Allow NC_EBADTYPID to transparently skip over datasets - * which have a datatype that netCDF-4 doesn't undertand - * (currently), but break out of iteration for other - * errors. - */ - if(NC_EBADTYPID != retval) + * which have a datatype that netCDF-4 doesn't undertand + * (currently), but break out of iteration for other + * errors. */ + if (retval != NC_EBADTYPID) BAIL(H5_ITER_ERROR); else retval = H5_ITER_CONT; diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 58ccf9c4d8..08e19cbce5 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -217,7 +217,7 @@ nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) * @param ncid File ID. * @param name Name. * @param xtype Type. - * @param ndims Number of dims. + * @param ndims Number of dims. HDF5 has maximim of 32. * @param dimidsp Array of dim IDs. * @param varidp Gets the var ID. * @@ -259,6 +259,10 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, BAIL(retval); assert(grp && h5); + /* HDF5 allows maximum of 32 dimensions. */ + if (ndims > H5S_MAX_RANK) + BAIL(NC_EMAXDIMS); + /* If it's not in define mode, strict nc3 files error out, * otherwise switch to define mode. This will also check that the * file is writable. */ diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 43e0186ee9..00a77f2365 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -3,8 +3,9 @@ * conditions. */ /** * @file - * This file is part of netcdf-4, a netCDF-like interface for HDF5, or a - * HDF5 backend for netCDF, depending on your point of view. + * @internal This file is part of netcdf-4, a netCDF-like interface + * for HDF5, or a HDF5 backend for netCDF, depending on your point of + * view. * * This file contains functions internal to the netcdf4 library. None of * the functions in this file are exposed in the exetnal API. These @@ -380,7 +381,6 @@ nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype, /* Determine an appropriate HDF5 datatype */ if (xtype == NC_NAT) - /* NAT = 'Not A Type' (c.f. NaN) */ return NC_EBADTYPE; else if (xtype == NC_CHAR || xtype == NC_STRING) { @@ -550,15 +550,17 @@ put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) { hid_t datasetid = 0, locid; hid_t attid = 0, spaceid = 0, file_typeid = 0; + hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0; hsize_t dims[1]; /* netcdf attributes always 1-D. */ htri_t attr_exists; - int retval = NC_NOERR; + int reuse_att = 0; /* Will be true if we can re-use an existing att. */ void *data; int phoney_data = 99; + int retval = NC_NOERR; assert(att->hdr.name); - LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d att->len %d", - __func__, varid, att->hdr.id, att->hdr.name, + LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d " + "att->len %d", __func__, varid, att->hdr.id, att->hdr.name, att->nc_typeid, att->len)); /* If the file is read-only, return an error. */ @@ -575,15 +577,6 @@ put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) locid = datasetid; } - /* Delete the att if it exists already. */ - if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0) - BAIL(NC_EHDFERR); - if (attr_exists) - { - if (H5Adelete(locid, att->hdr.name) < 0) - BAIL(NC_EHDFERR); - } - /* Get the length ready, and find the HDF type we'll be * writing. */ dims[0] = att->len; @@ -640,6 +633,41 @@ put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) BAIL(NC_EATTMETA); } } + + /* Does the att exists already? */ + if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0) + BAIL(NC_EHDFERR); + if (attr_exists) + { + hssize_t npoints; + + /* Open the attribute. */ + if ((existing_attid = H5Aopen(locid, att->hdr.name, H5P_DEFAULT)) < 0) + BAIL(NC_EATTMETA); + + /* Find the type of the existing attribute. */ + if ((existing_att_typeid = H5Aget_type(existing_attid)) < 0) + BAIL(NC_EATTMETA); + + /* How big is the attribute? */ + if ((existing_spaceid = H5Aget_space(existing_attid)) < 0) + BAIL(NC_EATTMETA); + if ((npoints = H5Sget_simple_extent_npoints(existing_spaceid)) < 0) + BAIL(NC_EATTMETA); + + /* Delete the attribute. */ + if (file_typeid != existing_att_typeid || npoints != att->len) + { + if (H5Adelete(locid, att->hdr.name) < 0) + BAIL(NC_EHDFERR); + } + else + { + reuse_att++; + } + } + + /* Create the attribute. */ if ((attid = H5Acreate(locid, att->hdr.name, file_typeid, spaceid, H5P_DEFAULT)) < 0) BAIL(NC_EATTMETA); @@ -653,8 +681,14 @@ put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) BAIL2(NC_EHDFERR); if (attid > 0 && H5Aclose(attid) < 0) BAIL2(NC_EHDFERR); + if (existing_att_typeid && H5Tclose(existing_att_typeid)) + BAIL2(NC_EHDFERR); + if (existing_attid > 0 && H5Aclose(existing_attid) < 0) + BAIL2(NC_EHDFERR); if (spaceid > 0 && H5Sclose(spaceid) < 0) BAIL2(NC_EHDFERR); + if (existing_spaceid > 0 && H5Sclose(existing_spaceid) < 0) + BAIL2(NC_EHDFERR); return retval; } @@ -670,16 +704,16 @@ put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) * @author Ed Hartnett */ static int -write_attlist(NCindex* attlist, int varid, NC_GRP_INFO_T *grp) +write_attlist(NCindex *attlist, int varid, NC_GRP_INFO_T *grp) { NC_ATT_INFO_T *att; int retval; int i; - for(i=0;idirty) { LOG((4, "%s: writing att %s to varid %d", __func__, att->hdr.name, varid)); @@ -811,8 +845,8 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid if ((access_plistid = H5Pcreate(H5P_DATASET_ACCESS)) < 0) BAIL(NC_EHDFERR); - /* RJ: this suppose to be FALSE that is defined in H5 private.h as 0 */ - if (H5Pset_obj_track_times(plistid,0)<0) + /* Turn off object tracking times in HDF5. */ + if (H5Pset_obj_track_times(plistid, 0) < 0) BAIL(NC_EHDFERR); /* Find the HDF5 type of the dataset. */ @@ -1921,11 +1955,13 @@ nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order) int i; assert(grp && grp->hdr.name && grp->hdf_grpid); - LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name, bad_coord_order)); + LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name, + bad_coord_order)); /* Write global attributes for this group. */ if ((retval = write_attlist(grp->att, NC_GLOBAL, grp))) return retval; + /* Set the pointers to the beginning of the list of dims & vars in this * group. */ dim_index = 0; diff --git a/libhdf5/nc4info.c b/libhdf5/nc4info.c index 866fe450d8..b04a39274d 100644 --- a/libhdf5/nc4info.c +++ b/libhdf5/nc4info.c @@ -8,158 +8,274 @@ */ #include "config.h" +#include "nc4internal.h" #include "hdf5internal.h" +#include "nclist.h" +#include "ncbytes.h" +/* Various Constants */ +#define NCPROPS_MAX_NAME 1024 /* max key name size */ +#define NCPROPS_MAX_VALUE 1024 /* max value size */ #define HDF5_MAX_NAME 1024 /**< HDF5 max name. */ +#define ESCAPECHARS "\\=|," + /** @internal Check NetCDF return code. */ #define NCHECK(expr) {if((expr)!=NC_NOERR) {goto done;}} /** @internal Check HDF5 return code. */ #define HCHECK(expr) {if((expr)<0) {ncstat = NC_EHDFERR; goto done;}} +static int globalpropinitialized = 0; struct NCPROPINFO globalpropinfo; /**< Global property info. */ +/* Forward */ +static int properties_parse(const char* text0, NClist* pairs); + /** - * @internal Initialize file info. + * @internal Initialize default provenance info + * This will only be used for newly created files + * or for opened files that do not contain an _NCProperties + * attribute. * * @return ::NC_NOERR No error. * @author Dennis Heimbigner */ int -NC4_fileinfo_init(void) +NC4_provenance_init(void) { int stat = NC_NOERR; + int i; + NClist* other = NULL; + char* name = NULL; + char* value = NULL; unsigned major,minor,release; - /* Build nc properties */ + if(globalpropinitialized) + return stat; + + /* Build _NCProperties info */ + + /* Initialize globalpropinfo */ memset((void*)&globalpropinfo,0,sizeof(globalpropinfo)); globalpropinfo.version = NCPROPS_VERSION; - globalpropinfo.netcdfver[0] = '\0'; - globalpropinfo.hdf5ver[0] = '\0'; + globalpropinfo.properties = nclistnew(); + if(globalpropinfo.properties == NULL) + {stat = NC_ENOMEM; goto done;} + /* Insert primary library version as first entry */ + if((value = strdup(PACKAGE_VERSION)) == NULL) + {stat = NC_ENOMEM; goto done;} + + if((name = strdup(NCPNCLIB2)) == NULL) + {stat = NC_ENOMEM; goto done;} + nclistpush(globalpropinfo.properties,name); + name = NULL; + nclistpush(globalpropinfo.properties,value); + value = NULL; + + /* Insert the HDF5 as underlying storage format library */ stat = NC4_hdf5get_libversion(&major,&minor,&release); if(stat) goto done; - snprintf(globalpropinfo.hdf5ver,sizeof(globalpropinfo.hdf5ver), - "%1u.%1u.%1u",major,minor,release); - strncpy(globalpropinfo.netcdfver,PACKAGE_VERSION,sizeof(globalpropinfo.netcdfver)); + { + char sversion[64]; + snprintf(sversion,sizeof(sversion),"%1u.%1u.%1u",major,minor,release); + if((value = strdup(sversion)) == NULL) + {stat = NC_ENOMEM; goto done;} + } + if((name = strdup(NCPHDF5LIB2)) == NULL) + {stat = NC_ENOMEM; goto done;} + nclistpush(globalpropinfo.properties,name); + name = NULL; + nclistpush(globalpropinfo.properties,value); + value = NULL; + + /* Add any extra fields */ + /*Parse them into an NClist */ + other = nclistnew(); + if(other == NULL) {stat = NC_ENOMEM; goto done;} +#ifdef NCPROPERTIES + stat = properties_parse(NCPROPERTIES_EXTRA,other); + if(stat) goto done; +#endif + /* merge into the properties list */ + for(i=0;iversion = 0; - ncprops->hdf5ver[0] = '\0'; - ncprops->netcdfver[0] = '\0'; + text = strdup(text0); + if(text == NULL) return NC_ENOMEM; - len = strlen(text); - if(len == 0) return NC_NOERR; - propdata = (char*)malloc(len+1); - if(propdata == NULL) return NC_ENOMEM; - memcpy(propdata,text,len+1); - propdata[len] = '\0'; /* guarantee */ + /* For back compatibility with version 1, translate '|' -> ',' */ + for(p=text;*p;p++) { + if(*p == NCPROPSSEP1) + *p = NCPROPSSEP2; + } /* Walk and fill in ncinfo */ - p = propdata; + p = text; while(*p) { - char* name = p; - char* value = NULL; - char* q = strchr(p,'='); - if(q == NULL) - {ret = NC_EINVAL; goto done;} - *q++ = '\0'; - value = p = q; - q = strchr(p,NCPROPSSEP); - if(q == NULL) q = (p+strlen(p)); else* q++ = '\0'; - p = q; - if(value != NULL) { - if(strcmp(name,NCPVERSION) == 0) { - int v = atoi(value); - if(v < 0) v = 0; - ncprops->version = v; - } else if(strcmp(name,NCPNCLIBVERSION) == 0) - strncpy(ncprops->netcdfver,value,sizeof(ncprops->netcdfver)-1); - else if(strcmp(name,NCPHDF5LIBVERSION) == 0) - strncpy(ncprops->hdf5ver,value,sizeof(ncprops->hdf5ver)-1); - /* else ignore */ - } + char* name = p; + char* value = NULL; + char* next = NULL; + + /* Delimit whole (key,value) pair */ + q = locate(p,NCPROPSSEP2); + if(*q != '\0') /* Never go beyond the final nul term */ + *q++ = '\0'; + next = q; + /* split key and value */ + q = locate(p,'='); + name = p; + *q++ = '\0'; + value = q; + /* Set up p for next iteration */ + p = next; + nclistpush(pairs,strdup(name)); + nclistpush(pairs,strdup(value)); } - /* Guarantee null term */ - ncprops->netcdfver[sizeof(ncprops->netcdfver)-1] = '\0'; - ncprops->hdf5ver[sizeof(ncprops->hdf5ver)-1] = '\0'; done: - if(propdata != NULL) free(propdata); + if(text) free(text); return ret; } + +/* Utility to transfer a string to a buffer with escaping */ +static void +escapify(NCbytes* buffer, const char* s) +{ + const char* p; + for(p=s;*p;p++) { + if(strchr(ESCAPECHARS,*p) != NULL) + ncbytesappend(buffer,'\\'); + ncbytesappend(buffer,*p); + } +} + +/* Utility to copy contents of the dfalt into an NCPROPINFO object */ +static int +propinfo_default(struct NCPROPINFO* dst, const struct NCPROPINFO* dfalt) +{ + int i; + if(dst->properties == NULL) { + dst->properties = nclistnew(); + if(dst->properties == NULL) return NC_ENOMEM; + } + dst->version = dfalt->version; + for(i=0;iproperties);i++) { + char* s = nclistget(dfalt->properties,i); + s = strdup(s); + if(s == NULL) return NC_ENOMEM; + nclistpush(dst->properties,s); + } + return NC_NOERR; +} + /** - * @internal Get properties attribure. + * @internal Build _NCProperties attribute value. * - * @param h5 Pointer to HDF5 file info struct. + * Convert a NCPROPINFO instance to a single string. + * + * @param info Properties info. + * @param propdatap Pointer that gets properties string. * * @return ::NC_NOERR No error. + * @return ::NC_EINVAL failed. * @author Dennis Heimbigner */ -static int -NC4_get_propattr(NC_FILE_INFO_T* h5) +int +NC4_buildpropinfo(struct NCPROPINFO* info, char** propdatap) { - int ncstat = NC_NOERR; - size_t size; - H5T_class_t t_class; - hid_t grp = -1; - hid_t attid = -1; - hid_t aspace = -1; - hid_t atype = -1; - hid_t ntype = -1; - char* text = NULL; + int stat = NC_NOERR; + int i; + NCbytes* buffer = NULL; + char sversion[64]; - /* Get root group */ - grp = h5->root_grp->hdf_grpid; /* get root group */ - /* Try to extract the NCPROPS attribute */ - if(H5Aexists(grp,NCPROPS) > 0) { /* Does exist */ - attid = H5Aopen_name(grp, NCPROPS); - aspace = H5Aget_space(attid); /* dimensions of attribute data */ - atype = H5Aget_type(attid); - /* Verify that atype and size */ - t_class = H5Tget_class(atype); - if(t_class != H5T_STRING) {ncstat = NC_EATTMETA; goto done;} - size = H5Tget_size(atype); - if(size == 0) goto done; - text = (char*)malloc(size+1); - if(text == NULL) - {ncstat = NC_ENOMEM; goto done;} - HCHECK((ntype = H5Tget_native_type(atype, H5T_DIR_ASCEND))); - HCHECK((H5Aread(attid, ntype, text))); - /* Make sure its null terminated */ - text[size] = '\0'; - /* Try to parse text */ - ncstat = NC4_properties_parse(&h5->fileinfo->propattr,text); + if(info == NULL || info->version == 0 || propdatap == NULL) + {stat = NC_EINVAL; goto done;} + + *propdatap = NULL; + + buffer = ncbytesnew(); + if(!buffer) {stat = NC_ENOMEM; goto done;} + + /* start with version */ + ncbytescat(buffer,NCPVERSION); + ncbytesappend(buffer,'='); + snprintf(sversion,sizeof(sversion),"%d",info->version); + ncbytescat(buffer,sversion); + + for(i=0;iproperties);i+=2) { + char* value, *name; + name = nclistget(info->properties,i); + if(name == NULL) continue; + value = nclistget(info->properties,i+1); + ncbytesappend(buffer,NCPROPSSEP2); /* terminate last entry */ + escapify(buffer,name); + ncbytesappend(buffer,'='); + escapify(buffer,value); } + /* Force null termination */ + ncbytesnull(buffer); + *propdatap = ncbytesextract(buffer); + done: - if(attid >= 0) HCHECK((H5Aclose(attid))); - if(aspace >= 0) HCHECK((H5Sclose(aspace))); - if(ntype >= 0) HCHECK((H5Tclose(ntype))); - if(atype >= 0) HCHECK((H5Tclose(atype))); - if(text != NULL) free(text); - return ncstat; + if(buffer != NULL) ncbytesfree(buffer); + return stat; } +#if 0 /** * @internal Write the properties attribute to file. * @@ -169,19 +285,15 @@ NC4_get_propattr(NC_FILE_INFO_T* h5) * @author Dennis Heimbigner */ int -NC4_put_propattr(NC_FILE_INFO_T* h5) +NC4_put_ncproperties(NC_FILE_INFO_T* file) { int ncstat = NC_NOERR; - hid_t grp = -1; - hid_t attid = -1; - hid_t aspace = -1; - hid_t atype = -1; char* text = NULL; /* Get root group */ grp = h5->root_grp->hdf_grpid; /* get root group */ /* See if the NCPROPS attribute exists */ - if(H5Aexists(grp,NCPROPS) == 0) { /* Does not exist */ + if(H5Aexists(grp,NCPROPS) <= 0) { /* Does not exist */ ncstat = NC4_buildpropinfo(&h5->fileinfo->propattr,&text); if(text == NULL || ncstat != NC_NOERR) { goto done; @@ -205,85 +317,315 @@ NC4_put_propattr(NC_FILE_INFO_T* h5) if(atype >= 0) HCHECK((H5Tclose(atype))); return ncstat; } +#endif /** * @internal * - * @param h5 Pointer to HDF5 file info struct. - * @param init Initialization. + * Construct the provenance information for a newly created file + * using dfalt as the default. + * Note that creation of the _NCProperties attribute is deferred + * to the sync_netcdf4_file function. + * + * @param file Pointer to file object. + * @param dfalt * * @return ::NC_NOERR No error. + * [Note: other errors are reported via LOG()] * @author Dennis Heimbigner */ int -NC4_get_fileinfo(NC_FILE_INFO_T* h5, struct NCPROPINFO* init) +NC4_set_provenance(NC_FILE_INFO_T* file, const struct NCPROPINFO* dfalt) { int ncstat = NC_NOERR; + struct NCPROVENANCE* provenance = NULL; + int superblock = -1; + + assert(file->provenance == NULL); + provenance = calloc(1,sizeof(struct NCPROVENANCE)); + if(provenance == NULL) {ncstat = NC_ENOMEM; goto done;} - /* Allocate the fileinfo in h5 */ - h5->fileinfo = (struct NCFILEINFO*)calloc(1,sizeof(struct NCFILEINFO)); - if(h5->fileinfo == NULL) + /* Initialize from the default */ + provenance->propattr.version = globalpropinfo.version; + provenance->propattr.properties = nclistnew(); + if(provenance->propattr.properties == NULL) {ncstat = NC_ENOMEM; goto done;} + + /* Get the superblock number */ + if((ncstat = NC4_hdf5get_superblock(file,&superblock))) + goto done; + provenance->superblockversion = superblock; - /* Get superblock version */ - NCHECK((ncstat = NC4_hdf5get_superblock(h5,&h5->fileinfo->superblockversion))); - /* Get properties attribute if not already defined */ - if(init == NULL) { - NCHECK((ncstat = NC4_get_propattr(h5))); - } else { /*init != NULL*/ - h5->fileinfo->propattr = *init; /* Initialize */ + /* add in the dfalt values */ + if(dfalt != NULL) { + int i; + for(i=0;iproperties);i++) { + nclistpush(provenance->propattr.properties,nclistget(dfalt->properties,i)); + } } + done: + if(ncstat) { + LOG((0,"Could not create _NCProperties attribute")); + (void)NC4_free_provenance(provenance); + } else + file->provenance = provenance; + return NC_NOERR; +} + +/** + * @internal + * + * Construct the provenance information for a newly opened file + * Using the specified _NCProperties value. If NULL, then + * initialize using dfalt. + * + * @param file Pointer to file object. + * @param propstring The contents of _NCProperties + * @param dfalt + * + * @return ::NC_NOERR No error. + * @return ::NC_ENOMEM + * @return ::NC_EINVAL + * @author Dennis Heimbigner + */ +int +NC4_get_provenance(NC_FILE_INFO_T* file, const char* propstring, const struct NCPROPINFO* dfalt) +{ + int ncstat = NC_NOERR; + struct NCPROVENANCE* provenance; + char *name = NULL; + char *value = NULL; + int v = 0; + int superblock = -1; + + assert(file->provenance == NULL); + if((file->provenance = calloc(1,sizeof(struct NCPROVENANCE))) == NULL) + {ncstat = NC_ENOMEM; goto done;} + provenance = file->provenance; + if((provenance->propattr.properties = nclistnew()) == NULL) + {ncstat = NC_ENOMEM; goto done;} + + /* Set the superblock */ + if((ncstat = NC4_hdf5get_superblock(file,&superblock))) + goto done; + provenance->superblockversion = superblock; + + if(propstring == NULL) { + /* Use dfalt */ + if((ncstat=propinfo_default(&provenance->propattr,dfalt))) + goto done; + } else { + NClist* list = provenance->propattr.properties; + if((ncstat=properties_parse(propstring,list))) + goto done; + /* Check the version and remove from properties list*/ + if(nclistlength(list) < 2) + {ncstat = NC_EINVAL; goto done;} /* bad _NCProperties attribute */ + /* Extract the purported version=... */ + name = nclistremove(list,0); + value = nclistremove(list,0); + if(strcmp(name,NCPVERSION) == 0) { + if(sscanf(value,"%d",&v) != 1) + {ncstat = NC_EINVAL; goto done;} /* illegal version */ + if(v <= 0 || v > NCPROPS_VERSION) + {ncstat = NC_EINVAL; goto done;} /* unknown version */ + provenance->propattr.version = v; + } else + {ncstat = NC_EINVAL; goto done;} /* bad _NCProperties attribute */ +#if 0 + /* Now, rebuild from version 1 to version 2 if necessary */ + if(provenance->propattr.version == 1) { + int i; + for(i=0;ipropattr.properties) + nclistfreeall(prov->propattr.properties); + free(prov); + return NC_NOERR; +} - if(info == NULL || info->version == 0) return NC_EINVAL; - if(propdatap == NULL) - return NC_NOERR; - *propdatap = NULL; +/* HDF5 Specific attribute read/write of _NCProperties */ +int +NC4_read_ncproperties(NC_FILE_INFO_T* h5) +{ + int retval = NC_NOERR; + hid_t hdf5grpid = -1; + hid_t attid = -1; + hid_t aspace = -1; + hid_t atype = -1; + hid_t ntype = -1; + char* text = NULL; + H5T_class_t t_class; + hsize_t size; + + hdf5grpid = h5->root_grp->hdf_grpid; - /* compute attribute length */ - total = 0; - total += strlen(NCPVERSION); - total += strlen("=00000000"); - if(strlen(info->netcdfver) > 0) { - total += 1; /*|NCPROPSEP|*/ - total += strlen(NCPNCLIBVERSION); - total += strlen("="); - total += strlen(info->netcdfver); + if(H5Aexists(hdf5grpid,NCPROPS) <= 0) { /* Does not exist */ + /* File did not contain a _NCProperties attribute */ + retval=NC4_get_provenance(h5,NULL,&globalpropinfo); + goto done; } - if(strlen(info->hdf5ver) > 0) { - total += 1; /*|NCPROPSEP|*/ - total += strlen(NCPHDF5LIBVERSION); - total += strlen("="); - total += strlen(info->hdf5ver); + + /* NCPROPS Attribute exists, make sure it is legitimate */ + attid = H5Aopen_name(hdf5grpid, NCPROPS); + assert(attid > 0); + aspace = H5Aget_space(attid); + atype = H5Aget_type(attid); + /* Verify atype and size */ + t_class = H5Tget_class(atype); + if(t_class != H5T_STRING) + {retval = NC_EINVAL; goto done;} + size = H5Tget_size(atype); + if(size == 0) + {retval = NC_EINVAL; goto done;} + text = (char*)malloc(1+(size_t)size); + if(text == NULL) + {retval = NC_ENOMEM; goto done;} + if((ntype = H5Tget_native_type(atype, H5T_DIR_DEFAULT)) < 0) + {retval = NC_EHDFERR; goto done;} + if((H5Aread(attid, ntype, text)) < 0) + {retval = NC_EHDFERR; goto done;} + /* Make sure its null terminated */ + text[(size_t)size] = '\0'; + /* Process the _NCProperties value */ + if((retval = NC4_get_provenance(h5, text, &globalpropinfo))) + goto done; + +done: + if(text != NULL) free(text); + /* Close out the HDF5 objects */ + if(attid > 0 && H5Aclose(attid) < 0) retval = NC_EHDFERR; + if(aspace > 0 && H5Sclose(aspace) < 0) retval = NC_EHDFERR; + if(atype > 0 && H5Tclose(atype) < 0) retval = NC_EHDFERR; + if(ntype > 0 && H5Tclose(ntype) < 0) retval = NC_EHDFERR; + + /* For certain errors, actually fail, else log that attribute was invalid and ignore */ + if(retval != NC_ENOMEM && retval != NC_EHDFERR) { + LOG((0,"Invalid _NCProperties attribute")); + retval = NC_NOERR; } - propdata = (char*)malloc(total+1); - if(propdata == NULL) - return NC_ENOMEM; - snprintf(propdata,total+1, - "%s=%d|%s=%s|%s=%s", - NCPVERSION,info->version, - NCPNCLIBVERSION,info->netcdfver, - NCPHDF5LIBVERSION,info->hdf5ver); - /* Force null termination */ - propdata[total] = '\0'; - *propdatap = propdata; + return retval; +} - return NC_NOERR; +int +NC4_write_ncproperties(NC_FILE_INFO_T* h5) +{ + int retval = NC_NOERR; + hid_t hdf5grpid = -1; + hid_t attid = -1; + hid_t aspace = -1; + hid_t atype = -1; + char* text = NULL; + size_t len = 0; + + /* If the file is read-only, return an error. */ + if (h5->no_write) + {retval = NC_EPERM; goto done;} + + hdf5grpid = h5->root_grp->hdf_grpid; + + if(H5Aexists(hdf5grpid,NCPROPS) > 0) /* Already exists, no overwrite */ + goto done; + + /* Build the attribute string */ + if((retval = NC4_buildpropinfo(&h5->provenance->propattr,&text))) + goto done; + + /* Build the HDF5 string type */ + if ((atype = H5Tcopy(H5T_C_S1)) < 0) + {retval = NC_EHDFERR; goto done;} + if (H5Tset_strpad(atype, H5T_STR_NULLTERM) < 0) + {retval = NC_EHDFERR; goto done;} + if(H5Tset_cset(atype, H5T_CSET_ASCII) < 0) + {retval = NC_EHDFERR; goto done;} + + /* Create NCPROPS attribute */ + + len = strlen(text); + if(H5Tset_size(atype, len) < 0) + {retval = NC_EFILEMETA; goto done;} + if((aspace = H5Screate(H5S_SCALAR)) < 0) + {retval = NC_EFILEMETA; goto done;} + if ((attid = H5Acreate(hdf5grpid, NCPROPS, atype, aspace, H5P_DEFAULT)) < 0) + {retval = NC_EFILEMETA; goto done;} + if (H5Awrite(attid, atype, text) < 0) + {retval = NC_EFILEMETA; goto done;} + +done: + if(text != NULL) free(text); + /* Close out the HDF5 objects */ + if(attid > 0 && H5Aclose(attid) < 0) retval = NC_EHDFERR; + if(aspace > 0 && H5Sclose(aspace) < 0) retval = NC_EHDFERR; + if(atype > 0 && H5Tclose(atype) < 0) retval = NC_EHDFERR; + + /* For certain errors, actually fail, else log that attribute was invalid and ignore */ + switch (retval) { + case NC_ENOMEM: + case NC_EHDFERR: + case NC_EPERM: + case NC_EFILEMETA: + case NC_NOERR: + break; + default: + LOG((0,"Invalid _NCProperties attribute")); + retval = NC_NOERR; + break; + } + return retval; } + +/* Debugging */ + +void +ncprintpropinfo(struct NCPROPINFO* info) +{ + int i; + fprintf(stderr,"[%p] version=%d\n",info,info->version); + for(i=0;iproperties);i+=2) { + char* name = nclistget(info->properties,i); + char* value = nclistget(info->properties,i+1); + fprintf(stderr,"\t[%d] name=|%s| value=|%s|\n",i,name,value); + } +} + +void +ncprintprovenance(struct NCPROVENANCE* prov) +{ + fprintf(stderr,"[%p] superblockversion=%d\n",prov,prov->superblockversion); + ncprintpropinfo(&prov->propattr); +} + diff --git a/libhdf5/nc4mem.c b/libhdf5/nc4mem.c index 6b50689a34..c9771cd758 100644 --- a/libhdf5/nc4mem.c +++ b/libhdf5/nc4mem.c @@ -31,8 +31,6 @@ #define HDrealloc(M,Z) realloc(M,Z) #endif /* HDrealloc */ -extern hid_t NC4_image_init(NC_FILE_INFO_T* h5); - int NC4_open_image_file(NC_FILE_INFO_T* h5) { @@ -43,13 +41,13 @@ NC4_open_image_file(NC_FILE_INFO_T* h5) if(h5->mem.memio.memory == NULL || h5->mem.memio.size == 0) {stat = NC_EINVAL; goto done;} - /* Figure out the flags */ - h5->mem.flags = 0; + /* Figure out the image flags */ + h5->mem.imageflags = 0; if(h5->mem.locked) { - h5->mem.flags |= (H5LT_FILE_IMAGE_DONT_COPY | H5LT_FILE_IMAGE_DONT_RELEASE); + h5->mem.imageflags |= (H5LT_FILE_IMAGE_DONT_COPY | H5LT_FILE_IMAGE_DONT_RELEASE); } if(!h5->no_write) - h5->mem.flags |= H5LT_FILE_IMAGE_OPEN_RW; + h5->mem.imageflags |= H5LT_FILE_IMAGE_OPEN_RW; /* Create the file but using our version of H5LTopen_file_image */ hdfid = NC4_image_init(h5); @@ -67,21 +65,22 @@ int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t initialsz) { int stat = NC_NOERR; - int hdfid; + hid_t hdfid; /* Create the file but using our version of H5LTopen_file_image */ h5->mem.created = 1; h5->mem.initialsize = initialsz; + h5->mem.imageflags |= H5LT_FILE_IMAGE_OPEN_RW; hdfid = NC4_image_init(h5); if(hdfid < 0) {stat = NC_EHDFERR; goto done;} - /* Remember HDF5 file identifier. */ ((NC_HDF5_FILE_INFO_T *)h5->format_file_info)->hdfid = hdfid; done: return stat; } +#if 0 int NC4_extract_file_image(NC_FILE_INFO_T* h5) { @@ -89,14 +88,19 @@ NC4_extract_file_image(NC_FILE_INFO_T* h5) hid_t fapl; herr_t herr; NC_memio mem; + hid_t hdfid = -1; assert(h5 && h5->format_file_info && !h5->no_write); - /* Get the file access property list */ - fapl = h5->mem.fapl; - if(fapl < 0) + if((hdfid = ((NC_HDF5_FILE_INFO_T *)h5->format_file_info)->hdfid) < 0) {stat = NC_EHDFERR; goto done;} + /* The h5->mem.memio struct should contain what we want */ + + + /* Get the file access property list */ + if((fapl = H5Fget_access_plist(hdfid)) < 0) + {stat = NC_EHDFERR; goto done;} memset(&mem,0,sizeof(mem)); herr = H5Pget_file_image(fapl, &mem.memory, &mem.size); if(herr < 0) @@ -106,7 +110,8 @@ NC4_extract_file_image(NC_FILE_INFO_T* h5) /* Close FAPL */ if (H5Pclose(fapl) < 0) {stat = NC_EHDFERR; goto done;} - h5->mem.fapl = 0; + done: return stat; } +#endif diff --git a/libhdf5/nc4memcb.c b/libhdf5/nc4memcb.c index 84cb00f79b..2af02e4aff 100644 --- a/libhdf5/nc4memcb.c +++ b/libhdf5/nc4memcb.c @@ -19,7 +19,7 @@ /* * @internal Inmemory support * - * This code is derived from H5Lt.c#H5LTopen_file_image. + * This code is derived from H5LT.c#H5LTopen_file_image. * In order to make the netcdf inmemory code work, it is necessary * to modify some of the callback functions; specifically * image_malloc, image_realloc, and image_memcpy. @@ -96,7 +96,8 @@ #endif #undef TRACE -#define CATCH +#undef CATCH +#undef TRACE_UDATA #ifdef TRACE #define CATCH @@ -104,18 +105,23 @@ #ifdef TRACE #include -static void trace(const char* fcn, void* _udata, ...); -static void traceend(const char* fcn, void* _udata); +static void trace(const char* fcn, H5FD_file_image_op_t op, void* _udata, ...); +static void traceend(const char* fcn, void* _udata, uintptr_t retval); +static const char* traceop(H5FD_file_image_op_t op); +static char* traceflags(int flags); + /* In case we do not have variadic macros */ -#define TRACE1(fcn,udata,x1) trace(fcn,udata,x1) -#define TRACE2(fcn,udata,x1,x2) trace(fcn,udata,x1,x2) -#define TRACE3(fcn,udata,x1,x2,x3) trace(fcn,udata,x1,x2,x3) -#define TRACEEND(fcn,udata) traceend(fcn,udata); +#define TRACE0(fcn,op,udata) trace(fcn,op,udata) +#define TRACE1(fcn,op,udata,x1) trace(fcn,op,udata,x1) +#define TRACE2(fcn,op,udata,x1,x2) trace(fcn,op,udata,x1,x2) +#define TRACE3(fcn,op,udata,x1,x2,x3) trace(fcn,op,udata,x1,x2,x3) +#define TRACEEND(fcn,udata,retval) traceend(fcn,udata,(uintptr_t)retval); #else /*!TRACE*/ -#define TRACE1(fcn,udata,x1) -#define TRACE2(fcn,udata,x1,x2) -#define TRACE3(fcn,udata,x1,x2,x3) -#define TRACEEND(fcn,udata) +#define TRACE0(fcn,op,udata) +#define TRACE1(fcn,op,udata,x1) +#define TRACE2(fcn,op,udata,x1,x2) +#define TRACE3(fcn,op,udata,x1,x2,x3) +#define TRACEEND(fcn,udata,retval) #endif #ifdef CATCH @@ -235,20 +241,26 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; void * return_value = NULL; - TRACE1("malloc",_udata, size); + TRACE1("malloc", file_image_op, _udata, size); +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif switch ( file_image_op ) { /* the app buffer is "copied" to only one FAPL. Afterwards, FAPLs can be "copied" */ case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: + /* It appears that the fapl memory is never created as we use it, so + we expect the udata ptr to be either null or same as the app buffer.*/ + assert(udata->fapl_image_ptr == NULL || udata->fapl_image_ptr == udata->app_image_ptr); + if (udata->app_image_ptr == NULL) goto out; if (udata->app_image_size != size) goto out; - if (udata->fapl_image_ptr != NULL) + if (udata->fapl_image_ptr != NULL) goto out; if (udata->fapl_image_size != 0) goto out; @@ -259,6 +271,7 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata udata->fapl_image_size = udata->app_image_size; return_value = udata->fapl_image_ptr; udata->fapl_ref_count++; + return_value = udata->fapl_image_ptr; break; case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: @@ -273,10 +286,10 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata udata->fapl_ref_count++; break; - case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET: - if(!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET: + if (udata->fapl_image_ptr == NULL) goto out; - /* fake the malloc by returning the original memory */ + /* fake the malloc by returning the current memory */ return_value = udata->fapl_image_ptr; break; @@ -310,7 +323,7 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata goto out; } /* end switch */ - TRACEEND("malloc",_udata); + TRACEEND("malloc",_udata,return_value); return(return_value); @@ -343,11 +356,13 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_ { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; - TRACE3("memcpy",_udata,dest,src,size); + TRACE3("memcpy", file_image_op, _udata,dest,src,size); +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif switch(file_image_op) { case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: @@ -361,6 +376,15 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_ goto out; if (udata->fapl_ref_count == 0) goto out; + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) { + if(src != dest) { + memcpy(dest,src,size); +#ifdef TRACE + fprintf(stderr,"\t>>>> memcpy(%p,%p,%ld)\n", + dest,src,(unsigned long)size); +#endif + } + } break; case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: @@ -406,10 +430,7 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_ goto out; } /* end switch */ -#ifdef TRACE - fprintf(stderr,"trace: memcpy: dest=%p\n",dest); -#endif - TRACEEND("memcpy",_udata); + TRACEEND("memcpy",_udata,dest); return(dest); out: @@ -433,61 +454,85 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_ * *------------------------------------------------------------------------- */ + +/* This warning is from H5FDcore.c" + Be careful of non-Posix realloc() that doesn't understand + what to do when the first argument is null. +*/ + +/* Modified: +1. If the realloc new size is <= existing size, + then pretend we did a realloc and return success. + This avoids unneccessary heap operations. +2. If the H5LT_FILE_IMAGE_DONT_COPY or + H5LT_FILE_IMAGE_DONT_RELEASE flag is set and the + realloc new size is > existing size, then fail + because the realloc() call may change the address + of the buffer. The new address cannot be + communicated to the application to release it. +3. Otherwise, use realloc(). Note that this may have the + side effect of freeing the previous memory chunk. +*/ static void * local_image_realloc(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *_udata) { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; void * return_value = NULL; - TRACE2("realloc",_udata, ptr, size); + TRACE2("realloc", file_image_op, _udata, ptr, size); +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif /* realloc() is not allowed if the image is open in read-only mode */ if (!(udata->flags & H5LT_FILE_IMAGE_OPEN_RW)) goto out; - if (file_image_op == H5FD_FILE_IMAGE_OP_FILE_RESIZE) { - if (udata->vfd_image_ptr != ptr) - goto out; + /* DONT_COPY => DONT_RELEASE */ + assert(((udata->flags & H5LT_FILE_IMAGE_DONT_COPY)?(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE):1)); - if (udata->vfd_ref_count != 1) - goto out; + /* Note that the fapl pointer is never realloc'd */ + if (file_image_op == H5FD_FILE_IMAGE_OP_FILE_RESIZE) { - /* Modified: - 1. If the realloc new size is <= existing size, - then pretend we did a realloc and return success. - This avoids unneccessary heap operations. - 2. If the H5LT_FILE_IMAGE_DONT_COPY or - H5LT_FILE_IMAGE_DONT_RELEASE flag is set and the - realloc new size is > existing size, then fail - because the realloc() call may change the address - of the buffer. The new address cannot be - communicated to the application to release it. - 3. Otherwise, use realloc(). Note that this may have the - side effect of freeing the previous memory chunk. - */ - if(size <= udata->vfd_image_size) { - /* Ok, pretend we did a realloc */ - } else if((udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE) - || (udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) { - goto out; /* realloc MAY violate these flags */ - } else { - if (NULL == (udata->vfd_image_ptr = HDrealloc(ptr, size))) - goto out; - udata->vfd_image_size = size; + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) { /* buffer modification is allowed */ + /* Divide code based on whether ptr == NULL or not */ + if(ptr == NULL) { + /* From realloc man page: If ptr is NULL, then the call is equivalent to malloc(size), + for all values of size; if size is equal to zero, and ptr is not NULL, then the call + is equivalent to free(ptr). */ + udata->vfd_image_ptr = malloc(size); + udata->vfd_ref_count++; + } else { /* ptr != NULL */ + if(udata->vfd_image_ptr != ptr) + goto out; + if (udata->vfd_ref_count != 1) + goto out; + udata->vfd_image_ptr = realloc(ptr, size); + if(NULL == udata->vfd_image_ptr) { + LOG((0,"image_realloc: unable to allocate memory block of size: %lu bytes",(unsigned long)size)); + goto out; + } +#ifdef TRACE + fprintf(stderr,"\t>>>> realloc(%p,%ld)=>%p\n",ptr,(unsigned long)size,udata->vfd_image_ptr); +#endif + } + udata->vfd_image_size = size; + } else { /* Cannot realloc, so fake it */ + if(size <= udata->vfd_image_size) { + /* Ok, pretend we did a realloc but just change size*/ + udata->vfd_image_size = size; + } else + goto out; } return_value = udata->vfd_image_ptr; } /* end if */ else goto out; -#ifdef TRACE - fprintf(stderr,"trace: realloc: return=%p\n",return_value); -#endif - TRACEEND("realloc",_udata); + TRACEEND("realloc",_udata,return_value); return(return_value); out: @@ -515,11 +560,13 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; - TRACE1("free",_udata,ptr); + TRACE1("free", file_image_op, _udata, ptr); +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif switch(file_image_op) { case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE: @@ -530,14 +577,15 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) udata->fapl_ref_count--; - /* release the shared buffer only if indicated by the respective flag and there are no outstanding references */ - if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 && - !(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) { - free(udata->fapl_image_ptr); - udata->app_image_ptr = NULL; - udata->fapl_image_ptr = NULL; - udata->vfd_image_ptr = NULL; - } /* end if */ + /* For the way we use it, it should still be the case that + the fapl pointer is same as image_ptr, so we do not need + to do anything */ + assert(udata->fapl_image_ptr == udata->app_image_ptr); + /* clean up */ +#if 0 + udata->app_image_ptr = NULL; + udata->fapl_image_ptr = NULL; +#endif break; case H5FD_FILE_IMAGE_OP_FILE_CLOSE: @@ -548,17 +596,17 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) udata->vfd_ref_count--; - /* release the shared buffer only if indicated by the respective flag and there are no outstanding references */ - if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 && - !(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) { - free(udata->vfd_image_ptr); - udata->app_image_ptr = NULL; - udata->fapl_image_ptr = NULL; - udata->vfd_image_ptr = NULL; - } /* end if */ + udata->h5->mem.memio.memory = udata->vfd_image_ptr; + udata->h5->mem.memio.size = udata->vfd_image_size; + /* clean up */ +#if 0 + udata->app_image_ptr = NULL; + udata->fapl_image_ptr = NULL; +#endif + udata->vfd_image_ptr = NULL; break; - /* added unused labels to keep the compiler quite */ + /* added unused labels to keep the compiler quiet */ case H5FD_FILE_IMAGE_OP_NO_OP: case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: @@ -569,7 +617,7 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) goto out; } /* end switch */ - TRACEEND("free",_udata); + TRACEEND("free",_udata,1); return(SUCCEED); out: @@ -598,18 +646,24 @@ local_udata_copy(void *_udata) { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + TRACE0("udata_copy", 0, _udata); + +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif - if (udata->ref_count == 0) - goto out; - + /* never copy */ + if (udata->ref_count == 0) + goto out; udata->ref_count++; + TRACEEND("udata_copy",udata,1); return(udata); out: + TRACEFAIL("udata_copy"); return NULL; } /* end udata_copy */ @@ -634,63 +688,75 @@ local_udata_free(void *_udata) { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + TRACE0("udata_free", 0, _udata); + +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif if (udata->ref_count == 0) goto out; - udata->ref_count--; /* checks that there are no references outstanding before deallocating udata */ - if (udata->ref_count == 0 && udata->fapl_ref_count == 0 && - udata->vfd_ref_count == 0) + if (udata->ref_count == 0 && udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0) { free(udata); - +#ifdef TRACE_UDATA + fprintf(stderr,"\t>>>> freed: udata=%p\n",udata); +#endif + udata = NULL; + } + TRACEEND("udata_free",udata,1); return(SUCCEED); out: + TRACEFAIL("udata_free"); return(FAIL); } /* end udata_free */ + /* End of callbacks definitions for file image operations */ hid_t NC4_image_init(NC_FILE_INFO_T* h5) { hid_t fapl = -1, file_id = -1; /* HDF5 identifiers */ - unsigned file_open_flags;/* Flags for image open */ + unsigned file_open_flags = 0;/* Flags for hdf5 open */ char file_name[64]; /* Filename buffer */ size_t alloc_incr; /* Buffer allocation increment */ size_t min_incr = 65536; /* Minimum buffer increment */ double buf_prcnt = 0.1f; /* Percentage of buffer size to set as increment */ size_t buf_size = 0; - void* buf_ptr = h5->mem.memio.memory; - unsigned flags = h5->mem.flags; + void* buf_ptr = NULL; + unsigned imageflags; + int create = 0; + H5LT_file_image_ud_t *udata = NULL; /* Pointer to udata structure */ - static long file_name_counter; H5FD_file_image_callbacks_t callbacks = {&local_image_malloc, &local_image_memcpy, &local_image_realloc, &local_image_free, &local_udata_copy, &local_udata_free, (void *)NULL}; + static long file_name_counter; + + imageflags = h5->mem.imageflags; + create = h5->mem.created; + /* check arguments */ - if (buf_ptr == NULL) { - if(h5->mem.created) { + if (h5->mem.memio.memory == NULL) { + if(create) { if(h5->mem.memio.size == 0) h5->mem.memio.size = DEFAULT_CREATE_MEMSIZE; h5->mem.memio.memory = malloc(h5->mem.memio.size); } else goto out; /* open requires an input buffer */ - } - // reset + } else if(h5->mem.memio.size == 0) + goto out; + + /* alias for convenience */ buf_size = h5->mem.memio.size; buf_ptr = h5->mem.memio.memory; - /* validate */ - if (buf_ptr == NULL || buf_size == 0) - goto out; - if (flags & (unsigned)~(H5LT_FILE_IMAGE_ALL)) - goto out; /* Create FAPL to transmit file image */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) @@ -708,14 +774,10 @@ NC4_image_init(NC_FILE_INFO_T* h5) if (H5Pset_fapl_core(fapl, alloc_incr, FALSE) < 0) goto out; - /* Set callbacks for file image ops ONLY if the file image is NOT copied */ - if (flags & H5LT_FILE_IMAGE_DONT_COPY) - // if (0) + /* Set callbacks for file image ops always */ { - H5LT_file_image_ud_t *udata; /* Pointer to udata structure */ - /* Allocate buffer to communicate user data to callbacks */ - if (NULL == (udata = (H5LT_file_image_ud_t *)malloc(sizeof(H5LT_file_image_ud_t)))) + if (NULL == (udata = (H5LT_file_image_ud_t *)calloc(1,sizeof(H5LT_file_image_ud_t)))) goto out; /* Initialize udata with info about app buffer containing file image and flags */ @@ -727,7 +789,7 @@ NC4_image_init(NC_FILE_INFO_T* h5) udata->vfd_image_ptr = NULL; udata->vfd_image_size = 0; udata->vfd_ref_count = 0; - udata->flags = flags; + udata->flags = imageflags; udata->ref_count = 1; /* corresponding to the first FAPL */ udata->h5 = h5; @@ -735,28 +797,27 @@ NC4_image_init(NC_FILE_INFO_T* h5) callbacks.udata = (void *)udata; /* Set file image callbacks */ - if (H5Pset_file_image_callbacks(fapl, &callbacks) < 0) { - free(udata); + if (H5Pset_file_image_callbacks(fapl, &callbacks) < 0) goto out; - } /* end if */ } /* Assign file image in user buffer to FAPL */ if (H5Pset_file_image(fapl, buf_ptr, buf_size) < 0) goto out; - /* set file open flags */ - if (flags & H5LT_FILE_IMAGE_OPEN_RW) + /* define a unique file name */ + snprintf(file_name, (sizeof(file_name) - 1), "file_image_%ld", file_name_counter++); + + /* set file open/create flags */ + if(create) + file_open_flags = H5F_ACC_TRUNC; /* H5Fcreate does not like H5F_ACC_RDWR */ + else if (imageflags & H5LT_FILE_IMAGE_OPEN_RW) file_open_flags = H5F_ACC_RDWR; else file_open_flags = H5F_ACC_RDONLY; - /* define a unique file name */ - snprintf(file_name, (sizeof(file_name) - 1), "file_image_%ld", file_name_counter++); - /* Assign file image in FAPL to the core file driver */ - if(h5->mem.created) { - file_open_flags |= H5F_ACC_TRUNC; + if(create) { if ((file_id = H5Fcreate(file_name, file_open_flags, H5P_DEFAULT, fapl)) < 0) goto out; } else { @@ -764,76 +825,133 @@ NC4_image_init(NC_FILE_INFO_T* h5) goto out; } - h5->mem.fapl = fapl; - - /* Return file identifier */ - return file_id; + h5->mem.udata = udata; -out: +done: + /* Reclaim the fapl object */ H5E_BEGIN_TRY { if(fapl >= 0) H5Pclose(fapl); } H5E_END_TRY; - return -1; + /* Reclaim udata */ + /* Return file identifier */ + return file_id; + +out: + if(udata != NULL) free(udata); + file_id = -1; + goto done; } /* end H5LTopen_file_image() */ +void +NC4_image_finalize(void* _udata) +{ + if(_udata != NULL) { + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t*)_udata; +#if 0 + if(udata->app_image_ptr != NULL) free(udata->app_image_ptr); + if(udata->fapl_image_ptr != NULL) free(udata->fapl_image_ptr); + if(udata->vfd_image_ptr != NULL) free(udata->vfd_image_ptr); +#endif + free(udata); + } +} + #ifdef TRACE -static void +static char* printudata(H5LT_file_image_ud_t* udata) { - if(udata == NULL) return; - fprintf(stderr," flags=0x%0x",udata->flags); - fprintf(stderr," flags=0x%0x",udata->flags); - fprintf(stderr," ref_count=%d",udata->ref_count); - fprintf(stderr," app=(%p,%lld)",udata->app_image_ptr,(long long)udata->app_image_size); - fprintf(stderr," fapl=(%p,%lld)[%d]",udata->fapl_image_ptr,(long long)udata->fapl_image_size,udata->fapl_ref_count); - fprintf(stderr," vfd=(%p,%lld)[%d]",udata->vfd_image_ptr,(long long)udata->vfd_image_size,udata->vfd_ref_count); + char buf[8192]; + char tmp[8192]; + char* flags = ""; + + buf[0] = '\0'; + if(udata == NULL) return strdup(""); + strlcat(buf,"flags=",sizeof(buf)); + flags = traceflags(udata->flags); + strlcat(buf,flags,sizeof(buf)); + if(flags != NULL) free(flags); + snprintf(tmp,sizeof(tmp)," ref_count=%d",udata->ref_count); + strlcat(buf,tmp,sizeof(tmp)); + snprintf(tmp,sizeof(tmp)," app=(%p,%lld)",udata->app_image_ptr,(long long)udata->app_image_size); + strlcat(buf,tmp,sizeof(tmp)); + snprintf(tmp,sizeof(tmp)," fapl=(%p,%lld)[%d]",udata->fapl_image_ptr,(long long)udata->fapl_image_size,udata->fapl_ref_count); + strlcat(buf,tmp,sizeof(tmp)); + snprintf(tmp,sizeof(tmp)," vfd=(%p,%lld)[%d]",udata->vfd_image_ptr,(long long)udata->vfd_image_size,udata->vfd_ref_count); + strlcat(buf,tmp,sizeof(tmp)); + return strdup(buf); } -static void trace(const char* fcn, void* _udata, ...) +static void +trace(const char* fcn, H5FD_file_image_op_t op, void* _udata, ...) { H5LT_file_image_ud_t *udata = NULL; va_list ap; + char buf[16000]; + char tmp[8192]; + char* ud; va_start(ap, _udata); /* Requires the last fixed parameter (to get the address) */ udata = (H5LT_file_image_ud_t *)_udata; + buf[0] = '\0'; + snprintf(tmp,sizeof(tmp),"trace [ %s: op=%s: ",fcn,traceop(op)); + strlcat(buf,tmp,sizeof(tmp)); if(strcmp("malloc",fcn)==0) { size_t size = va_arg(ap,size_t); - fprintf(stderr,"trace: %s: size=%lld udata=",fcn,(long long)size); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"size=%lld",(long long)size); } else if(strcmp("realloc",fcn)==0) { void* ptr = va_arg(ap,void*); size_t size = va_arg(ap,size_t); - fprintf(stderr,"trace: %s: ptr=%p, size=%lld udata=",fcn,ptr,(long long)size); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"ptr=%p, size=%lld",ptr,(long long)size); } else if(strcmp("free",fcn)==0) { void* ptr = va_arg(ap,void*); - fprintf(stderr,"trace: %s: ptr=%p udata=",fcn,ptr); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"ptr=%p",ptr); } else if(strcmp("memcpy",fcn)==0) { void* dest = va_arg(ap,void*); void* src = va_arg(ap,void*); size_t size = va_arg(ap,size_t); - fprintf(stderr,"trace: %s: dest=%p src=%p size=%lld udata=",fcn,dest,src,(long long)size); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"dest=%p, src=%p, size=%lld",dest,src,(long long)size); + } else if(strcmp("udata_copy",fcn)==0) { +#ifdef TRACE_UDATA + snprintf(tmp,sizeof(tmp),"udata=%p",udata); +#endif + } else if(strcmp("udata_free",fcn)==0) { +#ifdef TRACE_UDATA + snprintf(tmp,sizeof(tmp),"udata=%p",udata); +#endif } else { - fprintf(stderr,"unknown fcn: %s\n",fcn); + snprintf(tmp,sizeof(tmp),"unknown fcn: %s",fcn); } + strlcat(buf,tmp,sizeof(buf)); + ud = printudata(udata); + strlcat(buf,"\n\tudata=",sizeof(buf)); + strlcat(buf,ud,sizeof(tmp)); + free(ud); + strlcat(buf,"\n",sizeof(buf)); va_end(ap); + fprintf(stderr,"%s",buf); fflush(stderr); } -static void traceend(const char* fcn, void* _udata) +static void +traceend(const char* fcn, void* _udata, uintptr_t retval) { + char buf[16000]; + char tmp[8192]; + char* ud; + const char* tab = " "; + + buf[0] = '\0'; H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; - fprintf(stderr,"traceend: %s: udata=",fcn); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"%s]: retval=%p",tab,(void*)retval); + strlcat(buf,tmp,sizeof(buf)); + strlcat(buf," udata=",sizeof(buf)); + ud = printudata(udata); + strlcat(buf,ud,sizeof(tmp)); + free(ud); + strlcat(buf,"\n",sizeof(buf)); + fprintf(stderr,"%s",buf); fflush(stderr); } @@ -843,7 +961,68 @@ static void traceend(const char* fcn, void* _udata) static void tracefail(const char* fcn) { - fprintf(stderr,"fail: %s",fcn); + fprintf(stderr,"fail: %s\n",fcn); fflush(stderr); } #endif /*CATCH*/ + +#ifdef TRACE +static char* +traceflags(int flags) +{ + int i; + char buf[8192]; + char tmp[8192]; + buf[0] = '\0'; + for(i=0;i<16;i++) { + tmp[0] = '\0'; + if((flags & 1< 0) strlcat(tmp,"|",sizeof(tmp)); + switch(1< #include #include -#define access(path,mode) _access(path,mode) #endif #include "ncdispatch.h" @@ -116,6 +115,8 @@ static int memio_close(ncio* nciop, int); static int readfile(const char* path, NC_memio*); static int writefile(const char* path, NCMEMIO*); static int fileiswriteable(const char* path); +static int fileisreadable(const char* path); +static int fileexists(const char* path); /* Mnemonic */ #define DOOPEN 1 @@ -197,7 +198,7 @@ memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEM if(fIsSet(ioflags,NC_DISKLESS)) memio->diskless = 1; - if(fIsSet(ioflags,NC_INMEMORY) && !memio->diskless) + if(fIsSet(ioflags,NC_INMEMORY)) memio->inmemory = 1; if(fIsSet(ioflags,NC_WRITE) && !fIsSet(ioflags,NC_NOCLOBBER) && memio->diskless) memio->persist = 1; @@ -248,8 +249,8 @@ memio_create(const char* path, int ioflags, return status; if(memio->persist) { - /* Verify the file is writeable */ - if(!fileiswriteable(path)) + /* Verify the file is writeable or does not exist*/ + if(fileexists(path) && !fileiswriteable(path)) {status = EPERM; goto unwind_open;} } @@ -374,9 +375,11 @@ fprintf(stderr,"memio_open: initial memory: %lu/%lu\n",(unsigned long)memio->mem #endif if(memio->persist) { - /* Verify the file is writeable */ + /* Verify the file is writeable and exists */ + if(!fileexists(path)) + {status = ENOENT; goto unwind_open;} if(!fileiswriteable(path)) - {status = EPERM; goto unwind_open;} + {status = EACCES; goto unwind_open;} } /* Use half the filesize as the blocksize ; why? */ @@ -645,11 +648,39 @@ memio_extract(ncio* const nciop, size_t* sizep, void** memoryp) return status; } +/* Return 1 if file exists, 0 otherwise */ +static int +fileexists(const char* path) +{ + int ok; + /* See if the file exists at all */ + ok = NCaccess(path,ACCESS_MODE_EXISTS); + if(ok < 0) /* file does not exist */ + return 0; + return 1; +} + +/* Return 1 if file is writeable, return 0 otherwise; + assumes fileexists has been checked already */ static int fileiswriteable(const char* path) { int ok; - ok = access(path,O_RDWR); + /* if W is ok */ + ok = NCaccess(path,ACCESS_MODE_W); + if(ok < 0) + return 0; + return 1; +} + +/* Return 1 if file is READABLE, return 0 otherwise; + assumes fileexists has been checked already */ +static int +fileisreadable(const char* path) +{ + int ok; + /* if RW is ok */ + ok = NCaccess(path,ACCESS_MODE_R); if(ok < 0) return 0; return 1; @@ -714,14 +745,14 @@ writefile(const char* path, NCMEMIO* memio) size_t count = 0; char* p = NULL; - /* Open the file for writing*/ + /* Open/create the file for writing*/ #ifdef _MSC_VER - f = NCfopen(path,"rwb"); + f = NCfopen(path,"wb"); #else - f = NCfopen(path,"rw"); + f = NCfopen(path,"w"); #endif if(f == NULL) - {status = errno; goto done;} + {status = errno; goto done;} rewind(f); count = memio->size; p = memio->memory; diff --git a/libsrc/nc3internal.c b/libsrc/nc3internal.c index 79c6fbb1e8..4f689fd0bc 100644 --- a/libsrc/nc3internal.c +++ b/libsrc/nc3internal.c @@ -1044,7 +1044,7 @@ NC3_create(const char *path, int ioflags, int use_parallel, void* parameters, NC_Dispatch* dispatch, NC* nc) { - int status; + int status = NC_NOERR; void *xp = NULL; int sizeof_off_t = 0; NC3_INFO* nc3 = NULL; diff --git a/libsrc4/error4.c b/libsrc4/error4.c index 30f3d94ea2..bdd2345a55 100644 --- a/libsrc4/error4.c +++ b/libsrc4/error4.c @@ -32,7 +32,7 @@ extern int nc_log_level; nc_log(0, "this computer will explode in %d seconds", i); After the first arg (the severity), use the rest like a normal - printf statement. Output will appear on stdout. + printf statement. Output will appear on stderr. This function is heavily based on the function in section 15.5 of the C FAQ. */ @@ -50,18 +50,18 @@ nc_log(int severity, const char *fmt, ...) /* If the severity is zero, this is an error. Otherwise insert that many tabs before the message. */ if (!severity) - fprintf(stdout, "ERROR: "); + fprintf(stderr, "ERROR: "); for (t=0; tfileinfo->propattr.version == 0) + if(h5->provenance->propattr.version == 0) return NC_ENOTATT; if(mem_type == NC_NAT) mem_type = NC_CHAR; if(mem_type != NC_CHAR) return NC_ECHAR; if(filetypep) *filetypep = NC_CHAR; - stat = NC4_buildpropinfo(&h5->fileinfo->propattr, &propdata); + stat = NC4_buildpropinfo(&h5->provenance->propattr, &propdata); if(stat != NC_NOERR) return stat; len = strlen(propdata); if(lenp) *lenp = len; @@ -70,7 +70,7 @@ nc4_get_att_special(NC_FILE_INFO_T* h5, const char* name, if(filetypep) *filetypep = NC_INT; if(lenp) *lenp = 1; if(strcmp(name,SUPERBLOCKATT)==0) - iv = (unsigned long long)h5->fileinfo->superblockversion; + iv = (unsigned long long)h5->provenance->superblockversion; else /* strcmp(name,ISNETCDF4ATT)==0 */ iv = NC4_isnetcdf4(h5); if(mem_type == NC_NAT) mem_type = NC_INT; diff --git a/libsrc4/nc4dispatch.c b/libsrc4/nc4dispatch.c index beb168a9f9..00b33d21b6 100644 --- a/libsrc4/nc4dispatch.c +++ b/libsrc4/nc4dispatch.c @@ -37,8 +37,8 @@ NC4_sync, NC4_abort, NC4_close, NC4_set_fill, -NC4_inq_base_pe, -NC4_set_base_pe, +NC_NOTNC3_inq_base_pe, +NC_NOTNC3_set_base_pe, NC4_inq_format, NC4_inq_format_extended, diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 4795cea70d..8d2f384a88 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -780,9 +780,9 @@ nc4_dim_list_add(NC_GRP_INFO_T *grp, const char *name, size_t len, /* Assign the dimension ID. */ if (assignedid >= 0) - new_dim->hdr.id = assignedid; + new_dim->hdr.id = assignedid; else - new_dim->hdr.id = grp->nc4_info->next_dimid++; + new_dim->hdr.id = grp->nc4_info->next_dimid++; /* Remember the name and create a hash. */ if (!(new_dim->hdr.name = strdup(name))) diff --git a/libsrc4/ncfunc.c b/libsrc4/ncfunc.c index 361ea9ba52..c443963161 100644 --- a/libsrc4/ncfunc.c +++ b/libsrc4/ncfunc.c @@ -17,37 +17,6 @@ #include "nc4internal.h" #include "nc4dispatch.h" -/** - * @internal This function only does anything for netcdf-3 files. - * - * @param ncid File ID (ignored). - * @param pe Processor element (ignored). - * - * @return ::NC_ENOTNC3 Not a netCDF classic format file. - * @author Ed Hartnett - */ -int -NC4_set_base_pe(int ncid, int pe) -{ - return NC_ENOTNC3; -} - -/** - * @internal This function only does anything for netcdf-3 files. - * - * @param ncid File ID (ignored). - * @param pe Pointer to processor element. Ignored if NULL. Gets a 0 - * if present. - * - * @return ::NC_ENOTNC3 Not a netCDF classic format file. - * @author Ed Hartnett - */ -int -NC4_inq_base_pe(int ncid, int *pe) -{ - return NC_ENOTNC3; -} - /** * @internal Get the format (i.e. NC_FORMAT_NETCDF4 pr * NC_FORMAT_NETCDF4_CLASSIC) of an open netCDF-4 file. diff --git a/nc_test/Make0 b/nc_test/Make0 index 04c1b940b5..3ab0b8d357 100644 --- a/nc_test/Make0 +++ b/nc_test/Make0 @@ -1,17 +1,17 @@ # Test c output -T=tst_open_mem +T=nc_test #H58=8 H510=10 -#ARGS=f03_1820.nc +ARGS=persist -SRC=test_put.c test_get.c test_read.c test_write.c util.c error.c +#SRC=test_put.c test_get.c test_read.c test_write.c util.c error.c -#CMD=valgrind --leak-check=full +#CMD=env HDF5_DEBUG=trace #CMD=export NETCDF_LOG_LEVEL=5 ;gdb --args +CMD=valgrind --leak-check=full CMD=gdb --args -#CMD=env HDF5_DEBUG=trace #PAR=1 diff --git a/nc_test/run_diskless.sh b/nc_test/run_diskless.sh index 3f66e0773d..5807c02f36 100755 --- a/nc_test/run_diskless.sh +++ b/nc_test/run_diskless.sh @@ -5,7 +5,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh set -e - +set -x # Get the target OS and CPU CPU=`uname -p` OS=`uname` diff --git a/nc_test/run_inmemory.sh b/nc_test/run_inmemory.sh index 4d8dd027a5..b4515c5d1b 100755 --- a/nc_test/run_inmemory.sh +++ b/nc_test/run_inmemory.sh @@ -25,26 +25,19 @@ echo "*** Testing in-memory operations" HASNC4=`${top_builddir}/nc-config --has-nc4` # Execute the core of the inmemory tests -if ! ${execdir}/tst_inmemory; then - echo "FAIL: tst.inmemory" -fi -if ! ${execdir}/tst_open_mem ${srcdir}/${OMEMFILE}; then - echo "FAIL: tst_open_mem" -fi +${execdir}/tst_inmemory +exit +${execdir}/tst_open_mem ${srcdir}/${OMEMFILE} echo "**** Test ncdump of the resulting inmemory data" ${NCDUMP} -n "${FILE3}" ${FILE3}.nc > ${FILE3}.cdl ${NCDUMP} -n "${FILE3}" ${CREATE3}.nc > ${CREATE3}.cdl -if ! diff -wb ${FILE3}.cdl ${CREATE3}.cdl ; then - echo "FAIL: ${FILE3}.cdl vs ${CREATE3}.cdl -fi +diff -wb ${FILE3}.cdl ${CREATE3}.cdl if test "x$HASNC4" = "xyes" ; then ${NCDUMP} ${FILE4}.nc > ${FILE4}.cdl ${NCDUMP} ${CREATE4}.nc > ${CREATE4}.cdl -if diff -wb ${FILE4}.cdl ${CREATE4}.cdl ; then - echo "FAIL: ${FILE4}.cdl vs ${CREATE4}.cdl -fi +diff -wb ${FILE4}.cdl ${CREATE4}.cdl # cleanup rm -f ${FILE3}.nc ${FILE4}.nc ${CREATE3}.nc ${CREATE4}.nc diff --git a/nc_test/tst_diskless.c b/nc_test/tst_diskless.c index c5da2c0aed..321de11f6a 100644 --- a/nc_test/tst_diskless.c +++ b/nc_test/tst_diskless.c @@ -7,12 +7,12 @@ redistribution conditions. #undef DDBG -#include -#include -#include "err_macros.h" +#include "config.h" #include #include -#include +#include "netcdf.h" +#include "nc_tests.h" +#include "err_macros.h" /* netcdf tst_diskless { @@ -47,17 +47,12 @@ void fail(int line) { /* Control flags */ static int flags, persist, usenetcdf4, mmap; +/* Remove a file; do not care if it does not exist */ static void removefile(int persist, char* filename) { if(persist) { - if(remove(filename) != 0) { - if(errno != ENOENT) { - fprintf(stderr,"Could not remove file: %s: %d\n",filename,errno); - perror(""); - exit(1); - } - } + remove(filename); } } diff --git a/nc_test/tst_inmemory.c b/nc_test/tst_inmemory.c index c91fd4458d..76e0e199b1 100644 --- a/nc_test/tst_inmemory.c +++ b/nc_test/tst_inmemory.c @@ -442,11 +442,13 @@ test_open(const char* path, NC_memio* filedata, int mode) { int stat = NC_NOERR; NC_memio duplicate; - NC_memio* finaldata = NULL; + NC_memio finaldata; int ncid; int xmode = mode; /* modified mode */ - finaldata = calloc(1,sizeof(NC_memio)); + finaldata.memory = NULL; + finaldata.size = 0; + finaldata.flags = 0; fprintf(stderr,"\n\t***Test open 1: nc_open_mem(): read-only\n"); CHECK(duplicatememory(filedata,&duplicate,0)); @@ -460,13 +462,13 @@ test_open(const char* path, NC_memio* filedata, int mode) duplicate.flags = NC_MEMIO_LOCKED; CHECK(nc_open_memio(path, xmode, &duplicate, &ncid)) CHECK(verify_file(ncid,!MODIFIED)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); /* Verify that finaldata is same */ - if(finaldata->size != duplicate.size) CHECK(NC_EINVAL); - if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL); - free(finaldata->memory); + if(finaldata.size != duplicate.size) CHECK(NC_EINVAL); + if(finaldata.memory != duplicate.memory) CHECK(NC_EINVAL); + free(finaldata.memory); finaldata.memory = NULL; fprintf(stderr,"\n\t***Test open 3: nc_open_memio(): read-write, copy\n"); xmode |= NC_WRITE; /* allow file to be modified */ @@ -475,13 +477,13 @@ test_open(const char* path, NC_memio* filedata, int mode) /* modify file */ CHECK(modify_file(ncid)); CHECK(verify_file(ncid,MODIFIED)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); /* Verify that finaldata is same */ - if(finaldata->size < filedata->size) CHECK(NC_EINVAL); + if(finaldata.size < filedata->size) CHECK(NC_EINVAL); /* As a safeguard, the memory in duplicate should have been set to NULL*/ - free(finaldata->memory); + free(finaldata.memory); finaldata.memory = NULL; fprintf(stderr,"\n\t***Test open 4: nc_open_memio(): read-write, locked, extra space\n"); /* Store the filedata in a memory chunk that leaves room for modification */ @@ -493,14 +495,15 @@ test_open(const char* path, NC_memio* filedata, int mode) /* modify file */ CHECK(modify_file(ncid)); CHECK(verify_file(ncid,MODIFIED)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); - /* Check returned finaldata */ - if(finaldata->size != duplicate.size) CHECK(NC_EINVAL); - if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL); - free(finaldata->memory); - + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); + /* Check returned finaldata: + should have same memory but + actual used final size should not exceed the original */ + if(finaldata.size > duplicate.size) CHECK(NC_EINVAL); + if(finaldata.memory != duplicate.memory) CHECK(NC_EINVAL); + free(finaldata.memory); finaldata.memory = NULL; return stat; } @@ -508,33 +511,36 @@ static int test_create(const char* path, int mode) { int stat = NC_NOERR; - NC_memio* finaldata = NULL; + NC_memio finaldata; int ncid; int xmode = mode; + finaldata.memory = NULL; + finaldata.size = 0; + finaldata.flags = 0; + fprintf(stderr,"\n\t***Test create 1: nc_create_memio(): no initialsize\n"); CHECK(nc_create_mem(path, xmode, 0, &ncid)) /* create file metadata */ CHECK(define_metadata(ncid)); CHECK(verify_file(ncid,!MODIFIED)); - finaldata = calloc(1,sizeof(NC_memio)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); - free(finaldata->memory); - + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); + free(finaldata.memory); fprintf(stderr,"\n\t***Test create 2: nc_create_memio(): initialsize; save file\n"); CHECK(nc_create_mem(path, xmode, LARGE_SPACE, &ncid)) /* create file metadata */ CHECK(define_metadata(ncid)); CHECK(verify_file(ncid,!MODIFIED)); - finaldata = calloc(1,sizeof(NC_memio)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); /* Write out the final data as a .nc file */ - CHECK(writefile(path,finaldata)); - free(finaldata->memory); + CHECK(writefile(path,&finaldata)); + if(finaldata.memory != NULL) + free(finaldata.memory); + finaldata.memory = NULL; return stat; } @@ -584,15 +590,17 @@ test_xfail(const char* path, int mode, NC_memio* filedata) /* With HDF5 1.8.20, and possibly other versions, this tests causes a seg fault in the HDF5 Library. So until it is fixed, just leave well enough alone */ - NC_memio* finaldata = NULL; - finaldata = calloc(1,sizeof(NC_memio)); + NC_memio finaldata; + memset(&finaldata,0,sizeof(finaldata)); CHECK(duplicatememory(filedata,&duplicate,0)); duplicate.flags = NC_MEMIO_LOCKED; xmode |= NC_WRITE; CHECK(nc_open_memio(XFAIL, xmode, &duplicate, &ncid)) XCHECK(modify_file(ncid)); CHECK(nc_abort(ncid)); - free(finaldata->memory); + if(finaldata.memory != NULL) + free(finaldata.memory); + finaldata.memory = NULL; } return stat; @@ -605,11 +613,6 @@ main(int argc, char **argv) NC_memio filedata3; NC_memio filedata4; -#ifdef USE_NETCDF4 - nc_set_log_level(0); - H5Eprint1(stderr); -#endif - fprintf(stderr,"\n*** Testing the inmemory API: netcdf-3.\n"); CHECK(create_reference_file(FILE3,NC_NETCDF3,&filedata3)); /* netcdf-3 */ CHECK(test_open(FILE3,&filedata3,NC_NETCDF3)); diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 6d2d6b91ef..e1a09fa8d9 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -101,4 +101,4 @@ IF(ENABLE_METADATA_PERF) add_sh_test(nc_test4 perftest) ENDIF() -ADD_EXTRA_DIST(findplugin.in) +ADD_EXTRA_DIST(findplugin.in run_par_test.sh.in) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 1040d8b857..13603c2672 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -116,7 +116,8 @@ endif # with --enable-parallel-tests. if TEST_PARALLEL4 check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3 \ -tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r +tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r \ +tst_mode TESTS += run_par_test.sh endif @@ -125,7 +126,7 @@ check_PROGRAMS += bigmeta openbigmeta tst_attsperf TESTS += tst_attsperf perftest.sh endif -EXTRA_DIST = run_par_test.sh run_bm.sh run_bm_test1.sh \ +EXTRA_DIST = run_par_test.sh.in run_bm.sh run_bm_test1.sh \ run_bm_test2.sh run_bm_radar_2D.sh run_bm_radar_2D_compression1.sh \ run_par_bm_test.sh run_bm_elena.sh run_par_bm_radar_2D.sh \ run_bm_radar_2D_endianness1.sh run_tst_chunks.sh ref_chunks1.cdl \ @@ -139,14 +140,14 @@ ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \ ref_szip.cdl tst_filter.sh bzip2.cdl filtered.cdl unfiltered.cdl \ ref_bzip2.c findplugin.in perftest.sh - CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \ bm_radar.nc bm_radar1.nc radar_*.txt tst_floats_1D.cdl floats_1D_3.nc \ floats_1D.cdl tst_*.nc tst_floats2_*.cdl tst_ints2_*.cdl \ tst_shorts2_*.cdl tst_elena_*.cdl tst_simple*.cdl tst_chunks.cdl \ pr_A1.* tauu_A1.* usi_01.* thetau_01.* tst_*.h5 tst_grp_rename.cdl \ tst_grp_rename.dmp ref_grp_rename.cdl foo1.nc tst_*.h4 test.nc \ -testszip.nc test.h5 szip_dump.cdl perftest.txt bigmeta.nc bigvars.nc +testszip.nc test.h5 szip_dump.cdl perftest.txt bigmeta.nc bigvars.nc \ +run_par_test.sh DISTCLEANFILES = findplugin.sh diff --git a/nc_test4/run_par_test.sh b/nc_test4/run_par_test.sh deleted file mode 100755 index 427beb7ca2..0000000000 --- a/nc_test4/run_par_test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh - -if test "x$srcdir" = x ; then srcdir=`pwd`; fi -. ../test_common.sh - -# This shell runs some parallel I/O tests. - -set -e -echo -echo "Testing MPI parallel I/O with various other mode flags..." -mpiexec -n 1 ./tst_mode -echo -echo "Testing MPI parallel I/O without netCDF..." -mpiexec -n 4 ./tst_mpi_parallel -echo -echo "Testing very simple parallel I/O with 4 processors..." -mpiexec -n 4 ./tst_parallel -echo -echo "Testing simple parallel I/O with 16 processors..." -mpiexec -n 16 ./tst_parallel3 -echo -echo "num_proc time(s) write_rate(B/s)" -mpiexec -n 1 ./tst_parallel4 -mpiexec -n 2 ./tst_parallel4 -mpiexec -n 4 ./tst_parallel4 -mpiexec -n 8 ./tst_parallel4 - -#mpiexec -n 16 ./tst_parallel4 -#mpiexec -n 32 ./tst_parallel4 -#mpiexec -n 64 ./tst_parallel4 -echo -echo "Testing collective writes with some 0 element writes..." -mpiexec -n 4 ./tst_parallel5 - -echo -echo "Parallel Performance Test for NASA" -mpiexec -n 4 ./tst_nc4perf - -echo -echo "Parallel I/O test for Collective I/O, contributed by HDF Group." -mpiexec -n 1 ./tst_simplerw_coll_r -mpiexec -n 2 ./tst_simplerw_coll_r -mpiexec -n 4 ./tst_simplerw_coll_r diff --git a/nc_test4/run_par_test.sh.in b/nc_test4/run_par_test.sh.in new file mode 100644 index 0000000000..032aee4de2 --- /dev/null +++ b/nc_test4/run_par_test.sh.in @@ -0,0 +1,43 @@ +#!/bin/sh + +# This .in file is processed at build time into a shell that runs some +# parallel I/O tests for HDF5 parallel I/O. + +# Ed Hartnett, Dennis Heimbigner, Ward Fisher + +set -e +echo +echo "Testing MPI parallel I/O with various other mode flags..." +@MPIEXEC@ -n 1 ./tst_mode +echo +echo "Testing MPI parallel I/O without netCDF..." +@MPIEXEC@ -n 4 ./tst_mpi_parallel +echo +echo "Testing very simple parallel I/O with 4 processors..." +@MPIEXEC@ -n 4 ./tst_parallel +echo +echo "Testing simple parallel I/O with 16 processors..." +@MPIEXEC@ -n 16 ./tst_parallel3 +echo +echo "num_proc time(s) write_rate(B/s)" +@MPIEXEC@ -n 1 ./tst_parallel4 +@MPIEXEC@ -n 2 ./tst_parallel4 +@MPIEXEC@ -n 4 ./tst_parallel4 +@MPIEXEC@ -n 8 ./tst_parallel4 + +#@MPIEXEC@ -n 16 ./tst_parallel4 +#@MPIEXEC@ -n 32 ./tst_parallel4 +#@MPIEXEC@ -n 64 ./tst_parallel4 +echo +echo "Testing collective writes with some 0 element writes..." +@MPIEXEC@ -n 4 ./tst_parallel5 + +echo +echo "Parallel Performance Test for NASA" +@MPIEXEC@ -n 4 ./tst_nc4perf + +echo +echo "Parallel I/O test for Collective I/O, contributed by HDF Group." +@MPIEXEC@ -n 1 ./tst_simplerw_coll_r +@MPIEXEC@ -n 2 ./tst_simplerw_coll_r +@MPIEXEC@ -n 4 ./tst_simplerw_coll_r diff --git a/nc_test4/tst_atts.c b/nc_test4/tst_atts.c index 44e94808a9..11475a24e4 100644 --- a/nc_test4/tst_atts.c +++ b/nc_test4/tst_atts.c @@ -178,12 +178,12 @@ main(int argc, char **argv) /* Try and write a new att. Won't work. */ if (nc_put_att_text(ncid, NC_GLOBAL, OLD_NAME_2, strlen(CONTENTS_2), - CONTENTS_2) != NC_EINDEFINE) ERR; + CONTENTS_2) != NC_ENOTINDEFINE) ERR; /* This will not work. Overwriting att must be same length or * shorter if not in define mode. */ if (nc_put_att_text(ncid, NC_GLOBAL, OLD_NAME, strlen(CONTENTS_2), - CONTENTS_2) != NC_EINDEFINE) ERR; + CONTENTS_2) != NC_ENOTINDEFINE) ERR; /* Now overwrite the att. */ if (nc_put_att_text(ncid, NC_GLOBAL, OLD_NAME, strlen(CONTENTS_3), diff --git a/nc_test4/tst_dims2.c b/nc_test4/tst_dims2.c index 4811bc2a42..cebf9e3e9f 100644 --- a/nc_test4/tst_dims2.c +++ b/nc_test4/tst_dims2.c @@ -4,7 +4,7 @@ Test netcdf-4 dimensions some more. - $Id: tst_dims2.c,v 1.17 2010/05/25 13:53:04 ed Exp $ + Ed Hartnett */ #include @@ -121,7 +121,7 @@ main(int argc, char **argv) if (strcmp(name_in, "time") || xtype_in != NC_CHAR || ndims_in != NDIMS2 || natts_in != 0) ERR; for (i = 0; i < NDIMS2; i++) - if (time_dimids[i] != dimids_in[i]) ERR; + if (time_dimids[i] != dimids_in[i]) ERR; if (nc_get_vara_text(ncid, time_id, time_index, time_count, ttext_in)) ERR; ttext_in[TL] = 0; /* Add a NULL so strcmp will work. */ if (strcmp(ttext, ttext_in)) ERR; @@ -175,7 +175,7 @@ main(int argc, char **argv) time_count[0] = 1; time_count[1] = TL; for (time_index[0] = 0; time_index[0] < NUM_TIMES; time_index[0]++) - if(nc_put_vara_text(ncid, time_id, time_index, time_count, ttext)) ERR; + if(nc_put_vara_text(ncid, time_id, time_index, time_count, ttext)) ERR; /*close dataset*/ if (nc_close(ncid)) ERR; @@ -188,7 +188,7 @@ main(int argc, char **argv) if (strcmp(name_in, "time") || xtype_in != NC_CHAR || ndims_in != NDIMS2 || natts_in != 0) ERR; for (i = 0; i < NDIMS2; i++) - if (time_dimids[i] != dimids_in[i]) ERR; + if (time_dimids[i] != dimids_in[i]) ERR; time_index[0] = 0; if (nc_get_vara_text(ncid, time_id, time_index, time_count, ttext_in)) ERR; ttext_in[TL] = 0; /* Add a NULL so strcmp will work. */ @@ -225,7 +225,7 @@ main(int argc, char **argv) if (strcmp(name_in, TIME) || xtype_in != xtype || ndims_in != NDIMS || natts_in != 0) ERR; for (i = 0; i < NDIMS; i++) - if (dimids[i] != dimids_in[i]) ERR; + if (dimids[i] != dimids_in[i]) ERR; if (nc_close(ncid)) ERR; @@ -236,7 +236,7 @@ main(int argc, char **argv) if (strcmp(name_in, TIME) || xtype_in != xtype || ndims_in != NDIMS || natts_in != 0) ERR; for (i = 0; i < NDIMS; i++) - if (dimids[i] != dimids_in[i]) ERR; + if (dimids[i] != dimids_in[i]) ERR; if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; @@ -252,7 +252,7 @@ main(int argc, char **argv) /* Initialize some phony data. */ for (i = 0; i < MAX_VALUES; i++) - value[i] = MAX_VALUES - i; + value[i] = MAX_VALUES - i; /* Create a file with 2 unlimited dims, and one var that uses * both of them. */ @@ -292,13 +292,13 @@ main(int argc, char **argv) int varid, ncid, timeDimID, beamDimID; int i, j; int value[2000]; - size_t time_recs, beam_recs; /* count of records in each dimension */ - size_t time_len, beam_len; /* actual dimension lengths in each dimension */ + size_t time_recs, beam_recs; /* count of records in each dimension */ + size_t time_len, beam_len; /* actual dimension lengths in each dimension */ size_t start[] = {0, 0}; size_t count[] = {1, 1}; for (i = 0; i < 2000; i++) - value[i] = 2000 - i; + value[i] = 2000 - i; if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; @@ -317,19 +317,19 @@ main(int argc, char **argv) beam_recs = 0; for (j = 0; j < 100; j++) { - if (j > 500) - count[1] = j; - else - count[1] = 1000-j; - - if (nc_put_vara_int(ncid, varid, start, count, value)) ERR; - time_recs = MAX(time_recs, start[0] + count[0]); - beam_recs = MAX(beam_recs, start[1] + count[1]); - if (nc_inq_dimlen(ncid, timeDimID, &time_len)) ERR; - if (time_len != time_recs) ERR; - if (nc_inq_dimlen(ncid, beamDimID, &beam_len)) ERR; - if (beam_len != beam_recs) ERR; - start[0]++; + if (j > 500) + count[1] = j; + else + count[1] = 1000-j; + + if (nc_put_vara_int(ncid, varid, start, count, value)) ERR; + time_recs = MAX(time_recs, start[0] + count[0]); + beam_recs = MAX(beam_recs, start[1] + count[1]); + if (nc_inq_dimlen(ncid, timeDimID, &time_len)) ERR; + if (time_len != time_recs) ERR; + if (nc_inq_dimlen(ncid, beamDimID, &beam_len)) ERR; + if (beam_len != beam_recs) ERR; + start[0]++; } if (nc_close(ncid)) ERR; @@ -370,7 +370,7 @@ main(int argc, char **argv) if (strcmp(name_in, TIME) || xtype_in != xtype || ndims_in != NDIMS || natts_in != 0) ERR; for (i = 0; i < NDIMS; i++) - if (dimids[i] != dimids_in[i]) ERR; + if (dimids[i] != dimids_in[i]) ERR; /* Write some time values. */ index[0] = 0; @@ -389,7 +389,7 @@ main(int argc, char **argv) if (strcmp(name_in, TIME) || xtype_in != xtype || ndims_in != NDIMS || natts_in != 0) ERR; for (i = 0; i < NDIMS; i++) - if (dimids[i] != dimids_in[i]) ERR; + if (dimids[i] != dimids_in[i]) ERR; if (nc_inq_dim(ncid, dimids[0], name_in, &len_in)) ERR; if (strcmp(name_in, TIME) || len_in != 1) ERR; if (nc_close(ncid)) ERR; @@ -402,7 +402,7 @@ main(int argc, char **argv) count[0] = 1; count[1] = TL; /* Note we are not writing NULL char. */ for (index[0] = 1; index[0] < NUM_RECS; index[0]++) - if (nc_put_vara_text(ncid, varid, index, count, ttext)) ERR; + if (nc_put_vara_text(ncid, varid, index, count, ttext)) ERR; /* Check things again. */ if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR; @@ -411,12 +411,36 @@ main(int argc, char **argv) if (strcmp(name_in, TIME) || xtype_in != xtype || ndims_in != NDIMS || natts_in != 0) ERR; for (i = 0; i < NDIMS; i++) - if (dimids[i] != dimids_in[i]) ERR; + if (dimids[i] != dimids_in[i]) ERR; if (nc_inq_dim(ncid, dimids[0], name_in, &len_in)) ERR; if (strcmp(name_in, TIME) || len_in != NUM_RECS) ERR; if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; + printf("*** Checking many dimensions..."); + { +#define NDIMS50 50 +#define MANY_DIM_VAR "brain" + int dimids[NDIMS50]; + int varid, ncid; + int i; + + /* Create a file with many dimensions. */ + if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + for (i = 0; i < NDIMS50; i++) + { + char dim_name[NC_MAX_NAME + 1]; + + sprintf(dim_name, "dim_%d", i); + if (nc_def_dim(ncid, dim_name, 1, &dimids[i])) ERR; + } + + /* This will fail because HDF5 only allows 32 dimensions. */ + if (nc_def_var(ncid, MANY_DIM_VAR, NC_INT, NDIMS50, dimids, + &varid) != NC_EMAXDIMS) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; FINAL_RESULTS; } diff --git a/nc_test4/tst_enums.c b/nc_test4/tst_enums.c index 1fec6e7fcd..977d73cb4b 100644 --- a/nc_test4/tst_enums.c +++ b/nc_test4/tst_enums.c @@ -4,7 +4,7 @@ Test netcdf-4 enum types. - $Id: tst_enums.c,v 1.17 2009/02/14 14:09:45 ed Exp $ + Ed Hartnett */ #include @@ -37,7 +37,7 @@ main(int argc, char **argv) int value_in; /* Can't use the same name twice! */ char member_name[NUM_MEMBERS][NC_MAX_NAME + 1] = {"Mene1", "Mene2", - "Tekel", "Upharsin"}; + "Tekel", "Upharsin"}; int member_value[NUM_MEMBERS] = {0, 99, 81232, 12}; /* Create a file. */ @@ -46,23 +46,23 @@ main(int argc, char **argv) /* Create an enum type. */ if (nc_def_enum(ncid, NC_INT, TYPE_NAME, &typeid)) ERR; for (i = 0; i < NUM_MEMBERS; i++) - if (nc_insert_enum(ncid, typeid, member_name[i], - &member_value[i])) ERR; + if (nc_insert_enum(ncid, typeid, member_name[i], + &member_value[i])) ERR; /* Check it out. */ if (nc_inq_user_type(ncid, typeid, name_in, &base_size_in, &base_nc_type_in, - &nfields_in, &class_in)) ERR; + &nfields_in, &class_in)) ERR; if (strcmp(name_in, TYPE_NAME) || base_size_in != sizeof(int) || - base_nc_type_in != NC_INT || nfields_in != NUM_MEMBERS || class_in != NC_ENUM) ERR; + base_nc_type_in != NC_INT || nfields_in != NUM_MEMBERS || class_in != NC_ENUM) ERR; if (nc_inq_enum(ncid, typeid, name_in, &base_nc_type, &base_size_in, &num_members)) ERR; if (strcmp(name_in, TYPE_NAME) || base_nc_type != NC_INT || - num_members != NUM_MEMBERS) ERR; + num_members != NUM_MEMBERS) ERR; for (i = 0; i < NUM_MEMBERS; i++) { - if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; - if (strcmp(name_in, member_name[i]) || value_in != member_value[i]) ERR; - if (nc_inq_enum_ident(ncid, typeid, member_value[i], name_in)) ERR; - if (strcmp(name_in, member_name[i])) ERR; + if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; + if (strcmp(name_in, member_name[i]) || value_in != member_value[i]) ERR; + if (nc_inq_enum_ident(ncid, typeid, member_value[i], name_in)) ERR; + if (strcmp(name_in, member_name[i])) ERR; } /* Write the file. */ @@ -77,19 +77,19 @@ main(int argc, char **argv) /* Check it out. */ if (nc_inq_user_type(ncid, typeids[0], name_in, &base_size_in, &base_nc_type_in, - &nfields_in, &class_in)) ERR; + &nfields_in, &class_in)) ERR; if (strcmp(name_in, TYPE_NAME) || base_size_in != sizeof(int) || - base_nc_type_in != NC_INT || nfields_in != NUM_MEMBERS || class_in != NC_ENUM) ERR; + base_nc_type_in != NC_INT || nfields_in != NUM_MEMBERS || class_in != NC_ENUM) ERR; if (nc_inq_type(ncid, typeids[0], name_in, &base_size_in)) ERR; if (strcmp(name_in, TYPE_NAME) || base_size_in != sizeof(int)) ERR; if (nc_inq_enum(ncid, typeids[0], name_in, &base_nc_type, &base_size_in, &num_members)) ERR; if (strcmp(name_in, TYPE_NAME) || base_nc_type != NC_INT || num_members != NUM_MEMBERS) ERR; for (i = 0; i < NUM_MEMBERS; i++) { - if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; - if (strcmp(name_in, member_name[i]) || value_in != member_value[i]) ERR; - if (nc_inq_enum_ident(ncid, typeid, member_value[i], name_in)) ERR; - if (strcmp(name_in, member_name[i])) ERR; + if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; + if (strcmp(name_in, member_name[i]) || value_in != member_value[i]) ERR; + if (nc_inq_enum_ident(ncid, typeid, member_value[i], name_in)) ERR; + if (strcmp(name_in, member_name[i])) ERR; } if (nc_close(ncid)) ERR; @@ -105,8 +105,8 @@ main(int argc, char **argv) printf("*** testing enum attribute..."); { char brady_name[NUM_BRADYS][NC_MAX_NAME + 1] = {"Mike", "Carol", "Greg", "Marsha", - "Peter", "Jan", "Bobby", "Whats-her-face", - "Alice"}; + "Peter", "Jan", "Bobby", "Whats-her-face", + "Alice"}; unsigned char brady_value[NUM_BRADYS] = {8, 7, 6 , 5, 4, 3, 2, 1, 0}; unsigned char data[BRADY_DIM_LEN] = {0, 4, 8}; unsigned char value_in; @@ -117,23 +117,23 @@ main(int argc, char **argv) /* Create an enum type based on unsigned bytes. */ if (nc_def_enum(ncid, NC_UBYTE, BRADYS, &typeid)) ERR; for (i = 0; i < NUM_BRADYS; i++) - if (nc_insert_enum(ncid, typeid, brady_name[i], - &brady_value[i])) ERR; + if (nc_insert_enum(ncid, typeid, brady_name[i], + &brady_value[i])) ERR; /* Check it out. */ if (nc_inq_user_type(ncid, typeid, name_in, &base_size_in, &base_nc_type_in, - &nfields_in, &class_in)) ERR; + &nfields_in, &class_in)) ERR; if (strcmp(name_in, BRADYS) || base_size_in != 1 || - base_nc_type_in != NC_UBYTE || nfields_in != NUM_BRADYS || class_in != NC_ENUM) ERR; + base_nc_type_in != NC_UBYTE || nfields_in != NUM_BRADYS || class_in != NC_ENUM) ERR; if (nc_inq_enum(ncid, typeid, name_in, &base_nc_type, &base_size_in, &num_members)) ERR; if (strcmp(name_in, BRADYS) || base_nc_type != NC_UBYTE || base_size_in != 1 || - num_members != NUM_BRADYS) ERR; + num_members != NUM_BRADYS) ERR; for (i = 0; i < NUM_BRADYS; i++) { - if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; - if (strcmp(name_in, brady_name[i]) || value_in != brady_value[i]) ERR; - if (nc_inq_enum_ident(ncid, typeid, brady_value[i], name_in)) ERR; - if (strcmp(name_in, brady_name[i])) ERR; + if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; + if (strcmp(name_in, brady_name[i]) || value_in != brady_value[i]) ERR; + if (nc_inq_enum_ident(ncid, typeid, brady_value[i], name_in)) ERR; + if (strcmp(name_in, brady_name[i])) ERR; } /* Write an att of this enum type. */ @@ -147,18 +147,18 @@ main(int argc, char **argv) /* Check it out. */ if (nc_inq_user_type(ncid, typeid, name_in, &base_size_in, &base_nc_type_in, - &nfields_in, &class_in)) ERR; + &nfields_in, &class_in)) ERR; if (strcmp(name_in, BRADYS) || base_size_in != 1 || - base_nc_type_in != NC_UBYTE || nfields_in != NUM_BRADYS || class_in != NC_ENUM) ERR; + base_nc_type_in != NC_UBYTE || nfields_in != NUM_BRADYS || class_in != NC_ENUM) ERR; if (nc_inq_enum(ncid, typeid, name_in, &base_nc_type, &base_size_in, &num_members)) ERR; if (strcmp(name_in, BRADYS) || base_nc_type != NC_UBYTE || base_size_in != 1 || - num_members != NUM_BRADYS) ERR; + num_members != NUM_BRADYS) ERR; for (i = 0; i < NUM_BRADYS; i++) { - if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; - if (strcmp(name_in, brady_name[i]) || value_in != brady_value[i]) ERR; - if (nc_inq_enum_ident(ncid, typeid, brady_value[i], name_in)) ERR; - if (strcmp(name_in, brady_name[i])) ERR; + if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; + if (strcmp(name_in, brady_name[i]) || value_in != brady_value[i]) ERR; + if (nc_inq_enum_ident(ncid, typeid, brady_value[i], name_in)) ERR; + if (strcmp(name_in, brady_name[i])) ERR; } if (nc_close(ncid)) ERR; @@ -176,109 +176,109 @@ main(int argc, char **argv) printf("*** testing enum fill value ..."); { - int dimid, varid; - size_t num_members_in; - int class_in; - unsigned char value_in; - unsigned char att_value_in; - - enum clouds { /* a C enumeration */ - CLEAR=0, - CUMULONIMBUS=1, - STRATUS=2, - STRATOCUMULUS=3, - CUMULUS=4, - ALTOSTRATUS=5, - NIMBOSTRATUS=6, - ALTOCUMULUS=7, - CIRROSTRATUS=8, - CIRROCUMULUS=9, - CIRRUS=10, - MISSING=255}; - - struct { - char *name; - unsigned char value; - } cloud_types[] = { - {"Clear", CLEAR}, - {"Cumulonimbus", CUMULONIMBUS}, - {"Stratus", STRATUS}, - {"Stratocumulus", STRATOCUMULUS}, - {"Cumulus", CUMULUS}, - {"Altostratus", ALTOSTRATUS}, - {"Nimbostratus", NIMBOSTRATUS}, - {"Altocumulus", ALTOCUMULUS}, - {"Cirrostratus", CIRROSTRATUS}, - {"Cirrocumulus", CIRROCUMULUS}, - {"Cirrus", CIRRUS}, - {"Missing", MISSING} - }; - int var_dims[VAR2_RANK]; - unsigned char att_val; - unsigned char cloud_data[DIM2_LEN] = { - CLEAR, STRATUS, CLEAR, CUMULONIMBUS, MISSING}; - unsigned char cloud_data_in[DIM2_LEN]; - - if (nc_create(FILE_NAME, NC_CLOBBER | NC_NETCDF4, &ncid)) ERR; - - /* Create an enum type. */ - if (nc_def_enum(ncid, NC_UBYTE, TYPE2_NAME, &typeid)) ERR; - num_members = (sizeof cloud_types) / (sizeof cloud_types[0]); - for (i = 0; i < num_members; i++) - if (nc_insert_enum(ncid, typeid, cloud_types[i].name, - &cloud_types[i].value)) ERR; - - /* Declare a station dimension */ - if (nc_def_dim(ncid, DIM2_NAME, DIM2_LEN, &dimid)) ERR; - /* Declare a variable of the enum type */ - var_dims[0] = dimid; - if (nc_def_var(ncid, VAR2_NAME, typeid, VAR2_RANK, var_dims, &varid)) ERR; - /* Create and write a variable attribute of the enum type */ - att_val = MISSING; - if (nc_put_att(ncid, varid, ATT2_NAME, typeid, ATT2_LEN, &att_val)) ERR; - if (nc_enddef(ncid)) ERR; - /* Store some data of the enum type */ - if(nc_put_var(ncid, varid, cloud_data)) ERR; - /* Write the file. */ - if (nc_close(ncid)) ERR; - - /* Check it out. */ - - /* Reopen the file. */ - if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; - - if (nc_inq_user_type(ncid, typeid, name_in, &base_size_in, &base_nc_type_in, - &nfields_in, &class_in)) ERR; - if (strcmp(name_in, TYPE2_NAME) || - base_size_in != sizeof(unsigned char) || - base_nc_type_in != NC_UBYTE || - nfields_in != num_members || - class_in != NC_ENUM) ERR; - if (nc_inq_enum(ncid, typeid, name_in, - &base_nc_type_in, &base_size_in, &num_members_in)) ERR; - if (strcmp(name_in, TYPE2_NAME) || - base_nc_type_in != NC_UBYTE || - num_members_in != num_members) ERR; - for (i = 0; i < num_members; i++) - { - if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; - if (strcmp(name_in, cloud_types[i].name) || - value_in != cloud_types[i].value) ERR; - if (nc_inq_enum_ident(ncid, typeid, cloud_types[i].value, - name_in)) ERR; - if (strcmp(name_in, cloud_types[i].name)) ERR; - } - if (nc_inq_varid(ncid, VAR2_NAME, &varid)) ERR; - - if (nc_get_att(ncid, varid, ATT2_NAME, &att_value_in)) ERR; - if (att_value_in != MISSING) ERR; - - if(nc_get_var(ncid, varid, cloud_data_in)) ERR; - for (i = 0; i < DIM2_LEN; i++) { - if (cloud_data_in[i] != cloud_data[i]) ERR; - } - - if (nc_close(ncid)) ERR; + int dimid, varid; + size_t num_members_in; + int class_in; + unsigned char value_in; + unsigned char att_value_in; + + enum clouds { /* a C enumeration */ + CLEAR=0, + CUMULONIMBUS=1, + STRATUS=2, + STRATOCUMULUS=3, + CUMULUS=4, + ALTOSTRATUS=5, + NIMBOSTRATUS=6, + ALTOCUMULUS=7, + CIRROSTRATUS=8, + CIRROCUMULUS=9, + CIRRUS=10, + MISSING=255}; + + struct { + char *name; + unsigned char value; + } cloud_types[] = { + {"Clear", CLEAR}, + {"Cumulonimbus", CUMULONIMBUS}, + {"Stratus", STRATUS}, + {"Stratocumulus", STRATOCUMULUS}, + {"Cumulus", CUMULUS}, + {"Altostratus", ALTOSTRATUS}, + {"Nimbostratus", NIMBOSTRATUS}, + {"Altocumulus", ALTOCUMULUS}, + {"Cirrostratus", CIRROSTRATUS}, + {"Cirrocumulus", CIRROCUMULUS}, + {"Cirrus", CIRRUS}, + {"Missing", MISSING} + }; + int var_dims[VAR2_RANK]; + unsigned char att_val; + unsigned char cloud_data[DIM2_LEN] = { + CLEAR, STRATUS, CLEAR, CUMULONIMBUS, MISSING}; + unsigned char cloud_data_in[DIM2_LEN]; + + if (nc_create(FILE_NAME, NC_CLOBBER | NC_NETCDF4, &ncid)) ERR; + + /* Create an enum type. */ + if (nc_def_enum(ncid, NC_UBYTE, TYPE2_NAME, &typeid)) ERR; + num_members = (sizeof cloud_types) / (sizeof cloud_types[0]); + for (i = 0; i < num_members; i++) + if (nc_insert_enum(ncid, typeid, cloud_types[i].name, + &cloud_types[i].value)) ERR; + + /* Declare a station dimension */ + if (nc_def_dim(ncid, DIM2_NAME, DIM2_LEN, &dimid)) ERR; + /* Declare a variable of the enum type */ + var_dims[0] = dimid; + if (nc_def_var(ncid, VAR2_NAME, typeid, VAR2_RANK, var_dims, &varid)) ERR; + /* Create and write a variable attribute of the enum type */ + att_val = MISSING; + if (nc_put_att(ncid, varid, ATT2_NAME, typeid, ATT2_LEN, &att_val)) ERR; + if (nc_enddef(ncid)) ERR; + /* Store some data of the enum type */ + if(nc_put_var(ncid, varid, cloud_data)) ERR; + /* Write the file. */ + if (nc_close(ncid)) ERR; + + /* Check it out. */ + + /* Reopen the file. */ + if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; + + if (nc_inq_user_type(ncid, typeid, name_in, &base_size_in, &base_nc_type_in, + &nfields_in, &class_in)) ERR; + if (strcmp(name_in, TYPE2_NAME) || + base_size_in != sizeof(unsigned char) || + base_nc_type_in != NC_UBYTE || + nfields_in != num_members || + class_in != NC_ENUM) ERR; + if (nc_inq_enum(ncid, typeid, name_in, + &base_nc_type_in, &base_size_in, &num_members_in)) ERR; + if (strcmp(name_in, TYPE2_NAME) || + base_nc_type_in != NC_UBYTE || + num_members_in != num_members) ERR; + for (i = 0; i < num_members; i++) + { + if (nc_inq_enum_member(ncid, typeid, i, name_in, &value_in)) ERR; + if (strcmp(name_in, cloud_types[i].name) || + value_in != cloud_types[i].value) ERR; + if (nc_inq_enum_ident(ncid, typeid, cloud_types[i].value, + name_in)) ERR; + if (strcmp(name_in, cloud_types[i].name)) ERR; + } + if (nc_inq_varid(ncid, VAR2_NAME, &varid)) ERR; + + if (nc_get_att(ncid, varid, ATT2_NAME, &att_value_in)) ERR; + if (att_value_in != MISSING) ERR; + + if(nc_get_var(ncid, varid, cloud_data_in)) ERR; + for (i = 0; i < DIM2_LEN; i++) { + if (cloud_data_in[i] != cloud_data[i]) ERR; + } + + if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; @@ -295,19 +295,19 @@ main(int argc, char **argv) /* Check it out. */ if (nc_inq_user_type(ncid, typeid, name_in, &base_size_in, &base_nc_type_in, - &nfields_in, &class_in)) ERR; + &nfields_in, &class_in)) ERR; if (strcmp(name_in, GEEKY_NAME) || base_size_in != 1 || - base_nc_type_in != NC_UBYTE || nfields_in != 0 || class_in != NC_ENUM) ERR; + base_nc_type_in != NC_UBYTE || nfields_in != 0 || class_in != NC_ENUM) ERR; if (nc_inq_enum(ncid, typeid, name_in, &base_nc_type, &base_size_in, &num_members)) ERR; if (strcmp(name_in, GEEKY_NAME) || base_nc_type != NC_UBYTE || base_size_in != 1 || - num_members != 0) ERR; + num_members != 0) ERR; /* Close the file. */ if (nc_close(ncid) != NC_EINVAL) ERR; if (nc_redef(ncid)) ERR; if (nc_insert_enum(ncid, typeid, "name", &ubyte_value)) ERR; - if (nc_close(ncid)) ERR; + if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; printf("*** testing enum ncdump..."); @@ -327,38 +327,37 @@ main(int argc, char **argv) unsigned long long field_value[NUM_ENUM_FIELDS] = {1000000, 2000000, 3000000, 4000000, 5000000, 6000000, 7000000, 8000000}; unsigned long long data = 1000000, data_in; - + /* Create a netcdf-4 file. */ /*nc_set_log_level(3);*/ if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; - + /* Create a dimension. */ if (nc_def_dim(ncid, DIM_NAME, DIMSIZE, &dimid)) ERR; - + /* Create an enum type. */ if (nc_def_enum(ncid, NC_UINT64, ENUM_NAME, &typeid)) ERR; for (f = 0; f < NUM_ENUM_FIELDS; f++) if (nc_insert_enum(ncid, typeid, field_name[f], &field_value[f])) ERR; - + /* Create one var. */ if (nc_def_var(ncid, ENUM_VAR_NAME, typeid, NDIMS1, &dimid, &v1id)) ERR; - + /* Write metadata to file. */ if (nc_enddef(ncid)) ERR; - + /* Write phoney data. */ if (nc_put_vara(ncid, v1id, start, count, &data)) ERR; - + if (nc_sync(ncid)) ERR; /* Read phoney data. */ if (nc_get_vara(ncid, v1id, start, count, &data_in)) ERR; if (data_in != data) ERR; - + /* Close the netcdf file. */ if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; FINAL_RESULTS; } - diff --git a/nc_test4/tst_files6.c b/nc_test4/tst_files6.c index fa265f2e30..3b05554895 100644 --- a/nc_test4/tst_files6.c +++ b/nc_test4/tst_files6.c @@ -3,13 +3,19 @@ See COPYRIGHT file for conditions of use. Test netcdf files a bit. + + Ed Hartnett */ #include #include "err_macros.h" -#include "netcdf.h" +#include "hdf5internal.h" #define FILE_NAME "tst_files6.nc" +#define HDF5_FILE_NAME "tst_files6.h5" +#define GROUP_NAME "Britany_Fans" +#define GROUP_NAME_2 "Toxic_Fans" +#define TRUE_FANS "True_Toxic_Fans" int main(int argc, char **argv) @@ -54,5 +60,28 @@ main(int argc, char **argv) if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; + printf("*** testing HDF5 file with circular group structure..."); + { + hid_t hdfid, grpid, grpid2, fapl_id; + int ncid; + + /* First use HDF5 to create a file with circular group + * struct. */ + if ((hdfid = H5Fcreate(HDF5_FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, + H5P_DEFAULT)) < 0) ERR; + if ((grpid = H5Gcreate2(hdfid, GROUP_NAME, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) ERR; + if ((grpid2 = H5Gcreate2(grpid, GROUP_NAME_2, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) ERR; + if (H5Lcreate_soft(GROUP_NAME, grpid2, TRUE_FANS, H5P_DEFAULT, + H5P_DEFAULT) < 0) ERR; + if (H5Fclose(hdfid) < 0) ERR; + + H5close(); /* Force HDF5 to forget about this file. */ + + /* Now try and open it with netCDF. It will not work. */ + if (nc_open(HDF5_FILE_NAME, NC_NOWRITE, &ncid) != NC_EHDFERR) ERR; + } + SUMMARIZE_ERR; FINAL_RESULTS; } diff --git a/nc_test4/tst_parallel.c b/nc_test4/tst_parallel.c index 171c04751b..f09b1f4a96 100644 --- a/nc_test4/tst_parallel.c +++ b/nc_test4/tst_parallel.c @@ -40,7 +40,7 @@ main(int argc, char **argv) int ncid, v1id, dimids[NDIMS]; size_t start[NDIMS], count[NDIMS]; - int data[DIMSIZE * DIMSIZE], i, res; + int i, res; int slab_data[DIMSIZE * DIMSIZE / 4]; /* one slab */ char file_name[NC_MAX_NAME + 1]; @@ -82,10 +82,6 @@ main(int argc, char **argv) /* Create phony data. We're going to write a 24x24 array of ints, in 4 sets of 144. */ - /*printf("mpi_rank*QTR_DATA=%d (mpi_rank+1)*QTR_DATA-1=%d\n", - mpi_rank*QTR_DATA, (mpi_rank+1)*QTR_DATA);*/ - for (i = mpi_rank * QTR_DATA; i < (mpi_rank + 1) * QTR_DATA; i++) - data[i] = mpi_rank; for (i = 0; i < DIMSIZE * DIMSIZE / 4; i++) slab_data[i] = mpi_rank; diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index 1bb5a72a75..ae20d6cffa 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -60,5 +60,5 @@ ADD_SUBDIRECTORY(expectremote3) ## Specify files to be distributed by 'make dist' FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) -SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am) +SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} findtestserver.c.in CMakeLists.txt Makefile.am) ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 32d7a80a46..33e4152eb2 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -83,12 +83,14 @@ EXTRA_DIST = tst_ncdap3.sh \ tst_longremote3.sh \ tst_filelists.sh tst_urls.sh tst_utils.sh \ t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \ - t_ncf330.c tst_ber.sh tst_curlopt.sh + t_ncf330.c tst_ber.sh findtestserver.c.in CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe # This should only be left behind if using parallel io CLEANFILES += tmp_* +DISTCLEANFILES = findtestserver4.c + # This rule are used if someone wants to rebuild t_dap3a.c # Otherwise never invoked, but records how to do it. t_dap3a.c: t_dap.c diff --git a/ncdap_test/findtestserver.c b/ncdap_test/findtestserver.c index 69a324cbb1..9ad5e697ef 100644 --- a/ncdap_test/findtestserver.c +++ b/ncdap_test/findtestserver.c @@ -8,6 +8,7 @@ #define XSTRINGIFY(s) #s #define STRINGIFY(s) XSTRINGIFY(s) + /** usage: findtestserver dap2|dap4 suffix [serverlist] diff --git a/ncdap_test/findtestserver.c.in b/ncdap_test/findtestserver.c.in new file mode 100644 index 0000000000..9ad5e697ef --- /dev/null +++ b/ncdap_test/findtestserver.c.in @@ -0,0 +1,79 @@ +#include "config.h" +#include +#include +#include +#include "nctestserver.h" + +/* Support stringification of -D macros */ +#define XSTRINGIFY(s) #s +#define STRINGIFY(s) XSTRINGIFY(s) + + +/** +usage: findtestserver dap2|dap4 suffix [serverlist] + +Given a partial suffix path, try to find a +server for which a request to server + suffix +returns some kind of result using the +specified protocol. This indicates that the +server is up and running. Return the complete +url for the server plus the path. +If serverlist is present, then is should be a comma +separated list of servers (host+port) to try. +It defaults to REMOTETESTSERVERS. +*/ + +static void +usage() +{ + fprintf(stderr,"usage: findtestserver dap2|dap4 suffix [serverlist]\n"); + exit(1); +} + + +int +main(int argc, char** argv) +{ + const char* url = NULL; + const char* servlet = NULL; + const char* proto = NULL; + const char* serverlist = NULL; + int isdap4 = 0; /* 1 => dap4 */ + + argc--; argv++; + if(argc < 2) + usage(); + proto = strdup(argv[0]); + servlet = strdup(argv[1]); + if(argc >= 3) + serverlist = strdup(argv[2]); + +#ifdef ENABLE_DAP + if(strcasecmp(proto,"dap2")==0) + isdap4 = 0; + else +#endif +#ifdef ENABLE_DAP4 + if(strcasecmp(proto,"dap4")==0) + isdap4 = 1; + else +#endif + usage(); + + if(serverlist == NULL) { +#ifdef REMOTETESTSERVERS + serverlist = strdup(REMOTETESTSERVERS); +#endif + } + if(serverlist == NULL || strlen(serverlist) == 0) + fprintf(stderr,"Cannot determine a server list"); + + url = nc_findtestserver(servlet,isdap4,serverlist); + if(url == NULL) { + url = ""; + fprintf(stderr,"not found: %s\n",servlet); + } + printf("%s",url); + fflush(stdout); + exit(0); +} diff --git a/ncdap_test/testurl.sh b/ncdap_test/testurl.sh index 95ba9eb30f..b851e8c79a 100755 --- a/ncdap_test/testurl.sh +++ b/ncdap_test/testurl.sh @@ -56,7 +56,7 @@ locreset if test "x$NOP" != x1 ; then echo "***Testing url prefix parameters" -buildurl $PREFIX "" +buildurl "$PREFIX" "" # Invoke ncdump to extract the URL echo "command: ${NCDUMP} -h $url" @@ -66,7 +66,7 @@ if test "x${SHOW}" = x1 ; then cat ./tmp ; fi # Test that maxstrlen works as alias for stringlength echo "***Testing maxstrlen=stringlength alias" -buildurl $STRLEN "" +buildurl "$STRLEN" "" # Invoke ncdump to extract the URL echo "command: ${NCDUMP} -h $url" ${NCDUMP} "$url" >./tmp_testurl 2> ./errtmp_testurl @@ -82,7 +82,7 @@ fi locreset if test "x$NOS" != x1 ; then echo "***Testing url suffix parameters" -buildurl "" $SUFFIX +buildurl "" "$SUFFIX" # Invoke ncdump to extract the URL ${NCDUMP} -h "$url" >./tmp_testurl 2> ./errtmp_testurl if test "x${SHOW}" = x1 ; then cat ./tmp_testurl ; fi @@ -92,7 +92,7 @@ locreset if test "x$NOB" != x1 ; then echo "***Testing url prefix+suffix parameters" -buildurl $BOTHP $BOTHS +buildurl "$BOTHP" "$BOTHS" # Invoke ncdump to extract the URL ${NCDUMP} -h "$url" >./tmp_testurl 2> ./errtmp_testurl if test "x${SHOW}" = x1 ; then cat ./tmp_testurl ; fi diff --git a/ncdump/Make0 b/ncdump/Make0 index 1d02f46ce8..c18b783306 100644 --- a/ncdump/Make0 +++ b/ncdump/Make0 @@ -1,12 +1,12 @@ # Test c output -T=tst_chunking +T=tst_fileinfo #ARGS=./x.nc #TF=test_atomic_array.cdl #CMD=valgrind --leak-check=full -#CMD=gdb --args +CMD=gdb --args #HDF4=1 #PAR=1 diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 1df11844aa..4bf673d9e0 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -7,6 +7,7 @@ #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT = export SETX=1; # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am @@ -130,7 +131,7 @@ tst_hdf5_offset.sh run_ncgen_nc4_tests.sh tst_nccopy3_subset.sh \ ref_nccopy3_subset.nc ref_test_corrupt_magic.nc tst_ncgen_shared.sh \ tst_ncgen4.sh tst_ncgen4_classic.sh tst_ncgen4_diff.sh \ tst_ncgen4_cycle.sh tst_null_byte_padding.sh \ -ref_null_byte_padding_test.nc ref_tst_irish_rover.nc +ref_null_byte_padding_test.nc ref_tst_irish_rover.nc ref_provenance_v1.nc # The L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. diff --git a/ncdump/nccopy.1 b/ncdump/nccopy.1 index 1eda799713..e80dbbd5c3 100644 --- a/ncdump/nccopy.1 +++ b/ncdump/nccopy.1 @@ -21,6 +21,8 @@ nccopy \%[\-e \fI cache_elems \fP] \%[\-r] \%[\-F \fI filterspec \fP] +\%[\-L \fI n \fP] +\%[\-M \fI n \fP] \%\fI infile \fP \%\fI outfile \fP .hy @@ -252,6 +254,10 @@ Read netCDF classic or 64-bit offset input file into a diskless netCDF file in memory before copying. Requires that input file be small enough to fit into memory. For \fBnccopy\fP, this doesn't seem to provide any significant speedup, so may not be a useful option. +.IP "\fB \-L \fP \fIn\fP" +Set the log level; only usable if nccopy supports netCDF-4 (enhanced). +.IP "\fB \-M \fP \fIn\fP" +Set the minimum chunk size; only usable if nccopy supports netCDF-4 (enhanced). .IP "\fB \-F \fP \fIfilterspec\fP" For netCDF-4 output, including netCDF-4 classic model, specify a filter to apply to an specified variable in the output. As a rule, the filter diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index bae9930a9e..5f0f2cb8eb 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -1978,14 +1978,17 @@ usage(void) [-r] read whole input file into diskless file on open (classic or 64-bit offset or cdf5 formats only)\n\ [-F filterspec] specify the compression algorithm to apply to an output variable.\n\ [-Ln] set log level to n (>= 0); ignored if logging isn't enabled.\n\ + [-Mn] set minimum chunk size to n bytes (n >= 0)\n\ infile name of netCDF input file\n\ outfile name for netCDF output file\n" /* Don't document this flaky option until it works better */ /* [-x] use experimental computed estimates for variable-specific chunk caches\n\ */ - error("[-k kind] [-[3|4|6|7]] [-d n] [-s] [-c chunkspec] [-u] [-w] [-[v|V] varlist] [-[g|G] grplist] [-m n] [-h n] [-e n] [-r] infile outfile\n%s\nnetCDF library version %s", - USAGE, nc_inq_libvers()); + + error("%s [-k kind] [-[3|4|6|7]] [-d n] [-s] [-c chunkspec] [-u] [-w] [-[v|V] varlist] [-[g|G] grplist] [-m n] [-h n] [-e n] [-r] [-F filterspec] [-Ln] [-Mn] infile outfile\n%s\nnetCDF library version %s", + progname, USAGE, nc_inq_libvers()); + } int @@ -2160,7 +2163,7 @@ main(int argc, char**argv) case 'M': /* set min chunk size */ #ifdef USE_NETCDF4 if(optarg == NULL) - option_min_chunk_bytes = -1; + option_min_chunk_bytes = 0; else option_min_chunk_bytes = atol(optarg); if(option_min_chunk_bytes < 0) diff --git a/ncdump/ref_provenance_v1.nc b/ncdump/ref_provenance_v1.nc new file mode 100644 index 0000000000..2ccec37935 Binary files /dev/null and b/ncdump/ref_provenance_v1.nc differ diff --git a/ncdump/ref_tst_ncf213.cdl b/ncdump/ref_tst_ncf213.cdl index e298e74fc7..291e689015 100644 --- a/ncdump/ref_tst_ncf213.cdl +++ b/ncdump/ref_tst_ncf213.cdl @@ -44,7 +44,7 @@ variables: var5:_NoFill = "true" ; // global attributes: - :_NCProperties = "version=1|netcdflibversion=0.0.0|hdf5libversion=0.0.0" ; + :_NCProperties = "version=2,netcdf=4.6.2-development,hdf5=1.10.1" ; :_SuperblockVersion = 0 ; :_IsNetcdf4 = 1 ; :_Format = "netCDF-4" ; diff --git a/ncdump/ref_tst_special_atts.cdl b/ncdump/ref_tst_special_atts.cdl index 200ead025f..b846dde9d4 100644 --- a/ncdump/ref_tst_special_atts.cdl +++ b/ncdump/ref_tst_special_atts.cdl @@ -45,7 +45,7 @@ variables: var5:_NoFill = "true" ; // global attributes: - :_NCProperties = "version=1|netcdflibversion=0.0.0|hdf5libversion=0.0.0" ; + :_NCProperties = "version=2,netcdf=4.6.2-development,hdf5=1.10.1" ; :_SuperblockVersion = 0 ; :_IsNetcdf4 = 1 ; :_Format = "netCDF-4" ; diff --git a/ncdump/tst_fileinfo.c b/ncdump/tst_fileinfo.c index 6ae107a1b9..1748b8c1b8 100644 --- a/ncdump/tst_fileinfo.c +++ b/ncdump/tst_fileinfo.c @@ -113,9 +113,6 @@ main(int argc, char **argv) if(nc_def_grp(root,GROUPNAME,&grpid)!=0) ERR; /* Create global attribute in the group */ if(nc_put_att_int(grpid,NC_GLOBAL,INT_ATT_NAME,NC_INT,1,&data)!=0) ERR; - /* Create _NCProperties as var attr and as subgroup attribute */ - if(nc_put_att_text(grpid,NC_GLOBAL,NCPROPS,strlen(sdata),sdata)!=0) ERR; - if(nc_put_att_text(root,varid,NCPROPS,strlen(sdata),sdata)!=0) ERR; /* Create var + dimension to cause e.g. dimscales to appear */ if(nc_def_dim(root,DIMNAME,(size_t)4,&dimid)!=0) ERR; if(nc_def_var(root,DIMNAME,NC_INT,1,&dimid,&varid)!=0) ERR; /* same name */ diff --git a/ncdump/tst_fileinfo.sh b/ncdump/tst_fileinfo.sh index fb6c730b95..28bff55185 100755 --- a/ncdump/tst_fileinfo.sh +++ b/ncdump/tst_fileinfo.sh @@ -1,28 +1,34 @@ #!/bin/sh +# Author: Dennis Heimbigner if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh set -e -set -x echo "" EXIT=0 -NCF=${top_srcdir}/ncdump/nc4_fileinfo.nc -HDF=${top_srcdir}/ncdump/hdf5_fileinfo.hdf -NF=${top_srcdir}/ncdump/ref_tst_compounds4.nc +NCF="./nc4_fileinfo.nc" +HDF="./hdf5_fileinfo.hdf" + +NF="${top_srcdir}/ncdump/ref_tst_compounds4.nc" +NPV1="${top_srcdir}/ncdump/ref_provenance_v1.nc" + +# Create various files +${execdir}/tst_fileinfo # Do a false negative test -rm -f ./tmp_tst_fileinfo -if $NCDUMP -s $NF | fgrep '_IsNetcdf4 = 0' > ./tmp_tst_fileinfo ; then +rm -f ./tst_fileinfo.tmp +if $NCDUMP -s $NF | fgrep '_IsNetcdf4 = 0' > ./tst_fileinfo.tmp ; then echo "Pass: False negative for file: $NF" else echo "FAIL: False negative for file: $NF" + EXIT=1 fi -rm -f ./tmp_tst_fileinfo -if ${execdir}/tst_fileinfo > /dev/null ; then +rm -f ./tst_fileinfo.tmp +if test -e $NCF ; then # look at the _IsNetcdf4 flag N_IS=`${NCDUMP} -s $NCF | fgrep '_IsNetcdf4' | tr -d ' ;'` N_IS=`echo $N_IS | cut -d= -f2` @@ -30,16 +36,31 @@ if ${execdir}/tst_fileinfo > /dev/null ; then H_IS=`echo $H_IS | cut -d= -f2` if test "x$N_IS" = 'x0' ;then echo "FAIL: $NCF is marked as not netcdf-4" + EXIT=1 fi if test "x$H_IS" = 'x1' ;then echo "FAIL: $HDF is marked as netcdf-4" + EXIT=1 fi else -echo "FAIL: tst_fileinfo" -EXIT=1 + echo "FAIL: tst_fileinfo: $NCF does not exist" + EXIT=1 +fi + +# Test what happens when we read a file that used provenance version 1 +rm -f ./tst_fileinfo.tmp ./tst_fileinfo2.tmp +$NCDUMP -hs $NPV1 >tst_fileinfo2.tmp +fgrep '_NCProperties' ./tst_fileinfo.tmp +if ! fgrep 'version=1' tst_fileinfo.tmp ; then + echo "FAIL: $NPV1 is not marked as version=1" + EXIT=1 fi rm -f $NCF rm -f $HDF +rm -f *.tmp +if test "x$EXIT" = x0 ; then +echo "*** Pass all tests" +fi exit $EXIT diff --git a/ncdump/tst_nccopy3.sh b/ncdump/tst_nccopy3.sh index 449e896865..c657868970 100755 --- a/ncdump/tst_nccopy3.sh +++ b/ncdump/tst_nccopy3.sh @@ -34,6 +34,7 @@ fi echo "*** Testing netCDF-3 features of nccopy on ncdump/*.nc files" for i in $TESTFILES ; do echo "*** Testing nccopy $i.nc nccopy3_copy_of_$i.nc ..." +ls -l $i.nc ${NCCOPY} $i.nc nccopy3_copy_of_$i.nc ${NCDUMP} -n nccopy3_copy_of_$i $i.nc > tmp_tst_nccopy3.cdl ${NCDUMP} nccopy3_copy_of_$i.nc > nccopy3_copy_of_$i.cdl diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index 80bc6ac3e5..9b914a25bb 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -23,7 +23,7 @@ ${execdir}/tst_compress echo "*** Testing netCDF-4 features of nccopy on ncdump/*.nc files" for i in $TESTFILES ; do echo "*** Test nccopy $i.nc copy_of_$i.nc ..." - $NCCOPY $i.nc copy_of_$i.nc + ${NCCOPY} $i.nc copy_of_$i.nc ${NCDUMP} -n copy_of_$i $i.nc > tmp.cdl ${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl # echo "*** compare " with copy_of_$i.cdl @@ -33,27 +33,27 @@ done # echo "*** Testing compression of deflatable files ..." ./tst_compress echo "*** Test nccopy -d1 can compress a classic format file ..." -$NCCOPY -d1 tst_inflated.nc tst_deflated.nc +${NCCOPY} -d1 tst_inflated.nc tst_deflated.nc if test `wc -c < tst_deflated.nc` -ge `wc -c < tst_inflated.nc`; then exit 1 fi echo "*** Test nccopy -d1 can compress a netCDF-4 format file ..." -$NCCOPY -d1 tst_inflated4.nc tst_deflated.nc +${NCCOPY} -d1 tst_inflated4.nc tst_deflated.nc if test `wc -c < tst_deflated.nc` -ge `wc -c < tst_inflated4.nc`; then exit 1 fi echo "*** Test nccopy -d1 -s can compress a classic model netCDF-4 file even more ..." -$NCCOPY -d1 -s tst_inflated.nc tmp.nc +${NCCOPY} -d1 -s tst_inflated.nc tmp.nc if test `wc -c < tmp.nc` -ge `wc -c < tst_inflated.nc`; then exit 1 fi echo "*** Test nccopy -d1 -s can compress a netCDF-4 file even more ..." -$NCCOPY -d1 -s tst_inflated4.nc tmp.nc +${NCCOPY} -d1 -s tst_inflated4.nc tmp.nc if test `wc -c < tmp.nc` -ge `wc -c < tst_inflated4.nc`; then exit 1 fi echo "*** Test nccopy -d0 turns off compression, shuffling of compressed, shuffled file ..." -$NCCOPY -d0 tst_inflated4.nc tmp.nc +${NCCOPY} -d0 tst_inflated4.nc tmp.nc ${NCDUMP} -sh tmp.nc > tmp.cdl if fgrep '_DeflateLevel' < tmp.cdl ; then exit 1 @@ -66,7 +66,7 @@ rm tst_deflated.nc tst_inflated.nc tst_inflated4.nc tmp.nc tmp.cdl echo "*** Testing nccopy -d1 -s on ncdump/*.nc files" for i in $TESTFILES ; do echo "*** Test nccopy -d1 -s $i.nc copy_of_$i.nc ..." - $NCCOPY -d1 -s $i.nc copy_of_$i.nc + ${NCCOPY} -d1 -s $i.nc copy_of_$i.nc ${NCDUMP} -n copy_of_$i $i.nc > tmp.cdl ${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl # echo "*** compare " with copy_of_$i.cdl @@ -75,20 +75,20 @@ ${NCDUMP} copy_of_$i.nc > copy_of_$i.cdl done ./tst_chunking echo "*** Test that nccopy -c can chunk and unchunk files" -$NCCOPY tst_chunking.nc tmp.nc +${NCCOPY} -M0 tst_chunking.nc tmp.nc ${NCDUMP} tmp.nc > tmp.cdl -$NCCOPY -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ tst_chunking.nc tmp-chunked.nc +${NCCOPY} -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ tst_chunking.nc tmp-chunked.nc ${NCDUMP} -n tmp tmp-chunked.nc > tmp-chunked.cdl diff tmp.cdl tmp-chunked.cdl -$NCCOPY -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ tmp-chunked.nc tmp-unchunked.nc +${NCCOPY} -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ tmp-chunked.nc tmp-unchunked.nc ${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl diff tmp.cdl tmp-unchunked.cdl -$NCCOPY -c // tmp-chunked.nc tmp-unchunked.nc +${NCCOPY} -c // tmp-chunked.nc tmp-unchunked.nc ${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl diff tmp.cdl tmp-unchunked.cdl echo "*** Test that nccopy -c works as intended for record dimension default (1)" ${NCGEN} -b -o tst_bug321.nc $srcdir/tst_bug321.cdl -$NCCOPY -k nc7 -c"lat/2,lon/2" tst_bug321.nc tmp.nc +${NCCOPY} -k nc7 -c"lat/2,lon/2" tst_bug321.nc tmp.nc ${NCDUMP} -n tst_bug321 tmp.nc > tmp.cdl diff -b $srcdir/tst_bug321.cdl tmp.cdl # echo "*** Test that nccopy compression with chunking can improve compression" diff --git a/ncdump/tst_netcdf4.sh b/ncdump/tst_netcdf4.sh index 15c41721d3..a98e19bb79 100755 --- a/ncdump/tst_netcdf4.sh +++ b/ncdump/tst_netcdf4.sh @@ -7,6 +7,19 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +# Remove the version information from _NCProperties +cleanncprops() { + src="$1" + dst="$2" + rm -f $dst + cat $src \ + | sed -e 's/_SuperblockVersion = 1/_SuperblockVersion = 0/' \ + | sed -e 's/\(netcdflibversion\|netcdf\)=.*|/\1=NNNN|/' \ + | sed -e 's/\(hdf5libversion\|hdf5\)=.*"/\1=HHHH"/' \ + | grep -v '_NCProperties' \ + | cat >$dst +} + ERR() { RES=$? if [ $RES -ne 0 ]; then @@ -84,14 +97,11 @@ fi echo "*** Running tst_special_atts.c to create test files." ${execdir}/tst_special_atts ; ERR -${NCDUMP} -c -s tst_special_atts.nc \ - | sed 's/e+0/e+/g' \ - | sed -e 's/netcdflibversion=.*[|]/netcdflibversion=0.0.0|/' \ - | sed -e 's/hdf5libversion=.*"/hdf5libversion=0.0.0"/' \ - | sed -e 's|_SuperblockVersion = [0-9]|_SuperblockVersion = 0|' \ - | cat > tst_special_atts.cdl ; ERR +${NCDUMP} -c -s tst_special_atts.nc > tst_special_atts.cdl ; ERR +cleanncprops tst_special_atts.cdl tst_special_atts.tmp +cleanncprops $srcdir/ref_tst_special_atts.cdl ref_tst_special_atts.tmp echo "*** comparing tst_special_atts.cdl with ref_tst_special_atts.cdl..." -diff -b tst_special_atts.cdl $srcdir/ref_tst_special_atts.cdl ; ERR +diff -b tst_special_atts.tmp ref_tst_special_atts.tmp ; ERR #echo "" #echo "*** Testing ncdump on file with corrupted header " @@ -107,5 +117,6 @@ diff -b tst_special_atts.cdl $srcdir/ref_tst_special_atts.cdl ; ERR #echo "*** creating tst_output_irish_rover.cdl from ref_tst_irish_rover.nc..." #${NCDUMP} ref_tst_irish_rover.nc > tst_output_irish_rover.cdl +rm -f *.tmp echo "*** All ncgen and ncdump test output for netCDF-4 format passed!" exit 0 diff --git a/ncdump/tst_netcdf4_4.sh b/ncdump/tst_netcdf4_4.sh index 458cc1bd63..44d2c01156 100755 --- a/ncdump/tst_netcdf4_4.sh +++ b/ncdump/tst_netcdf4_4.sh @@ -2,11 +2,24 @@ # This shell script runs extra tests ncdump for netcdf-4 # Dennis Heimbigner, Ward Fisher -if test "x$srcdir" = x ; then srcdir=`pwd`; fi +if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh set -e +# Remove the version information from _NCProperties +cleanncprops() { + src="$1" + dst="$2" + rm -f $dst + cat $src \ + | sed -e 's/_SuperblockVersion = 1/_SuperblockVersion = 0/' \ + | sed -e 's/\(netcdflibversion\|netcdf\)=.*|/\1=NNNN|/' \ + | sed -e 's/\(hdf5libversion\|hdf5\)=.*"/\1=HHHH"/' \ + | grep -v '_NCProperties' \ + | cat >$dst +} + echo "" echo "*** Running extra netcdf-4 tests." @@ -17,7 +30,7 @@ echo "*** Running extra netcdf-4 tests." # 'ncdump ./ref_tst_compounds2'. This causes the test to fail. # But, 'srcdir' is necessary for make distcheck. # -# Short term solution, use sed when on windows/MSYS to +# Short term solution, use sed when on windows/MSYS to # remove the './','../../ncdump'. # # I am undoing this because libdispatch/dwinpath.c @@ -30,8 +43,10 @@ ${execdir}/tst_string_data echo "*** dumping tst_string_data.nc to tst_string_data.cdl..." ${NCDUMP} tst_string_data.nc > tst_string_data.cdl +cleanncprops tst_string_data.cdl tst_string_data.tmp +cleanncprops ${srcdir}/ref_tst_string_data.cdl ref_tst_string_data.tmp echo "*** comparing tst_string_data.cdl with ref_tst_string_data.cdl..." -diff -b tst_string_data.cdl ${top_srcdir}/ncdump/ref_tst_string_data.cdl +diff -b tst_string_data.tmp ref_tst_string_data.tmp #echo '*** testing non-coordinate variable of same name as dimension...' #${NCGEN} -v4 -b -o tst_noncoord.nc ${top_srcdir}/ncdump/ref_tst_noncoord.cdl @@ -50,17 +65,19 @@ diff -b tst_compounds4.cdl ${top_srcdir}/ncdump/ref_tst_compounds4.cdl # Exercise Jira NCF-213 bug fix # rm -f tst_ncf213.cdl tst_ncf213.nc -${NCGEN} -b -o tst_ncf213.nc ${top_srcdir}/ncdump/ref_tst_ncf213.cdl -${NCDUMP} -s -h tst_ncf213.nc \ - | sed -e 's/netcdflibversion=.*[|]/netcdflibversion=0.0.0|/' \ - | sed -e 's/hdf5libversion=.*"/hdf5libversion=0.0.0"/' \ - | sed -e 's|_SuperblockVersion = [0-9]|_SuperblockVersion = 0|' \ - | cat >tst_ncf213.cdl +# Remove specific _NCProperties values +${NCGEN} -b -o tst_ncf213.nc $srcdir/ref_tst_ncf213.cdl +${NCDUMP} -s -h tst_ncf213.nc > tst_ncf213.cdl +cleanncprops tst_ncf213.cdl tst_ncf213.tmp +cleanncprops ${srcdir}/ref_tst_ncf213.cdl ref_tst_ncf213.tmp # Now compare ok=1; -if diff -b ${top_srcdir}/ncdump/ref_tst_ncf213.cdl tst_ncf213.cdl ; then ok=1; else ok=0; fi +if diff -b tst_ncf213.tmp ref_tst_ncf213.tmp ; then ok=1; else ok=0; fi + # cleanup -# rm -f tst_ncf213.cdl tst_ncf213.nc +#rm -f tst_ncf213.cdl tst_ncf213.nc +rm -f *.tmp + if test $ok = 0 ; then echo "*** FAIL: NCF-213 Bug Fix test" exit 1 @@ -68,4 +85,3 @@ fi echo "*** All ncgen and ncdump extra test output for netCDF-4 format passed!" exit 0 - diff --git a/ncgen/genc.c b/ncgen/genc.c index 58eaf4e81d..727f09c483 100644 --- a/ncgen/genc.c +++ b/ncgen/genc.c @@ -517,7 +517,7 @@ genc_definespecialattributes(Symbol* vsym) level = 9; /* default */ else level = special->_FilterParams[0]; - if(level < 0 || level > 9) + if(level > 9) derror("Illegal deflate level"); else { bbprintf0(stmt, @@ -525,7 +525,7 @@ genc_definespecialattributes(Symbol* vsym) groupncid(vsym->container), varncid(vsym), (special->_Shuffle == 1?"NC_SHUFFLE":"NC_NOSHUFFLE"), - (level >= 0?1:0), + 1, level); codedump(stmt); } diff --git a/ncgen/main.c b/ncgen/main.c index 8af6909cd3..90ebbaf17e 100644 --- a/ncgen/main.c +++ b/ncgen/main.c @@ -212,6 +212,7 @@ usage(void) " [-x]" " [-N datasetname]" " [-L loglevel]" +" [-H]" " [file ... ]", progname); derror("netcdf library version %s", nc_inq_libvers()); @@ -293,10 +294,10 @@ main( } l_flag = L_BINARY; break; - case 'h': + case 'H': header_only = 1; break; - case 'H': + case 'h': usage(); goto done; case 'l': /* specify language, instead of using -c or -f or -b */ diff --git a/oc2/ochttp.c b/oc2/ochttp.c index 3f0d772f79..cbb1b76cba 100644 --- a/oc2/ochttp.c +++ b/oc2/ochttp.c @@ -312,7 +312,7 @@ ocping(const char* url) if (cstat != CURLE_OK) goto done; - /* use a very short timeout: 10 seconds */ + /* use a very short timeout: 5 seconds */ cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)5)); if (cstat != CURLE_OK) goto done;