Merge branch 'master' into dapcurlopt.dmh

CMakeLists.txt

@@ -315,6 +315,8 @@ SET(CHUNK_CACHE_PREEMPTION 0.75 CACHE STRING "Default file chunk cache preemptio
 SET(MAX_DEFAULT_CACHE_SIZE 67108864 CACHE STRING "Default maximum cache size.")
 SET(NETCDF_LIB_NAME "" CACHE STRING "Default name of the netcdf library.")
 SET(TEMP_LARGE "." CACHE STRING "Where to put large temp files if large file tests are run.")
+SET(NCPROPERTIES_EXTRA "" CACHE STRNG "Specify extra pairs for _NCProperties.")
+SET(MPIEXEC "mpiexec" CACHE STRING "Command to run MPI programs if parallel tests are run.")
 
 IF(NOT NETCDF_LIB_NAME STREQUAL "")
   SET(MOD_NETCDF_NAME ON)
@@ -735,6 +737,7 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4)
   ENDIF()
 
   CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Pset_libver_bounds "" HDF5_HAS_LIBVER_BOUNDS)
+  CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5free_memory "" HDF5_HAS_H5FREE)
 
   IF(HDF5_PARALLEL)
 	SET(HDF5_CC h5pcc)
@@ -986,7 +989,7 @@ IF(NOT ENABLE_NETCDF_4 AND ENABLE_EXAMPLE_TESTS)
   SET(ENABLE_EXAMPLE_TESTS OFF)
 ENDIF()
 
-# Enable Parallel (different than pnetcdf).
+# Enable Parallel IO with netCDF-4/HDF5 files using HDF5 parallel I/O.
 SET(STATUS_PARALLEL "OFF")
 OPTION(ENABLE_PARALLEL4 "Build netCDF-4 with parallel IO" "${HDF5_PARALLEL}")
 IF(ENABLE_PARALLEL4 AND ENABLE_NETCDF_4)
@@ -998,10 +1001,16 @@ IF(ENABLE_PARALLEL4 AND ENABLE_NETCDF_4)
     SET(USE_PARALLEL ON CACHE BOOL "")
     SET(USE_PARALLEL4 ON CACHE BOOL "")
     SET(STATUS_PARALLEL "ON")
+    configure_file("${netCDF_SOURCE_DIR}/nc_test4/run_par_test.sh.in"
+      "${netCDF_BINARY_DIR}/tmp/run_par_test.sh" @ONLY NEWLINE_STYLE LF)
+    FILE(COPY "${netCDF_BINARY_DIR}/tmp/run_par_test.sh"
+      DESTINATION ${netCDF_BINARY_DIR}/nc_test4
+      FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+
   ENDIF()
 ENDIF()
 
-# Options to enable parallel IO, tests.
+# Options to enable parallel IO for classic formats with parallel-netcdf library.
 SET(STATUS_PNETCDF "OFF")
 OPTION(ENABLE_PNETCDF "Build with parallel I/O for classic and 64-bit offset files using parallel-netcdf." OFF)
 
@@ -1984,6 +1993,12 @@ IF(ENABLE_EXAMPLES)
 configure_file(${CMAKE_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF)
 ENDIF()
 
+#####
+# Build ncdap_test|dap4_test/findtestserver[4].c
+#####
+configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF)
+configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF)
+
 ####
 # Export files
 ####

RELEASE_NOTES.md

@@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.7.0 - TBD
 
-
+* [Enhancement] Create a new version of _NCProperties provenance attribute. This version (version 2) supports arbitrary key-value pairs. It is the default when new files are created. Version 1 continues to be accepted.
 * [Enhancement] Allow user to set http read buffersize for DAP2 and  DAP4 using the tag HTTP.READ.BUFFERSIZE in the .daprc file.
 * [Enhancement] Allow user to set http keepalive for DAP2 and  DAP4 using the tag HTTP.KEEPALIVE in the .daprc file (see the OPeNDAP documentation for details).
 * [Enhancement] Support DAP4 remote tests using a new remote test server locatedon the Unidata JetStream project.

cf

@@ -3,7 +3,7 @@
 DB=1
 #X=-x
 
-#FAST=1
+FAST=1
 #PROF=1
 
 HDF5=1
@@ -108,10 +108,10 @@ FLAGS="$FLAGS --enable-extreme-numbers"
 #FLAGS="$FLAGS --disable-testsets"
 #FLAGS="$FLAGS --disable-dap-remote-tests"
 #FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server
-#FLAGS="$FLAGS --enable-doxygen"
-#FLAGS="$FLAGS --enable-internal-docs"
+FLAGS="$FLAGS --enable-doxygen"
+FLAGS="$FLAGS --enable-internal-docs"
 FLAGS="$FLAGS --enable-logging"
-FLAGS="$FLAGS --disable-diskless"
+#FLAGS="$FLAGS --disable-diskless"
 #FLAGS="$FLAGS --enable-mmap"
 #FLAGS="$FLAGS --with-udunits"
 #FLAGS="$FLAGS --with-libcf"
@@ -120,6 +120,7 @@ FLAGS="$FLAGS --disable-diskless"
 #FLAGS="$FLAGS --disable-silent-rules"
 #FLAGS="$FLAGS --disable-filter-testing"
 #FLAGS="$FLAGS --enable-metadata-perf"
+#FLAGS="$FLAGS --with-ncproperties-extra=key1=value1,key2=value2"
 
 if test "x$TESTSERVERS" != x ; then
 FLAGS="$FLAGS --with-testservers=$TESTSERVERS"
@@ -184,9 +185,6 @@ CFLAGS="${CFLAGS} -pg"
 LDFLAGS="${LDFLAGS} -pg"
 fi
 
-
-#FLAGS="${FLAGS} --enable-stdio"
-
 export PATH
 export CC
 export CPPFLAGS
@@ -199,17 +197,13 @@ export CXXFLAGS
 DISTCHECK_CONFIGURE_FLAGS="$FLAGS"
 export DISTCHECK_CONFIGURE_FLAGS
 
-if test "x$NB" != x -o "x$FAST" = x ; then
+if test "x$NB" != x1 && test "x$FAST" != x1 ; then
 ${MAKE} distclean >/dev/null 2>&1
 fi
-if test -z "$NB" ; then
+if test "x$NB" != x1 ; then
   if autoreconf -i --force ; then ok=1; else exit ; fi
 fi
 
-if test -z "$FAST" ; then
-  if test -f Makefile ; then ${MAKE} distclean >/dev/null 2>&1 ; fi
-fi
-
 sh $X ./configure ${FLAGS}
 for c in $cmds; do
 printenv LD_LIBRARY_PATH

cf.cmake

@@ -6,6 +6,8 @@ DAP=1
 #CDF5=1
 #HDF4=1
 
+export SETX=1
+
 for arg in "$@" ; do
 case "$arg" in
 vs|VS) VS=1 ;;
@@ -60,14 +62,17 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true"
 FLAGS="$FLAGS -DENABLE_LOGGING=true"
 #FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true"
 #FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true"
-FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
+#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
 
 # Disables
 FLAGS="$FLAGS -DENABLE_EXAMPLES=false"
 FLAGS="$FLAGS -DENABLE_CONVERSION_WARNINGS=false"
 #FLAGS="$FLAGS -DENABLE_TESTS=false"
 #FLAGS="$FLAGS -DENABLE_DISKLESS=false"
 
+# Withs
+FLAGS="$FLAGS -DNCPROPERTIES_EXTRA=\"key1=value1|key2=value2\""
+
 rm -fr build
 mkdir build
 cd build

config.h.cmake.in

@@ -381,6 +381,9 @@ are set when opening a binary file on Windows. */
 /* min blocksize for posixio. */
 #cmakedefine NCIO_MINBLOCKSIZE ${NCIO_MINBLOCKSIZE}
 
+/* Add extra properties to _NCProperties attribute */
+#cmakedefine NCPROPERTIES_EXTRA ${NCPROPERTIES_EXTRA}
+
 /* no IEEE float on this platform */
 #cmakedefine NO_IEEE_FLOAT 1
 

configure.ac

@@ -1,16 +1,13 @@
 #                                               -*- Autoconf -*-
 ## Process this file with autoconf to produce a configure script.
 
-# This is part of Unidata's netCDF package. Copyright 2005-2012, see
+# This is part of Unidata's netCDF package. Copyright 2005-2018, see
 # the COPYRIGHT file for more information.
 # Ed Hartnett, Ward Fisher, Dennis Heimbigner
 
 # Recall that ${VAR-exp} expands to $VAR if var is set (even to null),
 # and to exp otherwise.
 
-## This puts the cvs ID tag in the output configure script.
-AC_REVISION([$Id: configure.ac,v 1.450 2010/05/28 19:42:47 dmh Exp $])
-
 # Running autoconf on this file will trigger a warning if
 # autoconf is not at least the specified version.
 AC_PREREQ([2.59])
@@ -172,8 +169,12 @@ AC_DEFINE([USE_FSYNC], [1], [if true, include experimental fsync code])
 fi
 
 # Temporary until JNA bug is fixed (which is probably never).
-# See Jira NCF-298
-AC_MSG_CHECKING([if jna bug workaround is enabledd])
+# The problem being solved is this:
+# > On Windows using the microsoft runtime, it is an error
+# > for one library to free memory allocated by a different library.
+# This is probably only an issue when using the netcdf-c library
+# via JNA under Java.
+AC_MSG_CHECKING([if jna bug workaround is enabled])
 AC_ARG_ENABLE([jna],
               [AS_HELP_STRING([--enable-jna],
                               [enable jna bug workaround])],
@@ -255,6 +256,15 @@ AC_ARG_ENABLE([parallel-tests],
 test "x$enable_parallel_tests" = xyes || enable_parallel_tests=no
 AC_MSG_RESULT($enable_parallel_tests)
 
+# Did the user specify an MPI launcher other than mpiexec?
+AC_MSG_CHECKING([whether a user specified program to run mpi programs])
+AC_ARG_WITH([mpiexec],
+              [AS_HELP_STRING([--with-mpiexec=<command>],
+                              [Specify command to launch MPI parallel tests.])],
+            [MPIEXEC=$with_mpiexec], [MPIEXEC=mpiexec])
+AC_MSG_RESULT([$MPIEXEC])
+AC_SUBST([MPIEXEC], [$MPIEXEC])
+
 # Did the user specify a default chunk size?
 AC_MSG_CHECKING([whether a default chunk size in bytes was specified])
 AC_ARG_WITH([default-chunk-size],
@@ -592,6 +602,18 @@ AC_MSG_RESULT($TEMP_LARGE)
 #AC_SUBST(TEMP_LARGE)
 AC_DEFINE_UNQUOTED([TEMP_LARGE], ["$TEMP_LARGE"], [Place to put very large netCDF test files.])
 
+# Specify extra values to add to _NCProperties attribute
+# --with-ncproperties-extra="<name>=<value>|...".
+# Note: need to figure out a way to do this programmatically also
+AC_MSG_CHECKING([Extra values for _NCProperties])
+AC_ARG_WITH([ncproperties-extra],
+            [AS_HELP_STRING([--with-ncproperties-extra="<name>=<value>,...],
+                            [specify extra pairs for _NCProperties])],
+            [NCPROPERTIES_EXTRA=$with_ncproperties_extra],
+            [NCPROPERTIES_EXTRA=""])
+AC_MSG_RESULT([$NCPROPERTIES_EXTRA])
+AC_DEFINE_UNQUOTED([NCPROPERTIES_EXTRA], ["$NCPROPERTIES_EXTRA"], [Extra pairs for _NCProperties])
+
 # Did the user specify a user-defined format 0?
 AC_MSG_CHECKING([whether user-defined format 0 was specified])
 AC_ARG_WITH([udf0],
@@ -1505,13 +1527,18 @@ AC_SUBST([MSVC], [])
 AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
 AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
 
+# DAP 2/4 findtestserver[4].c setup
+AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)
+AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
+
 #####
 # End netcdf_meta.h definitions.
 #####
 
 AC_MSG_NOTICE([generating header files and makefiles])
+AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh])
+AC_CONFIG_FILES([nc-config], [chmod 755 nc-config])
 AC_CONFIG_FILES([Makefile
-                 nc-config
                  netcdf.pc
                  libnetcdf.settings
                  postinstall.sh
@@ -1548,11 +1575,7 @@ AC_CONFIG_FILES([Makefile
                  ncdap_test/expectremote3/Makefile
                  dap4_test/Makefile
                  plugins/Makefile
-                 ],
-                 [test -f  nc-config && chmod 755 nc-config])
-
+                 ])
 AC_OUTPUT()
 
-    #mv -f ${abs_top_srcdir}/test_common.sh ${abs_top_builddir}/test_common.sh
-
 cat libnetcdf.settings

dap4_test/Makefile.am

@@ -8,8 +8,11 @@ include $(top_srcdir)/lib_flags.am
 
 #TEST_EXTENSIONS = .sh
 
+#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
 #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
-
+#TESTS_ENVIRONMENT = export SETX=1;
+
 LDADD = ${top_builddir}/liblib/libnetcdf.la
 AM_CPPFLAGS += -I$(top_srcdir)/libdap4
 
@@ -62,6 +65,8 @@ CLEANFILES = *.exe
 # This should only be left behind if using parallel io
 CLEANFILES += tmp_*
 
+DISTCLEANFILES = findtestserver4.c
+
 # One last thing
 BUILT_SOURCES = .daprc
 

dap4_test/d4test_common.sh

@@ -24,8 +24,8 @@ cd ${srcdir}/baselineremote; BASELINEREM=`pwd` ; cd ${WD}
 BASELINEH=${BASELINEREM}
 
 setresultdir() {
-rm -fr ./$1
-mkdir -p ./$1
+rm -fr ${builddir}/$1
+mkdir -p ${builddir}/$1
 }
 
 FAILURES=

dap4_test/test_raw.sh

@@ -42,29 +42,29 @@ resultclean() {
 }
 
 setresultdir results_test_raw
+
 if test "x${RESET}" = x1 ; then rm -fr ${BASELINERAW}/*.dmp ; fi
 for f in $F ; do
     echo "testing: $f"
-    URL="[dap4]file://${DAPTESTFILES}/${f}"
-    if ! ${VG} ${NCDUMP} "${URL}" > ./results_test_raw/${f}.dmp; then
+    URL="[log][dap4]file://${DAPTESTFILES}/${f}"
+    if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_raw/${f}.dmp; then
         failure "${URL}"
     fi
     if test "x${TEST}" = x1 ; then
-	if ! diff -wBb ${BASELINERAW}/${f}.dmp ./results_test_raw/${f}.dmp ; then
+	if ! diff -wBb ${BASELINERAW}/${f}.dmp ${builddir}/results_test_raw/${f}.dmp ; then
 	    failure "diff ${f}.dmp"
 	fi
     elif test "x${RESET}" = x1 ; then
 	echo "${f}:"
-	cp ./results_test_raw/${f}.dmp ${BASELINERAW}/${f}.dmp
+	cp ${builddir}/results_test_raw/${f}.dmp ${BASELINERAW}/${f}.dmp
     elif test "x${DIFF}" = x1 ; then
 	echo "hdrtest: ${f}"
-	rm -f ./tr1 ./tr2 ./tb1 ./tb2
 	baseclean
-        if ! diff -wBb ./${BASELINERAW}/${f}.dmp ./${BASELINE}/${f}.ncdump ; then
-          failure diff -wBb ./${BASELINERAW}/${f}.dmp ./${BASELINE}/${f}.ncdump
+        if ! diff -wBb ${BASELINERAW}/${f}.dmp ${BASELINE}/${f}.ncdump ; then
+          failure diff -wBb ${BASELINERAW}/${f}.dmp ${BASELINE}/${f}.ncdump
 	fi
     fi
 done
-rm -rf ./results_test_raw
+rm -rf ${builddir}/results_test_raw
 
 finish