Improve S3 documentation, testing and support

.github/workflows/run_tests_s3.yml

@@ -0,0 +1,153 @@
+###
+# Test S3 Support
+#  -- derived from run_tests_ubuntu.yml
+###
+
+###
+# Build hdf5 dependencies and cache them in a combined directory.
+###
+
+name: Run S3 netCDF Tests (under Ubuntu Linux)
+
+on: [workflow_dispatch]
+
+jobs:
+
+  build-deps-serial:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        hdf5: [ 1.10.8, 1.12.2, 1.14.0 ]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install System dependencies
+        shell: bash -l {0}
+        run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev
+
+        ###
+        # Installing libhdf5
+        ###
+      - name: Cache libhdf5-${{ matrix.hdf5 }}
+        id: cache-hdf5
+        uses: actions/cache@v3
+        with:
+          path: ~/environments/${{ matrix.hdf5 }}
+          key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }}
+
+
+      - name: Build libhdf5-${{ matrix.hdf5 }}
+        if: steps.cache-hdf5.outputs.cache-hit != 'true'
+        run: |
+          set -x
+
+          wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2
+          tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2
+          pushd hdf5-${{ matrix.hdf5 }}
+          ./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} --enable-hl --with-szlib
+          make -j
+          make install -j
+          popd
+          
+
+  #####
+  # S3 Autotools-based tests.
+  #####
+  ##
+  # Serial
+  ##
+  nc-ac-tests-s3-serial:
+
+    needs: build-deps-serial
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        hdf5: [ 1.14.0 ]
+    steps:
+
+      - uses: actions/checkout@v3
+
+      - name: Install System dependencies
+        shell: bash -l {0}
+        run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev openssl libssl-dev
+
+        ###
+        # Set Environmental Variables
+        ###
+
+      - run: echo "CFLAGS=-I${HOME}/environments/${{ matrix.hdf5 }}/include" >> $GITHUB_ENV
+      - run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV
+      - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV
+
+
+        ###
+        # Fetch Cache
+        ###
+
+      - name: Fetch HDF Cache
+        id: cache-hdf
+        uses: actions/cache@v3
+        with:
+          path: ~/environments/${{ matrix.hdf5 }}
+          key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }}
+
+      - name: Check Cache
+        shell: bash -l {0}
+        run: ls ${HOME}/environments && ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib
+
+        ###
+        # Configure and build
+        ###
+
+      - name: Run autoconf
+        shell: bash -l {0}
+        run: autoreconf -if
+
+      - name: Configure
+        shell: bash -l {0}
+        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf5 --disable-dap --enable-external-server-tests --enable-s3 --enable-s3-internal --with-s3-testing=public
+        if: ${{ success() }}
+
+      - name: Look at config.log if error
+        shell: bash -l {0}
+        run: cat config.log
+        if: ${{ failure() }}
+
+      - name: Print Summary
+        shell: bash -l {0}
+        run: cat libnetcdf.settings
+
+      - name: Build Library and Utilities
+        shell: bash -l {0}
+        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j
+        if: ${{ success() }}
+
+      - name: Build Tests
+        shell: bash -l {0}
+        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j
+        if: ${{ success() }}
+
+      - name: Run Tests
+        shell: bash -l {0}
+        env:
+          AWS_PROFILE: ${{ secrets.DEFAULT_PROFILE }}
+        run: |
+          mkdir -p ~/.aws
+          echo "" > ~/.aws/config
+          chmod go-x ~/.aws/config
+          echo "${AWS_PROFILE}" >> ~/.aws/config
+          LD_LIBRARY_PATH="/home/runner/work/netcdf-c/netcdf-c/liblib/.libs:${LD_LIBRARY_PATH}"
+          CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j
+        if: ${{ success() }}
+
+  #####
+  # S3 CMake-based tests.
+  #####
+  ##
+  # Serial
+  ##
+  # T.B.D. nc-cmake-tests-s3-serial:

.github/workflows/run_tests_ubuntu.yml

@@ -17,12 +17,11 @@ jobs:
         hdf5: [ 1.10.8, 1.12.2, 1.14.0 ]
 
     steps:
-
       - uses: actions/checkout@v3
 
       - name: Install System dependencies
         shell: bash -l {0}
-        run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen
+        run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen openssl
 
         ###
         # Installing libhdf5
@@ -132,7 +131,7 @@ jobs:
 
       - name: Install System dependencies
         shell: bash -l {0}
-        run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen
+        run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen valgrind
 
         ###
         # Set Environmental Variables
@@ -192,7 +191,9 @@ jobs:
 
       - name: Run Tests
         shell: bash -l {0}
-        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j
+        run: |
+          LD_LIBRARY_PATH="/home/runner/work/netcdf-c/netcdf-c/liblib/.libs:${LD_LIBRARY_PATH}"
+          CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j
         if: ${{ success() }}
 
   ##
@@ -627,4 +628,3 @@ jobs:
           cd build
           LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 --rerun-failed --output-on-failure -VV
         if: ${{ failure() }}
-

.github/workflows/run_tests_win_cygwin.yml

@@ -30,6 +30,7 @@ jobs:
             libhdf4-devel zipinfo libxml2-devel perl zlib-devel
             libzstd-devel libbz2-devel libaec-devel libzip-devel
             libdeflate-devel gcc-core libcurl-devel libiconv-devel
+            libssl-devel libcrypt-devel
 
       - name: (Autotools) Run autoconf and friends
         run: |
@@ -40,10 +41,9 @@ jobs:
 
       - name: (Autotools) Configure in-tree build
         run: >-
-          /bin/dash ./configure --enable-hdf5 --enable-shared
-          --disable-static --enable-dap --disable-dap-remote-tests
-          --enable-plugins --disable-nczarr-filters
-          --disable-nczarr-s3 --disable-nczarr-s3-tests --disable-nczarr
+           /bin/dash ./configure --enable-hdf5 --enable-shared
+           --disable-static --enable-dap --disable-dap-remote-tests
+           --enable-plugins --enable-nczarr
 
       - name: Look at config.log if error
         if: ${{ failure() }}
@@ -66,4 +66,5 @@ jobs:
 
       - name: (Autotools) Build and run tests
         timeout-minutes: 30
-        run: make check -j8 SHELL=/bin/dash
+        run: |
+          make check -j8 SHELL=/bin/dash

.github/workflows/run_tests_win_mingw.yml

@@ -37,7 +37,7 @@ jobs:
         run: autoreconf -if
 
       - name: (Autotools) Configure Build
-        run: ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --disable-static --disable-byterange --disable-dap-remote-tests --disable-logging --enable-plugins --disable-nczarr-filters --disable-nczarr-s3 --disable-nczarr-s3-tests
+        run: ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --disable-static --disable-byterange --disable-dap-remote-tests --disable-logging --enable-plugins --disable-nczarr-filters --disable-s3
         if: ${{ success() }}
 
       - name: (Autotools) Look at config.log if error
@@ -54,7 +54,7 @@ jobs:
       - name: Check for plugins
         run: |
           dir ./plugins
-          if test -e ./plugins/.libs ; then dir ./plugins/.libs ; fi
+          if test -f ./plugins/.libs ; then dir ./plugins/.libs; fi
 
       - name: (Autotools) Build and Run Tests
         run: make check -j 8 LDFLAGS="-Wl,--export-all-symbols"

CMakeLists.txt

@@ -1270,47 +1270,66 @@ ENDIF()
 
 # Options for S3 Support
 OPTION(ENABLE_S3 "Enable S3 support." OFF)
+OPTION(ENABLE_S3_INTERNAL "Enable S3 Internal support." OFF)
 OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF)
-OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF)
+
+# Control S3 Testing: Multi-valued option
+SET(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none)  PUBLIC")
+    SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC)  #
+IF(WITH_S3_TESTING STREQUAL "")
+  SET(WITH_S3_TESTING OFF CACHE STRING "") # Default
+ENDIF()
+
+IF(WITH_S3_TESTING)
+message(WARNING "**** DO NOT USE WITH_S3_TESTING=ON UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***")
+ENDIF()
 
 # ENABLE_NCZARR_S3 is now an alias for ENABLE_S3 (but...)
 if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3)
   SET(ENABLE_S3 ON CACHE BOOL "NCARR S3" FORCE) # For back compatibility
 ENDIF()
 UNSET(ENABLE_NCZARR_S3)
 
-IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_S3)
-  message(FATAL_ERROR "S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=NO")
-  SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCZARR S3 TESTS" FORCE)
-ENDIF()
-
 # Note we check for the library after checking for enable_s3
 # because for some reason this screws up if we unconditionally test for sdk
 # and it is not available. Fix someday
 IF(ENABLE_S3)
-  # See if aws-s3-sdk is available
-  find_package(AWSSDK REQUIRED COMPONENTS s3;core)
-  IF(AWSSDK_FOUND)
-    SET(service s3;core)
-    AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES)
-    SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE)
-  ELSE()
-    SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
+  IF(NOT ENABLE_S3_INTERNAL)
+    # See if aws-s3-sdk is available
+    find_package(AWSSDK REQUIRED COMPONENTS s3;core)
+    IF(AWSSDK_FOUND)
+      SET(service s3;core)
+      AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES)
+      SET(ENABLE_S3_AWS ON CACHE BOOL "S3 AWS" FORCE)
+    ELSE()
+      SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
+    ENDIF()
   ENDIF()
 ELSE()
-  SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
+  SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
 ENDIF()
 
-IF(NOT ENABLE_S3_SDK)
-  IF(ENABLE_S3)
+# Unless/until we get aws-sdk-cpp working for Windows, force use of internal
+IF(ENABLE_S3 AND MSVC)
+  SET(ENABLE_S3_INTERNAL ON CACHE BOOL "S3 Intern" FORCE)
+ENDIF()
+
+IF(ENABLE_S3)
+  IF(NOT ENABLE_S3_AWS AND NOT ENABLE_S3_INTERNAL)
     message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_S3=NO")
-    SET(ENABLE_S3 OFF CACHE BOOL "S3 support" FORCE)
-    SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
+    SET(ENABLE_S3 OFF CACHE BOOL "S3 support" FORCE)        
+  ENDIF()
+  IF(ENABLE_S3_AWS AND ENABLE_S3_INTERNAL)
+    message(WARNING "Both aws-sdk-cpp and s3-internal enabled => use s3-internal")
+    SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)        
   ENDIF()
 ENDIF()
 
-IF(ENABLE_NCZARR_S3_TESTS)
-  message(WARNING "**** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***")
+IF(NOT ENABLE_S3)
+IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING)
+  message(WARNING "S3 support is disabled => WITH_S3_TESTING=OFF")
+  SET(WITH_S3_TESTING OFF CACHE STRING "" FORCE)
+ENDIF()
 ENDIF()
 
 # Start disabling if curl not found
@@ -1890,6 +1909,7 @@ ENDIF()
 # Check for various functions.
 CHECK_FUNCTION_EXISTS(fsync HAVE_FSYNC)
 CHECK_FUNCTION_EXISTS(strlcat   HAVE_STRLCAT)
+CHECK_FUNCTION_EXISTS(strlcpy  HAVE_STRLCPY)
 CHECK_FUNCTION_EXISTS(strdup  HAVE_STRDUP)
 CHECK_FUNCTION_EXISTS(strndup HAVE_STRNDUP)
 CHECK_FUNCTION_EXISTS(strtoll HAVE_STRTOLL)
@@ -2298,6 +2318,10 @@ ENDIF()
 IF(ENABLE_DAP4)
   ADD_SUBDIRECTORY(libdap4)
   ADD_SUBDIRECTORY(libncxml)
+ELSE()
+  IF(ENABLE_S3_INTERNAL)
+    ADD_SUBDIRECTORY(libncxml)
+  ENDIF()
 ENDIF()
 
 IF(ENABLE_PLUGINS)
@@ -2540,9 +2564,10 @@ is_enabled(USE_CDF5 HAS_CDF5)
 is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
 is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
 is_enabled(ENABLE_S3 HAS_S3)
-is_enabled(ENABLE_S3_SDK HAS_S3_SDK)
+is_enabled(ENABLE_S3_AWS HAS_S3_AWS)
+is_enabled(ENABLE_S3_INTERNAL HAS_S3_INTERNAL)
+is_enabled(HAS_HDF5_ROS3 HAS_HDF5_ROS3)
 is_enabled(ENABLE_NCZARR HAS_NCZARR)
-is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
 is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP)
 is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
 is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
@@ -2555,6 +2580,26 @@ is_enabled(HAVE_ZSTD HAS_ZSTD)
 is_enabled(HAVE_BLOSC HAS_BLOSC)
 is_enabled(HAVE_BZ2 HAS_BZ2)
 
+if(ENABLE_S3_INTERNAL)
+SET(WHICH_S3_SDK "internal")
+SET(NC_WHICH_S3_SDK "internal")
+elseif(ENABLE_S3_AWS)
+SET(WHICH_S3_SDK "aws-sdk-cpp")
+SET(NC_WHICH_S3_SDK "aws-sdk-cpp")
+else()
+SET(WHICH_S3_SDK "none")
+SET(NC_WHICH_S3_SDK "none")
+endif()
+
+if(WITH_S3_TESTING STREQUAL PUBLIC)
+SET(DO_S3_TESTING "public")
+elseif(WITH_S3_TESTING)
+SET(DO_S3_TESTING "yes")
+elseif(NOT WITH_S3_TESTING)
+SET(DO_S3_TESTING "no")
+else()
+SET(DO_S3_TESTING "no")
+endif()
 
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"