HDF5: Update to 1.14.3 (#7908)

* Update HDF5 to 1.14.3, disable Fortran * Remove OpenMP dependency * HDF5: Build Fortran again * HDF5: Do not use lld * HDF5: Correct FreeBSD, OpenMPI problems * FreeBSD: Debug __float128 * HDF5: Debug FreeBSD * HDF5: Correct FreeBSD Fortran settings * HDF5: Add CompilerSupportLibraries_jll for libgfortran * HDF5: Correct MPItrampoline setup * Build Windows using cross compile, remove msys2 sources * Fix LibraryProduct indent, use Fortran * Add lib64 path to FCFLAGS * Disable Microsoft MPI, refactor lib64 library flag support * HDF5: Clean up build script --------- Co-authored-by: Mark Kittisopikul <markkitt@gmail.com>
JuliaPackaging · Jan 8, 2024 · 8e03326 · 8e03326
1 parent 228a143
commit 8e03326
Show file tree

Hide file tree

Showing 67 changed files with 2,401 additions and 9,816 deletions.
diff --git a/H/HDF5/build_tarballs.jl b/H/HDF5/build_tarballs.jl
@@ -6,54 +6,18 @@ const YGGDRASIL_DIR = "../.."
 include(joinpath(YGGDRASIL_DIR, "platforms", "mpi.jl"))
 
 name = "HDF5"
-version = v"1.14.2"
+version = v"1.14.3"
 
 # Collection of sources required to complete build
 sources = [
     ArchiveSource("https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(version.major).$(version.minor)/hdf5-$(version)/src/hdf5-$(version).tar.bz2",
-                  "ea3c5e257ef322af5e77fc1e52ead3ad6bf3bb4ac06480dd17ee3900d7a24cfb"),
+                  "9425f224ed75d1280bb46d6f26923dd938f9040e7eaebf57e66ec7357c08f917"),
     DirectorySource("./bundled"),
-
-    # We don't build HDF5 on Windows; instead, we use packages from msys there:
-
-    # 32-bit Windows from https://packages.msys2.org/package/mingw-w64-i686-hdf5
-    ArchiveSource("https://mirror.msys2.org/mingw/mingw32/mingw-w64-i686-hdf5-1.14.2-2-any.pkg.tar.zst",
-                  "ab053fdafb3e0c456751fed9fe5cc2fa339042046b4de677ce2848cd8b6d3b3f"; unpack_target="i686-w64-mingw32"),
-    ArchiveSource("https://mirror.msys2.org/mingw/mingw32/mingw-w64-i686-zlib-1.3-1-any.pkg.tar.zst",
-                  "21bacf3a43073749a4cbdf407c7f1da92bab56c80925b1205f7c4cb289c724a1"; unpack_target="i686-w64-mingw32"),
-    # We need some special compiler support libraries from mingw for i686 (libgcc_s_dw2)
-    ArchiveSource("https://mirror.msys2.org/mingw/mingw32/mingw-w64-i686-gcc-libs-13.2.0-2-any.pkg.tar.zst",
-                  "2dae8189318a91067cca895572b2b46183bfd2ee97a55127a84f4f418f0b32f3"; unpack_target="i686-w64-mingw32"),
-
-    # 64-bit Windows from https://packages.msys2.org/package/mingw-w64-x86_64-hdf5
-    ArchiveSource("https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-hdf5-1.14.2-2-any.pkg.tar.zst",
-                  "19a0a28d32c8990a29e001b77fe2deeb4946ff6c7d0949dbf756dfe1b9b40e8a"; unpack_target="x86_64-w64-mingw32"),
-    ArchiveSource("https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-zlib-1.3-1-any.pkg.tar.zst",
-                  "254a6c5a8a27d1b787377a3e70a39cceb200b47c5f15f4ab5bfa1431b718ef98"; unpack_target="x86_64-w64-mingw32"),
-
 ]
 
 # Bash recipe for building across all platforms
 script = raw"""
-cd ${WORKSPACE}/srcdir
-
-# We don't build HDF5 on Windows; instead, we use packages from msys there:
-if [[ ${target} == *mingw* ]]; then
-    cd ${target}/mingw${nbits}
-
-    mkdir -p ${libdir} ${includedir}
-    rm -f lib/{*_cpp*,*fortran*,*f90*} # we do not need these
-    rm -f bin/{*_cpp*,*fortran*,*f90*} # we do not need these
-    
-    mv -v lib/libhdf5*.dll.a ${prefix}/lib
-    mv -v bin/*.dll ${libdir}
-    mv -v include/* ${includedir}
-
-    install_license share/doc/hdf5/COPYING
-    exit 0
-fi
-
-cd hdf5-*
+cd ${WORKSPACE}/srcdir/hdf5-*
 
 if [[ ${target} == *-mingw* ]]; then
     atomic_patch -p1 ${WORKSPACE}/srcdir/patches/h5ls.c.patch
@@ -65,105 +29,10 @@ fi
 # HDF5 assumes that some MPI constants are C constants, but they are not
 atomic_patch -p1 ${WORKSPACE}/srcdir/patches/mpi.patch
 
-# Idea:
-# - provide the registered filter plugins (BZIP2, JPEG, LZF, BLOSC, MAFISC, LZ4, Bitshuffle, and ZFP)
-
-# Building via `configure` instead of via `cmake` has one advantage:
-# The `h5cc` etc. scripts are generated, and some other packages might expect these.
-if false; then
-
-# Option 1: Build with cmake
-# This is now outdated, e.g. it doesn't enable C++ nor Fortran.
-
-# Patch `CMakeLists.txt`:
-# - HDF5 would also try to build and run `H5detect` to collect ABI information.
-#   We know this information, and thus can provide it manually.
-# - HDF5 would try to build and run `H5make_libsettings` to collect
-#   build-time information. That information seems entirely optional, so
-#   we do mostly nothing instead.
-atomic_patch -p1 ${WORKSPACE}/srcdir/patches/CMakeLists.txt.patch
-
-# Prepare the pre-generated file `H5Tinit.c` that cmake will expect:
-case "${target}" in
-    aarch64-apple-darwin*)
-        cat ../../files/H5Tinit-darwin-arm64v8.c
-        ;;
-    aarch64-linux-*)
-        cat ../../files/H5Tinit-debian-arm64v8.c
-        ;;
-    arm-linux-*)
-        cat ../../files/H5Tinit-debian-arm32v7.c
-        ;;
-    i686-linux-*)
-        cat ../../files/H5Tinit-debian-i386.c
-        ;;
-    i686-w64-mingw32)
-        # sizeof(long double) == 12
-        # layout seems to be 16-bit sign+exponent and 64-bit mantissa
-        # same as for Linux
-        cat ../../files/H5Tinit-debian-i386.c
-        ;;
-    powerpc64le-linux-*)
-        cat ../../files/H5Tinit-debian-ppc64le.c
-        ;;
-    x86_64-apple-darwin*)
-        cat ../../files/H5Tinit-darwin-amd64.c
-        ;;
-    x86_64-linux-* | x86_64-*-freebsd*)
-        cat ../../files/H5Tinit-debian-amd64.c
-        ;;
-    x86_64-w64-mingw32)
-        # sizeof(long double) == 16
-        # layout seems to be 16-bit sign+exponent and 64-bit mantissa
-        # same as for Linux
-        cat ../../files/H5Tinit-debian-amd64.c 
-        ;;
-    *)
-        echo "Unsupported target architecture ${target}" >&2
-        exit 1
-        ;;
-esac >src/H5Tinit.c
-echo 'char H5libhdf5_settings[]="";' >src/H5lib_settings.c
-
-mkdir build
-pushd build
-
-cmake \
-    -DCMAKE_FIND_ROOT_PATH=${prefix} \
-    -DCMAKE_INSTALL_PREFIX=${prefix} \
-    -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \
-    -DBUILD_STATIC_LIBS=OFF \
-    -DBUILD_TESTING=OFF \
-    -DHDF5_BUILD_EXAMPLES=OFF \
-    -DHDF5_BUILD_HL_LIB=ON \
-    -DHDF5_BUILD_TOOLS=ON \
-    -DHAVE_IOEO_EXITCODE=0 \
-    -DTEST_LFS_WORKS_RUN=0 \
-    -DH5_LDOUBLE_TO_LONG_SPECIAL_RUN=1 \
-    -DH5_LDOUBLE_TO_LONG_SPECIAL_RUN__TRYRUN_OUTPUT= \
-    -DH5_LONG_TO_LDOUBLE_SPECIAL_RUN=1 \
-    -DH5_LONG_TO_LDOUBLE_SPECIAL_RUN__TRYRUN_OUTPUT= \
-    -DH5_LDOUBLE_TO_LLONG_ACCURATE_RUN=0 \
-    -DH5_LDOUBLE_TO_LLONG_ACCURATE_RUN__TRYRUN_OUTPUT= \
-    -DH5_LLONG_TO_LDOUBLE_CORRECT_RUN=0 \
-    -DH5_LLONG_TO_LDOUBLE_CORRECT_RUN__TRYRUN_OUTPUT= \
-    -DH5_DISABLE_SOME_LDOUBLE_CONV_RUN=1 \
-    -DH5_DISABLE_SOME_LDOUBLE_CONV_RUN__TRYRUN_OUTPUT= \
-    ..
-cmake --build . --config RelWithDebInfo --parallel ${nproc}
-cmake --build . --config RelWithDebInfo --parallel ${nproc} --target install
-
-popd
-
-else
-
-# Option 2: Build with configure
-# This is the currently supported way.
-
 # Patch `configure.ac`:
 atomic_patch -p1 ${WORKSPACE}/srcdir/patches/configure.ac.patch
 
-# Prepare the files `H5init.c` and `config.saved` that contain predetermined
+# Prepare the file `config.saved` that contains predetermined
 # configuration information
 mkdir saved
 case "${target}" in
@@ -182,7 +51,6 @@ case "${target}" in
     i686-w64-mingw32)
         # sizeof(long double) == 12
         # layout seems to be 16-bit sign+exponent and 64-bit mantissa
-        # same as for Linux
         cp ../files/msys2-mingw32/* saved
         ;;
     powerpc64le-linux-*)
@@ -195,13 +63,12 @@ case "${target}" in
         cp ../files/debian-amd64/* saved
         ;;
     x86_64-*-freebsd*)
-        # same as for Linux
+        # no __float128
         cp ../files/freebsd-amd64/* saved
         ;;
     x86_64-w64-mingw32)
         # sizeof(long double) == 16
         # layout seems to be 16-bit sign+exponent and 64-bit mantissa
-        # same as for Linux
         cp ../files/msys2-mingw64/* saved
         ;;
     *)
@@ -240,6 +107,9 @@ fi
 FLAGS=()
 if [[ ${target} == *-mingw* ]]; then
     FLAGS+=(LDFLAGS='-no-undefined')
+    # For OpenSSL's libcrypto for ROS3-VFD
+    export CFLAGS="${CFLAGS} -L${prefix}/lib64"
+    export FCFLAGS="${FCFLAGS} -L${prefix}/lib64"
 fi
 
 # Check which VFD are available
@@ -253,45 +123,47 @@ elif [[ ${target} == *-w64-mingw32 ]]; then
 fi
 
 # Configure MPI
-ENABLE_PARALLEL=yes
-if grep -q MPICH_NAME ${prefix}/include/mpi.h; then
-    # MPICH
-    export CC=mpicc
-    export CXX=mpicxx
-    export FC=mpifort
-elif grep -q MPITRAMPOLINE_MPI_H ${prefix}/include/mpi.h; then
-    # MPItrampoline
-    export MPITRAMPOLINE_CC="$(which $CC)"
-    export MPITRAMPOLINE_CXX="$(which $CXX)"
-    export MPITRAMPOLINE_FC="$(which $FC)"
-    export CC=mpicc
-    export CXX=mpicxx
-    export FC=mpifort
-elif grep -q MSMPI_VER ${prefix}/include/mpi.h; then
+if grep -q MSMPI_VER ${prefix}/include/mpi.h; then
     # Microsoft MPI
     if [[ ${target} == i686-* ]]; then
         # 32-bit system
         # Do not enable MPI; the function MPI_File_close is not defined
         # in the 32-bit version of Microsoft MPI 10.1.12498.18
-        ENABLE_PARALLEL=no
-    else
+        :
+    elif false; then
+        # DISABLED
+        # 64-bit system
+        # Do not enable MPI
+        # Mingw-w64 runtime failure:
+        # 32 bit pseudo relocation at 0000000007828E2C out of range, targeting 00007FFDE78BAD90, yielding the value 00007FFDE0091F60.
+        # Consider: https://www.symscape.com/configure-msmpi-for-mingw-w64
+        # gendef msmpi.dll - creates msmpi.def
+        # x86_64-w64-mingw32-dlltool -d msmpi.def -l libmsmpi.a -D msmpi.dll - creates libmsmpi.a
+
         # Hide static libraries
         rm ${prefix}/lib/msmpi*.lib
         # Make shared libraries visible
         ln -s msmpi.dll ${libdir}/libmsmpi.dll
-        export FCFLAGS="$FCFLAGS -I${prefix}/src -I${prefix}/include -fno-range-check"
+        ENABLE_PARALLEL=yes
+        export FCFLAGS="${FCFLAGS} -I${prefix}/src -I${prefix}/include -fno-range-check"
         export LIBS="-L${libdir} -lmsmpi"
     fi
-elif grep -q OMPI_MAJOR_VERSION ${prefix}/include/mpi.h; then
-    # OpenMPI
+else
+    ENABLE_PARALLEL=yes
+    export MPITRAMPOLINE_CC="${CC}"
+    export MPITRAMPOLINE_CXX="${CXX}"
+    export MPITRAMPOLINE_FC="${FC}"
     export CC=mpicc
     export CXX=mpicxx
     export FC=mpifort
-else
-    # Unknown MPI
-    exit 1
 fi
 
+# This is a bug in HDF5; see
+# <https://github.com/HDFGroup/hdf5/issues/3925>. The file
+# `config/freebsd` includes `config/classic-fflags` which is
+# missing.
+: >../config/classic-fflags
+
 ../configure \
     --prefix=${prefix} \
     --build=${MACHTYPE} \
@@ -339,12 +211,10 @@ fi
 
 # Patch the generated `Makefile`:
 # (We could instead patch `Makefile.in`, or maybe even `Makefile.am`.)
-# - HDF5 would also try to build and run `H5detect` to collect ABI information.
-#   We know this information, and thus can provide it manually.
-# - HDF5 would try to build and run `H5make_libsettings` to collect
-#   build-time information. That information seems entirely optional, so
-#   we do mostly nothing instead.
-atomic_patch -p1 ${WORKSPACE}/srcdir/patches/src-Makefile.patch
+# HDF5 would otherwise try to build and run code to determine what
+# integer and real types are available in Fortran. This doesn't work
+# while cross-compiling. We thus provide pre-recorded information
+# instead (see `config.saved` above).
 atomic_patch -p1 ${WORKSPACE}/srcdir/patches/fortran-src-Makefile.patch
 atomic_patch -p1 ${WORKSPACE}/srcdir/patches/hl-fortran-src-Makefile.patch
 
@@ -357,8 +227,6 @@ make install
 
 popd
 
-fi
-
 install_license COPYING
 """
 
@@ -371,12 +239,12 @@ augment_platform_block = """
 # These are the platforms we will build for by default, unless further
 # platforms are passed in on the command line
 platforms = supported_platforms()
-# TODO: Don't expand ABIs for Windows since we're not providing either C++ or Fortran bindings there.
 platforms = expand_cxxstring_abis(platforms)
 platforms = expand_gfortran_versions(platforms)
 
-# TODO: Don't require MPI for Windows since we're using the non-MPI msys libraries there.
-platforms, platform_dependencies = MPI.augment_platforms(platforms; MPItrampoline_compat="5.3.0")
+platforms, platform_dependencies = MPI.augment_platforms(platforms; MPItrampoline_compat="5.3.1", OpenMPI_compat="4.1.6")
+# TODO: Use MPI only on non-Windows platforms
+# platforms = [filter(!Sys.iswindows, mpi_platforms); filter(Sys.iswindows, platforms)]
 
 # Avoid platforms where the MPI implementation isn't supported
 # OpenMPI
@@ -387,48 +255,39 @@ platforms = filter(p -> !(p["mpi"] == "mpitrampoline" && Sys.isfreebsd(p)), plat
 
 # The products that we will ensure are always built
 products = [
-    # Since we use the msys binaries for Windows, we can only define
-    # those products that are provided by msys as well. These are
-    # just the regular and the high-level libraries.
-
     # # HDF5 tools
-    # ExecutableProduct("h5clear", :h5clear),
-    # ExecutableProduct("h5copy", :h5copy),
-    # ExecutableProduct("h5debug", :h5debug),
-    # ExecutableProduct("h5delete", :h5delete),
-    # ExecutableProduct("h5diff", :h5diff),
-    # ExecutableProduct("h5dump", :h5dump),
-    # ExecutableProduct("h5format_convert", :h5format_convert),
-    # ExecutableProduct("h5import", :h5import),
-    # ExecutableProduct("h5jam",:h5jam),
-    # ExecutableProduct("h5ls", :h5ls),
-    # ExecutableProduct("h5mkgrp", :h5mkgrp),
-    # ExecutableProduct("h5perf_serial",:h5perf_serial),
-    # ExecutableProduct("h5repack", :h5repack),
-    # ExecutableProduct("h5repart", :h5repart),
-    # ExecutableProduct("h5stat", :h5stat),
-    # ExecutableProduct("h5unjam", :h5unjam),
-    # ExecutableProduct("h5watch", :h5watch),
+    ExecutableProduct("h5clear", :h5clear),
+    ExecutableProduct("h5copy", :h5copy),
+    ExecutableProduct("h5debug", :h5debug),
+    ExecutableProduct("h5delete", :h5delete),
+    ExecutableProduct("h5diff", :h5diff),
+    ExecutableProduct("h5dump", :h5dump),
+    ExecutableProduct("h5format_convert", :h5format_convert),
+    ExecutableProduct("h5import", :h5import),
+    ExecutableProduct("h5jam",:h5jam),
+    ExecutableProduct("h5ls", :h5ls),
+    ExecutableProduct("h5mkgrp", :h5mkgrp),
+    ExecutableProduct("h5perf_serial",:h5perf_serial),
+    ExecutableProduct("h5repack", :h5repack),
+    ExecutableProduct("h5repart", :h5repart),
+    ExecutableProduct("h5stat", :h5stat),
+    ExecutableProduct("h5unjam", :h5unjam),
+    ExecutableProduct("h5watch", :h5watch),
 
     # HDF5 libraries
     LibraryProduct("libhdf5", :libhdf5),
-    # LibraryProduct("libhdf5_cpp", :libhdf5_cpp),
-    # LibraryProduct("libhdf5_fortran", :libhdf5_fortran),
+    LibraryProduct("libhdf5_cpp", :libhdf5_cpp),
+    LibraryProduct("libhdf5_fortran", :libhdf5_fortran),
     LibraryProduct("libhdf5_hl", :libhdf5_hl),
-    # LibraryProduct("libhdf5_hl_cpp", :libhdf5_hl_cpp),
-    # LibraryProduct("libhdf5hl_fortran", :libhdf5_hl_fortran),
+    LibraryProduct("libhdf5_hl_cpp", :libhdf5_hl_cpp),
+    LibraryProduct("libhdf5hl_fortran", :libhdf5_hl_fortran),
 ]
 
 # Dependencies that must be installed before this package can be built
 dependencies = [
-    # For OpenMP we use libomp from `LLVMOpenMP_jll` where we use LLVM as compiler (BSD 
-    # systems), and libgomp from `CompilerSupportLibraries_jll` everywhere else. 
-    Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae");
-               platforms=filter(!Sys.isbsd, platforms)),
-    Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e");
-               platforms=filter(Sys.isbsd, platforms)),
+    # To ensure that the correct version of libgfortran is found at runtime
+    Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae")),
     Dependency("LibCURL_jll"),
-    # The msys Windows libraries require OpenSSL@3
     Dependency("OpenSSL_jll"; compat="3.0.8"),
     Dependency("Zlib_jll"),
     # Dependency("dlfcn_win32_jll"; platforms=filter(Sys.iswindows, platforms)),
@@ -443,4 +302,4 @@ ENV["MPITRAMPOLINE_DELAY_INIT"] = "1"
 # Build the tarballs, and possibly a `build.jl` as well.
 # GCC 5 reports an ICE on i686-linux-gnu-libgfortran3-cxx11-mpi+mpich
 build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies;
-               augment_platform_block, julia_compat="1.6", preferred_gcc_version=v"6")
+               augment_platform_block, clang_use_lld=false, julia_compat="1.6", preferred_gcc_version=v"6")