Upgrade the vendored dune to 3.14.0 and other packages to their latest release

.github/scripts/main/create-ocaml-cache.sh

@@ -9,16 +9,19 @@ OCAML_LOCAL="$4"
 PLATFORM="$5"
 
 if [[ $OCAML_BRANCH -gt 407 ]]; then
+  make -C src_ext dune-local.stamp
+  cd src_ext/dune-local
+  ocaml boot/bootstrap.ml
+  cp _boot/dune.exe "$PREFIX/bin/dune$EXE"
+  cd ../..
+
+  git clean -dfX
+
   if [[ -n $GITHUB_BASE_REF ]]; then
     git tag combak
     git fetch origin $GITHUB_BASE_REF
     git checkout origin/$GITHUB_BASE_REF
   fi
-  make -C src_ext dune-local.stamp
-  cd src_ext/dune-local
-  ocaml bootstrap.ml
-  cp dune.exe "$PREFIX/bin/dune$EXE"
-  cd ../..
 
   ./configure --with-vendored-deps
   make

.github/workflows/ci.ml

@@ -15,7 +15,7 @@ open Lib
 
 let ocamls = [
   (* Fully supported versions *)
-   "4.08.1"; "4.09.1"; "4.10.2"; "4.11.2"; "4.12.1"; "4.13.1"; "5.0.0"; "5.1.0"; "4.14.1";
+   "4.08.1"; "4.09.1"; "4.10.2"; "4.11.2"; "4.12.1"; "4.13.1"; "5.0.0"; "5.1.1"; "4.14.1";
 ]
 
 (* Entry point for the workflow. Workflows are specified as continuations where
@@ -487,8 +487,8 @@ let main oc : unit =
     ("CYGWIN_EPOCH", "3");
   ] in
   let keys = [
-    ("archives", "archives-1-${{ hashFiles('src_ext/Makefile.sources', 'src_ext/Makefile', '.github/scripts/common/preamble.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/archives-cache.sh') }}-${{ env.OPAM_REPO_SHA }}");
-    ("ocaml-cache", "${{ hashFiles('.github/scripts/main/ocaml-cache.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/create-ocaml-cache.sh') }}");
+    ("archives", "archives-1-${{ hashFiles('src_ext/Makefile.dune', 'src_ext/Makefile.sources', 'src_ext/Makefile', '.github/scripts/common/preamble.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/archives-cache.sh') }}-${{ env.OPAM_REPO_SHA }}");
+    ("ocaml-cache", "${{ hashFiles('src_ext/Makefile.dune', '.github/scripts/main/ocaml-cache.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/create-ocaml-cache.sh') }}");
     ("cygwin", "${{ hashFiles('.github/scripts/cygwin.cmd') }}-${{ env.CYGWIN_EPOCH }}");
     ("opam-bs-cache", "${{ hashFiles('.github/scripts/main/opam-bs-cache.sh', '*.opam', '.github/scripts/main/preamble.sh') }}");
   ] in

.github/workflows/main.yml

@@ -56,10 +56,10 @@ jobs:
     - name: Determine cache keys
       id: keys
       run: |
-        echo archives=archives-1-${{ hashFiles('src_ext/Makefile.sources', 'src_ext/Makefile', '.github/scripts/common/preamble.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/archives-cache.sh') }}-${{ env.OPAM_REPO_SHA }}
-        echo archives=archives-1-${{ hashFiles('src_ext/Makefile.sources', 'src_ext/Makefile', '.github/scripts/common/preamble.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/archives-cache.sh') }}-${{ env.OPAM_REPO_SHA }} >> $GITHUB_OUTPUT
-        echo ocaml-cache=${{ hashFiles('.github/scripts/main/ocaml-cache.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/create-ocaml-cache.sh') }}
-        echo ocaml-cache=${{ hashFiles('.github/scripts/main/ocaml-cache.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/create-ocaml-cache.sh') }} >> $GITHUB_OUTPUT
+        echo archives=archives-1-${{ hashFiles('src_ext/Makefile.dune', 'src_ext/Makefile.sources', 'src_ext/Makefile', '.github/scripts/common/preamble.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/archives-cache.sh') }}-${{ env.OPAM_REPO_SHA }}
+        echo archives=archives-1-${{ hashFiles('src_ext/Makefile.dune', 'src_ext/Makefile.sources', 'src_ext/Makefile', '.github/scripts/common/preamble.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/archives-cache.sh') }}-${{ env.OPAM_REPO_SHA }} >> $GITHUB_OUTPUT
+        echo ocaml-cache=${{ hashFiles('src_ext/Makefile.dune', '.github/scripts/main/ocaml-cache.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/create-ocaml-cache.sh') }}
+        echo ocaml-cache=${{ hashFiles('src_ext/Makefile.dune', '.github/scripts/main/ocaml-cache.sh', '.github/scripts/main/preamble.sh', '.github/scripts/main/create-ocaml-cache.sh') }} >> $GITHUB_OUTPUT
         echo cygwin=${{ hashFiles('.github/scripts/cygwin.cmd') }}-${{ env.CYGWIN_EPOCH }}
         echo cygwin=${{ hashFiles('.github/scripts/cygwin.cmd') }}-${{ env.CYGWIN_EPOCH }} >> $GITHUB_OUTPUT
         echo opam-bs-cache=${{ hashFiles('.github/scripts/main/opam-bs-cache.sh', '*.opam', '.github/scripts/main/preamble.sh') }}
@@ -106,7 +106,7 @@ jobs:
     needs: Analyse
     strategy:
       matrix:
-        ocamlv: [ 4.08.1, 4.09.1, 4.10.2, 4.11.2, 4.12.1, 4.13.1, 5.0.0, 5.1.0, 4.14.1 ]
+        ocamlv: [ 4.08.1, 4.09.1, 4.10.2, 4.11.2, 4.12.1, 4.13.1, 5.0.0, 5.1.1, 4.14.1 ]
       fail-fast: true
     steps:
     - name: Install bubblewrap

.gitignore

@@ -41,6 +41,7 @@ src_ext/seq/
 src_ext/sha/
 src_ext/spdx_licenses/
 src_ext/stdlib-shims/
+src_ext/swhid_core/
 src_ext/topkg/
 Opam.Runtime.*/
 *.tar.bz2

Makefile

@@ -21,7 +21,7 @@ else
 endif
 
 ifeq ($(DUNE),)
-  DUNE_EXE = src_ext/dune-local/dune.exe
+  DUNE_EXE = src_ext/dune-local/_boot/dune.exe
   ifeq ($(shell command -v cygpath 2>/dev/null),)
     DUNE := $(DUNE_EXE)
   else
@@ -48,11 +48,11 @@ else
   DUNE_PROFILE_ARG = --profile=$(DUNE_PROFILE)
 endif
 
-src_ext/dune-local/dune.exe: src_ext/dune-local.stamp $(DUNE_SECONDARY)
+src_ext/dune-local/_boot/dune.exe: src_ext/dune-local.stamp $(DUNE_SECONDARY)
 ifeq ($(DUNE_SECONDARY),)
-	cd src_ext/dune-local && ocaml bootstrap.ml
+	cd src_ext/dune-local && ocaml boot/bootstrap.ml
 else
-	cd src_ext/dune-local && ( unset OCAMLLIB ; unset CAML_LD_LIBRARY_PATH ; PATH="$(dir $(realpath $(DUNE_SECONDARY))):$$PATH" ../../$(DUNE_SECONDARY) bootstrap.ml )
+	cd src_ext/dune-local && ( unset OCAMLLIB ; unset CAML_LD_LIBRARY_PATH ; PATH="$(dir $(realpath $(DUNE_SECONDARY))):$$PATH" ../../$(DUNE_SECONDARY) boot/bootstrap.ml )
 endif
 
 src_ext/dune-local.stamp:

master_changes.md

@@ -84,9 +84,14 @@ users)
   * Do not check for cppo in the configure script (not used directly anymore since #5498) [#5794 @kit-ty-kate]
   * Upgrade vendored cmdliner to 1.2.0 [#5797 @kit-ty-kate]
   * Add winsymlinks:native to the CYGWIN environment variable when installing a package on Windows [#5793 @kit-ty-kate - fix #5782]
+  * Upgrade the vendored dune to 3.14.0 [#5869 @kit-ty-kate]
+  * Upgrade the vendored re to 1.11.0 [#5869 @dra27]
+  * Upgrade the vendored ocamlgraph to 2.1.0 [#5869 @dra27]
+  * Upgrade the vendored opam-file-format to 2.1.6 [#5869 @dra27]
 
 ## Infrastructure
   * Fix depexts CI workflow and ensure all workflows run on master push [#5788 @dra27]
+  * Update src_ext/Makefile.dune along with src_ext/Makefile.sources [#5869 @dra27]
 
 ## Release scripts
 
@@ -126,6 +131,8 @@ users)
 
 ## Github Actions
   * Update checkout action to v4 [#5851 @rjbou]
+  * Test OCaml 5.1.1 [#5869 @dra27]
+  * Fix the cache key [#5869 @dra27]
 
 ## Doc
   * Fix a typo in the documentation of `opam lint --recursive` [#5812 @Khady]

src_ext/Makefile

@@ -102,7 +102,7 @@ reset-lib-pkg:
 	@rm -rf ../bootstrap/ocaml/lib/ocaml/site-lib ../bootstrap/ocaml/etc *.pkgbuild
 
 ifeq ($(DUNE),)
-DUNE_DEP=dune-local/_boot/install/default/bin/dune$(EXE)
+DUNE_DEP=dune-local/_boot/dune$(EXE)
 DUNE_CLONE=dune-local.stamp
 ifeq ($(shell command -v cygpath 2>/dev/null),)
 DUNE:=$(DUNE_DEP)
@@ -114,8 +114,8 @@ DUNE_DEP=
 DUNE_CLONE=
 endif
 
-dune-local/_boot/install/default/bin/dune$(EXE): $(DUNE_CLONE)
-	cd dune-local && ocaml bootstrap.ml && ./boot.exe --release
+dune-local/_boot/dune$(EXE): $(DUNE_CLONE)
+	cd dune-local && ocaml boot/bootstrap.ml
 
 build-pkg: clone-pkg $(PKG_EXTS:=.pkgbuild)
 	@

src_ext/Makefile.dune

@@ -1,3 +1,3 @@
 # NB If minimum OCaml version for Dune changes, update DUNE_SECONDARY in configure.ac
-URL_dune-local = https://github.com/ocaml/dune/releases/download/3.5.0/dune-3.5.0.tbz
-MD5_dune-local = 46d1bde258ef60f60f053e88fe9d937b
+URL_dune-local = https://github.com/ocaml/dune/releases/download/3.14.0/dune-3.14.0.tbz
+MD5_dune-local = bff778fff4996b890e5af3da7ecf5baa

src_ext/Makefile.packages

@@ -115,8 +115,8 @@ dose3-pkg-build:
 	dune install "--prefix=$(OCAMLROOT)" -p dose3 dose3
 
 dune-local-pkg-build:
-	ocaml bootstrap.ml
-	cp dune.exe $(OCAMLBIN)/dune$(EXT_EXE)
+	ocaml boot/bootstrap.ml
+	cp _boot/dune.exe $(OCAMLBIN)/dune$(EXT_EXE)
 
 mccs-pkg-build: stublibs
 	dune build @install -p mccs

src_ext/Makefile.sources

@@ -13,8 +13,8 @@ MD5_base64 = bfdd16aa8c136412878109df8791fc01
 
 $(call PKG_SAME,base64)
 
-URL_re = https://github.com/ocaml/ocaml-re/releases/download/1.10.4/re-1.10.4.tbz
-MD5_re = 68c427f8b55507f8b61c613ee6a5b3da
+URL_re = https://github.com/ocaml/ocaml-re/releases/download/1.11.0/re-1.11.0.tbz
+MD5_re = e0199e32947fd33fcc1b8e69de2308a1
 
 $(call PKG_SAME,re)
 
@@ -23,8 +23,8 @@ MD5_cmdliner = b860881cc90c68b703dca0f35bdd4cdb
 
 $(call PKG_SAME,cmdliner)
 
-URL_ocamlgraph = https://github.com/backtracking/ocamlgraph/releases/download/2.0.0/ocamlgraph-2.0.0.tbz
-MD5_ocamlgraph = 2d07fcf3501e1d4997c03fa94cea22f0
+URL_ocamlgraph = https://github.com/backtracking/ocamlgraph/releases/download/2.1.0/ocamlgraph-2.1.0.tbz
+MD5_ocamlgraph = 4200d8f223aa7a32b4024e4553821c7c
 
 $(call PKG_SAME,ocamlgraph)
 
@@ -53,8 +53,8 @@ MD5_0install-solver = 030edc9b1d3676c06d51397ffb5a737d
 
 $(call PKG_SAME,0install-solver)
 
-URL_opam-file-format = https://github.com/ocaml/opam-file-format/archive/refs/tags/2.1.5.tar.gz
-MD5_opam-file-format = 46dadff2565d8371cdc606d33d408fc4
+URL_opam-file-format = https://github.com/ocaml/opam-file-format/archive/refs/tags/2.1.6.tar.gz
+MD5_opam-file-format = 706ce5fc3e77db746a4c8b11d79cefef
 
 $(call PKG_SAME,opam-file-format)
 

src_ext/update-sources.sh

@@ -1,46 +1,52 @@
 #!/usr/bin/env bash
 
-cd $(dirname $0)
-echo -n "Checking packages for new versions in opam: "
-DISAGREEMENTS=()
-while read name prefix version url; do
-  package=$name
-  case "$package" in
-    findlib) package=ocamlfind;;
-    dune-local) package=dune;;
-  esac
-  latest=$(opam show $package -f all-versions | sed -e 's/  base//')
-  latest=${latest##* }
-  package_url=$(opam show $package.$latest -f url.src: | sed -e 's/"//g')
-  md5=$(sed -n -e "s/MD5$prefix$name *= *\(.*\)/\1/p" Makefile.sources)
-  package_md5=$(opam show $package.$latest -f url.checksum: | sed -n -e "/md5/s/.*md5=\([a-fA-F0-9]\{32\}\).*/\1/p")
-  if [[ -z $package_md5 ]] ; then
-    echo -e "\n$name: [\033[1;33mWARN\033[m] no md5 given in opam, downloading $package_url to check"
-    package_md5=$(curl -LSs $package_url | md5sum | cut -f1 -d' ')
-  fi
-  if [[ $package_url = $url ]] ; then
-    if [[ $package_md5 = $md5 ]] ; then
-      echo -ne "[\033[0;32m$name\033[m] "
-      if [[ $latest != $version ]] ; then
-        DISAGREEMENTS+=" $name ($version vs $latest in opam)"
-      fi
-    else
-      echo -e "\n$name: [\033[1;33mWARN\033[m] MD5 is wrong for (should be $package_md5 according to opam)"
+function process
+{
+  while read name prefix version url; do
+    package=$name
+    case "$package" in
+      findlib) package=ocamlfind;;
+      dune-local) package=dune;;
+    esac
+    latest=$(opam show $package -f all-versions | sed -e 's/  base//')
+    latest=${latest##* }
+    package_url=$(opam show $package.$latest -f url.src: | sed -e 's/"//g')
+    md5=$(sed -n -e "s/MD5$prefix$name *= *\(.*\)/\1/p" "$1")
+    package_md5=$(opam show $package.$latest -f url.checksum: | sed -n -e "/md5/s/.*md5=\([a-fA-F0-9]\{32\}\).*/\1/p")
+    if [[ -z $package_md5 ]] ; then
+      echo -e "\n$name: [\033[1;33mWARN\033[m] no md5 given in opam, downloading $package_url to check"
+      package_md5=$(curl -LSs $package_url | md5sum | cut -f1 -d' ')
     fi
-  else
-    if [[ $package_md5 = $md5 ]] ; then
-      echo -e "\n$name: [\033[1;33mWARN\033[m] URL is wrong for $name (should be $package_url according to opam)"
+    if [[ $package_url = $url ]] ; then
+      if [[ $package_md5 = $md5 ]] ; then
+        echo -ne "[\033[0;32m$name\033[m] "
+        if [[ $latest != $version ]] ; then
+          DISAGREEMENTS+=" $name ($version vs $latest in opam)"
+        fi
+      else
+        echo -e "\n$name: [\033[1;33mWARN\033[m] MD5 is wrong for (should be $package_md5 according to opam)"
+      fi
     else
-      if [[ $latest = $version ]] ; then
-        echo -e "\n$name: [\033[1;33mWARN\033[m] URL and MD5 are wrong for $name (should be $package_url (md5=$package_md5) according to opam)"
+      if [[ $package_md5 = $md5 ]] ; then
+        echo -e "\n$name: [\033[1;33mWARN\033[m] URL is wrong for $name (should be $package_url according to opam)"
       else
-        echo -ne "[\033[0;31m$name\033[m: \033[1m$latest\033[m] "
-        sed -e "s/\(URL$prefix$name *= *\).*/\1${package_url////\\/}/" -e "s/\(MD5$prefix$name *= *\).*/\1$package_md5/" Makefile.sources > Makefile.sources.tmp
-        mv Makefile.sources.tmp Makefile.sources
+        if [[ $latest = $version ]] ; then
+          echo -e "\n$name: [\033[1;33mWARN\033[m] URL and MD5 are wrong for $name (should be $package_url (md5=$package_md5) according to opam)"
+        else
+          echo -ne "[\033[0;31m$name\033[m: \033[1m$latest\033[m] "
+          sed -e "s/\(URL$prefix$name *= *\).*/\1${package_url////\\/}/" -e "s/\(MD5$prefix$name *= *\).*/\1$package_md5/" "$1" > "$1.tmp"
+          mv "$1.tmp" "$1"
+        fi
       fi
     fi
-  fi
-done < <(grep -F URL_ Makefile.sources | sed -e "s/URL\(_\(PKG_\)\?\)\([^ =]*\) *= *\(.*\/\(\([^0-9][^-]*\)*-\)\?v\?\)\([0-9.]\+\([-+.][^\/]*\)\?\)\(\.tbz\|\.tar\.gz\)/\3 \1 \7 \4\7\9/" | sort)
+  done < <(grep -F URL_ "$1" | sed -e "s/URL\(_\(PKG_\)\?\)\([^ =]*\) *= *\(.*\/\(\([^0-9][^-]*\)*-\)\?v\?\)\([0-9.]\+\([-+.][^\/]*\)\?\)\(\.tbz\|\.tar\.gz\)/\3 \1 \7 \4\7\9/" | sort)
+}
+
+cd $(dirname $0)
+echo -n "Checking packages for new versions in opam: "
+DISAGREEMENTS=()
+process Makefile.sources
+process Makefile.dune
 echo -e "\nComplete."
 if [[ ${#DISAGREEMENTS[@]} -gt 0 ]] ; then
   echo "Disagreements over version:${DISAGREEMENTS[@]}"