Make SHA computation faster by using ocaml-sha

[kit-ty-kate]

In the spirit of #3217 and #4823 here is a PR that removes the runtime dependency on openssl.

In the expected range of use of opam it also makes installation and updates faster than before.
Here is the benchmark i used to show both correctness and speed:

let fmt = Printf.sprintf

let digestif file =
  let ci = open_in_bin file in
  let sha = Digestif.SHA512.init () in
  let buf = Bytes.create 1024 in
  let rec aux sha =
    match input ci buf 0 1024 with
    | 0 | exception End_of_file -> Digestif.SHA512.get sha
    | len -> aux (Digestif.SHA512.feed_bytes sha ~len buf)
  in
  let sha = aux sha in
  close_in ci;
  Digestif.SHA512.to_hex sha

let time name f =
  let timer = OpamConsole.timer () in
  let sha = f () in
  print_endline (fmt "%s calculated %s in %fs" name sha (timer ()))

let () =
  let file = Sys.argv.(1) in
  OpamCoreConfig.init ~debug_level:1 ();
  time "digestif" (fun () -> digestif file);
  time "ocaml-sha" (fun () -> Sha512.to_hex (Sha512.file file));
  time "openssl" (fun () -> OpamHash.to_string (OpamHash.compute ~kind:`SHA512 file));
  time "opam-core" (fun () -> OpamSHA.hash `SHA512 file);
  ()

Using these typical files:

opam-repository/packages/coq/coq.8.13.0/files/coq.install: 172B
opam-repository/packages/ocaml-base-compiler/ocaml-base-compiler.4.02.1/files/fix-gcc10.patch: 756B
opam-repository/packages/ocaml-base-compiler/ocaml-base-compiler.4.02.1/files/gpr1330.patch: 2.0K
irmin-2.10.2.tbz: 285K

and this large atypical file:

FreeBSD-13.0-RELEASE-amd64-bootonly.iso: 347M

Here is the result I get for SHA512:

$ ./a.out opam-repository/packages/coq/coq.8.13.0/files/coq.install
digestif calculated b501737b4dbd22adc1c0377d744448056fb1dc493caf72... in 0.001532s
ocaml-sha calculated b501737b4dbd22adc1c0377d744448056fb1dc493caf72c... in 0.000036s
openssl calculated b501737b4dbd22adc1c0377d744448056fb1dc493caf72c... in 0.003682s
opam-core calculated b501737b4dbd22adc1c0377d744448056fb1dc493caf72c... in 0.000059s

$ ./a.out opam-repository/packages/ocaml-base-compiler/ocaml-base-compiler.4.02.1/files/fix-gcc10.patch
digestif calculated e71423ff0dad6261fb369f6c30a51062b2fe3df7a1b4bf2f7424c756... in 0.001612s
ocaml-sha calculated e71423ff0dad6261fb369f6c30a51062b2fe3df7a1b4bf2f7424c756... in 0.000126s
digestif calculated e71423ff0dad6261fb369f6c30a51062b2fe3df7a1b4bf2f7424c756... in 0.004505s
opam-core calculated e71423ff0dad6261fb369f6c30a51062b2fe3df7a1b4bf2f7424c756... in 0.000088s

$ ./a.out opam-repository/packages/ocaml-base-compiler/ocaml-base-compiler.4.02.1/files/gpr1330.patch
digestif calculated f691fb074f601ebe7accf41d9dcf5ad8793ad8b8cab13... in 0.001678s
ocaml-sha calculated f691fb074f601ebe7accf41d9dcf5ad8793ad8b8cab13fc... in 0.000119s
openssl calculated f691fb074f601ebe7accf41d9dcf5ad8793ad8b8cab13fc... in 0.004745s
opam-core calculated f691fb074f601ebe7accf41d9dcf5ad8793ad8b8cab13fc... in 0.000157s

$ ./a.out irmin-2.10.2.tbz
digestif calculated bbc03417d6eb87d99bee391c489d23a586b0a260e4c797f5e... in 0.002477s
ocaml-sha calculated bbc03417d6eb87d99bee391c489d23a586b0a260e4c797f5e... in 0.002120s
openssl calculated bbc03417d6eb87d99bee391c489d23a586b0a260e4c797f5e... in 0.005695s
opam-core calculated bbc03417d6eb87d99bee391c489d23a586b0a260e4c797f5e... in 0.006051s

$ ./a.out FreeBSD-13.0-RELEASE-amd64-bootonly.iso
digestif calculated 4bb54bbe258a7369cf6255c2324c26eeb302a2c294bbef... in 1.430754s
ocaml-sha calculated 4bb54bbe258a7369cf6255c2324c26eeb302a2c294bbef... in 1.418716s
openssl calculated 4bb54bbe258a7369cf6255c2324c26eeb302a2c294bbef... in 0.843393s
opam-core calculated 4bb54bbe258a7369cf6255c2324c26eeb302a2c294bbef... in 4.579879s

The same order of result can be observed with SHA256. e.g.

$ ./a.out irmin-2.10.2.tbz
digestif calculated 088741b619471c8198e74aaf9f4aeb4a9997abec8b9abcbfb3443fd27bfb433f in 0.001824s
ocaml-sha calculated 088741b619471c8198e74aaf9f4aeb4a9997abec8b9abcbfb3443fd27bfb433f in 0.002390s
openssl calculated 088741b619471c8198e74aaf9f4aeb4a9997abec8b9abcbfb3443fd27bfb433f in 0.003565s
opam-core calculated 088741b619471c8198e74aaf9f4aeb4a9997abec8b9abcbfb3443fd27bfb433f in 0.010110s

Awaiting review to put the rest of the work (vendoring) for this PR to pass CI.
This PR is not urgent at all and rebase should be easy so no rush to put this before opam 2.2.0

Fixes #5037
Would help #4158

[AltGr]

Nice, thanks for all this! I am a bit surprised by the poor results of Digestif here; I think that to be fair you should use its bigstrings functions and an mmap, to avoid the extra memory copy of the whole data — like is done in OpamSHA. Then I'd expect it to be as fast as OpamSHA in all cases ?

We'll need to remember to check how the static linking scripts fare with the new C code for next release (I think it should be fine 🤞 but...)

Another precision: there are lots of cases where opam computes hashes from already in-memory strings. The timings would be negligible in this case, except if we need to exec() an external binary to compute the hash like it used to be the case with openssl before we had OpamSHA.

[kit-ty-kate]

Note by @AltGr: this PR in its current state would make it possible to remove the dependency towards bigarray entirely (see #5049)

[rjbou]

Some things around to fix, but on the idea, ok!

[rjbou]

As there is usage replacement in OpamSHA, should we in the same time update Digest usage?

[rjbou]

why renaming these functions from sha256 to sha256_string ?

[kit-ty-kate]

It’s not a renaming, only removal of functions that are not used.

[rjbou]

fixed the error is

ocamlfind install sha _build/install/default/sha/META _build/install/default/sha/libsha_stubs.a _build/install/default/sha/sha.* _build/install/default/stublibs/*.so
ocamlfind: _build/install/default/sha/META: No such file or directory
make[2]: Leaving directory '/home/runner/work/opam/opam/src_ext/sha'
make[2]: *** [../Makefile.packages:157: sha-pkg-build] Error 2
make[1]: *** [Makefile:125: sha.pkgbuild] Error 2
make[1]: *** Waiting for unfinished jobs....

[rjbou]

(new) error is

  + make all admin
  dune build --profile=dev --root .  --promote-install-files -- opam-installer.install opam.install
      ocamlopt src/core/.opam_core.objs/native/opamSHA.{cmx,o} (exit 2)
  (cd _build/default && /home/runner/work/opam/opam/bootstrap/ocaml/bin/ocamlopt.opt -w [...] -o src/core/.opam_core.objs/native/opamSHA.cmx -c -impl src/core/opamSHA.ml)
  File "src/core/opamSHA.ml", line 11, characters 23-36:
  11 | let sha256_file file = Sha256.to_hex (Sha256.file file)
                              ^^^^^^^^^^^^^
  Error: Unbound module Sha256
  make: *** [Makefile:143: build-opam] Error 1