Add macOS support

lib/build.ml

@@ -3,7 +3,15 @@ open Capnp_rpc_lwt
 open Lwt.Infix
 
 module Git = Current_git
-module Image = Current_docker.Raw.Image
+
+(* TODO: Make macOS use docker images *)
+type base =
+  | Docker of Current_docker.Raw.Image.t
+  | MacOS of string
+
+let base_to_string = function
+  | Docker img -> Current_docker.Raw.Image.hash img
+  | MacOS base -> base
 
 let ( >>!= ) = Lwt_result.bind
 
@@ -84,7 +92,7 @@ module Op = struct
     config : t;
     master : Current_git.Commit.t;
     urgent : ([`High | `Low] -> bool) option;
-    base : Image.t;                           (* The image with the OCaml compiler to use. *)
+    base : base;
   }
 
   let id = "ci-ocluster-build"
@@ -123,7 +131,7 @@ module Op = struct
   let run { config = { connection; timeout }; master; urgent; base } job { Key.pool; commit; variant; ty } () =
     let master = Current_git.Commit.hash master in
     let build_spec ~for_docker =
-      let base = Image.hash base in
+      let base = base_to_string base in
       match ty with
       | `Opam (`List_revdeps, pkg) -> Opam_build.revdeps ~for_docker ~base ~variant ~pkg
       | `Opam (`Build { revdep; lower_bounds; with_tests; opam_version }, pkg) -> Opam_build.spec ~for_docker ~opam_version ~base ~variant ~revdep ~lower_bounds ~with_tests ~pkg
@@ -151,7 +159,7 @@ module Op = struct
         | `Opam (`List_revdeps, pkg)
         | `Opam (`Build _, pkg) -> OpamPackage.to_string pkg
       in
-      Printf.sprintf "%s-%s-%s" (Image.hash base) pkg (Git.Commit_id.hash commit)
+      Printf.sprintf "%s-%s-%s" (base_to_string base) pkg (Git.Commit_id.hash commit)
     in
     Current.Job.log job "Using cache hint %S" cache_hint;
     Current.Job.log job "Using OBuilder spec:@.%s@." spec_str;

lib/build.mli

@@ -11,6 +11,10 @@ module Spec : sig
     t
 end
 
+type base =
+  | Docker of Current_docker.Raw.Image.t
+  | MacOS of string
+
 type t
 
 val config : timeout:int64 -> [ `Submission_f4e8a768b32a7c42 ] Capnp_rpc_lwt.Sturdy_ref.t -> t
@@ -19,7 +23,7 @@ val v :
   t ->
   label:string ->
   spec:Spec.t Current.t ->
-  base:Current_docker.Raw.Image.t Current.t ->
+  base:base Current.t ->
   master:Current_git.Commit.t Current.t ->
   urgent:([`High | `Low] -> bool) option Current.t ->
   Current_git.Commit_id.t Current.t ->
@@ -29,7 +33,7 @@ val list_revdeps :
   t ->
   platform:Platform.t ->
   pkgopt:PackageOpt.t Current.t ->
-  base:Current_docker.Raw.Image.t Current.t ->
+  base:base Current.t ->
   master:Current_git.Commit.t Current.t ->
   after:unit Current.t ->
   Current_git.Commit_id.t Current.t ->

lib/opam_build.ml

@@ -44,7 +44,38 @@ let opam_install ~variant ~opam_version ~pin ~lower_bounds ~with_tests ~pkg =
 
 let setup_repository ~variant ~for_docker ~opam_version =
   let open Obuilder_spec in
+  let home_dir = match Variant.os variant with
+    | `macOS -> None
+    | `linux -> Some "/home/opam"
+  in
+  let prefix = match Variant.os variant with
+    | `macOS -> "~/local"
+    | `linux -> "/usr"
+  in
+  let ln = match Variant.os variant with
+    | `macOS -> "ln"
+    | `linux -> "sudo ln"
+  in
+  let opam_version_str = match opam_version with
+    | `V2_0 -> "2.0"
+    | `V2_1 -> "2.1"
+  in
+  let opam_repo_args = match Variant.os variant with
+    | `macOS -> " -k local" (* TODO: (copy ...) do not copy the content of .git or something like that and make the subsequent opam pin fail *)
+    | `linux -> ""
+  in
+  let opamrc = match Variant.os variant with
+    (* NOTE: [for_docker] is required because docker does not support bubblewrap in docker build *)
+    (* docker run has --privileged but docker build does not have it *)
+    (* so we need to remove the part re-enabling the sandbox. *)
+    | `linux when not for_docker -> " --config .opamrc-sandbox"
+    | `macOS | `linux -> ""
+    (* TODO: On macOS, the sandbox is always (and should be) enabled by default but does not have those ~/.opamrc-sandbox files *)
+  in
   user ~uid:1000 ~gid:1000 ::
+  (match home_dir with Some home_dir -> [workdir home_dir] | None -> []) @
+  (* TODO: macOS seems to have a bug in (copy ...) so I am forced to remove the (workdir ...) here.
+     Otherwise the "opam pin" after the "opam repository set-url" will fail (cannot find the new package for some reason) *)
   run "for pkg in $(opam pin list --short); do opam pin remove \"$pkg\"; done" :: (* The ocaml/opam base images have a pin to their compiler package.
                                                                                      Such pin is useless for opam 2.0 as we don't use --unlock-base,
                                                                                      and causes issues for opam 2.1 as it allows to upgrade the compiler
@@ -53,28 +84,16 @@ let setup_repository ~variant ~for_docker ~opam_version =
   run "opam repository remove -a multicore || true" :: (* We remove this non-standard repository
                                                           because we don't have access and it hosts
                                                           non-official packages *)
-  run "sudo ln -f /usr/bin/opam-%s /usr/bin/opam"
-    (match opam_version with
-     | `V2_0 -> "2.0"
-     | `V2_1 -> "2.1"
-    ) ::
-  (* NOTE: [for_docker] is required because docker does not support bubblewrap in docker build *)
-  (* docker run has --privileged but docker build does not have it *)
-  (* so we need to remove the part re-enabling the sandbox. *)
-  (* NOTE: On alpine-3.12 bwrap fails with "capset failed: Operation not permitted". *)
-  let sandboxing_not_supported =
-    let distro = variant.Variant.distribution in
-    String.equal distro (Dockerfile_distro.tag_of_distro (`Alpine `V3_12)) ||
-    for_docker
-  in
-  run "opam init --reinit%s -ni" (if sandboxing_not_supported then "" else " --config ~/.opamrc-sandbox") ::
+  run "%s -f %s/bin/opam-%s %s/bin/opam" ln prefix opam_version_str prefix ::
+  run "opam init --reinit%s -ni" opamrc ::
   env "OPAMDOWNLOADJOBS" "1" :: (* Try to avoid github spam detection *)
   env "OPAMERRLOGLEN" "0" :: (* Show the whole log if it fails *)
   env "OPAMSOLVERTIMEOUT" "500" :: (* Increase timeout. Poor mccs is doing its best *)
   env "OPAMPRECISETRACKING" "1" :: (* Mitigate https://github.com/ocaml/opam/issues/3997 *)
   [
-    copy ["."] ~dst:"/src/";
-    run "opam repository set-url --strict default file:///src";
+    run "rm -rf opam-repository/";
+    copy ["."] ~dst:"opam-repository/";
+    run "opam repository set-url%s --strict default opam-repository/" opam_repo_args;
   ]
 
 let set_personality ~variant =

lib/variant.ml

@@ -20,5 +20,17 @@ let distribution t = t.distribution
 
 let pp f t = Fmt.pf f "%s/%s" (docker_tag t) (Ocaml_version.string_of_arch t.arch)
 
+let macos_distributions = [
+  "macos-homebrew";
+  (* TODO: Add macos-macports *)
+]
+
+(* TODO: Remove that when macOS uses ocaml-dockerfile *)
+let os {distribution; _} =
+  if List.exists (String.equal distribution) macos_distributions then
+    `macOS
+  else
+    `linux
+
 let v ~arch ~distro ~compiler:(ocaml_version, ocaml_variant) =
   { arch; distribution = distro; ocaml_version; ocaml_variant }

service/conf.ml

@@ -22,9 +22,17 @@ end
 (** Maximum time for one build. *)
 let build_timeout = Duration.of_hour 1
 
-let pool_of_arch = function
-  | `X86_64 | `I386 -> "linux-x86_64"
-  | `Aarch32 | `Aarch64 -> "linux-arm64"
-  | `Ppc64le -> "linux-ppc64"
-  | `S390x -> "linux-s390x"
-  | `Riscv64 -> "linux-riscv64"
+let pool_of_arch variant =
+  let open Opam_repo_ci in
+  let os = match Variant.os variant with
+    | `macOS -> "macos"
+    | `linux -> "linux"
+  in
+  let arch = match variant.Variant.arch with
+    | `X86_64 | `I386 -> "x86_64"
+    | `Aarch32 | `Aarch64 -> "arm64"
+    | `Ppc64le -> "ppc64"
+    | `S390x -> "s390x"
+    | `Riscv64 -> "riscv64"
+  in
+  os^"-"^arch

service/pipeline.ml

@@ -125,7 +125,7 @@ let build_with_cluster ~ocluster ~analysis ~lint ~master source =
   let pkgs = Current.map (List.filter_map get_significant_available_pkg) pkgs in
   let build ~opam_version ~lower_bounds ~revdeps label variant =
     let arch = Variant.arch variant in
-    let pool = Conf.pool_of_arch arch in
+    let pool = Conf.pool_of_arch variant in
     let platform = {Platform.label; pool; variant} in
     let analysis = with_label label analysis in
     let pkgs =
@@ -140,11 +140,15 @@ let build_with_cluster ~ocluster ~analysis ~lint ~master source =
         let pkg = Current.map (fun {PackageOpt.pkg; urgent = _} -> pkg) pkgopt in
         let urgent = Current.return None in
         let base =
-          let+ repo_id =
-            Docker.peek ~schedule:weekly ~arch:(Ocaml_version.to_docker_arch arch)
-              ("ocaml/opam:" ^ Variant.docker_tag variant)
-          in
-          Current_docker.Raw.Image.of_hash repo_id
+          match Variant.os variant with
+          | `macOS ->
+              Current.return (Build.MacOS (Variant.docker_tag variant))
+          | `linux -> (* TODO: Use docker images as base for both macOS and linux *)
+              let+ repo_id =
+                Docker.peek ~schedule:weekly ~arch:(Ocaml_version.to_docker_arch arch)
+                  ("ocaml/opam:" ^ Variant.docker_tag variant)
+              in
+              Build.Docker (Current_docker.Raw.Image.of_hash repo_id)
         in
         let image =
           let spec = build_spec ~platform ~opam_version pkg in
@@ -196,8 +200,8 @@ let build_with_cluster ~ocluster ~analysis ~lint ~master source =
     end
   in
   let distributions ~opam_version =
-    Current.list_seq begin
-      let default_compiler = Ocaml_version.to_string default_compiler in
+    let default_compiler = Ocaml_version.to_string default_compiler in
+    let linux_distributions =
       Dockerfile_distro.active_distros `X86_64 |>
       List.fold_left (fun acc distro ->
         if Dockerfile_distro.compare distro master_distro = 0 (* TODO: Add Dockerfile_distro.equal *)
@@ -208,7 +212,15 @@ let build_with_cluster ~ocluster ~analysis ~lint ~master source =
           let variant = Variant.v ~arch:`X86_64 ~distro ~compiler:(default_compiler, None) in
           build ~opam_version ~lower_bounds:false ~revdeps:false distro variant :: acc
       ) []
-    end
+    in
+    let macos_distributions =
+      Variant.macos_distributions |>
+      List.map (fun distro ->
+        let variant = Variant.v ~arch:`X86_64 ~distro ~compiler:(default_compiler, None) in
+        build ~opam_version ~lower_bounds:false ~revdeps:false distro variant
+      )
+    in
+    Current.list_seq (macos_distributions @ linux_distributions)
   in
   let+ analysis = Node.action `Analysed analysis
   and+ lint = Node.action `Linted lint

web-ui/github.ml

@@ -124,6 +124,8 @@ let statuses ss =
     | StatusTree.Leaf (_, x) ->
         status x []
     | StatusTree.Branch (b, None, ss) ->
+        (* TODO: Remove that *)
+        let b = if b = "macos-homebrew" then b^" (experimental)" else b in
         li ~a:[a_class ["none"]] [txt b; ul ~a:[a_class ["statuses"]] (List.map render_status ss)]
     | StatusTree.Branch (_, Some (((NotStarted | Aborted | Failed _ | Undefined _), _) as x), _) ->
         status x [] (* Do not show children of a node that has failed (guarenties in service/pipeline.ml means that only successful parents have children with meaningful error messages) *)