Vendoring solver service (#634)

Vendoring solver-service and delete local solver

The solver code has moved to https://github.com/ocurrent/solver-service and can now be run as a cluster job.
It's also possible to run the local pipeline with remote solver(cap file) or local solver(spawning a local solver).

Co-authored-by: Tim McGilchrist <timmcgil@gmail.com>

.gitmodules

@@ -14,3 +14,7 @@
 	path = ocaml-dockerfile
 	url = https://github.com/ocurrent/ocaml-dockerfile.git
 	branch = master
+[submodule "solver-service"]
+	path = solver-service
+	url = https://github.com/ocurrent/solver-service.git
+	branch = solver-ci

Dockerfile

@@ -14,6 +14,10 @@ COPY --chown=opam \
 	ocluster/ocluster-api.opam \
 	ocluster/current_ocluster.opam \
 	/src/ocluster/
+COPY --chown=opam \
+	solver-service/solver-service-api.opam \
+	solver-service/solver-service.opam \
+	/src/solver-service/
 COPY --chown=opam \
 	ocaml-version/ocaml-version.opam \
 	/src/ocaml-version/
@@ -32,8 +36,10 @@ RUN opam pin add -yn current_docker.dev "./ocurrent" && \
     opam pin add -yn ocaml-version.dev "./ocaml-version" && \
     opam pin add -yn dockerfile.dev "./ocaml-dockerfile" && \
     opam pin add -yn dockerfile-opam.dev "./ocaml-dockerfile" && \
+    opam pin add -yn solver-service-api.dev "./solver-service" && \
+    opam pin add -yn solver-service.dev "./solver-service" && \
     opam pin add -yn ocluster-api.dev "./ocluster"
-COPY --chown=opam ocaml-ci.opam ocaml-ci-service.opam ocaml-ci-api.opam ocaml-ci-solver.opam /src/
+COPY --chown=opam ocaml-ci.opam ocaml-ci-service.opam ocaml-ci-api.opam /src/
 RUN opam-2.1 install -y --deps-only .
 ADD --chown=opam . .
 RUN opam-2.1 exec -- dune build ./_build/install/default/bin/ocaml-ci-service
@@ -48,7 +54,7 @@ ENTRYPOINT ["dumb-init", "/usr/local/bin/ocaml-ci-service"]
 ENV OCAMLRUNPARAM=a=2
 # Enable experimental for docker manifest support
 ENV DOCKER_CLI_EXPERIMENTAL=enabled
-COPY --from=build /src/_build/install/default/bin/ocaml-ci-service /src/_build/install/default/bin/ocaml-ci-solver /usr/local/bin/
+COPY --from=build /src/_build/install/default/bin/ocaml-ci-service /src/_build/install/default/bin/solver-service /usr/local/bin/
 # Create migration directory
 RUN mkdir -p /migrations
 COPY --from=build /src/migrations /migrations

Dockerfile.gitlab

@@ -15,6 +15,10 @@ COPY --chown=opam \
 	ocluster/ocluster-api.opam \
 	ocluster/current_ocluster.opam \
 	/src/ocluster/
+COPY --chown=opam \
+	solver-service/solver-service-api.opam \
+	solver-service/solver-service.opam \
+	/src/solver-service/
 COPY --chown=opam \
 	ocaml-version/ocaml-version.opam \
 	/src/ocaml-version/
@@ -34,8 +38,10 @@ RUN opam pin add -yn current_docker.dev "./ocurrent" && \
     opam pin add -yn ocaml-version.dev "./ocaml-version" && \
     opam pin add -yn dockerfile.dev "./ocaml-dockerfile" && \
     opam pin add -yn dockerfile-opam.dev "./ocaml-dockerfile" && \
+    opam pin add -yn solver-service-api.dev "./solver-service" && \
+    opam pin add -yn solver-service.dev "./solver-service" && \
     opam pin add -yn ocluster-api.dev "./ocluster"
-COPY --chown=opam ocaml-ci.opam ocaml-ci-gitlab.opam ocaml-ci-service.opam ocaml-ci-api.opam ocaml-ci-solver.opam /src/
+COPY --chown=opam ocaml-ci.opam ocaml-ci-gitlab.opam ocaml-ci-service.opam ocaml-ci-api.opam /src/
 RUN opam-2.1 install -y --deps-only .
 ADD --chown=opam . .
 RUN opam-2.1 exec -- dune build ./_build/install/default/bin/ocaml-ci-gitlab
@@ -50,7 +56,7 @@ ENTRYPOINT ["dumb-init", "/usr/local/bin/ocaml-ci-gitlab"]
 ENV OCAMLRUNPARAM=a=2
 # Enable experimental for docker manifest support
 ENV DOCKER_CLI_EXPERIMENTAL=enabled
-COPY --from=build /src/_build/install/default/bin/ocaml-ci-gitlab /src/_build/install/default/bin/ocaml-ci-solver /usr/local/bin/
+COPY --from=build /src/_build/install/default/bin/ocaml-ci-gitlab /src/_build/install/default/bin/solver-service /usr/local/bin/
 # Create migration directory
 RUN mkdir -p /migrations
 COPY --from=build /src/migrations /migrations

api/dune

@@ -1,7 +1,11 @@
 (library
  (name ocaml_ci_api)
  (public_name ocaml-ci-api)
- (libraries unix capnp-rpc-lwt current_rpc ppx_deriving_yojson.runtime)
+ (libraries
+  capnp-rpc-lwt
+  current_rpc
+  ppx_deriving_yojson.runtime
+  solver-service-api)
  (flags
   (:standard -w -53-55))
  (preprocess

api/raw.ml

@@ -1 +1,2 @@
 include Schema.MakeRPC (Capnp_rpc_lwt)
+module Solve = Solver_service_api.Raw

api/schema.capnp

@@ -139,11 +139,3 @@ interface CI {
   orgsDetailed       @2 () -> (orgs :List(OrgInfo));
   # Get a list of organisations and related information for this CI capability.
 }
-
-interface Log {
-  write @0 (msg :Text);
-}
-
-interface Solver {
-  solve @0 (request :Text, log :Log) -> (response :Text);
-}

api/solver.ml

@@ -1,48 +1 @@
-open Lwt.Infix
-open Capnp_rpc_lwt
-
-module Log = struct
-  module X = Raw.Client.Log
-
-  type t = X.t Capability.t
-
-  let pp_timestamp f x =
-    let open Unix in
-    let tm = gmtime x in
-    Fmt.pf f "%04d-%02d-%02d %02d:%02d.%02d" (tm.tm_year + 1900) (tm.tm_mon + 1)
-      tm.tm_mday tm.tm_hour tm.tm_min tm.tm_sec
-
-  let write t msg =
-    let open X.Write in
-    let message_size = 150 + String.length msg in
-    let request, params =
-      Capability.Request.create ~message_size Params.init_pointer
-    in
-    Params.msg_set params msg;
-    Capability.call_for_unit_exn t method_id request
-
-  let info t fmt =
-    let now = Unix.gettimeofday () in
-    let k msg =
-      let thread = write t msg in
-      Lwt.on_failure thread (fun ex ->
-          Format.eprintf "Log.info(%S) failed: %a@." msg Fmt.exn ex)
-    in
-    Fmt.kstr k ("%a [INFO] @[" ^^ fmt ^^ "@]@.") pp_timestamp now
-end
-
-module X = Raw.Client.Solver
-
-type t = X.t Capability.t
-
-let solve t ~log reqs =
-  let open X.Solve in
-  let request, params = Capability.Request.create Params.init_pointer in
-  Params.request_set params
-    (Worker.Solve_request.to_yojson reqs |> Yojson.Safe.to_string);
-  Params.log_set params (Some log);
-  Capability.call_for_value_exn t method_id request >|= Results.response_get
-  >|= fun json ->
-  match Worker.Solve_response.of_yojson (Yojson.Safe.from_string json) with
-  | Ok x -> x
-  | Error ex -> failwith ex
+include Solver_service_api.Solver

api/worker.ml

@@ -1,49 +1 @@
-(** Communication between ocaml-ci and the workers. *)
-
-(** Variables describing a build environment. *)
-module Vars = struct
-  type t = {
-    arch : string;
-    os : string;
-    os_family : string;
-    os_distribution : string;
-    os_version : string;
-    ocaml_package : string;
-    ocaml_version : string;
-    opam_version : string;
-  }
-  [@@deriving yojson]
-end
-
-(** A set of packages for a single build. *)
-module Selection = struct
-  type t = {
-    id : string;  (** The platform ID from the request. *)
-    compat_pkgs : string list;
-        (** Local root packages compatible with the platform. *)
-    packages : string list;  (** The selected packages ("name.version"). *)
-    commit : string;  (** A commit in opam-repository to use. *)
-  }
-  [@@deriving yojson, ord]
-end
-
-(** A request to select sets of packages for the builds. *)
-module Solve_request = struct
-  type t = {
-    opam_repository_commit : string;  (** Commit in opam repository to use. *)
-    root_pkgs : (string * string) list;
-        (** Name and contents of top-level opam files. *)
-    pinned_pkgs : (string * string) list;
-        (** Name and contents of other pinned opam files. *)
-    platforms : (string * Vars.t) list;  (** Possible build platforms, by ID. *)
-  }
-  [@@deriving yojson]
-end
-
-(** The response from the solver. *)
-module Solve_response = struct
-  type ('a, 'b) result = ('a, 'b) Stdlib.result = Ok of 'a | Error of 'b
-  [@@deriving yojson]
-
-  type t = (Selection.t list, [ `Msg of string ]) result [@@deriving yojson]
-end
+include Solver_service_api.Worker

dune

@@ -20,4 +20,5 @@
  opam-0install-solver
  ocluster
  ocaml-version
+ solver-service
  ocaml-dockerfile)

dune-project

@@ -54,7 +54,7 @@
   current_ocluster
   (capnp-rpc-unix (>= 1.2))
   ocaml-ci-api
-  ocaml-ci-solver
+  solver-service
   ocluster-api
   obuilder-spec
   (timedesc (>= 0.9.0))
@@ -84,27 +84,6 @@
   ocaml-ci-service
   (ocaml (>= 4.13))
   (logs (>= 0.7.0))))
-(package
- (name ocaml-ci-solver)
- (synopsis "Choose package versions to test")
- (depends
-  (ppx_deriving_yojson (>= 3.6.1))
-  (ppx_deriving (>= 5.1))
-  yojson
-  (lwt (>= 5.6.1))
-  logs
-  fmt
-  ocaml-ci
-  ocaml-ci-api
-  (conf-libev (<> :os "win32"))
-  (opam-0install (>= "0.4.3"))
-  (git-unix (>= 3.9.0))
-  (timedesc (>= 0.9.0))
-  (ocaml (>= 4.14.0))
-  (capnp-rpc-unix (>= 1.2)))
- (conflicts
-  (carton (< 0.4.2))) ; workaround for mirage/ocaml-git#514
-)
 (package
  (name ocaml-ci-client)
  (synopsis "Command-line client for ocaml-ci")

gitlab/dune

@@ -42,7 +42,7 @@
 
 ; This is a hack to work around https://github.com/ocaml/dune/issues/3499
 ; We first build the binaries, then copy them to a new name, then install them.
-; This allows us to add in the run-time dependency on ocaml-ci-solver in the
+; This allows us to add in the run-time dependency on solver-service in the
 ; copy step.
 
 (install
@@ -60,13 +60,13 @@
 (rule
  (target main-copy.exe)
  (deps
-  (package ocaml-ci-solver))
+  (package solver-service))
  (action
   (copy main.exe main-copy.exe)))
 
 (rule
  (target local-copy.exe)
  (deps
-  (package ocaml-ci-solver))
+  (package solver-service))
  (action
   (copy local.exe local-copy.exe)))

gitlab/local.ml

@@ -6,7 +6,7 @@ let setup_log default_level =
   Prometheus_unix.Logging.init ?default_level ()
 
 let main () config mode repo : ('a, [ `Msg of string ]) result =
-  let solver = Ocaml_ci.Solver_pool.spawn_local () in
+  let solver = Ocaml_ci.Backend_solver.create None in
   let repo = Current_git.Local.v (Fpath.v repo) in
   let engine =
     Current.Engine.create ~config

gitlab/main.ml

@@ -125,9 +125,10 @@ module Gitlab = struct
 end
 
 let main () config mode app capnp_public_address capnp_listen_address
-    gitlab_auth submission_uri migrations : ('a, [ `Msg of string ]) result =
+    gitlab_auth submission_uri solver_uri migrations :
+    ('a, [ `Msg of string ]) result =
   Lwt_main.run
-    (let solver = Ocaml_ci.Solver_pool.spawn_local () in
+    (let solver = Ocaml_ci.Backend_solver.create solver_uri in
      run_capnp capnp_public_address capnp_listen_address
      >>= fun (vat, rpc_engine_resolver) ->
      let ocluster =
@@ -204,6 +205,17 @@ let migrations =
             the migration step is ignored."
          [ "migration-path" ])
 
+let submission_solver_service =
+  Arg.value
+  @@ Arg.opt Arg.(some Capnp_rpc_unix.sturdy_uri) None
+  @@ Arg.info
+       ~doc:
+         "The submission-solve.cap file for a scheduler service which handles \
+          a solver-worker. The cap file could be the same as \
+          $(b,--submission-service)."
+       ~docv:"FILE"
+       [ "submission-solver-service" ]
+
 let cmd =
   let doc = "Build OCaml projects on GitLab" in
   let info = Cmd.info "ocaml-ci-gitlab" ~doc in
@@ -219,6 +231,7 @@ let cmd =
         $ capnp_listen_address
         $ Current_gitlab.Auth.cmdliner
         $ submission_service
+        $ submission_solver_service
         $ migrations))
 
 let () = exit @@ Cmd.eval cmd

gitlab/pipeline.mli

@@ -1,12 +1,15 @@
 val local_test :
-  solver:Ocaml_ci_api.Solver.t -> Current_git.Local.t -> unit -> unit Current.t
+  solver:Ocaml_ci.Backend_solver.t ->
+  Current_git.Local.t ->
+  unit ->
+  unit Current.t
 (** [local_test ~solver repo] is a pipeline that tests local repository [repo]
     as the CI would. *)
 
 val v :
   ?ocluster:Cluster_api.Raw.Client.Submission.t Capnp_rpc_lwt.Sturdy_ref.t ->
   app:Current_gitlab.Api.t ->
-  solver:Ocaml_ci_api.Solver.t ->
+  solver:Ocaml_ci.Backend_solver.t ->
   migrations:string option ->
   unit ->
   unit Current.t

lib/analyse.ml

@@ -24,7 +24,7 @@ let read_file ~max_len path =
 
 (* A logging service that logs to [job]. *)
 let job_log job =
-  let module X = Ocaml_ci_api.Raw.Service.Log in
+  let module X = Ocaml_ci_api.Raw.Solve.Service.Log in
   X.local
   @@ object
        inherit X.service
@@ -146,15 +146,18 @@ module Analysis = struct
     in
     let request =
       {
-        Ocaml_ci_api.Worker.Solve_request.opam_repository_commit =
-          Current_git.Commit_id.hash opam_repository_commit;
+        Ocaml_ci_api.Worker.Solve_request.opam_repository_commits =
+          [
+            ( Current_git.Commit_id.repo opam_repository_commit,
+              Current_git.Commit_id.hash opam_repository_commit );
+          ];
         root_pkgs;
         pinned_pkgs;
         platforms;
       }
     in
     Capnp_rpc_lwt.Capability.with_ref (job_log job) @@ fun log ->
-    Ocaml_ci_api.Solver.solve solver request ~log >|= function
+    Backend_solver.solve solver job request ~log >|= function
     | Error (`Msg msg) -> Fmt.error_msg "Error from solver: %s" msg
     | Ok [] -> Fmt.error_msg "No solution found for any supported platform"
     | Ok x -> (
@@ -283,7 +286,7 @@ module Analysis = struct
 end
 
 module Examine = struct
-  type t = Ocaml_ci_api.Solver.t
+  type t = Backend_solver.t
 
   module Key = struct
     type t = Current_git.Commit.t

lib/analyse.mli

@@ -11,7 +11,7 @@ module Analysis : sig
     | `Opam_monorepo of Opam_monorepo.config list ]
 
   val of_dir :
-    solver:Ocaml_ci_api.Solver.t ->
+    solver:Backend_solver.t ->
     job:Current.Job.t ->
     platforms:(Variant.t * Ocaml_ci_api.Worker.Vars.t) list ->
     opam_repository_commit:Current_git.Commit_id.t ->
@@ -20,7 +20,7 @@ module Analysis : sig
 end
 
 val examine :
-  solver:Ocaml_ci_api.Solver.t ->
+  solver:Backend_solver.t ->
   platforms:Platform.t list Current.t ->
   opam_repository_commit:Current_git.Commit_id.t Current.t ->
   Current_git.Commit.t Current.t ->