Merge pull request #473 from talex5/split-prometheus

Move Prometheus libraries into a separate repository

Dockerfile.bridge-local-git

@@ -1,8 +1,6 @@
 FROM docker/datakit:client
 
 RUN opam pin add github --dev -n
-RUN opam pin add prometheus.dev /home/opam/src/datakit -n
-RUN opam pin add prometheus-app.dev /home/opam/src/datakit -n
 RUN opam pin add datakit-github.dev /home/opam/src/datakit -n
 RUN opam depext datakit-github && opam install datakit-github --deps
 COPY datakit-bridge-local-git.opam /home/opam/src/datakit/datakit-bridge-local-git.opam

Dockerfile.ci

@@ -1,5 +1,5 @@
 FROM ocaml/opam:alpine-3.5_ocaml-4.04.0
-ENV OPAMYES=1 OPAMERRLOGLEN=0
+ENV OPAMERRLOGLEN=0 OPAMYES=1
 RUN sudo apk add tzdata
 
 RUN opam depext -i opam-devel && sudo cp $(opam config var "opam-devel:lib")/opam /usr/bin/opam
@@ -8,13 +8,11 @@ RUN opam init -y
 RUN opam install ocaml -y
 RUN opam install depext -y
 
-RUN opam pin add -yn prometheus https://github.com/docker/datakit.git && \
-    opam pin add -yn prometheus-app https://github.com/docker/datakit.git && \
-    opam pin add -yn datakit-client https://github.com/docker/datakit.git && \
+RUN opam pin add -yn datakit-client https://github.com/docker/datakit.git && \
     opam pin add -yn github --dev && \
-    opam pin add -yn datakit-github https://github.com/docker/datakit.git && \
-    opam pin add -yn datakit-server https://github.com/docker/datakit.git && \
-    opam pin add -yn datakit https://github.com/docker/datakit.git
+    opam pin add -yn datakit-github https://github.com/talex5/datakit.git#split-prometheus && \
+    opam pin add -yn datakit-server https://github.com/talex5/datakit.git#split-prometheus && \
+    opam pin add -yn datakit https://github.com/talex5/datakit.git#split-prometheus
 
 ADD datakit-ci.opam /tmp/deps/opam
 RUN opam pin add -y tmp /tmp/deps/opam -n
@@ -26,4 +24,7 @@ RUN sudo chown opam /home/opam/datakit
 RUN opam pin add -k git datakit-client /home/opam/datakit -y
 RUN opam pin add -k git datakit-ci /home/opam/datakit -y
 
+# Needed if we decide to run the tests
+RUN opam pin add -k git datakit-github /home/opam/datakit -yn
+
 VOLUME /secrets

Dockerfile.client

@@ -1,5 +1,5 @@
 FROM ocaml/opam:alpine
-ENV OPAMYES=1 OPAMERRLOGLEN=0
+ENV OPAMERRLOGLEN=0 OPAMYES=1
 
 # TMP
 RUN cd /home/opam/opam-repository && git pull && opam update -y

Dockerfile.server

@@ -1,5 +1,5 @@
 FROM ocaml/opam:alpine-3.5_ocaml-4.04.0
-ENV OPAMYES=1 OPAMERRLOGLEN=0
+ENV OPAMERRLOGLEN=0 OPAMYES=1
 RUN sudo apk add tzdata
 
 RUN opam depext -i opam-devel && sudo cp $(opam config var "opam-devel:lib")/opam /usr/bin/opam
@@ -13,10 +13,6 @@ RUN opam depext -ui lwt inotify alcotest conf-libev lambda-term
 # cache opam install of dependencies
 COPY datakit-client.opam /home/opam/src/datakit/datakit-client.opam
 COPY datakit-server.opam /home/opam/src/datakit/datakit-server.opam
-COPY prometheus.opam /home/opam/src/datakit/prometheus.opam
-COPY prometheus-app.opam /home/opam/src/datakit/prometheus-app.opam
-RUN opam pin add prometheus.dev /home/opam/src/datakit -yn
-RUN opam pin add prometheus-app.dev /home/opam/src/datakit -yn
 RUN opam pin add datakit-server.dev /home/opam/src/datakit -yn
 RUN opam pin add datakit-client.dev /home/opam/src/datakit -yn
 RUN opam depext datakit-server && opam install datakit-server --deps

Makefile

@@ -5,14 +5,12 @@ PINOPTS=-y -k git
 
 TESTS = true
 
-.PHONY: all clean test bundle COMMIT exe prometheus ci
+.PHONY: all clean test bundle COMMIT exe ci
 
 all: datakit
 	@
 
 depends:
-	opam pin add ${PINOPTS} prometheus .
-	opam pin add ${PINOPTS} prometheus-app .
 	opam pin add ${PINOPTS} datakit-client .
 	opam pin add ${PINOPTS} datakit-server .
 	opam pin add ${PINOPTS} datakit-github .
@@ -34,13 +32,6 @@ github:
 bridge-local-git:
 	ocaml pkg/pkg.ml build -n datakit-bridge-local-git -q
 
-prometheus-app:
-	ocaml pkg/pkg.ml build -n prometheus-app -q --tests true
-	ocaml pkg/pkg.ml test _build/prometheus/tests/test.native
-
-prometheus:
-	ocaml pkg/pkg.ml build -n prometheus -q
-
 ci:
 	ocaml pkg/pkg.ml build -n datakit-ci -q --tests true
 	ocaml pkg/pkg.ml test _build/ci/tests/test_ci.native

_tags

@@ -1,7 +1,7 @@
 true: -traverse
 true : bin_annot, safe_string
 not <ci/**> and not <bridge/local/**>: warn_error(+1..49-3), warn(A-4-41-44)
-not <prometheus/**>: package(bytes lwt astring logs result cstruct fmt rresult)
+true: package(bytes lwt astring logs result cstruct fmt rresult)
 <ci/static/**>: traverse
 
 ### datakit-client
@@ -20,7 +20,7 @@ not <prometheus/**>: package(bytes lwt astring logs result cstruct fmt rresult)
 #### irmin-io
 <src/datakit/*>: package(conduit.lwt-unix irmin lwt.unix uri camlzip git tc)
 
-<src/datakit/*>: package(prometheus-app)
+<src/datakit/*>: package(prometheus-app.unix)
 <src/datakit/main.*>: package(cmdliner fmt.cli fmt.tty logs.fmt asetmap)
 <src/datakit/main.*>: package(git irmin irmin.git irmin.mem irmin-watcher)
 <src/datakit/main.*>: package(irmin.http cohttp.lwt irmin-watcher), thread
@@ -38,7 +38,7 @@ not <prometheus/**>: package(bytes lwt astring logs result cstruct fmt rresult)
 ### datakit-bridge
 
 <bridge/github/*>: package(uri datakit-client datakit-server.vfs asetmap prometheus)
-<bridge/github/main.*>: package(datakit-server.fs9p prometheus-app)
+<bridge/github/main.*>: package(datakit-server.fs9p prometheus-app.unix)
 <bridge/github/datakit_github_*>: package(github github-hooks.unix)
 <bridge/github/main.*>: package(cmdliner fmt.cli fmt.tty logs.fmt)
 <bridge/github/main.*>: package(github.unix github-hooks.unix)

appveyor.yml

@@ -6,7 +6,7 @@ environment:
   CYG_BASH: "%CYG_ROOT%\\bin\\bash -lc"
   TESTS: "false"
   OPAM_SWITCH: "4.03.0+mingw64c"
-  PINS: "prometheus:. prometheus-app:. datakit-server:. irmin.0.12.0:--dev hvsock:https://github.com/mirage/ocaml-hvsock.git#v0.11.1"
+  PINS: "prometheus:https://github.com/mirage/prometheus.git prometheus-app:https://github.com/mirage/prometheus.git datakit-server:. irmin.0.12.0:--dev hvsock:https://github.com/mirage/ocaml-hvsock.git#v0.11.1"
 
 install:
   - appveyor DownloadFile https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/appveyor-opam.sh

bridge/github/main.ml

@@ -86,7 +86,7 @@ type datakit_config = {
 
 let start () no_listen listen_urls datakit cap webhook resync_interval prometheus =
   quiet ();
-  let prometheus_threads = Prometheus_app.serve prometheus in
+  let prometheus_threads = Prometheus_unix.serve prometheus in
   set_signal_if_supported Sys.sigpipe Sys.Signal_ignore;
   set_signal_if_supported Sys.sigterm (Sys.Signal_handle (fun _ ->
       (* On Win32 we receive this signal on every failed Hyper-V
@@ -276,7 +276,7 @@ let term =
         bidirectional mapping between the GitHub API and a Git branch.";
   ] in
   Term.(pure start $ setup_log $ no_listen $ listen_urls
-        $ datakit $ capabilities $ webhook $ resync $ Prometheus_app.opts),
+        $ datakit $ capabilities $ webhook $ resync $ Prometheus_unix.opts),
   Term.info (Filename.basename Sys.argv.(0)) ~version:Version.v ~doc ~man
 
 let () = match Term.eval term with

ci/_tags

@@ -8,6 +8,6 @@ true: package(tls, channel, conduit.lwt-unix, io-page.unix)
 true: package(pbkdf, sexplib, ppx_sexp_conv)
 true: package(asetmap)
 true: package(github.unix datakit-github.client)
-true: package(prometheus-app)
+true: package(prometheus-app.unix)
 
 <src>: include

ci/self-ci/selfCI.ml

@@ -11,7 +11,6 @@ module Dockerfile = struct
     let label = label |> Utils.default file in
     Docker.create ~logs ~pool ~timeout ~label file
 
-  let prometheus  = v ~timeout:(30. *. minute) "Dockerfile.prometheus"
   let client      = v ~timeout:(30. *. minute) "Dockerfile.client"
   let ci          = v ~timeout:(30. *. minute) "Dockerfile.ci"
   let self_ci     = v ~timeout:(30. *. minute) "ci/self-ci/Dockerfile" ~label:"Dockerfile.self-ci"
@@ -39,7 +38,6 @@ let opam_test ?depexts pkg =
 
 let opam_test_ci = opam_test "datakit-ci" ~depexts:["conf-autoconf"; "conf-libpcre"]
 let opam_test_datakit = opam_test "datakit" ~depexts:["conf-gmp"; "conf-libpcre"; "conf-perl"; "conf-autoconf"]
-let opam_test_prometheus = opam_test "prometheus-app"
 
 module Tests = struct
   open Term.Infix
@@ -59,7 +57,6 @@ module Tests = struct
       method client      = build Dockerfile.client
       method client_4_02 = build Dockerfile.client     ~from:alpine_4_02
       method local_git   = build Dockerfile.local_git  ~from:self#client
-      method prometheus  = build Dockerfile.prometheus
       method ci          = build Dockerfile.ci
       method self_ci     = build Dockerfile.self_ci    ~from:self#ci
       method server      = build Dockerfile.server
@@ -87,7 +84,6 @@ module Tests = struct
         "local-git",   check_builds images#local_git;
         "libraries",   Term.wait_for_all [
           "server",      check_builds images#server;
-          "prometheus",  run_tests    images#prometheus   opam_test_prometheus;
           "client",      check_builds images#client;
           "client-4.02", check_builds images#client_4_02;
         ] >|= fun () -> "Library tests succeeded"

ci/src/cI_cache.ml

@@ -218,7 +218,7 @@ module Make(B : CI_s.BUILDER) = struct
               conn () >>= fun dk ->
               DK.branch dk branch_name >>*= fun branch ->
               Prometheus.Gauge.track_inprogress (Metrics.builds_in_progress (B.name t.builder)) @@ fun () ->
-              Prometheus.Summary.time (Metrics.build_time (B.name t.builder)) @@ fun () ->
+              Prometheus.Summary.time (Metrics.build_time (B.name t.builder)) Unix.gettimeofday @@ fun () ->
               DK.Branch.with_transaction branch (fun trans ->
                   ensure_removed trans Path.rebuild >>= fun () ->
                   ensure_removed trans Path.value >>= fun () ->

ci/src/cI_log_reporter.ml

@@ -22,11 +22,11 @@ module Metrics = struct
 
   let inc_messages =
     let help = "Total number of messages logged" in
-    let c = Counter.v_labels ~label_names:[| "level"; "src" |]
+    let c = Counter.v_labels ~label_names:["level"; "src"]
         ~help ~namespace ~subsystem "messages_total" in
     fun lvl src ->
       let lvl = Logs.level_to_string (Some lvl) in
-      Counter.inc_one @@ Counter.labels c [| lvl; src |]
+      Counter.inc_one @@ Counter.labels c [lvl; src]
 end
 
 let report src level ~over k msgf =

ci/src/cI_main.ml

@@ -3,7 +3,6 @@ open Datakit_github
 let key_bits = 4096
 
 open CI_utils
-open CI_utils.Infix
 open! Astring
 open Lwt.Infix
 
@@ -45,7 +44,7 @@ let make_session_backend = function
     `Redis (Lwt_pool.create 4 ~check connect)
 
 let start_lwt ~pr_store ~web_ui ~secrets_dir ~canaries ~config ~session_backend ~prometheus =
-  let prometheus_threads = Prometheus_app.serve prometheus in
+  let prometheus_threads = Prometheus_unix.serve prometheus in
   let { CI_config.web_config; projects } = config in
   let dashboards = Repo.Map.map (fun p -> p.CI_config.dashboards) projects in
   let projects = Repo.Map.map (fun p -> p.CI_config.tests) projects in
@@ -159,7 +158,7 @@ let run ?(info=default_info) config =
                    $ config
                    $ canaries
                    $ session_backend
-                   $ Prometheus_app.opts
+                   $ Prometheus_unix.opts
                   ) in
   match Term.eval (spec, info) with
   | `Error _ -> exit 1

ci/src/cI_web_utils.ml

@@ -14,17 +14,13 @@ module Metrics = struct
 
   let record_request =
     let help = "HTTP requests to web UI" in
-    let family = Counter.v_labels ~help ~label_names:[|"method"|] ~namespace ~subsystem "requests_total" in
-    fun req ->
-      let c = Counter.labels family [| Cohttp.(Code.string_of_method (Request.meth req)) |] in
-      Counter.inc_one c
+    let metric = Counter.v_label ~help ~label_name:"method" ~namespace ~subsystem "requests_total" in
+    fun req -> Counter.inc_one (metric (Cohttp.(Code.string_of_method (Request.meth req))))
 
   let record_response =
     let help = "HTTP responses from web UI" in
-    let family = Counter.v_labels ~help ~label_names:[|"code"|] ~namespace ~subsystem "responses_total" in
-    fun code ->
-      let c = Counter.labels family [| Cohttp.(Code.string_of_status code) |] in
-      Counter.inc_one c
+    let metric = Counter.v_label ~help ~label_name:"code" ~namespace ~subsystem "responses_total" in
+    fun code -> Counter.inc_one (metric (Cohttp.(Code.string_of_status code)))
 
   let requests_in_progress =
     let help = "HTTP requests currently being handled by the web UI" in
@@ -636,7 +632,7 @@ let pp_path =
 
 let callback ~routes _conn request body =
   Metrics.record_request request;
-  Prometheus.Summary.time Metrics.response_time_seconds @@ fun () ->
+  Prometheus.Summary.time Metrics.response_time_seconds Unix.gettimeofday @@ fun () ->
   Prometheus.Gauge.track_inprogress Metrics.requests_in_progress @@ fun () ->
   Wm.dispatch' routes ~body ~request
   >|= begin function

circle.yml

@@ -17,8 +17,6 @@ dependencies:
   - opam init --comp system -n https://github.com/ocaml/opam-repository.git
   - opam switch system
   - opam update && opam upgrade
-  - opam pin add prometheus . -n
-  - opam pin add prometheus-app . -n
   - opam pin add datakit-server . -n
   - opam pin add datakit . -n
   - opam install depext && opam depext osx-fsevents datakit

pkg/META.ci

@@ -1,6 +1,6 @@
 description = "Define CI pipelines on top of datakit-client"
 version = "%%VERSION%%"
-requires = "fmt astring datakit-client protocol-9p.unix astring cmdliner fmt.tty logs.fmt fmt.cli logs.cli cohttp.lwt tyxml webmachine session.webmachine multipart-form-data tls channel conduit.lwt-unix io-page.unix pbkdf sexplib asetmap github.unix session.redis-lwt datakit-github.client prometheus-app"
+requires = "fmt astring datakit-client protocol-9p.unix astring cmdliner fmt.tty logs.fmt fmt.cli logs.cli cohttp.lwt tyxml webmachine session.webmachine multipart-form-data tls channel conduit.lwt-unix io-page.unix pbkdf sexplib asetmap github.unix session.redis-lwt datakit-github.client prometheus-app.unix"
 archive(byte)   = "datakit-ci.cma"
 archive(native) = "datakit-ci.cmxa"
 plugin(byte)    = "datakit-ci.cma"

pkg/pkg.ml

@@ -4,7 +4,6 @@
 open Topkg
 
 let includes = function
-  | "prometheus" | "prometheus-app" -> ["prometheus"]
   | "datakit-ci" -> ["ci"]
   | "datakit" -> ["src"; "src/datakit"]
   | "datakit-server" -> ["src"; "src/datakit-server"]
@@ -37,8 +36,6 @@ let metas = [
   Pkg.meta_file ~install:false "pkg/META.server";
   Pkg.meta_file ~install:false "pkg/META.github";
   Pkg.meta_file ~install:false "pkg/META.ci";
-  Pkg.meta_file ~install:false "pkg/META.prometheus";
-  Pkg.meta_file ~install:false "pkg/META.prometheus-app";
 ]
 
 let opams =
@@ -50,8 +47,6 @@ let opams =
     Pkg.opam_file "datakit-server.opam" ~lint_deps_excluding ~install;
     Pkg.opam_file "datakit-github.opam" ~lint_deps_excluding ~install;
     Pkg.opam_file "datakit-ci.opam" ~lint_deps_excluding ~install;
-    Pkg.opam_file "prometheus.opam" ~lint_deps_excluding ~install;
-    Pkg.opam_file "prometheus-app.opam" ~lint_deps_excluding ~install;
   ]
 
 let () =
@@ -101,15 +96,4 @@ let () =
       Pkg.test  "ci/tests/test_ci" ~args:(Cmd.v "-q");
       Pkg.test  "ci/tests/exampleCI" ~run:false;
     ]
-  | "prometheus" -> Ok [
-      Pkg.lib   "pkg/META.prometheus"   ~dst:"META";
-      Pkg.lib   "prometheus.opam"       ~dst:"opam";
-      Pkg.mllib "prometheus/src/prometheus.mllib";
-    ]
-  | "prometheus-app" -> Ok [
-      Pkg.lib   "pkg/META.prometheus-app" ~dst:"META";
-      Pkg.lib   "prometheus-app.opam"     ~dst:"opam";
-      Pkg.mllib "prometheus/app/prometheus-app.mllib";
-      Pkg.test  "prometheus/tests/test" ~args:(Cmd.v "-q");
-    ]
   | other -> R.error_msgf "unknown package name: %s" other