diff --git a/Dockerfile.bridge-local-git b/Dockerfile.bridge-local-git index e335465c2..d93d44e3d 100644 --- a/Dockerfile.bridge-local-git +++ b/Dockerfile.bridge-local-git @@ -1,8 +1,6 @@ FROM docker/datakit:client RUN opam pin add github --dev -n -RUN opam pin add prometheus.dev /home/opam/src/datakit -n -RUN opam pin add prometheus-app.dev /home/opam/src/datakit -n RUN opam pin add datakit-github.dev /home/opam/src/datakit -n RUN opam depext datakit-github && opam install datakit-github --deps COPY datakit-bridge-local-git.opam /home/opam/src/datakit/datakit-bridge-local-git.opam diff --git a/Dockerfile.ci b/Dockerfile.ci index d4200f77c..a0e6c04b6 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,5 +1,5 @@ FROM ocaml/opam:alpine-3.5_ocaml-4.04.0 -ENV OPAMYES=1 OPAMERRLOGLEN=0 +ENV OPAMERRLOGLEN=0 OPAMYES=1 RUN sudo apk add tzdata RUN opam depext -i opam-devel && sudo cp $(opam config var "opam-devel:lib")/opam /usr/bin/opam @@ -8,13 +8,11 @@ RUN opam init -y RUN opam install ocaml -y RUN opam install depext -y -RUN opam pin add -yn prometheus https://github.com/docker/datakit.git && \ - opam pin add -yn prometheus-app https://github.com/docker/datakit.git && \ - opam pin add -yn datakit-client https://github.com/docker/datakit.git && \ +RUN opam pin add -yn datakit-client https://github.com/docker/datakit.git && \ opam pin add -yn github --dev && \ - opam pin add -yn datakit-github https://github.com/docker/datakit.git && \ - opam pin add -yn datakit-server https://github.com/docker/datakit.git && \ - opam pin add -yn datakit https://github.com/docker/datakit.git + opam pin add -yn datakit-github https://github.com/talex5/datakit.git#split-prometheus && \ + opam pin add -yn datakit-server https://github.com/talex5/datakit.git#split-prometheus && \ + opam pin add -yn datakit https://github.com/talex5/datakit.git#split-prometheus ADD datakit-ci.opam /tmp/deps/opam RUN opam pin add -y tmp /tmp/deps/opam -n @@ -26,4 +24,7 @@ RUN sudo chown opam /home/opam/datakit RUN opam pin add -k git datakit-client /home/opam/datakit -y RUN opam pin add -k git datakit-ci /home/opam/datakit -y +# Needed if we decide to run the tests +RUN opam pin add -k git datakit-github /home/opam/datakit -yn + VOLUME /secrets diff --git a/Dockerfile.client b/Dockerfile.client index 0a070b139..a9def3e7a 100644 --- a/Dockerfile.client +++ b/Dockerfile.client @@ -1,5 +1,5 @@ FROM ocaml/opam:alpine -ENV OPAMYES=1 OPAMERRLOGLEN=0 +ENV OPAMERRLOGLEN=0 OPAMYES=1 # TMP RUN cd /home/opam/opam-repository && git pull && opam update -y diff --git a/Dockerfile.prometheus b/Dockerfile.prometheus deleted file mode 100644 index a00ac5fc2..000000000 --- a/Dockerfile.prometheus +++ /dev/null @@ -1,19 +0,0 @@ -FROM ocaml/opam:alpine-3.5_ocaml-4.04.0 -ENV OPAMYES=1 OPAMERRLOGLEN=0 -RUN sudo apk add tzdata - -RUN opam depext -i opam-devel && sudo cp $(opam config var "opam-devel:lib")/opam /usr/bin/opam -RUN rm -rf ~/.opam -RUN opam init -y -RUN opam install ocaml -y -RUN OPAMYES=1 opam depext -i cohttp lwt fmt - -ADD prometheus.opam /tmp/deps/opam -RUN opam pin add -y tmp /tmp/deps/opam -n -RUN opam depext tmp -y -RUN opam install --deps-only tmp -y - -ADD . /home/opam/datakit -RUN sudo chown opam /home/opam/datakit -RUN opam pin add -y -k git prometheus /home/opam/datakit -n -RUN opam pin add -y -k git prometheus-app /home/opam/datakit diff --git a/Dockerfile.server b/Dockerfile.server index f8d34c4ad..9fbfd1b08 100644 --- a/Dockerfile.server +++ b/Dockerfile.server @@ -1,5 +1,5 @@ FROM ocaml/opam:alpine-3.5_ocaml-4.04.0 -ENV OPAMYES=1 OPAMERRLOGLEN=0 +ENV OPAMERRLOGLEN=0 OPAMYES=1 RUN sudo apk add tzdata RUN opam depext -i opam-devel && sudo cp $(opam config var "opam-devel:lib")/opam /usr/bin/opam @@ -13,10 +13,6 @@ RUN opam depext -ui lwt inotify alcotest conf-libev lambda-term # cache opam install of dependencies COPY datakit-client.opam /home/opam/src/datakit/datakit-client.opam COPY datakit-server.opam /home/opam/src/datakit/datakit-server.opam -COPY prometheus.opam /home/opam/src/datakit/prometheus.opam -COPY prometheus-app.opam /home/opam/src/datakit/prometheus-app.opam -RUN opam pin add prometheus.dev /home/opam/src/datakit -yn -RUN opam pin add prometheus-app.dev /home/opam/src/datakit -yn RUN opam pin add datakit-server.dev /home/opam/src/datakit -yn RUN opam pin add datakit-client.dev /home/opam/src/datakit -yn RUN opam depext datakit-server && opam install datakit-server --deps diff --git a/Makefile b/Makefile index 8a4cb6226..2c6835bbf 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,12 @@ PINOPTS=-y -k git TESTS = true -.PHONY: all clean test bundle COMMIT exe prometheus ci +.PHONY: all clean test bundle COMMIT exe ci all: datakit @ depends: - opam pin add ${PINOPTS} prometheus . - opam pin add ${PINOPTS} prometheus-app . opam pin add ${PINOPTS} datakit-client . opam pin add ${PINOPTS} datakit-server . opam pin add ${PINOPTS} datakit-github . @@ -34,13 +32,6 @@ github: bridge-local-git: ocaml pkg/pkg.ml build -n datakit-bridge-local-git -q -prometheus-app: - ocaml pkg/pkg.ml build -n prometheus-app -q --tests true - ocaml pkg/pkg.ml test _build/prometheus/tests/test.native - -prometheus: - ocaml pkg/pkg.ml build -n prometheus -q - ci: ocaml pkg/pkg.ml build -n datakit-ci -q --tests true ocaml pkg/pkg.ml test _build/ci/tests/test_ci.native diff --git a/_tags b/_tags index 30b67b3e9..e568e0151 100644 --- a/_tags +++ b/_tags @@ -1,7 +1,7 @@ true: -traverse true : bin_annot, safe_string not and not : warn_error(+1..49-3), warn(A-4-41-44) -not : package(bytes lwt astring logs result cstruct fmt rresult) +true: package(bytes lwt astring logs result cstruct fmt rresult) : traverse ### datakit-client @@ -20,7 +20,7 @@ not : package(bytes lwt astring logs result cstruct fmt rresult) #### irmin-io : package(conduit.lwt-unix irmin lwt.unix uri camlzip git tc) -: package(prometheus-app) +: package(prometheus-app.unix) : package(cmdliner fmt.cli fmt.tty logs.fmt asetmap) : package(git irmin irmin.git irmin.mem irmin-watcher) : package(irmin.http cohttp.lwt irmin-watcher), thread @@ -38,7 +38,7 @@ not : package(bytes lwt astring logs result cstruct fmt rresult) ### datakit-bridge : package(uri datakit-client datakit-server.vfs asetmap prometheus) -: package(datakit-server.fs9p prometheus-app) +: package(datakit-server.fs9p prometheus-app.unix) : package(github github-hooks.unix) : package(cmdliner fmt.cli fmt.tty logs.fmt) : package(github.unix github-hooks.unix) diff --git a/appveyor.yml b/appveyor.yml index afd25680d..c79a77185 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,7 +6,7 @@ environment: CYG_BASH: "%CYG_ROOT%\\bin\\bash -lc" TESTS: "false" OPAM_SWITCH: "4.03.0+mingw64c" - PINS: "prometheus:. prometheus-app:. datakit-server:. irmin.0.12.0:--dev hvsock:https://github.com/mirage/ocaml-hvsock.git#v0.11.1" + PINS: "prometheus:https://github.com/mirage/prometheus.git prometheus-app:https://github.com/mirage/prometheus.git datakit-server:. irmin.0.12.0:--dev hvsock:https://github.com/mirage/ocaml-hvsock.git#v0.11.1" install: - appveyor DownloadFile https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/appveyor-opam.sh diff --git a/bridge/github/main.ml b/bridge/github/main.ml index df04f2a2a..47bb37a92 100644 --- a/bridge/github/main.ml +++ b/bridge/github/main.ml @@ -86,7 +86,7 @@ type datakit_config = { let start () no_listen listen_urls datakit cap webhook resync_interval prometheus = quiet (); - let prometheus_threads = Prometheus_app.serve prometheus in + let prometheus_threads = Prometheus_unix.serve prometheus in set_signal_if_supported Sys.sigpipe Sys.Signal_ignore; set_signal_if_supported Sys.sigterm (Sys.Signal_handle (fun _ -> (* On Win32 we receive this signal on every failed Hyper-V @@ -276,7 +276,7 @@ let term = bidirectional mapping between the GitHub API and a Git branch."; ] in Term.(pure start $ setup_log $ no_listen $ listen_urls - $ datakit $ capabilities $ webhook $ resync $ Prometheus_app.opts), + $ datakit $ capabilities $ webhook $ resync $ Prometheus_unix.opts), Term.info (Filename.basename Sys.argv.(0)) ~version:Version.v ~doc ~man let () = match Term.eval term with diff --git a/ci/_tags b/ci/_tags index adeaed210..e79a24703 100644 --- a/ci/_tags +++ b/ci/_tags @@ -8,6 +8,6 @@ true: package(tls, channel, conduit.lwt-unix, io-page.unix) true: package(pbkdf, sexplib, ppx_sexp_conv) true: package(asetmap) true: package(github.unix datakit-github.client) -true: package(prometheus-app) +true: package(prometheus-app.unix) : include diff --git a/ci/self-ci/selfCI.ml b/ci/self-ci/selfCI.ml index 1b0dd5f4f..afeb99e07 100644 --- a/ci/self-ci/selfCI.ml +++ b/ci/self-ci/selfCI.ml @@ -11,7 +11,6 @@ module Dockerfile = struct let label = label |> Utils.default file in Docker.create ~logs ~pool ~timeout ~label file - let prometheus = v ~timeout:(30. *. minute) "Dockerfile.prometheus" let client = v ~timeout:(30. *. minute) "Dockerfile.client" let ci = v ~timeout:(30. *. minute) "Dockerfile.ci" let self_ci = v ~timeout:(30. *. minute) "ci/self-ci/Dockerfile" ~label:"Dockerfile.self-ci" @@ -39,7 +38,6 @@ let opam_test ?depexts pkg = let opam_test_ci = opam_test "datakit-ci" ~depexts:["conf-autoconf"; "conf-libpcre"] let opam_test_datakit = opam_test "datakit" ~depexts:["conf-gmp"; "conf-libpcre"; "conf-perl"; "conf-autoconf"] -let opam_test_prometheus = opam_test "prometheus-app" module Tests = struct open Term.Infix @@ -59,7 +57,6 @@ module Tests = struct method client = build Dockerfile.client method client_4_02 = build Dockerfile.client ~from:alpine_4_02 method local_git = build Dockerfile.local_git ~from:self#client - method prometheus = build Dockerfile.prometheus method ci = build Dockerfile.ci method self_ci = build Dockerfile.self_ci ~from:self#ci method server = build Dockerfile.server @@ -87,7 +84,6 @@ module Tests = struct "local-git", check_builds images#local_git; "libraries", Term.wait_for_all [ "server", check_builds images#server; - "prometheus", run_tests images#prometheus opam_test_prometheus; "client", check_builds images#client; "client-4.02", check_builds images#client_4_02; ] >|= fun () -> "Library tests succeeded" diff --git a/ci/src/cI_cache.ml b/ci/src/cI_cache.ml index f26fcdf6b..e9740bcdb 100644 --- a/ci/src/cI_cache.ml +++ b/ci/src/cI_cache.ml @@ -218,7 +218,7 @@ module Make(B : CI_s.BUILDER) = struct conn () >>= fun dk -> DK.branch dk branch_name >>*= fun branch -> Prometheus.Gauge.track_inprogress (Metrics.builds_in_progress (B.name t.builder)) @@ fun () -> - Prometheus.Summary.time (Metrics.build_time (B.name t.builder)) @@ fun () -> + Prometheus.Summary.time (Metrics.build_time (B.name t.builder)) Unix.gettimeofday @@ fun () -> DK.Branch.with_transaction branch (fun trans -> ensure_removed trans Path.rebuild >>= fun () -> ensure_removed trans Path.value >>= fun () -> diff --git a/ci/src/cI_log_reporter.ml b/ci/src/cI_log_reporter.ml index dbfc1423f..b0f3e1a7c 100644 --- a/ci/src/cI_log_reporter.ml +++ b/ci/src/cI_log_reporter.ml @@ -22,11 +22,11 @@ module Metrics = struct let inc_messages = let help = "Total number of messages logged" in - let c = Counter.v_labels ~label_names:[| "level"; "src" |] + let c = Counter.v_labels ~label_names:["level"; "src"] ~help ~namespace ~subsystem "messages_total" in fun lvl src -> let lvl = Logs.level_to_string (Some lvl) in - Counter.inc_one @@ Counter.labels c [| lvl; src |] + Counter.inc_one @@ Counter.labels c [lvl; src] end let report src level ~over k msgf = diff --git a/ci/src/cI_main.ml b/ci/src/cI_main.ml index 8e4f2e7f9..cf9af547f 100644 --- a/ci/src/cI_main.ml +++ b/ci/src/cI_main.ml @@ -3,7 +3,6 @@ open Datakit_github let key_bits = 4096 open CI_utils -open CI_utils.Infix open! Astring open Lwt.Infix @@ -45,7 +44,7 @@ let make_session_backend = function `Redis (Lwt_pool.create 4 ~check connect) let start_lwt ~pr_store ~web_ui ~secrets_dir ~canaries ~config ~session_backend ~prometheus = - let prometheus_threads = Prometheus_app.serve prometheus in + let prometheus_threads = Prometheus_unix.serve prometheus in let { CI_config.web_config; projects } = config in let dashboards = Repo.Map.map (fun p -> p.CI_config.dashboards) projects in let projects = Repo.Map.map (fun p -> p.CI_config.tests) projects in @@ -159,7 +158,7 @@ let run ?(info=default_info) config = $ config $ canaries $ session_backend - $ Prometheus_app.opts + $ Prometheus_unix.opts ) in match Term.eval (spec, info) with | `Error _ -> exit 1 diff --git a/ci/src/cI_web_utils.ml b/ci/src/cI_web_utils.ml index 63c7ec3ca..451392acc 100644 --- a/ci/src/cI_web_utils.ml +++ b/ci/src/cI_web_utils.ml @@ -14,17 +14,13 @@ module Metrics = struct let record_request = let help = "HTTP requests to web UI" in - let family = Counter.v_labels ~help ~label_names:[|"method"|] ~namespace ~subsystem "requests_total" in - fun req -> - let c = Counter.labels family [| Cohttp.(Code.string_of_method (Request.meth req)) |] in - Counter.inc_one c + let metric = Counter.v_label ~help ~label_name:"method" ~namespace ~subsystem "requests_total" in + fun req -> Counter.inc_one (metric (Cohttp.(Code.string_of_method (Request.meth req)))) let record_response = let help = "HTTP responses from web UI" in - let family = Counter.v_labels ~help ~label_names:[|"code"|] ~namespace ~subsystem "responses_total" in - fun code -> - let c = Counter.labels family [| Cohttp.(Code.string_of_status code) |] in - Counter.inc_one c + let metric = Counter.v_label ~help ~label_name:"code" ~namespace ~subsystem "responses_total" in + fun code -> Counter.inc_one (metric (Cohttp.(Code.string_of_status code))) let requests_in_progress = let help = "HTTP requests currently being handled by the web UI" in @@ -636,7 +632,7 @@ let pp_path = let callback ~routes _conn request body = Metrics.record_request request; - Prometheus.Summary.time Metrics.response_time_seconds @@ fun () -> + Prometheus.Summary.time Metrics.response_time_seconds Unix.gettimeofday @@ fun () -> Prometheus.Gauge.track_inprogress Metrics.requests_in_progress @@ fun () -> Wm.dispatch' routes ~body ~request >|= begin function diff --git a/circle.yml b/circle.yml index ecb2640c5..e66d9b605 100644 --- a/circle.yml +++ b/circle.yml @@ -17,8 +17,6 @@ dependencies: - opam init --comp system -n https://github.com/ocaml/opam-repository.git - opam switch system - opam update && opam upgrade - - opam pin add prometheus . -n - - opam pin add prometheus-app . -n - opam pin add datakit-server . -n - opam pin add datakit . -n - opam install depext && opam depext osx-fsevents datakit diff --git a/pkg/META.ci b/pkg/META.ci index 999210ad2..c27513f71 100644 --- a/pkg/META.ci +++ b/pkg/META.ci @@ -1,6 +1,6 @@ description = "Define CI pipelines on top of datakit-client" version = "%%VERSION%%" -requires = "fmt astring datakit-client protocol-9p.unix astring cmdliner fmt.tty logs.fmt fmt.cli logs.cli cohttp.lwt tyxml webmachine session.webmachine multipart-form-data tls channel conduit.lwt-unix io-page.unix pbkdf sexplib asetmap github.unix session.redis-lwt datakit-github.client prometheus-app" +requires = "fmt astring datakit-client protocol-9p.unix astring cmdliner fmt.tty logs.fmt fmt.cli logs.cli cohttp.lwt tyxml webmachine session.webmachine multipart-form-data tls channel conduit.lwt-unix io-page.unix pbkdf sexplib asetmap github.unix session.redis-lwt datakit-github.client prometheus-app.unix" archive(byte) = "datakit-ci.cma" archive(native) = "datakit-ci.cmxa" plugin(byte) = "datakit-ci.cma" diff --git a/pkg/META.prometheus b/pkg/META.prometheus deleted file mode 100644 index f3c22a492..000000000 --- a/pkg/META.prometheus +++ /dev/null @@ -1,7 +0,0 @@ -description = "Prometheus metrics reporting (client)" -version = "%%VERSION%%" -requires = "astring asetmap fmt lwt str unix" -archive(byte) = "prometheus.cma" -archive(native) = "prometheus.cmxa" -plugin(byte) = "prometheus.cma" -plugin(native) = "prometheus.cmxs" diff --git a/pkg/META.prometheus-app b/pkg/META.prometheus-app deleted file mode 100644 index 2ff02a4b2..000000000 --- a/pkg/META.prometheus-app +++ /dev/null @@ -1,8 +0,0 @@ -description = "Cohttp server for Prometheus metrics" -version = "%%VERSION%%" -requires = "prometheus cmdliner cohttp.lwt" -archive(byte) = "prometheus-app.cma" -archive(native) = "prometheus-app.cmxa" -plugin(byte) = "prometheus-app.cma" -plugin(native) = "prometheus-app.cmxs" -exists_if = "prometheus-app.cma" diff --git a/pkg/pkg.ml b/pkg/pkg.ml index dbbbef7a3..fd8120527 100755 --- a/pkg/pkg.ml +++ b/pkg/pkg.ml @@ -4,7 +4,6 @@ open Topkg let includes = function - | "prometheus" | "prometheus-app" -> ["prometheus"] | "datakit-ci" -> ["ci"] | "datakit" -> ["src"; "src/datakit"] | "datakit-server" -> ["src"; "src/datakit-server"] @@ -37,8 +36,6 @@ let metas = [ Pkg.meta_file ~install:false "pkg/META.server"; Pkg.meta_file ~install:false "pkg/META.github"; Pkg.meta_file ~install:false "pkg/META.ci"; - Pkg.meta_file ~install:false "pkg/META.prometheus"; - Pkg.meta_file ~install:false "pkg/META.prometheus-app"; ] let opams = @@ -50,8 +47,6 @@ let opams = Pkg.opam_file "datakit-server.opam" ~lint_deps_excluding ~install; Pkg.opam_file "datakit-github.opam" ~lint_deps_excluding ~install; Pkg.opam_file "datakit-ci.opam" ~lint_deps_excluding ~install; - Pkg.opam_file "prometheus.opam" ~lint_deps_excluding ~install; - Pkg.opam_file "prometheus-app.opam" ~lint_deps_excluding ~install; ] let () = @@ -101,15 +96,4 @@ let () = Pkg.test "ci/tests/test_ci" ~args:(Cmd.v "-q"); Pkg.test "ci/tests/exampleCI" ~run:false; ] - | "prometheus" -> Ok [ - Pkg.lib "pkg/META.prometheus" ~dst:"META"; - Pkg.lib "prometheus.opam" ~dst:"opam"; - Pkg.mllib "prometheus/src/prometheus.mllib"; - ] - | "prometheus-app" -> Ok [ - Pkg.lib "pkg/META.prometheus-app" ~dst:"META"; - Pkg.lib "prometheus-app.opam" ~dst:"opam"; - Pkg.mllib "prometheus/app/prometheus-app.mllib"; - Pkg.test "prometheus/tests/test" ~args:(Cmd.v "-q"); - ] | other -> R.error_msgf "unknown package name: %s" other diff --git a/prometheus-app.opam b/prometheus-app.opam deleted file mode 100644 index ac1cb4d0e..000000000 --- a/prometheus-app.opam +++ /dev/null @@ -1,27 +0,0 @@ -opam-version: "1.2" -maintainer: "datakit@docker.com" -authors: ["Thomas Leonard"] -license: "Apache" -homepage: "https://github.com/docker/datakit" -bug-reports: "https://github.com/docker/datakit/issues" -dev-repo: "https://github.com/docker/datakit.git" -doc: "https://docker.github.io/datakit/" - -build: [ - "ocaml" "pkg/pkg.ml" "build" "--pinned" "%{pinned}%" "--tests" "false" - "-n" "prometheus-app" -] -build-test: [ - ["ocaml" "pkg/pkg.ml" "build" "--pinned" "%{pinned}%" "--tests" "true" "-n" "prometheus-app"] - ["ocaml" "pkg/pkg.ml" "test" "_build/prometheus/tests/test.native"] -] - -depends: [ - "ocamlfind" {build} - "prometheus" - "fmt" - "cohttp" {>="0.20.0"} - "lwt" {>="2.5.0"} - "alcotest" {test} -] -available: [ocaml-version >= "4.03.0"] diff --git a/prometheus.opam b/prometheus.opam deleted file mode 100644 index da1a4fb85..000000000 --- a/prometheus.opam +++ /dev/null @@ -1,23 +0,0 @@ -opam-version: "1.2" -maintainer: "datakit@docker.com" -authors: ["Thomas Leonard"] -license: "Apache" -homepage: "https://github.com/docker/datakit" -bug-reports: "https://github.com/docker/datakit/issues" -dev-repo: "https://github.com/docker/datakit.git" -doc: "https://docker.github.io/datakit/" - -build: [ - "ocaml" "pkg/pkg.ml" "build" "--pinned" "%{pinned}%" "--tests" "false" - "-n" "prometheus" -] - -depends: [ - "ocamlfind" {build} - "astring" - "asetmap" - "fmt" - "lwt" {>="2.5.0"} - "alcotest" {test} -] -available: [ocaml-version >= "4.03.0"] diff --git a/prometheus/Makefile b/prometheus/Makefile deleted file mode 100644 index 121bfbcd7..000000000 --- a/prometheus/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -all: - make -C .. prometheus - make -C .. prometheus-app - -clean: - make -C .. clean diff --git a/prometheus/_tags b/prometheus/_tags deleted file mode 100644 index f3c988c73..000000000 --- a/prometheus/_tags +++ /dev/null @@ -1,7 +0,0 @@ -true: warn(A-4-58) -true: strict_sequence, safe_string, annot, bin_annot -true: package(lwt, astring, asetmap, fmt, str) - or : package(cohttp.lwt) -: package(alcotest) -: include -: include diff --git a/prometheus/app/_tags b/prometheus/app/_tags deleted file mode 100644 index 2c0f97fa7..000000000 --- a/prometheus/app/_tags +++ /dev/null @@ -1 +0,0 @@ -true: package(cohttp.lwt, uri, cmdliner) diff --git a/prometheus/app/prometheus-app.mllib b/prometheus/app/prometheus-app.mllib deleted file mode 100644 index 69f9135c6..000000000 --- a/prometheus/app/prometheus-app.mllib +++ /dev/null @@ -1 +0,0 @@ -Prometheus_app diff --git a/prometheus/app/prometheus_app.ml b/prometheus/app/prometheus_app.ml deleted file mode 100644 index 94d3e659e..000000000 --- a/prometheus/app/prometheus_app.ml +++ /dev/null @@ -1,179 +0,0 @@ -open Prometheus - -module Server = Cohttp_lwt_unix.Server - -let failf fmt = - Fmt.kstrf failwith fmt - -module TextFormat_0_0_4 = struct - let re_unquoted_escapes = Str.regexp "[\\\n]" - let re_quoted_escapes = Str.regexp "[\"\\\n]" - - let quote s = - match Str.matched_string s with - | "\\" -> "\\\\" - | "\n" -> "\\n" - | "\"" -> "\\\"" - | x -> failf "Unexpected match %S" x - - let output_metric_type f = function - | Counter -> Fmt.string f "counter" - | Gauge -> Fmt.string f "gauge" - | Summary -> Fmt.string f "summary" - (* | Histogram -> Fmt.string f "histogram" *) - - let output_unquoted f s = - Fmt.string f @@ Str.global_substitute re_unquoted_escapes quote s - - let output_quoted f s = - Fmt.string f @@ Str.global_substitute re_quoted_escapes quote s - - let output_value f v = - match classify_float v with - | FP_normal | FP_subnormal | FP_zero -> Fmt.float f v - | FP_infinite when v > 0.0 -> Fmt.string f "+Inf" - | FP_infinite -> Fmt.string f "-Inf" - | FP_nan -> Fmt.string f "Nan" - - let output_pairs f (label_names, label_values) = - let cont = ref false in - let output_pair name value = - if !cont then Fmt.string f ", " - else cont := true; - Fmt.pf f "%a=\"%a\"" LabelName.pp name output_quoted value - in - Array.iter2 output_pair label_names label_values - - let output_labels ~label_names f = function - | [||] -> () - | label_values -> Fmt.pf f "{%a}" output_pairs (label_names, label_values) - - let output_sample ~base ~label_names ~label_values f (ext, sample) = - Fmt.pf f "%a%s%a %a@." - MetricName.pp base ext - (output_labels ~label_names) label_values - output_value sample - - let output_metric ~name ~label_names f (label_values, samples) = - List.iter (output_sample ~base:name ~label_names ~label_values f) samples - - let output f = - MetricMap.iter (fun metric samples -> - let {MetricInfo.name; metric_type; help; label_names} = metric in - Fmt.pf f - "#HELP %a %a@.\ - #TYPE %a %a@.\ - %a" - MetricName.pp name output_unquoted help - MetricName.pp name output_metric_type metric_type - (LabelSetMap.pp ~sep:Fmt.nop (output_metric ~name ~label_names)) samples - ) -end - -module Runtime = struct - let start_time = Unix.gettimeofday () - - let current = ref (Gc.stat ()) - let update () = - current := Gc.stat () - - let simple_metric ~metric_type ~help name fn = - let info = { - MetricInfo. - name = MetricName.v name; - help; - metric_type; - label_names = [| |]; - } - in - let collect () = - LabelSetMap.singleton [| |] ["", fn ()] - in - info, collect - - let ocaml_gc_allocated_bytes = - simple_metric ~metric_type:Counter "ocaml_gc_allocated_bytes" Gc.allocated_bytes - ~help:"Total number of bytes allocated since the program was started." - - let ocaml_gc_major_words = - simple_metric ~metric_type:Counter "ocaml_gc_major_words" (fun () -> (!current).Gc.major_words) - ~help:"Number of words allocated in the major heap since the program was started." - - let ocaml_gc_minor_collections = - simple_metric ~metric_type:Counter "ocaml_gc_minor_collections" (fun () -> float_of_int (!current).Gc.minor_collections) - ~help:"Number of minor collection cycles completed since the program was started." - - let ocaml_gc_major_collections = - simple_metric ~metric_type:Counter "ocaml_gc_major_collections" (fun () -> float_of_int (!current).Gc.major_collections) - ~help:"Number of major collection cycles completed since the program was started." - - let ocaml_gc_heap_words = - simple_metric ~metric_type:Gauge "ocaml_gc_heap_words" (fun () -> float_of_int (!current).Gc.heap_words) - ~help:"Total size of the major heap, in words." - - let ocaml_gc_compactions = - simple_metric ~metric_type:Counter "ocaml_gc_compactions" (fun () -> float_of_int (!current).Gc.compactions) - ~help:"Number of heap compactions since the program was started." - - let ocaml_gc_top_heap_words = - simple_metric ~metric_type:Counter "ocaml_gc_top_heap_words" (fun () -> float_of_int (!current).Gc.top_heap_words) - ~help:"Maximum size reached by the major heap, in words." - - let process_cpu_seconds_total = - simple_metric ~metric_type:Counter "process_cpu_seconds_total" Sys.time - ~help:"Total user and system CPU time spent in seconds." - - let process_start_time_seconds = - simple_metric ~metric_type:Counter "process_start_time_seconds" (fun () -> start_time) - ~help:"Start time of the process since unix epoch in seconds." - - let metrics = [ - ocaml_gc_allocated_bytes; - ocaml_gc_major_words; - ocaml_gc_minor_collections; - ocaml_gc_major_collections; - ocaml_gc_heap_words; - ocaml_gc_compactions; - ocaml_gc_top_heap_words; - process_cpu_seconds_total; - process_start_time_seconds; - ] -end - -type config = int option - -let callback _conn req _body = - let open Cohttp in - let uri = Request.uri req in - match Request.meth req, Uri.path uri with - | `GET, "/metrics" -> - let data = Prometheus.CollectorRegistry.(collect default) in - let body = Fmt.to_to_string TextFormat_0_0_4.output data in - let headers = Header.init_with "Content-Type" "text/plain; version=0.0.4" in - Server.respond_string ~status:`OK ~headers ~body () - | _ -> Server.respond_error ~status:`Bad_request ~body:"Bad request" () - -let serve = function - | None -> [] - | Some port -> - let mode = `TCP (`Port port) in - let thread = Cohttp_lwt_unix.Server.create ~mode (Server.make ~callback ()) in - [thread] - -let listen_prometheus = - let open Cmdliner in - let doc = - Arg.info ~doc: - "Port on which to provide Prometheus metrics over HTTP, \ - of the form port or host:port" - ["listen-prometheus"] - in - Arg.(value & opt (some int) None doc) - -let opts = listen_prometheus - -let () = - CollectorRegistry.(register_pre_collect default) Runtime.update; - let add (info, collector) = - CollectorRegistry.(register default) info collector in - List.iter add Runtime.metrics diff --git a/prometheus/app/prometheus_app.mli b/prometheus/app/prometheus_app.mli deleted file mode 100644 index a691c111d..000000000 --- a/prometheus/app/prometheus_app.mli +++ /dev/null @@ -1,27 +0,0 @@ -(** Report metrics for Prometheus. - See: https://prometheus.io/ - - Notes: - - - This module is intended to be used by applications that export Prometheus metrics. - Libraries should only link against the `Prometheus` module. - - - This module automatically initialises itself and registers some standard collectors relating to - GC statistics, as recommended by Prometheus. - *) - -type config - -module TextFormat_0_0_4 : sig - val output : Prometheus.CollectorRegistry.snapshot Fmt.t - (** Format a snapshot in Prometheus's text format, version 0.0.4. *) -end - -val serve : config -> unit Lwt.t list -(** [serve config] starts a Cohttp server according to config. - It returns a singleton list containing the thread to monitor, - or an empty list if no server is configured. *) - -val opts : config Cmdliner.Term.t -(** [opts] is the extra command-line options to offer Prometheus - monitoring. *) diff --git a/prometheus/src/prometheus.ml b/prometheus/src/prometheus.ml deleted file mode 100644 index b75750d2a..000000000 --- a/prometheus/src/prometheus.ml +++ /dev/null @@ -1,231 +0,0 @@ -open! Astring -open Asetmap - -module type NAME_SPEC = sig - val valid : Str.regexp -end - -module type NAME = sig - type t = private string - val v : string -> t - val pp : t Fmt.t - val compare : t -> t -> int -end - -module Name(N : NAME_SPEC) : NAME = struct - type t = string - - let v name = - if not (Str.string_match N.valid name 0) then - failwith (Fmt.strf "Invalid name %S" name); - name - - let compare = String.compare - - let pp = Fmt.string -end - -module MetricName = Name(struct let valid = Str.regexp "^[a-zA-Z_:][a-zA-Z0-9_:]*$" end) -module LabelName = Name(struct let valid = Str.regexp "^[a-zA-Z_][a-zA-Z0-9_]*$" end) - -type metric_type = - | Counter - | Gauge - | Summary -(* - | Histogram -*) - -module LabelSet = struct - type t = string array - let compare (a:t) (b:t) = compare a b -end -module LabelSetMap = Map.Make(LabelSet) - -module MetricInfo = struct - type t = { - name : MetricName.t; - metric_type : metric_type; - help : string; - label_names : LabelName.t array; - } - - let pp_opt () = function - | None -> "" - | Some v -> v ^ "_" - - let v ~help ?(label_names=[||]) ~metric_type ?namespace ?subsystem name = - let name = Printf.sprintf "%a%a%s" pp_opt namespace pp_opt subsystem name in - { - name = MetricName.v name; - metric_type; - help; - label_names; - } - - let compare a b = MetricName.compare a.name b.name -end - -module MetricMap = Map.Make(MetricInfo) - -module CollectorRegistry = struct - type t = { - mutable metrics : (unit -> (string * float) list LabelSetMap.t) MetricMap.t; - mutable pre_collect : (unit -> unit) list; - } - - type snapshot = (string * float) list LabelSetMap.t MetricMap.t - - let create () = { - metrics = MetricMap.empty; - pre_collect = []; - } - - let default = create () - - let register_pre_collect t f = t.pre_collect <- f :: t.pre_collect - - let register t info collector = - assert (not (MetricMap.mem info t.metrics)); - t.metrics <- MetricMap.add info collector t.metrics - - let collect t = - List.iter (fun f -> f ()) t.pre_collect; - MetricMap.map (fun f -> f ()) t.metrics -end - -module type METRIC = sig - type family - type t - val v_labels : label_names:string array -> ?registry:CollectorRegistry.t -> help:string -> ?namespace:string -> ?subsystem:string -> string -> family - val labels : family -> string array -> t - val v_label : label_name:string -> ?registry:CollectorRegistry.t -> help:string -> ?namespace:string -> ?subsystem:string -> string -> (string -> t) - val v : ?registry:CollectorRegistry.t -> help:string -> ?namespace:string -> ?subsystem:string -> string -> t -end - -module type CHILD = sig - type t - val create : unit -> t - val values : t -> (string * float) list (* extension, value *) - val metric_type : metric_type -end - -module Metric(Child : CHILD) : sig - include METRIC with type t = Child.t -end = struct - type family = { - metric : MetricInfo.t; - mutable children : Child.t LabelSetMap.t; - } - - type t = Child.t - - let collect t = - LabelSetMap.map Child.values t.children - - let v_labels ~label_names ?(registry=CollectorRegistry.default) ~help ?namespace ?subsystem name = - let label_names = Array.map LabelName.v label_names in - let metric = MetricInfo.v ~metric_type:Child.metric_type ~help ~label_names ?namespace ?subsystem name in - let t = { - metric; - children = LabelSetMap.empty; - } in - CollectorRegistry.register registry metric (fun () -> collect t); - t - - let labels t label_values = - assert (Array.length t.metric.MetricInfo.label_names = Array.length label_values); - match LabelSetMap.find label_values t.children with - | Some child -> child - | None -> - let child = Child.create () in - t.children <- LabelSetMap.add label_values child t.children; - child - - let v_label ~label_name ?registry ~help ?namespace ?subsystem name = - let family = v_labels ~label_names:[|label_name|] ?registry ~help ?namespace ?subsystem name in - fun x -> labels family [| x |] - - let v ?registry ~help ?namespace ?subsystem name = - let family = v_labels ~help ?registry ?namespace ?subsystem name ~label_names:[||] in - labels family [||] -end - -module Counter = struct - include Metric(struct - type t = float ref - let create () = ref 0.0 - let values t = ["", !t] - let metric_type = Counter - end) - - let inc_one t = - t := !t +. 1.0 - - let inc t v = - assert (v >= 0.0); - t := !t +. v -end - -module Gauge = struct - include Metric(struct - type t = float ref - let create () = ref 0.0 - let values t = ["", !t] - let metric_type = Gauge - end) - - let inc t v = - t := !t +. v - let inc_one t = inc t 1.0 - - let dec t x = inc t (-. x) - let dec_one t = dec t 1.0 - - let set t v = - t := v - - let track_inprogress t fn = - inc_one t; - Lwt.finalize fn (fun () -> dec_one t; Lwt.return_unit) - - let time t fn = - let start = Unix.gettimeofday () in - Lwt.finalize fn - (fun () -> - let finish = Unix.gettimeofday () in - inc t (finish -. start); - Lwt.return_unit - ) -end - -module Summary = struct - module Child = struct - type t = { - mutable count : float; - mutable sum : float; - } - let create () = { count = 0.0; sum = 0.0 } - let values t = - [ - "_sum", t.sum; - "_count", t.count; - ] - let metric_type = Summary - end - include Metric(Child) - - let observe t v = - let open Child in - t.count <- t.count +. 1.0; - t.sum <- t.sum +. v - - let time t fn = - let start = Unix.gettimeofday () in - Lwt.finalize fn - (fun () -> - let finish = Unix.gettimeofday () in - observe t (finish -. start); - Lwt.return_unit - ) -end diff --git a/prometheus/src/prometheus.mli b/prometheus/src/prometheus.mli deleted file mode 100644 index aede609ce..000000000 --- a/prometheus/src/prometheus.mli +++ /dev/null @@ -1,136 +0,0 @@ -(** Collect metrics for Prometheus. - See: https://prometheus.io/ - - Notes: - - - The Prometheus docs require that client libraries are thread-safe. We interpret this to mean safe - with Lwt threads, NOT with native threading. - - - The [time] functions are unfortunate, as they create a dependency on [Unix]. - Maybe they should be moved elsewhere. - - - The API is rather limited. In particular, it is not yet possible to write your own collectors. - - - This library is intended to be a dependency of any library that might need to report metrics, - even though many applications will not enable it. Therefore it should have minimal dependencies. -*) - -open Asetmap - -type metric_type = - | Counter - | Gauge - | Summary - -module type NAME = sig - type t = private string - val v : string -> t - val pp : t Fmt.t - val compare : t -> t -> int -end - -module MetricName : NAME -module LabelName : NAME - -module MetricInfo : sig - type t = { - name : MetricName.t; - metric_type : metric_type; - help : string; - label_names : LabelName.t array; - } -end - -module LabelSetMap : Map.S with type key = string array -module MetricMap : Map.S with type key = MetricInfo.t - -module CollectorRegistry : sig - type t - (** A collection of metrics to be monitored. *) - - type snapshot = (string * float) list LabelSetMap.t MetricMap.t - (** The result of reading a set of metrics. *) - - val create : unit -> t - (** [create ()] is a fresh registry. This is mostly useful for testing. *) - - val default : t - (** The default registry. *) - - val collect : t -> snapshot - (** Read the current value of each metric. *) - - val register : t -> MetricInfo.t -> (unit -> (string * float) list LabelSetMap.t) -> unit - (** [register t metric collector] adds [metric] to the set of metrics being collected. - It will call [collector ()] to collect the values each time [collect] is called. *) - - val register_pre_collect : t -> (unit -> unit) -> unit - (** [register_pre_collect t fn] arranges for [fn ()] to be called at the start - of each collection. This is useful if one expensive call provides - information about multiple metrics. *) -end - -module type METRIC = sig - type family - (** A collection of metrics that are the same except for their labels. - e.g. "Number of HTTP responses" *) - - type t - (** A particular metric. - e.g. "Number of HTTP responses with code=404" *) - - val v_labels : label_names:string array -> ?registry:CollectorRegistry.t -> help:string -> ?namespace:string -> ?subsystem:string -> string -> family - (** [v_labels ~label_names ~help ~namespace ~subsystem name] is a family of metrics with full name - [namespace_subsystem_name] and documentation string [help]. Each metric in the family will provide - a value for each of the labels. - The new family is registered with [registry] (default: [CollectorRegistry.default]). *) - - val labels : family -> string array -> t - (** [labels family label_values] is the metric in [family] with these values for the labels. - The order of the values must be the same as the order of the [label_names] passed to [v_labels]; - you may wish to write a wrapper function with labelled arguments to avoid mistakes. - If this is called multiple times with the same set of values, the existing metric will be returned. *) - - val v_label : label_name:string -> ?registry:CollectorRegistry.t -> help:string -> ?namespace:string -> ?subsystem:string -> string -> (string -> t) - (** [v_label] is a convenience wrapper around [v_labels] for the case where there is a single label. - The result is a function from the single label's value to the metric. *) - - val v : ?registry:CollectorRegistry.t -> help:string -> ?namespace:string -> ?subsystem:string -> string -> t - (** [v] is a convenience wrapper around [v_labels] for the case where there are no labels. *) -end - -module Counter : sig - (** A counter is a cumulative metric that represents a single numerical value that only ever goes up. *) - - include METRIC - val inc_one : t -> unit - val inc : t -> float -> unit - (** [inc t v] increases [t] by [v], which must be non-negative. *) -end - -module Gauge : sig - (** A gauge is a metric that represents a single numerical value that can arbitrarily go up and down. *) - - include METRIC - - val inc_one : t -> unit - val inc : t -> float -> unit - - val dec_one : t -> unit - val dec : t -> float -> unit - - val set : t -> float -> unit - - val track_inprogress : t -> (unit -> 'a Lwt.t) -> 'a Lwt.t - val time : t -> (unit -> 'a Lwt.t) -> 'a Lwt.t -end - -module Summary : sig - (** A summary is a metric that records both the number of readings and their total. - This allows calculating the average. *) - - include METRIC - - val observe : t -> float -> unit - val time : t -> (unit -> 'a Lwt.t) -> 'a Lwt.t -end diff --git a/prometheus/src/prometheus.mllib b/prometheus/src/prometheus.mllib deleted file mode 100644 index 4129a7616..000000000 --- a/prometheus/src/prometheus.mllib +++ /dev/null @@ -1 +0,0 @@ -Prometheus diff --git a/prometheus/tests/test.ml b/prometheus/tests/test.ml deleted file mode 100644 index 18f24f0f8..000000000 --- a/prometheus/tests/test.ml +++ /dev/null @@ -1,37 +0,0 @@ -open! Astring -open Prometheus -open Prometheus_app - -let test_metrics () = - let registry = CollectorRegistry.create () in - let requests = - let label_names = [| "method"; "path" |] in - Counter.v_labels ~label_names ~registry ~help:"Requests" ~namespace:"dkci" ~subsystem:"tests" "requests" in - let m = Counter.v ~registry ~help:"Test \\counter:\n1" "tests" in - Counter.inc_one m; - let get_index = Counter.labels requests [| "GET"; "\"\\-\n" |] in - let post_login = Counter.labels requests [| "POST"; "/login" |] in - Counter.inc get_index 5.; - Counter.inc post_login 2.; - let post_login2 = Counter.labels requests [| "POST"; "/login" |] in - Counter.inc_one post_login2; - let output = Fmt.to_to_string TextFormat_0_0_4.output (CollectorRegistry.collect registry) in - Alcotest.(check string) "Text output" - "#HELP dkci_tests_requests Requests\n\ - #TYPE dkci_tests_requests counter\n\ - dkci_tests_requests{method=\"GET\", path=\"\\\"\\\\-\\n\"} 5\n\ - dkci_tests_requests{method=\"POST\", path=\"/login\"} 3\n\ - #HELP tests Test \\\\counter:\\n1\n\ - #TYPE tests counter\n\ - tests 1\n\ - " - output - -let test_set = [ - "Metrics", `Quick, test_metrics; -] - -let () = - Alcotest.run "prometheus" [ - "main", test_set; - ] diff --git a/prometheus/tests/test.mli b/prometheus/tests/test.mli deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/datakit/main.ml b/src/datakit/main.ml index 412fa8835..31237657b 100644 --- a/src/datakit/main.ml +++ b/src/datakit/main.ml @@ -163,7 +163,7 @@ let start ~listen_9p ~listen_http prometheus git = Log.app (fun l -> l "Starting %s %s ..." (Filename.basename Sys.argv.(0)) Version.v ); - let prometheus_threads = Prometheus_app.serve prometheus in + let prometheus_threads = Prometheus_unix.serve prometheus in let serve_http = match listen_http with | None -> [] | Some uri -> [http_server (Uri.of_string uri) git] @@ -291,7 +291,7 @@ let term = `S "DESCRIPTION"; `P "$(tname) is a Git-like database with a 9p interface."; ] in - Term.(pure start $ setup_log $ listen_9p $ listen_http $ Prometheus_app.opts + Term.(pure start $ setup_log $ listen_9p $ listen_http $ Prometheus_unix.opts $ git $ auto_push), Term.info (Filename.basename Sys.argv.(0)) ~version:Version.v ~doc ~man