refactor/rename is_it_up (#8)

* [grafana] Improve dashboard * [make] attach-app-0 * [elixir] better counter * [experiment] try montonic check strategy * [experiment] check strategy * [experiment] check strategy/metrics * refactor/rename * update tzdata - fix FunctionClauseError bitwalker/timex#542 * [fix] DNS SRV - update namespace * smarter handing of changed namespace * [fix] Confex resolve :libcluster
binarytemple · Jul 8, 2019 · 0ab4762 · 0ab4762
1 parent 2881161
commit 0ab4762
Show file tree

Hide file tree

Showing 22 changed files with 2,596 additions and 886 deletions.
diff --git a/Makefile b/Makefile
@@ -1,3 +1,5 @@
+attach-app-0:
+	kubectl -n elixir exec -t -i is-it-up-0 -- bin/is_it_up remote_console
 
 mix-run-0:
 	HTTP_PORT=4000 iex --cookie foo --name a@127.0.0.1 -S mix
@@ -6,17 +8,17 @@ mix-run-1:
 	HTTP_PORT=4001 iex --cookie foo --name b@127.0.0.1 -S mix
 
 build:
-	docker build  . -f ops/Dockerfile -t binarytemple/elixir_plug_poc:latest
+	docker build  . -f ops/Dockerfile -t binarytemple/is_it_up:latest
 
 docker-run: build
-	docker run -ti -p 4000:4000 -e'ERLANG_COOKIE=foo' --rm binarytemple/elixir_plug_poc console
+	docker run -ti -p 4000:4000 -e'ERLANG_COOKIE=foo' --rm binarytemple/is_it_up console
 
 push: build
-	docker push binarytemple/elixir_plug_poc:latest
+	docker push binarytemple/is_it_up:latest
 
 deploy-app:
 	kubectl create namespace elixir || true
-	kubectl apply --force --namespace elixir -f ops/app/elixir-plug-poc.yaml
+	kubectl apply --force --namespace elixir -f ops/app/is-it-up.yaml
 
 undeploy-app:
 	kubectl delete namespace elixir || true
@@ -41,4 +43,4 @@ port-forward-grafana:
 	$(MAKE) -C $(shell pwd)/ops/monitoring/ port-forward-grafana
 
 port-forward-elixir:
-	kubectl port-forward --namespace elixir elixir-plug-poc-0 14000:4000
+	kubectl port-forward --namespace elixir is-it-up-0 14000:4000
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@ It is now a testbed for deploying and monitoring clustered Elixir applications i
 ## Running under docker (downloading from docker hub)
 
 ```
-docker run -p 4000:4000 binarytemple/elixir_plug_poc:<version>
+docker run -p 4000:4000 binarytemple/is_it_up:<version>
 ```
 
 ## Running under docker (build it locally)
@@ -22,29 +22,29 @@ docker run whatever
 
 This is where it gets fancy, this will serve as a handly starting point for distributed Erlang under Kubernetes.
 
-The file `k8s/deploy/elixir-plug-poc.yaml` describes a kubernetes statefulset and service.
+The file `k8s/deploy/is-it-up.yaml` describes a kubernetes statefulset and service.
 
 When the file is applied to your kubernetes cluster, two pods will be created - with hostnames corresponding to the FQDN of the nodes.
 
 ```
-kubectl apply -f k8s/elixir-plug-poc.yaml
+kubectl apply -f k8s/is-it-up.yaml
 ```
 
 ```
-kubectl exec -t -i elixir-plug-poc-0 /bin/bash
+kubectl exec -t -i is-it-up-0 /bin/bash
 ```
 
 ```
 bash-4.4# hostname -f
-elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local
+is-it-up-0.is-it-up.default.svc.cluster.local
 ```
 
 You can discover the other node names using a SRV or wildcard query after removing the first dot from the hostname i.e.
 
 ```
-bash-4.4# dig *.elixir-plug-poc.default.svc.cluster.local
+bash-4.4# dig *.is-it-up.default.svc.cluster.local
 
-; <<>> DiG 9.12.4-P2 <<>> *.elixir-plug-poc.default.svc.cluster.local
+; <<>> DiG 9.12.4-P2 <<>> *.is-it-up.default.svc.cluster.local
 ;; global options: +cmd
 ;; Got answer:
 ;; WARNING: .local is reserved for Multicast DNS
@@ -53,13 +53,13 @@ bash-4.4# dig *.elixir-plug-poc.default.svc.cluster.local
 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 4, AUTHORITY: 0, ADDITIONAL: 0
 
 ;; QUESTION SECTION:
-;*.elixir-plug-poc.default.svc.cluster.local. IN        A
+;*.is-it-up.default.svc.cluster.local. IN        A
 
 ;; ANSWER SECTION:
-*.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.55
-*.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.57
-*.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.59
-*.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.60
+*.is-it-up.default.svc.cluster.local. 30 IN A 10.1.0.55
+*.is-it-up.default.svc.cluster.local. 30 IN A 10.1.0.57
+*.is-it-up.default.svc.cluster.local. 30 IN A 10.1.0.59
+*.is-it-up.default.svc.cluster.local. 30 IN A 10.1.0.60
 
 ;; Query time: 0 msec
 ;; SERVER: 10.96.0.10#53(10.96.0.10)
@@ -70,9 +70,9 @@ bash-4.4# dig *.elixir-plug-poc.default.svc.cluster.local
 Or SRV query :
 
 ```
-bash-4.4# dig SRV elixir-plug-poc.default.svc.cluster.local
+bash-4.4# dig SRV is-it-up.default.svc.cluster.local
 
-; <<>> DiG 9.12.4-P2 <<>> SRV elixir-plug-poc.default.svc.cluster.local
+; <<>> DiG 9.12.4-P2 <<>> SRV is-it-up.default.svc.cluster.local
 ;; global options: +cmd
 ;; Got answer:
 ;; WARNING: .local is reserved for Multicast DNS
@@ -81,19 +81,19 @@ bash-4.4# dig SRV elixir-plug-poc.default.svc.cluster.local
 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 4, AUTHORITY: 0, ADDITIONAL: 4
 
 ;; QUESTION SECTION:
-;elixir-plug-poc.default.svc.cluster.local. IN SRV
+;is-it-up.default.svc.cluster.local. IN SRV
 
 ;; ANSWER SECTION:
-elixir-plug-poc.default.svc.cluster.local. 30 IN SRV 10 25 0 elixir-plug-poc-3.elixir-plug-poc.default.svc.cluster.local.
-elixir-plug-poc.default.svc.cluster.local. 30 IN SRV 10 25 0 elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local.
-elixir-plug-poc.default.svc.cluster.local. 30 IN SRV 10 25 0 elixir-plug-poc-1.elixir-plug-poc.default.svc.cluster.local.
-elixir-plug-poc.default.svc.cluster.local. 30 IN SRV 10 25 0 elixir-plug-poc-2.elixir-plug-poc.default.svc.cluster.local.
+is-it-up.default.svc.cluster.local. 30 IN SRV 10 25 0 is-it-up-3.is-it-up.default.svc.cluster.local.
+is-it-up.default.svc.cluster.local. 30 IN SRV 10 25 0 is-it-up-0.is-it-up.default.svc.cluster.local.
+is-it-up.default.svc.cluster.local. 30 IN SRV 10 25 0 is-it-up-1.is-it-up.default.svc.cluster.local.
+is-it-up.default.svc.cluster.local. 30 IN SRV 10 25 0 is-it-up-2.is-it-up.default.svc.cluster.local.
 
 ;; ADDITIONAL SECTION:
-elixir-plug-poc-3.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.60
-elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.55
-elixir-plug-poc-1.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.57
-elixir-plug-poc-2.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.59
+is-it-up-3.is-it-up.default.svc.cluster.local. 30 IN A 10.1.0.60
+is-it-up-0.is-it-up.default.svc.cluster.local. 30 IN A 10.1.0.55
+is-it-up-1.is-it-up.default.svc.cluster.local. 30 IN A 10.1.0.57
+is-it-up-2.is-it-up.default.svc.cluster.local. 30 IN A 10.1.0.59
 
 ;; Query time: 0 msec
 ;; SERVER: 10.96.0.10#53(10.96.0.10)
@@ -106,54 +106,54 @@ elixir-plug-poc-2.elixir-plug-poc.default.svc.cluster.local. 30 IN A 10.1.0.59
 Lets attach to the running Elixir application inside the container :
 
 ```
-bin/elixir_plug_poc remote_console
+bin/is_it_up remote_console
 ```
 
 
 And verify `:inet_res.nslookup` is working correctly :
 
 ```
-iex(elixir_plug_poc@elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local)32> :inet_res.nslookup('elixir-plug-poc.default.svc.cluster.
+iex(is_it_up@is-it-up-0.is-it-up.default.svc.cluster.local)32> :inet_res.nslookup('is-it-up.default.svc.cluster.
 local', :any, :srv)
 {:ok,
  {:dns_rec, {:dns_header, 1, true, :query, true, false, true, true, false, 0},
-  [{:dns_query, 'elixir-plug-poc.default.svc.cluster.local', :srv, :any}],
+  [{:dns_query, 'is-it-up.default.svc.cluster.local', :srv, :any}],
   [
-    {:dns_rr, 'elixir-plug-poc.default.svc.cluster.local', :srv, :in, 0, 30,
-     {10, 25, 0, 'elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local'},
+    {:dns_rr, 'is-it-up.default.svc.cluster.local', :srv, :in, 0, 30,
+     {10, 25, 0, 'is-it-up-0.is-it-up.default.svc.cluster.local'},
      :undefined, [], false},
-    {:dns_rr, 'elixir-plug-poc.default.svc.cluster.local', :srv, :in, 0, 30,
-     {10, 25, 0, 'elixir-plug-poc-1.elixir-plug-poc.default.svc.cluster.local'},
+    {:dns_rr, 'is-it-up.default.svc.cluster.local', :srv, :in, 0, 30,
+     {10, 25, 0, 'is-it-up-1.is-it-up.default.svc.cluster.local'},
      :undefined, [], false},
-    {:dns_rr, 'elixir-plug-poc.default.svc.cluster.local', :srv, :in, 0, 30,
-     {10, 25, 0, 'elixir-plug-poc-2.elixir-plug-poc.default.svc.cluster.local'},
+    {:dns_rr, 'is-it-up.default.svc.cluster.local', :srv, :in, 0, 30,
+     {10, 25, 0, 'is-it-up-2.is-it-up.default.svc.cluster.local'},
      :undefined, [], false},
-    {:dns_rr, 'elixir-plug-poc.default.svc.cluster.local', :srv, :in, 0, 30,
-     {10, 25, 0, 'elixir-plug-poc-3.elixir-plug-poc.default.svc.cluster.local'},
+    {:dns_rr, 'is-it-up.default.svc.cluster.local', :srv, :in, 0, 30,
+     {10, 25, 0, 'is-it-up-3.is-it-up.default.svc.cluster.local'},
      :undefined, [], false}
   ], [],
   [
-    {:dns_rr, 'elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local', :a,
+    {:dns_rr, 'is-it-up-0.is-it-up.default.svc.cluster.local', :a,
      :in, 0, 30, {10, 1, 0, 55}, :undefined, [], false},
-    {:dns_rr, 'elixir-plug-poc-1.elixir-plug-poc.default.svc.cluster.local', :a,
+    {:dns_rr, 'is-it-up-1.is-it-up.default.svc.cluster.local', :a,
      :in, 0, 30, {10, 1, 0, 57}, :undefined, [], false},
-    {:dns_rr, 'elixir-plug-poc-2.elixir-plug-poc.default.svc.cluster.local', :a,
+    {:dns_rr, 'is-it-up-2.is-it-up.default.svc.cluster.local', :a,
      :in, 0, 30, {10, 1, 0, 59}, :undefined, [], false},
-    {:dns_rr, 'elixir-plug-poc-3.elixir-plug-poc.default.svc.cluster.local', :a,
+    {:dns_rr, 'is-it-up-3.is-it-up.default.svc.cluster.local', :a,
      :in, 0, 30, {10, 1, 0, 60}, :undefined, [], false}
   ]}}
 ```
 
 
 Verify distributed Erlang working correctly:
 
-    iex(elixir_plug_poc@elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local)11> Node.connect(:'elixir_plug_poc@elixir-plug-poc-1.elixir-plug-poc.default.svc.cluster.local')
+    iex(is_it_up@is-it-up-0.is-it-up.default.svc.cluster.local)11> Node.connect(:'is_it_up@is-it-up-1.is-it-up.default.svc.cluster.local')
     true
 
-    iex(elixir_plug_poc@elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local)20> Node.list
-    [:"elixir_plug_poc@elixir-plug-poc-1.elixir-plug-poc.default.svc.cluster.local"]
+    iex(is_it_up@is-it-up-0.is-it-up.default.svc.cluster.local)20> Node.list
+    [:"is_it_up@is-it-up-1.is-it-up.default.svc.cluster.local"]
 
-    iex(elixir_plug_poc@elixir-plug-poc-0.elixir-plug-poc.default.svc.cluster.local)25> Node.ping(:"elixir_plug_poc@elixir-plug-poc-1.elixir-plug-poc.default.svc.cluster.local")
+    iex(is_it_up@is-it-up-0.is-it-up.default.svc.cluster.local)25> Node.ping(:"is_it_up@is-it-up-1.is-it-up.default.svc.cluster.local")
     :pong
 
 

diff --git a/config/config.exs b/config/config.exs
@@ -20,9 +20,9 @@ config :logger, :console,
   format: "$time $metadata[$level] $message\n",
   metadata: [:user_id]
 
-config :elixir_plug_poc, :http_port, {:system, :integer, "HTTP_PORT", 4000}
-config :elixir_plug_poc, :check_init_delay, {:system, :integer, "CHECK_INIT_DELAY", 2}
-config :elixir_plug_poc, :check_interval, {:system, :integer, "CHECK_INTERVAL", 30}
-config :elixir_plug_poc, :check_host, {:system, :integer, "CHECK_HOST", "google.com"}
+config :is_it_up, :http_port, {:system, :integer, "HTTP_PORT", 4000}
+config :is_it_up, :check_init_delay, {:system, :integer, "CHECK_INIT_DELAY", 2}
+config :is_it_up, :check_interval, {:system, :integer, "CHECK_INTERVAL", 30}
+config :is_it_up, :check_host, {:system, :integer, "CHECK_HOST", "google.com"}
 
 import_config "#{Mix.env()}.exs"
diff --git a/config/prod.exs b/config/prod.exs
@@ -5,9 +5,9 @@ config :libcluster,
     k8s_example: [
       strategy: Cluster.Strategy.Kubernetes.DNSSRV,
       config: [
-        service: "elixir-plug-poc",
-        application_name: "elixir_plug_poc",
-        namespace: "default",
+        service: "is-it-up",
+        application_name: "is_it_up",
+        namespace: {:system, :string, "K8S_NAMESPACE", "elixir"},
         polling_interval: 10_000
       ]
     ]

diff --git a/config/test.exs b/config/test.exs
@@ -1,13 +1,11 @@
 use Mix.Config
 
-
 config :logger,
   level: :info,
   handle_otp_reports: false,
   handle_sasl_reports: false
 
-
-config :elixir_plug_poc, 
+config :is_it_up,
   http_port: 4001,
   check_init_delay: 2,
   check_interval: 30,

diff --git a/lib/elixir_plug_poc.ex b/lib/elixir_plug_poc.ex
diff --git a/lib/is_it_up/app.ex b/lib/is_it_up/app.ex
@@ -0,0 +1,29 @@
+defmodule IsItUp.App do
+  use Application
+
+  def start(_type, _args) do
+    Confex.resolve_env!(:logger)
+    Logger.configure(Application.get_all_env(:logger))
+    Confex.resolve_env!(:libcluster)
+    Confex.resolve_env!(:is_it_up)
+    IsItUp.Metrics.PlugExporter.setup()
+
+    http_port = Application.get_env(:is_it_up, :http_port)
+
+    topologies = Application.get_env(:libcluster, :topologies)
+
+    children = [
+      %{
+        id: IsItUp.Checker,
+        start: {IsItUp.Checker, :start_link, []},
+        restart: :permanent,
+        shutdown: 5000,
+        type: :worker
+      },
+      {Cluster.Supervisor, [topologies, [name: ClusterSupervisor]]},
+      Plug.Cowboy.child_spec(scheme: :http, plug:  IsItUp.Plug.Pipeline, options: [port: http_port])
+    ]
+
+    Supervisor.start_link(children, strategy: :one_for_one)
+  end
+end
diff --git a/lib/is_it_up/metrics/exporter.ex b/lib/is_it_up/metrics/exporter.ex
@@ -0,0 +1,6 @@
+defmodule IsItUp.Metrics.PlugExporter do
+  use Prometheus.PlugExporter
+  # def init(x) do
+  #   x
+  # end
+end