Add new server_status metric

This commits introduces new boolean metric - server_status with
server ip and server port labels.

The metric will represent an active or inactive state of a RADIUS
server where eradius tries to send RADIUS requests. By default all
possible endpoints are set to inactive. If eradius client sent a
request and received successfully response such primary/secondary
RADIUS server will be set to 'active' and that will be changed only
when next RADIUS request will be failed for this endpoint.

rebar.config

@@ -19,7 +19,9 @@
 	       deprecated_functions]}.
 
 {xref_ignores, [{prometheus_histogram, declare, 1},
-                {prometheus_histogram, observe, 3}]}.
+                {prometheus_histogram, observe, 3},
+                {prometheus_boolean,   declare, 1},
+                {prometheus_boolean,   set,     3}]}.
 
 
 %% == Plugins ==

src/eradius_client.erl

@@ -180,6 +180,7 @@ proceed_response(Request, {ok, Response, Secret, Authenticator}, _Peer = {_Serve
             ?LOG(error, "~s INF: Noreply for request ~p. Could not decode the request, reason: ~s", [printable_peer(ServerIP, Port), Request, Reason]),
             maybe_failover(Request, noreply, {ServerIP, Port}, Options);
         Decoded ->
+            update_server_status_metric(ServerIP, Port, true),
             update_client_response(Decoded#radius_request.cmd, MetricsInfo, Request),
             {ok, Response, Authenticator}
     end;
@@ -190,6 +191,7 @@ proceed_response(Request, Response, {_ServerName, {ServerIP, Port}}, TS1, Metric
     maybe_failover(Request, Response, {ServerIP, Port}, Options).
 
 maybe_failover(Request, Response, {ServerIP, Port}, Options) ->
+    update_server_status_metric(ServerIP, Port, false),
     case proplists:get_value(failover, Options, []) of
         [] ->
             Response;
@@ -476,6 +478,7 @@ store_upstream_servers(Server) ->
 
 %% private
 store_radius_server_from_pool(Addr, Port, Retries) when is_tuple(Addr) and is_integer(Port) and is_integer(Retries) ->
+    eradius_counter:set_boolean_metric(server_status, [Addr, Port], false),
     ets:insert(?MODULE, {{Addr, Port}, Retries, Retries});
 store_radius_server_from_pool(Addr, _, _) ->
     ?LOG(error, "bad IP address specified in RADIUS servers pool configuration ~p", [Addr]),
@@ -595,6 +598,9 @@ inc_responses_counter_accounting(MetricsInfo, #radius_request{attrs = Attrs}) ->
 inc_responses_counter_accounting(_, _) ->
     ok.
 
+update_server_status_metric(IP, Port, Value) ->
+    eradius_counter:set_boolean_metric(server_status, [IP, Port], Value).
+
 %% check if we can use persistent_term for config
 %% persistent term was added in OTP 21.2 but we can't
 %% check minor versions with macros so we're stuck waiting

src/eradius_counter.erl

@@ -3,7 +3,8 @@
 
 -module(eradius_counter).
 -export([init_counter/1, init_counter/2, inc_counter/2, dec_counter/2, reset_counter/1, reset_counter/2,
-         inc_request_counter/2, inc_reply_counter/2, observe/4, observe/5]).
+         inc_request_counter/2, inc_reply_counter/2, observe/4, observe/5,
+         set_boolean_metric/3]).
 
 -behaviour(gen_server).
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
@@ -82,6 +83,14 @@ aggregate({Servers, {ResetTS, Nass}}) ->
     NSum1 = [Value || {_Key, Value} <- orddict:to_list(NSums)],
     {Servers, {ResetTS, NSum1}}.
 
+%% @doc Set Value for the given prometheus boolean metric by the given Name with
+%% the given values
+set_boolean_metric(Name, Labels, Value) ->
+    case code:is_loaded(prometheus) of
+        {file, _} -> prometheus_boolean:set(Name, Labels, Value);
+        _ ->         ok
+    end.
+
 %% @doc Update the given histogram metric value
 %% NOTE: We use prometheus_histogram collector here instead of eradius_counter ets table because
 %% it is much easy to use histograms in this way. As we don't need to manage buckets and do
@@ -132,6 +141,13 @@ start_link() ->
 init([]) ->
     ets:new(?MODULE, [ordered_set, protected, named_table, {keypos, #nas_counter.key}, {write_concurrency,true}]),
     eradius:modules_ready([?MODULE]),
+    case code:is_loaded(prometheus) of
+        {file, _} ->
+            prometheus_boolean:declare([{name, server_status}, {labels, [server_ip, server_port]},
+                                        {help, "Status of an upstream RADIUS Server"}]);
+        _ ->
+            ok
+    end,
     {ok, #state{reset = eradius_lib:timestamp()}}.
 
 %% @private