[#73] adding system metric collector

config/prod.exs

@@ -13,8 +13,8 @@ config :poa_agent,
          {:eth_information, POAAgent.Plugins.Collectors.Eth.Information, 60_000, :eth_information, [url: "http://localhost:8545", name: "Elixir-NodeJS-Integration", contact: "myemail@gmail.com"]},
          {:eth_latest_block, POAAgent.Plugins.Collectors.Eth.LatestBlock, 500, :latest_block, [url: "http://localhost:8545"]},
          {:eth_stats, POAAgent.Plugins.Collectors.Eth.Stats, 5000, :eth_stats, [url: "http://localhost:8545"]},
-         {:eth_pending, POAAgent.Plugins.Collectors.Eth.Pending, 500, :eth_pending, [url: "http://localhost:8545"]}
-
+         {:eth_pending, POAAgent.Plugins.Collectors.Eth.Pending, 500, :eth_pending, [url: "http://localhost:8545"]},
+         {:system_collector, POAAgent.Plugins.Collectors.System.Stats, 5_000, :system_metrics, []}
        ]
 
 # configuration for transfers. The format for each collector is {collector_process_id, module, args}
@@ -26,8 +26,8 @@ config :poa_agent,
              identifier: "elixirNodeJSIntegration",
 
              # Authentication parameters
-             user: "BK3eiZcT",
-             password: "MPr1n9B-ipvpYbj",
+             user: "rUN7afCO",
+             password: "_3IC09xfMtAW4Hr",
              token_url: "https://localhost:4003/session"
            ]
          }
@@ -40,5 +40,6 @@ config :poa_agent,
          {:eth_latest_block, [:rest_transfer]},
          {:eth_stats, [:rest_transfer]},
          {:eth_pending, [:rest_transfer]},
-         {:eth_information, [:rest_transfer]}
+         {:eth_information, [:rest_transfer]},
+         {:system_collector, []}
        ]

lib/poa_agent/entity/system/statistics.ex

@@ -0,0 +1,37 @@
+defmodule POAAgent.Entity.System.Statistics do
+  @moduledoc false
+
+  alias POAAgent.Format.POAProtocol.Data
+
+  @type t :: %__MODULE__{
+    cpu_load: number,
+    memory_usage: number,
+    disk_usage: number
+  }
+
+  defstruct [
+    cpu_load: 0,
+    memory_usage: 0,
+    disk_usage: 0
+  ]
+
+  def new(cpu_load, memory_usage, disk_usage) do
+    %__MODULE__{
+      cpu_load: cpu_load,
+      memory_usage: memory_usage,
+      disk_usage: disk_usage
+    }
+  end
+
+  defimpl POAAgent.Entity.NameConvention do
+    def from_elixir_to_node(x) do
+      Map.from_struct(x)
+    end
+  end
+
+  defimpl Data.Format do
+    def to_data(x) do
+      Data.new("statistics", x)
+    end
+  end
+end

lib/poa_agent/plugins/collectors/eth/stats.ex

@@ -74,7 +74,7 @@ defmodule POAAgent.Plugins.Collectors.Eth.Stats do
          {:ok, mining} <- Ethereumex.HttpClient.eth_mining(),
          {:ok, hashrate} <- Ethereumex.HttpClient.eth_hashrate(),
          {:ok, syncing} <- Ethereumex.HttpClient.eth_syncing(),
-         {:ok, gas_price} <- Ethereumex.HttpClient.eth_gas_price()
+         {:ok, gas_price} <- eth_gas_price()
     do
       peers = String.to_integer(POAAgent.Format.Literal.Hex.decimalize(peers))
       hashrate = String.to_integer(POAAgent.Format.Literal.Hex.decimalize(hashrate))
@@ -114,6 +114,15 @@ defmodule POAAgent.Plugins.Collectors.Eth.Stats do
     }
   end
 
+  defp eth_gas_price do
+    try do
+      Ethereumex.HttpClient.eth_gas_price()
+    catch
+      :exit, _ ->
+        {:ok, "0x0"}
+    end
+  end
+
   defp uptime(tries, down) do
     ((tries - down) / tries) * 100
   end

lib/poa_agent/plugins/collectors/system/stats.ex

@@ -0,0 +1,93 @@
+defmodule POAAgent.Plugins.Collectors.System.Stats do
+  use POAAgent.Plugins.Collector
+
+  @moduledoc """
+  This module retrieves system metrics. Those metrics are the cpu usage, memory and disk.
+  In order to use it we have to add it to the config file like this:
+
+      {:system_collector, POAAgent.Plugins.Collectors.System.Stats, 60_000, :system_metrics, []}
+
+  In this example we are checking the system metrics every minute
+
+  """
+
+  alias POAAgent.Entity.System.Statistics
+
+  @typep internal_state :: %{
+    last_metrics: Statistics.t
+  }
+
+  @doc false
+  @spec init_collector(term()) :: {:ok, internal_state()}
+  def init_collector(_) do
+    stats = gather_metrics()
+
+    {:transfer, stats, %{last_metrics: stats}}
+  end
+
+  @doc false
+  @spec collect(internal_state()) :: term()
+  def collect(%{last_metrics: last_metrics} = state) do
+    case gather_metrics() do
+      ^last_metrics ->
+        {:notransfer, state}
+      metrics ->
+        {:transfer, metrics, %{state | last_metrics: metrics}}
+    end
+  end
+
+  @doc false
+  @spec metric_type() :: String.t
+  def metric_type do
+    "system_metrics"
+  end
+
+  @doc false
+  @spec terminate(internal_state()) :: :ok
+  def terminate(_state) do
+    :ok
+  end
+
+  defp gather_metrics() do
+    Statistics.new(cpu_load(), memory_usage(), disk_usage())
+  end
+
+  defp cpu_load do
+    {total, amount} =
+      case :cpu_sup.util([:per_cpu]) do
+        cpu_info when is_list(cpu_info) ->
+          cpu_info
+          |> Enum.reduce({0, 0}, fn({_, load, _, _}, {total, acc}) ->
+                                   {total + 100.0, acc + load}
+                                 end)
+        _ ->
+          {100, 0}
+      end
+
+    percentage(total, amount)
+  end
+
+  defp memory_usage do
+    memory = :memsup.get_system_memory_data()
+
+    {total, amount} =
+      case Keyword.get(memory, :total_memory) do
+        nil ->
+          {100, 0}
+        total_memory ->
+          {total_memory, :erlang.memory(:total)}
+      end
+
+    percentage(total, amount)
+  end
+
+  defp disk_usage do
+    [{_, _, usage} | _] = :disksup.get_disk_data() |> Enum.sort(&(elem(&1, 2) >= elem(&2, 2)))
+    usage
+  end
+
+  defp percentage(total, amount) do
+    100 * amount / total
+  end
+
+end

lib/poa_agent/plugins/transfers/http/rest.ex

@@ -110,6 +110,7 @@ defmodule POAAgent.Plugins.Transfers.HTTP.REST do
       {:ok, %HTTPoison.Response{status_code: 401}} ->
         Logger.warn("Error 401, getting a new Token")
         jwt_token = new_token(state)
+        Logger.info("The new Token is #{inspect jwt_token}")
         result = HTTPoison.post(address, event, [@content_type_header, bearer_auth_header(jwt_token)])
         {result, %State{state | token: jwt_token}}
       {:ok, %HTTPoison.Response{status_code: error} = result} ->

mix.exs

@@ -19,7 +19,7 @@ defmodule POAAgent.MixProject do
 
   def application do
     [
-      extra_applications: [:logger],
+      extra_applications: [:logger, :os_mon],
       mod: {POAAgent.Application, []}
     ]
   end

test/poa_agent/entity/protocol_test.exs

@@ -5,9 +5,10 @@ defmodule POAAgent.Entity.ProtocolTest do
   alias POAAgent.Entity.Ethereum.Block
   alias POAAgent.Entity.Ethereum.History
   alias POAAgent.Entity.Ethereum.Pending
-  alias POAAgent.Entity.Ethereum.Statistics
+  alias POAAgent.Entity.Ethereum
   alias POAAgent.Entity.Host.Information
   alias POAAgent.Entity.Host.Latency
+  alias POAAgent.Entity.System
   alias POAAgent.Entity
 
   test "Data protocol test for block entity" do
@@ -39,7 +40,7 @@ defmodule POAAgent.Entity.ProtocolTest do
   end
 
   test "Data protocol test for statistics entity" do
-    entity = %Statistics{}
+    entity = %Ethereum.Statistics{}
     formated_entity = format_entity(entity)
 
     assert %Data{type: "statistics", body: formated_entity} == Data.Format.to_data(entity)
@@ -52,6 +53,13 @@ defmodule POAAgent.Entity.ProtocolTest do
     assert %Data{type: "latency", body: formated_entity} == Data.Format.to_data(entity)
   end
 
+  test "Data protocol test for System Stats entity" do
+    entity = %System.Statistics{}
+    formated_entity = format_entity(entity)
+
+    assert %Data{type: "statistics", body: formated_entity} == Data.Format.to_data(entity)
+  end
+
   defp format_entity(entity) do
     entity
     |> Entity.NameConvention.from_elixir_to_node()

test/poa_agent/plugins/collectors/system/stats_test.exs

@@ -0,0 +1,34 @@
+defmodule POAAgent.Plugins.Collectors.System.StatsTest do
+  use ExUnit.Case
+
+  alias POAAgent.Plugins.Collectors.System.Stats
+  alias POAAgent.Entity.System.Statistics
+
+  test "sending stats to the transfer when the collector starts and after a while" do
+    echo_transfer = :echo_transfer
+    {:ok, _echo} = EchoTransfer.start(echo_transfer)
+
+    args = %{
+      name: :system_metrics,
+      transfers: [echo_transfer],
+      frequency: 1000,
+      label: :system_metrics,
+      args: []
+    }
+
+    {:ok, _pid} = Stats.start_link(args)
+
+    assert_receive {:system_metrics, metric}, 20_000
+    assert %Statistics{cpu_load: cpu_load, memory_usage: memory_usage, disk_usage: disk_usage} = metric
+    assert is_number(cpu_load)
+    assert is_number(memory_usage)
+    assert is_number(disk_usage)
+
+    # we should receiver system metrics again
+    assert_receive {:system_metrics, metric}, 20_000
+    assert %Statistics{cpu_load: cpu_load, memory_usage: memory_usage, disk_usage: disk_usage} = metric
+    assert is_number(cpu_load)
+    assert is_number(memory_usage)
+    assert is_number(disk_usage)
+  end
+end