feat: use telemetry in omg_status

apps/omg/config/config.exs

@@ -14,9 +14,4 @@ config :omg, :eip_712_domain,
   version: "1",
   salt: "0xfad5c7f626d80f9256ef01929f3beb96e058b8b4b0e3fe52d84f054c0e2a7a83"
 
-config :omg, OMG.Utils.Tracer,
-  service: :omg,
-  adapter: SpandexDatadog.Adapter,
-  env: {:system, "APP_ENV"}
-
 import_config "#{Mix.env()}.exs"

apps/omg/config/test.exs

@@ -6,5 +6,3 @@ config :omg,
   coordinator_eth_height_check_interval_ms: 100,
   client_monitor_interval_ms: 50,
   environment: :test
-
-config :omg, OMG.Utils.Tracer, env: "test"

apps/omg/lib/omg/application.ex

@@ -20,13 +20,9 @@ defmodule OMG.Application do
 
   use Application
   alias OMG.Alert.AlarmHandler
-  alias OMG.Utils.Metrics
 
   def start(_type, _args) do
-    :ok = DeferredConfig.populate(:statix)
-    :ok = DeferredConfig.populate(:omg)
     :ok = AlarmHandler.install()
-    :ok = Metrics.connect()
 
     OMG.Supervisor.start_link()
   end

apps/omg/lib/omg/ethereum_event_listener.ex

@@ -18,13 +18,11 @@ defmodule OMG.EthereumEventListener do
   """
 
   alias OMG.EthereumEventListener.Core
-  alias OMG.Recorder
   alias OMG.RootChainCoordinator
   alias OMG.RootChainCoordinator.SyncGuide
-  alias OMG.Utils.Metrics
 
   use GenServer
-  use Metrics
+  use OMG.Status.Metric.Measure
   use OMG.Utils.LoggerExt
 
   @type config() :: %{
@@ -81,21 +79,20 @@ defmodule OMG.EthereumEventListener do
 
     {:ok, _} = schedule_get_events()
     :ok = RootChainCoordinator.check_in(height_to_check_in, service_name)
-
-    name =
-      service_name
-      |> Atom.to_string()
-      |> Kernel.<>(".Recorder")
-      |> String.to_atom()
-
-    {:ok, _} = Recorder.start_link(%Recorder{name: name, parent: self()})
+    {:ok, _} = :timer.send_interval(Application.fetch_env!(:omg, :metrics_collection_interval), self(), :send_metrics)
 
     _ = Logger.info("Started #{inspect(__MODULE__)} for #{service_name}, synced_height: #{inspect(height_to_check_in)}")
 
     {:noreply, {initial_state, callbacks_map}}
   end
 
+  def handle_info(:send_metrics, {core, _callbacks} = state) do
+    :ok = :telemetry.execute([:process, __MODULE__], %{}, core)
+    {:noreply, state}
+  end
+
   def handle_info(:sync, state), do: do_sync(state)
+
   @decorate measure_start()
   defp do_sync({%Core{} = core, _callbacks} = state) do
     case RootChainCoordinator.get_sync_info() do
@@ -112,7 +109,7 @@ defmodule OMG.EthereumEventListener do
   end
 
   defp sync_height(
-         {%Core{service_name: service_name} = state, callbacks},
+         {%Core{} = state, callbacks},
          %SyncGuide{sync_height: sync_height} = sync_info
        ) do
     {:ok, events, db_updates, height_to_check_in, new_state} =
@@ -121,7 +118,6 @@ defmodule OMG.EthereumEventListener do
       |> Core.get_events(sync_height)
 
     {:ok, db_updates_from_callback} = callbacks.process_events_callback.(events)
-    Metrics.increment(service_name |> Atom.to_string(), length(events))
     :ok = OMG.DB.multi_update(db_updates ++ db_updates_from_callback)
     :ok = RootChainCoordinator.check_in(height_to_check_in, state.service_name)
 

apps/omg/lib/omg/ethereum_event_listener/core.ex

@@ -138,6 +138,7 @@ defmodule OMG.EthereumEventListener.Core do
 
     height_to_check_in = get_height_to_check_in(new_state)
     db_update = [{:put, update_key, height_to_check_in}]
+    :ok = :telemetry.execute([:process, __MODULE__], %{events: events}, state)
     {:ok, events, db_update, height_to_check_in, new_state}
   end
 end

apps/omg/lib/omg/ethereum_event_listener/measure.ex

@@ -0,0 +1,37 @@
+# Copyright 2019 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.EthereumEventListener.Measure do
+  @moduledoc """
+  Counting business metrics sent to DataDog
+  """
+
+  import OMG.Status.Metric.Event, only: [name: 2]
+  alias OMG.Status.Metric.Datadog
+  @supported_events [[:process, OMG.EthereumEventListener], [:process, OMG.EthereumEventListener.Core]]
+  def supported_events, do: @supported_events
+
+  def handle_event([:process, OMG.EthereumEventListener.Core], %{events: events}, state, _config) do
+    :ok = Datadog.gauge(name(state.service_name, :events), length(events))
+  end
+
+  def handle_event([:process, OMG.EthereumEventListener], %{}, state, _config) do
+    value =
+      self()
+      |> Process.info(:message_queue_len)
+      |> elem(1)
+
+    :ok = Datadog.gauge(name(state.service_name, :message_queue_len), value)
+  end
+end

apps/omg/lib/omg/root_chain_coordinator.ex

@@ -17,12 +17,11 @@ defmodule OMG.RootChainCoordinator do
   """
 
   alias OMG.EthereumHeight
-  alias OMG.Recorder
   alias OMG.RootChainCoordinator.Core
 
   use GenServer
   use OMG.Utils.LoggerExt
-  use OMG.Utils.Metrics
+  use OMG.Status.Metric.Measure
 
   defmodule SyncGuide do
     @moduledoc """
@@ -88,8 +87,26 @@ defmodule OMG.RootChainCoordinator do
     |> Map.keys()
     |> request_sync()
 
-    {:ok, _} = Recorder.start_link(%Recorder{name: __MODULE__.Recorder, parent: self()})
+    {:ok, _} = :timer.send_interval(Application.fetch_env!(:omg, :metrics_collection_interval), self(), :send_metrics)
 
+    _ = Logger.info("Started #{inspect(__MODULE__)}")
+
+    {:noreply, state}
+  end
+
+  def handle_info(:send_metrics, state) do
+    :ok = :telemetry.execute([:process, __MODULE__], %{}, state)
+    {:noreply, state}
+  end
+
+  def handle_info(:update_root_chain_height, state) do
+    {:ok, root_chain_height} = OMG.EthereumHeight.get()
+    {:ok, state} = Core.update_root_chain_height(state, root_chain_height)
+    {:noreply, state}
+  end
+
+  def handle_info({:DOWN, _ref, :process, pid, _}, state) do
+    {:ok, state} = Core.check_out(state, pid)
     {:noreply, state}
   end
 
@@ -108,17 +125,6 @@ defmodule OMG.RootChainCoordinator do
     {:reply, {:ok, Core.get_ethereum_heights(state)}, state}
   end
 
-  def handle_info(:update_root_chain_height, state) do
-    {:ok, root_chain_height} = OMG.EthereumHeight.get()
-    {:ok, state} = Core.update_root_chain_height(state, root_chain_height)
-    {:noreply, state}
-  end
-
-  def handle_info({:DOWN, _ref, :process, pid, _}, state) do
-    {:ok, state} = Core.check_out(state, pid)
-    {:noreply, state}
-  end
-
   defp schedule_get_ethereum_height(interval) do
     :timer.send_interval(interval, self(), :update_root_chain_height)
   end

apps/omg/lib/omg/root_chain_coordinator/measure.ex

@@ -0,0 +1,31 @@
+# Copyright 2019 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule OMG.RootChainCoordinator.Measure do
+  @moduledoc """
+  Counting business metrics sent to DataDog
+  """
+
+  import OMG.Status.Metric.Event, only: [name: 2]
+  alias OMG.Status.Metric.Datadog
+
+  def handle_event([:process, OMG.RootChainCoordinator], _, state, _config) do
+    value =
+      self()
+      |> Process.info(:message_queue_len)
+      |> elem(1)
+
+    :ok = Datadog.gauge(name(state.service_name, :message_queue_len), value)
+  end
+end

apps/omg/lib/omg/state.ex

@@ -21,14 +21,13 @@ defmodule OMG.State do
   alias OMG.DB
   alias OMG.Eth
   alias OMG.Fees
-  alias OMG.Recorder
   alias OMG.State.Core
   alias OMG.State.Transaction
   alias OMG.State.Transaction.Validator
   alias OMG.Utxo
 
   use GenServer
-  use OMG.Utils.Metrics
+  use OMG.Status.Metric.Measure
   use OMG.Utils.LoggerExt
 
   @type exec_error :: Validator.exec_error()
@@ -94,7 +93,6 @@ defmodule OMG.State do
     {:ok, utxos_query_result} = DB.utxos()
     {:ok, height_query_result} = DB.get_single_value(:child_top_block_number)
     {:ok, last_deposit_query_result} = DB.get_single_value(:last_deposit_child_blknum)
-    {:ok, _} = :timer.send_interval(Application.fetch_env!(:omg, :metrics_collection_interval), self(), :send_metrics)
     {:ok, [utxos_query_result, height_query_result, last_deposit_query_result], {:continue, :setup}}
   end
 
@@ -109,7 +107,15 @@ defmodule OMG.State do
                last_deposit_query_result,
                child_block_interval
              ) do
-        _ = Logger.info("Started State, height: #{height_query_result}, deposit height: #{last_deposit_query_result}")
+        _ =
+          Logger.info(
+            "Started #{inspect(__MODULE__)}, height: #{height_query_result}, deposit height: #{
+              last_deposit_query_result
+            }"
+          )
+
+        {:ok, _} =
+          :timer.send_interval(Application.fetch_env!(:omg, :metrics_collection_interval), self(), :send_metrics)
 
         result
       else
@@ -121,16 +127,11 @@ defmodule OMG.State do
           other
       end
 
-    {:ok, _} = Recorder.start_link(%Recorder{name: __MODULE__.Recorder, parent: self()})
-
     {:noreply, state}
   end
 
   def handle_info(:send_metrics, state) do
-    _ =
-      Core.Metrics.calculate(state)
-      |> Enum.map(fn {key, value} -> OMG.Utils.Metrics.gauge(key, value) end)
-
+    :ok = :telemetry.execute([:process, __MODULE__], %{}, state)
     {:noreply, state}
   end