From a7232d75cd6f5357782beb92238b189d0d4e429b Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 00:08:45 +0100 Subject: [PATCH 01/48] Adding initial keys for hydrating caches --- config/dev.exs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/config/dev.exs b/config/dev.exs index f50c59bf7..b97bad4d0 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -77,6 +77,26 @@ config :archethic, Archethic.Crypto.NodeKeystore.Origin.TPMImpl end) +config :archethic, Archethic.Utils.HydratingCache.CachesManager, + uco_service: [ + {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, [["usd", "eur"]], + 3000}, + {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], 3000}, + {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, [["usd", "eur"]], + 3000} + ], + coco_service: [ + {:coin_paprika, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, + [["usd", "eur"]], 3000}, + {:coin_gecko, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, + [["usd", "eur"]], 3000}, + {:coin_market_cap, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, + [["usd", "eur"]], 3000} + ] + config :archethic, Archethic.Governance.Pools, initial_members: [ technical_council: [ From d0f69c5f11fc3588e09017cf0eeaec10b2e82c87 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 00:09:09 +0100 Subject: [PATCH 02/48] Adding hydrating cache code --- .../utils/hydrating_cache/cache_entry.ex | 204 ++++++++++++++++++ .../utils/hydrating_cache/caches_manager.ex | 89 ++++++++ .../utils/hydrating_cache/hydrating_cache.ex | 197 +++++++++++++++++ 3 files changed, 490 insertions(+) create mode 100644 lib/archethic/utils/hydrating_cache/cache_entry.ex create mode 100644 lib/archethic/utils/hydrating_cache/caches_manager.ex create mode 100644 lib/archethic/utils/hydrating_cache/hydrating_cache.ex diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex new file mode 100644 index 000000000..a9e537339 --- /dev/null +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -0,0 +1,204 @@ +defmodule Archethic.Utils.HydratingCache.CacheEntry.StateData do + @moduledoc """ + Struct describing the state of a cache entry FSM + """ + + defstruct running_func_task: :undefined, + hydrating_func: :undefined, + getters: [], + ttl: :undefined, + refresh_interval: :undefined, + key: :undefined, + value: :"$$undefined", + timer_func: :undefined, + timer_discard: :undefined +end + +defmodule Archethic.Utils.HydratingCache.CacheEntry do + @moduledoc """ + This module is a finite state machine implementing a cache entry. + There is one such Cache Entry FSM running per registered key. + + It is responsible for : + - receiving request to get the value for the key it is associated with + - Run the hydrating function associated with this key + - Manage various timers + """ + alias Archethic.Utils.HydratingCache.CacheEntry + @behaviour :gen_statem + + use Task + require Logger + + def start_link([fun, key, ttl, refresh_interval]) do + :gen_statem.start_link(__MODULE__, [fun, key, ttl, refresh_interval], []) + end + + @impl :gen_statem + def init([fun, key, ttl, refresh_interval]) do + ## Hydrate the value + {:ok, :running, + %CacheEntry.StateData{ + hydrating_func: fun, + key: key, + ttl: ttl, + refresh_interval: refresh_interval + }} + end + + @impl :gen_statem + def callback_mode, do: [:handle_event_function, :state_enter] + + @impl :gen_statem + def handle_event({:call, from}, :get, :idle, data) do + ## Value is requested while fsm is iddle, return the value + IO.puts("Sending value to #{inspect({from, data.value})}") + {:next_state, :idle, data, [{:reply, from, data.value}]} + end + + def handle_event(:cast, {:get, from}, :idle, data) do + ## Value is requested while fsm is iddle, return the value + send(from, {:ok, data.value}) + {:next_state, :idle, data} + end + + ## call for value while hydrating function is running + ## We return previous value not to block the caller + def handle_event({:call, from}, :get, :running, data) do + {:next_state, :running, data, [{:reply, from, data.value}]} + end + + def handle_event(:cast, {:get, from}, :running, data) when data.value == :"$$undefined" do + ## Getting value when a function is running and no previous value is available + ## Register this getter to send value later on + previous_getters = data.getters + {:next_state, :running, %CacheEntry.StateData{data | getters: previous_getters ++ [from]}} + end + + def handle_event(:cast, {:get, from}, :running, data) do + ## Getting value while function is running but previous value is available + ## Return vurrent value + send(from, {:ok, data.value}) + {:next_state, :running, data} + end + + def handle_event({:call, from}, {:register, fun, key, ttl, refresh_interval}, :running, data) do + ## Registering a new hydrating function while previous one is running + ## We stop the task + Task.shutdown(data.running_func_task, :brutal_kill) + + ## And the timers triggering it and discarding value + _ = :timer.cancel(data.timer_func) + _ = :timer.cancel(data.timer_discard) + + ## Start new timer to hydrate at refresh interval + timer = :timer.send_interval(refresh_interval, self(), :hydrate) + + ## We trigger the update ( to trigger or not could be set at registering option ) + {:repeat_state, + %CacheEntry.StateData{ + data + | hydrating_func: fun, + key: key, + ttl: ttl, + refresh_interval: refresh_interval, + timer_func: timer + }, [{:reply, from, :ok}]} + end + + def handle_event({:call, from}, {:register, fun, key, ttl, refresh_interval}, _state, data) do + IO.puts("Registering new hydrating function #{inspect(fun)} for key #{inspect(key)}") + ## Setting hydrating function in other cases + ## Hydrating function not running, we just stop the timers + _ = :timer.cancel(data.timer_func) + _ = :timer.cancel(data.timer_discard) + + ## Fill state with new hydrating function parameters + data = + Map.merge(data, %{ + :hydrating_func => fun, + :ttl => ttl, + :refresh_interval => refresh_interval + }) + + timer = :timer.send_interval(refresh_interval, self(), :hydrate) + + ## We trigger the update ( to trigger or not could be set at registering option ) + {:next_state, :running, + %CacheEntry.StateData{ + data + | hydrating_func: fun, + key: key, + ttl: ttl, + refresh_interval: refresh_interval, + timer_func: timer + }, [{:reply, from, :ok}]} + end + + def handle_event(:info, :hydrate, :idle, data) do + ## Time to rehydrate + ## Hydrating the key, go to running state + {:next_state, :running, data} + end + + def handle_event(:enter, _event, :running, data) do + ## At entering running state, we start the hydrating task + me = self() + + hydrating_task = + spawn(fn -> + value = data.hydrating_func.() + :gen_statem.cast(me, {:new_value, data.key, value}) + end) + + ## we stay in running state + {:next_state, :running, %CacheEntry.StateData{data | running_func_task: hydrating_task}} + end + + def handle_event(:info, :discarded, state, data) do + ## Value is discarded + + Logger.warning( + "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} discarded" + ) + + {:next_state, state, %CacheEntry.StateData{data | value: :discarded}} + end + + def handle_event(:cast, {:new_value, key, {:ok, value}}, :running, data) do + ## We got result from hydrating function + Logger.debug( + "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} got value #{inspect({key, value})}" + ) + + ## notify waiiting getters + Enum.each(data.getters, fn getter -> send(getter, {:ok, value}) end) + ## We could do error control here, like unregistering the running func. + ## atm, we will keep it + _ = :timer.cancel(data.timer_discard) + me = self() + {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) + + {:next_state, :idle, + %CacheEntry.StateData{ + data + | running_func_task: :undefined, + value: value, + getters: [], + timer_discard: new_timer + }} + end + + def handle_event(:cast, {:new_value, key, {:error, reason}}, :running, data) do + ## Got error new value for key + Logger.warning( + "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} got error value #{inspect({key, {:error, reason}})}" + ) + + {:next_state, :idle, %CacheEntry.StateData{data | running_func_task: :undefined}} + end + + def handle_event(_type, _event, _state, data) do + {:keep_state, data} + end +end diff --git a/lib/archethic/utils/hydrating_cache/caches_manager.ex b/lib/archethic/utils/hydrating_cache/caches_manager.ex new file mode 100644 index 000000000..9d1133330 --- /dev/null +++ b/lib/archethic/utils/hydrating_cache/caches_manager.ex @@ -0,0 +1,89 @@ +defmodule Archethic.Utils.HydratingCache.CachesManager do + @moduledoc """ + This module is used to manage (create and delete) hydrating caches. + At start it will read the configuration and start a cache per service. + """ + use GenServer + require Logger + alias Archethic.Utils.HydratingCache + + def start_link(args \\ [], opts \\ [name: __MODULE__]) do + GenServer.start_link(__MODULE__, args, opts) + end + + @doc """ + Start a new hydrating cache process to hold the values from a service. + This is a synchronous call, it will block until the cache is ready ( all keys are hydrated ) + """ + @spec new_service_sync( + name :: String.t(), + initial_keys :: list() + ) :: {:error, any} | {:ok, pid} + def new_service_sync(name, initial_keys) do + GenServer.call(__MODULE__, {:new_service_sync, name, initial_keys}) + end + + @doc """ + Start a new hydrating cache process to hold the values from a service. + This is an asynchronous call, it will return immediately. + """ + def new_service_async(name, keys) do + GenServer.cast(__MODULE__, {:new_service_async, name, keys, self()}) + end + + @doc """ + Sync call to end a service cache. + """ + def end_service_sync(name) do + GenServer.call(__MODULE__, {:end_service, name}) + end + + @impl true + def init(_args) do + manager_conf = Application.get_env(:archethic, __MODULE__) + + {:ok, caches_sup} = + DynamicSupervisor.start_link( + name: Archethic.Utils.HydratingCache.Manager.CachesSupervisor, + strategy: :one_for_one + ) + + Logger.info( + "Starting hydrating cache manager #{inspect(__MODULE__)} with conf #{inspect(manager_conf)}" + ) + + Enum.each(manager_conf, fn {service, keys} -> + Logger.info("Starting new service #{service}") + new_service_async(service, keys) + end) + + {:ok, %{:caches_sup => caches_sup}} + end + + @impl true + def handle_call({:new_service_sync, name, initial_keys}, _from, state) do + {:ok, pid} = + DynamicSupervisor.start_child( + state.caches_sup, + HydratingCache.child_spec([{name, initial_keys}, []]) + ) + + {:reply, {:ok, pid}, state} + end + + @impl true + def handle_cast({:new_service_async, name, keys, _requester}, state) do + IO.inspect("Starting new service genserver #{name}") + + DynamicSupervisor.start_child(state.caches_sup, %{ + id: name, + start: {HydratingCache, :start_link, [{name, keys}]} + }) + + {:noreply, state} + end + + def handle_cast(_, state) do + {:noreply, state} + end +end diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex new file mode 100644 index 000000000..bc7b9e660 --- /dev/null +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -0,0 +1,197 @@ +defmodule Archethic.Utils.HydratingCache do + @moduledoc """ + GenServer implementing the hydrating cache itself. + It receives queries from clients requesting the cache, and manage the cache entries FSMs + """ + alias Archethic.Utils.HydratingCache.CacheEntry + + use GenServer + + require Logger + + @type result :: + {:ok, any()} + | {:error, :timeout} + | {:error, :not_registered} + + def start_link({name, initial_keys}) do + GenServer.start_link(__MODULE__, [name, initial_keys], name: :"#{__MODULE__}.#{name}") + end + + @doc ~s""" + Registers a function that will be computed periodically to update the cache. + + Arguments: + - `fun`: a 0-arity function that computes the value and returns either + `{:ok, value}` or `{:error, reason}`. + - `key`: associated with the function and is used to retrieve the stored + value. + - `ttl` ("time to live"): how long (in milliseconds) the value is stored + before it is discarded if the value is not refreshed. + - `refresh_interval`: how often (in milliseconds) the function is + recomputed and the new value stored. `refresh_interval` must be strictly + smaller than `ttl`. After the value is refreshed, the `ttl` counter is + restarted. + + The value is stored only if `{:ok, value}` is returned by `fun`. If `{:error, + reason}` is returned, the value is not stored and `fun` must be retried on + the next run. + """ + @spec register_function( + hydrating_cache :: pid(), + fun :: (() -> {:ok, any()} | {:error, any()}), + key :: any, + ttl :: non_neg_integer(), + refresh_interval :: non_neg_integer() + ) :: :ok + def register_function(hydrating_cache, fun, key, ttl, refresh_interval) + when is_function(fun, 0) and is_integer(ttl) and ttl > 0 and + is_integer(refresh_interval) and + refresh_interval < ttl do + GenServer.call(hydrating_cache, {:register, fun, key, ttl, refresh_interval}) + end + + @doc ~s""" + Get the value associated with `key`. + + Details: + - If the value for `key` is stored in the cache, the value is returned + immediately. + - If a recomputation of the function is in progress, the last stored value + is returned. + - If the value for `key` is not stored in the cache but a computation of + the function associated with this `key` is in progress, wait up to + `timeout` milliseconds. If the value is computed within this interval, + the value is returned. If the computation does not finish in this + interval, `{:error, :timeout}` is returned. + - If `key` is not associated with any function, return `{:error, + :not_registered}` + """ + @spec get(pid(), any(), non_neg_integer(), Keyword.t()) :: result + def get(cache, key, timeout \\ 30_000, _opts \\ []) + when is_integer(timeout) and timeout > 0 do + Logger.debug("Getting key #{inspect(key)} from hydrating cache #{inspect(cache)}") + + GenServer.call(cache, {:get, key}, timeout) + end + + @impl GenServer + def init([name, keys]) do + Logger.info("Starting Hydrating cache for service #{inspect(name)}") + + ## start a dynamic supervisor for the cache entries/keys + {:ok, keys_sup} = + DynamicSupervisor.start_link( + name: :"Archethic.Utils.HydratingCache.CacheEntry.KeysSupervisor.#{name}", + strategy: :one_for_one + ) + + me = self() + + ## start a supervisor to manage the initial keys insertion workers + {:ok, initial_keys_worker_sup} = Task.Supervisor.start_link() + + ## Registering initial keys + _ = + Task.Supervisor.async_stream_nolink( + initial_keys_worker_sup, + keys, + fn + {provider, mod, func, params, refresh_rate} -> + Logger.debug( + "Registering hydrating function. Provider: #{inspect(provider)} Hydrating function: + #{inspect(mod)}.#{inspect(func)}(#{inspect(params)}) Refresh rate: #{inspect(refresh_rate)}" + ) + + GenServer.call( + me, + {:register, fn -> apply(mod, func, params) end, provider, 10000, refresh_rate} + ) + + other -> + Logger.error("Hydrating cache: Invalid configuration entry: #{inspect(other)}") + end, + on_timeout: :kill_task + ) + |> Stream.filter(&match?({:ok, {:ok, _}}, &1)) + |> Enum.to_list() + + ## stop the initial keys worker supervisor + Supervisor.stop(initial_keys_worker_sup) + {:ok, %{:keys => keys, keys_sup: keys_sup}} + end + + @impl true + + def handle_call({:get, key}, _from, state) do + case Map.get(state, key, :undefined) do + :undefined -> + {:reply, {:error, :not_registered}, state} + + pid -> + value = :gen_statem.call(pid, :get) + IO.puts("value #{inspect(value)}") + + {:reply, value, state} + end + end + + def handle_call({:register, fun, key, ttl, refresh_interval}, _from, state) do + ## Called when asked to register a function + case Map.get(state, key) do + nil -> + ## New key, we start a cache entry fsm + {:ok, pid} = + DynamicSupervisor.start_child( + state.keys_sup, + {CacheEntry, [fun, key, ttl, refresh_interval]} + ) + + {:reply, :ok, Map.put(state, key, pid)} + + pid -> + ## Key already exists, no need to start fsm + case :gen_statem.call(pid, {:register, fun, key, ttl, refresh_interval}) do + :ok -> + {:reply, :ok, Map.put(state, key, pid)} + + error -> + {:reply, {:error, error}, Map.put(state, key, pid)} + end + end + end + + def handle_call(unmanaged, _from, state) do + Logger.warning("Cache received unmanaged call: #{inspect(unmanaged)}") + {:reply, :ok, state} + end + + @impl true + def handle_cast({:get, from, key}, state) do + case Map.get(state, key, :undefined) do + :undefined -> + send(from, {:error, :not_registered}) + {:noreply, state} + + pid -> + :gen_statem.cast(pid, {:get, from}) + {:noreply, state} + end + end + + def handle_cast({:register, fun, key, ttl, refresh_interval}, state) do + handle_call({:register, fun, key, ttl, refresh_interval}, nil, state) + {:noreply, state} + end + + def handle_cast(unmanaged, state) do + Logger.warning("Cache received unmanaged cast: #{inspect(unmanaged)}") + {:noreply, state} + end + + @impl true + def handle_info(unmanaged, state) do + Logger.warning("Cache received unmanaged info: #{inspect(unmanaged)}") + {:noreply, state} + end +end From d0042cabddeb92d29e652948ec2aa4358569f3b1 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 00:09:23 +0100 Subject: [PATCH 03/48] Adding cache to supervisor --- lib/archethic/oracle_chain/supervisor.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/archethic/oracle_chain/supervisor.ex b/lib/archethic/oracle_chain/supervisor.ex index 40ff8694b..43d2a9e50 100644 --- a/lib/archethic/oracle_chain/supervisor.ex +++ b/lib/archethic/oracle_chain/supervisor.ex @@ -8,6 +8,7 @@ defmodule Archethic.OracleChain.Supervisor do alias Archethic.OracleChain.Scheduler alias Archethic.Utils + alias Archethic.Utils.HydratingCache.CachesManager def start_link(args \\ []) do Supervisor.start_link(__MODULE__, args) @@ -19,7 +20,8 @@ defmodule Archethic.OracleChain.Supervisor do children = [ MemTable, MemTableLoader, - {Scheduler, scheduler_conf} + {Scheduler, scheduler_conf}, + CachesManager ] Supervisor.init(Utils.configurable_children(children), strategy: :one_for_one) From c9f70d1334e7aae667a5ef2262d6266bce65f501 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 00:09:51 +0100 Subject: [PATCH 04/48] Using cache in agregator --- .../oracle_chain/services/uco_price.ex | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index add86f999..e6c9fbcd5 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -8,6 +8,8 @@ defmodule Archethic.OracleChain.Services.UCOPrice do alias Archethic.OracleChain.Services.Impl alias Archethic.Utils + alias Archethic.Utils.HydratingCache + @behaviour Impl @precision_digits 5 @@ -21,32 +23,33 @@ defmodule Archethic.OracleChain.Services.UCOPrice do {:ok, fetching_tasks_supervisor} = Task.Supervisor.start_link() ## retrieve prices from configured providers and filter results marked as errors prices = - Task.Supervisor.async_stream_nolink( - fetching_tasks_supervisor, - providers(), - fn provider -> - case provider.fetch(@pairs) do - {:ok, _prices} = result -> - result - - {:error, reason} -> - provider_name = provider |> to_string() |> String.split(".") |> List.last() - - Logger.warning( - "Service UCOPrice cannot fetch values from " <> - "provider: #{inspect(provider_name)} with reason : #{inspect(reason)}." - ) - - {:error, provider} - end - end, - on_timeout: :kill_task - ) - |> Stream.filter(&match?({:ok, {:ok, _}}, &1)) - |> Stream.map(fn - {_, {_, result = %{}}} -> + Enum.map(providers(), fn provider -> + case HydratingCache.get(:"Elixir.Archethic.Utils.HydratingCache.uco_service", provider) do + {:error, reason} -> + Logger.warning( + "Service UCOPrice cannot fetch values from provider: #{inspect(provider)} with reason : #{inspect(reason)}." + ) + + [] + + result -> + {provider, result} + end + end) + |> List.flatten() + |> Enum.filter(fn + {_, %{}} -> + true + + other -> + Logger.error("Service UCOPrice cannot fetch values from provider: #{inspect(other)}.") + false + end) + |> Enum.map(fn + {_, result = %{}} -> result end) + ## Here stream looks like : [%{"eur"=>[0.44], "usd"=[0.32]}, ..., %{"eur"=>[0.42, 0.43], "usd"=[0.35]}] |> Enum.reduce(%{}, &agregate_providers_data/2) |> Enum.reduce(%{}, fn {currency, values}, acc -> From fd2083970bf8ab18842d9286ea6c837df7338b54 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 00:19:48 +0100 Subject: [PATCH 05/48] Increasing deault ttl --- lib/archethic/utils/hydrating_cache/hydrating_cache.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index bc7b9e660..d693d736b 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -105,7 +105,7 @@ defmodule Archethic.Utils.HydratingCache do GenServer.call( me, - {:register, fn -> apply(mod, func, params) end, provider, 10000, refresh_rate} + {:register, fn -> apply(mod, func, params) end, provider, 75_000, refresh_rate} ) other -> From 121acc1b66cd0a5f323a460890ccbbad41d6d074 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 13:11:00 +0100 Subject: [PATCH 06/48] Fixing querying when no initial value plus bugfixes --- .../oracle_chain/services/uco_price.ex | 2 +- .../utils/hydrating_cache/cache_entry.ex | 35 ++++++++++++------- .../utils/hydrating_cache/hydrating_cache.ex | 28 +++++++++++++-- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index e6c9fbcd5..fccd0da48 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -32,7 +32,7 @@ defmodule Archethic.OracleChain.Services.UCOPrice do [] - result -> + {:ok, result} -> {provider, result} end end) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index a9e537339..359f39f28 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -50,9 +50,8 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do def callback_mode, do: [:handle_event_function, :state_enter] @impl :gen_statem - def handle_event({:call, from}, :get, :idle, data) do + def handle_event({:call, from}, {:get, _from}, :idle, data) do ## Value is requested while fsm is iddle, return the value - IO.puts("Sending value to #{inspect({from, data.value})}") {:next_state, :idle, data, [{:reply, from, data.value}]} end @@ -62,15 +61,25 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:next_state, :idle, data} end - ## call for value while hydrating function is running - ## We return previous value not to block the caller - def handle_event({:call, from}, :get, :running, data) do + ## Call for value while hydrating function is running and we have no previous value + ## We register the caller to send value later on, and we indicate caller to block + def handle_event({:call, _from}, {:get, from}, :running, data) + when data.value == :"$$undefined" do + previous_getters = data.getters + + {:keep_state, %CacheEntry.StateData{data | getters: previous_getters ++ [from]}, + [{:reply, from, {:ok, :answer_delayed}}]} + end + + ## Call for value while hydrating function is running and we have a previous value + ## We return the value to caller + def handle_event({:call, from}, {:get, _from}, :running, data) do {:next_state, :running, data, [{:reply, from, data.value}]} end + ## Getting value when a function is running and no previous value is available + ## Register this getter to send value later on def handle_event(:cast, {:get, from}, :running, data) when data.value == :"$$undefined" do - ## Getting value when a function is running and no previous value is available - ## Register this getter to send value later on previous_getters = data.getters {:next_state, :running, %CacheEntry.StateData{data | getters: previous_getters ++ [from]}} end @@ -172,20 +181,20 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ) ## notify waiiting getters - Enum.each(data.getters, fn getter -> send(getter, {:ok, value}) end) + Enum.each(data.getters, fn {pid, _ref} -> send(pid, {:ok, value}) end) ## We could do error control here, like unregistering the running func. ## atm, we will keep it _ = :timer.cancel(data.timer_discard) - me = self() - {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) + # me = self() + # {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) {:next_state, :idle, %CacheEntry.StateData{ data | running_func_task: :undefined, - value: value, - getters: [], - timer_discard: new_timer + value: {:ok, value}, + getters: [] + # timer_discard: new_timer }} end diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index d693d736b..4f5285d17 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -72,7 +72,28 @@ defmodule Archethic.Utils.HydratingCache do when is_integer(timeout) and timeout > 0 do Logger.debug("Getting key #{inspect(key)} from hydrating cache #{inspect(cache)}") - GenServer.call(cache, {:get, key}, timeout) + case GenServer.call(cache, {:get, key}, timeout) do + {:ok, :answer_delayed} -> + Logger.debug( + "waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(cache)}" + ) + + receive do + {:ok, value} -> + {:ok, value} + # code + after + timeout -> + {:error, :timeout} + end + + {:ok, value} -> + Logger.debug( + "Got value #{inspect(value)} for key #{inspect(key)} from hydrating cache #{inspect(cache)}" + ) + + {:ok, value} + end end @impl GenServer @@ -123,13 +144,14 @@ defmodule Archethic.Utils.HydratingCache do @impl true - def handle_call({:get, key}, _from, state) do + def handle_call({:get, key}, from, state) do case Map.get(state, key, :undefined) do :undefined -> {:reply, {:error, :not_registered}, state} pid -> - value = :gen_statem.call(pid, :get) + value = :gen_statem.call(pid, {:get, from}) + IO.puts("value #{inspect(value)}") {:reply, value, state} From dfda3ace11e6bdca33f7b7ad9378c6a694e5090a Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 13:18:14 +0100 Subject: [PATCH 07/48] Fixing specs --- lib/archethic/utils/hydrating_cache/hydrating_cache.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index 4f5285d17..3f11b1220 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -67,8 +67,8 @@ defmodule Archethic.Utils.HydratingCache do - If `key` is not associated with any function, return `{:error, :not_registered}` """ - @spec get(pid(), any(), non_neg_integer(), Keyword.t()) :: result - def get(cache, key, timeout \\ 30_000, _opts \\ []) + @spec get(atom(), any(), non_neg_integer(), Keyword.t()) :: result + def get(cache, key, timeout \\ 3_000, _opts \\ []) when is_integer(timeout) and timeout > 0 do Logger.debug("Getting key #{inspect(key)} from hydrating cache #{inspect(cache)}") From 1e7c4a92aec16b26f2d9082f0eb73070b56d0b97 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 17:44:11 +0100 Subject: [PATCH 08/48] Starting cache as first child --- lib/archethic/oracle_chain/supervisor.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/archethic/oracle_chain/supervisor.ex b/lib/archethic/oracle_chain/supervisor.ex index 43d2a9e50..2090ea85e 100644 --- a/lib/archethic/oracle_chain/supervisor.ex +++ b/lib/archethic/oracle_chain/supervisor.ex @@ -18,10 +18,10 @@ defmodule Archethic.OracleChain.Supervisor do scheduler_conf = Application.get_env(:archethic, Scheduler) children = [ + CachesManager, MemTable, MemTableLoader, - {Scheduler, scheduler_conf}, - CachesManager + {Scheduler, scheduler_conf} ] Supervisor.init(Utils.configurable_children(children), strategy: :one_for_one) From 05ec2ce428d7b39781bd4e603615cf4ef48b133b Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 17:44:49 +0100 Subject: [PATCH 09/48] Bug fixes and refinement --- .../utils/hydrating_cache/cache_entry.ex | 30 ++++++++++++++----- .../utils/hydrating_cache/caches_manager.ex | 6 ++-- .../utils/hydrating_cache/hydrating_cache.ex | 15 ++++++---- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 359f39f28..51403c775 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -36,9 +36,12 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do @impl :gen_statem def init([fun, key, ttl, refresh_interval]) do + timer = :timer.send_interval(refresh_interval, self(), :hydrate) + ## Hydrate the value {:ok, :running, %CacheEntry.StateData{ + timer_func: timer, hydrating_func: fun, key: key, ttl: ttl, @@ -51,6 +54,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do @impl :gen_statem def handle_event({:call, from}, {:get, _from}, :idle, data) do + IO.puts("==> Get in idle state") ## Value is requested while fsm is iddle, return the value {:next_state, :idle, data, [{:reply, from, data.value}]} end @@ -93,8 +97,12 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do def handle_event({:call, from}, {:register, fun, key, ttl, refresh_interval}, :running, data) do ## Registering a new hydrating function while previous one is running - ## We stop the task - Task.shutdown(data.running_func_task, :brutal_kill) + + ## We stop the hydrating task if it is already running + case data.running_func_task do + pid when is_pid(pid) -> Process.exit(pid, :brutal_kill) + _ -> :ok + end ## And the timers triggering it and discarding value _ = :timer.cancel(data.timer_func) @@ -175,26 +183,34 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end def handle_event(:cast, {:new_value, key, {:ok, value}}, :running, data) do + ## Stop timer on value ttl + _ = :timer.cancel(data.timer_discard) + ## We got result from hydrating function Logger.debug( "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} got value #{inspect({key, value})}" ) ## notify waiiting getters - Enum.each(data.getters, fn {pid, _ref} -> send(pid, {:ok, value}) end) + Enum.each(data.getters, fn {pid, _ref} -> + IO.puts("Sending value to #{inspect(pid)}") + send(pid, {:ok, value}) + end) + ## We could do error control here, like unregistering the running func. ## atm, we will keep it _ = :timer.cancel(data.timer_discard) - # me = self() - # {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) + + me = self() + {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) {:next_state, :idle, %CacheEntry.StateData{ data | running_func_task: :undefined, value: {:ok, value}, - getters: [] - # timer_discard: new_timer + getters: [], + timer_discard: new_timer }} end diff --git a/lib/archethic/utils/hydrating_cache/caches_manager.ex b/lib/archethic/utils/hydrating_cache/caches_manager.ex index 9d1133330..4dd7cb77e 100644 --- a/lib/archethic/utils/hydrating_cache/caches_manager.ex +++ b/lib/archethic/utils/hydrating_cache/caches_manager.ex @@ -40,7 +40,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do @impl true def init(_args) do - manager_conf = Application.get_env(:archethic, __MODULE__) + manager_conf = Application.get_env(:archethic, __MODULE__, []) {:ok, caches_sup} = DynamicSupervisor.start_link( @@ -65,7 +65,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do {:ok, pid} = DynamicSupervisor.start_child( state.caches_sup, - HydratingCache.child_spec([{name, initial_keys}, []]) + HydratingCache.child_spec([name, initial_keys, []]) ) {:reply, {:ok, pid}, state} @@ -77,7 +77,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do DynamicSupervisor.start_child(state.caches_sup, %{ id: name, - start: {HydratingCache, :start_link, [{name, keys}]} + start: {HydratingCache, :start_link, [name, keys]} }) {:noreply, state} diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index 3f11b1220..cb169b0cb 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -14,7 +14,7 @@ defmodule Archethic.Utils.HydratingCache do | {:error, :timeout} | {:error, :not_registered} - def start_link({name, initial_keys}) do + def start_link(name, initial_keys \\ []) do GenServer.start_link(__MODULE__, [name, initial_keys], name: :"#{__MODULE__}.#{name}") end @@ -68,31 +68,34 @@ defmodule Archethic.Utils.HydratingCache do :not_registered}` """ @spec get(atom(), any(), non_neg_integer(), Keyword.t()) :: result - def get(cache, key, timeout \\ 3_000, _opts \\ []) + def get(cache, key, timeout \\ 1_000, _opts \\ []) when is_integer(timeout) and timeout > 0 do Logger.debug("Getting key #{inspect(key)} from hydrating cache #{inspect(cache)}") case GenServer.call(cache, {:get, key}, timeout) do {:ok, :answer_delayed} -> Logger.debug( - "waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(cache)}" + "waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(cache)} #{inspect(self())}" ) receive do {:ok, value} -> {:ok, value} + + other -> + Logger.info("Unexpected return value #{inspect(other)}") # code after timeout -> {:error, :timeout} end - {:ok, value} -> + other_result -> Logger.debug( - "Got value #{inspect(value)} for key #{inspect(key)} from hydrating cache #{inspect(cache)}" + "Got value #{inspect(other_result)} for key #{inspect(key)} from hydrating cache #{inspect(cache)}" ) - {:ok, value} + other_result end end From 1525fafc7dbe9a0054bcc79410746ac885956171 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Thu, 19 Jan 2023 17:45:05 +0100 Subject: [PATCH 10/48] Adding tests --- .../hydrating_cache/hydrating_cache_test.exs | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 test/archethic/utils/hydrating_cache/hydrating_cache_test.exs diff --git a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs new file mode 100644 index 000000000..25046daae --- /dev/null +++ b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs @@ -0,0 +1,239 @@ +defmodule CacheTest do + alias Archethic.Utils.HydratingCache + use ExUnit.Case + require Logger + + test "If `key` is not associated with any function, return `{:error, :not_registered}`" do + {:ok, pid} = HydratingCache.start_link(:test_service) + assert HydratingCache.get(pid, "unexisting_key") == {:error, :not_registered} + end + + test "If value stored, it is returned immediatly" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + result = + HydratingCache.register_function( + pid, + fn -> + {:ok, 1} + end, + "simple_func", + 50_000, + 10_000 + ) + + assert result == :ok + ## WAit a little to be sure value is registered and not being refreshed + :timer.sleep(500) + r = HydratingCache.get(pid, "simple_func", 10_000) + assert r == {:ok, 1} + end + + test "Getting value for key while function is running first time make process wait and return value" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + result = + HydratingCache.register_function( + pid, + fn -> + Logger.info("Hydrating function Sleeping 3 secs") + :timer.sleep(3000) + {:ok, 1} + end, + "test_long_function", + 50_000, + 9000 + ) + + assert result == :ok + + r = HydratingCache.get(pid, "test_long_function", 10_000) + assert r == {:ok, 1} + end + + test "Getting value for key while function is running first time returns timeout after ttl" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + result = + HydratingCache.register_function( + pid, + fn -> + Logger.info("Hydrating function Sleeping 3 secs") + :timer.sleep(3000) + {:ok, 1} + end, + "test_get_ttl", + 50_000, + 9000 + ) + + assert result == :ok + + ## get and wait up to 1 second + r = HydratingCache.get(pid, "test_get_ttl", 1000) + assert r == {:error, :timeout} + end + + test "Hydrating function runs periodically" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + :persistent_term.put("test", 0) + + result = + HydratingCache.register_function( + pid, + fn -> + IO.puts("Hydrating function incrementing value") + value = :persistent_term.get("test") + value = value + 1 + :persistent_term.put("test", value) + {:ok, value} + end, + "test_inc", + 50000, + 1000 + ) + + assert result == :ok + + :timer.sleep(5000) + {:ok, value} = HydratingCache.get(pid, "test_inc", 3000) + + assert value >= 5 + end + + test "Update hydrating function while another one is running returns new hydrating value from new function" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + result = + HydratingCache.register_function( + pid, + fn -> + :timer.sleep(5000) + {:ok, 1} + end, + "test_reregister", + 50000, + 10000 + ) + + assert result == :ok + + _result = + HydratingCache.register_function( + pid, + fn -> + {:ok, 2} + end, + "test_reregister", + 50000, + 10000 + ) + + :timer.sleep(5000) + {:ok, value} = HydratingCache.get(pid, "test_reregister", 4000) + + assert value == 2 + end + + test "Getting value while function is running and previous value is available returns value" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + _ = + HydratingCache.register_function( + pid, + fn -> + {:ok, 1} + end, + "test_reregister", + 50000, + 10000 + ) + + _ = + HydratingCache.register_function( + pid, + fn -> + :timer.sleep(5000) + {:ok, 2} + end, + "test_reregister", + 50000, + 10000 + ) + + {:ok, value} = HydratingCache.get(pid, "test_reregister", 4000) + + assert value == 1 + end + + test "Two hydrating function can run at same time" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + _ = + HydratingCache.register_function( + pid, + fn -> + :timer.sleep(5000) + {:ok, :result_timed} + end, + "timed_value", + 80000, + 70000 + ) + + _ = + HydratingCache.register_function( + pid, + fn -> + {:ok, :result} + end, + "direct_value", + 80000, + 70000 + ) + + ## We query the value with timeout smaller than timed function + {:ok, _value} = HydratingCache.get(pid, "direct_value", 2000) + end + + test "Querying key while first refreshed will block the calling process until refreshed and provide the value" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + _ = + HydratingCache.register_function( + pid, + fn -> + :timer.sleep(2000) + {:ok, :valid_result} + end, + "delayed_result", + 80000, + 70000 + ) + + ## We query the value with timeout smaller than timed function + {:ok, value} = HydratingCache.get(pid, "delayed_result", 3000) + assert value == :valid_result + end + + test "Querying key while first refreshed will block the calling process until timeout" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + _ = + HydratingCache.register_function( + pid, + fn -> + :timer.sleep(2000) + {:ok, :valid_result} + end, + "delayed_result", + 80000, + 70000 + ) + + ## We query the value with timeout smaller than timed function + result = HydratingCache.get(pid, "delayed_result", 1000) + assert result == {:error, :timeout} + end +end From fc89bdb4dac1a6e4af6f0d49885c06e2e0ed4d84 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Fri, 20 Jan 2023 00:33:02 +0100 Subject: [PATCH 11/48] Adding test for manager + refinement --- .../hydrating_cache/caches_manager_test.exs | 25 +++++++++++++++++++ .../hydrating_cache/hydrating_cache_test.exs | 7 +++--- 2 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 test/archethic/utils/hydrating_cache/caches_manager_test.exs diff --git a/test/archethic/utils/hydrating_cache/caches_manager_test.exs b/test/archethic/utils/hydrating_cache/caches_manager_test.exs new file mode 100644 index 000000000..4151a6e91 --- /dev/null +++ b/test/archethic/utils/hydrating_cache/caches_manager_test.exs @@ -0,0 +1,25 @@ +defmodule HydratingCacheTest do + alias Archethic.Utils.HydratingCache + alias Archethic.Utils.HydratingCache.CachesManager + use ExUnit.Case + require Logger + + test "starting service from manager returns value once first hydrating have been done" do + CachesManager.new_service_async("test_services", [ + {:key1, __MODULE__, :waiting_function, [4000], 6000}, + {:key2, __MODULE__, :waiting_function, [2000], 6000}, + {:key3, __MODULE__, :waiting_function, [4000], 6000} + ]) + + assert HydratingCache.get( + :"Elixir.Archethic.Utils.HydratingCache.test_services", + "key2", + 3000 + ) == {:ok, 1} + end + + def waiting_function(delay \\ 1000) do + :timer.sleep(delay) + {:ok, 1} + end +end diff --git a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs index 25046daae..2537cbc46 100644 --- a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs +++ b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs @@ -1,4 +1,4 @@ -defmodule CacheTest do +defmodule HydratingCacheTest do alias Archethic.Utils.HydratingCache use ExUnit.Case require Logger @@ -204,7 +204,7 @@ defmodule CacheTest do HydratingCache.register_function( pid, fn -> - :timer.sleep(2000) + :timer.sleep(4000) {:ok, :valid_result} end, "delayed_result", @@ -213,8 +213,7 @@ defmodule CacheTest do ) ## We query the value with timeout smaller than timed function - {:ok, value} = HydratingCache.get(pid, "delayed_result", 3000) - assert value == :valid_result + assert {:ok, :valid_result} = HydratingCache.get(pid, "delayed_result", 5000) end test "Querying key while first refreshed will block the calling process until timeout" do From 1de4d8fcb2e8e7e0b2093cbd7941f6975def2990 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Fri, 20 Jan 2023 10:56:58 +0100 Subject: [PATCH 12/48] Addressing comments --- .../utils/hydrating_cache/cache_entry.ex | 21 ++++++++++--------- .../utils/hydrating_cache/caches_manager.ex | 1 + 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 51403c775..47d300716 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -3,15 +3,17 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry.StateData do Struct describing the state of a cache entry FSM """ - defstruct running_func_task: :undefined, - hydrating_func: :undefined, - getters: [], - ttl: :undefined, - refresh_interval: :undefined, - key: :undefined, - value: :"$$undefined", - timer_func: :undefined, - timer_discard: :undefined + defstruct([ + :running_func_task, + :hydrating_func, + :ttl, + :refresh_interval, + :key, + :timer_func, + :timer_discard, + getters: [], + value: :"$$undefined" + ]) end defmodule Archethic.Utils.HydratingCache.CacheEntry do @@ -54,7 +56,6 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do @impl :gen_statem def handle_event({:call, from}, {:get, _from}, :idle, data) do - IO.puts("==> Get in idle state") ## Value is requested while fsm is iddle, return the value {:next_state, :idle, data, [{:reply, from, data.value}]} end diff --git a/lib/archethic/utils/hydrating_cache/caches_manager.ex b/lib/archethic/utils/hydrating_cache/caches_manager.ex index 4dd7cb77e..825101b19 100644 --- a/lib/archethic/utils/hydrating_cache/caches_manager.ex +++ b/lib/archethic/utils/hydrating_cache/caches_manager.ex @@ -28,6 +28,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do This is an asynchronous call, it will return immediately. """ def new_service_async(name, keys) do + IO.inspect("Starting new service #{name}") GenServer.cast(__MODULE__, {:new_service_async, name, keys, self()}) end From 45c2406376b41c962611ece534c5614eaa3a3d1a Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Sun, 22 Jan 2023 00:13:55 +0100 Subject: [PATCH 13/48] Uncreasing oracle service frequency to be closer to real use ( and removing coco service ) --- config/dev.exs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index b97bad4d0..af7c26141 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -81,20 +81,13 @@ config :archethic, Archethic.Utils.HydratingCache.CachesManager, uco_service: [ {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, [["usd", "eur"]], - 3000}, + 30000}, {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], 3000}, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], + 30000}, {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, [["usd", "eur"]], - 3000} - ], - coco_service: [ - {:coin_paprika, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, - [["usd", "eur"]], 3000}, - {:coin_gecko, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, - [["usd", "eur"]], 3000}, - {:coin_market_cap, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, - [["usd", "eur"]], 3000} + 30000} ] config :archethic, Archethic.Governance.Pools, From aa0f0cb63af945895b326bf04f9010f2a2d1d35f Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Sun, 22 Jan 2023 00:19:42 +0100 Subject: [PATCH 14/48] Adding 3 secs timeout when fethcing value from cache --- lib/archethic/oracle_chain/services/uco_price.ex | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index fccd0da48..e716741c8 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -24,7 +24,11 @@ defmodule Archethic.OracleChain.Services.UCOPrice do ## retrieve prices from configured providers and filter results marked as errors prices = Enum.map(providers(), fn provider -> - case HydratingCache.get(:"Elixir.Archethic.Utils.HydratingCache.uco_service", provider) do + case HydratingCache.get( + :"Elixir.Archethic.Utils.HydratingCache.uco_service", + provider, + 3000 + ) do {:error, reason} -> Logger.warning( "Service UCOPrice cannot fetch values from provider: #{inspect(provider)} with reason : #{inspect(reason)}." @@ -49,7 +53,6 @@ defmodule Archethic.OracleChain.Services.UCOPrice do {_, result = %{}} -> result end) - ## Here stream looks like : [%{"eur"=>[0.44], "usd"=[0.32]}, ..., %{"eur"=>[0.42, 0.43], "usd"=[0.35]}] |> Enum.reduce(%{}, &agregate_providers_data/2) |> Enum.reduce(%{}, fn {currency, values}, acc -> From 76bf4f5504ef5da6a058c8e1c4e2dc49caadc38d Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Sun, 22 Jan 2023 00:20:38 +0100 Subject: [PATCH 15/48] Managing case when value is requested but couldn't have been initialised --- .../utils/hydrating_cache/cache_entry.ex | 45 +++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 47d300716..05d0bc13f 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -39,7 +39,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do @impl :gen_statem def init([fun, key, ttl, refresh_interval]) do timer = :timer.send_interval(refresh_interval, self(), :hydrate) - + Logger.debug("Cache entry started for key #{inspect(key)}") ## Hydrate the value {:ok, :running, %CacheEntry.StateData{ @@ -55,30 +55,57 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do def callback_mode, do: [:handle_event_function, :state_enter] @impl :gen_statem - def handle_event({:call, from}, {:get, _from}, :idle, data) do + + def handle_event( + {:call, from}, + {:get, _requester}, + :idle, + data = %CacheEntry.StateData{:value => :"$$undefined"} + ) do + IO.puts("Cache entry Call: value for key #{inspect(data.key)} when idle and no value") + ## Value is requested while fsm is iddle, return the value + + {:keep_state, data, [{:reply, from, {:error, :not_initialized}}]} + end + + def handle_event({:call, from}, {:get, _requester}, :idle, data) do + IO.puts("Cache entry Call: value for key #{inspect(data.key)} is requested while idle") ## Value is requested while fsm is iddle, return the value {:next_state, :idle, data, [{:reply, from, data.value}]} end - def handle_event(:cast, {:get, from}, :idle, data) do + def handle_event(:cast, {:get, requester}, :idle, data) do + IO.puts("Cache entry Cast: value for key #{inspect(data.key)} is requested while idle") ## Value is requested while fsm is iddle, return the value - send(from, {:ok, data.value}) + send(requester, {:ok, data.value}) {:next_state, :idle, data} end ## Call for value while hydrating function is running and we have no previous value ## We register the caller to send value later on, and we indicate caller to block - def handle_event({:call, _from}, {:get, from}, :running, data) - when data.value == :"$$undefined" do + def handle_event( + {:call, from}, + {:get, requester}, + :running, + data = %CacheEntry.StateData{value: :"$$undefined"} + ) do + IO.puts( + "Cache entry Call: value for key #{inspect(data.key)} is requested while running and no value #{inspect(requester)} self:#{inspect(self())}" + ) + previous_getters = data.getters - {:keep_state, %CacheEntry.StateData{data | getters: previous_getters ++ [from]}, + {:keep_state, %CacheEntry.StateData{data | getters: previous_getters ++ [requester]}, [{:reply, from, {:ok, :answer_delayed}}]} end ## Call for value while hydrating function is running and we have a previous value ## We return the value to caller - def handle_event({:call, from}, {:get, _from}, :running, data) do + def handle_event({:call, from}, {:get, _requester}, :running, data) do + IO.puts( + "Cache entry Call: value for key #{inspect(data.key)} is requested while running and value" + ) + {:next_state, :running, data, [{:reply, from, data.value}]} end @@ -125,7 +152,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end def handle_event({:call, from}, {:register, fun, key, ttl, refresh_interval}, _state, data) do - IO.puts("Registering new hydrating function #{inspect(fun)} for key #{inspect(key)}") + Logger.info("Registering new hydrating function #{inspect(fun)} for key #{inspect(key)}") ## Setting hydrating function in other cases ## Hydrating function not running, we just stop the timers _ = :timer.cancel(data.timer_func) From fc63a3b83ea6df0c62b79e6dcdff563cc5355093 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Sun, 22 Jan 2023 00:21:01 +0100 Subject: [PATCH 16/48] Adding Trace --- lib/archethic/utils/hydrating_cache/caches_manager.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/archethic/utils/hydrating_cache/caches_manager.ex b/lib/archethic/utils/hydrating_cache/caches_manager.ex index 825101b19..6b03506a9 100644 --- a/lib/archethic/utils/hydrating_cache/caches_manager.ex +++ b/lib/archethic/utils/hydrating_cache/caches_manager.ex @@ -81,6 +81,8 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do start: {HydratingCache, :start_link, [name, keys]} }) + Logger.info("Started new service #{name}") + {:noreply, state} end From 992276f2029e169d0a7d15a9ede8f8c8716f5954 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Sun, 22 Jan 2023 00:23:01 +0100 Subject: [PATCH 17/48] Adding cast to request hydrating function registration from api --- .../utils/hydrating_cache/hydrating_cache.ex | 60 ++++++++++++++++--- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index cb169b0cb..4484964ff 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -1,6 +1,7 @@ defmodule Archethic.Utils.HydratingCache do @moduledoc """ GenServer implementing the hydrating cache itself. + There should be one Hydrating per service ( ex : UCO price, meteo etc...) It receives queries from clients requesting the cache, and manage the cache entries FSMs """ alias Archethic.Utils.HydratingCache.CacheEntry @@ -70,7 +71,9 @@ defmodule Archethic.Utils.HydratingCache do @spec get(atom(), any(), non_neg_integer(), Keyword.t()) :: result def get(cache, key, timeout \\ 1_000, _opts \\ []) when is_integer(timeout) and timeout > 0 do - Logger.debug("Getting key #{inspect(key)} from hydrating cache #{inspect(cache)}") + Logger.debug( + "Getting key #{inspect(key)} from hydrating cache #{inspect(cache)} for #{inspect(self())}" + ) case GenServer.call(cache, {:get, key}, timeout) do {:ok, :answer_delayed} -> @@ -80,6 +83,10 @@ defmodule Archethic.Utils.HydratingCache do receive do {:ok, value} -> + Logger.debug( + "Got delayed value #{inspect(value)} for key #{inspect(key)} from hydrating cache #{inspect(cache)}" + ) + {:ok, value} other -> @@ -87,6 +94,10 @@ defmodule Archethic.Utils.HydratingCache do # code after timeout -> + Logger.warn( + "Timeout waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(cache)}" + ) + {:error, :timeout} end @@ -116,6 +127,8 @@ defmodule Archethic.Utils.HydratingCache do {:ok, initial_keys_worker_sup} = Task.Supervisor.start_link() ## Registering initial keys + Logger.info("Init Registering initial keys for #{inspect(name)}") + _ = Task.Supervisor.async_stream_nolink( initial_keys_worker_sup, @@ -123,14 +136,18 @@ defmodule Archethic.Utils.HydratingCache do fn {provider, mod, func, params, refresh_rate} -> Logger.debug( - "Registering hydrating function. Provider: #{inspect(provider)} Hydrating function: + "Init asking Registering hydrating function. Provider: #{inspect(provider)} Hydrating function: #{inspect(mod)}.#{inspect(func)}(#{inspect(params)}) Refresh rate: #{inspect(refresh_rate)}" ) - GenServer.call( - me, - {:register, fn -> apply(mod, func, params) end, provider, 75_000, refresh_rate} - ) + g = + GenServer.cast( + me, + {:register, fn -> apply(mod, func, params) end, provider, 75_000, refresh_rate} + ) + + Logger.debug("Finished requesting registration for #{inspect(provider)}") + g other -> Logger.error("Hydrating cache: Invalid configuration entry: #{inspect(other)}") @@ -140,8 +157,10 @@ defmodule Archethic.Utils.HydratingCache do |> Stream.filter(&match?({:ok, {:ok, _}}, &1)) |> Enum.to_list() + Logger.info("Hydrating cache: Init Finished registering initial keys") ## stop the initial keys worker supervisor Supervisor.stop(initial_keys_worker_sup) + Logger.info("Hydrating cache: Init Finished stopping initial keys worker supervisor") {:ok, %{:keys => keys, keys_sup: keys_sup}} end @@ -150,28 +169,34 @@ defmodule Archethic.Utils.HydratingCache do def handle_call({:get, key}, from, state) do case Map.get(state, key, :undefined) do :undefined -> + Logger.debug("HydratingCache no entry for #{inspect(state)}") {:reply, {:error, :not_registered}, state} pid -> + Logger.debug("HydratingCache found entry #{inspect(pid)} for #{inspect(self())}") value = :gen_statem.call(pid, {:get, from}) - IO.puts("value #{inspect(value)}") + Logger.debug("Cache entry returned #{inspect(value)}") {:reply, value, state} end end def handle_call({:register, fun, key, ttl, refresh_interval}, _from, state) do + Logger.debug("Registering hydrating function for #{inspect(key)}") ## Called when asked to register a function case Map.get(state, key) do nil -> ## New key, we start a cache entry fsm + Logger.debug("Starting cache entry for #{inspect(key)}") + {:ok, pid} = DynamicSupervisor.start_child( state.keys_sup, {CacheEntry, [fun, key, ttl, refresh_interval]} ) + Logger.debug("Started cache entry for #{inspect({key, pid})}") {:reply, :ok, Map.put(state, key, pid)} pid -> @@ -205,8 +230,25 @@ defmodule Archethic.Utils.HydratingCache do end def handle_cast({:register, fun, key, ttl, refresh_interval}, state) do - handle_call({:register, fun, key, ttl, refresh_interval}, nil, state) - {:noreply, state} + case Map.get(state, key) do + nil -> + ## New key, we start a cache entry fsm + Logger.debug("CAST Starting cache entry for #{inspect(key)}") + + {:ok, pid} = + DynamicSupervisor.start_child( + state.keys_sup, + {CacheEntry, [fun, key, ttl, refresh_interval]} + ) + + Logger.debug("CAST Started cache entry for #{inspect({key, pid})}") + {:noreply, Map.put(state, key, pid)} + + pid -> + ## Key already exists, no need to start fsm + _ = :gen_statem.call(pid, {:register, fun, key, ttl, refresh_interval}) + {:noreply, Map.put(state, key, pid)} + end end def handle_cast(unmanaged, state) do From 0b43778c625a5d77dbdabc08c98814f7080bc305 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Sun, 22 Jan 2023 00:23:21 +0100 Subject: [PATCH 18/48] Adding tests --- .../hydrating_cache/caches_manager_test.exs | 13 ++-- .../hydrating_cache/hydrating_cache_test.exs | 64 ++++++++++++++++++- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/test/archethic/utils/hydrating_cache/caches_manager_test.exs b/test/archethic/utils/hydrating_cache/caches_manager_test.exs index 4151a6e91..4c0e85094 100644 --- a/test/archethic/utils/hydrating_cache/caches_manager_test.exs +++ b/test/archethic/utils/hydrating_cache/caches_manager_test.exs @@ -6,15 +6,18 @@ defmodule HydratingCacheTest do test "starting service from manager returns value once first hydrating have been done" do CachesManager.new_service_async("test_services", [ - {:key1, __MODULE__, :waiting_function, [4000], 6000}, - {:key2, __MODULE__, :waiting_function, [2000], 6000}, - {:key3, __MODULE__, :waiting_function, [4000], 6000} + {:key1, __MODULE__, :waiting_function, [2000], 6000}, + {:key2, __MODULE__, :waiting_function, [1000], 6000}, + {:key3, __MODULE__, :waiting_function, [2000], 6000} ]) + ## wait a little so at least keys are registered + :timer.sleep(500) + assert HydratingCache.get( :"Elixir.Archethic.Utils.HydratingCache.test_services", - "key2", - 3000 + :key2, + 1700 ) == {:ok, 1} end diff --git a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs index 2537cbc46..1d4e59188 100644 --- a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs +++ b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs @@ -233,6 +233,68 @@ defmodule HydratingCacheTest do ## We query the value with timeout smaller than timed function result = HydratingCache.get(pid, "delayed_result", 1000) - assert result == {:error, :timeout} + assert result = {:error, :timeout} + end + + test "Multiple process can wait for a delayed value" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + _ = + HydratingCache.register_function( + pid, + fn -> + IO.puts("Hydrating function Sleeping 3 secs") + :timer.sleep(3000) + IO.puts("Hydrating function done") + {:ok, :valid_result} + end, + "delayed_result", + 80000, + 70000 + ) + + ## We query the value with timeout smaller than timed function + results = + Task.async_stream(1..10, fn _ -> HydratingCache.get(pid, "delayed_result", 4000) end) + + assert Enum.all?(results, fn + {:ok, {:ok, :valid_result}} -> true + other -> IO.puts("Unk #{inspect(other)}") + end) + end + + ## Resilience tests + test "If hydrating function crash, key fsm will still be oprationnal" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + _ = + HydratingCache.register_function( + pid, + fn -> + ## Trigger badmatch + exit(1) + {:ok, :badmatch} + end, + :key, + 80000, + 70000 + ) + + :timer.sleep(1000) + + _ = + HydratingCache.register_function( + pid, + fn -> + ## Trigger badmatch + {:ok, :value} + end, + :key, + 80000, + 70000 + ) + + result = HydratingCache.get(pid, :key, 3000) + assert result == {:ok, :value} end end From b58c91daef5ddab5b2d6993f5b43195c6fe45cb7 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Sun, 22 Jan 2023 00:54:59 +0100 Subject: [PATCH 19/48] Cleaning code --- .../utils/hydrating_cache/cache_entry.ex | 23 ++-------- .../utils/hydrating_cache/caches_manager.ex | 8 ++-- .../utils/hydrating_cache/hydrating_cache.ex | 44 +++---------------- 3 files changed, 14 insertions(+), 61 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 05d0bc13f..6b82926c5 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -38,8 +38,9 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do @impl :gen_statem def init([fun, key, ttl, refresh_interval]) do + # start hydrating at the needed refresh interval timer = :timer.send_interval(refresh_interval, self(), :hydrate) - Logger.debug("Cache entry started for key #{inspect(key)}") + ## Hydrate the value {:ok, :running, %CacheEntry.StateData{ @@ -62,20 +63,17 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do :idle, data = %CacheEntry.StateData{:value => :"$$undefined"} ) do - IO.puts("Cache entry Call: value for key #{inspect(data.key)} when idle and no value") ## Value is requested while fsm is iddle, return the value {:keep_state, data, [{:reply, from, {:error, :not_initialized}}]} end def handle_event({:call, from}, {:get, _requester}, :idle, data) do - IO.puts("Cache entry Call: value for key #{inspect(data.key)} is requested while idle") ## Value is requested while fsm is iddle, return the value {:next_state, :idle, data, [{:reply, from, data.value}]} end def handle_event(:cast, {:get, requester}, :idle, data) do - IO.puts("Cache entry Cast: value for key #{inspect(data.key)} is requested while idle") ## Value is requested while fsm is iddle, return the value send(requester, {:ok, data.value}) {:next_state, :idle, data} @@ -89,10 +87,6 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do :running, data = %CacheEntry.StateData{value: :"$$undefined"} ) do - IO.puts( - "Cache entry Call: value for key #{inspect(data.key)} is requested while running and no value #{inspect(requester)} self:#{inspect(self())}" - ) - previous_getters = data.getters {:keep_state, %CacheEntry.StateData{data | getters: previous_getters ++ [requester]}, @@ -102,10 +96,6 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ## Call for value while hydrating function is running and we have a previous value ## We return the value to caller def handle_event({:call, from}, {:get, _requester}, :running, data) do - IO.puts( - "Cache entry Call: value for key #{inspect(data.key)} is requested while running and value" - ) - {:next_state, :running, data, [{:reply, from, data.value}]} end @@ -152,7 +142,6 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end def handle_event({:call, from}, {:register, fun, key, ttl, refresh_interval}, _state, data) do - Logger.info("Registering new hydrating function #{inspect(fun)} for key #{inspect(key)}") ## Setting hydrating function in other cases ## Hydrating function not running, we just stop the timers _ = :timer.cancel(data.timer_func) @@ -210,18 +199,14 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:next_state, state, %CacheEntry.StateData{data | value: :discarded}} end - def handle_event(:cast, {:new_value, key, {:ok, value}}, :running, data) do + def handle_event(:cast, {:new_value, _key, {:ok, value}}, :running, data) do ## Stop timer on value ttl _ = :timer.cancel(data.timer_discard) ## We got result from hydrating function - Logger.debug( - "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} got value #{inspect({key, value})}" - ) - ## notify waiiting getters + ## notify waiting getters Enum.each(data.getters, fn {pid, _ref} -> - IO.puts("Sending value to #{inspect(pid)}") send(pid, {:ok, value}) end) diff --git a/lib/archethic/utils/hydrating_cache/caches_manager.ex b/lib/archethic/utils/hydrating_cache/caches_manager.ex index 6b03506a9..d2cd76d0a 100644 --- a/lib/archethic/utils/hydrating_cache/caches_manager.ex +++ b/lib/archethic/utils/hydrating_cache/caches_manager.ex @@ -12,14 +12,16 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do end @doc """ - Start a new hydrating cache process to hold the values from a service. - This is a synchronous call, it will block until the cache is ready ( all keys are hydrated ) + Start a new hydrating cache process to hold the values from a service. + This is a synchronous call, it will block until the cache is started and + hydrating process for initial keys is started. """ @spec new_service_sync( name :: String.t(), initial_keys :: list() ) :: {:error, any} | {:ok, pid} def new_service_sync(name, initial_keys) do + Logger.info("Starting new service sync #{name}") GenServer.call(__MODULE__, {:new_service_sync, name, initial_keys}) end @@ -28,7 +30,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do This is an asynchronous call, it will return immediately. """ def new_service_async(name, keys) do - IO.inspect("Starting new service #{name}") + Logger.info("Starting new service async #{name}") GenServer.cast(__MODULE__, {:new_service_async, name, keys, self()}) end diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index 4484964ff..444afc431 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -77,21 +77,13 @@ defmodule Archethic.Utils.HydratingCache do case GenServer.call(cache, {:get, key}, timeout) do {:ok, :answer_delayed} -> - Logger.debug( - "waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(cache)} #{inspect(self())}" - ) - receive do {:ok, value} -> - Logger.debug( - "Got delayed value #{inspect(value)} for key #{inspect(key)} from hydrating cache #{inspect(cache)}" - ) - {:ok, value} other -> Logger.info("Unexpected return value #{inspect(other)}") - # code + {:error, :unexpected_value} after timeout -> Logger.warn( @@ -102,10 +94,6 @@ defmodule Archethic.Utils.HydratingCache do end other_result -> - Logger.debug( - "Got value #{inspect(other_result)} for key #{inspect(key)} from hydrating cache #{inspect(cache)}" - ) - other_result end end @@ -127,28 +115,17 @@ defmodule Archethic.Utils.HydratingCache do {:ok, initial_keys_worker_sup} = Task.Supervisor.start_link() ## Registering initial keys - Logger.info("Init Registering initial keys for #{inspect(name)}") - _ = Task.Supervisor.async_stream_nolink( initial_keys_worker_sup, keys, fn {provider, mod, func, params, refresh_rate} -> - Logger.debug( - "Init asking Registering hydrating function. Provider: #{inspect(provider)} Hydrating function: - #{inspect(mod)}.#{inspect(func)}(#{inspect(params)}) Refresh rate: #{inspect(refresh_rate)}" + GenServer.cast( + me, + {:register, fn -> apply(mod, func, params) end, provider, 75_000, refresh_rate} ) - g = - GenServer.cast( - me, - {:register, fn -> apply(mod, func, params) end, provider, 75_000, refresh_rate} - ) - - Logger.debug("Finished requesting registration for #{inspect(provider)}") - g - other -> Logger.error("Hydrating cache: Invalid configuration entry: #{inspect(other)}") end, @@ -157,10 +134,8 @@ defmodule Archethic.Utils.HydratingCache do |> Stream.filter(&match?({:ok, {:ok, _}}, &1)) |> Enum.to_list() - Logger.info("Hydrating cache: Init Finished registering initial keys") ## stop the initial keys worker supervisor Supervisor.stop(initial_keys_worker_sup) - Logger.info("Hydrating cache: Init Finished stopping initial keys worker supervisor") {:ok, %{:keys => keys, keys_sup: keys_sup}} end @@ -169,15 +144,11 @@ defmodule Archethic.Utils.HydratingCache do def handle_call({:get, key}, from, state) do case Map.get(state, key, :undefined) do :undefined -> - Logger.debug("HydratingCache no entry for #{inspect(state)}") + Logger.warning("HydratingCache no entry for #{inspect(state)}") {:reply, {:error, :not_registered}, state} pid -> - Logger.debug("HydratingCache found entry #{inspect(pid)} for #{inspect(self())}") value = :gen_statem.call(pid, {:get, from}) - - Logger.debug("Cache entry returned #{inspect(value)}") - {:reply, value, state} end end @@ -188,15 +159,12 @@ defmodule Archethic.Utils.HydratingCache do case Map.get(state, key) do nil -> ## New key, we start a cache entry fsm - Logger.debug("Starting cache entry for #{inspect(key)}") - {:ok, pid} = DynamicSupervisor.start_child( state.keys_sup, {CacheEntry, [fun, key, ttl, refresh_interval]} ) - Logger.debug("Started cache entry for #{inspect({key, pid})}") {:reply, :ok, Map.put(state, key, pid)} pid -> @@ -233,7 +201,6 @@ defmodule Archethic.Utils.HydratingCache do case Map.get(state, key) do nil -> ## New key, we start a cache entry fsm - Logger.debug("CAST Starting cache entry for #{inspect(key)}") {:ok, pid} = DynamicSupervisor.start_child( @@ -241,7 +208,6 @@ defmodule Archethic.Utils.HydratingCache do {CacheEntry, [fun, key, ttl, refresh_interval]} ) - Logger.debug("CAST Started cache entry for #{inspect({key, pid})}") {:noreply, Map.put(state, key, pid)} pid -> From f9c3d506f33250649233d681c2b85300967fc038 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Sun, 22 Jan 2023 20:04:23 +0100 Subject: [PATCH 20/48] Adding timeout support for ttl plus refinement --- config/dev.exs | 10 +- .../utils/hydrating_cache/cache_entry.ex | 43 ++++-- .../utils/hydrating_cache/caches_manager.ex | 4 +- .../utils/hydrating_cache/hydrating_cache.ex | 42 +++--- .../hydrating_cache/caches_manager_test.exs | 6 +- .../hydrating_cache/hydrating_cache_test.exs | 135 ++++++++---------- 6 files changed, 124 insertions(+), 116 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index af7c26141..300f80aed 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -77,17 +77,19 @@ config :archethic, Archethic.Crypto.NodeKeystore.Origin.TPMImpl end) +## Format : {service, [{key, module, function, args, refresh_interval_ms, ttl_ms}, ...]} +## If ttl = :infinity, the cache will never expire config :archethic, Archethic.Utils.HydratingCache.CachesManager, uco_service: [ {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, [["usd", "eur"]], - 30000}, + 30000, :infinity}, {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], - 30000}, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], 30000, + :infinity}, {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, [["usd", "eur"]], - 30000} + 30000, :infinity} ] config :archethic, Archethic.Governance.Pools, diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 6b82926c5..9c7f144a6 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -32,12 +32,12 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do use Task require Logger - def start_link([fun, key, ttl, refresh_interval]) do - :gen_statem.start_link(__MODULE__, [fun, key, ttl, refresh_interval], []) + def start_link([fun, key, refresh_interval, ttl]) do + :gen_statem.start_link(__MODULE__, [fun, key, refresh_interval, ttl], []) end @impl :gen_statem - def init([fun, key, ttl, refresh_interval]) do + def init([fun, key, refresh_interval, ttl]) do # start hydrating at the needed refresh interval timer = :timer.send_interval(refresh_interval, self(), :hydrate) @@ -113,12 +113,12 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:next_state, :running, data} end - def handle_event({:call, from}, {:register, fun, key, ttl, refresh_interval}, :running, data) do + def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, :running, data) do ## Registering a new hydrating function while previous one is running ## We stop the hydrating task if it is already running case data.running_func_task do - pid when is_pid(pid) -> Process.exit(pid, :brutal_kill) + pid when is_pid(pid) -> Process.exit(pid, :normal) _ -> :ok end @@ -141,7 +141,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do }, [{:reply, from, :ok}]} end - def handle_event({:call, from}, {:register, fun, key, ttl, refresh_interval}, _state, data) do + def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, _state, data) do ## Setting hydrating function in other cases ## Hydrating function not running, we just stop the timers _ = :timer.cancel(data.timer_func) @@ -155,7 +155,15 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do :refresh_interval => refresh_interval }) - timer = :timer.send_interval(refresh_interval, self(), :hydrate) + timer = + case ttl do + :infinity -> + nil + + value when is_number(value) -> + {:ok, t} = :timer.send_interval(refresh_interval, self(), :hydrate) + t + end ## We trigger the update ( to trigger or not could be set at registering option ) {:next_state, :running, @@ -181,6 +189,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do hydrating_task = spawn(fn -> + Logger.info("Running hydrating function for key :#{inspect(data.key)}") value = data.hydrating_func.() :gen_statem.cast(me, {:new_value, data.key, value}) end) @@ -196,7 +205,8 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} discarded" ) - {:next_state, state, %CacheEntry.StateData{data | value: :discarded}} + {:next_state, state, + %CacheEntry.StateData{data | value: {:error, :discarded}, timer_discard: nil}} end def handle_event(:cast, {:new_value, _key, {:ok, value}}, :running, data) do @@ -210,9 +220,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do send(pid, {:ok, value}) end) - ## We could do error control here, like unregistering the running func. - ## atm, we will keep it - _ = :timer.cancel(data.timer_discard) + ## Start timer to discard new value me = self() {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) @@ -233,7 +241,18 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} got error value #{inspect({key, {:error, reason}})}" ) - {:next_state, :idle, %CacheEntry.StateData{data | running_func_task: :undefined}} + ## We reprogram the timer to hydrate, even if previous call failled. Error control could occur here + me = self() + {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) + + {:next_state, :idle, + %CacheEntry.StateData{ + data + | running_func_task: :undefined, + getters: [], + timer_discard: nil, + timer_func: new_timer + }} end def handle_event(_type, _event, _state, data) do diff --git a/lib/archethic/utils/hydrating_cache/caches_manager.ex b/lib/archethic/utils/hydrating_cache/caches_manager.ex index d2cd76d0a..0c2ba3d01 100644 --- a/lib/archethic/utils/hydrating_cache/caches_manager.ex +++ b/lib/archethic/utils/hydrating_cache/caches_manager.ex @@ -65,6 +65,8 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do @impl true def handle_call({:new_service_sync, name, initial_keys}, _from, state) do + Logger.info("Starting new service sync : #{name}") + {:ok, pid} = DynamicSupervisor.start_child( state.caches_sup, @@ -76,7 +78,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do @impl true def handle_cast({:new_service_async, name, keys, _requester}, state) do - IO.inspect("Starting new service genserver #{name}") + Logger.info("Starting new service async : #{name}") DynamicSupervisor.start_child(state.caches_sup, %{ id: name, diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index 444afc431..63a29ca79 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -23,16 +23,18 @@ defmodule Archethic.Utils.HydratingCache do Registers a function that will be computed periodically to update the cache. Arguments: + - `hydrating_cache`: the pid of the hydrating cache. - `fun`: a 0-arity function that computes the value and returns either `{:ok, value}` or `{:error, reason}`. - `key`: associated with the function and is used to retrieve the stored value. - - `ttl` ("time to live"): how long (in milliseconds) the value is stored - before it is discarded if the value is not refreshed. - `refresh_interval`: how often (in milliseconds) the function is recomputed and the new value stored. `refresh_interval` must be strictly smaller than `ttl`. After the value is refreshed, the `ttl` counter is restarted. + - `ttl` ("time to live"): how long (in milliseconds) the value is stored + before it is discarded if the value is not refreshed. + The value is stored only if `{:ok, value}` is returned by `fun`. If `{:error, reason}` is returned, the value is not stored and `fun` must be retried on @@ -42,14 +44,14 @@ defmodule Archethic.Utils.HydratingCache do hydrating_cache :: pid(), fun :: (() -> {:ok, any()} | {:error, any()}), key :: any, - ttl :: non_neg_integer(), - refresh_interval :: non_neg_integer() + refresh_interval :: non_neg_integer(), + ttl :: non_neg_integer() ) :: :ok - def register_function(hydrating_cache, fun, key, ttl, refresh_interval) + def register_function(hydrating_cache, fun, key, refresh_interval, ttl) when is_function(fun, 0) and is_integer(ttl) and ttl > 0 and is_integer(refresh_interval) and refresh_interval < ttl do - GenServer.call(hydrating_cache, {:register, fun, key, ttl, refresh_interval}) + GenServer.call(hydrating_cache, {:register, fun, key, refresh_interval, ttl}) end @doc ~s""" @@ -69,25 +71,25 @@ defmodule Archethic.Utils.HydratingCache do :not_registered}` """ @spec get(atom(), any(), non_neg_integer(), Keyword.t()) :: result - def get(cache, key, timeout \\ 1_000, _opts \\ []) + def get(hydrating_cache, key, timeout \\ 1_000, _opts \\ []) when is_integer(timeout) and timeout > 0 do Logger.debug( - "Getting key #{inspect(key)} from hydrating cache #{inspect(cache)} for #{inspect(self())}" + "Getting key #{inspect(key)} from hydrating cache #{inspect(hydrating_cache)} for #{inspect(self())}" ) - case GenServer.call(cache, {:get, key}, timeout) do + case GenServer.call(hydrating_cache, {:get, key}, timeout) do {:ok, :answer_delayed} -> receive do {:ok, value} -> {:ok, value} other -> - Logger.info("Unexpected return value #{inspect(other)}") + Logger.warning("Unexpected return value #{inspect(other)}") {:error, :unexpected_value} after timeout -> - Logger.warn( - "Timeout waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(cache)}" + Logger.warning( + "Timeout waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(hydrating_cache)}" ) {:error, :timeout} @@ -120,10 +122,10 @@ defmodule Archethic.Utils.HydratingCache do initial_keys_worker_sup, keys, fn - {provider, mod, func, params, refresh_rate} -> + {provider, mod, func, params, refresh_rate, ttl} -> GenServer.cast( me, - {:register, fn -> apply(mod, func, params) end, provider, 75_000, refresh_rate} + {:register, fn -> apply(mod, func, params) end, provider, refresh_rate, ttl} ) other -> @@ -153,7 +155,7 @@ defmodule Archethic.Utils.HydratingCache do end end - def handle_call({:register, fun, key, ttl, refresh_interval}, _from, state) do + def handle_call({:register, fun, key, refresh_interval, ttl}, _from, state) do Logger.debug("Registering hydrating function for #{inspect(key)}") ## Called when asked to register a function case Map.get(state, key) do @@ -162,14 +164,14 @@ defmodule Archethic.Utils.HydratingCache do {:ok, pid} = DynamicSupervisor.start_child( state.keys_sup, - {CacheEntry, [fun, key, ttl, refresh_interval]} + {CacheEntry, [fun, key, refresh_interval, ttl]} ) {:reply, :ok, Map.put(state, key, pid)} pid -> ## Key already exists, no need to start fsm - case :gen_statem.call(pid, {:register, fun, key, ttl, refresh_interval}) do + case :gen_statem.call(pid, {:register, fun, key, refresh_interval, ttl}) do :ok -> {:reply, :ok, Map.put(state, key, pid)} @@ -197,7 +199,7 @@ defmodule Archethic.Utils.HydratingCache do end end - def handle_cast({:register, fun, key, ttl, refresh_interval}, state) do + def handle_cast({:register, fun, key, refresh_interval, ttl}, state) do case Map.get(state, key) do nil -> ## New key, we start a cache entry fsm @@ -205,14 +207,14 @@ defmodule Archethic.Utils.HydratingCache do {:ok, pid} = DynamicSupervisor.start_child( state.keys_sup, - {CacheEntry, [fun, key, ttl, refresh_interval]} + {CacheEntry, [fun, key, refresh_interval, ttl]} ) {:noreply, Map.put(state, key, pid)} pid -> ## Key already exists, no need to start fsm - _ = :gen_statem.call(pid, {:register, fun, key, ttl, refresh_interval}) + _ = :gen_statem.call(pid, {:register, fun, key, refresh_interval, ttl}) {:noreply, Map.put(state, key, pid)} end end diff --git a/test/archethic/utils/hydrating_cache/caches_manager_test.exs b/test/archethic/utils/hydrating_cache/caches_manager_test.exs index 4c0e85094..60fcc8a14 100644 --- a/test/archethic/utils/hydrating_cache/caches_manager_test.exs +++ b/test/archethic/utils/hydrating_cache/caches_manager_test.exs @@ -6,9 +6,9 @@ defmodule HydratingCacheTest do test "starting service from manager returns value once first hydrating have been done" do CachesManager.new_service_async("test_services", [ - {:key1, __MODULE__, :waiting_function, [2000], 6000}, - {:key2, __MODULE__, :waiting_function, [1000], 6000}, - {:key3, __MODULE__, :waiting_function, [2000], 6000} + {:key1, __MODULE__, :waiting_function, [2000], 6000, 8000}, + {:key2, __MODULE__, :waiting_function, [1000], 6000, 8000}, + {:key3, __MODULE__, :waiting_function, [2000], 6000, 8000} ]) ## wait a little so at least keys are registered diff --git a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs index 1d4e59188..a080963ef 100644 --- a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs +++ b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs @@ -18,8 +18,8 @@ defmodule HydratingCacheTest do {:ok, 1} end, "simple_func", - 50_000, - 10_000 + 10_000, + 15_000 ) assert result == :ok @@ -29,55 +29,10 @@ defmodule HydratingCacheTest do assert r == {:ok, 1} end - test "Getting value for key while function is running first time make process wait and return value" do - {:ok, pid} = HydratingCache.start_link(:test_service) - - result = - HydratingCache.register_function( - pid, - fn -> - Logger.info("Hydrating function Sleeping 3 secs") - :timer.sleep(3000) - {:ok, 1} - end, - "test_long_function", - 50_000, - 9000 - ) - - assert result == :ok - - r = HydratingCache.get(pid, "test_long_function", 10_000) - assert r == {:ok, 1} - end - - test "Getting value for key while function is running first time returns timeout after ttl" do - {:ok, pid} = HydratingCache.start_link(:test_service) - - result = - HydratingCache.register_function( - pid, - fn -> - Logger.info("Hydrating function Sleeping 3 secs") - :timer.sleep(3000) - {:ok, 1} - end, - "test_get_ttl", - 50_000, - 9000 - ) - - assert result == :ok - - ## get and wait up to 1 second - r = HydratingCache.get(pid, "test_get_ttl", 1000) - assert r == {:error, :timeout} - end - test "Hydrating function runs periodically" do {:ok, pid} = HydratingCache.start_link(:test_service) - :persistent_term.put("test", 0) + :persistent_term.put("test", 1) result = HydratingCache.register_function( @@ -90,16 +45,16 @@ defmodule HydratingCacheTest do {:ok, value} end, "test_inc", - 50000, - 1000 + 1_000, + 50_000 ) assert result == :ok - :timer.sleep(5000) + :timer.sleep(3000) {:ok, value} = HydratingCache.get(pid, "test_inc", 3000) - assert value >= 5 + assert value >= 3 end test "Update hydrating function while another one is running returns new hydrating value from new function" do @@ -113,8 +68,8 @@ defmodule HydratingCacheTest do {:ok, 1} end, "test_reregister", - 50000, - 10000 + 10_000, + 50_000 ) assert result == :ok @@ -126,8 +81,8 @@ defmodule HydratingCacheTest do {:ok, 2} end, "test_reregister", - 50000, - 10000 + 10_000, + 50_000 ) :timer.sleep(5000) @@ -146,8 +101,8 @@ defmodule HydratingCacheTest do {:ok, 1} end, "test_reregister", - 50000, - 10000 + 10_000, + 50_000 ) _ = @@ -158,8 +113,8 @@ defmodule HydratingCacheTest do {:ok, 2} end, "test_reregister", - 50000, - 10000 + 10_000, + 50_000 ) {:ok, value} = HydratingCache.get(pid, "test_reregister", 4000) @@ -178,8 +133,8 @@ defmodule HydratingCacheTest do {:ok, :result_timed} end, "timed_value", - 80000, - 70000 + 70_000, + 80_000 ) _ = @@ -189,8 +144,8 @@ defmodule HydratingCacheTest do {:ok, :result} end, "direct_value", - 80000, - 70000 + 70_000, + 80_000 ) ## We query the value with timeout smaller than timed function @@ -208,8 +163,8 @@ defmodule HydratingCacheTest do {:ok, :valid_result} end, "delayed_result", - 80000, - 70000 + 70_000, + 80_000 ) ## We query the value with timeout smaller than timed function @@ -227,13 +182,12 @@ defmodule HydratingCacheTest do {:ok, :valid_result} end, "delayed_result", - 80000, - 70000 + 70_000, + 80_000 ) ## We query the value with timeout smaller than timed function - result = HydratingCache.get(pid, "delayed_result", 1000) - assert result = {:error, :timeout} + assert {:error, :timeout} = HydratingCache.get(pid, "delayed_result", 1000) end test "Multiple process can wait for a delayed value" do @@ -249,8 +203,8 @@ defmodule HydratingCacheTest do {:ok, :valid_result} end, "delayed_result", - 80000, - 70000 + 70_000, + 80_000 ) ## We query the value with timeout smaller than timed function @@ -276,8 +230,8 @@ defmodule HydratingCacheTest do {:ok, :badmatch} end, :key, - 80000, - 70000 + 70_000, + 80_000 ) :timer.sleep(1000) @@ -290,11 +244,40 @@ defmodule HydratingCacheTest do {:ok, :value} end, :key, - 80000, - 70000 + 70_000, + 80_000 ) result = HydratingCache.get(pid, :key, 3000) assert result == {:ok, :value} end + + ## This could occur if hydrating function takes time to answer. + ## In this case, getting the value would return the old value, unless too + ## much time occur where it would be discarded because of ttl + test "value gets discarded after some time" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + _ = + HydratingCache.register_function( + pid, + fn -> + case :persistent_term.get("flag", nil) do + 1 -> + :timer.sleep(3_000) + + nil -> + :persistent_term.put("flag", 1) + {:ok, :value} + end + end, + :key, + 500, + 1_000 + ) + + :timer.sleep(1_100) + result = HydratingCache.get(pid, :key, 3000) + assert result == {:error, :discarded} + end end From 4df5ba5f4136ab649f115177034326cc1ded28be Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 11:15:28 +0100 Subject: [PATCH 21/48] Using prior value if no new value for a curreny ( fixes #836 ) --- lib/archethic/oracle_chain/services/uco_price.ex | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index e716741c8..9a0d9dab4 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -83,7 +83,7 @@ defmodule Archethic.OracleChain.Services.UCOPrice do previous_values ++ values end) - {_currency, _values}, acc -> + _, acc -> acc end) end @@ -98,7 +98,10 @@ defmodule Archethic.OracleChain.Services.UCOPrice do {:ok, prices_now} -> Enum.all?(@pairs, fn pair -> - compare_price(Map.fetch!(prices_prior, pair), Map.fetch!(prices_now, pair)) + compare_price( + Map.fetch!(prices_prior, pair), + Map.get(prices_now, pair, Map.fetch!(prices_prior, pair)) + ) end) end end From bfe85bddc7754b6c8dc5b901abdc49e9aefc5b9e Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 15:56:12 +0100 Subject: [PATCH 22/48] Fixing TTL bug --- .../utils/hydrating_cache/cache_entry.ex | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 9c7f144a6..53ef9fe92 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -39,7 +39,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do @impl :gen_statem def init([fun, key, refresh_interval, ttl]) do # start hydrating at the needed refresh interval - timer = :timer.send_interval(refresh_interval, self(), :hydrate) + {:ok, timer} = :timer.send_interval(refresh_interval, self(), :hydrate) ## Hydrate the value {:ok, :running, @@ -123,11 +123,11 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end ## And the timers triggering it and discarding value - _ = :timer.cancel(data.timer_func) - _ = :timer.cancel(data.timer_discard) + _ = maybe_stop_timer(data.timer_func) + _ = maybe_stop_timer(data.timer_discard) ## Start new timer to hydrate at refresh interval - timer = :timer.send_interval(refresh_interval, self(), :hydrate) + {:ok, timer} = :timer.send_interval(refresh_interval, self(), :hydrate) ## We trigger the update ( to trigger or not could be set at registering option ) {:repeat_state, @@ -144,8 +144,8 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, _state, data) do ## Setting hydrating function in other cases ## Hydrating function not running, we just stop the timers - _ = :timer.cancel(data.timer_func) - _ = :timer.cancel(data.timer_discard) + _ = maybe_stop_timer(data.timer_func) + _ = maybe_stop_timer(data.timer_discard) ## Fill state with new hydrating function parameters data = @@ -155,15 +155,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do :refresh_interval => refresh_interval }) - timer = - case ttl do - :infinity -> - nil - - value when is_number(value) -> - {:ok, t} = :timer.send_interval(refresh_interval, self(), :hydrate) - t - end + {:ok, timer} = :timer.send_interval(refresh_interval, self(), :hydrate) ## We trigger the update ( to trigger or not could be set at registering option ) {:next_state, :running, @@ -211,7 +203,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do def handle_event(:cast, {:new_value, _key, {:ok, value}}, :running, data) do ## Stop timer on value ttl - _ = :timer.cancel(data.timer_discard) + _ = maybe_stop_timer(data.timer_discard) ## We got result from hydrating function @@ -220,10 +212,18 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do send(pid, {:ok, value}) end) - ## Start timer to discard new value - + ## Start timer to discard new value if needed me = self() - {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) + + new_timer = + case data.ttl do + value when is_number(value) -> + {:ok, t} = :timer.send_after(value, me, :discarded) + t + + _ -> + :ok + end {:next_state, :idle, %CacheEntry.StateData{ @@ -258,4 +258,12 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do def handle_event(_type, _event, _state, data) do {:keep_state, data} end + + defp maybe_stop_timer(ref = {_, _}) do + :timer.cancel(ref) + end + + defp maybe_stop_timer(_ref) do + :ok + end end From 2a8508f87d84eb0a711d3cfdea82f098b63f4573 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 18:34:24 +0100 Subject: [PATCH 23/48] Fixing legacy tests + bugfixes --- .../utils/hydrating_cache/cache_entry.ex | 11 +- .../utils/hydrating_cache/hydrating_cache.ex | 10 +- .../archethic/oracle_chain/scheduler_test.exs | 58 ++--- .../oracle_chain/services/uco_price_test.exs | 237 +++++++----------- test/archethic/oracle_chain/services_test.exs | 96 +++---- 5 files changed, 167 insertions(+), 245 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 53ef9fe92..8d112f8aa 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -243,7 +243,16 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ## We reprogram the timer to hydrate, even if previous call failled. Error control could occur here me = self() - {:ok, new_timer} = :timer.send_after(data.ttl, me, :discarded) + + new_timer = + case data.ttl do + value when is_number(value) -> + {:ok, t} = :timer.send_after(value, me, :discarded) + t + + _ -> + :ok + end {:next_state, :idle, %CacheEntry.StateData{ diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index 63a29ca79..842b2e288 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -46,7 +46,7 @@ defmodule Archethic.Utils.HydratingCache do key :: any, refresh_interval :: non_neg_integer(), ttl :: non_neg_integer() - ) :: :ok + ) :: term() def register_function(hydrating_cache, fun, key, refresh_interval, ttl) when is_function(fun, 0) and is_integer(ttl) and ttl > 0 and is_integer(refresh_interval) and @@ -70,8 +70,8 @@ defmodule Archethic.Utils.HydratingCache do - If `key` is not associated with any function, return `{:error, :not_registered}` """ - @spec get(atom(), any(), non_neg_integer(), Keyword.t()) :: result - def get(hydrating_cache, key, timeout \\ 1_000, _opts \\ []) + @spec get(atom(), any(), non_neg_integer()) :: {:ok, term()} | {:error, atom()} + def get(hydrating_cache, key, timeout \\ 1_000) when is_integer(timeout) and timeout > 0 do Logger.debug( "Getting key #{inspect(key)} from hydrating cache #{inspect(hydrating_cache)} for #{inspect(self())}" @@ -102,7 +102,7 @@ defmodule Archethic.Utils.HydratingCache do @impl GenServer def init([name, keys]) do - Logger.info("Starting Hydrating cache for service #{inspect(name)}") + Logger.info("Starting Hydrating cache for service #{inspect("#{__MODULE__}.#{name}")}") ## start a dynamic supervisor for the cache entries/keys {:ok, keys_sup} = @@ -146,7 +146,7 @@ defmodule Archethic.Utils.HydratingCache do def handle_call({:get, key}, from, state) do case Map.get(state, key, :undefined) do :undefined -> - Logger.warning("HydratingCache no entry for #{inspect(state)}") + Logger.warning("HydratingCache no entry for #{inspect(key)}") {:reply, {:error, :not_registered}, state} pid -> diff --git a/test/archethic/oracle_chain/scheduler_test.exs b/test/archethic/oracle_chain/scheduler_test.exs index 004bc5a0c..b784525a3 100644 --- a/test/archethic/oracle_chain/scheduler_test.exs +++ b/test/archethic/oracle_chain/scheduler_test.exs @@ -18,6 +18,8 @@ defmodule Archethic.OracleChain.SchedulerTest do alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData + alias Archethic.Utils.HydratingCache + import ArchethicCase, only: [setup_before_send_tx: 0] import Mox @@ -107,14 +109,11 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, _} = :sys.get_state(pid) - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity} + ]) # polling_date = # "0 * * * *" @@ -185,14 +184,11 @@ defmodule Archethic.OracleChain.SchedulerTest do }} end) - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity} + ]) send(pid, :poll) @@ -223,14 +219,11 @@ defmodule Archethic.OracleChain.SchedulerTest do available?: true }) - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity} + ]) summary_date = "0 0 0 * *" @@ -332,14 +325,11 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, %{polling_timer: timer1}} = :sys.get_state(pid) - MockUCOPriceProvider1 - |> stub(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) - - MockUCOPriceProvider2 - |> stub(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) - - MockUCOPriceProvider3 - |> stub(:fetch, fn _pairs -> {:ok, %{"usd" => [0.2]}} end) + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity} + ]) send(pid, :poll) @@ -465,4 +455,8 @@ defmodule Archethic.OracleChain.SchedulerTest do :persistent_term.put(:archethic_up, nil) end end + + def fetch(values) do + values + end end diff --git a/test/archethic/oracle_chain/services/uco_price_test.exs b/test/archethic/oracle_chain/services/uco_price_test.exs index e57d916a1..59964c112 100644 --- a/test/archethic/oracle_chain/services/uco_price_test.exs +++ b/test/archethic/oracle_chain/services/uco_price_test.exs @@ -3,119 +3,78 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do alias Archethic.OracleChain.Services.UCOPrice + alias Archethic.Utils.HydratingCache + import Mox test "fetch/0 should retrieve some data and build a map with the oracle name in it" do - MockUCOPriceProvider1 - |> expect(:fetch, fn pairs -> - res = - Enum.map(pairs, fn pair -> - {pair, [:rand.uniform_real()]} - end) - |> Enum.into(%{}) - - {:ok, res} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn pairs -> - res = - Enum.map(pairs, fn pair -> - {pair, [:rand.uniform_real()]} - end) - |> Enum.into(%{}) - - {:ok, res} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn pairs -> - res = - Enum.map(pairs, fn pair -> - {pair, [:rand.uniform_real()]} - end) - |> Enum.into(%{}) - - {:ok, res} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity} + ]) assert {:ok, %{"eur" => _, "usd" => _}} = UCOPrice.fetch() end test "fetch/0 should retrieve some data and build a map with the oracle name in it and keep the precision to 5" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30000, :infinity} + ]) assert {:ok, %{"eur" => 0.12346, "usd" => 0.12345}} = UCOPrice.fetch() end describe "verify/1" do test "should return true if the prices are the good one" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.11]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.30, 0.40], "usd" => [0.12, 0.13]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.50], "usd" => [0.14]}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.20], "usd" => [0.11]}}], 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.30, 0.40], "usd" => [0.12, 0.13]}}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.50], "usd" => [0.14]}}], 30000, :infinity} + ]) assert {:ok, %{"eur" => 0.35, "usd" => 0.125}} == UCOPrice.fetch() end test "should return false if the prices have deviated" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity} + ]) assert false == UCOPrice.verify?(%{"eur" => 0.10, "usd" => 0.14}) end end test "should return the median value when multiple providers queried" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.30], "usd" => [0.12]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.40], "usd" => [0.12]}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.30], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.40], "usd" => [0.12]}}], + 30000, :infinity} + ]) assert true == UCOPrice.verify?(%{"eur" => 0.30, "usd" => 0.12}) end @@ -140,26 +99,17 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do Application.put_env(:archethic, Archethic.OracleChain.Services.UCOPrice, new_uco_env) - ## Define mocks expectations - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.30], "usd" => [0.12]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.40], "usd" => [0.12]}} - end) - - MockUCOPriceProvider4 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.50], "usd" => [0.12]}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.30], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.40], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.12]}}], + 30000, :infinity} + ]) ## Restore original environment Application.put_env(:archethic, Archethic.OracleChain.Services.UCOPrice, old_env) @@ -168,59 +118,46 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do end test "verify?/1 should return false when no data are returned from all providers" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [], "usd" => []}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [], "usd" => []}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [], "usd" => []}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30000, + :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30000, + :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30000, + :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30000, + :infinity} + ]) assert false == UCOPrice.verify?(%{}) end test "should report values even if a provider returns an error" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.50], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:error, :error_message} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.60], "usd" => [0.12]}} - end) + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:error, :error_message}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.60], "usd" => [0.12]}}], + 30000, :infinity} + ]) assert {:ok, %{"eur" => 0.55, "usd" => 0.12}} = UCOPrice.fetch() end test "should handle a service timing out" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.50], "usd" => [0.10]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - :timer.sleep(5_000) - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.50], "usd" => [0.10]}} - end) + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.10]}}], + 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [:timer.sleep(5_000)], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.10]}}], + 30000, :infinity} + ]) assert true == UCOPrice.verify?(%{"eur" => 0.50, "usd" => 0.10}) end + + def fetch(values) do + values + end end diff --git a/test/archethic/oracle_chain/services_test.exs b/test/archethic/oracle_chain/services_test.exs index 00e960253..36e891c7d 100644 --- a/test/archethic/oracle_chain/services_test.exs +++ b/test/archethic/oracle_chain/services_test.exs @@ -2,63 +2,46 @@ defmodule Archethic.OracleChain.ServicesTest do use ExUnit.Case alias Archethic.OracleChain.Services - + alias Archethic.Utils.HydratingCache import Mox describe "fetch_new_data/1" do test "should return the new data when no previous content" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity} + ]) assert %{uco: %{"eur" => 0.20, "usd" => 0.12}} = Services.fetch_new_data() end test "should not return the new data when the previous content is the same" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity} + ]) assert %{} = Services.fetch_new_data(%{uco: %{"eur" => 0.20, "usd" => 0.12}}) end test "should return the new data when the previous content is not the same" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity} + ]) assert %{uco: %{"eur" => 0.20, "usd" => 0.12}} = Services.fetch_new_data(%{"uco" => %{"eur" => 0.19, "usd" => 0.15}}) @@ -66,21 +49,20 @@ defmodule Archethic.OracleChain.ServicesTest do end test "verify_correctness?/1 should true when the data is correct" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30000, :infinity} + ]) assert true == Services.verify_correctness?(%{"uco" => %{"eur" => 0.20, "usd" => 0.12}}) end + + def fetch(values) do + values + end end From 6f31aee8d9cc285b303bf9b2586a81063f4b3b04 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 18:42:20 +0100 Subject: [PATCH 24/48] Fixing credo warnings --- test/archethic/oracle_chain/services_test.exs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/archethic/oracle_chain/services_test.exs b/test/archethic/oracle_chain/services_test.exs index 36e891c7d..bad29eebf 100644 --- a/test/archethic/oracle_chain/services_test.exs +++ b/test/archethic/oracle_chain/services_test.exs @@ -10,11 +10,11 @@ defmodule Archethic.OracleChain.ServicesTest do _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity}, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity}, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity} + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity} ]) assert %{uco: %{"eur" => 0.20, "usd" => 0.12}} = Services.fetch_new_data() @@ -23,11 +23,11 @@ defmodule Archethic.OracleChain.ServicesTest do test "should not return the new data when the previous content is the same" do HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity} + 30_000, :infinity} ]) assert %{} = Services.fetch_new_data(%{uco: %{"eur" => 0.20, "usd" => 0.12}}) @@ -36,11 +36,11 @@ defmodule Archethic.OracleChain.ServicesTest do test "should return the new data when the previous content is not the same" do HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity} + 30_000, :infinity} ]) assert %{uco: %{"eur" => 0.20, "usd" => 0.12}} = @@ -52,11 +52,11 @@ defmodule Archethic.OracleChain.ServicesTest do _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity} + 30_000, :infinity} ]) assert true == Services.verify_correctness?(%{"uco" => %{"eur" => 0.20, "usd" => 0.12}}) From c1271cfc367226fd79a48ea1264f7bde67318744 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 18:48:19 +0100 Subject: [PATCH 25/48] Fixing Credo second round --- config/dev.exs | 6 +- .../archethic/oracle_chain/scheduler_test.exs | 32 +++++--- .../oracle_chain/services/uco_price_test.exs | 73 ++++++++----------- 3 files changed, 52 insertions(+), 59 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index 300f80aed..afd48de25 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -83,13 +83,13 @@ config :archethic, Archethic.Utils.HydratingCache.CachesManager, uco_service: [ {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, [["usd", "eur"]], - 30000, :infinity}, + 30_000, :infinity}, {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], 30000, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], 30_000, :infinity}, {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, [["usd", "eur"]], - 30000, :infinity} + 30_000, :infinity} ] config :archethic, Archethic.Governance.Pools, diff --git a/test/archethic/oracle_chain/scheduler_test.exs b/test/archethic/oracle_chain/scheduler_test.exs index b784525a3..9251e0284 100644 --- a/test/archethic/oracle_chain/scheduler_test.exs +++ b/test/archethic/oracle_chain/scheduler_test.exs @@ -110,9 +110,11 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, _} = :sys.get_state(pid) HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity} + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, + :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, + :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, :infinity} ]) # polling_date = @@ -185,9 +187,11 @@ defmodule Archethic.OracleChain.SchedulerTest do end) HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity} + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, + :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, + :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, :infinity} ]) send(pid, :poll) @@ -220,9 +224,11 @@ defmodule Archethic.OracleChain.SchedulerTest do }) HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity} + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, + :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, + :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, :infinity} ]) summary_date = @@ -326,9 +332,11 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, %{polling_timer: timer1}} = :sys.get_state(pid) HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30000, :infinity} + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, + :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, + :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, :infinity} ]) send(pid, :poll) diff --git a/test/archethic/oracle_chain/services/uco_price_test.exs b/test/archethic/oracle_chain/services/uco_price_test.exs index 59964c112..05e00871a 100644 --- a/test/archethic/oracle_chain/services/uco_price_test.exs +++ b/test/archethic/oracle_chain/services/uco_price_test.exs @@ -11,11 +11,11 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity} + 30_000, :infinity} ]) assert {:ok, %{"eur" => _, "usd" => _}} = UCOPrice.fetch() @@ -25,11 +25,11 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30000, :infinity}, + [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30000, :infinity}, + [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30000, :infinity} + [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30_000, :infinity} ]) assert {:ok, %{"eur" => 0.12346, "usd" => 0.12345}} = UCOPrice.fetch() @@ -40,11 +40,11 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.11]}}], 30000, :infinity}, + [{:ok, %{"eur" => [0.20], "usd" => [0.11]}}], 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.30, 0.40], "usd" => [0.12, 0.13]}}], 30000, :infinity}, + [{:ok, %{"eur" => [0.30, 0.40], "usd" => [0.12, 0.13]}}], 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.50], "usd" => [0.14]}}], 30000, :infinity} + [{:ok, %{"eur" => [0.50], "usd" => [0.14]}}], 30_000, :infinity} ]) assert {:ok, %{"eur" => 0.35, "usd" => 0.125}} == UCOPrice.fetch() @@ -54,11 +54,11 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity}, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity}, + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30000, :infinity} + [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity} ]) assert false == UCOPrice.verify?(%{"eur" => 0.10, "usd" => 0.14}) @@ -69,11 +69,11 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.30], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.40], "usd" => [0.12]}}], - 30000, :infinity} + 30_000, :infinity} ]) assert true == UCOPrice.verify?(%{"eur" => 0.30, "usd" => 0.12}) @@ -85,48 +85,33 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl ) - ## Backup old environment variable, and update it with fourth provider - old_env = Application.get_env(:archethic, Archethic.OracleChain.Services.UCOPrice) - - new_uco_env = - old_env - |> Keyword.replace(:providers, [ - MockUCOPriceProvider1, - MockUCOPriceProvider2, - MockUCOPriceProvider3, - MockUCOPriceProvider4 - ]) - Application.put_env(:archethic, Archethic.OracleChain.Services.UCOPrice, new_uco_env) _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.30], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.40], "usd" => [0.12]}}], - 30000, :infinity}, + 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.12]}}], - 30000, :infinity} + 30_000, :infinity} ]) - ## Restore original environment - Application.put_env(:archethic, Archethic.OracleChain.Services.UCOPrice, old_env) - assert false == UCOPrice.verify?(%{"eur" => 0.35, "usd" => 0.12}) end test "verify?/1 should return false when no data are returned from all providers" do _ = HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30000, + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30000, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30000, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30000, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30_000, :infinity} ]) @@ -136,10 +121,10 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should report values even if a provider returns an error" do HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.12]}}], - 30000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:error, :error_message}], 30000, :infinity}, + 30_000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:error, :error_message}], 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.60], "usd" => [0.12]}}], - 30000, :infinity} + 30_000, :infinity} ]) assert {:ok, %{"eur" => 0.55, "usd" => 0.12}} = UCOPrice.fetch() @@ -148,10 +133,10 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should handle a service timing out" do HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.10]}}], - 30000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [:timer.sleep(5_000)], 30000, :infinity}, + 30_000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [:timer.sleep(5_000)], 30_000, :infinity}, {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.10]}}], - 30000, :infinity} + 30_000, :infinity} ]) assert true == UCOPrice.verify?(%{"eur" => 0.50, "usd" => 0.10}) From fb07570abb406cba07785bf12998613f624c3d2d Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 18:50:43 +0100 Subject: [PATCH 26/48] Fixing formatting --- config/dev.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index afd48de25..cc2e2fa7e 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -85,8 +85,8 @@ config :archethic, Archethic.Utils.HydratingCache.CachesManager, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, [["usd", "eur"]], 30_000, :infinity}, {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], 30_000, - :infinity}, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], + 30_000, :infinity}, {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, [["usd", "eur"]], 30_000, :infinity} From 8527cce26d29ee3bcb375cc7f648571a6927a770 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 18:58:34 +0100 Subject: [PATCH 27/48] Fixing @impl syntax --- lib/archethic/utils/hydrating_cache/caches_manager.ex | 6 +++--- lib/archethic/utils/hydrating_cache/hydrating_cache.ex | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/caches_manager.ex b/lib/archethic/utils/hydrating_cache/caches_manager.ex index 0c2ba3d01..7fb813a20 100644 --- a/lib/archethic/utils/hydrating_cache/caches_manager.ex +++ b/lib/archethic/utils/hydrating_cache/caches_manager.ex @@ -41,7 +41,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do GenServer.call(__MODULE__, {:end_service, name}) end - @impl true + @impl GenServer def init(_args) do manager_conf = Application.get_env(:archethic, __MODULE__, []) @@ -63,7 +63,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do {:ok, %{:caches_sup => caches_sup}} end - @impl true + @impl GenServer def handle_call({:new_service_sync, name, initial_keys}, _from, state) do Logger.info("Starting new service sync : #{name}") @@ -76,7 +76,7 @@ defmodule Archethic.Utils.HydratingCache.CachesManager do {:reply, {:ok, pid}, state} end - @impl true + @impl GenServer def handle_cast({:new_service_async, name, keys, _requester}, state) do Logger.info("Starting new service async : #{name}") diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index 842b2e288..8a391d639 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -141,8 +141,7 @@ defmodule Archethic.Utils.HydratingCache do {:ok, %{:keys => keys, keys_sup: keys_sup}} end - @impl true - + @impl GenServer def handle_call({:get, key}, from, state) do case Map.get(state, key, :undefined) do :undefined -> @@ -186,7 +185,7 @@ defmodule Archethic.Utils.HydratingCache do {:reply, :ok, state} end - @impl true + @impl GenServer def handle_cast({:get, from, key}, state) do case Map.get(state, key, :undefined) do :undefined -> @@ -224,7 +223,7 @@ defmodule Archethic.Utils.HydratingCache do {:noreply, state} end - @impl true + @impl GenServer def handle_info(unmanaged, state) do Logger.warning("Cache received unmanaged info: #{inspect(unmanaged)}") {:noreply, state} From 52bc9ffaf82a274699da3f547432e65d22678d47 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 19:03:01 +0100 Subject: [PATCH 28/48] Fixing unref variable... --- test/archethic/oracle_chain/services/uco_price_test.exs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/archethic/oracle_chain/services/uco_price_test.exs b/test/archethic/oracle_chain/services/uco_price_test.exs index 05e00871a..25db48fa3 100644 --- a/test/archethic/oracle_chain/services/uco_price_test.exs +++ b/test/archethic/oracle_chain/services/uco_price_test.exs @@ -5,8 +5,6 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do alias Archethic.Utils.HydratingCache - import Mox - test "fetch/0 should retrieve some data and build a map with the oracle name in it" do _ = HydratingCache.start_link(:uco_service, [ @@ -84,9 +82,6 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do Mox.defmock(MockUCOPriceProvider4, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl ) - - Application.put_env(:archethic, Archethic.OracleChain.Services.UCOPrice, new_uco_env) - _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], From f1095610a449fa1f6da16ad9fe3ff0b1ca6539c1 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 19:06:03 +0100 Subject: [PATCH 29/48] Formatting --- test/archethic/oracle_chain/services/uco_price_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/archethic/oracle_chain/services/uco_price_test.exs b/test/archethic/oracle_chain/services/uco_price_test.exs index 25db48fa3..913dae276 100644 --- a/test/archethic/oracle_chain/services/uco_price_test.exs +++ b/test/archethic/oracle_chain/services/uco_price_test.exs @@ -82,6 +82,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do Mox.defmock(MockUCOPriceProvider4, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl ) + _ = HydratingCache.start_link(:uco_service, [ {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], From 448917c4fd2a81b6b0028d7d5f9ea1d442d63026 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 19:10:02 +0100 Subject: [PATCH 30/48] Fixing incorrect module name --- test/archethic/utils/hydrating_cache/caches_manager_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/archethic/utils/hydrating_cache/caches_manager_test.exs b/test/archethic/utils/hydrating_cache/caches_manager_test.exs index 60fcc8a14..2d560d509 100644 --- a/test/archethic/utils/hydrating_cache/caches_manager_test.exs +++ b/test/archethic/utils/hydrating_cache/caches_manager_test.exs @@ -1,4 +1,4 @@ -defmodule HydratingCacheTest do +defmodule CachesManagerTest do alias Archethic.Utils.HydratingCache alias Archethic.Utils.HydratingCache.CachesManager use ExUnit.Case From e3a2084b9f00d71a36d73bccd685a0a267ae8ae2 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 19:23:22 +0100 Subject: [PATCH 31/48] Fixing legacy tests --- test/archethic/oracle_chain_test.exs | 28 +++++++++---------- .../hydrating_cache/caches_manager_test.exs | 11 ++++---- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/test/archethic/oracle_chain_test.exs b/test/archethic/oracle_chain_test.exs index 71d9a1211..b56f83d22 100644 --- a/test/archethic/oracle_chain_test.exs +++ b/test/archethic/oracle_chain_test.exs @@ -6,24 +6,20 @@ defmodule Archethic.OracleChainTest do alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData + alias Archethic.Utils.HydratingCache import Mox test "valid_services_content?/1 should verify the oracle transaction's content correctness" do - MockUCOPriceProvider1 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider2 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) - - MockUCOPriceProvider3 - |> expect(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.20], "usd" => [0.12]}} - end) + _ = + HydratingCache.start_link(:uco_service, [ + {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30_000, :infinity}, + {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30_000, :infinity}, + {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], + 30_000, :infinity} + ]) content = %{ @@ -63,4 +59,8 @@ defmodule Archethic.OracleChainTest do assert true == OracleChain.valid_summary?(content, chain) end + + def fetch(values) do + values + end end diff --git a/test/archethic/utils/hydrating_cache/caches_manager_test.exs b/test/archethic/utils/hydrating_cache/caches_manager_test.exs index 2d560d509..573910203 100644 --- a/test/archethic/utils/hydrating_cache/caches_manager_test.exs +++ b/test/archethic/utils/hydrating_cache/caches_manager_test.exs @@ -6,9 +6,9 @@ defmodule CachesManagerTest do test "starting service from manager returns value once first hydrating have been done" do CachesManager.new_service_async("test_services", [ - {:key1, __MODULE__, :waiting_function, [2000], 6000, 8000}, - {:key2, __MODULE__, :waiting_function, [1000], 6000, 8000}, - {:key3, __MODULE__, :waiting_function, [2000], 6000, 8000} + {:key1, __MODULE__, :fetch, [2000], 6000, 8000}, + {:key2, __MODULE__, :fetch, [1000], 6000, 8000}, + {:key3, __MODULE__, :fetch, [2000], 6000, 8000} ]) ## wait a little so at least keys are registered @@ -21,8 +21,7 @@ defmodule CachesManagerTest do ) == {:ok, 1} end - def waiting_function(delay \\ 1000) do - :timer.sleep(delay) - {:ok, 1} + def fetch(values) do + values end end From 6f33f1dea7ce001ad6d64e530225b476f9abd69f Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 23 Jan 2023 19:33:19 +0100 Subject: [PATCH 32/48] Reverting changes on cache manager --- .../utils/hydrating_cache/caches_manager_test.exs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/archethic/utils/hydrating_cache/caches_manager_test.exs b/test/archethic/utils/hydrating_cache/caches_manager_test.exs index 573910203..4dd651366 100644 --- a/test/archethic/utils/hydrating_cache/caches_manager_test.exs +++ b/test/archethic/utils/hydrating_cache/caches_manager_test.exs @@ -6,9 +6,9 @@ defmodule CachesManagerTest do test "starting service from manager returns value once first hydrating have been done" do CachesManager.new_service_async("test_services", [ - {:key1, __MODULE__, :fetch, [2000], 6000, 8000}, - {:key2, __MODULE__, :fetch, [1000], 6000, 8000}, - {:key3, __MODULE__, :fetch, [2000], 6000, 8000} + {:key1, __MODULE__, :waiting_function, [2000], 6000, 8000}, + {:key2, __MODULE__, :waiting_function, [1000], 6000, 8000}, + {:key3, __MODULE__, :waiting_function, [2000], 6000, 8000} ]) ## wait a little so at least keys are registered @@ -21,7 +21,7 @@ defmodule CachesManagerTest do ) == {:ok, 1} end - def fetch(values) do - values + def waiting_function(delay \\ 1000) do + {:ok, 1} end end From 62a86e807786e8386796e2ad86ec5466bf01fc11 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Tue, 31 Jan 2023 17:26:45 +0100 Subject: [PATCH 33/48] Removing cache manager, better hydrating cache name management --- config/config.exs | 9 +- config/dev.exs | 15 - config/test.exs | 3 - .../oracle_chain/services/uco_price.ex | 10 +- lib/archethic/oracle_chain/supervisor.ex | 18 +- .../utils/hydrating_cache/cache_entry.ex | 9 +- .../utils/hydrating_cache/caches_manager.ex | 96 ----- .../utils/hydrating_cache/hydrating_cache.ex | 13 +- .../archethic/oracle_chain/scheduler_test.exs | 120 +++++-- .../oracle_chain/services/uco_price_test.exs | 338 +++++++++++++----- test/archethic/oracle_chain/services_test.exs | 126 +++++-- test/archethic/oracle_chain_test.exs | 32 +- .../hydrating_cache/caches_manager_test.exs | 27 -- test/test_helper.exs | 4 + 14 files changed, 493 insertions(+), 327 deletions(-) delete mode 100644 lib/archethic/utils/hydrating_cache/caches_manager.ex delete mode 100644 test/archethic/utils/hydrating_cache/caches_manager_test.exs diff --git a/config/config.exs b/config/config.exs index 394188787..eb9500433 100644 --- a/config/config.exs +++ b/config/config.exs @@ -137,11 +137,14 @@ config :archethic, Archethic.OracleChain, uco: Archethic.OracleChain.Services.UCOPrice ] +## UCO Price Oracle configuration +## Format : {key, refresh_interval_ms, ttl_ms}, ...]} +## If ttl = :infinity, the cache will never expire config :archethic, Archethic.OracleChain.Services.UCOPrice, providers: [ - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika + {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 60_000, :infinity}, + {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 60_000, :infinity}, + {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 60_000, :infinity} ] config :archethic, ArchethicWeb.FaucetController, diff --git a/config/dev.exs b/config/dev.exs index cc2e2fa7e..f50c59bf7 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -77,21 +77,6 @@ config :archethic, Archethic.Crypto.NodeKeystore.Origin.TPMImpl end) -## Format : {service, [{key, module, function, args, refresh_interval_ms, ttl_ms}, ...]} -## If ttl = :infinity, the cache will never expire -config :archethic, Archethic.Utils.HydratingCache.CachesManager, - uco_service: [ - {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, [["usd", "eur"]], - 30_000, :infinity}, - {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [["usd", "eur"]], - 30_000, :infinity}, - {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, [["usd", "eur"]], - 30_000, :infinity} - ] - config :archethic, Archethic.Governance.Pools, initial_members: [ technical_council: [ diff --git a/config/test.exs b/config/test.exs index cbccee44f..5283128df 100755 --- a/config/test.exs +++ b/config/test.exs @@ -90,9 +90,6 @@ config :archethic, Archethic.OracleChain.Scheduler, polling_interval: "0 0 * * * *", summary_interval: "0 0 0 * * *" -config :archethic, Archethic.OracleChain.Services.UCOPrice, - providers: [MockUCOPriceProvider1, MockUCOPriceProvider2, MockUCOPriceProvider3] - # -----Start-of-Networking-tests-configs----- config :archethic, Archethic.Networking, validate_node_ip: false diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index 9a0d9dab4..a9905f46b 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -23,11 +23,11 @@ defmodule Archethic.OracleChain.Services.UCOPrice do {:ok, fetching_tasks_supervisor} = Task.Supervisor.start_link() ## retrieve prices from configured providers and filter results marked as errors prices = - Enum.map(providers(), fn provider -> + Enum.map(providers(), fn {provider, _, _} -> case HydratingCache.get( - :"Elixir.Archethic.Utils.HydratingCache.uco_service", + Archethic.Utils.HydratingCache.UcoPrice, provider, - 3000 + 3_000 ) do {:error, reason} -> Logger.warning( @@ -100,7 +100,7 @@ defmodule Archethic.OracleChain.Services.UCOPrice do Enum.all?(@pairs, fn pair -> compare_price( Map.fetch!(prices_prior, pair), - Map.get(prices_now, pair, Map.fetch!(prices_prior, pair)) + Map.get(prices_now, pair, Map.get(prices_prior, pair, 0.0)) ) end) end @@ -156,6 +156,6 @@ defmodule Archethic.OracleChain.Services.UCOPrice do def parse_data(_), do: {:error, :invalid_data} defp providers do - Application.get_env(:archethic, __MODULE__) |> Keyword.fetch!(:providers) + Application.get_env(:archethic, __MODULE__, []) |> Keyword.get(:providers, []) end end diff --git a/lib/archethic/oracle_chain/supervisor.ex b/lib/archethic/oracle_chain/supervisor.ex index 2090ea85e..f8781980d 100644 --- a/lib/archethic/oracle_chain/supervisor.ex +++ b/lib/archethic/oracle_chain/supervisor.ex @@ -8,7 +8,11 @@ defmodule Archethic.OracleChain.Supervisor do alias Archethic.OracleChain.Scheduler alias Archethic.Utils - alias Archethic.Utils.HydratingCache.CachesManager + alias Archethic.Utils.HydratingCache + + require Logger + + @pairs ["usd", "eur"] def start_link(args \\ []) do Supervisor.start_link(__MODULE__, args) @@ -17,8 +21,18 @@ defmodule Archethic.OracleChain.Supervisor do def init(_args) do scheduler_conf = Application.get_env(:archethic, Scheduler) + ## Cook hydrating cache parameters from configuration + uco_service_providers = + :archethic + |> Application.get_env(Archethic.OracleChain.Services.UCOPrice, []) + |> Keyword.get(:providers, []) + |> IO.inspect(label: "Providers") + |> Enum.map(fn {mod, refresh_rate, ttl} -> + {mod, mod, :fetch, [@pairs], refresh_rate, ttl} + end) + children = [ - CachesManager, + {HydratingCache, [Archethic.Utils.HydratingCache.UcoPrice, uco_service_providers]}, MemTable, MemTableLoader, {Scheduler, scheduler_conf} diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 8d112f8aa..4a2e9985f 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -202,11 +202,11 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end def handle_event(:cast, {:new_value, _key, {:ok, value}}, :running, data) do + ## We got result from hydrating function + ## Stop timer on value ttl _ = maybe_stop_timer(data.timer_discard) - ## We got result from hydrating function - ## notify waiting getters Enum.each(data.getters, fn {pid, _ref} -> send(pid, {:ok, value}) @@ -241,7 +241,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} got error value #{inspect({key, {:error, reason}})}" ) - ## We reprogram the timer to hydrate, even if previous call failled. Error control could occur here + ## Error values can be discarded me = self() new_timer = @@ -259,8 +259,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do data | running_func_task: :undefined, getters: [], - timer_discard: nil, - timer_func: new_timer + timer_discard: new_timer }} end diff --git a/lib/archethic/utils/hydrating_cache/caches_manager.ex b/lib/archethic/utils/hydrating_cache/caches_manager.ex deleted file mode 100644 index 7fb813a20..000000000 --- a/lib/archethic/utils/hydrating_cache/caches_manager.ex +++ /dev/null @@ -1,96 +0,0 @@ -defmodule Archethic.Utils.HydratingCache.CachesManager do - @moduledoc """ - This module is used to manage (create and delete) hydrating caches. - At start it will read the configuration and start a cache per service. - """ - use GenServer - require Logger - alias Archethic.Utils.HydratingCache - - def start_link(args \\ [], opts \\ [name: __MODULE__]) do - GenServer.start_link(__MODULE__, args, opts) - end - - @doc """ - Start a new hydrating cache process to hold the values from a service. - This is a synchronous call, it will block until the cache is started and - hydrating process for initial keys is started. - """ - @spec new_service_sync( - name :: String.t(), - initial_keys :: list() - ) :: {:error, any} | {:ok, pid} - def new_service_sync(name, initial_keys) do - Logger.info("Starting new service sync #{name}") - GenServer.call(__MODULE__, {:new_service_sync, name, initial_keys}) - end - - @doc """ - Start a new hydrating cache process to hold the values from a service. - This is an asynchronous call, it will return immediately. - """ - def new_service_async(name, keys) do - Logger.info("Starting new service async #{name}") - GenServer.cast(__MODULE__, {:new_service_async, name, keys, self()}) - end - - @doc """ - Sync call to end a service cache. - """ - def end_service_sync(name) do - GenServer.call(__MODULE__, {:end_service, name}) - end - - @impl GenServer - def init(_args) do - manager_conf = Application.get_env(:archethic, __MODULE__, []) - - {:ok, caches_sup} = - DynamicSupervisor.start_link( - name: Archethic.Utils.HydratingCache.Manager.CachesSupervisor, - strategy: :one_for_one - ) - - Logger.info( - "Starting hydrating cache manager #{inspect(__MODULE__)} with conf #{inspect(manager_conf)}" - ) - - Enum.each(manager_conf, fn {service, keys} -> - Logger.info("Starting new service #{service}") - new_service_async(service, keys) - end) - - {:ok, %{:caches_sup => caches_sup}} - end - - @impl GenServer - def handle_call({:new_service_sync, name, initial_keys}, _from, state) do - Logger.info("Starting new service sync : #{name}") - - {:ok, pid} = - DynamicSupervisor.start_child( - state.caches_sup, - HydratingCache.child_spec([name, initial_keys, []]) - ) - - {:reply, {:ok, pid}, state} - end - - @impl GenServer - def handle_cast({:new_service_async, name, keys, _requester}, state) do - Logger.info("Starting new service async : #{name}") - - DynamicSupervisor.start_child(state.caches_sup, %{ - id: name, - start: {HydratingCache, :start_link, [name, keys]} - }) - - Logger.info("Started new service #{name}") - - {:noreply, state} - end - - def handle_cast(_, state) do - {:noreply, state} - end -end diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index 8a391d639..b2303ecae 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -15,8 +15,13 @@ defmodule Archethic.Utils.HydratingCache do | {:error, :timeout} | {:error, :not_registered} + def start_link([name, initial_keys]) do + Logger.info("Starting cache #{inspect(name)}") + GenServer.start_link(__MODULE__, [name, initial_keys], name: name) + end + def start_link(name, initial_keys \\ []) do - GenServer.start_link(__MODULE__, [name, initial_keys], name: :"#{__MODULE__}.#{name}") + start_link([name, initial_keys]) end @doc ~s""" @@ -45,12 +50,12 @@ defmodule Archethic.Utils.HydratingCache do fun :: (() -> {:ok, any()} | {:error, any()}), key :: any, refresh_interval :: non_neg_integer(), - ttl :: non_neg_integer() + ttl :: non_neg_integer() | :infinity ) :: term() def register_function(hydrating_cache, fun, key, refresh_interval, ttl) - when is_function(fun, 0) and is_integer(ttl) and ttl > 0 and + when is_function(fun, 0) and is_integer(refresh_interval) and - refresh_interval < ttl do + (refresh_interval < ttl or ttl == :infinity) do GenServer.call(hydrating_cache, {:register, fun, key, refresh_interval, ttl}) end diff --git a/test/archethic/oracle_chain/scheduler_test.exs b/test/archethic/oracle_chain/scheduler_test.exs index 9251e0284..e74e33ada 100644 --- a/test/archethic/oracle_chain/scheduler_test.exs +++ b/test/archethic/oracle_chain/scheduler_test.exs @@ -109,13 +109,29 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, _} = :sys.get_state(pid) - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, - :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, - :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) # polling_date = # "0 * * * *" @@ -186,13 +202,29 @@ defmodule Archethic.OracleChain.SchedulerTest do }} end) - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, - :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, - :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) send(pid, :poll) @@ -223,13 +255,29 @@ defmodule Archethic.OracleChain.SchedulerTest do available?: true }) - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, - :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, - :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) summary_date = "0 0 0 * *" @@ -331,13 +379,29 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, %{polling_timer: timer1}} = :sys.get_state(pid) - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, - :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, - :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"usd" => [0.2]}}], 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) send(pid, :poll) diff --git a/test/archethic/oracle_chain/services/uco_price_test.exs b/test/archethic/oracle_chain/services/uco_price_test.exs index 913dae276..c86555eea 100644 --- a/test/archethic/oracle_chain/services/uco_price_test.exs +++ b/test/archethic/oracle_chain/services/uco_price_test.exs @@ -6,136 +6,282 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do alias Archethic.Utils.HydratingCache test "fetch/0 should retrieve some data and build a map with the oracle name in it" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) assert {:ok, %{"eur" => _, "usd" => _}} = UCOPrice.fetch() end test "fetch/0 should retrieve some data and build a map with the oracle name in it and keep the precision to 5" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}}], 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) assert {:ok, %{"eur" => 0.12346, "usd" => 0.12345}} = UCOPrice.fetch() end describe "verify/1" do test "should return true if the prices are the good one" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.11]}}], 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.30, 0.40], "usd" => [0.12, 0.13]}}], 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.50], "usd" => [0.14]}}], 30_000, :infinity} - ]) - - assert {:ok, %{"eur" => 0.35, "usd" => 0.125}} == UCOPrice.fetch() + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"eur" => [0.10], "usd" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"eur" => [0.20], "usd" => [0.30]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"eur" => [0.30], "usd" => [0.40]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) + + assert {:ok, %{"eur" => 0.20, "usd" => 0.30}} == UCOPrice.fetch() end test "should return false if the prices have deviated" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) assert false == UCOPrice.verify?(%{"eur" => 0.10, "usd" => 0.14}) end end test "should return the median value when multiple providers queried" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.30], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.40], "usd" => [0.12]}}], - 30_000, :infinity} - ]) - - assert true == UCOPrice.verify?(%{"eur" => 0.30, "usd" => 0.12}) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.40], "eur" => [0.40]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) + + assert true == UCOPrice.verify?(%{"eur" => 0.30, "usd" => 0.30}) end test "should return the average of median values when a even number of providers queried" do - ## Define a fourth mock to have even number of mocks - Mox.defmock(MockUCOPriceProvider4, - for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl - ) - - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.30], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.40], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.12]}}], - 30_000, :infinity} - ]) - - assert false == UCOPrice.verify?(%{"eur" => 0.35, "usd" => 0.12}) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.20, 0.30], "eur" => [0.20, 0.30]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.40], "eur" => [0.40]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) + + assert {:ok, %{"eur" => 0.25, "usd" => 0.25}} == UCOPrice.fetch() end test "verify?/1 should return false when no data are returned from all providers" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30_000, - :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30_000, - :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30_000, - :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [], "usd" => []}}], 30_000, - :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [], "eur" => []}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [], "eur" => []}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [], "eur" => []}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) assert false == UCOPrice.verify?(%{}) end test "should report values even if a provider returns an error" do - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:error, :error_message}], 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.60], "usd" => [0.12]}}], - 30_000, :infinity} - ]) - - assert {:ok, %{"eur" => 0.55, "usd" => 0.12}} = UCOPrice.fetch() + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + ## If service returns an error, old value will be returned + ## we are so inserting a previous value + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> + {:ok, %{"usd" => [0.20], "eur" => [0.20]}} + end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:error, :error_message} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) + + assert {:ok, %{"eur" => 0.20, "usd" => 0.20}} = UCOPrice.fetch() end test "should handle a service timing out" do - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.10]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [:timer.sleep(5_000)], 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.50], "usd" => [0.10]}}], - 30_000, :infinity} - ]) - - assert true == UCOPrice.verify?(%{"eur" => 0.50, "usd" => 0.10}) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> + {:ok, %{"usd" => [0.20], "eur" => [0.20]}} + end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> + :timer.sleep(5_000) + {:ok, {:error, :error_message}} + end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) + + assert {:ok, %{"eur" => 0.20, "usd" => 0.20}} == UCOPrice.fetch() end def fetch(values) do diff --git a/test/archethic/oracle_chain/services_test.exs b/test/archethic/oracle_chain/services_test.exs index bad29eebf..ad94f3fd9 100644 --- a/test/archethic/oracle_chain/services_test.exs +++ b/test/archethic/oracle_chain/services_test.exs @@ -7,41 +7,85 @@ defmodule Archethic.OracleChain.ServicesTest do describe "fetch_new_data/1" do test "should return the new data when no previous content" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, - [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) assert %{uco: %{"eur" => 0.20, "usd" => 0.12}} = Services.fetch_new_data() end test "should not return the new data when the previous content is the same" do - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) assert %{} = Services.fetch_new_data(%{uco: %{"eur" => 0.20, "usd" => 0.12}}) end test "should return the new data when the previous content is not the same" do - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) assert %{uco: %{"eur" => 0.20, "usd" => 0.12}} = Services.fetch_new_data(%{"uco" => %{"eur" => 0.19, "usd" => 0.15}}) @@ -49,15 +93,29 @@ defmodule Archethic.OracleChain.ServicesTest do end test "verify_correctness?/1 should true when the data is correct" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) assert true == Services.verify_correctness?(%{"uco" => %{"eur" => 0.20, "usd" => 0.12}}) end diff --git a/test/archethic/oracle_chain_test.exs b/test/archethic/oracle_chain_test.exs index b56f83d22..548239078 100644 --- a/test/archethic/oracle_chain_test.exs +++ b/test/archethic/oracle_chain_test.exs @@ -11,15 +11,29 @@ defmodule Archethic.OracleChainTest do import Mox test "valid_services_content?/1 should verify the oracle transaction's content correctness" do - _ = - HydratingCache.start_link(:uco_service, [ - {MockUCOPriceProvider1, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider2, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity}, - {MockUCOPriceProvider3, __MODULE__, :fetch, [{:ok, %{"eur" => [0.20], "usd" => [0.12]}}], - 30_000, :infinity} - ]) + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + 30_000, + :infinity + ) + + HydratingCache.register_function( + Archethic.Utils.HydratingCache.UcoPrice, + fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + 30_000, + :infinity + ) content = %{ diff --git a/test/archethic/utils/hydrating_cache/caches_manager_test.exs b/test/archethic/utils/hydrating_cache/caches_manager_test.exs deleted file mode 100644 index 4dd651366..000000000 --- a/test/archethic/utils/hydrating_cache/caches_manager_test.exs +++ /dev/null @@ -1,27 +0,0 @@ -defmodule CachesManagerTest do - alias Archethic.Utils.HydratingCache - alias Archethic.Utils.HydratingCache.CachesManager - use ExUnit.Case - require Logger - - test "starting service from manager returns value once first hydrating have been done" do - CachesManager.new_service_async("test_services", [ - {:key1, __MODULE__, :waiting_function, [2000], 6000, 8000}, - {:key2, __MODULE__, :waiting_function, [1000], 6000, 8000}, - {:key3, __MODULE__, :waiting_function, [2000], 6000, 8000} - ]) - - ## wait a little so at least keys are registered - :timer.sleep(500) - - assert HydratingCache.get( - :"Elixir.Archethic.Utils.HydratingCache.test_services", - :key2, - 1700 - ) == {:ok, 1} - end - - def waiting_function(delay \\ 1000) do - {:ok, 1} - end -end diff --git a/test/test_helper.exs b/test/test_helper.exs index 12c512426..33658baa4 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -37,4 +37,8 @@ Mox.defmock(MockIPLookup, for: Archethic.Networking.IPLookup.Impl) Mox.defmock(MockRemoteDiscovery, for: Archethic.Networking.IPLookup.Impl) Mox.defmock(MockNATDiscovery, for: Archethic.Networking.IPLookup.Impl) +Mox.defmock(MockUCOPriceProvider1, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) +Mox.defmock(MockUCOPriceProvider2, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) +Mox.defmock(MockUCOPriceProvider3, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) + # -----End-of-Networking-Mocks ------ From b184d848fe5594b79ee86b1a1ac54f5c144a96b7 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Tue, 31 Jan 2023 17:30:26 +0100 Subject: [PATCH 34/48] Adressing Credo remarks --- lib/archethic/oracle_chain/supervisor.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/archethic/oracle_chain/supervisor.ex b/lib/archethic/oracle_chain/supervisor.ex index f8781980d..2985d8019 100644 --- a/lib/archethic/oracle_chain/supervisor.ex +++ b/lib/archethic/oracle_chain/supervisor.ex @@ -26,7 +26,6 @@ defmodule Archethic.OracleChain.Supervisor do :archethic |> Application.get_env(Archethic.OracleChain.Services.UCOPrice, []) |> Keyword.get(:providers, []) - |> IO.inspect(label: "Providers") |> Enum.map(fn {mod, refresh_rate, ttl} -> {mod, mod, :fetch, [@pairs], refresh_rate, ttl} end) From 2a1cc084e7b30d45628874edf2049abc8233a4eb Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 6 Feb 2023 11:33:19 +0100 Subject: [PATCH 35/48] Addressing MR comments plus bugfixes --- .../utils/hydrating_cache/cache_entry.ex | 137 ++++++++---------- .../utils/hydrating_cache/hydrating_cache.ex | 93 +++--------- .../hydrating_cache/hydrating_cache_test.exs | 9 +- 3 files changed, 87 insertions(+), 152 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index 4a2e9985f..c986a52fc 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -1,21 +1,3 @@ -defmodule Archethic.Utils.HydratingCache.CacheEntry.StateData do - @moduledoc """ - Struct describing the state of a cache entry FSM - """ - - defstruct([ - :running_func_task, - :hydrating_func, - :ttl, - :refresh_interval, - :key, - :timer_func, - :timer_discard, - getters: [], - value: :"$$undefined" - ]) -end - defmodule Archethic.Utils.HydratingCache.CacheEntry do @moduledoc """ This module is a finite state machine implementing a cache entry. @@ -27,23 +9,36 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do - Manage various timers """ alias Archethic.Utils.HydratingCache.CacheEntry - @behaviour :gen_statem + use GenStateMachine, callback_mode: [:handle_event_function, :state_enter] use Task require Logger + defstruct([ + :running_func_task, + :hydrating_func, + :ttl, + :refresh_interval, + :key, + :timer_func, + :timer_discard, + getters: [], + value: :"$$undefined" + ]) + + @spec start_link([...]) :: :ignore | {:error, any} | {:ok, pid} def start_link([fun, key, refresh_interval, ttl]) do - :gen_statem.start_link(__MODULE__, [fun, key, refresh_interval, ttl], []) + GenStateMachine.start_link(__MODULE__, [fun, key, refresh_interval, ttl], []) end - @impl :gen_statem + @impl GenStateMachine def init([fun, key, refresh_interval, ttl]) do # start hydrating at the needed refresh interval - {:ok, timer} = :timer.send_interval(refresh_interval, self(), :hydrate) + {:ok, timer} = :timer.send_after(refresh_interval, self(), :hydrate) ## Hydrate the value {:ok, :running, - %CacheEntry.StateData{ + %CacheEntry{ timer_func: timer, hydrating_func: fun, key: key, @@ -52,19 +47,14 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do }} end - @impl :gen_statem - def callback_mode, do: [:handle_event_function, :state_enter] - - @impl :gen_statem - + @impl GenStateMachine def handle_event( {:call, from}, {:get, _requester}, :idle, - data = %CacheEntry.StateData{:value => :"$$undefined"} + data = %CacheEntry{:value => :"$$undefined"} ) do ## Value is requested while fsm is iddle, return the value - {:keep_state, data, [{:reply, from, {:error, :not_initialized}}]} end @@ -73,23 +63,17 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:next_state, :idle, data, [{:reply, from, data.value}]} end - def handle_event(:cast, {:get, requester}, :idle, data) do - ## Value is requested while fsm is iddle, return the value - send(requester, {:ok, data.value}) - {:next_state, :idle, data} - end - ## Call for value while hydrating function is running and we have no previous value ## We register the caller to send value later on, and we indicate caller to block def handle_event( {:call, from}, {:get, requester}, :running, - data = %CacheEntry.StateData{value: :"$$undefined"} + data = %CacheEntry{value: :"$$undefined"} ) do previous_getters = data.getters - {:keep_state, %CacheEntry.StateData{data | getters: previous_getters ++ [requester]}, + {:keep_state, %CacheEntry{data | getters: previous_getters ++ [requester]}, [{:reply, from, {:ok, :answer_delayed}}]} end @@ -99,20 +83,6 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:next_state, :running, data, [{:reply, from, data.value}]} end - ## Getting value when a function is running and no previous value is available - ## Register this getter to send value later on - def handle_event(:cast, {:get, from}, :running, data) when data.value == :"$$undefined" do - previous_getters = data.getters - {:next_state, :running, %CacheEntry.StateData{data | getters: previous_getters ++ [from]}} - end - - def handle_event(:cast, {:get, from}, :running, data) do - ## Getting value while function is running but previous value is available - ## Return vurrent value - send(from, {:ok, data.value}) - {:next_state, :running, data} - end - def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, :running, data) do ## Registering a new hydrating function while previous one is running @@ -127,11 +97,11 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do _ = maybe_stop_timer(data.timer_discard) ## Start new timer to hydrate at refresh interval - {:ok, timer} = :timer.send_interval(refresh_interval, self(), :hydrate) + {:ok, timer} = :timer.send_after(refresh_interval, self(), :hydrate) ## We trigger the update ( to trigger or not could be set at registering option ) {:repeat_state, - %CacheEntry.StateData{ + %CacheEntry{ data | hydrating_func: fun, key: key, @@ -141,7 +111,11 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do }, [{:reply, from, :ok}]} end - def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, _state, data) do + def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, state, data) do + Logger.debug( + "Registering hydrating function for key :#{inspect(key)} state:#{inspect(state)} data: #{inspect(data)}" + ) + ## Setting hydrating function in other cases ## Hydrating function not running, we just stop the timers _ = maybe_stop_timer(data.timer_func) @@ -155,11 +129,11 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do :refresh_interval => refresh_interval }) - {:ok, timer} = :timer.send_interval(refresh_interval, self(), :hydrate) + {:ok, timer} = :timer.send_after(refresh_interval, self(), :hydrate) ## We trigger the update ( to trigger or not could be set at registering option ) {:next_state, :running, - %CacheEntry.StateData{ + %CacheEntry{ data | hydrating_func: fun, key: key, @@ -183,11 +157,11 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do spawn(fn -> Logger.info("Running hydrating function for key :#{inspect(data.key)}") value = data.hydrating_func.() - :gen_statem.cast(me, {:new_value, data.key, value}) + GenStateMachine.cast(me, {:new_value, data.key, value}) end) ## we stay in running state - {:next_state, :running, %CacheEntry.StateData{data | running_func_task: hydrating_task}} + {:next_state, :running, %CacheEntry{data | running_func_task: hydrating_task}} end def handle_event(:info, :discarded, state, data) do @@ -197,16 +171,18 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} discarded" ) - {:next_state, state, - %CacheEntry.StateData{data | value: {:error, :discarded}, timer_discard: nil}} + {:next_state, state, %CacheEntry{data | value: {:error, :discarded}, timer_discard: nil}} end def handle_event(:cast, {:new_value, _key, {:ok, value}}, :running, data) do ## We got result from hydrating function - + Logger.debug("Got new value for key :#{inspect(data.key)}") ## Stop timer on value ttl _ = maybe_stop_timer(data.timer_discard) + ## Start hydrating timer + {:ok, timer_hydrate} = :timer.send_after(data.refresh_interval, self(), :hydrate) + ## notify waiting getters Enum.each(data.getters, fn {pid, _ref} -> send(pid, {:ok, value}) @@ -215,23 +191,23 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ## Start timer to discard new value if needed me = self() - new_timer = + {:ok, timer_ttl} = case data.ttl do - value when is_number(value) -> - {:ok, t} = :timer.send_after(value, me, :discarded) - t + ttl when is_number(ttl) -> + :timer.send_after(ttl, me, :discarded) _ -> - :ok + {:ok, nil} end {:next_state, :idle, - %CacheEntry.StateData{ + %CacheEntry{ data | running_func_task: :undefined, value: {:ok, value}, getters: [], - timer_discard: new_timer + timer_func: timer_hydrate, + timer_discard: timer_ttl }} end @@ -244,22 +220,25 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ## Error values can be discarded me = self() - new_timer = + {:ok, timer_ttl} = case data.ttl do - value when is_number(value) -> - {:ok, t} = :timer.send_after(value, me, :discarded) - t + ttl when is_number(ttl) -> + :timer.send_after(ttl, me, :discarded) _ -> - :ok + {:ok, nil} end + ## Start hydrating timer + {:ok, timer_hydrate} = :timer.send_after(data.refresh_interval, self(), :hydrate) + {:next_state, :idle, - %CacheEntry.StateData{ + %CacheEntry{ data | running_func_task: :undefined, getters: [], - timer_discard: new_timer + timer_func: timer_hydrate, + timer_discard: timer_ttl }} end @@ -267,11 +246,11 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:keep_state, data} end - defp maybe_stop_timer(ref = {_, _}) do - :timer.cancel(ref) + defp maybe_stop_timer(tref = {_, _}) do + :timer.cancel(tref) end - defp maybe_stop_timer(_ref) do + defp maybe_stop_timer(_else) do :ok end end diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index b2303ecae..5a82f4748 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -7,6 +7,7 @@ defmodule Archethic.Utils.HydratingCache do alias Archethic.Utils.HydratingCache.CacheEntry use GenServer + @vsn Mix.Project.config()[:version] require Logger @@ -16,7 +17,6 @@ defmodule Archethic.Utils.HydratingCache do | {:error, :not_registered} def start_link([name, initial_keys]) do - Logger.info("Starting cache #{inspect(name)}") GenServer.start_link(__MODULE__, [name, initial_keys], name: name) end @@ -106,7 +106,7 @@ defmodule Archethic.Utils.HydratingCache do end @impl GenServer - def init([name, keys]) do + def init([name, initial_keys]) do Logger.info("Starting Hydrating cache for service #{inspect("#{__MODULE__}.#{name}")}") ## start a dynamic supervisor for the cache entries/keys @@ -116,53 +116,40 @@ defmodule Archethic.Utils.HydratingCache do strategy: :one_for_one ) - me = self() - - ## start a supervisor to manage the initial keys insertion workers - {:ok, initial_keys_worker_sup} = Task.Supervisor.start_link() + ## Create child for each initial key + child_specs = + initial_keys + |> Enum.map(fn {provider, mod, func, params, refresh_interval, ttl} -> + {CacheEntry, [fn -> apply(mod, func, params) end, provider, refresh_interval, ttl]} + end) ## Registering initial keys - _ = - Task.Supervisor.async_stream_nolink( - initial_keys_worker_sup, - keys, - fn - {provider, mod, func, params, refresh_rate, ttl} -> - GenServer.cast( - me, - {:register, fn -> apply(mod, func, params) end, provider, refresh_rate, ttl} - ) + keys = + Enum.reduce(child_specs, %{}, fn child = {_, [_, provider, _, _]}, acc -> + {:ok, cache_entry} = DynamicSupervisor.start_child(keys_sup, child) + Map.put(acc, provider, cache_entry) + end) - other -> - Logger.error("Hydrating cache: Invalid configuration entry: #{inspect(other)}") - end, - on_timeout: :kill_task - ) - |> Stream.filter(&match?({:ok, {:ok, _}}, &1)) - |> Enum.to_list() - - ## stop the initial keys worker supervisor - Supervisor.stop(initial_keys_worker_sup) - {:ok, %{:keys => keys, keys_sup: keys_sup}} + {:ok, %{keys: keys, keys_sup: keys_sup}} end @impl GenServer - def handle_call({:get, key}, from, state) do - case Map.get(state, key, :undefined) do - :undefined -> + def handle_call({:get, key}, from, state = %{keys: keys}) do + case Map.get(keys, key) do + nil -> Logger.warning("HydratingCache no entry for #{inspect(key)}") {:reply, {:error, :not_registered}, state} pid -> - value = :gen_statem.call(pid, {:get, from}) + value = GenStateMachine.call(pid, {:get, from}) {:reply, value, state} end end - def handle_call({:register, fun, key, refresh_interval, ttl}, _from, state) do + def handle_call({:register, fun, key, refresh_interval, ttl}, _from, state = %{keys: keys}) do Logger.debug("Registering hydrating function for #{inspect(key)}") ## Called when asked to register a function - case Map.get(state, key) do + case Map.get(keys, key) do nil -> ## New key, we start a cache entry fsm {:ok, pid} = @@ -171,16 +158,16 @@ defmodule Archethic.Utils.HydratingCache do {CacheEntry, [fun, key, refresh_interval, ttl]} ) - {:reply, :ok, Map.put(state, key, pid)} + {:reply, :ok, %{state | keys: Map.put(keys, key, pid)}} pid -> ## Key already exists, no need to start fsm - case :gen_statem.call(pid, {:register, fun, key, refresh_interval, ttl}) do + case GenStateMachine.call(pid, {:register, fun, key, refresh_interval, ttl}) do :ok -> - {:reply, :ok, Map.put(state, key, pid)} + {:reply, :ok, %{state | keys: Map.put(keys, key, pid)}} error -> - {:reply, {:error, error}, Map.put(state, key, pid)} + {:reply, {:error, error}, state} end end end @@ -191,38 +178,6 @@ defmodule Archethic.Utils.HydratingCache do end @impl GenServer - def handle_cast({:get, from, key}, state) do - case Map.get(state, key, :undefined) do - :undefined -> - send(from, {:error, :not_registered}) - {:noreply, state} - - pid -> - :gen_statem.cast(pid, {:get, from}) - {:noreply, state} - end - end - - def handle_cast({:register, fun, key, refresh_interval, ttl}, state) do - case Map.get(state, key) do - nil -> - ## New key, we start a cache entry fsm - - {:ok, pid} = - DynamicSupervisor.start_child( - state.keys_sup, - {CacheEntry, [fun, key, refresh_interval, ttl]} - ) - - {:noreply, Map.put(state, key, pid)} - - pid -> - ## Key already exists, no need to start fsm - _ = :gen_statem.call(pid, {:register, fun, key, refresh_interval, ttl}) - {:noreply, Map.put(state, key, pid)} - end - end - def handle_cast(unmanaged, state) do Logger.warning("Cache received unmanaged cast: #{inspect(unmanaged)}") {:noreply, state} diff --git a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs index a080963ef..43e986074 100644 --- a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs +++ b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs @@ -101,7 +101,7 @@ defmodule HydratingCacheTest do {:ok, 1} end, "test_reregister", - 10_000, + 40_000, 50_000 ) @@ -109,11 +109,12 @@ defmodule HydratingCacheTest do HydratingCache.register_function( pid, fn -> + Logger.info("Hydrating function sleeping 5 secs") :timer.sleep(5000) {:ok, 2} end, "test_reregister", - 10_000, + 40_000, 50_000 ) @@ -218,14 +219,14 @@ defmodule HydratingCacheTest do end ## Resilience tests - test "If hydrating function crash, key fsm will still be oprationnal" do + test "If hydrating function crash, key fsm will still be operationnal" do {:ok, pid} = HydratingCache.start_link(:test_service) _ = HydratingCache.register_function( pid, fn -> - ## Trigger badmatch + ## Exit hydrating function exit(1) {:ok, :badmatch} end, From e3a42703a97dce1b27294edbd3cde6fa9cd26db4 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 6 Feb 2023 20:33:05 +0100 Subject: [PATCH 36/48] Adding get_all to get all keys from hydrating cache plus tests --- .../oracle_chain/services/uco_price.ex | 34 +------- .../utils/hydrating_cache/cache_entry.ex | 13 +-- .../utils/hydrating_cache/hydrating_cache.ex | 60 +++++++++++++- .../hydrating_cache/hydrating_cache_test.exs | 80 +++++++++++++++++-- 4 files changed, 141 insertions(+), 46 deletions(-) diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index a9905f46b..aeb7bf2ef 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -23,36 +23,8 @@ defmodule Archethic.OracleChain.Services.UCOPrice do {:ok, fetching_tasks_supervisor} = Task.Supervisor.start_link() ## retrieve prices from configured providers and filter results marked as errors prices = - Enum.map(providers(), fn {provider, _, _} -> - case HydratingCache.get( - Archethic.Utils.HydratingCache.UcoPrice, - provider, - 3_000 - ) do - {:error, reason} -> - Logger.warning( - "Service UCOPrice cannot fetch values from provider: #{inspect(provider)} with reason : #{inspect(reason)}." - ) - - [] - - {:ok, result} -> - {provider, result} - end - end) - |> List.flatten() - |> Enum.filter(fn - {_, %{}} -> - true + HydratingCache.get_all(Archethic.Utils.HydratingCache.UcoPrice) - other -> - Logger.error("Service UCOPrice cannot fetch values from provider: #{inspect(other)}.") - false - end) - |> Enum.map(fn - {_, result = %{}} -> - result - end) ## Here stream looks like : [%{"eur"=>[0.44], "usd"=[0.32]}, ..., %{"eur"=>[0.42, 0.43], "usd"=[0.35]}] |> Enum.reduce(%{}, &agregate_providers_data/2) |> Enum.reduce(%{}, fn {currency, values}, acc -> @@ -154,8 +126,4 @@ defmodule Archethic.OracleChain.Services.UCOPrice do end def parse_data(_), do: {:error, :invalid_data} - - defp providers do - Application.get_env(:archethic, __MODULE__, []) |> Keyword.get(:providers, []) - end end diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic/utils/hydrating_cache/cache_entry.ex index c986a52fc..9eaaf5e41 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic/utils/hydrating_cache/cache_entry.ex @@ -72,6 +72,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do data = %CacheEntry{value: :"$$undefined"} ) do previous_getters = data.getters + Logger.warning("Get Value but undefined #{inspect(data)}") {:keep_state, %CacheEntry{data | getters: previous_getters ++ [requester]}, [{:reply, from, {:ok, :answer_delayed}}]} @@ -111,10 +112,8 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do }, [{:reply, from, :ok}]} end - def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, state, data) do - Logger.debug( - "Registering hydrating function for key :#{inspect(key)} state:#{inspect(state)} data: #{inspect(data)}" - ) + def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, _state, data) do + Logger.info("Registering hydrating function for key :#{inspect(key)}") ## Setting hydrating function in other cases ## Hydrating function not running, we just stop the timers @@ -150,6 +149,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end def handle_event(:enter, _event, :running, data) do + Logger.info("Entering running state for key :#{inspect(data.key)}") ## At entering running state, we start the hydrating task me = self() @@ -176,7 +176,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do def handle_event(:cast, {:new_value, _key, {:ok, value}}, :running, data) do ## We got result from hydrating function - Logger.debug("Got new value for key :#{inspect(data.key)}") + Logger.debug("Got new value for key :#{inspect(data.key)} #{inspect(value)}") ## Stop timer on value ttl _ = maybe_stop_timer(data.timer_discard) @@ -185,7 +185,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ## notify waiting getters Enum.each(data.getters, fn {pid, _ref} -> - send(pid, {:ok, value}) + send(pid, {:delayed_value, data.key, {:ok, value}}) end) ## Start timer to discard new value if needed @@ -247,6 +247,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end defp maybe_stop_timer(tref = {_, _}) do + IO.puts("Cancelling timer #{inspect(tref)}") :timer.cancel(tref) end diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index 5a82f4748..ea91dd7fa 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -85,8 +85,8 @@ defmodule Archethic.Utils.HydratingCache do case GenServer.call(hydrating_cache, {:get, key}, timeout) do {:ok, :answer_delayed} -> receive do - {:ok, value} -> - {:ok, value} + {:delayed_value, _, value} -> + value other -> Logger.warning("Unexpected return value #{inspect(other)}") @@ -105,6 +105,10 @@ defmodule Archethic.Utils.HydratingCache do end end + def get_all(hydrating_cache) do + GenServer.call(hydrating_cache, :get_all) + end + @impl GenServer def init([name, initial_keys]) do Logger.info("Starting Hydrating cache for service #{inspect("#{__MODULE__}.#{name}")}") @@ -146,6 +150,58 @@ defmodule Archethic.Utils.HydratingCache do end end + def handle_call(:get_all, from, state = %{keys: keys}) do + Logger.debug( + "Getting all keys from hydrating cache, current keys are #{inspect(keys)} sup: #{inspect(state.keys_sup)}" + ) + + {:ok, fetching_values_supervisor} = Task.Supervisor.start_link() + + result = + Task.Supervisor.async_stream_nolink( + fetching_values_supervisor, + keys, + fn {key, pid} -> + IO.puts("Calling #{inspect(pid)} for key #{inspect(key)} from is #{inspect(from)}") + + case GenStateMachine.call(pid, {:get, {self(), nil}}) do + {:ok, :answer_delayed} -> + receive do + {:delayed_value, _, {:ok, value}} -> + Logger.debug( + "Got delayed value for key #{inspect(value)} from hydrating cache #{inspect(self())}" + ) + + {:ok, value} + + other -> + Logger.warning("Unexpected return value #{inspect(other)}") + {:error, :unexpected_value} + after + 3_000 -> + Logger.warning( + "Timeout waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(self())}" + ) + + {:error, :timeout} + end + + other -> + other + end + end, + on_timeout: :kill_task + ) + |> Stream.filter(&match?({:ok, {:ok, _}}, &1)) + |> Stream.map(fn + {_, {_, result}} -> + result + end) + |> Enum.to_list() + + {:reply, result, state} + end + def handle_call({:register, fun, key, refresh_interval, ttl}, _from, state = %{keys: keys}) do Logger.debug("Registering hydrating function for #{inspect(key)}") ## Called when asked to register a function diff --git a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs index 43e986074..ec06b47ac 100644 --- a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs +++ b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs @@ -38,7 +38,6 @@ defmodule HydratingCacheTest do HydratingCache.register_function( pid, fn -> - IO.puts("Hydrating function incrementing value") value = :persistent_term.get("test") value = value + 1 :persistent_term.put("test", value) @@ -198,9 +197,7 @@ defmodule HydratingCacheTest do HydratingCache.register_function( pid, fn -> - IO.puts("Hydrating function Sleeping 3 secs") - :timer.sleep(3000) - IO.puts("Hydrating function done") + :timer.sleep(2_000) {:ok, :valid_result} end, "delayed_result", @@ -214,7 +211,6 @@ defmodule HydratingCacheTest do assert Enum.all?(results, fn {:ok, {:ok, :valid_result}} -> true - other -> IO.puts("Unk #{inspect(other)}") end) end @@ -281,4 +277,78 @@ defmodule HydratingCacheTest do result = HydratingCache.get(pid, :key, 3000) assert result == {:error, :discarded} end + + test "Can get a value while another request is waiting for results" do + {:ok, pid} = HydratingCache.start_link(:test_service) + + _ = + HydratingCache.register_function( + pid, + fn -> + :timer.sleep(10_000) + {:ok, :value} + end, + :key1, + 30_000, + 40_000 + ) + + :erlang.spawn(fn -> HydratingCache.get(pid, :key1, 35_000) end) + + _ = + HydratingCache.register_function( + pid, + fn -> + {:ok, :value2} + end, + :key2, + 500, + 1_000 + ) + + result = HydratingCache.get(pid, :key2, 3000) + assert result == {:ok, :value2} + end + + test "can retrieve all values beside erroneous ones" do + {:ok, pid} = + HydratingCache.start_link(:test_service, [ + {"key1", __MODULE__, :val_hydrating_function, [10], 30_000, 40_000}, + {"key2", __MODULE__, :failval_hydrating_function, [20], 30_000, 40_000}, + {"key3", __MODULE__, :val_hydrating_function, [30], 30_000, 40_000} + ]) + + :timer.sleep(1000) + values = HydratingCache.get_all(pid) + assert values == [10, 30] + end + + test "Retrieving all values supports delayed values" do + {:ok, pid} = + HydratingCache.start_link(:test_service, [ + {"key1", __MODULE__, :val_hydrating_function, [10], 30_000, 40_000}, + {"key2", __MODULE__, :timed_hydrating_function, [2000, 20], 30_000, 40_000}, + {"key3", __MODULE__, :failval_hydrating_function, [30], 30_000, 40_000} + ]) + + :timer.sleep(1000) + values = HydratingCache.get_all(pid) + assert values == [10, 20] + end + + def val_hydrating_function(value) do + Logger.debug("Hydrating value #{value}") + {:ok, value} + end + + def failval_hydrating_function(value) do + Logger.debug("Hydrating value #{value}") + {:error, value} + end + + def timed_hydrating_function(delay, value) do + Logger.debug("Timed Hydrating value #{value}") + :timer.sleep(delay) + {:ok, value} + end end From cc167f8850985104d8c03d1994d1217b931eb064 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 6 Feb 2023 20:36:55 +0100 Subject: [PATCH 37/48] Removing trace --- lib/archethic/utils/hydrating_cache/hydrating_cache.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index ea91dd7fa..ad71062c2 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -162,8 +162,6 @@ defmodule Archethic.Utils.HydratingCache do fetching_values_supervisor, keys, fn {key, pid} -> - IO.puts("Calling #{inspect(pid)} for key #{inspect(key)} from is #{inspect(from)}") - case GenStateMachine.call(pid, {:get, {self(), nil}}) do {:ok, :answer_delayed} -> receive do From 21fe8e8ba24d5fa6beddd47952414a5a174137e8 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 6 Feb 2023 20:49:21 +0100 Subject: [PATCH 38/48] Fixing unused variable --- lib/archethic/utils/hydrating_cache/hydrating_cache.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex index ad71062c2..9e0a18cdc 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic/utils/hydrating_cache/hydrating_cache.ex @@ -150,7 +150,7 @@ defmodule Archethic.Utils.HydratingCache do end end - def handle_call(:get_all, from, state = %{keys: keys}) do + def handle_call(:get_all, _from, state = %{keys: keys}) do Logger.debug( "Getting all keys from hydrating cache, current keys are #{inspect(keys)} sup: #{inspect(state.keys_sup)}" ) From 12099ff4df7fb4670f5a5dd272d1d47e8c81d916 Mon Sep 17 00:00:00 2001 From: Yan Guiborat Date: Mon, 6 Feb 2023 21:19:16 +0100 Subject: [PATCH 39/48] Preventing pipeline race condition --- .../hydrating_cache/hydrating_cache_test.exs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs index ec06b47ac..93084fce6 100644 --- a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs +++ b/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs @@ -9,7 +9,7 @@ defmodule HydratingCacheTest do end test "If value stored, it is returned immediatly" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_normal) result = HydratingCache.register_function( @@ -30,7 +30,7 @@ defmodule HydratingCacheTest do end test "Hydrating function runs periodically" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_periodic) :persistent_term.put("test", 1) @@ -57,7 +57,7 @@ defmodule HydratingCacheTest do end test "Update hydrating function while another one is running returns new hydrating value from new function" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_hydrating) result = HydratingCache.register_function( @@ -91,7 +91,7 @@ defmodule HydratingCacheTest do end test "Getting value while function is running and previous value is available returns value" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_running) _ = HydratingCache.register_function( @@ -123,7 +123,7 @@ defmodule HydratingCacheTest do end test "Two hydrating function can run at same time" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_simultaneous) _ = HydratingCache.register_function( @@ -153,7 +153,7 @@ defmodule HydratingCacheTest do end test "Querying key while first refreshed will block the calling process until refreshed and provide the value" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_refresh) _ = HydratingCache.register_function( @@ -172,7 +172,7 @@ defmodule HydratingCacheTest do end test "Querying key while first refreshed will block the calling process until timeout" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_block) _ = HydratingCache.register_function( @@ -191,7 +191,7 @@ defmodule HydratingCacheTest do end test "Multiple process can wait for a delayed value" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_delayed) _ = HydratingCache.register_function( @@ -216,7 +216,7 @@ defmodule HydratingCacheTest do ## Resilience tests test "If hydrating function crash, key fsm will still be operationnal" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_crash) _ = HydratingCache.register_function( @@ -279,7 +279,7 @@ defmodule HydratingCacheTest do end test "Can get a value while another request is waiting for results" do - {:ok, pid} = HydratingCache.start_link(:test_service) + {:ok, pid} = HydratingCache.start_link(:test_service_wait) _ = HydratingCache.register_function( @@ -312,7 +312,7 @@ defmodule HydratingCacheTest do test "can retrieve all values beside erroneous ones" do {:ok, pid} = - HydratingCache.start_link(:test_service, [ + HydratingCache.start_link(:test_service_get_all, [ {"key1", __MODULE__, :val_hydrating_function, [10], 30_000, 40_000}, {"key2", __MODULE__, :failval_hydrating_function, [20], 30_000, 40_000}, {"key3", __MODULE__, :val_hydrating_function, [30], 30_000, 40_000} @@ -325,7 +325,7 @@ defmodule HydratingCacheTest do test "Retrieving all values supports delayed values" do {:ok, pid} = - HydratingCache.start_link(:test_service, [ + HydratingCache.start_link(:test_service_get_all_delayed, [ {"key1", __MODULE__, :val_hydrating_function, [10], 30_000, 40_000}, {"key2", __MODULE__, :timed_hydrating_function, [2000, 20], 30_000, 40_000}, {"key3", __MODULE__, :failval_hydrating_function, [30], 30_000, 40_000} From deba1f51b1584cfc0b2b06239b818ad4b1ca2f59 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Wed, 1 Mar 2023 11:23:22 +0100 Subject: [PATCH 40/48] Move HydratingCache into ArchethicCache folder --- .../oracle_chain/services/uco_price.ex | 4 +- lib/archethic/oracle_chain/supervisor.ex | 6 +- .../hydrating_cache.ex | 6 +- .../hydrating_cache/cache_entry.ex | 24 ++++---- .../archethic/oracle_chain/scheduler_test.exs | 26 ++++---- .../oracle_chain/services/uco_price_test.exs | 60 +++++++++---------- test/archethic/oracle_chain/services_test.exs | 26 ++++---- test/archethic/oracle_chain_test.exs | 8 +-- .../hydrating_cache_test.exs | 10 +--- 9 files changed, 81 insertions(+), 89 deletions(-) rename lib/{archethic/utils/hydrating_cache => archethic_cache}/hydrating_cache.ex (97%) rename lib/{archethic/utils => archethic_cache}/hydrating_cache/cache_entry.ex (93%) rename test/{archethic/utils/hydrating_cache => archethic_cache}/hydrating_cache_test.exs (96%) diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index aeb7bf2ef..d40a82951 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -8,7 +8,7 @@ defmodule Archethic.OracleChain.Services.UCOPrice do alias Archethic.OracleChain.Services.Impl alias Archethic.Utils - alias Archethic.Utils.HydratingCache + alias ArchethicCache.HydratingCache @behaviour Impl @@ -23,7 +23,7 @@ defmodule Archethic.OracleChain.Services.UCOPrice do {:ok, fetching_tasks_supervisor} = Task.Supervisor.start_link() ## retrieve prices from configured providers and filter results marked as errors prices = - HydratingCache.get_all(Archethic.Utils.HydratingCache.UcoPrice) + HydratingCache.get_all(HydratingCache.UcoPrice) ## Here stream looks like : [%{"eur"=>[0.44], "usd"=[0.32]}, ..., %{"eur"=>[0.42, 0.43], "usd"=[0.35]}] |> Enum.reduce(%{}, &agregate_providers_data/2) diff --git a/lib/archethic/oracle_chain/supervisor.ex b/lib/archethic/oracle_chain/supervisor.ex index 2985d8019..3f8d10e17 100644 --- a/lib/archethic/oracle_chain/supervisor.ex +++ b/lib/archethic/oracle_chain/supervisor.ex @@ -8,9 +8,7 @@ defmodule Archethic.OracleChain.Supervisor do alias Archethic.OracleChain.Scheduler alias Archethic.Utils - alias Archethic.Utils.HydratingCache - - require Logger + alias ArchethicCache.HydratingCache @pairs ["usd", "eur"] @@ -31,7 +29,7 @@ defmodule Archethic.OracleChain.Supervisor do end) children = [ - {HydratingCache, [Archethic.Utils.HydratingCache.UcoPrice, uco_service_providers]}, + {HydratingCache, [HydratingCache.UcoPrice, uco_service_providers]}, MemTable, MemTableLoader, {Scheduler, scheduler_conf} diff --git a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex b/lib/archethic_cache/hydrating_cache.ex similarity index 97% rename from lib/archethic/utils/hydrating_cache/hydrating_cache.ex rename to lib/archethic_cache/hydrating_cache.ex index 9e0a18cdc..31663d661 100644 --- a/lib/archethic/utils/hydrating_cache/hydrating_cache.ex +++ b/lib/archethic_cache/hydrating_cache.ex @@ -1,10 +1,10 @@ -defmodule Archethic.Utils.HydratingCache do +defmodule ArchethicCache.HydratingCache do @moduledoc """ GenServer implementing the hydrating cache itself. There should be one Hydrating per service ( ex : UCO price, meteo etc...) It receives queries from clients requesting the cache, and manage the cache entries FSMs """ - alias Archethic.Utils.HydratingCache.CacheEntry + alias __MODULE__.CacheEntry use GenServer @vsn Mix.Project.config()[:version] @@ -116,7 +116,7 @@ defmodule Archethic.Utils.HydratingCache do ## start a dynamic supervisor for the cache entries/keys {:ok, keys_sup} = DynamicSupervisor.start_link( - name: :"Archethic.Utils.HydratingCache.CacheEntry.KeysSupervisor.#{name}", + name: :"ArchethicCache.HydratingCache.CacheEntry.KeysSupervisor.#{name}", strategy: :one_for_one ) diff --git a/lib/archethic/utils/hydrating_cache/cache_entry.ex b/lib/archethic_cache/hydrating_cache/cache_entry.ex similarity index 93% rename from lib/archethic/utils/hydrating_cache/cache_entry.ex rename to lib/archethic_cache/hydrating_cache/cache_entry.ex index 9eaaf5e41..ab5da2cab 100644 --- a/lib/archethic/utils/hydrating_cache/cache_entry.ex +++ b/lib/archethic_cache/hydrating_cache/cache_entry.ex @@ -1,4 +1,4 @@ -defmodule Archethic.Utils.HydratingCache.CacheEntry do +defmodule ArchethicCache.HydratingCache.CacheEntry do @moduledoc """ This module is a finite state machine implementing a cache entry. There is one such Cache Entry FSM running per registered key. @@ -8,7 +8,6 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do - Run the hydrating function associated with this key - Manage various timers """ - alias Archethic.Utils.HydratingCache.CacheEntry use GenStateMachine, callback_mode: [:handle_event_function, :state_enter] use Task @@ -38,7 +37,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ## Hydrate the value {:ok, :running, - %CacheEntry{ + %__MODULE__{ timer_func: timer, hydrating_func: fun, key: key, @@ -52,7 +51,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:call, from}, {:get, _requester}, :idle, - data = %CacheEntry{:value => :"$$undefined"} + data = %__MODULE__{:value => :"$$undefined"} ) do ## Value is requested while fsm is iddle, return the value {:keep_state, data, [{:reply, from, {:error, :not_initialized}}]} @@ -69,12 +68,12 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:call, from}, {:get, requester}, :running, - data = %CacheEntry{value: :"$$undefined"} + data = %__MODULE__{value: :"$$undefined"} ) do previous_getters = data.getters Logger.warning("Get Value but undefined #{inspect(data)}") - {:keep_state, %CacheEntry{data | getters: previous_getters ++ [requester]}, + {:keep_state, %__MODULE__{data | getters: previous_getters ++ [requester]}, [{:reply, from, {:ok, :answer_delayed}}]} end @@ -102,7 +101,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ## We trigger the update ( to trigger or not could be set at registering option ) {:repeat_state, - %CacheEntry{ + %__MODULE__{ data | hydrating_func: fun, key: key, @@ -132,7 +131,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do ## We trigger the update ( to trigger or not could be set at registering option ) {:next_state, :running, - %CacheEntry{ + %__MODULE__{ data | hydrating_func: fun, key: key, @@ -161,7 +160,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end) ## we stay in running state - {:next_state, :running, %CacheEntry{data | running_func_task: hydrating_task}} + {:next_state, :running, %__MODULE__{data | running_func_task: hydrating_task}} end def handle_event(:info, :discarded, state, data) do @@ -171,7 +170,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} discarded" ) - {:next_state, state, %CacheEntry{data | value: {:error, :discarded}, timer_discard: nil}} + {:next_state, state, %__MODULE__{data | value: {:error, :discarded}, timer_discard: nil}} end def handle_event(:cast, {:new_value, _key, {:ok, value}}, :running, data) do @@ -201,7 +200,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end {:next_state, :idle, - %CacheEntry{ + %__MODULE__{ data | running_func_task: :undefined, value: {:ok, value}, @@ -233,7 +232,7 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do {:ok, timer_hydrate} = :timer.send_after(data.refresh_interval, self(), :hydrate) {:next_state, :idle, - %CacheEntry{ + %__MODULE__{ data | running_func_task: :undefined, getters: [], @@ -247,7 +246,6 @@ defmodule Archethic.Utils.HydratingCache.CacheEntry do end defp maybe_stop_timer(tref = {_, _}) do - IO.puts("Cancelling timer #{inspect(tref)}") :timer.cancel(tref) end diff --git a/test/archethic/oracle_chain/scheduler_test.exs b/test/archethic/oracle_chain/scheduler_test.exs index e74e33ada..7cac55814 100644 --- a/test/archethic/oracle_chain/scheduler_test.exs +++ b/test/archethic/oracle_chain/scheduler_test.exs @@ -18,7 +18,7 @@ defmodule Archethic.OracleChain.SchedulerTest do alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData - alias Archethic.Utils.HydratingCache + alias ArchethicCache.HydratingCache import ArchethicCase, only: [setup_before_send_tx: 0] @@ -110,7 +110,7 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, _} = :sys.get_state(pid) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -118,7 +118,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -126,7 +126,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -203,7 +203,7 @@ defmodule Archethic.OracleChain.SchedulerTest do end) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -211,7 +211,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -219,7 +219,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -256,7 +256,7 @@ defmodule Archethic.OracleChain.SchedulerTest do }) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -264,7 +264,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -272,7 +272,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -380,7 +380,7 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, %{polling_timer: timer1}} = :sys.get_state(pid) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -388,7 +388,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -396,7 +396,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, diff --git a/test/archethic/oracle_chain/services/uco_price_test.exs b/test/archethic/oracle_chain/services/uco_price_test.exs index c86555eea..b7fa880b0 100644 --- a/test/archethic/oracle_chain/services/uco_price_test.exs +++ b/test/archethic/oracle_chain/services/uco_price_test.exs @@ -3,11 +3,11 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do alias Archethic.OracleChain.Services.UCOPrice - alias Archethic.Utils.HydratingCache + alias ArchethicCache.HydratingCache test "fetch/0 should retrieve some data and build a map with the oracle name in it" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -15,7 +15,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -23,7 +23,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -35,7 +35,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "fetch/0 should retrieve some data and build a map with the oracle name in it and keep the precision to 5" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -43,7 +43,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -51,7 +51,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -64,7 +64,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do describe "verify/1" do test "should return true if the prices are the good one" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"eur" => [0.10], "usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -72,7 +72,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"eur" => [0.20], "usd" => [0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -80,7 +80,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"eur" => [0.30], "usd" => [0.40]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -92,7 +92,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should return false if the prices have deviated" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -100,7 +100,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -108,7 +108,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -121,7 +121,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should return the median value when multiple providers queried" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -129,7 +129,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -137,7 +137,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.40], "eur" => [0.40]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -149,7 +149,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should return the average of median values when a even number of providers queried" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -157,7 +157,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20, 0.30], "eur" => [0.20, 0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -165,7 +165,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.40], "eur" => [0.40]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -177,7 +177,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "verify?/1 should return false when no data are returned from all providers" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [], "eur" => []}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -185,7 +185,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [], "eur" => []}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -193,7 +193,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [], "eur" => []}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -205,7 +205,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should report values even if a provider returns an error" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -215,7 +215,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ## If service returns an error, old value will be returned ## we are so inserting a previous value HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20], "eur" => [0.20]}} end, @@ -225,7 +225,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:error, :error_message} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -233,7 +233,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -245,7 +245,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should handle a service timing out" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -253,7 +253,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.20], "eur" => [0.20]}} end, @@ -263,7 +263,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> :timer.sleep(5_000) {:ok, {:error, :error_message}} @@ -274,7 +274,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, diff --git a/test/archethic/oracle_chain/services_test.exs b/test/archethic/oracle_chain/services_test.exs index ad94f3fd9..83d5012cf 100644 --- a/test/archethic/oracle_chain/services_test.exs +++ b/test/archethic/oracle_chain/services_test.exs @@ -2,13 +2,13 @@ defmodule Archethic.OracleChain.ServicesTest do use ExUnit.Case alias Archethic.OracleChain.Services - alias Archethic.Utils.HydratingCache + alias ArchethicCache.HydratingCache import Mox describe "fetch_new_data/1" do test "should return the new data when no previous content" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -16,7 +16,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -24,7 +24,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -36,7 +36,7 @@ defmodule Archethic.OracleChain.ServicesTest do test "should not return the new data when the previous content is the same" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -44,7 +44,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -52,7 +52,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -64,7 +64,7 @@ defmodule Archethic.OracleChain.ServicesTest do test "should return the new data when the previous content is not the same" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -72,7 +72,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -80,7 +80,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -94,7 +94,7 @@ defmodule Archethic.OracleChain.ServicesTest do test "verify_correctness?/1 should true when the data is correct" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -102,7 +102,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -110,7 +110,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, diff --git a/test/archethic/oracle_chain_test.exs b/test/archethic/oracle_chain_test.exs index 548239078..c06386c7d 100644 --- a/test/archethic/oracle_chain_test.exs +++ b/test/archethic/oracle_chain_test.exs @@ -6,13 +6,13 @@ defmodule Archethic.OracleChainTest do alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData - alias Archethic.Utils.HydratingCache + alias ArchethicCache.HydratingCache import Mox test "valid_services_content?/1 should verify the oracle transaction's content correctness" do HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -20,7 +20,7 @@ defmodule Archethic.OracleChainTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -28,7 +28,7 @@ defmodule Archethic.OracleChainTest do ) HydratingCache.register_function( - Archethic.Utils.HydratingCache.UcoPrice, + HydratingCache.UcoPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, diff --git a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs b/test/archethic_cache/hydrating_cache_test.exs similarity index 96% rename from test/archethic/utils/hydrating_cache/hydrating_cache_test.exs rename to test/archethic_cache/hydrating_cache_test.exs index 93084fce6..62a36638f 100644 --- a/test/archethic/utils/hydrating_cache/hydrating_cache_test.exs +++ b/test/archethic_cache/hydrating_cache_test.exs @@ -1,7 +1,7 @@ -defmodule HydratingCacheTest do - alias Archethic.Utils.HydratingCache +defmodule ArchethicCache.HydratingCacheTest do + alias ArchethicCache.HydratingCache + use ExUnit.Case - require Logger test "If `key` is not associated with any function, return `{:error, :not_registered}`" do {:ok, pid} = HydratingCache.start_link(:test_service) @@ -108,7 +108,6 @@ defmodule HydratingCacheTest do HydratingCache.register_function( pid, fn -> - Logger.info("Hydrating function sleeping 5 secs") :timer.sleep(5000) {:ok, 2} end, @@ -337,17 +336,14 @@ defmodule HydratingCacheTest do end def val_hydrating_function(value) do - Logger.debug("Hydrating value #{value}") {:ok, value} end def failval_hydrating_function(value) do - Logger.debug("Hydrating value #{value}") {:error, value} end def timed_hydrating_function(delay, value) do - Logger.debug("Timed Hydrating value #{value}") :timer.sleep(delay) {:ok, value} end From cf8282ebd87bf65699694860a3e3addb6fac61cd Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Wed, 1 Mar 2023 18:45:16 +0100 Subject: [PATCH 41/48] Refactor hydrating cache configuration --- config/config.exs | 19 ++++-- config/test.exs | 2 + .../oracle_chain/services/uco_price.ex | 2 +- lib/archethic/oracle_chain/supervisor.ex | 39 +++++++------ lib/archethic_cache/hydrating_cache.ex | 22 ++++--- .../archethic/oracle_chain/scheduler_test.exs | 24 ++++---- .../oracle_chain/services/uco_price_test.exs | 58 +++++++++---------- test/archethic/oracle_chain/services_test.exs | 25 ++++---- test/archethic/oracle_chain_test.exs | 8 +-- test/archethic_cache/hydrating_cache_test.exs | 12 ++-- test/test_helper.exs | 4 -- 11 files changed, 110 insertions(+), 105 deletions(-) diff --git a/config/config.exs b/config/config.exs index eb9500433..70e7c5d25 100644 --- a/config/config.exs +++ b/config/config.exs @@ -137,14 +137,21 @@ config :archethic, Archethic.OracleChain, uco: Archethic.OracleChain.Services.UCOPrice ] -## UCO Price Oracle configuration -## Format : {key, refresh_interval_ms, ttl_ms}, ...]} -## If ttl = :infinity, the cache will never expire +currencies = ["eur", "usd"] + config :archethic, Archethic.OracleChain.Services.UCOPrice, providers: [ - {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 60_000, :infinity}, - {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 60_000, :infinity}, - {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 60_000, :infinity} + # Format: {id, MFA, refresh_interval, ttl} + # Note: ID=MODULE only work as long as there isn't two providers using the same module + {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, + {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [currencies]}, 60_000, + :infinity}, + {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, + {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, [currencies]}, + 60_000, :infinity}, + {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, + {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, [currencies]}, + 60_000, :infinity} ] config :archethic, ArchethicWeb.FaucetController, diff --git a/config/test.exs b/config/test.exs index 5283128df..4809489ac 100755 --- a/config/test.exs +++ b/config/test.exs @@ -90,6 +90,8 @@ config :archethic, Archethic.OracleChain.Scheduler, polling_interval: "0 0 * * * *", summary_interval: "0 0 0 * * *" +config :archethic, Archethic.OracleChain.Services.UCOPrice, providers: [] + # -----Start-of-Networking-tests-configs----- config :archethic, Archethic.Networking, validate_node_ip: false diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index d40a82951..4bde2d1db 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -23,7 +23,7 @@ defmodule Archethic.OracleChain.Services.UCOPrice do {:ok, fetching_tasks_supervisor} = Task.Supervisor.start_link() ## retrieve prices from configured providers and filter results marked as errors prices = - HydratingCache.get_all(HydratingCache.UcoPrice) + HydratingCache.get_all(__MODULE__) ## Here stream looks like : [%{"eur"=>[0.44], "usd"=[0.32]}, ..., %{"eur"=>[0.42, 0.43], "usd"=[0.35]}] |> Enum.reduce(%{}, &agregate_providers_data/2) diff --git a/lib/archethic/oracle_chain/supervisor.ex b/lib/archethic/oracle_chain/supervisor.ex index 3f8d10e17..8f1c60cae 100644 --- a/lib/archethic/oracle_chain/supervisor.ex +++ b/lib/archethic/oracle_chain/supervisor.ex @@ -8,9 +8,8 @@ defmodule Archethic.OracleChain.Supervisor do alias Archethic.OracleChain.Scheduler alias Archethic.Utils - alias ArchethicCache.HydratingCache - @pairs ["usd", "eur"] + alias ArchethicCache.HydratingCache def start_link(args \\ []) do Supervisor.start_link(__MODULE__, args) @@ -19,22 +18,28 @@ defmodule Archethic.OracleChain.Supervisor do def init(_args) do scheduler_conf = Application.get_env(:archethic, Scheduler) - ## Cook hydrating cache parameters from configuration - uco_service_providers = - :archethic - |> Application.get_env(Archethic.OracleChain.Services.UCOPrice, []) - |> Keyword.get(:providers, []) - |> Enum.map(fn {mod, refresh_rate, ttl} -> - {mod, mod, :fetch, [@pairs], refresh_rate, ttl} - end) - - children = [ - {HydratingCache, [HydratingCache.UcoPrice, uco_service_providers]}, - MemTable, - MemTableLoader, - {Scheduler, scheduler_conf} - ] + children = + [ + MemTable, + MemTableLoader, + {Scheduler, scheduler_conf} + ] ++ self_hydrating_caches() Supervisor.init(Utils.configurable_children(children), strategy: :one_for_one) end + + # all oracle services should use a self-hydrating cache + defp self_hydrating_caches() do + Application.get_env(:archethic, Archethic.OracleChain, []) + |> Keyword.get(:services, []) + |> Keyword.values() + |> Enum.map(fn service_module -> + # we expect a list of providers in config for each services + providers = + Application.get_env(:archethic, service_module, []) + |> Keyword.get(:providers, []) + + {HydratingCache, [service_module, providers]} + end) + end end diff --git a/lib/archethic_cache/hydrating_cache.ex b/lib/archethic_cache/hydrating_cache.ex index 31663d661..54e501533 100644 --- a/lib/archethic_cache/hydrating_cache.ex +++ b/lib/archethic_cache/hydrating_cache.ex @@ -1,7 +1,7 @@ defmodule ArchethicCache.HydratingCache do @moduledoc """ GenServer implementing the hydrating cache itself. - There should be one Hydrating per service ( ex : UCO price, meteo etc...) + There should be one Hydrating per service ( ex : UCO price, weather forecast etc...) It receives queries from clients requesting the cache, and manage the cache entries FSMs """ alias __MODULE__.CacheEntry @@ -120,18 +120,16 @@ defmodule ArchethicCache.HydratingCache do strategy: :one_for_one ) - ## Create child for each initial key - child_specs = - initial_keys - |> Enum.map(fn {provider, mod, func, params, refresh_interval, ttl} -> - {CacheEntry, [fn -> apply(mod, func, params) end, provider, refresh_interval, ttl]} - end) - - ## Registering initial keys + ## create child for each initial key keys = - Enum.reduce(child_specs, %{}, fn child = {_, [_, provider, _, _]}, acc -> - {:ok, cache_entry} = DynamicSupervisor.start_child(keys_sup, child) - Map.put(acc, provider, cache_entry) + Enum.reduce(initial_keys, %{}, fn {key, {mod, func, args}, refresh_interval, ttl}, acc -> + {:ok, cache_entry} = + DynamicSupervisor.start_child( + keys_sup, + {CacheEntry, [fn -> apply(mod, func, args) end, key, refresh_interval, ttl]} + ) + + Map.put(acc, key, cache_entry) end) {:ok, %{keys: keys, keys_sup: keys_sup}} diff --git a/test/archethic/oracle_chain/scheduler_test.exs b/test/archethic/oracle_chain/scheduler_test.exs index 7cac55814..e16aa89e6 100644 --- a/test/archethic/oracle_chain/scheduler_test.exs +++ b/test/archethic/oracle_chain/scheduler_test.exs @@ -110,7 +110,7 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, _} = :sys.get_state(pid) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -118,7 +118,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -126,7 +126,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -203,7 +203,7 @@ defmodule Archethic.OracleChain.SchedulerTest do end) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -211,7 +211,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -219,7 +219,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -256,7 +256,7 @@ defmodule Archethic.OracleChain.SchedulerTest do }) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -264,7 +264,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -272,7 +272,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -380,7 +380,7 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, %{polling_timer: timer1}} = :sys.get_state(pid) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -388,7 +388,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -396,7 +396,7 @@ defmodule Archethic.OracleChain.SchedulerTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, diff --git a/test/archethic/oracle_chain/services/uco_price_test.exs b/test/archethic/oracle_chain/services/uco_price_test.exs index b7fa880b0..ffd6d557a 100644 --- a/test/archethic/oracle_chain/services/uco_price_test.exs +++ b/test/archethic/oracle_chain/services/uco_price_test.exs @@ -7,7 +7,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "fetch/0 should retrieve some data and build a map with the oracle name in it" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -15,7 +15,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -23,7 +23,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -35,7 +35,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "fetch/0 should retrieve some data and build a map with the oracle name in it and keep the precision to 5" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -43,7 +43,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -51,7 +51,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -64,7 +64,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do describe "verify/1" do test "should return true if the prices are the good one" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"eur" => [0.10], "usd" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -72,7 +72,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"eur" => [0.20], "usd" => [0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -80,7 +80,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"eur" => [0.30], "usd" => [0.40]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -92,7 +92,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should return false if the prices have deviated" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -100,7 +100,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -108,7 +108,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -121,7 +121,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should return the median value when multiple providers queried" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -129,7 +129,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -137,7 +137,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.40], "eur" => [0.40]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -149,7 +149,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should return the average of median values when a even number of providers queried" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -157,7 +157,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20, 0.30], "eur" => [0.20, 0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -165,7 +165,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.40], "eur" => [0.40]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -177,7 +177,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "verify?/1 should return false when no data are returned from all providers" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [], "eur" => []}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -185,7 +185,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [], "eur" => []}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -193,7 +193,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [], "eur" => []}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -205,7 +205,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should report values even if a provider returns an error" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -215,7 +215,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ## If service returns an error, old value will be returned ## we are so inserting a previous value HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20], "eur" => [0.20]}} end, @@ -225,7 +225,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:error, :error_message} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -233,7 +233,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -245,7 +245,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do test "should handle a service timing out" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -253,7 +253,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.20], "eur" => [0.20]}} end, @@ -263,7 +263,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> :timer.sleep(5_000) {:ok, {:error, :error_message}} @@ -274,7 +274,7 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, diff --git a/test/archethic/oracle_chain/services_test.exs b/test/archethic/oracle_chain/services_test.exs index 83d5012cf..f535d9867 100644 --- a/test/archethic/oracle_chain/services_test.exs +++ b/test/archethic/oracle_chain/services_test.exs @@ -3,12 +3,11 @@ defmodule Archethic.OracleChain.ServicesTest do alias Archethic.OracleChain.Services alias ArchethicCache.HydratingCache - import Mox describe "fetch_new_data/1" do test "should return the new data when no previous content" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -16,7 +15,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -24,7 +23,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -36,7 +35,7 @@ defmodule Archethic.OracleChain.ServicesTest do test "should not return the new data when the previous content is the same" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -44,7 +43,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -52,7 +51,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -64,7 +63,7 @@ defmodule Archethic.OracleChain.ServicesTest do test "should return the new data when the previous content is not the same" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -72,7 +71,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -80,7 +79,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, @@ -94,7 +93,7 @@ defmodule Archethic.OracleChain.ServicesTest do test "verify_correctness?/1 should true when the data is correct" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -102,7 +101,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -110,7 +109,7 @@ defmodule Archethic.OracleChain.ServicesTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, diff --git a/test/archethic/oracle_chain_test.exs b/test/archethic/oracle_chain_test.exs index c06386c7d..04991d379 100644 --- a/test/archethic/oracle_chain_test.exs +++ b/test/archethic/oracle_chain_test.exs @@ -8,11 +8,9 @@ defmodule Archethic.OracleChainTest do alias Archethic.TransactionChain.TransactionData alias ArchethicCache.HydratingCache - import Mox - test "valid_services_content?/1 should verify the oracle transaction's content correctness" do HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, 30_000, @@ -20,7 +18,7 @@ defmodule Archethic.OracleChainTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, 30_000, @@ -28,7 +26,7 @@ defmodule Archethic.OracleChainTest do ) HydratingCache.register_function( - HydratingCache.UcoPrice, + Archethic.OracleChain.Services.UCOPrice, fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, 30_000, diff --git a/test/archethic_cache/hydrating_cache_test.exs b/test/archethic_cache/hydrating_cache_test.exs index 62a36638f..b5231c336 100644 --- a/test/archethic_cache/hydrating_cache_test.exs +++ b/test/archethic_cache/hydrating_cache_test.exs @@ -312,9 +312,9 @@ defmodule ArchethicCache.HydratingCacheTest do test "can retrieve all values beside erroneous ones" do {:ok, pid} = HydratingCache.start_link(:test_service_get_all, [ - {"key1", __MODULE__, :val_hydrating_function, [10], 30_000, 40_000}, - {"key2", __MODULE__, :failval_hydrating_function, [20], 30_000, 40_000}, - {"key3", __MODULE__, :val_hydrating_function, [30], 30_000, 40_000} + {"key1", {__MODULE__, :val_hydrating_function, [10]}, 30_000, 40_000}, + {"key2", {__MODULE__, :failval_hydrating_function, [20]}, 30_000, 40_000}, + {"key3", {__MODULE__, :val_hydrating_function, [30]}, 30_000, 40_000} ]) :timer.sleep(1000) @@ -325,9 +325,9 @@ defmodule ArchethicCache.HydratingCacheTest do test "Retrieving all values supports delayed values" do {:ok, pid} = HydratingCache.start_link(:test_service_get_all_delayed, [ - {"key1", __MODULE__, :val_hydrating_function, [10], 30_000, 40_000}, - {"key2", __MODULE__, :timed_hydrating_function, [2000, 20], 30_000, 40_000}, - {"key3", __MODULE__, :failval_hydrating_function, [30], 30_000, 40_000} + {"key1", {__MODULE__, :val_hydrating_function, [10]}, 30_000, 40_000}, + {"key2", {__MODULE__, :timed_hydrating_function, [2000, 20]}, 30_000, 40_000}, + {"key3", {__MODULE__, :failval_hydrating_function, [30]}, 30_000, 40_000} ]) :timer.sleep(1000) diff --git a/test/test_helper.exs b/test/test_helper.exs index 33658baa4..12c512426 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -37,8 +37,4 @@ Mox.defmock(MockIPLookup, for: Archethic.Networking.IPLookup.Impl) Mox.defmock(MockRemoteDiscovery, for: Archethic.Networking.IPLookup.Impl) Mox.defmock(MockNATDiscovery, for: Archethic.Networking.IPLookup.Impl) -Mox.defmock(MockUCOPriceProvider1, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) -Mox.defmock(MockUCOPriceProvider2, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) -Mox.defmock(MockUCOPriceProvider3, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) - # -----End-of-Networking-Mocks ------ From 5b4a6c229550a66e32696571be2b69b42c16a568 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 2 Mar 2023 14:35:07 +0100 Subject: [PATCH 42/48] replace erlang timer with elixir's --- .../hydrating_cache/cache_entry.ex | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/lib/archethic_cache/hydrating_cache/cache_entry.ex b/lib/archethic_cache/hydrating_cache/cache_entry.ex index ab5da2cab..d78a4ac3d 100644 --- a/lib/archethic_cache/hydrating_cache/cache_entry.ex +++ b/lib/archethic_cache/hydrating_cache/cache_entry.ex @@ -33,7 +33,7 @@ defmodule ArchethicCache.HydratingCache.CacheEntry do @impl GenStateMachine def init([fun, key, refresh_interval, ttl]) do # start hydrating at the needed refresh interval - {:ok, timer} = :timer.send_after(refresh_interval, self(), :hydrate) + timer = Process.send_after(self(), :hydrate, refresh_interval) ## Hydrate the value {:ok, :running, @@ -97,7 +97,7 @@ defmodule ArchethicCache.HydratingCache.CacheEntry do _ = maybe_stop_timer(data.timer_discard) ## Start new timer to hydrate at refresh interval - {:ok, timer} = :timer.send_after(refresh_interval, self(), :hydrate) + timer = Process.send_after(self(), :hydrate, refresh_interval) ## We trigger the update ( to trigger or not could be set at registering option ) {:repeat_state, @@ -127,7 +127,7 @@ defmodule ArchethicCache.HydratingCache.CacheEntry do :refresh_interval => refresh_interval }) - {:ok, timer} = :timer.send_after(refresh_interval, self(), :hydrate) + timer = Process.send_after(self(), :hydrate, refresh_interval) ## We trigger the update ( to trigger or not could be set at registering option ) {:next_state, :running, @@ -180,7 +180,7 @@ defmodule ArchethicCache.HydratingCache.CacheEntry do _ = maybe_stop_timer(data.timer_discard) ## Start hydrating timer - {:ok, timer_hydrate} = :timer.send_after(data.refresh_interval, self(), :hydrate) + timer_hydrate = Process.send_after(self(), :hydrate, data.refresh_interval) ## notify waiting getters Enum.each(data.getters, fn {pid, _ref} -> @@ -190,13 +190,13 @@ defmodule ArchethicCache.HydratingCache.CacheEntry do ## Start timer to discard new value if needed me = self() - {:ok, timer_ttl} = + timer_ttl = case data.ttl do ttl when is_number(ttl) -> - :timer.send_after(ttl, me, :discarded) + Process.send_after(me, :discarded, ttl) _ -> - {:ok, nil} + nil end {:next_state, :idle, @@ -219,17 +219,17 @@ defmodule ArchethicCache.HydratingCache.CacheEntry do ## Error values can be discarded me = self() - {:ok, timer_ttl} = + timer_ttl = case data.ttl do ttl when is_number(ttl) -> - :timer.send_after(ttl, me, :discarded) + Process.send_after(me, :discarded, ttl) _ -> - {:ok, nil} + nil end ## Start hydrating timer - {:ok, timer_hydrate} = :timer.send_after(data.refresh_interval, self(), :hydrate) + timer_hydrate = Process.send_after(self(), :hydrate, data.refresh_interval) {:next_state, :idle, %__MODULE__{ @@ -245,11 +245,6 @@ defmodule ArchethicCache.HydratingCache.CacheEntry do {:keep_state, data} end - defp maybe_stop_timer(tref = {_, _}) do - :timer.cancel(tref) - end - - defp maybe_stop_timer(_else) do - :ok - end + defp maybe_stop_timer(tref) when is_reference(tref), do: Process.cancel_timer(tref) + defp maybe_stop_timer(_), do: :ok end From bc7fed68cb2e0bb5f91b7573f776dac35f0fd310 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 2 Mar 2023 14:41:45 +0100 Subject: [PATCH 43/48] remove obsolete mocks --- test/archethic/reward_test.exs | 15 --------------- test/test_helper.exs | 5 ----- 2 files changed, 20 deletions(-) diff --git a/test/archethic/reward_test.exs b/test/archethic/reward_test.exs index 677911fec..c882565e1 100644 --- a/test/archethic/reward_test.exs +++ b/test/archethic/reward_test.exs @@ -57,21 +57,6 @@ defmodule Archethic.RewardTest do end test "get_transfers should create transfer transaction" do - MockUCOPriceProvider1 - |> stub(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.10], "usd" => [0.10]}} - end) - - MockUCOPriceProvider2 - |> stub(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.10], "usd" => [0.10]}} - end) - - MockUCOPriceProvider3 - |> stub(:fetch, fn _pairs -> - {:ok, %{"eur" => [0.10], "usd" => [0.10]}} - end) - address = :crypto.strong_rand_bytes(32) token_address1 = :crypto.strong_rand_bytes(32) token_address2 = :crypto.strong_rand_bytes(32) diff --git a/test/test_helper.exs b/test/test_helper.exs index 12c512426..1b412afc3 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -23,11 +23,6 @@ Mox.defmock(MockCrypto.SharedSecretsKeystore, for: Archethic.Crypto.SharedSecret Mox.defmock(MockDB, for: Archethic.DB) Mox.defmock(MockGeoIP, for: Archethic.P2P.GeoPatch.GeoIP) - -Mox.defmock(MockUCOPriceProvider1, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) -Mox.defmock(MockUCOPriceProvider2, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) -Mox.defmock(MockUCOPriceProvider3, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) - Mox.defmock(MockMetricsCollector, for: Archethic.Metrics.Collector) # -----Start-of-Networking-Mocks----- From f1965ea8a1817fdd10a7522065d3fff306ae0fb9 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 2 Mar 2023 15:18:08 +0100 Subject: [PATCH 44/48] 300ms tests instead of 21s --- test/archethic_cache/hydrating_cache_test.exs | 444 ++++++++---------- 1 file changed, 193 insertions(+), 251 deletions(-) diff --git a/test/archethic_cache/hydrating_cache_test.exs b/test/archethic_cache/hydrating_cache_test.exs index b5231c336..6ddc2b0d8 100644 --- a/test/archethic_cache/hydrating_cache_test.exs +++ b/test/archethic_cache/hydrating_cache_test.exs @@ -11,22 +11,17 @@ defmodule ArchethicCache.HydratingCacheTest do test "If value stored, it is returned immediatly" do {:ok, pid} = HydratingCache.start_link(:test_service_normal) - result = - HydratingCache.register_function( - pid, - fn -> - {:ok, 1} - end, - "simple_func", - 10_000, - 15_000 - ) - - assert result == :ok - ## WAit a little to be sure value is registered and not being refreshed - :timer.sleep(500) - r = HydratingCache.get(pid, "simple_func", 10_000) - assert r == {:ok, 1} + HydratingCache.register_function( + pid, + fn -> + {:ok, 1} + end, + "simple_func", + 10_000, + :infinity + ) + + assert {:ok, 1} = HydratingCache.get(pid, "simple_func") end test "Hydrating function runs periodically" do @@ -34,179 +29,139 @@ defmodule ArchethicCache.HydratingCacheTest do :persistent_term.put("test", 1) - result = - HydratingCache.register_function( - pid, - fn -> - value = :persistent_term.get("test") - value = value + 1 - :persistent_term.put("test", value) - {:ok, value} - end, - "test_inc", - 1_000, - 50_000 - ) - - assert result == :ok - - :timer.sleep(3000) - {:ok, value} = HydratingCache.get(pid, "test_inc", 3000) - - assert value >= 3 + HydratingCache.register_function( + pid, + fn -> + value = :persistent_term.get("test") + value = value + 1 + :persistent_term.put("test", value) + {:ok, value} + end, + "test_inc", + 10, + :infinity + ) + + Process.sleep(50) + assert {:ok, value} = HydratingCache.get(pid, "test_inc") + assert value >= 5 end test "Update hydrating function while another one is running returns new hydrating value from new function" do {:ok, pid} = HydratingCache.start_link(:test_service_hydrating) - result = - HydratingCache.register_function( - pid, - fn -> - :timer.sleep(5000) - {:ok, 1} - end, - "test_reregister", - 10_000, - 50_000 - ) - - assert result == :ok - - _result = - HydratingCache.register_function( - pid, - fn -> - {:ok, 2} - end, - "test_reregister", - 10_000, - 50_000 - ) - - :timer.sleep(5000) - {:ok, value} = HydratingCache.get(pid, "test_reregister", 4000) - - assert value == 2 + HydratingCache.register_function( + pid, + fn -> + Process.sleep(5000) + {:ok, 1} + end, + "test_reregister", + 10_000, + :infinity + ) + + HydratingCache.register_function( + pid, + fn -> + {:ok, 2} + end, + "test_reregister", + 10_000, + :infinity + ) + + Process.sleep(50) + assert {:ok, 2} = HydratingCache.get(pid, "test_reregister") end test "Getting value while function is running and previous value is available returns value" do {:ok, pid} = HydratingCache.start_link(:test_service_running) - _ = - HydratingCache.register_function( - pid, - fn -> - {:ok, 1} - end, - "test_reregister", - 40_000, - 50_000 - ) - - _ = - HydratingCache.register_function( - pid, - fn -> - :timer.sleep(5000) - {:ok, 2} - end, - "test_reregister", - 40_000, - 50_000 - ) - - {:ok, value} = HydratingCache.get(pid, "test_reregister", 4000) - - assert value == 1 + HydratingCache.register_function( + pid, + fn -> + {:ok, 1} + end, + "test_reregister", + 10_000, + :infinity + ) + + HydratingCache.register_function( + pid, + fn -> + Process.sleep(5000) + {:ok, 2} + end, + "test_reregister", + 10_000, + :infinity + ) + + assert {:ok, 1} = HydratingCache.get(pid, "test_reregister") end test "Two hydrating function can run at same time" do {:ok, pid} = HydratingCache.start_link(:test_service_simultaneous) - _ = - HydratingCache.register_function( - pid, - fn -> - :timer.sleep(5000) - {:ok, :result_timed} - end, - "timed_value", - 70_000, - 80_000 - ) - - _ = - HydratingCache.register_function( - pid, - fn -> - {:ok, :result} - end, - "direct_value", - 70_000, - 80_000 - ) - - ## We query the value with timeout smaller than timed function - {:ok, _value} = HydratingCache.get(pid, "direct_value", 2000) - end - - test "Querying key while first refreshed will block the calling process until refreshed and provide the value" do - {:ok, pid} = HydratingCache.start_link(:test_service_refresh) - - _ = - HydratingCache.register_function( - pid, - fn -> - :timer.sleep(4000) - {:ok, :valid_result} - end, - "delayed_result", - 70_000, - 80_000 - ) - - ## We query the value with timeout smaller than timed function - assert {:ok, :valid_result} = HydratingCache.get(pid, "delayed_result", 5000) + HydratingCache.register_function( + pid, + fn -> + Process.sleep(5000) + {:ok, :result_timed} + end, + "timed_value", + 10_000, + :infinity + ) + + HydratingCache.register_function( + pid, + fn -> + {:ok, :result} + end, + "direct_value", + 10_000, + :infinity + ) + + assert {:ok, :result} = HydratingCache.get(pid, "direct_value") end test "Querying key while first refreshed will block the calling process until timeout" do {:ok, pid} = HydratingCache.start_link(:test_service_block) - _ = - HydratingCache.register_function( - pid, - fn -> - :timer.sleep(2000) - {:ok, :valid_result} - end, - "delayed_result", - 70_000, - 80_000 - ) + HydratingCache.register_function( + pid, + fn -> + Process.sleep(5000) + {:ok, :valid_result} + end, + "delayed_result", + 10_000, + :infinity + ) ## We query the value with timeout smaller than timed function - assert {:error, :timeout} = HydratingCache.get(pid, "delayed_result", 1000) + assert {:error, :timeout} = HydratingCache.get(pid, "delayed_result", 1) end test "Multiple process can wait for a delayed value" do {:ok, pid} = HydratingCache.start_link(:test_service_delayed) - _ = - HydratingCache.register_function( - pid, - fn -> - :timer.sleep(2_000) - {:ok, :valid_result} - end, - "delayed_result", - 70_000, - 80_000 - ) + HydratingCache.register_function( + pid, + fn -> + Process.sleep(100) + {:ok, :valid_result} + end, + "delayed_result", + 10_000, + :infinity + ) - ## We query the value with timeout smaller than timed function - results = - Task.async_stream(1..10, fn _ -> HydratingCache.get(pid, "delayed_result", 4000) end) + results = Task.async_stream(1..10, fn _ -> HydratingCache.get(pid, "delayed_result") end) assert Enum.all?(results, fn {:ok, {:ok, :valid_result}} -> true @@ -217,35 +172,30 @@ defmodule ArchethicCache.HydratingCacheTest do test "If hydrating function crash, key fsm will still be operationnal" do {:ok, pid} = HydratingCache.start_link(:test_service_crash) - _ = - HydratingCache.register_function( - pid, - fn -> - ## Exit hydrating function - exit(1) - {:ok, :badmatch} - end, - :key, - 70_000, - 80_000 - ) - - :timer.sleep(1000) - - _ = - HydratingCache.register_function( - pid, - fn -> - ## Trigger badmatch - {:ok, :value} - end, - :key, - 70_000, - 80_000 - ) - - result = HydratingCache.get(pid, :key, 3000) - assert result == {:ok, :value} + HydratingCache.register_function( + pid, + fn -> + ## Exit hydrating function + exit(1) + end, + :key, + 10_000, + :infinity + ) + + Process.sleep(1) + + HydratingCache.register_function( + pid, + fn -> + {:ok, :value} + end, + :key, + 10_000, + :infinity + ) + + assert {:ok, :value} = HydratingCache.get(pid, :key) end ## This could occur if hydrating function takes time to answer. @@ -254,85 +204,77 @@ defmodule ArchethicCache.HydratingCacheTest do test "value gets discarded after some time" do {:ok, pid} = HydratingCache.start_link(:test_service) - _ = - HydratingCache.register_function( - pid, - fn -> - case :persistent_term.get("flag", nil) do - 1 -> - :timer.sleep(3_000) - - nil -> - :persistent_term.put("flag", 1) - {:ok, :value} - end - end, - :key, - 500, - 1_000 - ) - - :timer.sleep(1_100) - result = HydratingCache.get(pid, :key, 3000) - assert result == {:error, :discarded} + HydratingCache.register_function( + pid, + fn -> + case :persistent_term.get("flag", nil) do + 1 -> + Process.sleep(5000) + + nil -> + :persistent_term.put("flag", 1) + {:ok, :value} + end + end, + :key, + 10, + 20 + ) + + assert {:ok, :value} = HydratingCache.get(pid, :key) + Process.sleep(25) + assert {:error, :discarded} = HydratingCache.get(pid, :key) end test "Can get a value while another request is waiting for results" do {:ok, pid} = HydratingCache.start_link(:test_service_wait) - _ = - HydratingCache.register_function( - pid, - fn -> - :timer.sleep(10_000) - {:ok, :value} - end, - :key1, - 30_000, - 40_000 - ) - - :erlang.spawn(fn -> HydratingCache.get(pid, :key1, 35_000) end) - - _ = - HydratingCache.register_function( - pid, - fn -> - {:ok, :value2} - end, - :key2, - 500, - 1_000 - ) - - result = HydratingCache.get(pid, :key2, 3000) - assert result == {:ok, :value2} + HydratingCache.register_function( + pid, + fn -> + Process.sleep(5000) + {:ok, :value} + end, + :key1, + 10_000, + :infinity + ) + + :erlang.spawn(fn -> HydratingCache.get(pid, :key1, 15_000) end) + + HydratingCache.register_function( + pid, + fn -> + {:ok, :value2} + end, + :key2, + 10_000, + :infinity + ) + + assert {:ok, :value2} = HydratingCache.get(pid, :key2) end test "can retrieve all values beside erroneous ones" do {:ok, pid} = HydratingCache.start_link(:test_service_get_all, [ - {"key1", {__MODULE__, :val_hydrating_function, [10]}, 30_000, 40_000}, - {"key2", {__MODULE__, :failval_hydrating_function, [20]}, 30_000, 40_000}, - {"key3", {__MODULE__, :val_hydrating_function, [30]}, 30_000, 40_000} + {"key1", {__MODULE__, :val_hydrating_function, [10]}, 10_00, :infinity}, + {"key2", {__MODULE__, :failval_hydrating_function, [20]}, 10_00, :infinity}, + {"key3", {__MODULE__, :val_hydrating_function, [30]}, 10_00, :infinity} ]) - :timer.sleep(1000) - values = HydratingCache.get_all(pid) - assert values == [10, 30] + assert [10, 30] = HydratingCache.get_all(pid) end test "Retrieving all values supports delayed values" do {:ok, pid} = HydratingCache.start_link(:test_service_get_all_delayed, [ - {"key1", {__MODULE__, :val_hydrating_function, [10]}, 30_000, 40_000}, - {"key2", {__MODULE__, :timed_hydrating_function, [2000, 20]}, 30_000, 40_000}, - {"key3", {__MODULE__, :failval_hydrating_function, [30]}, 30_000, 40_000} + {"key1", {__MODULE__, :val_hydrating_function, [10]}, 10_000, :infinity}, + {"key2", {__MODULE__, :timed_hydrating_function, [50, 20]}, 10_000, :infinity}, + {"key3", {__MODULE__, :failval_hydrating_function, [30]}, 10_000, :infinity} ]) - :timer.sleep(1000) - values = HydratingCache.get_all(pid) - assert values == [10, 20] + assert [10, 20] = HydratingCache.get_all(pid) end def val_hydrating_function(value) do @@ -344,7 +286,7 @@ defmodule ArchethicCache.HydratingCacheTest do end def timed_hydrating_function(delay, value) do - :timer.sleep(delay) + Process.sleep(delay) {:ok, value} end end From 3c895d840c9bf2a9b9cab0a3c625809f5855290d Mon Sep 17 00:00:00 2001 From: Samuel Manzanera Date: Fri, 10 Mar 2023 14:55:24 +0100 Subject: [PATCH 45/48] Refactor --- config/config.exs | 22 +- config/test.exs | 13 +- lib/archethic/oracle_chain/services.ex | 12 +- .../oracle_chain/services/cache_supervisor.ex | 16 + .../oracle_chain/services/hydrating_cache.ex | 120 +++++ lib/archethic/oracle_chain/services/impl.ex | 1 + .../services/provider_cache_supervisor.ex | 56 +++ .../oracle_chain/services/uco_price.ex | 27 +- lib/archethic/oracle_chain/supervisor.ex | 30 +- lib/archethic_cache/hydrating_cache.ex | 243 ---------- .../hydrating_cache/cache_entry.ex | 250 ----------- .../archethic/oracle_chain/scheduler_test.exs | 140 +----- .../services/hydrating_cache_test.exs | 104 +++++ .../oracle_chain/services/uco_price_test.exs | 419 ++++++++---------- test/archethic/oracle_chain/services_test.exs | 117 +---- test/archethic/oracle_chain_test.exs | 37 +- test/archethic_cache/hydrating_cache_test.exs | 292 ------------ test/test_helper.exs | 4 + 18 files changed, 593 insertions(+), 1310 deletions(-) create mode 100644 lib/archethic/oracle_chain/services/cache_supervisor.ex create mode 100644 lib/archethic/oracle_chain/services/hydrating_cache.ex create mode 100644 lib/archethic/oracle_chain/services/provider_cache_supervisor.ex delete mode 100644 lib/archethic_cache/hydrating_cache.ex delete mode 100644 lib/archethic_cache/hydrating_cache/cache_entry.ex create mode 100644 test/archethic/oracle_chain/services/hydrating_cache_test.exs delete mode 100644 test/archethic_cache/hydrating_cache_test.exs diff --git a/config/config.exs b/config/config.exs index 70e7c5d25..8c883d538 100644 --- a/config/config.exs +++ b/config/config.exs @@ -137,22 +137,14 @@ config :archethic, Archethic.OracleChain, uco: Archethic.OracleChain.Services.UCOPrice ] -currencies = ["eur", "usd"] - config :archethic, Archethic.OracleChain.Services.UCOPrice, - providers: [ - # Format: {id, MFA, refresh_interval, ttl} - # Note: ID=MODULE only work as long as there isn't two providers using the same module - {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - {Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, :fetch, [currencies]}, 60_000, - :infinity}, - {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - {Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, :fetch, [currencies]}, - 60_000, :infinity}, - {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - {Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, :fetch, [currencies]}, - 60_000, :infinity} - ] + providers: %{ + # Coingecko limits to 10-30 calls, with 30s delay we would be under the limitation + Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko => [refresh_interval: 30_000], + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap => [refresh_interval: 10_000], + # Coinpaprika limits to 25K req/mo; with 2min delay we can reach ~21K + Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika => [refresh_interval: 120_000] + } config :archethic, ArchethicWeb.FaucetController, seed: diff --git a/config/test.exs b/config/test.exs index 4809489ac..66a0f6fb6 100755 --- a/config/test.exs +++ b/config/test.exs @@ -90,7 +90,18 @@ config :archethic, Archethic.OracleChain.Scheduler, polling_interval: "0 0 * * * *", summary_interval: "0 0 0 * * *" -config :archethic, Archethic.OracleChain.Services.UCOPrice, providers: [] +config :archethic, Archethic.OracleChain.ServiceCacheSupervisor, enabled: false + +config :archethic, Archethic.OracleChain, + services: [ + uco: MockUCOPrice + ] + +config :archethic, Archethic.OracleChain.Services.UCOPrice, + providers: %{ + MockUCOProvider1 => [refresh_interval: 1000], + MockUCOProvider2 => [refresh_interval: 1000] + } # -----Start-of-Networking-tests-configs----- diff --git a/lib/archethic/oracle_chain/services.ex b/lib/archethic/oracle_chain/services.ex index 0a223251f..eeec98ac3 100644 --- a/lib/archethic/oracle_chain/services.ex +++ b/lib/archethic/oracle_chain/services.ex @@ -12,7 +12,7 @@ defmodule Archethic.OracleChain.Services do def fetch_new_data(previous_content \\ %{}) do Enum.map(services(), fn {service, handler} -> Logger.debug("Fetching #{service} oracle data...") - {service, apply(handler, :fetch, [])} + {service, handler.fetch()} end) |> Enum.filter(fn {service, {:ok, data}} -> @@ -82,4 +82,14 @@ defmodule Archethic.OracleChain.Services do defp services do Application.get_env(:archethic, Archethic.OracleChain) |> Keyword.fetch!(:services) end + + @doc """ + List all the service cache supervisor specs + """ + @spec cache_service_supervisor_specs() :: list(Supervisor.child_spec()) + def cache_service_supervisor_specs do + Enum.map(services(), fn {_service_name, handler} -> + handler.cache_child_spec() + end) + end end diff --git a/lib/archethic/oracle_chain/services/cache_supervisor.ex b/lib/archethic/oracle_chain/services/cache_supervisor.ex new file mode 100644 index 000000000..2a84a86e9 --- /dev/null +++ b/lib/archethic/oracle_chain/services/cache_supervisor.ex @@ -0,0 +1,16 @@ +defmodule Archethic.OracleChain.ServiceCacheSupervisor do + @moduledoc false + + use Supervisor + + alias Archethic.OracleChain.Services + + def start_link(arg \\ []) do + Supervisor.start_link(__MODULE__, arg) + end + + def init(_arg) do + children = Services.cache_service_supervisor_specs() + Supervisor.init(children, strategy: :one_for_one) + end +end diff --git a/lib/archethic/oracle_chain/services/hydrating_cache.ex b/lib/archethic/oracle_chain/services/hydrating_cache.ex new file mode 100644 index 000000000..c401ec25d --- /dev/null +++ b/lib/archethic/oracle_chain/services/hydrating_cache.ex @@ -0,0 +1,120 @@ +defmodule Archethic.OracleChain.Services.HydratingCache do + @moduledoc """ + This module is responsible for : + - Run the hydrating function associated with this key at a given interval + - Discard the value after some time + - Return the value when requested + """ + use GenServer + + defmodule State do + @moduledoc false + defstruct([ + :mfa, + :ttl, + :ttl_ref, + :refresh_interval, + :value, + :hydrating_task + ]) + end + + @spec start_link(keyword()) :: + {:ok, GenServer.on_start()} | {:error, term()} + def start_link(arg \\ []) do + GenServer.start_link(__MODULE__, arg, Keyword.take(arg, [:name])) + end + + @spec get(GenServer.server(), integer()) :: {:ok, any()} | :error + def get(server, timeout \\ 5000) do + try do + GenServer.call(server, :get, timeout) + catch + :exit, {:timeout, _} -> + :error + end + end + + def init(options) do + refresh_interval = Keyword.fetch!(options, :refresh_interval) + mfa = Keyword.fetch!(options, :mfa) + ttl = Keyword.get(options, :ttl, :infinity) + + # start hydrating as soon as init is done + Process.send_after(self(), :hydrate, 0) + + ## Hydrate the value + {:ok, + %State{ + mfa: mfa, + ttl: ttl, + refresh_interval: refresh_interval + }} + end + + def handle_call(:get, _from, state = %State{value: nil}) do + {:reply, :error, state} + end + + def handle_call(:get, _from, state = %State{value: value}) when value != nil do + {:reply, {:ok, value}, state} + end + + def handle_info( + :hydrate, + state = %State{ + refresh_interval: refresh_interval, + mfa: {m, f, a} + } + ) do + hydrating_task = + Task.async(fn -> + try do + {:ok, apply(m, f, a)} + rescue + _ -> + :error + end + end) + + # start a new hydrate timer + Process.send_after(self(), :hydrate, refresh_interval) + + {:noreply, %State{state | hydrating_task: hydrating_task}} + end + + def handle_info( + {ref, result}, + state = %State{ttl_ref: ttl_ref, ttl: ttl, hydrating_task: %Task{ref: ref_task}} + ) + when ref == ref_task do + # cancel current ttl if any + if is_reference(ttl_ref) do + Process.cancel_timer(ttl_ref) + end + + # start new ttl timer + ttl_ref = + if is_integer(ttl) do + Process.send_after(self(), :discard_value, ttl) + else + nil + end + + new_state = %{state | ttl_ref: ttl_ref, hydrating_task: nil} + + case result do + {:ok, value} -> + {:noreply, %{new_state | value: value}} + + _ -> + {:noreply, new_state} + end + end + + def handle_info({:DOWN, _ref, :process, _, _}, state), do: {:noreply, state} + + def handle_info(:discard_value, state) do + {:noreply, %State{state | value: nil, ttl_ref: nil}} + end +end diff --git a/lib/archethic/oracle_chain/services/impl.ex b/lib/archethic/oracle_chain/services/impl.ex index 6510c609d..3d25681d7 100644 --- a/lib/archethic/oracle_chain/services/impl.ex +++ b/lib/archethic/oracle_chain/services/impl.ex @@ -1,6 +1,7 @@ defmodule Archethic.OracleChain.Services.Impl do @moduledoc false + @callback cache_child_spec() :: Supervisor.child_spec() @callback fetch() :: {:ok, %{required(String.t()) => any()}} | {:error, any()} @callback verify?(%{required(String.t()) => any()}) :: boolean @callback parse_data(map()) :: {:ok, map()} | :error diff --git a/lib/archethic/oracle_chain/services/provider_cache_supervisor.ex b/lib/archethic/oracle_chain/services/provider_cache_supervisor.ex new file mode 100644 index 000000000..0cbaa6b07 --- /dev/null +++ b/lib/archethic/oracle_chain/services/provider_cache_supervisor.ex @@ -0,0 +1,56 @@ +defmodule Archethic.OracleChain.Services.ProviderCacheSupervisor do + @moduledoc """ + Supervise the several self-hydrating cache for the providers + """ + + use Supervisor + + alias Archethic.OracleChain.Services.HydratingCache + + def start_link(arg) do + Supervisor.start_link(__MODULE__, arg, name: __MODULE__) + end + + def init(arg) do + fetch_args = Keyword.fetch!(arg, :fetch_args) + providers = Keyword.fetch!(arg, :providers) + + provider_child_specs = + Enum.map(providers, fn {provider, opts} -> + refresh_interval = Keyword.get(opts, :refresh_interval, 60_000) + + Supervisor.child_spec( + {HydratingCache, + [ + refresh_interval: refresh_interval, + mfa: {provider, :fetch, [fetch_args]}, + name: cache_name(provider) + ]}, + id: cache_name(provider) + ) + end) + + children = provider_child_specs + + Supervisor.init( + children, + strategy: :one_for_one + ) + end + + defp cache_name(module), do: :"#{module}Cache" + + @doc """ + Return the values from the several provider caches + """ + @spec get_values(list(module())) :: list(any()) + def get_values(providers) do + providers + |> Enum.map(fn {provider, _} -> cache_name(provider) end) + |> Enum.map(&HydratingCache.get/1) + |> Enum.filter(&match?({:ok, {:ok, _}}, &1)) + |> Enum.map(fn + {:ok, {:ok, val}} -> val + end) + end +end diff --git a/lib/archethic/oracle_chain/services/uco_price.ex b/lib/archethic/oracle_chain/services/uco_price.ex index 4bde2d1db..888f51445 100644 --- a/lib/archethic/oracle_chain/services/uco_price.ex +++ b/lib/archethic/oracle_chain/services/uco_price.ex @@ -6,9 +6,9 @@ defmodule Archethic.OracleChain.Services.UCOPrice do require Logger alias Archethic.OracleChain.Services.Impl - alias Archethic.Utils + alias Archethic.OracleChain.Services.ProviderCacheSupervisor - alias ArchethicCache.HydratingCache + alias Archethic.Utils @behaviour Impl @@ -16,16 +16,25 @@ defmodule Archethic.OracleChain.Services.UCOPrice do @pairs ["usd", "eur"] + @impl Impl + def cache_child_spec do + Supervisor.child_spec({ProviderCacheSupervisor, providers: providers(), fetch_args: @pairs}, + id: CacheSupervisor + ) + end + + defp providers do + Application.get_env(:archethic, __MODULE__) |> Keyword.fetch!(:providers) + end + @impl Impl @spec fetch() :: {:ok, %{required(String.t()) => any()}} | {:error, any()} def fetch do - ## Start a supervisor for the feching tasks - {:ok, fetching_tasks_supervisor} = Task.Supervisor.start_link() - ## retrieve prices from configured providers and filter results marked as errors + # retrieve prices from configured providers and filter results marked as errors + # Here stream looks like : [%{"eur"=>[0.44], "usd"=[0.32]}, ..., %{"eur"=>[0.42, 0.43], "usd"=[0.35]}] prices = - HydratingCache.get_all(__MODULE__) - - ## Here stream looks like : [%{"eur"=>[0.44], "usd"=[0.32]}, ..., %{"eur"=>[0.42, 0.43], "usd"=[0.35]}] + providers() + |> ProviderCacheSupervisor.get_values() |> Enum.reduce(%{}, &agregate_providers_data/2) |> Enum.reduce(%{}, fn {currency, values}, acc -> price = @@ -41,11 +50,9 @@ defmodule Archethic.OracleChain.Services.UCOPrice do Map.put(acc, currency, price) end) - Supervisor.stop(fetching_tasks_supervisor, :normal, 3_000) {:ok, prices} end - @spec agregate_providers_data(map(), map()) :: map() defp agregate_providers_data(provider_results, acc) do provider_results |> Enum.reduce(acc, fn diff --git a/lib/archethic/oracle_chain/supervisor.ex b/lib/archethic/oracle_chain/supervisor.ex index 8f1c60cae..bc2337541 100644 --- a/lib/archethic/oracle_chain/supervisor.ex +++ b/lib/archethic/oracle_chain/supervisor.ex @@ -6,11 +6,10 @@ defmodule Archethic.OracleChain.Supervisor do alias Archethic.OracleChain.MemTable alias Archethic.OracleChain.MemTableLoader alias Archethic.OracleChain.Scheduler + alias Archethic.OracleChain.ServiceCacheSupervisor alias Archethic.Utils - alias ArchethicCache.HydratingCache - def start_link(args \\ []) do Supervisor.start_link(__MODULE__, args) end @@ -18,28 +17,13 @@ defmodule Archethic.OracleChain.Supervisor do def init(_args) do scheduler_conf = Application.get_env(:archethic, Scheduler) - children = - [ - MemTable, - MemTableLoader, - {Scheduler, scheduler_conf} - ] ++ self_hydrating_caches() + children = [ + MemTable, + MemTableLoader, + {Scheduler, scheduler_conf}, + ServiceCacheSupervisor + ] Supervisor.init(Utils.configurable_children(children), strategy: :one_for_one) end - - # all oracle services should use a self-hydrating cache - defp self_hydrating_caches() do - Application.get_env(:archethic, Archethic.OracleChain, []) - |> Keyword.get(:services, []) - |> Keyword.values() - |> Enum.map(fn service_module -> - # we expect a list of providers in config for each services - providers = - Application.get_env(:archethic, service_module, []) - |> Keyword.get(:providers, []) - - {HydratingCache, [service_module, providers]} - end) - end end diff --git a/lib/archethic_cache/hydrating_cache.ex b/lib/archethic_cache/hydrating_cache.ex deleted file mode 100644 index 54e501533..000000000 --- a/lib/archethic_cache/hydrating_cache.ex +++ /dev/null @@ -1,243 +0,0 @@ -defmodule ArchethicCache.HydratingCache do - @moduledoc """ - GenServer implementing the hydrating cache itself. - There should be one Hydrating per service ( ex : UCO price, weather forecast etc...) - It receives queries from clients requesting the cache, and manage the cache entries FSMs - """ - alias __MODULE__.CacheEntry - - use GenServer - @vsn Mix.Project.config()[:version] - - require Logger - - @type result :: - {:ok, any()} - | {:error, :timeout} - | {:error, :not_registered} - - def start_link([name, initial_keys]) do - GenServer.start_link(__MODULE__, [name, initial_keys], name: name) - end - - def start_link(name, initial_keys \\ []) do - start_link([name, initial_keys]) - end - - @doc ~s""" - Registers a function that will be computed periodically to update the cache. - - Arguments: - - `hydrating_cache`: the pid of the hydrating cache. - - `fun`: a 0-arity function that computes the value and returns either - `{:ok, value}` or `{:error, reason}`. - - `key`: associated with the function and is used to retrieve the stored - value. - - `refresh_interval`: how often (in milliseconds) the function is - recomputed and the new value stored. `refresh_interval` must be strictly - smaller than `ttl`. After the value is refreshed, the `ttl` counter is - restarted. - - `ttl` ("time to live"): how long (in milliseconds) the value is stored - before it is discarded if the value is not refreshed. - - - The value is stored only if `{:ok, value}` is returned by `fun`. If `{:error, - reason}` is returned, the value is not stored and `fun` must be retried on - the next run. - """ - @spec register_function( - hydrating_cache :: pid(), - fun :: (() -> {:ok, any()} | {:error, any()}), - key :: any, - refresh_interval :: non_neg_integer(), - ttl :: non_neg_integer() | :infinity - ) :: term() - def register_function(hydrating_cache, fun, key, refresh_interval, ttl) - when is_function(fun, 0) and - is_integer(refresh_interval) and - (refresh_interval < ttl or ttl == :infinity) do - GenServer.call(hydrating_cache, {:register, fun, key, refresh_interval, ttl}) - end - - @doc ~s""" - Get the value associated with `key`. - - Details: - - If the value for `key` is stored in the cache, the value is returned - immediately. - - If a recomputation of the function is in progress, the last stored value - is returned. - - If the value for `key` is not stored in the cache but a computation of - the function associated with this `key` is in progress, wait up to - `timeout` milliseconds. If the value is computed within this interval, - the value is returned. If the computation does not finish in this - interval, `{:error, :timeout}` is returned. - - If `key` is not associated with any function, return `{:error, - :not_registered}` - """ - @spec get(atom(), any(), non_neg_integer()) :: {:ok, term()} | {:error, atom()} - def get(hydrating_cache, key, timeout \\ 1_000) - when is_integer(timeout) and timeout > 0 do - Logger.debug( - "Getting key #{inspect(key)} from hydrating cache #{inspect(hydrating_cache)} for #{inspect(self())}" - ) - - case GenServer.call(hydrating_cache, {:get, key}, timeout) do - {:ok, :answer_delayed} -> - receive do - {:delayed_value, _, value} -> - value - - other -> - Logger.warning("Unexpected return value #{inspect(other)}") - {:error, :unexpected_value} - after - timeout -> - Logger.warning( - "Timeout waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(hydrating_cache)}" - ) - - {:error, :timeout} - end - - other_result -> - other_result - end - end - - def get_all(hydrating_cache) do - GenServer.call(hydrating_cache, :get_all) - end - - @impl GenServer - def init([name, initial_keys]) do - Logger.info("Starting Hydrating cache for service #{inspect("#{__MODULE__}.#{name}")}") - - ## start a dynamic supervisor for the cache entries/keys - {:ok, keys_sup} = - DynamicSupervisor.start_link( - name: :"ArchethicCache.HydratingCache.CacheEntry.KeysSupervisor.#{name}", - strategy: :one_for_one - ) - - ## create child for each initial key - keys = - Enum.reduce(initial_keys, %{}, fn {key, {mod, func, args}, refresh_interval, ttl}, acc -> - {:ok, cache_entry} = - DynamicSupervisor.start_child( - keys_sup, - {CacheEntry, [fn -> apply(mod, func, args) end, key, refresh_interval, ttl]} - ) - - Map.put(acc, key, cache_entry) - end) - - {:ok, %{keys: keys, keys_sup: keys_sup}} - end - - @impl GenServer - def handle_call({:get, key}, from, state = %{keys: keys}) do - case Map.get(keys, key) do - nil -> - Logger.warning("HydratingCache no entry for #{inspect(key)}") - {:reply, {:error, :not_registered}, state} - - pid -> - value = GenStateMachine.call(pid, {:get, from}) - {:reply, value, state} - end - end - - def handle_call(:get_all, _from, state = %{keys: keys}) do - Logger.debug( - "Getting all keys from hydrating cache, current keys are #{inspect(keys)} sup: #{inspect(state.keys_sup)}" - ) - - {:ok, fetching_values_supervisor} = Task.Supervisor.start_link() - - result = - Task.Supervisor.async_stream_nolink( - fetching_values_supervisor, - keys, - fn {key, pid} -> - case GenStateMachine.call(pid, {:get, {self(), nil}}) do - {:ok, :answer_delayed} -> - receive do - {:delayed_value, _, {:ok, value}} -> - Logger.debug( - "Got delayed value for key #{inspect(value)} from hydrating cache #{inspect(self())}" - ) - - {:ok, value} - - other -> - Logger.warning("Unexpected return value #{inspect(other)}") - {:error, :unexpected_value} - after - 3_000 -> - Logger.warning( - "Timeout waiting for delayed value for key #{inspect(key)} from hydrating cache #{inspect(self())}" - ) - - {:error, :timeout} - end - - other -> - other - end - end, - on_timeout: :kill_task - ) - |> Stream.filter(&match?({:ok, {:ok, _}}, &1)) - |> Stream.map(fn - {_, {_, result}} -> - result - end) - |> Enum.to_list() - - {:reply, result, state} - end - - def handle_call({:register, fun, key, refresh_interval, ttl}, _from, state = %{keys: keys}) do - Logger.debug("Registering hydrating function for #{inspect(key)}") - ## Called when asked to register a function - case Map.get(keys, key) do - nil -> - ## New key, we start a cache entry fsm - {:ok, pid} = - DynamicSupervisor.start_child( - state.keys_sup, - {CacheEntry, [fun, key, refresh_interval, ttl]} - ) - - {:reply, :ok, %{state | keys: Map.put(keys, key, pid)}} - - pid -> - ## Key already exists, no need to start fsm - case GenStateMachine.call(pid, {:register, fun, key, refresh_interval, ttl}) do - :ok -> - {:reply, :ok, %{state | keys: Map.put(keys, key, pid)}} - - error -> - {:reply, {:error, error}, state} - end - end - end - - def handle_call(unmanaged, _from, state) do - Logger.warning("Cache received unmanaged call: #{inspect(unmanaged)}") - {:reply, :ok, state} - end - - @impl GenServer - def handle_cast(unmanaged, state) do - Logger.warning("Cache received unmanaged cast: #{inspect(unmanaged)}") - {:noreply, state} - end - - @impl GenServer - def handle_info(unmanaged, state) do - Logger.warning("Cache received unmanaged info: #{inspect(unmanaged)}") - {:noreply, state} - end -end diff --git a/lib/archethic_cache/hydrating_cache/cache_entry.ex b/lib/archethic_cache/hydrating_cache/cache_entry.ex deleted file mode 100644 index d78a4ac3d..000000000 --- a/lib/archethic_cache/hydrating_cache/cache_entry.ex +++ /dev/null @@ -1,250 +0,0 @@ -defmodule ArchethicCache.HydratingCache.CacheEntry do - @moduledoc """ - This module is a finite state machine implementing a cache entry. - There is one such Cache Entry FSM running per registered key. - - It is responsible for : - - receiving request to get the value for the key it is associated with - - Run the hydrating function associated with this key - - Manage various timers - """ - use GenStateMachine, callback_mode: [:handle_event_function, :state_enter] - - use Task - require Logger - - defstruct([ - :running_func_task, - :hydrating_func, - :ttl, - :refresh_interval, - :key, - :timer_func, - :timer_discard, - getters: [], - value: :"$$undefined" - ]) - - @spec start_link([...]) :: :ignore | {:error, any} | {:ok, pid} - def start_link([fun, key, refresh_interval, ttl]) do - GenStateMachine.start_link(__MODULE__, [fun, key, refresh_interval, ttl], []) - end - - @impl GenStateMachine - def init([fun, key, refresh_interval, ttl]) do - # start hydrating at the needed refresh interval - timer = Process.send_after(self(), :hydrate, refresh_interval) - - ## Hydrate the value - {:ok, :running, - %__MODULE__{ - timer_func: timer, - hydrating_func: fun, - key: key, - ttl: ttl, - refresh_interval: refresh_interval - }} - end - - @impl GenStateMachine - def handle_event( - {:call, from}, - {:get, _requester}, - :idle, - data = %__MODULE__{:value => :"$$undefined"} - ) do - ## Value is requested while fsm is iddle, return the value - {:keep_state, data, [{:reply, from, {:error, :not_initialized}}]} - end - - def handle_event({:call, from}, {:get, _requester}, :idle, data) do - ## Value is requested while fsm is iddle, return the value - {:next_state, :idle, data, [{:reply, from, data.value}]} - end - - ## Call for value while hydrating function is running and we have no previous value - ## We register the caller to send value later on, and we indicate caller to block - def handle_event( - {:call, from}, - {:get, requester}, - :running, - data = %__MODULE__{value: :"$$undefined"} - ) do - previous_getters = data.getters - Logger.warning("Get Value but undefined #{inspect(data)}") - - {:keep_state, %__MODULE__{data | getters: previous_getters ++ [requester]}, - [{:reply, from, {:ok, :answer_delayed}}]} - end - - ## Call for value while hydrating function is running and we have a previous value - ## We return the value to caller - def handle_event({:call, from}, {:get, _requester}, :running, data) do - {:next_state, :running, data, [{:reply, from, data.value}]} - end - - def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, :running, data) do - ## Registering a new hydrating function while previous one is running - - ## We stop the hydrating task if it is already running - case data.running_func_task do - pid when is_pid(pid) -> Process.exit(pid, :normal) - _ -> :ok - end - - ## And the timers triggering it and discarding value - _ = maybe_stop_timer(data.timer_func) - _ = maybe_stop_timer(data.timer_discard) - - ## Start new timer to hydrate at refresh interval - timer = Process.send_after(self(), :hydrate, refresh_interval) - - ## We trigger the update ( to trigger or not could be set at registering option ) - {:repeat_state, - %__MODULE__{ - data - | hydrating_func: fun, - key: key, - ttl: ttl, - refresh_interval: refresh_interval, - timer_func: timer - }, [{:reply, from, :ok}]} - end - - def handle_event({:call, from}, {:register, fun, key, refresh_interval, ttl}, _state, data) do - Logger.info("Registering hydrating function for key :#{inspect(key)}") - - ## Setting hydrating function in other cases - ## Hydrating function not running, we just stop the timers - _ = maybe_stop_timer(data.timer_func) - _ = maybe_stop_timer(data.timer_discard) - - ## Fill state with new hydrating function parameters - data = - Map.merge(data, %{ - :hydrating_func => fun, - :ttl => ttl, - :refresh_interval => refresh_interval - }) - - timer = Process.send_after(self(), :hydrate, refresh_interval) - - ## We trigger the update ( to trigger or not could be set at registering option ) - {:next_state, :running, - %__MODULE__{ - data - | hydrating_func: fun, - key: key, - ttl: ttl, - refresh_interval: refresh_interval, - timer_func: timer - }, [{:reply, from, :ok}]} - end - - def handle_event(:info, :hydrate, :idle, data) do - ## Time to rehydrate - ## Hydrating the key, go to running state - {:next_state, :running, data} - end - - def handle_event(:enter, _event, :running, data) do - Logger.info("Entering running state for key :#{inspect(data.key)}") - ## At entering running state, we start the hydrating task - me = self() - - hydrating_task = - spawn(fn -> - Logger.info("Running hydrating function for key :#{inspect(data.key)}") - value = data.hydrating_func.() - GenStateMachine.cast(me, {:new_value, data.key, value}) - end) - - ## we stay in running state - {:next_state, :running, %__MODULE__{data | running_func_task: hydrating_task}} - end - - def handle_event(:info, :discarded, state, data) do - ## Value is discarded - - Logger.warning( - "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} discarded" - ) - - {:next_state, state, %__MODULE__{data | value: {:error, :discarded}, timer_discard: nil}} - end - - def handle_event(:cast, {:new_value, _key, {:ok, value}}, :running, data) do - ## We got result from hydrating function - Logger.debug("Got new value for key :#{inspect(data.key)} #{inspect(value)}") - ## Stop timer on value ttl - _ = maybe_stop_timer(data.timer_discard) - - ## Start hydrating timer - timer_hydrate = Process.send_after(self(), :hydrate, data.refresh_interval) - - ## notify waiting getters - Enum.each(data.getters, fn {pid, _ref} -> - send(pid, {:delayed_value, data.key, {:ok, value}}) - end) - - ## Start timer to discard new value if needed - me = self() - - timer_ttl = - case data.ttl do - ttl when is_number(ttl) -> - Process.send_after(me, :discarded, ttl) - - _ -> - nil - end - - {:next_state, :idle, - %__MODULE__{ - data - | running_func_task: :undefined, - value: {:ok, value}, - getters: [], - timer_func: timer_hydrate, - timer_discard: timer_ttl - }} - end - - def handle_event(:cast, {:new_value, key, {:error, reason}}, :running, data) do - ## Got error new value for key - Logger.warning( - "Key :#{inspect(data.key)}, Hydrating func #{inspect(data.hydrating_func)} got error value #{inspect({key, {:error, reason}})}" - ) - - ## Error values can be discarded - me = self() - - timer_ttl = - case data.ttl do - ttl when is_number(ttl) -> - Process.send_after(me, :discarded, ttl) - - _ -> - nil - end - - ## Start hydrating timer - timer_hydrate = Process.send_after(self(), :hydrate, data.refresh_interval) - - {:next_state, :idle, - %__MODULE__{ - data - | running_func_task: :undefined, - getters: [], - timer_func: timer_hydrate, - timer_discard: timer_ttl - }} - end - - def handle_event(_type, _event, _state, data) do - {:keep_state, data} - end - - defp maybe_stop_timer(tref) when is_reference(tref), do: Process.cancel_timer(tref) - defp maybe_stop_timer(_), do: :ok -end diff --git a/test/archethic/oracle_chain/scheduler_test.exs b/test/archethic/oracle_chain/scheduler_test.exs index e16aa89e6..6d47b405c 100644 --- a/test/archethic/oracle_chain/scheduler_test.exs +++ b/test/archethic/oracle_chain/scheduler_test.exs @@ -109,42 +109,20 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, _} = :sys.get_state(pid) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) - - # polling_date = - # "0 * * * *" - # |> Crontab.CronExpression.Parser.parse!(true) - # |> Crontab.Scheduler.get_next_run_date!(DateTime.to_naive(DateTime.utc_now())) - # |> DateTime.from_naive!("Etc/UTC") - summary_date = "0 0 0 * *" |> Crontab.CronExpression.Parser.parse!(true) |> Crontab.Scheduler.get_next_run_date!(DateTime.to_naive(DateTime.utc_now())) |> DateTime.from_naive!("Etc/UTC") + MockUCOPrice + |> expect(:fetch, fn -> + {:ok, %{usd: 0.2}} + end) + |> expect(:parse_data, fn _ -> + {:ok, %{usd: 0.2}} + end) + send(pid, :poll) assert_receive {:transaction_sent, @@ -186,6 +164,11 @@ defmodule Archethic.OracleChain.SchedulerTest do available?: true }) + MockUCOPrice + |> expect(:fetch, fn -> + {:ok, %{usd: 0.2}} + end) + MockDB |> expect(:get_transaction, fn _, _, _ -> {:ok, @@ -202,30 +185,6 @@ defmodule Archethic.OracleChain.SchedulerTest do }} end) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) - send(pid, :poll) refute_receive {:transaction_sent, _} @@ -255,29 +214,13 @@ defmodule Archethic.OracleChain.SchedulerTest do available?: true }) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + MockUCOPrice + |> expect(:fetch, fn -> + {:ok, %{usd: 0.2}} + end) + |> expect(:parse_data, fn _ -> + {:ok, %{usd: 0.2}} + end) summary_date = "0 0 0 * *" @@ -285,13 +228,6 @@ defmodule Archethic.OracleChain.SchedulerTest do |> Crontab.Scheduler.get_next_run_date!(DateTime.to_naive(DateTime.utc_now())) |> DateTime.from_naive!("Etc/UTC") - # summary_date2 = - # "0 0 0 * *" - # |> Crontab.CronExpression.Parser.parse!(true) - # |> Crontab.Scheduler.get_next_run_dates(DateTime.to_naive(DateTime.utc_now())) - # |> Enum.at(1) - # |> DateTime.from_naive!("Etc/UTC") - MockDB |> expect(:get_transaction_chain, fn _, _, _ -> {[ @@ -345,11 +281,6 @@ defmodule Archethic.OracleChain.SchedulerTest do data: %TransactionData{content: content} }} - # assert polling_address == - # Crypto.derive_oracle_keypair(summary_date2, 1) - # |> elem(0) - # |> Crypto.hash() - # assert {:ok, %{"uco" => %{"usd" => 0.2}}} = Services.parse_data(Jason.decode!(content)) end @@ -379,29 +310,10 @@ defmodule Archethic.OracleChain.SchedulerTest do assert {:scheduled, %{polling_timer: timer1}} = :sys.get_state(pid) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + MockUCOPrice + |> expect(:fetch, fn -> + {:ok, %{usd: 0.2}} + end) send(pid, :poll) @@ -527,8 +439,4 @@ defmodule Archethic.OracleChain.SchedulerTest do :persistent_term.put(:archethic_up, nil) end end - - def fetch(values) do - values - end end diff --git a/test/archethic/oracle_chain/services/hydrating_cache_test.exs b/test/archethic/oracle_chain/services/hydrating_cache_test.exs new file mode 100644 index 000000000..1e2233886 --- /dev/null +++ b/test/archethic/oracle_chain/services/hydrating_cache_test.exs @@ -0,0 +1,104 @@ +defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do + alias Archethic.OracleChain.Services.HydratingCache + + use ExUnit.Case + + test "should receive the same value until next refresh" do + {:ok, pid} = + HydratingCache.start_link( + mfa: {DateTime, :utc_now, []}, + refresh_interval: 100, + ttl: :infinity + ) + + # 1ms required just so it has the time to receive the :hydrate msg + Process.sleep(1) + {:ok, date} = HydratingCache.get(pid) + + Process.sleep(10) + assert {:ok, ^date} = HydratingCache.get(pid) + + Process.sleep(10) + assert {:ok, ^date} = HydratingCache.get(pid) + + Process.sleep(10) + assert {:ok, ^date} = HydratingCache.get(pid) + + Process.sleep(10) + assert {:ok, ^date} = HydratingCache.get(pid) + + Process.sleep(100) + {:ok, date2} = HydratingCache.get(pid) + assert date != date2 + end + + test "should discard the value after the ttl is reached" do + {:ok, pid} = + HydratingCache.start_link( + mfa: {DateTime, :utc_now, []}, + refresh_interval: 100, + ttl: 50 + ) + + # 1ms required just so it has the time to receive the :hydrate msg + Process.sleep(1) + {:ok, _date} = HydratingCache.get(pid) + + Process.sleep(50) + assert :error = HydratingCache.get(pid) + end + + test "should refresh the value after a discard" do + {:ok, pid} = + HydratingCache.start_link( + mfa: {DateTime, :utc_now, []}, + refresh_interval: 100, + ttl: 50 + ) + + # 1ms required just so it has the time to receive the :hydrate msg + Process.sleep(1) + {:ok, date} = HydratingCache.get(pid) + + Process.sleep(50) + assert :error = HydratingCache.get(pid) + + Process.sleep(50) + {:ok, date2} = HydratingCache.get(pid) + + assert date != date2 + end + + test "should not crash if the module is undefined" do + {:ok, pid} = + HydratingCache.start_link( + mfa: {NonExisting, :function, []}, + refresh_interval: 100, + ttl: 50 + ) + + # 1ms required just so it has the time to receive the :hydrate msg + Process.sleep(1) + assert :error = HydratingCache.get(pid) + + Process.sleep(50) + assert :error = HydratingCache.get(pid) + + Process.sleep(100) + assert :error = HydratingCache.get(pid) + end + + test "should await the timeout to clean the value" do + {:ok, pid} = + HydratingCache.start_link( + mfa: {Process, :sleep, [100]}, + refresh_interval: 100, + ttl: :infinity + ) + + # 1ms required just so it has the time to receive the :hydrate msg + Process.sleep(1) + + assert :error = HydratingCache.get(pid, 1) + end +end diff --git a/test/archethic/oracle_chain/services/uco_price_test.exs b/test/archethic/oracle_chain/services/uco_price_test.exs index ffd6d557a..c23f6fb91 100644 --- a/test/archethic/oracle_chain/services/uco_price_test.exs +++ b/test/archethic/oracle_chain/services/uco_price_test.exs @@ -1,290 +1,243 @@ defmodule Archethic.OracleChain.Services.UCOPriceTest do - use ExUnit.Case + use ExUnit.Case, async: false + alias Archethic.OracleChain.Services.HydratingCache alias Archethic.OracleChain.Services.UCOPrice - alias ArchethicCache.HydratingCache + import Mox + import ExUnit.CaptureLog - test "fetch/0 should retrieve some data and build a map with the oracle name in it" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) + setup :verify_on_exit! + setup :set_mox_global - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + describe "fetch/0" do + test "should retrieve some data and build a map with the oracle name in it" do + MockUCOProvider1 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.12], "eur" => [0.20]}} + end) - assert {:ok, %{"eur" => _, "usd" => _}} = UCOPrice.fetch() - end + MockUCOProvider2 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.12], "eur" => [0.20]}} + end) - test "fetch/0 should retrieve some data and build a map with the oracle name in it and keep the precision to 5" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider1Cache, + mfa: {MockUCOProvider1, :fetch, [["usd", "eur"]]} + ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider2Cache, + mfa: {MockUCOProvider2, :fetch, [["usd", "eur"]]} + ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"eur" => [0.123456789], "usd" => [0.123454789]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + Process.sleep(1) - assert {:ok, %{"eur" => 0.12346, "usd" => 0.12345}} = UCOPrice.fetch() - end + assert {:ok, %{"eur" => 0.20, "usd" => 0.12}} = UCOPrice.fetch() + end - describe "verify/1" do - test "should return true if the prices are the good one" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"eur" => [0.10], "usd" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity + test "should retrieve some data and build a map with the oracle name in it and keep the precision to 5" do + MockUCOProvider1 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.123454789], "eur" => [0.123456789]}} + end) + + MockUCOProvider2 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.123454789], "eur" => [0.123456789]}} + end) + + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider1Cache, + mfa: {MockUCOProvider1, :fetch, [["usd", "eur"]]} ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"eur" => [0.20], "usd" => [0.30]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider2Cache, + mfa: {MockUCOProvider2, :fetch, [["usd", "eur"]]} ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"eur" => [0.30], "usd" => [0.40]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + Process.sleep(1) - assert {:ok, %{"eur" => 0.20, "usd" => 0.30}} == UCOPrice.fetch() + assert {:ok, %{"eur" => 0.12346, "usd" => 0.12345}} = UCOPrice.fetch() end - test "should return false if the prices have deviated" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) + test "should handle a service timing out" do + MockUCOProvider1 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.20], "eur" => [0.20]}} + end) + + MockUCOProvider2 + |> expect(:fetch, fn _ -> + :timer.sleep(5_000) + {:ok, {:error, :error_message}} + end) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider1Cache, + mfa: {MockUCOProvider1, :fetch, [["usd", "eur"]]} ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider2Cache, + mfa: {MockUCOProvider2, :fetch, [["usd", "eur"]]} ) - assert false == UCOPrice.verify?(%{"eur" => 0.10, "usd" => 0.14}) + Process.sleep(1) + + assert {:ok, %{"eur" => 0.20, "usd" => 0.20}} = UCOPrice.fetch() end - end - test "should return the median value when multiple providers queried" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) + test "should return the median value when multiple providers queried" do + MockUCOProvider1 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.20], "eur" => [0.10]}} + end) + + MockUCOProvider2 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.30], "eur" => [0.40]}} + end) + + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider1Cache, + mfa: {MockUCOProvider1, :fetch, [["usd", "eur"]]} + ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider2Cache, + mfa: {MockUCOProvider2, :fetch, [["usd", "eur"]]} + ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.40], "eur" => [0.40]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + Process.sleep(1) - assert true == UCOPrice.verify?(%{"eur" => 0.30, "usd" => 0.30}) + assert {:ok, %{"eur" => 0.25, "usd" => 0.25}} = UCOPrice.fetch() + end end - test "should return the average of median values when a even number of providers queried" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) + describe "verify/1" do + test "should return true if the prices are the good one" do + MockUCOProvider1 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.20], "eur" => [0.10]}} + end) + + MockUCOProvider2 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.30], "eur" => [0.40]}} + end) + + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider1Cache, + mfa: {MockUCOProvider1, :fetch, [["usd", "eur"]]} + ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.20, 0.30], "eur" => [0.20, 0.30]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider2Cache, + mfa: {MockUCOProvider2, :fetch, [["usd", "eur"]]} + ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.40], "eur" => [0.40]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + Process.sleep(1) - assert {:ok, %{"eur" => 0.25, "usd" => 0.25}} == UCOPrice.fetch() - end + assert UCOPrice.verify?(%{"eur" => 0.25, "usd" => 0.25}) + end - test "verify?/1 should return false when no data are returned from all providers" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [], "eur" => []}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) + test "should return false if the prices have deviated" do + MockUCOProvider1 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.30], "eur" => [0.20]}} + end) + + MockUCOProvider2 + |> expect(:fetch, fn _ -> + {:ok, %{"usd" => [0.40], "eur" => [0.50]}} + end) + + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider1Cache, + mfa: {MockUCOProvider1, :fetch, [["usd", "eur"]]} + ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [], "eur" => []}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider2Cache, + mfa: {MockUCOProvider2, :fetch, [["usd", "eur"]]} + ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [], "eur" => []}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + Process.sleep(1) - assert false == UCOPrice.verify?(%{}) + refute UCOPrice.verify?(%{"eur" => 0.25, "usd" => 0.25}) + end end - test "should report values even if a provider returns an error" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) + test "verify?/1 should return false when no data are returned from all providers" do + MockUCOProvider1 + |> expect(:fetch, fn _ -> + {:error, ""} + end) - ## If service returns an error, old value will be returned - ## we are so inserting a previous value - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> - {:ok, %{"usd" => [0.20], "eur" => [0.20]}} - end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) + MockUCOProvider2 + |> expect(:fetch, fn _ -> + {:error, ""} + end) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:error, :error_message} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider1Cache, + mfa: {MockUCOProvider1, :fetch, [["usd", "eur"]]} ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider2Cache, + mfa: {MockUCOProvider2, :fetch, [["usd", "eur"]]} ) - assert {:ok, %{"eur" => 0.20, "usd" => 0.20}} = UCOPrice.fetch() + Process.sleep(1) + + {result, log} = with_log(fn -> UCOPrice.verify?(%{"eur" => 0.25, "usd" => 0.25}) end) + assert result == false + assert log =~ "Cannot fetch UCO price - reason: no data from any service." end - test "should handle a service timing out" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.10], "eur" => [0.10]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) + test "should report values even if a provider returns an error" do + MockUCOProvider1 + |> expect(:fetch, fn _ -> + {:error, ""} + end) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> - {:ok, %{"usd" => [0.20], "eur" => [0.20]}} - end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) + MockUCOProvider2 + |> expect(:fetch, fn _ -> + {:ok, %{"eur" => [0.25], "usd" => [0.25]}} + end) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> - :timer.sleep(5_000) - {:ok, {:error, :error_message}} - end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider1Cache, + mfa: {MockUCOProvider1, :fetch, [["usd", "eur"]]} ) - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.30], "eur" => [0.30]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity + HydratingCache.start_link( + refresh_interval: 1000, + name: MockUCOProvider2Cache, + mfa: {MockUCOProvider2, :fetch, [["usd", "eur"]]} ) - assert {:ok, %{"eur" => 0.20, "usd" => 0.20}} == UCOPrice.fetch() - end + Process.sleep(1) - def fetch(values) do - values + assert UCOPrice.verify?(%{"eur" => 0.25, "usd" => 0.25}) end end diff --git a/test/archethic/oracle_chain/services_test.exs b/test/archethic/oracle_chain/services_test.exs index f535d9867..9ed91fa71 100644 --- a/test/archethic/oracle_chain/services_test.exs +++ b/test/archethic/oracle_chain/services_test.exs @@ -2,89 +2,33 @@ defmodule Archethic.OracleChain.ServicesTest do use ExUnit.Case alias Archethic.OracleChain.Services - alias ArchethicCache.HydratingCache + + import Mox describe "fetch_new_data/1" do test "should return the new data when no previous content" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + MockUCOPrice + |> expect(:fetch, fn -> + {:ok, %{"eur" => 0.20, "usd" => 0.12}} + end) assert %{uco: %{"eur" => 0.20, "usd" => 0.12}} = Services.fetch_new_data() end test "should not return the new data when the previous content is the same" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + MockUCOPrice + |> expect(:fetch, fn -> + {:ok, %{"eur" => 0.20, "usd" => 0.12}} + end) assert %{} = Services.fetch_new_data(%{uco: %{"eur" => 0.20, "usd" => 0.12}}) end test "should return the new data when the previous content is not the same" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + MockUCOPrice + |> expect(:fetch, fn -> + {:ok, %{"eur" => 0.20, "usd" => 0.12}} + end) assert %{uco: %{"eur" => 0.20, "usd" => 0.12}} = Services.fetch_new_data(%{"uco" => %{"eur" => 0.19, "usd" => 0.15}}) @@ -92,34 +36,11 @@ defmodule Archethic.OracleChain.ServicesTest do end test "verify_correctness?/1 should true when the data is correct" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) - - assert true == Services.verify_correctness?(%{"uco" => %{"eur" => 0.20, "usd" => 0.12}}) - end + MockUCOPrice + |> expect(:verify?, fn _ -> + true + end) - def fetch(values) do - values + assert Services.verify_correctness?(%{"uco" => %{"eur" => 0.20, "usd" => 0.12}}) end end diff --git a/test/archethic/oracle_chain_test.exs b/test/archethic/oracle_chain_test.exs index 04991d379..70a970e33 100644 --- a/test/archethic/oracle_chain_test.exs +++ b/test/archethic/oracle_chain_test.exs @@ -6,32 +6,12 @@ defmodule Archethic.OracleChainTest do alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData - alias ArchethicCache.HydratingCache - test "valid_services_content?/1 should verify the oracle transaction's content correctness" do - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko, - 30_000, - :infinity - ) - - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap, - 30_000, - :infinity - ) + import Mox - HydratingCache.register_function( - Archethic.OracleChain.Services.UCOPrice, - fn -> {:ok, %{"usd" => [0.12], "eur" => [0.20]}} end, - Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika, - 30_000, - :infinity - ) + test "valid_services_content?/1 should verify the oracle transaction's content correctness" do + MockUCOPrice + |> expect(:verify?, fn _ -> true end) content = %{ @@ -69,10 +49,11 @@ defmodule Archethic.OracleChainTest do } ] - assert true == OracleChain.valid_summary?(content, chain) - end + MockUCOPrice + |> expect(:parse_data, fn _ -> + {:ok, %{"eur" => 0.20, "usd" => 0.12}} + end) - def fetch(values) do - values + assert true == OracleChain.valid_summary?(content, chain) end end diff --git a/test/archethic_cache/hydrating_cache_test.exs b/test/archethic_cache/hydrating_cache_test.exs deleted file mode 100644 index 6ddc2b0d8..000000000 --- a/test/archethic_cache/hydrating_cache_test.exs +++ /dev/null @@ -1,292 +0,0 @@ -defmodule ArchethicCache.HydratingCacheTest do - alias ArchethicCache.HydratingCache - - use ExUnit.Case - - test "If `key` is not associated with any function, return `{:error, :not_registered}`" do - {:ok, pid} = HydratingCache.start_link(:test_service) - assert HydratingCache.get(pid, "unexisting_key") == {:error, :not_registered} - end - - test "If value stored, it is returned immediatly" do - {:ok, pid} = HydratingCache.start_link(:test_service_normal) - - HydratingCache.register_function( - pid, - fn -> - {:ok, 1} - end, - "simple_func", - 10_000, - :infinity - ) - - assert {:ok, 1} = HydratingCache.get(pid, "simple_func") - end - - test "Hydrating function runs periodically" do - {:ok, pid} = HydratingCache.start_link(:test_service_periodic) - - :persistent_term.put("test", 1) - - HydratingCache.register_function( - pid, - fn -> - value = :persistent_term.get("test") - value = value + 1 - :persistent_term.put("test", value) - {:ok, value} - end, - "test_inc", - 10, - :infinity - ) - - Process.sleep(50) - assert {:ok, value} = HydratingCache.get(pid, "test_inc") - assert value >= 5 - end - - test "Update hydrating function while another one is running returns new hydrating value from new function" do - {:ok, pid} = HydratingCache.start_link(:test_service_hydrating) - - HydratingCache.register_function( - pid, - fn -> - Process.sleep(5000) - {:ok, 1} - end, - "test_reregister", - 10_000, - :infinity - ) - - HydratingCache.register_function( - pid, - fn -> - {:ok, 2} - end, - "test_reregister", - 10_000, - :infinity - ) - - Process.sleep(50) - assert {:ok, 2} = HydratingCache.get(pid, "test_reregister") - end - - test "Getting value while function is running and previous value is available returns value" do - {:ok, pid} = HydratingCache.start_link(:test_service_running) - - HydratingCache.register_function( - pid, - fn -> - {:ok, 1} - end, - "test_reregister", - 10_000, - :infinity - ) - - HydratingCache.register_function( - pid, - fn -> - Process.sleep(5000) - {:ok, 2} - end, - "test_reregister", - 10_000, - :infinity - ) - - assert {:ok, 1} = HydratingCache.get(pid, "test_reregister") - end - - test "Two hydrating function can run at same time" do - {:ok, pid} = HydratingCache.start_link(:test_service_simultaneous) - - HydratingCache.register_function( - pid, - fn -> - Process.sleep(5000) - {:ok, :result_timed} - end, - "timed_value", - 10_000, - :infinity - ) - - HydratingCache.register_function( - pid, - fn -> - {:ok, :result} - end, - "direct_value", - 10_000, - :infinity - ) - - assert {:ok, :result} = HydratingCache.get(pid, "direct_value") - end - - test "Querying key while first refreshed will block the calling process until timeout" do - {:ok, pid} = HydratingCache.start_link(:test_service_block) - - HydratingCache.register_function( - pid, - fn -> - Process.sleep(5000) - {:ok, :valid_result} - end, - "delayed_result", - 10_000, - :infinity - ) - - ## We query the value with timeout smaller than timed function - assert {:error, :timeout} = HydratingCache.get(pid, "delayed_result", 1) - end - - test "Multiple process can wait for a delayed value" do - {:ok, pid} = HydratingCache.start_link(:test_service_delayed) - - HydratingCache.register_function( - pid, - fn -> - Process.sleep(100) - {:ok, :valid_result} - end, - "delayed_result", - 10_000, - :infinity - ) - - results = Task.async_stream(1..10, fn _ -> HydratingCache.get(pid, "delayed_result") end) - - assert Enum.all?(results, fn - {:ok, {:ok, :valid_result}} -> true - end) - end - - ## Resilience tests - test "If hydrating function crash, key fsm will still be operationnal" do - {:ok, pid} = HydratingCache.start_link(:test_service_crash) - - HydratingCache.register_function( - pid, - fn -> - ## Exit hydrating function - exit(1) - end, - :key, - 10_000, - :infinity - ) - - Process.sleep(1) - - HydratingCache.register_function( - pid, - fn -> - {:ok, :value} - end, - :key, - 10_000, - :infinity - ) - - assert {:ok, :value} = HydratingCache.get(pid, :key) - end - - ## This could occur if hydrating function takes time to answer. - ## In this case, getting the value would return the old value, unless too - ## much time occur where it would be discarded because of ttl - test "value gets discarded after some time" do - {:ok, pid} = HydratingCache.start_link(:test_service) - - HydratingCache.register_function( - pid, - fn -> - case :persistent_term.get("flag", nil) do - 1 -> - Process.sleep(5000) - - nil -> - :persistent_term.put("flag", 1) - {:ok, :value} - end - end, - :key, - 10, - 20 - ) - - assert {:ok, :value} = HydratingCache.get(pid, :key) - Process.sleep(25) - assert {:error, :discarded} = HydratingCache.get(pid, :key) - end - - test "Can get a value while another request is waiting for results" do - {:ok, pid} = HydratingCache.start_link(:test_service_wait) - - HydratingCache.register_function( - pid, - fn -> - Process.sleep(5000) - {:ok, :value} - end, - :key1, - 10_000, - :infinity - ) - - :erlang.spawn(fn -> HydratingCache.get(pid, :key1, 15_000) end) - - HydratingCache.register_function( - pid, - fn -> - {:ok, :value2} - end, - :key2, - 10_000, - :infinity - ) - - assert {:ok, :value2} = HydratingCache.get(pid, :key2) - end - - test "can retrieve all values beside erroneous ones" do - {:ok, pid} = - HydratingCache.start_link(:test_service_get_all, [ - {"key1", {__MODULE__, :val_hydrating_function, [10]}, 10_00, :infinity}, - {"key2", {__MODULE__, :failval_hydrating_function, [20]}, 10_00, :infinity}, - {"key3", {__MODULE__, :val_hydrating_function, [30]}, 10_00, :infinity} - ]) - - assert [10, 30] = HydratingCache.get_all(pid) - end - - test "Retrieving all values supports delayed values" do - {:ok, pid} = - HydratingCache.start_link(:test_service_get_all_delayed, [ - {"key1", {__MODULE__, :val_hydrating_function, [10]}, 10_000, :infinity}, - {"key2", {__MODULE__, :timed_hydrating_function, [50, 20]}, 10_000, :infinity}, - {"key3", {__MODULE__, :failval_hydrating_function, [30]}, 10_000, :infinity} - ]) - - assert [10, 20] = HydratingCache.get_all(pid) - end - - def val_hydrating_function(value) do - {:ok, value} - end - - def failval_hydrating_function(value) do - {:error, value} - end - - def timed_hydrating_function(delay, value) do - Process.sleep(delay) - {:ok, value} - end -end diff --git a/test/test_helper.exs b/test/test_helper.exs index 1b412afc3..485a18088 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -33,3 +33,7 @@ Mox.defmock(MockRemoteDiscovery, for: Archethic.Networking.IPLookup.Impl) Mox.defmock(MockNATDiscovery, for: Archethic.Networking.IPLookup.Impl) # -----End-of-Networking-Mocks ------ + +Mox.defmock(MockUCOPrice, for: Archethic.OracleChain.Services.Impl) +Mox.defmock(MockUCOProvider1, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) +Mox.defmock(MockUCOProvider2, for: Archethic.OracleChain.Services.UCOPrice.Providers.Impl) From ef4047cef9e17453483fe6e0ef38be7de7b646b2 Mon Sep 17 00:00:00 2001 From: Samuel Manzanera Date: Fri, 17 Mar 2023 15:12:26 +0100 Subject: [PATCH 46/48] Improve task monitoring and tests The renewal of the hydrating message is done after the task is completed The timers are persisted in the state to have a better monitoring The tests are now using :erlang.trace to avoid undeterministic behaviors based on time. --- .../oracle_chain/services/hydrating_cache.ex | 49 ++++++++++++------- .../services/hydrating_cache_test.exs | 28 ++++++----- 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/lib/archethic/oracle_chain/services/hydrating_cache.ex b/lib/archethic/oracle_chain/services/hydrating_cache.ex index c401ec25d..e92f0f9b5 100644 --- a/lib/archethic/oracle_chain/services/hydrating_cache.ex +++ b/lib/archethic/oracle_chain/services/hydrating_cache.ex @@ -7,15 +7,18 @@ defmodule Archethic.OracleChain.Services.HydratingCache do """ use GenServer + require Logger + defmodule State do @moduledoc false defstruct([ :mfa, :ttl, - :ttl_ref, + :ttl_timer, :refresh_interval, :value, - :hydrating_task + :hydrating_task, + :hydrating_timer ]) end @@ -41,14 +44,15 @@ defmodule Archethic.OracleChain.Services.HydratingCache do ttl = Keyword.get(options, :ttl, :infinity) # start hydrating as soon as init is done - Process.send_after(self(), :hydrate, 0) + hydrating_timer = Process.send_after(self(), :hydrate, 0) ## Hydrate the value {:ok, %State{ mfa: mfa, ttl: ttl, - refresh_interval: refresh_interval + refresh_interval: refresh_interval, + hydrating_timer: hydrating_timer }} end @@ -63,7 +67,6 @@ defmodule Archethic.OracleChain.Services.HydratingCache do def handle_info( :hydrate, state = %State{ - refresh_interval: refresh_interval, mfa: {m, f, a} } ) do @@ -72,42 +75,54 @@ defmodule Archethic.OracleChain.Services.HydratingCache do try do {:ok, apply(m, f, a)} rescue - _ -> - :error + e -> + {:error, e} end end) - # start a new hydrate timer - Process.send_after(self(), :hydrate, refresh_interval) - {:noreply, %State{state | hydrating_task: hydrating_task}} end def handle_info( {ref, result}, - state = %State{ttl_ref: ttl_ref, ttl: ttl, hydrating_task: %Task{ref: ref_task}} + state = %State{ + mfa: {m, f, a}, + refresh_interval: refresh_interval, + ttl_timer: ttl_timer, + ttl: ttl, + hydrating_task: %Task{ref: ref_task} + } ) when ref == ref_task do # cancel current ttl if any - if is_reference(ttl_ref) do - Process.cancel_timer(ttl_ref) + if is_reference(ttl_timer) do + Process.cancel_timer(ttl_timer) end # start new ttl timer - ttl_ref = + ttl_timer = if is_integer(ttl) do Process.send_after(self(), :discard_value, ttl) else nil end - new_state = %{state | ttl_ref: ttl_ref, hydrating_task: nil} + # start a new hydrate timer + hydrating_timer = Process.send_after(self(), :hydrate, refresh_interval) + + new_state = %{ + state + | ttl_timer: ttl_timer, + hydrating_task: nil, + hydrating_timer: hydrating_timer + } case result do {:ok, value} -> {:noreply, %{new_state | value: value}} - _ -> + {:error, reason} -> + Logger.error("#{m}.#{f}.#{inspect(a)} returns an error: #{inspect(reason)}") {:noreply, new_state} end end @@ -115,6 +130,6 @@ defmodule Archethic.OracleChain.Services.HydratingCache do def handle_info({:DOWN, _ref, :process, _, _}, state), do: {:noreply, state} def handle_info(:discard_value, state) do - {:noreply, %State{state | value: nil, ttl_ref: nil}} + {:noreply, %State{state | value: nil, ttl_timer: nil}} end end diff --git a/test/archethic/oracle_chain/services/hydrating_cache_test.exs b/test/archethic/oracle_chain/services/hydrating_cache_test.exs index 1e2233886..a0b3931c5 100644 --- a/test/archethic/oracle_chain/services/hydrating_cache_test.exs +++ b/test/archethic/oracle_chain/services/hydrating_cache_test.exs @@ -2,6 +2,7 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do alias Archethic.OracleChain.Services.HydratingCache use ExUnit.Case + @moduletag capture_log: true test "should receive the same value until next refresh" do {:ok, pid} = @@ -11,7 +12,8 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do ttl: :infinity ) - # 1ms required just so it has the time to receive the :hydrate msg + :erlang.trace(pid, true, [:receive]) + assert_receive {:trace, ^pid, :receive, :hydrate} Process.sleep(1) {:ok, date} = HydratingCache.get(pid) @@ -27,7 +29,9 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do Process.sleep(10) assert {:ok, ^date} = HydratingCache.get(pid) - Process.sleep(100) + assert_receive {:trace, ^pid, :receive, {:DOWN, _, _, _, _}} + assert_receive {:trace, ^pid, :receive, :hydrate} + Process.sleep(1) {:ok, date2} = HydratingCache.get(pid) assert date != date2 end @@ -40,9 +44,11 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do ttl: 50 ) - # 1ms required just so it has the time to receive the :hydrate msg + :erlang.trace(pid, true, [:receive]) + assert_receive {:trace, ^pid, :receive, :hydrate} Process.sleep(1) - {:ok, _date} = HydratingCache.get(pid) + + assert {:ok, _date} = HydratingCache.get(pid) Process.sleep(50) assert :error = HydratingCache.get(pid) @@ -56,7 +62,8 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do ttl: 50 ) - # 1ms required just so it has the time to receive the :hydrate msg + :erlang.trace(pid, true, [:receive]) + assert_receive {:trace, ^pid, :receive, :hydrate} Process.sleep(1) {:ok, date} = HydratingCache.get(pid) @@ -77,11 +84,9 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do ttl: 50 ) - # 1ms required just so it has the time to receive the :hydrate msg - Process.sleep(1) - assert :error = HydratingCache.get(pid) - - Process.sleep(50) + :erlang.trace(pid, true, [:receive]) + assert_receive {:trace, ^pid, :receive, :hydrate} + assert_receive {:trace, ^pid, :receive, {_ref, {:error, %UndefinedFunctionError{}}}} assert :error = HydratingCache.get(pid) Process.sleep(100) @@ -96,7 +101,8 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do ttl: :infinity ) - # 1ms required just so it has the time to receive the :hydrate msg + :erlang.trace(pid, true, [:receive]) + assert_receive {:trace, ^pid, :receive, :hydrate} Process.sleep(1) assert :error = HydratingCache.get(pid, 1) From 04706b361f14880375f59f46e00145d8bdc9c39c Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Wed, 22 Mar 2023 11:49:20 +0100 Subject: [PATCH 47/48] Hydrating Cache accepts CRON intervals --- .../oracle_chain/services/hydrating_cache.ex | 12 +++++++++++- .../services/hydrating_cache_test.exs | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/archethic/oracle_chain/services/hydrating_cache.ex b/lib/archethic/oracle_chain/services/hydrating_cache.ex index e92f0f9b5..f8b4fba1f 100644 --- a/lib/archethic/oracle_chain/services/hydrating_cache.ex +++ b/lib/archethic/oracle_chain/services/hydrating_cache.ex @@ -7,6 +7,7 @@ defmodule Archethic.OracleChain.Services.HydratingCache do """ use GenServer + alias Archethic.Utils require Logger defmodule State do @@ -15,6 +16,7 @@ defmodule Archethic.OracleChain.Services.HydratingCache do :mfa, :ttl, :ttl_timer, + # refresh_interval :: Int | CronInterval :refresh_interval, :value, :hydrating_task, @@ -108,7 +110,7 @@ defmodule Archethic.OracleChain.Services.HydratingCache do end # start a new hydrate timer - hydrating_timer = Process.send_after(self(), :hydrate, refresh_interval) + hydrating_timer = Process.send_after(self(), :hydrate, next_tick_in_seconds(refresh_interval)) new_state = %{ state @@ -132,4 +134,12 @@ defmodule Archethic.OracleChain.Services.HydratingCache do def handle_info(:discard_value, state) do {:noreply, %State{state | value: nil, ttl_timer: nil}} end + + defp next_tick_in_seconds(refresh_interval) do + if is_binary(refresh_interval) do + Utils.time_offset(refresh_interval) * 1000 + else + refresh_interval + end + end end diff --git a/test/archethic/oracle_chain/services/hydrating_cache_test.exs b/test/archethic/oracle_chain/services/hydrating_cache_test.exs index a0b3931c5..3d3e33f08 100644 --- a/test/archethic/oracle_chain/services/hydrating_cache_test.exs +++ b/test/archethic/oracle_chain/services/hydrating_cache_test.exs @@ -36,6 +36,25 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do assert date != date2 end + test "should work with cron interval" do + {:ok, pid} = + HydratingCache.start_link( + mfa: {DateTime, :utc_now, []}, + refresh_interval: "* * * * *", + ttl: :infinity + ) + + :erlang.trace(pid, true, [:receive]) + assert_receive {:trace, ^pid, :receive, :hydrate} + Process.sleep(1) + {:ok, date} = HydratingCache.get(pid) + + # minimum interval is 1s + Process.sleep(1005) + {:ok, date2} = HydratingCache.get(pid) + assert date != date2 + end + test "should discard the value after the ttl is reached" do {:ok, pid} = HydratingCache.start_link( From ef646d78101529e0813439f1ce1cc4921b5d4fbf Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Wed, 22 Mar 2023 15:37:24 +0100 Subject: [PATCH 48/48] Add a timeout for the hydrating function --- .../oracle_chain/services/hydrating_cache.ex | 15 ++++++++++++++- .../services/hydrating_cache_test.exs | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/lib/archethic/oracle_chain/services/hydrating_cache.ex b/lib/archethic/oracle_chain/services/hydrating_cache.ex index f8b4fba1f..cb05f1fa0 100644 --- a/lib/archethic/oracle_chain/services/hydrating_cache.ex +++ b/lib/archethic/oracle_chain/services/hydrating_cache.ex @@ -20,7 +20,8 @@ defmodule Archethic.OracleChain.Services.HydratingCache do :refresh_interval, :value, :hydrating_task, - :hydrating_timer + :hydrating_timer, + :hydrating_function_timeout ]) end @@ -44,6 +45,7 @@ defmodule Archethic.OracleChain.Services.HydratingCache do refresh_interval = Keyword.fetch!(options, :refresh_interval) mfa = Keyword.fetch!(options, :mfa) ttl = Keyword.get(options, :ttl, :infinity) + hydrating_function_timeout = Keyword.get(options, :hydrating_function_timeout, 5000) # start hydrating as soon as init is done hydrating_timer = Process.send_after(self(), :hydrate, 0) @@ -53,6 +55,7 @@ defmodule Archethic.OracleChain.Services.HydratingCache do %State{ mfa: mfa, ttl: ttl, + hydrating_function_timeout: hydrating_function_timeout, refresh_interval: refresh_interval, hydrating_timer: hydrating_timer }} @@ -69,6 +72,7 @@ defmodule Archethic.OracleChain.Services.HydratingCache do def handle_info( :hydrate, state = %State{ + hydrating_function_timeout: hydrating_function_timeout, mfa: {m, f, a} } ) do @@ -82,9 +86,18 @@ defmodule Archethic.OracleChain.Services.HydratingCache do end end) + # we make sure that our hydrating function does not hang + Process.send_after(self(), {:kill_hydrating_task, hydrating_task}, hydrating_function_timeout) + {:noreply, %State{state | hydrating_task: hydrating_task}} end + def handle_info({:kill_hydrating_task, task}, state) do + Task.shutdown(task, :brutal_kill) + + {:noreply, state} + end + def handle_info( {ref, result}, state = %State{ diff --git a/test/archethic/oracle_chain/services/hydrating_cache_test.exs b/test/archethic/oracle_chain/services/hydrating_cache_test.exs index 3d3e33f08..efe80b19a 100644 --- a/test/archethic/oracle_chain/services/hydrating_cache_test.exs +++ b/test/archethic/oracle_chain/services/hydrating_cache_test.exs @@ -126,4 +126,23 @@ defmodule ArchethicCache.OracleChain.Services.HydratingCacheTest do assert :error = HydratingCache.get(pid, 1) end + + test "should kill the task if hydrating function takes longer than timeout" do + {:ok, pid} = + HydratingCache.start_link( + mfa: {Process, :sleep, [100]}, + refresh_interval: 100, + hydrating_function_timeout: 10, + ttl: :infinity + ) + + :erlang.trace(pid, true, [:receive]) + assert_receive {:trace, ^pid, :receive, :hydrate} + assert_receive {:trace, ^pid, :receive, {:kill_hydrating_task, _}} + + refute Process.alive?(:sys.get_state(pid).hydrating_task.pid) + assert Process.alive?(pid) + + assert :error = HydratingCache.get(pid) + end end