From 3521ddec319a14b07f6822aeecfa1f7c28d6efcb Mon Sep 17 00:00:00 2001 From: bchamagne Date: Fri, 17 Nov 2023 15:02:39 +0100 Subject: [PATCH] duplicate coin market cap oracle provider because they will soon change the url --- config/config.exs | 7 +- .../uco_price/providers/coin_gecko.ex | 3 + .../providers/coin_marketcap_archethic.ex | 141 ++++++++++++++++++ ..._marketcap.ex => coin_marketcap_uniris.ex} | 5 +- .../uco_price/providers/coin_paprika.ex | 3 + .../uco_price/coin_marketcap_test.exs | 13 +- 6 files changed, 167 insertions(+), 5 deletions(-) create mode 100644 lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap_archethic.ex rename lib/archethic/oracle_chain/services/uco_price/providers/{coin_marketcap.ex => coin_marketcap_uniris.ex} (97%) diff --git a/config/config.exs b/config/config.exs index 86a7660a27..8cdb0df116 100644 --- a/config/config.exs +++ b/config/config.exs @@ -144,7 +144,12 @@ config :archethic, Archethic.OracleChain.Services.UCOPrice, providers: %{ # Coingecko limits to 10-30 calls, with 30s delay we would be under the limitation Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko => [refresh_interval: 30_000], - Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap => [refresh_interval: 10_000], + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCapArchethic => [ + refresh_interval: 10_000 + ], + Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCapUniris => [ + refresh_interval: 10_000 + ], # Coinpaprika limits to 25K req/mo; with 2min delay we can reach ~21K Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika => [refresh_interval: 120_000] } diff --git a/lib/archethic/oracle_chain/services/uco_price/providers/coin_gecko.ex b/lib/archethic/oracle_chain/services/uco_price/providers/coin_gecko.ex index 7ca8cce5f3..51393ee6c8 100644 --- a/lib/archethic/oracle_chain/services/uco_price/providers/coin_gecko.ex +++ b/lib/archethic/oracle_chain/services/uco_price/providers/coin_gecko.ex @@ -10,6 +10,9 @@ defmodule Archethic.OracleChain.Services.UCOPrice.Providers.Coingecko do @impl Impl @spec fetch(list(binary())) :: {:ok, %{required(String.t()) => any()}} | {:error, any()} def fetch(pairs) when is_list(pairs) do + IO.puts("ooooooo") + IO.puts("CoinGECKO fetch") + IO.puts("ooooooo") pairs_str = Enum.join(pairs, ",") query = diff --git a/lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap_archethic.ex b/lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap_archethic.ex new file mode 100644 index 0000000000..2bea3bd13d --- /dev/null +++ b/lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap_archethic.ex @@ -0,0 +1,141 @@ +defmodule Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCapArchethic do + @moduledoc false + + alias Archethic.OracleChain.Services.UCOPrice.Providers.Impl + + @behaviour Impl + require Logger + + @impl Impl + @spec fetch(list(binary())) :: {:ok, %{required(String.t()) => any()}} | {:error, any()} + def fetch(pairs) when is_list(pairs) do + IO.puts("ooooooo") + IO.puts("CoinMCA fetch") + IO.puts("ooooooo") + query = 'https://coinmarketcap.com/currencies/archethic/markets/' + + httpc_options = [ + ssl: [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 3, + customize_hostname_check: [ + match_fun: :public_key.pkix_verify_hostname_match_fun(:https) + ] + ], + connect_timeout: 1000, + timeout: 2000 + ] + + returned_prices = + Task.Supervisor.async_stream_nolink( + Archethic.TaskSupervisor, + pairs, + fn pair -> + headers = [ + {'user-agent', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'}, + {'accept', 'text/html'}, + {'accept-language', 'en-US,en;q=0.9,es;q=0.8'}, + {'upgrade-insecure-requests', '1'}, + {'Cookie', 'currency=#{pair}'} + ] + + with {:ok, {{_, 200, 'OK'}, _headers, body}} <- + :httpc.request(:get, {query, headers}, httpc_options, []), + {:ok, document} <- Floki.parse_document(body) do + price = + extract_methods() + |> Enum.reduce_while(nil, fn extract_fn, acc -> + try do + {:halt, extract_fn.(document)} + rescue + _ -> + {:cont, acc} + catch + _ -> + {:cont, acc} + end + end) + + if is_number(price) do + {:ok, {pair, [price]}} + else + {:error, :not_a_number} + end + else + {:ok, {{_, _, status}, _, _}} -> + {:error, status} + + :error -> + {:error, "invalid content"} + + {:error, _} = e -> + e + end + end + ) + |> Stream.filter(&match?({:ok, {:ok, _}}, &1)) + |> Stream.map(fn {:ok, {:ok, val}} -> val end) + |> Enum.into(%{}) + + {:ok, returned_prices} + end + + defp extract_methods() do + [ + &extract_method1/1, + &extract_method2/1, + &extract_method3/1, + # if every other failed + &fallback_error/1 + ] + end + + defp extract_method1(document) do + Floki.find(document, "div.priceTitle > div.priceValue > span") + |> Floki.text() + |> String.graphemes() + |> Enum.filter(&(&1 in [".", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"])) + |> Enum.into("") + |> String.to_float() + end + + defp extract_method2(document) do + regex = ~r/price today is (.+) with a/ + + Floki.find(document, "meta[name=description]") + |> Floki.attribute("content") + |> Enum.join() + |> then(&Regex.run(regex, &1, capture: :all_but_first)) + |> Enum.join() + |> String.graphemes() + |> Enum.filter(&(&1 in [".", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"])) + |> Enum.into("") + |> String.to_float() + end + + defp extract_method3(document) do + document + |> Floki.find("#__NEXT_DATA__") + |> Floki.text(js: true) + |> Jason.decode!() + |> get_in(["props", "pageProps", "info", "statistics", "price"]) + end + + defp fallback_error(document) do + path = "/tmp/coinmarketcap.html" + + case File.write(path, Floki.raw_html(document, pretty: true)) do + :ok -> + Logger.warning( + "Coinmarketcap failed to be parsed, you may find the page we receive at #{path}" + ) + + {:error, _posix} -> + Logger.warning("Coinmarketcap failed to be parsed") + end + + throw("coinmarketcap failed to parse") + end +end diff --git a/lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap.ex b/lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap_uniris.ex similarity index 97% rename from lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap.ex rename to lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap_uniris.ex index 9cd4615404..b6596338e8 100644 --- a/lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap.ex +++ b/lib/archethic/oracle_chain/services/uco_price/providers/coin_marketcap_uniris.ex @@ -1,4 +1,4 @@ -defmodule Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap do +defmodule Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCapUniris do @moduledoc false alias Archethic.OracleChain.Services.UCOPrice.Providers.Impl @@ -10,6 +10,9 @@ defmodule Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap do @impl Impl @spec fetch(list(binary())) :: {:ok, %{required(String.t()) => any()}} | {:error, any()} def fetch(pairs) when is_list(pairs) do + IO.puts("ooooooo") + IO.puts("CoinMCU fetch") + IO.puts("ooooooo") query = 'https://coinmarketcap.com/currencies/uniris/markets/' httpc_options = [ diff --git a/lib/archethic/oracle_chain/services/uco_price/providers/coin_paprika.ex b/lib/archethic/oracle_chain/services/uco_price/providers/coin_paprika.ex index 223933f08e..dc8d1a3b16 100644 --- a/lib/archethic/oracle_chain/services/uco_price/providers/coin_paprika.ex +++ b/lib/archethic/oracle_chain/services/uco_price/providers/coin_paprika.ex @@ -10,6 +10,9 @@ defmodule Archethic.OracleChain.Services.UCOPrice.Providers.CoinPaprika do @impl Impl @spec fetch(list(binary())) :: {:ok, %{required(String.t()) => any()}} | {:error, any()} def fetch(pairs) when is_list(pairs) do + IO.puts("ooooooo") + IO.puts("CoinPaprika fetch") + IO.puts("ooooooo") pairs_str = Enum.join(pairs, ",") httpc_options = [ diff --git a/test/archethic/oracle_chain/services/uco_price/coin_marketcap_test.exs b/test/archethic/oracle_chain/services/uco_price/coin_marketcap_test.exs index 3911b9a8f4..e56e682c62 100644 --- a/test/archethic/oracle_chain/services/uco_price/coin_marketcap_test.exs +++ b/test/archethic/oracle_chain/services/uco_price/coin_marketcap_test.exs @@ -1,11 +1,18 @@ defmodule Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCapTest do use ExUnit.Case - alias Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCap + alias Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCapUniris + alias Archethic.OracleChain.Services.UCOPrice.Providers.CoinMarketCapArchethic @tag oracle_provider: true - test "fetch/1 should get the current UCO price from CoinMarketCap" do - assert {:ok, %{"eur" => prices}} = CoinMarketCap.fetch(["eur"]) + test "fetch/1 should get the current UCO price from CoinMarketCap (uniris)" do + assert {:ok, %{"eur" => prices}} = CoinMarketCapUniris.fetch(["eur"]) + assert is_list(prices) + end + + @tag oracle_provider: true + test "fetch/1 should get the current UCO price from CoinMarketCap (archethic)" do + assert {:ok, %{"eur" => prices}} = CoinMarketCapArchethic.fetch(["eur"]) assert is_list(prices) end end