diff --git a/config/config.exs b/config/config.exs index d14e69a2d..5e8f147a0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -151,6 +151,16 @@ config :archethic, Archethic.Networking.IPLookup.RemoteDiscovery, # -----End-of-Networking-configs ------ +config :archethic_web, + # The tx_cache is stored on RAM + # 750MB should hold a minimum 250 transactions + tx_cache_bytes: 750 * 1024 * 1024, + + # The file_cache is stored on DISK + # 5GB should hold 2000 average size pages + # https://httparchive.org/reports/page-weight + file_cache_bytes: 5 * 1024 * 1024 * 1024 + config :esbuild, version: "0.12.18", default: [ diff --git a/lib/archethic/metrics/parser.ex b/lib/archethic/metrics/parser.ex index bb1bd260f..da3e25b96 100755 --- a/lib/archethic/metrics/parser.ex +++ b/lib/archethic/metrics/parser.ex @@ -272,10 +272,16 @@ defmodule Archethic.Metrics.Parser do ...> name: "vm_memory_atom", ...> type: "gauge" ...> }, + ...> %{ + ...> metrics: [%{value: 4}, %{value: 1}], + ...> name: "cache_hit", + ...> type: "counter" + ...> } ...> ] |> Parser.reduce_metrics() %{ "archethic_contract_parsing_duration" => %{count: 2, sum: 10.0}, - "vm_memory_atom" => 1589609 + "vm_memory_atom" => 1589609, + "cache_hit" => 5 } """ @spec reduce_metrics(list(metric())) :: @@ -287,9 +293,21 @@ defmodule Archethic.Metrics.Parser do metric = %{type: "gauge"}, acc -> Map.merge(acc, map_gauge(metric)) + + metric = %{type: "counter"}, acc -> + Map.merge(acc, map_counter(metric)) end) end + defp map_counter(%{name: name, metrics: metrics}) do + metrics = + Enum.reduce(metrics, 0, fn %{value: value}, acc -> + acc + value + end) + + %{name => metrics} + end + defp map_gauge(%{name: name, metrics: [metric | _]}) do value = Map.get(metric, :value, 0) %{name => value} diff --git a/lib/archethic/telemetry.ex b/lib/archethic/telemetry.ex index e83d9286d..1112e01f4 100644 --- a/lib/archethic/telemetry.ex +++ b/lib/archethic/telemetry.ex @@ -189,7 +189,13 @@ defmodule Archethic.Telemetry do reporter_options: [buckets: [0.01, 0.025, 0.05, 0.1, 0.3, 0.5, 0.8, 1, 1.5, 2, 5, 10]], measurement: :duration, tags: [:nb_summaries] - ) + ), + + # Archethic Web + counter("archethic_web.hosting.cache_file.hit.count"), + counter("archethic_web.hosting.cache_file.miss.count"), + counter("archethic_web.hosting.cache_ref_tx.hit.count"), + counter("archethic_web.hosting.cache_ref_tx.miss.count") ] end end diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex new file mode 100644 index 000000000..0d63096f5 --- /dev/null +++ b/lib/archethic_cache/lru.ex @@ -0,0 +1,159 @@ +defmodule ArchethicCache.LRU do + @moduledoc """ + A cache that stores the values in an ETS table. + There are hooks available to be able to add effects (ex: write to disk). + + It keeps track of the order and bytes in the genserver state. + The `bytes_used` are tracked in here because if we just monitor ETS table size, we would not be able to have a disk cache. + The `keys` are used to determine the Least Recent Used (first is the most recent used, last is the least recent used). + + We do not store the values directly in ETS but we insert a pair {size, value} instead. + Because size can be modified with the hooks + (ex: For LRUDisk, we discard the value from the ETS table, but still want to know the size written to disk) + """ + + @spec start_link(GenServer.name(), non_neg_integer(), keyword()) :: + {:ok, binary()} | {:error, term()} + def start_link(name, max_bytes, opts \\ []) do + GenServer.start_link(__MODULE__, [name, max_bytes, opts], name: name) + end + + @spec put(GenServer.server(), term(), term()) :: :ok + def put(server, key, value) do + GenServer.cast(server, {:put, key, value}) + end + + @spec get(GenServer.server(), term()) :: nil | term() + def get(server, key) do + GenServer.call(server, {:get, key}) + end + + @spec purge(GenServer.server()) :: :ok + def purge(server) do + GenServer.call(server, :purge) + end + + def init([name, max_bytes, opts]) do + table = :ets.new(:"aecache_#{name}", [:set, {:read_concurrency, true}]) + + {:ok, + %{ + table: table, + bytes_max: max_bytes, + bytes_used: 0, + keys: [], + put_fn: Keyword.get(opts, :put_fn, fn _key, value -> value end), + get_fn: Keyword.get(opts, :get_fn, fn _key, value -> value end), + evict_fn: Keyword.get(opts, :evict_fn, fn _key, _value -> :ok end) + }} + end + + def handle_call({:get, key}, _from, state = %{table: table, keys: keys, get_fn: get_fn}) do + {reply, new_state} = + case :ets.lookup(table, key) do + [{^key, {_size, value}}] -> + { + get_fn.(key, value), + %{state | keys: keys |> move_front(key)} + } + + [] -> + {nil, state} + end + + {:reply, reply, new_state} + end + + def handle_call(:purge, _from, state = %{table: table, evict_fn: evict_fn}) do + # we call the evict_fn to be able to clean effects (ex: file written to disk) + :ets.foldr( + fn {key, {_size, value}}, acc -> + evict_fn.(key, value) + acc + 1 + end, + 0, + table + ) + + :ets.delete_all_objects(table) + {:reply, :ok, %{state | keys: [], bytes_used: 0}} + end + + def handle_cast( + {:put, key, value}, + state = %{table: table, bytes_max: bytes_max, put_fn: put_fn, evict_fn: evict_fn} + ) do + size = :erlang.external_size(value) + + if size > bytes_max do + {:noreply, state} + else + # maybe evict some keys to make space + state = + evict_until(state, fn %{bytes_used: bytes_used, bytes_max: bytes_max} -> + bytes_used + size <= bytes_max + end) + + case :ets.lookup(table, key) do + [] -> + value_to_store = put_fn.(key, value) + + :ets.insert(table, {key, {size, value_to_store}}) + + new_state = %{ + state + | keys: [key | state.keys], + bytes_used: state.bytes_used + size + } + + {:noreply, new_state} + + [{^key, {old_size, old_value}}] -> + # this is a replacement, we need to evict to update the bytes_used + evict_fn.(key, old_value) + value_to_store = put_fn.(key, value) + + :ets.insert(table, {key, {size, value_to_store}}) + + new_state = %{ + state + | keys: state.keys |> move_front(key), + bytes_used: state.bytes_used + size - old_size + } + + {:noreply, new_state} + end + end + end + + defp evict_until( + state = %{table: table, keys: keys, evict_fn: evict_fn, bytes_used: bytes_used}, + predicate + ) do + if predicate.(state) do + state + else + case Enum.reverse(keys) do + [] -> + state + + [oldest_key | rest] -> + [{_, {size, oldest_value}}] = :ets.take(table, oldest_key) + evict_fn.(oldest_key, oldest_value) + + evict_until( + %{ + state + | bytes_used: bytes_used - size, + keys: rest + }, + predicate + ) + end + end + end + + defp move_front(list, item) do + [item | List.delete(list, item)] + end +end diff --git a/lib/archethic_cache/lru_disk.ex b/lib/archethic_cache/lru_disk.ex new file mode 100644 index 000000000..b7872b5bf --- /dev/null +++ b/lib/archethic_cache/lru_disk.ex @@ -0,0 +1,67 @@ +defmodule ArchethicCache.LRUDisk do + @moduledoc """ + Wraps the LRU genserver and adds hooks to write / read from disk. + The value is always a binary. + """ + alias ArchethicCache.LRU + alias Archethic.Crypto + + require Logger + + @spec start_link(GenServer.name(), non_neg_integer(), binary()) :: + {:ok, binary()} | {:error, term()} + def start_link(name, max_bytes, cache_dir) do + cache_dir = Path.join(cache_dir, "#{name}") + :ok = reset_directory(cache_dir) + + LRU.start_link(name, max_bytes, + put_fn: fn key, value -> + # write to disk + File.write!(key_to_path(cache_dir, key), value, [:exclusive, :binary]) + + # store a nil value in the LRU's ETS table + nil + end, + get_fn: fn key, nil -> + # called only if the key is already in LRU's ETS table + case File.read(key_to_path(cache_dir, key)) do + {:ok, bin} -> + bin + + {:error, _} -> + nil + end + end, + evict_fn: fn key, nil -> + case File.rm(key_to_path(cache_dir, key)) do + :ok -> + :ok + + {:error, _} -> + :ok + end + end + ) + end + + @spec put(GenServer.server(), term(), binary()) :: :ok + defdelegate put(server, key, value), to: LRU, as: :put + + @spec get(GenServer.server(), term()) :: nil | binary() + defdelegate get(server, key), to: LRU, as: :get + + @spec purge(GenServer.server()) :: :ok + defdelegate purge(server), to: LRU, as: :purge + + defp reset_directory(dir) do + File.rm_rf!(dir) + File.mkdir_p!(dir) + end + + defp key_to_path(cache_dir, key) do + Path.join( + cache_dir, + Crypto.hash(:erlang.term_to_binary(key)) |> Base.url_encode64(padding: false) + ) + end +end diff --git a/lib/archethic_web/controllers/aeweb_root_controller.ex b/lib/archethic_web/controllers/aeweb_root_controller.ex index 19af2217a..c88c3764f 100644 --- a/lib/archethic_web/controllers/aeweb_root_controller.ex +++ b/lib/archethic_web/controllers/aeweb_root_controller.ex @@ -5,40 +5,7 @@ defmodule ArchethicWeb.AEWebRootController do use ArchethicWeb, :controller - def index(conn, params = %{"url_path" => url_path}) do - cache_headers = WebHostingController.get_cache_headers(conn) - - case WebHostingController.get_website(params, cache_headers) do - {:ok, file_content, encoding, mime_type, cached?, etag} -> - WebHostingController.send_response(conn, file_content, encoding, mime_type, cached?, etag) - - {:error, {:is_a_directory, transaction}} -> - {:ok, listing_html, encoding, mime_type, cached?, etag} = - WebHostingController.DirectoryListing.list( - conn.request_path, - params, - transaction, - cache_headers - ) - - WebHostingController.send_response(conn, listing_html, encoding, mime_type, cached?, etag) - - {:error, :file_not_found} -> - # If file is not found, returning default file (url can be handled by index file) - case url_path do - [] -> - send_resp(conn, 404, "Not Found") - - ["index.html"] -> - send_resp(conn, 400, "Not Found") - - _path -> - params = Map.put(params, "url_path", ["index.html"]) - index(conn, params) - end - - _ -> - send_resp(conn, 404, "Not Found") - end + def index(conn, params) do + WebHostingController.web_hosting(conn, params) end end diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex index d9daea210..c7a193250 100644 --- a/lib/archethic_web/controllers/api/web_hosting_controller.ex +++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex @@ -3,11 +3,15 @@ defmodule ArchethicWeb.API.WebHostingController do use ArchethicWeb, :controller - alias Archethic.{Crypto, TransactionChain.Transaction} + alias Archethic.Crypto require Logger - alias ArchethicWeb.API.WebHostingController.{Resources, DirectoryListing} + alias ArchethicWeb.API.WebHostingController.{ + Resources, + DirectoryListing, + ReferenceTransaction + } @spec web_hosting(Plug.Conn.t(), params :: map()) :: Plug.Conn.t() def web_hosting(conn, params = %{"url_path" => []}) do @@ -44,12 +48,12 @@ defmodule ArchethicWeb.API.WebHostingController do {:error, :invalid_encoding} -> send_resp(conn, 400, "Invalid file encoding") - {:error, {:is_a_directory, txn}} -> + {:error, {:is_a_directory, reference_transaction}} -> {:ok, listing_html, encoding, mime_type, cached?, etag} = DirectoryListing.list( conn.request_path, params, - txn, + reference_transaction, cache_headers ) @@ -70,9 +74,8 @@ defmodule ArchethicWeb.API.WebHostingController do | {:error, :website_not_found} | {:error, :invalid_content} | {:error, :file_not_found} - | {:error, :is_a_directory} | {:error, :invalid_encoding} - | {:error, {:is_a_directory, any()}} + | {:error, {:is_a_directory, ReferenceTransaction.t()}} | {:error, any()} def get_website(params = %{"address" => address}, cache_headers) do @@ -80,14 +83,17 @@ defmodule ArchethicWeb.API.WebHostingController do with {:ok, address} <- Base.decode16(address, case: :mixed), true <- Crypto.valid_address?(address), - {:ok, txn = %Transaction{}} <- Archethic.get_last_transaction(address), + {:ok, reference_transaction} <- ReferenceTransaction.fetch_last(address), {:ok, file_content, encoding, mime_type, cached?, etag} <- - Resources.load(txn, url_path, cache_headers) do + Resources.load(reference_transaction, url_path, cache_headers) do {:ok, file_content, encoding, mime_type, cached?, etag} else er when er in [:error, false] -> {:error, :invalid_address} + {:error, %Jason.DecodeError{}} -> + {:error, :invalid_content} + {:error, reason} when reason in [:transaction_not_exists, :transaction_invalid] -> {:error, :website_not_found} diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex b/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex index dfd68a71d..b3a6469ca 100644 --- a/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex +++ b/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex @@ -1,14 +1,14 @@ defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do @moduledoc false - alias Archethic.TransactionChain.{Transaction, Transaction.ValidationStamp, TransactionData} + alias ArchethicWeb.API.WebHostingController.ReferenceTransaction require Logger @spec list( request_path :: String.t(), params :: map(), - transaction :: Transaction.t(), + reference_transaction :: ReferenceTransaction.t(), cached_headers :: list() ) :: {:ok, listing_html :: binary() | nil, encoding :: nil | binary(), mime_type :: binary(), @@ -16,39 +16,32 @@ defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do def list( request_path, params, - %Transaction{ - address: last_address, - data: %TransactionData{content: content}, - validation_stamp: %ValidationStamp{timestamp: timestamp} + %ReferenceTransaction{ + address: address, + timestamp: timestamp, + json_content: json_content }, cache_headers ) do url_path = Map.get(params, "url_path", []) mime_type = "text/html" - case get_cache(cache_headers, last_address, url_path) do + case get_cache(cache_headers, address, url_path) do {cached? = true, etag} -> {:ok, nil, nil, mime_type, cached?, etag} {cached? = false, etag} -> - case Jason.decode(content) do - {:error, err = %Jason.DecodeError{}} -> - {:error, err} - - {:ok, json_content} -> - assigns = - do_list( - request_path, - url_path, - elem(get_metadata(json_content), 1), - timestamp, - last_address - ) - - {:ok, - Phoenix.View.render_to_iodata(ArchethicWeb.DirListingView, "index.html", assigns), - nil, mime_type, cached?, etag} - end + assigns = + do_list( + request_path, + url_path, + elem(get_metadata(json_content), 1), + timestamp, + address + ) + + {:ok, Phoenix.View.render_to_iodata(ArchethicWeb.DirListingView, "index.html", assigns), + nil, mime_type, cached?, etag} end end diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/reference_transaction.ex b/lib/archethic_web/controllers/api/web_hosting_controller/reference_transaction.ex new file mode 100644 index 000000000..35e081384 --- /dev/null +++ b/lib/archethic_web/controllers/api/web_hosting_controller/reference_transaction.ex @@ -0,0 +1,71 @@ +defmodule ArchethicWeb.API.WebHostingController.ReferenceTransaction do + @moduledoc """ + ReferenceTransaction is a subset of a transaction + It is meant to be cached so we use a lighter structure + """ + alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.Transaction.ValidationStamp + alias Archethic.TransactionChain.TransactionData + alias Archethic.TransactionChain.TransactionData.Ownership + alias ArchethicCache.LRU + + @enforce_keys [:address, :json_content, :timestamp, :ownerships] + defstruct [:address, :json_content, :timestamp, :ownerships] + + @type t() :: %__MODULE__{ + address: binary(), + json_content: map(), + timestamp: DateTime.t(), + ownerships: list(Ownership.t()) + } + + @doc """ + Fetch the reference transaction either from cache, or from the network. + """ + @spec fetch(binary()) :: {:ok, t()} | {:error, term()} + def fetch(address) do + # started by ArchethicWeb.Supervisor + cache_server = :web_hosting_cache_ref_tx + cache_key = address + + case LRU.get(cache_server, cache_key) do + nil -> + with {:ok, transaction} <- Archethic.search_transaction(address), + {:ok, reference_transaction} <- from_transaction(transaction) do + :telemetry.execute([:archethic_web, :hosting, :cache_ref_tx, :miss], %{count: 1}) + LRU.put(cache_server, cache_key, reference_transaction) + {:ok, reference_transaction} + end + + reference_transaction -> + :telemetry.execute([:archethic_web, :hosting, :cache_ref_tx, :hit], %{count: 1}) + {:ok, reference_transaction} + end + end + + @doc """ + Fetch the latest reference transaction of the chain, either from cache, or from the network. + """ + @spec fetch_last(binary()) :: {:ok, t()} | {:error, term()} + def fetch_last(address) do + with {:ok, last_address} <- Archethic.get_last_transaction_address(address) do + fetch(last_address) + end + end + + defp from_transaction(%Transaction{ + address: address, + data: %TransactionData{content: content, ownerships: ownerships}, + validation_stamp: %ValidationStamp{timestamp: timestamp} + }) do + with {:ok, json_content} <- Jason.decode(content) do + {:ok, + %__MODULE__{ + address: address, + json_content: json_content, + timestamp: timestamp, + ownerships: ownerships + }} + end + end +end diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex index 090ad0856..33716d0a2 100644 --- a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex +++ b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex @@ -2,45 +2,48 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do @moduledoc false alias Archethic.TransactionChain.{Transaction, TransactionData} + alias ArchethicCache.LRUDisk + alias ArchethicWeb.API.WebHostingController.ReferenceTransaction require Logger - @spec load(tx :: Transaction.t(), url_path :: list(), cache_headers :: list()) :: + @spec load( + reference_transaction :: ReferenceTransaction.t(), + url_path :: list(), + cache_headers :: list() + ) :: {:ok, file_content :: binary() | nil, encoding :: binary() | nil, mime_type :: binary(), cached? :: boolean(), etag :: binary()} | {:error, - :invalid_content - | :file_not_found - | {:is_a_directory, tx :: Transaction.t()} + :file_not_found + | {:is_a_directory, ReferenceTransaction.t()} | :invalid_encoding | any()} def load( - tx = %Transaction{ - address: last_address, - data: %TransactionData{content: content} + reference_transaction = %ReferenceTransaction{ + address: address, + json_content: json_content }, url_path, cache_headers ) do - with {:ok, json_content} <- Jason.decode(content), - {:ok, metadata, _aeweb_version} <- get_metadata(json_content), + with {:ok, metadata, _aeweb_version} <- get_metadata(json_content), {:ok, file, mime_type, resource_path} <- get_file(metadata, url_path), - {cached?, etag} <- get_cache(cache_headers, last_address, url_path), + {cached?, etag} <- get_cache(cache_headers, address, url_path), {:ok, file_content, encoding} <- get_file_content(file, cached?, resource_path) do {:ok, file_content, encoding, mime_type, cached?, etag} else - {:error, %Jason.DecodeError{}} -> - {:error, :invalid_content} + {:error, :invalid_encoding} -> + {:error, :invalid_encoding} {:error, :file_not_found} -> {:error, :file_not_found} {:error, :get_metadata} -> - {:error, - "Error: Cant access metadata and aewebversion, Reftx: #{Base.encode16(last_address)}"} + {:error, "Error: Cant access metadata and aewebversion, Reftx: #{Base.encode16(address)}"} {:error, :is_a_directory} -> - {:error, {:is_a_directory, tx}} + {:error, {:is_a_directory, reference_transaction}} error -> error @@ -62,14 +65,14 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do # index file @spec get_file(metadata :: map(), url_path :: list()) :: {:ok, file :: map(), mime_type :: binary(), resource_path :: binary()} - | {:error, :is_a_directory | :file_not_found | :invalid_encoding} + | {:error, :is_a_directory | :file_not_found} defp get_file(metadata, []) do - case Map.get(metadata, "index.html", :error) do - :error -> + case Map.get(metadata, "index.html") do + nil -> {:error, :is_a_directory} - value -> - {:ok, value, MIME.from_path("index.html"), "index.html"} + file -> + {:ok, file, MIME.from_path("index.html"), "index.html"} end end @@ -81,7 +84,15 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do if is_a_directory?(metadata, resource_path) do {:error, :is_a_directory} else - {:error, :file_not_found} + # Handle JS History API by serving index.html instead of a 404 + # We loose the ability to return real 404 errors + case Map.get(metadata, "index.html") do + nil -> + {:error, :file_not_found} + + file -> + {:ok, file, MIME.from_path("index.html"), "index.html"} + end end file -> @@ -91,46 +102,58 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do @spec get_file_content(file_metadata :: map(), cached? :: boolean(), resource_path :: binary()) :: {:ok, nil | binary(), nil | binary()} - | {:error, :encoding_error | :file_not_found | :invalid_encoding | any()} - defp get_file_content(_, true, _), do: {:ok, nil, nil} - + | {:error, :file_not_found | :invalid_encoding | any()} defp get_file_content( file_metadata = %{"addresses" => address_list}, _cached? = false, resource_path ) do - try do - file_content = - Enum.reduce(address_list, "", fn address, acc -> - {:ok, address_bin} = Base.decode16(address, case: :mixed) + with {:ok, file_content} <- do_get_file_content(address_list, resource_path), + {:ok, encoding} <- access(file_metadata, "encoding", nil) do + {:ok, file_content, encoding} + end + end + + defp get_file_content(_, true, _), do: {:ok, nil, nil} + defp get_file_content(_, _, _), do: {:error, :file_not_found} - {:ok, %Transaction{data: %TransactionData{content: tx_content}}} = - Archethic.search_transaction(address_bin) + defp do_get_file_content(address_list, resource_path) do + # started by ArchethicWeb.Supervisor + cache_server = :web_hosting_cache_file + cache_key = {address_list, resource_path} - {:ok, decoded_content} = Jason.decode(tx_content) + case LRUDisk.get(cache_server, cache_key) do + nil -> + encoded_file_content = + Enum.reduce(address_list, "", fn address, acc -> + {:ok, address_bin} = Base.decode16(address, case: :mixed) - {:ok, res_content} = access(decoded_content, resource_path) + {:ok, %Transaction{data: %TransactionData{content: tx_content}}} = + Archethic.search_transaction(address_bin) - acc <> res_content - end) + {:ok, decoded_content} = Jason.decode(tx_content) - {:ok, file_content} = Base.url_decode64(file_content, padding: false) - {:ok, encoding} = access(file_metadata, "encoding", nil) - {:ok, file_content, encoding} - rescue - MatchError -> - {:error, :invalid_encoding} + {:ok, res_content} = access(decoded_content, resource_path) - ArgumentError -> - {:error, :encoding_error} + acc <> res_content + end) - error -> - {:error, error} + case Base.url_decode64(encoded_file_content, padding: false) do + {:ok, decoded_file_content} -> + :telemetry.execute([:archethic_web, :hosting, :cache_file, :miss], %{count: 1}) + LRUDisk.put(cache_server, cache_key, decoded_file_content) + {:ok, decoded_file_content} + + :error -> + {:error, :invalid_encoding} + end + + decoded_file_content -> + :telemetry.execute([:archethic_web, :hosting, :cache_file, :hit], %{count: 1}) + {:ok, decoded_file_content} end end - defp get_file_content(_, false, _), do: {:error, :file_not_found} - @spec access(map(), key :: binary(), any()) :: {:error, :file_not_found} | {:ok, any()} defp access(map, key, default \\ :file_not_found) do case Map.get(map, key, default) do diff --git a/lib/archethic_web/domain.ex b/lib/archethic_web/domain.ex index 89baa706d..f6ade4e51 100644 --- a/lib/archethic_web/domain.ex +++ b/lib/archethic_web/domain.ex @@ -5,9 +5,8 @@ defmodule ArchethicWeb.Domain do alias Archethic alias Archethic.Crypto - alias Archethic.TransactionChain.Transaction - alias Archethic.TransactionChain.TransactionData alias Archethic.TransactionChain.TransactionData.Ownership + alias ArchethicWeb.API.WebHostingController.ReferenceTransaction require Logger @@ -51,16 +50,11 @@ defmodule ArchethicWeb.Domain do with {:ok, tx_address} <- lookup_dnslink_address(domain), {:ok, tx_address} <- Base.decode16(tx_address, case: :mixed), {:ok, - %Transaction{ - type: :hosting, - data: %TransactionData{ - content: content, - ownerships: [ownership = %Ownership{secret: secret} | _] - } - }} <- - Archethic.get_last_transaction(tx_address), - {:ok, json} <- Jason.decode(content), - {:ok, cert_pem} <- Map.fetch(json, "sslCertificate"), + %ReferenceTransaction{ + json_content: json_content, + ownerships: [ownership = %Ownership{secret: secret} | _] + }} <- ReferenceTransaction.fetch_last(tx_address), + {:ok, cert_pem} <- Map.fetch(json_content, "sslCertificate"), %{extensions: extensions} <- EasySSL.parse_pem(cert_pem), {:ok, san} <- Map.fetch(extensions, :subjectAltName), ^domain <- String.split(san, ":") |> List.last(), diff --git a/lib/archethic_web/supervisor.ex b/lib/archethic_web/supervisor.ex index bd9402af6..bff0cf733 100644 --- a/lib/archethic_web/supervisor.ex +++ b/lib/archethic_web/supervisor.ex @@ -4,7 +4,10 @@ defmodule ArchethicWeb.Supervisor do use Supervisor alias Archethic.Networking + alias Archethic.Utils + alias ArchethicCache.LRU + alias ArchethicCache.LRUDisk alias ArchethicWeb.Endpoint alias ArchethicWeb.{FaucetRateLimiter, TransactionSubscriber, TransactionCache} alias ArchethicWeb.ExplorerLive.TopTransactionsCache @@ -29,7 +32,9 @@ defmodule ArchethicWeb.Supervisor do {Phoenix.PubSub, [name: ArchethicWeb.PubSub, adapter: Phoenix.PubSub.PG2]}, Endpoint, {Absinthe.Subscription, Endpoint}, - TransactionSubscriber + TransactionSubscriber, + web_hosting_cache_ref_tx(), + web_hosting_cache_file() ] |> add_faucet_rate_limit_child() @@ -53,4 +58,31 @@ defmodule ArchethicWeb.Supervisor do children end end + + # this is used in web_hosting_controller.ex + # it does not store an entire transaction, but a triplet {address, json_content, timestamp} + defp web_hosting_cache_ref_tx() do + %{ + id: :web_hosting_cache_ref_tx, + start: + {LRU, :start_link, + [ + :web_hosting_cache_ref_tx, + Application.fetch_env!(:archethic_web, :tx_cache_bytes) + ]} + } + end + + defp web_hosting_cache_file() do + %{ + id: :web_hosting_cache_file, + start: + {LRUDisk, :start_link, + [ + :web_hosting_cache_file, + Application.fetch_env!(:archethic_web, :file_cache_bytes), + Path.join(Utils.mut_dir(), "aeweb") + ]} + } + end end diff --git a/test/archethic_cache/lru_disk_test.exs b/test/archethic_cache/lru_disk_test.exs new file mode 100644 index 000000000..cc3a3f095 --- /dev/null +++ b/test/archethic_cache/lru_disk_test.exs @@ -0,0 +1,167 @@ +defmodule ArchethicCache.LRUDiskTest do + use ExUnit.Case, async: false + + @moduledoc """ + the tests are independent because the ETS table dies with the process and the disk is cleared on init + """ + + @cache_dir Path.join(Archethic.Utils.mut_dir(), "aecache") + + alias ArchethicCache.LRUDisk + + describe "single disk cache" do + test "should return nil when key is not in cache" do + {:ok, pid} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir) + + assert nil == LRUDisk.get(pid, :key1) + end + + test "should cache binaries" do + {:ok, pid} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir) + + LRUDisk.put(pid, :key1, "my binary") + LRUDisk.put(pid, :key2, "my binary2") + LRUDisk.put(pid, :key3, "my binary3") + + assert "my binary" == LRUDisk.get(pid, :key1) + assert "my binary2" == LRUDisk.get(pid, :key2) + assert "my binary3" == LRUDisk.get(pid, :key3) + end + + test "should be able to replace binaries" do + {:ok, pid} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir) + + LRUDisk.put(pid, :key1, "my binary") + LRUDisk.put(pid, :key1, "my binary2") + + assert "my binary2" == LRUDisk.get(pid, :key1) + assert 1 == length(File.ls!(cache_dir_for_ls(:my_cache))) + end + + test "should evict some cached values when there is not enough space available" do + binary = get_a_binary_of_bytes(200) + + {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir) + + LRUDisk.put(pid, :key1, binary) + LRUDisk.put(pid, :key2, binary) + LRUDisk.put(pid, :key3, get_a_binary_of_bytes(400)) + + assert nil == LRUDisk.get(pid, :key1) + assert nil == LRUDisk.get(pid, :key2) + assert 1 == length(File.ls!(cache_dir_for_ls(:my_cache))) + end + + test "should evict the LRU" do + binary = get_a_binary_of_bytes(200) + + {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir) + + LRUDisk.put(pid, :key1, binary) + LRUDisk.put(pid, :key2, binary) + LRUDisk.get(pid, :key1) + LRUDisk.put(pid, :key3, binary) + + assert ^binary = LRUDisk.get(pid, :key1) + assert ^binary = LRUDisk.get(pid, :key3) + assert nil == LRUDisk.get(pid, :key2) + assert 2 == length(File.ls!(cache_dir_for_ls(:my_cache))) + end + + test "should not cache a binary bigger than cache size" do + binary = get_a_binary_of_bytes(500) + + {:ok, pid} = LRUDisk.start_link(:my_cache, 200, @cache_dir) + + assert :ok == LRUDisk.put(pid, :key1, binary) + assert nil == LRUDisk.get(pid, :key1) + assert Enum.empty?(File.ls!(cache_dir_for_ls(:my_cache))) + end + + test "should not crash if an external intervention deletes the file or folder" do + binary = get_a_binary_of_bytes(400) + + server = :my_cache + + start_supervised!(%{ + id: ArchethicCache.LRUDisk, + start: {ArchethicCache.LRUDisk, :start_link, [server, 500, @cache_dir]} + }) + + LRUDisk.put(server, :key1, binary) + + assert ^binary = LRUDisk.get(server, :key1) + + # example of external intervention + File.rm_rf!(cache_dir_for_ls(server)) + + # we loose the cached value + assert nil == LRUDisk.get(server, :key1) + + pid_before_crash = Process.whereis(server) + + # capture_log is used to hide the LRU process terminating + # because we don't want red in our logs when it's expected + # ps: only use it with async: false + ExUnit.CaptureLog.capture_log(fn -> + # if we try to add new values, it will crash the LRU process (write to a non existing dir) + # the cache is restarted from a blank state (recreate dir) by the supervisor + # the caller will not crash (it's a genserver.cast) + LRUDisk.put(server, :key1, binary) + + # allow some time for supervisor to restart the LRU + Process.sleep(100) + end) + + pid_after_crash = Process.whereis(server) + assert Process.alive?(pid_after_crash) + refute Process.alive?(pid_before_crash) + + # cache should automatically restart later + LRUDisk.put(server, :key1, binary) + assert ^binary = LRUDisk.get(server, :key1) + end + + test "should remove when purged" do + binary = get_a_binary_of_bytes(400) + + {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir) + + LRUDisk.put(pid, :key1, binary) + LRUDisk.put(pid, :key2, binary) + LRUDisk.purge(pid) + assert nil == LRUDisk.get(pid, :key1) + assert nil == LRUDisk.get(pid, :key2) + + assert Enum.empty?(File.ls!(cache_dir_for_ls(:my_cache))) + end + end + + describe "multiple disk caches" do + test "should not conflict each other" do + {:ok, pid} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir) + assert {:error, _} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir) + + {:ok, pid2} = LRUDisk.start_link(:my_cache2, 10 * 1024, @cache_dir) + LRUDisk.put(pid, :key1, "value1a") + LRUDisk.put(pid2, :key1, "value1b") + + assert "value1a" == LRUDisk.get(pid, :key1) + assert "value1b" == LRUDisk.get(pid2, :key1) + assert 1 == length(File.ls!(cache_dir_for_ls(:my_cache))) + assert 1 == length(File.ls!(cache_dir_for_ls(:my_cache2))) + end + end + + defp cache_dir_for_ls(name), do: Path.join(@cache_dir, "#{name}") + + defp get_a_binary_of_bytes(bytes) do + get_a_binary_of_bytes(bytes, <<>>) + end + + defp get_a_binary_of_bytes(0, acc), do: acc + + defp get_a_binary_of_bytes(bytes, acc) do + get_a_binary_of_bytes(bytes - 1, <<0::8, acc::binary>>) + end +end diff --git a/test/archethic_cache/lru_test.exs b/test/archethic_cache/lru_test.exs new file mode 100644 index 000000000..87a2c2eea --- /dev/null +++ b/test/archethic_cache/lru_test.exs @@ -0,0 +1,112 @@ +defmodule ArchethicCache.LRUTest do + use ExUnit.Case, async: false + + @moduledoc """ + the tests are independent because the ETS table dies with the process + """ + + alias ArchethicCache.LRU + + describe "single in memory cache" do + test "should return nil when key is not in cache" do + {:ok, pid} = LRU.start_link(:my_cache, 10 * 1024) + + assert nil == LRU.get(pid, :key1) + end + + test "should cache any term" do + {:ok, pid} = LRU.start_link(:my_cache, 10 * 1024) + + LRU.put(pid, :key1, 1) + LRU.put(pid, :key2, :atom) + LRU.put(pid, :key3, %{a: 1}) + LRU.put(pid, {1, 2}, "binary") + + assert 1 == LRU.get(pid, :key1) + assert :atom == LRU.get(pid, :key2) + assert %{a: 1} == LRU.get(pid, :key3) + assert "binary" == LRU.get(pid, {1, 2}) + end + + test "should be able to replace a value" do + {:ok, pid} = LRU.start_link(:my_cache, 10 * 1024) + + LRU.put(pid, :key1, "value1a") + LRU.put(pid, :key1, "value1b") + + assert "value1b" == LRU.get(pid, :key1) + end + + test "should evict some cached values when there is not enough space available" do + binary = get_a_binary_of_bytes(200) + + {:ok, pid} = LRU.start_link(:my_cache, 500) + + LRU.put(pid, :key1, binary) + LRU.put(pid, :key2, binary) + LRU.put(pid, :key3, get_a_binary_of_bytes(400)) + + assert nil == LRU.get(pid, :key1) + assert nil == LRU.get(pid, :key2) + end + + test "should evict the LRU" do + binary = get_a_binary_of_bytes(200) + + {:ok, pid} = LRU.start_link(:my_cache, 500) + + LRU.put(pid, :key1, binary) + LRU.put(pid, :key2, binary) + LRU.get(pid, :key1) + LRU.put(pid, :key3, binary) + + assert ^binary = LRU.get(pid, :key1) + assert nil == LRU.get(pid, :key2) + end + + test "should not cache a binary bigger than cache size" do + binary = get_a_binary_of_bytes(500) + + {:ok, pid} = LRU.start_link(:my_cache, 200) + + assert :ok == LRU.put(pid, :key1, binary) + assert nil == LRU.get(pid, :key1) + end + + test "should remove all when purged" do + binary = get_a_binary_of_bytes(100) + + {:ok, pid} = LRU.start_link(:my_cache, 500) + + LRU.put(pid, :key1, binary) + LRU.put(pid, :key2, binary) + LRU.purge(pid) + assert nil == LRU.get(pid, :key1) + assert nil == LRU.get(pid, :key2) + end + end + + describe "multiple in memory caches" do + test "should not conflict each other" do + {:ok, pid} = LRU.start_link(:my_cache, 10 * 1024) + assert {:error, _} = LRU.start_link(:my_cache, 10 * 1024) + + {:ok, pid2} = LRU.start_link(:my_cache2, 10 * 1024) + LRU.put(pid, :key1, "value1a") + LRU.put(pid2, :key1, "value1b") + + assert "value1a" == LRU.get(pid, :key1) + assert "value1b" == LRU.get(pid2, :key1) + end + end + + defp get_a_binary_of_bytes(bytes) do + get_a_binary_of_bytes(bytes, <<>>) + end + + defp get_a_binary_of_bytes(0, acc), do: acc + + defp get_a_binary_of_bytes(bytes, acc) do + get_a_binary_of_bytes(bytes - 1, <<0::8, acc::binary>>) + end +end diff --git a/test/archethic_web/controllers/api/web_hosting_controller_test.exs b/test/archethic_web/controllers/api/web_hosting_controller_test.exs index f58f5f1b8..44e317bf8 100644 --- a/test/archethic_web/controllers/api/web_hosting_controller_test.exs +++ b/test/archethic_web/controllers/api/web_hosting_controller_test.exs @@ -1,5 +1,5 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do - use ArchethicCase + use ArchethicCase, async: false use ArchethicWeb.ConnCase alias Archethic.P2P @@ -15,9 +15,23 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData + alias Archethic.Utils + + alias ArchethicCache.LRU + alias ArchethicCache.LRUDisk + import Mox setup do + # There is a setup in ArchethicCase that removes the mut_dir() + # Since we need it for LRUDisk, we recreate it on every test + File.mkdir_p!(Path.join([Utils.mut_dir(), "aeweb", "web_hosting_cache_file"])) + + # clear cache on every test because most tests use the same address + # and cache is a global state + :ok = LRU.purge(:web_hosting_cache_ref_tx) + :ok = LRUDisk.purge(:web_hosting_cache_file) + P2P.add_and_connect_node(%Node{ ip: {127, 0, 0, 1}, port: 3000, @@ -63,7 +77,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do address: <<0, 0, 34, 84, 150, 163, 128, 213, 0, 92, 182, 131, 116, 233, 184, 180, 93, 126, 15, 80, 90, 66, 248, 205, 97, 203, 212, 60, 54, 132, 197, 203, 172, 186>>, - data: %TransactionData{content: "invalid"} + data: %TransactionData{content: "invalid"}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} end) @@ -122,7 +139,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do {:ok, %Transaction{ address: address, - data: %TransactionData{content: content} + data: %TransactionData{content: content}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} _, @@ -134,21 +154,24 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do _ -> {:ok, %Transaction{ - data: %TransactionData{content: content2} + data: %TransactionData{content: content2}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} end) :ok end - test "should return file does not exist", %{conn: conn} do + test "should return index.html on file not found (handle JS routing)", %{conn: conn} do conn = get( conn, "/api/web_hosting/0000225496a380d5005cb68374e9b8b45d7e0f505a42f8cd61cbd43c3684c5cbacba/file.html" ) - assert "Cannot find file content" = response(conn, 404) + assert "

Archethic

" = response(conn, 200) end test "should return default index.html file", %{conn: conn} do @@ -252,7 +275,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do {:ok, %Transaction{ address: address, - data: %TransactionData{content: content} + data: %TransactionData{content: content}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} _, @@ -264,7 +290,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do _ -> {:ok, %Transaction{ - data: %TransactionData{content: content2} + data: %TransactionData{content: content2}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} end) @@ -525,7 +554,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do {:ok, %Transaction{ address: address, - data: %TransactionData{content: content} + data: %TransactionData{content: content}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} _, @@ -537,7 +569,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do _ -> {:ok, %Transaction{ - data: %TransactionData{content: content2} + data: %TransactionData{content: content2}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} _, @@ -549,7 +584,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do _ -> {:ok, %Transaction{ - data: %TransactionData{content: content3} + data: %TransactionData{content: content3}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} end) @@ -616,7 +654,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do {:ok, %Transaction{ address: address, - data: %TransactionData{content: content} + data: %TransactionData{content: content}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} _, @@ -628,7 +669,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do _ -> {:ok, %Transaction{ - data: %TransactionData{content: content2} + data: %TransactionData{content: content2}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } }} end)