From 0284134026a6d7bfad497902fd518b419afdca36 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Fri, 2 Dec 2022 15:34:27 +0100
Subject: [PATCH 01/22] cache transactions in the webhostingctlr

---
 config/config.exs                             |  3 +++
 .../controllers/api/web_hosting_controller.ex | 19 +++++++++++++++++++
 lib/archethic_web/supervisor.ex               | 15 ++++++++++++++-
 mix.exs                                       |  1 +
 .../api/web_hosting_controller_test.exs       |  3 +++
 5 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/config/config.exs b/config/config.exs
index d14e69a2d..dd9ab18b0 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -151,6 +151,9 @@ config :archethic, Archethic.Networking.IPLookup.RemoteDiscovery,
 
 # -----End-of-Networking-configs ------
 
+config :archethic_web,
+  tx_cache_bytes: 100 * 1024 * 1024
+
 config :esbuild,
   version: "0.12.18",
   default: [
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex
index d9daea210..ba220e047 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex
@@ -84,6 +84,7 @@ defmodule ArchethicWeb.API.WebHostingController do
          {:ok, file_content, encoding, mime_type, cached?, etag} <-
            Resources.load(txn, url_path, cache_headers) do
       {:ok, file_content, encoding, mime_type, cached?, etag}
+
     else
       er when er in [:error, false] ->
         {:error, :invalid_address}
@@ -143,4 +144,22 @@ defmodule ArchethicWeb.API.WebHostingController do
         else: {conn, file_content}
     end
   end
+
+  @spec search_transaction(binary()) :: {:ok, Transaction.t()} | {:error, atom()}
+  defp search_transaction(address) do
+    case :lru.get(:cache_lru_tx, address) do
+      :undefined ->
+        case Archethic.search_transaction(address) do
+          {:ok, tx} ->
+            :lru.add(:cache_lru_tx, address, tx)
+            {:ok, tx}
+
+          {:error, reason} ->
+            {:error, reason}
+        end
+
+      tx ->
+        {:ok, tx}
+    end
+  end
 end
diff --git a/lib/archethic_web/supervisor.ex b/lib/archethic_web/supervisor.ex
index bd9402af6..8f1943965 100644
--- a/lib/archethic_web/supervisor.ex
+++ b/lib/archethic_web/supervisor.ex
@@ -29,7 +29,8 @@ defmodule ArchethicWeb.Supervisor do
         {Phoenix.PubSub, [name: ArchethicWeb.PubSub, adapter: Phoenix.PubSub.PG2]},
         Endpoint,
         {Absinthe.Subscription, Endpoint},
-        TransactionSubscriber
+        TransactionSubscriber,
+        cache_child_spec()
       ]
       |> add_faucet_rate_limit_child()
 
@@ -53,4 +54,16 @@ defmodule ArchethicWeb.Supervisor do
       children
     end
   end
+
+  defp cache_child_spec() do
+    %{
+      id: :cache_lru_tx,
+      start:
+        {:lru, :start_link,
+         [
+           {:local, :cache_lru_tx},
+           [max_size: Application.fetch_env!(:archethic_web, :tx_cache_bytes)]
+         ]}
+    }
+  end
 end
diff --git a/mix.exs b/mix.exs
index ad3ae8676..44cd318ed 100644
--- a/mix.exs
+++ b/mix.exs
@@ -57,6 +57,7 @@ defmodule Archethic.MixProject do
       {:mint, "~> 1.0"},
       {:ecto, "~> 3.9"},
       {:websockex, "~> 0.4"},
+      {:lru, "~> 2.4"},
 
       # Dev
       {:benchee, "~> 1.1"},
diff --git a/test/archethic_web/controllers/api/web_hosting_controller_test.exs b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
index f58f5f1b8..00cfe4209 100644
--- a/test/archethic_web/controllers/api/web_hosting_controller_test.exs
+++ b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
@@ -18,6 +18,9 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
   import Mox
 
   setup do
+    # clear cache on every test because most tests use the same address
+    :ok = :lru.purge(:cache_lru_tx)
+
     P2P.add_and_connect_node(%Node{
       ip: {127, 0, 0, 1},
       port: 3000,

From 28e06366b8e60f3131395c023f289cded5500b64 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Tue, 6 Dec 2022 10:45:25 +0100
Subject: [PATCH 02/22] Create a new LRU cache because of licensing issue with
 previous package

---
 config/config.exs                             |   3 +-
 lib/archethic_cache/lru.ex                    | 149 ++++++++++++++++++
 lib/archethic_cache/lru_disk.ex               |  74 +++++++++
 .../controllers/api/web_hosting_controller.ex |  19 ++-
 lib/archethic_web/supervisor.ex               |  26 ++-
 mix.exs                                       |   1 -
 test/archethic_cache/lru_disk_test.exs        | 141 +++++++++++++++++
 test/archethic_cache/lru_test.exs             | 112 +++++++++++++
 .../api/web_hosting_controller_test.exs       |   6 +-
 9 files changed, 518 insertions(+), 13 deletions(-)
 create mode 100644 lib/archethic_cache/lru.ex
 create mode 100644 lib/archethic_cache/lru_disk.ex
 create mode 100644 test/archethic_cache/lru_disk_test.exs
 create mode 100644 test/archethic_cache/lru_test.exs

diff --git a/config/config.exs b/config/config.exs
index dd9ab18b0..847f80f39 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -152,7 +152,8 @@ config :archethic, Archethic.Networking.IPLookup.RemoteDiscovery,
 # -----End-of-Networking-configs ------
 
 config :archethic_web,
-  tx_cache_bytes: 100 * 1024 * 1024
+  tx_cache_bytes: 128 * 1024 * 1024,
+  file_cache_bytes: 512 * 1024 * 1024
 
 config :esbuild,
   version: "0.12.18",
diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex
new file mode 100644
index 000000000..5fbaaae5f
--- /dev/null
+++ b/lib/archethic_cache/lru.ex
@@ -0,0 +1,149 @@
+defmodule ArchethicCache.LRU do
+  @moduledoc """
+  A cache that stores the values in an ETS table.
+  There are hooks available to be able to add effects (ex: write to disk).
+
+  It keeps track of the order and bytes in the genserver state.
+  The `bytes_used` are tracked in here because if we just monitor ETS table size, we would not be able to have a disk cache.
+  The `keys` are used to determine the Least Recent Used (first is the most recent used, last is the least recent used).
+  """
+
+  @spec start_link(GenServer.name(), non_neg_integer(), keyword()) ::
+          {:ok, binary()} | {:error, term()}
+  def start_link(name, max_bytes, opts \\ []) do
+    GenServer.start_link(__MODULE__, [name, max_bytes, opts], name: name)
+  end
+
+  @spec put(GenServer.server(), term(), term()) :: boolean()
+  def put(pid, key, value) do
+    GenServer.call(pid, {:put, key, value})
+  end
+
+  @spec get(GenServer.server(), term()) :: nil | term()
+  def get(pid, key) do
+    GenServer.call(pid, {:get, key})
+  end
+
+  @spec purge(GenServer.server()) :: :ok
+  def purge(pid) do
+    GenServer.call(pid, :purge)
+  end
+
+  def init([name, max_bytes, opts]) do
+    table = :ets.new(:"aecache_#{name}", [:set, {:read_concurrency, true}])
+
+    {:ok,
+     %{
+       table: table,
+       bytes_max: max_bytes,
+       bytes_used: 0,
+       keys: [],
+       put_fn:
+         Keyword.get(opts, :put_fn, fn _key, value ->
+           {value, :erlang.external_size(value)}
+         end),
+       get_fn:
+         Keyword.get(opts, :get_fn, fn _key, value ->
+           value
+         end),
+       evict_fn:
+         Keyword.get(opts, :evict_fn, fn _key, value ->
+           :erlang.external_size(value)
+         end)
+     }}
+  end
+
+  def handle_call({:get, key}, _from, state) do
+    {reply, new_state} =
+      case :ets.lookup(state.table, key) do
+        [{^key, value}] ->
+          {
+            state.get_fn.(key, value),
+            %{state | keys: state.keys |> move_front(key)}
+          }
+
+        [] ->
+          {nil, state}
+      end
+
+    {:reply, reply, new_state}
+  end
+
+  def handle_call({:put, key, value}, _from, state) do
+    size = :erlang.external_size(value)
+
+    if size > state.bytes_max do
+      {:reply, false, state}
+    else
+      # maybe evict some keys to make space
+      state =
+        evict_until(state, fn state ->
+          state.bytes_used + size <= state.bytes_max
+        end)
+
+      case :ets.lookup(state.table, key) do
+        [] ->
+          {value_to_store, size_to_store} = state.put_fn.(key, value)
+
+          :ets.insert(state.table, {key, value_to_store})
+
+          new_state = %{
+            state
+            | keys: [key | state.keys],
+              bytes_used: state.bytes_used + size_to_store
+          }
+
+          {:reply, true, new_state}
+
+        [{^key, old_value}] ->
+          # this is a replacement, we need to evict to update the bytes_used
+          bytes_evicted = state.evict_fn.(key, old_value)
+          {value_to_store, size_to_store} = state.put_fn.(key, value)
+
+          :ets.insert(state.table, {key, value_to_store})
+
+          new_state = %{
+            state
+            | keys: state.keys |> move_front(key),
+              bytes_used: state.bytes_used + size_to_store - bytes_evicted
+          }
+
+          {:reply, true, new_state}
+      end
+    end
+  end
+
+  def handle_call(:purge, _from, state) do
+    # we call the evict_fn to be able to clean effects (ex: file written to disk)
+    :ets.tab2list(state.table)
+    |> Enum.each(fn {key, value} ->
+      _ = state.evict_fn.(key, value)
+    end)
+
+    :ets.delete_all_objects(state.table)
+    {:reply, :ok, %{state | keys: [], bytes_used: 0}}
+  end
+
+  defp evict_until(state, predicate) do
+    if predicate.(state) do
+      state
+    else
+      oldest_key = List.last(state.keys)
+      [{_, oldest_value}] = :ets.take(state.table, oldest_key)
+      bytes_evicted = state.evict_fn.(oldest_key, oldest_value)
+
+      evict_until(
+        %{
+          state
+          | bytes_used: state.bytes_used - bytes_evicted,
+            keys: List.delete(state.keys, oldest_key)
+        },
+        predicate
+      )
+    end
+  end
+
+  defp move_front(list, item) do
+    [item | List.delete(list, item)]
+  end
+end
diff --git a/lib/archethic_cache/lru_disk.ex b/lib/archethic_cache/lru_disk.ex
new file mode 100644
index 000000000..666506ab0
--- /dev/null
+++ b/lib/archethic_cache/lru_disk.ex
@@ -0,0 +1,74 @@
+defmodule ArchethicCache.LRUDisk do
+  @moduledoc """
+  Wraps the LRU genserver and adds hooks to write / read from disk.
+  The value is always a binary.
+  """
+  alias ArchethicCache.LRU
+
+  require Logger
+
+  @spec start_link(GenServer.name(), non_neg_integer(), binary()) ::
+          {:ok, binary()} | {:error, term()}
+  def start_link(name, max_bytes, cache_dir) do
+    cache_dir = Path.join(cache_dir, "#{name}")
+    :ok = reset_directory(cache_dir)
+
+    LRU.start_link(name, max_bytes,
+      put_fn: fn key, value ->
+        # retry to create the dir everytime in case someones delete it
+        File.mkdir_p!(cache_dir)
+
+        # write to disk
+        File.write!(key_to_path(cache_dir, key), value, [:exclusive, :binary])
+
+        # return value to store in memory and size
+        # we use the size as value so it's available in the evict fn without doing a File.stat
+        size = byte_size(value)
+        {size, size}
+      end,
+      get_fn: fn key, _size ->
+        # called only if the key is already in LRU's ETS table
+        case File.read(key_to_path(cache_dir, key)) do
+          {:ok, bin} ->
+            bin
+
+          {:error, _} ->
+            nil
+        end
+      end,
+      evict_fn: fn key, size ->
+        case File.rm(key_to_path(cache_dir, key)) do
+          :ok ->
+            :ok
+
+          {:error, _} ->
+            :ok
+        end
+
+        # return size deleted
+        size
+      end
+    )
+  end
+
+  @spec put(GenServer.server(), term(), binary()) :: boolean()
+  defdelegate put(pid, key, value), to: LRU, as: :put
+
+  @spec get(GenServer.server(), term()) :: nil | binary()
+  defdelegate get(pid, key), to: LRU, as: :get
+
+  @spec purge(GenServer.server()) :: :ok
+  defdelegate purge(pid), to: LRU, as: :purge
+
+  defp reset_directory(dir) do
+    File.rm_rf!(dir)
+    File.mkdir_p!(dir)
+  end
+
+  defp key_to_path(cache_dir, key) do
+    # DISCUSS: use a proper hash function
+    # DISCUSS: this is flat and not site/file
+    hash = Base.encode64(:erlang.term_to_binary(key))
+    Path.join(cache_dir, hash)
+  end
+end
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex
index ba220e047..ed7cb5b0d 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex
@@ -3,7 +3,19 @@ defmodule ArchethicWeb.API.WebHostingController do
 
   use ArchethicWeb, :controller
 
+<<<<<<< HEAD
   alias Archethic.{Crypto, TransactionChain.Transaction}
+=======
+  alias Archethic
+  alias Archethic.Crypto
+  alias Archethic.TransactionChain.Transaction
+  alias Archethic.TransactionChain.Transaction.ValidationStamp
+  alias Archethic.TransactionChain.TransactionData
+
+  alias ArchethicCache.LRU
+
+  use Pathex
+>>>>>>> a774b69b (Create a new LRU cache because of licensing issue with previous package)
 
   require Logger
 
@@ -147,11 +159,12 @@ defmodule ArchethicWeb.API.WebHostingController do
 
   @spec search_transaction(binary()) :: {:ok, Transaction.t()} | {:error, atom()}
   defp search_transaction(address) do
-    case :lru.get(:cache_lru_tx, address) do
-      :undefined ->
+    # :cache_tx is started by the ArchethicWeb.Supervisor
+    case LRU.get(:cache_tx, address) do
+      nil ->
         case Archethic.search_transaction(address) do
           {:ok, tx} ->
-            :lru.add(:cache_lru_tx, address, tx)
+            LRU.put(:cache_tx, address, tx)
             {:ok, tx}
 
           {:error, reason} ->
diff --git a/lib/archethic_web/supervisor.ex b/lib/archethic_web/supervisor.ex
index 8f1943965..31a6d9776 100644
--- a/lib/archethic_web/supervisor.ex
+++ b/lib/archethic_web/supervisor.ex
@@ -5,6 +5,7 @@ defmodule ArchethicWeb.Supervisor do
 
   alias Archethic.Networking
 
+  alias ArchethicCache.LRU
   alias ArchethicWeb.Endpoint
   alias ArchethicWeb.{FaucetRateLimiter, TransactionSubscriber, TransactionCache}
   alias ArchethicWeb.ExplorerLive.TopTransactionsCache
@@ -30,7 +31,8 @@ defmodule ArchethicWeb.Supervisor do
         Endpoint,
         {Absinthe.Subscription, Endpoint},
         TransactionSubscriber,
-        cache_child_spec()
+        cache_tx(),
+        cache_file()
       ]
       |> add_faucet_rate_limit_child()
 
@@ -55,14 +57,26 @@ defmodule ArchethicWeb.Supervisor do
     end
   end
 
-  defp cache_child_spec() do
+  defp cache_tx() do
     %{
-      id: :cache_lru_tx,
+      id: :cache_tx,
       start:
-        {:lru, :start_link,
+        {LRU, :start_link,
          [
-           {:local, :cache_lru_tx},
-           [max_size: Application.fetch_env!(:archethic_web, :tx_cache_bytes)]
+           :cache_tx,
+           Application.fetch_env!(:archethic_web, :tx_cache_bytes)
+         ]}
+    }
+  end
+
+  defp cache_file() do
+    %{
+      id: :cache_file,
+      start:
+        {LRU, :start_link,
+         [
+           :cache_file,
+           Application.fetch_env!(:archethic_web, :file_cache_bytes)
          ]}
     }
   end
diff --git a/mix.exs b/mix.exs
index 44cd318ed..ad3ae8676 100644
--- a/mix.exs
+++ b/mix.exs
@@ -57,7 +57,6 @@ defmodule Archethic.MixProject do
       {:mint, "~> 1.0"},
       {:ecto, "~> 3.9"},
       {:websockex, "~> 0.4"},
-      {:lru, "~> 2.4"},
 
       # Dev
       {:benchee, "~> 1.1"},
diff --git a/test/archethic_cache/lru_disk_test.exs b/test/archethic_cache/lru_disk_test.exs
new file mode 100644
index 000000000..569387e0d
--- /dev/null
+++ b/test/archethic_cache/lru_disk_test.exs
@@ -0,0 +1,141 @@
+defmodule ArchethicCache.LRUDiskTest do
+  use ExUnit.Case, async: false
+
+  @moduledoc """
+  the tests are independent because the ETS table dies with the process and the disk is cleared on init
+  """
+
+  @cache_dir Path.join(Archethic.Utils.mut_dir(), "aecache")
+
+  alias ArchethicCache.LRUDisk
+
+  describe "single disk cache" do
+    test "should return nil when key is not in cache" do
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir)
+
+      assert nil == LRUDisk.get(pid, :key1)
+    end
+
+    test "should cache binaries" do
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir)
+
+      LRUDisk.put(pid, :key1, "my binary")
+      LRUDisk.put(pid, :key2, "my binary2")
+      LRUDisk.put(pid, :key3, "my binary3")
+
+      assert "my binary" == LRUDisk.get(pid, :key1)
+      assert "my binary2" == LRUDisk.get(pid, :key2)
+      assert "my binary3" == LRUDisk.get(pid, :key3)
+    end
+
+    test "should be able to replace binaries" do
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir)
+
+      LRUDisk.put(pid, :key1, "my binary")
+      LRUDisk.put(pid, :key1, "my binary2")
+
+      assert "my binary2" == LRUDisk.get(pid, :key1)
+      assert 1 == length(File.ls!(cache_dir_for_ls(:my_cache)))
+    end
+
+    test "should evict some cached values when there is not enough space available" do
+      binary = get_a_binary_of_bytes(200)
+
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir)
+
+      LRUDisk.put(pid, :key1, binary)
+      LRUDisk.put(pid, :key2, binary)
+      LRUDisk.put(pid, :key3, get_a_binary_of_bytes(400))
+
+      assert nil == LRUDisk.get(pid, :key1)
+      assert nil == LRUDisk.get(pid, :key2)
+      assert 1 == length(File.ls!(cache_dir_for_ls(:my_cache)))
+    end
+
+    test "should evict the LRU" do
+      binary = get_a_binary_of_bytes(200)
+
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir)
+
+      LRUDisk.put(pid, :key1, binary)
+      LRUDisk.put(pid, :key2, binary)
+      LRUDisk.get(pid, :key1)
+      LRUDisk.put(pid, :key3, binary)
+
+      assert ^binary = LRUDisk.get(pid, :key1)
+      assert ^binary = LRUDisk.get(pid, :key3)
+      assert nil == LRUDisk.get(pid, :key2)
+      assert 2 == length(File.ls!(cache_dir_for_ls(:my_cache)))
+    end
+
+    test "should not cache a binary bigger than cache size" do
+      binary = get_a_binary_of_bytes(500)
+
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 200, @cache_dir)
+
+      assert false == LRUDisk.put(pid, :key1, binary)
+      assert nil == LRUDisk.get(pid, :key1)
+      assert Enum.empty?(File.ls!(cache_dir_for_ls(:my_cache)))
+    end
+
+    test "should not crash if an external intervention deletes the file or folder" do
+      binary = get_a_binary_of_bytes(400)
+
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir)
+
+      LRUDisk.put(pid, :key1, binary)
+
+      assert ^binary = LRUDisk.get(pid, :key1)
+
+      # example of external intervention
+      File.rm_rf!(cache_dir_for_ls(:my_cache))
+
+      # we loose the cached value
+      assert nil == LRUDisk.get(pid, :key1)
+
+      # but if we try to add new values, it should resume
+      LRUDisk.put(pid, :key1, binary)
+      assert ^binary = LRUDisk.get(pid, :key1)
+    end
+
+    test "should remove when purged" do
+      binary = get_a_binary_of_bytes(400)
+
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir)
+
+      LRUDisk.put(pid, :key1, binary)
+      LRUDisk.purge(pid)
+      assert nil == LRUDisk.get(pid, :key1)
+
+      assert Enum.empty?(File.ls!(cache_dir_for_ls(:my_cache)))
+    end
+  end
+
+  describe "multiple disk caches" do
+    test "should not conflict each other" do
+      {:ok, pid} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir)
+      assert {:error, _} = LRUDisk.start_link(:my_cache, 10 * 1024, @cache_dir)
+
+      {:ok, pid2} = LRUDisk.start_link(:my_cache2, 10 * 1024, @cache_dir)
+      LRUDisk.put(pid, :key1, "value1a")
+      LRUDisk.put(pid2, :key1, "value1b")
+
+      assert "value1a" == LRUDisk.get(pid, :key1)
+      assert "value1b" == LRUDisk.get(pid2, :key1)
+      assert 1 == length(File.ls!(cache_dir_for_ls(:my_cache)))
+      assert 1 == length(File.ls!(cache_dir_for_ls(:my_cache2)))
+    end
+  end
+
+  defp cache_dir_for_ls(name), do: Path.join(@cache_dir, "#{name}")
+
+  defp get_a_binary_of_bytes(bytes) do
+    get_a_binary_of_bytes(bytes, <<>>)
+  end
+
+  defp get_a_binary_of_bytes(0, acc), do: acc
+
+  defp get_a_binary_of_bytes(bytes, acc) do
+    get_a_binary_of_bytes(bytes - 1, <<0::8, acc::binary>>)
+  end
+end
diff --git a/test/archethic_cache/lru_test.exs b/test/archethic_cache/lru_test.exs
new file mode 100644
index 000000000..cee94df55
--- /dev/null
+++ b/test/archethic_cache/lru_test.exs
@@ -0,0 +1,112 @@
+defmodule ArchethicCache.LRUTest do
+  use ExUnit.Case, async: false
+
+  @moduledoc """
+  the tests are independent because the ETS table dies with the process
+  """
+
+  alias ArchethicCache.LRU
+
+  describe "single in memory cache" do
+    test "should return nil when key is not in cache" do
+      {:ok, pid} = LRU.start_link(:my_cache, 10 * 1024)
+
+      assert nil == LRU.get(pid, :key1)
+    end
+
+    test "should cache any term" do
+      {:ok, pid} = LRU.start_link(:my_cache, 10 * 1024)
+
+      LRU.put(pid, :key1, 1)
+      LRU.put(pid, :key2, :atom)
+      LRU.put(pid, :key3, %{a: 1})
+      LRU.put(pid, {1, 2}, "binary")
+
+      assert 1 == LRU.get(pid, :key1)
+      assert :atom == LRU.get(pid, :key2)
+      assert %{a: 1} == LRU.get(pid, :key3)
+      assert "binary" == LRU.get(pid, {1, 2})
+    end
+
+    test "should be able to replace a value" do
+      {:ok, pid} = LRU.start_link(:my_cache, 10 * 1024)
+
+      LRU.put(pid, :key1, "value1a")
+      LRU.put(pid, :key1, "value1b")
+
+      assert "value1b" == LRU.get(pid, :key1)
+    end
+
+    test "should evict some cached values when there is not enough space available" do
+      binary = get_a_binary_of_bytes(200)
+
+      {:ok, pid} = LRU.start_link(:my_cache, 500)
+
+      LRU.put(pid, :key1, binary)
+      LRU.put(pid, :key2, binary)
+      LRU.put(pid, :key3, get_a_binary_of_bytes(400))
+
+      assert nil == LRU.get(pid, :key1)
+      assert nil == LRU.get(pid, :key2)
+    end
+
+    test "should evict the LRU" do
+      binary = get_a_binary_of_bytes(200)
+
+      {:ok, pid} = LRU.start_link(:my_cache, 500)
+
+      LRU.put(pid, :key1, binary)
+      LRU.put(pid, :key2, binary)
+      LRU.get(pid, :key1)
+      LRU.put(pid, :key3, binary)
+
+      assert ^binary = LRU.get(pid, :key1)
+      assert nil == LRU.get(pid, :key2)
+    end
+
+    test "should not cache a binary bigger than cache size" do
+      binary = get_a_binary_of_bytes(500)
+
+      {:ok, pid} = LRU.start_link(:my_cache, 200)
+
+      assert false == LRU.put(pid, :key1, binary)
+      assert nil == LRU.get(pid, :key1)
+    end
+
+    test "should remove all when purged" do
+      binary = get_a_binary_of_bytes(100)
+
+      {:ok, pid} = LRU.start_link(:my_cache, 500)
+
+      LRU.put(pid, :key1, binary)
+      LRU.put(pid, :key2, binary)
+      LRU.purge(pid)
+      assert nil == LRU.get(pid, :key1)
+      assert nil == LRU.get(pid, :key2)
+    end
+  end
+
+  describe "multiple in memory caches" do
+    test "should not conflict each other" do
+      {:ok, pid} = LRU.start_link(:my_cache, 10 * 1024)
+      assert {:error, _} = LRU.start_link(:my_cache, 10 * 1024)
+
+      {:ok, pid2} = LRU.start_link(:my_cache2, 10 * 1024)
+      LRU.put(pid, :key1, "value1a")
+      LRU.put(pid2, :key1, "value1b")
+
+      assert "value1a" == LRU.get(pid, :key1)
+      assert "value1b" == LRU.get(pid2, :key1)
+    end
+  end
+
+  defp get_a_binary_of_bytes(bytes) do
+    get_a_binary_of_bytes(bytes, <<>>)
+  end
+
+  defp get_a_binary_of_bytes(0, acc), do: acc
+
+  defp get_a_binary_of_bytes(bytes, acc) do
+    get_a_binary_of_bytes(bytes - 1, <<0::8, acc::binary>>)
+  end
+end
diff --git a/test/archethic_web/controllers/api/web_hosting_controller_test.exs b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
index 00cfe4209..69b926bf6 100644
--- a/test/archethic_web/controllers/api/web_hosting_controller_test.exs
+++ b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
@@ -1,5 +1,5 @@
 defmodule ArchethicWeb.API.WebHostingControllerTest do
-  use ArchethicCase
+  use ArchethicCase, async: false
   use ArchethicWeb.ConnCase
 
   alias Archethic.P2P
@@ -15,11 +15,13 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
   alias Archethic.TransactionChain.Transaction.ValidationStamp
   alias Archethic.TransactionChain.TransactionData
 
+  alias ArchethicCache.LRU
+
   import Mox
 
   setup do
     # clear cache on every test because most tests use the same address
-    :ok = :lru.purge(:cache_lru_tx)
+    :ok = LRU.purge(:cache_tx)
 
     P2P.add_and_connect_node(%Node{
       ip: {127, 0, 0, 1},

From 868c73ea1422b2ab26decbe6580110621276bf65 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Tue, 6 Dec 2022 11:27:48 +0100
Subject: [PATCH 03/22] fix child spec of cache_file

---
 lib/archethic_web/supervisor.ex | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/archethic_web/supervisor.ex b/lib/archethic_web/supervisor.ex
index 31a6d9776..aced1914b 100644
--- a/lib/archethic_web/supervisor.ex
+++ b/lib/archethic_web/supervisor.ex
@@ -4,8 +4,10 @@ defmodule ArchethicWeb.Supervisor do
   use Supervisor
 
   alias Archethic.Networking
+  alias Archethic.Utils
 
   alias ArchethicCache.LRU
+  alias ArchethicCache.LRUDisk
   alias ArchethicWeb.Endpoint
   alias ArchethicWeb.{FaucetRateLimiter, TransactionSubscriber, TransactionCache}
   alias ArchethicWeb.ExplorerLive.TopTransactionsCache
@@ -73,10 +75,11 @@ defmodule ArchethicWeb.Supervisor do
     %{
       id: :cache_file,
       start:
-        {LRU, :start_link,
+        {LRUDisk, :start_link,
          [
            :cache_file,
-           Application.fetch_env!(:archethic_web, :file_cache_bytes)
+           Application.fetch_env!(:archethic_web, :file_cache_bytes),
+           Path.join(Utils.mut_dir(), "aeweb")
          ]}
     }
   end

From a57be619ecb7e97eafd58de8ec8da632f69e3dce Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Tue, 6 Dec 2022 12:06:43 +0100
Subject: [PATCH 04/22] Use a disk cache to cache the files

---
 .../controllers/api/web_hosting_controller.ex       | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex
index ed7cb5b0d..7e3e68c06 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex
@@ -3,19 +3,7 @@ defmodule ArchethicWeb.API.WebHostingController do
 
   use ArchethicWeb, :controller
 
-<<<<<<< HEAD
   alias Archethic.{Crypto, TransactionChain.Transaction}
-=======
-  alias Archethic
-  alias Archethic.Crypto
-  alias Archethic.TransactionChain.Transaction
-  alias Archethic.TransactionChain.Transaction.ValidationStamp
-  alias Archethic.TransactionChain.TransactionData
-
-  alias ArchethicCache.LRU
-
-  use Pathex
->>>>>>> a774b69b (Create a new LRU cache because of licensing issue with previous package)
 
   require Logger
 
@@ -96,7 +84,6 @@ defmodule ArchethicWeb.API.WebHostingController do
          {:ok, file_content, encoding, mime_type, cached?, etag} <-
            Resources.load(txn, url_path, cache_headers) do
       {:ok, file_content, encoding, mime_type, cached?, etag}
-
     else
       er when er in [:error, false] ->
         {:error, :invalid_address}

From d1d3749caf2da25ca66afa89529b0959b626245e Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Tue, 6 Dec 2022 14:42:32 +0100
Subject: [PATCH 05/22] Use cast instead of call for lru.put/3

caller do not need to wait for the I/O to end to continue
---
 lib/archethic_cache/lru.ex             | 72 +++++++++++++-------------
 lib/archethic_cache/lru_disk.ex        |  2 +-
 test/archethic_cache/lru_disk_test.exs |  2 +-
 test/archethic_cache/lru_test.exs      |  2 +-
 4 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex
index 5fbaaae5f..f645c89bb 100644
--- a/lib/archethic_cache/lru.ex
+++ b/lib/archethic_cache/lru.ex
@@ -14,9 +14,9 @@ defmodule ArchethicCache.LRU do
     GenServer.start_link(__MODULE__, [name, max_bytes, opts], name: name)
   end
 
-  @spec put(GenServer.server(), term(), term()) :: boolean()
+  @spec put(GenServer.server(), term(), term()) :: :ok
   def put(pid, key, value) do
-    GenServer.call(pid, {:put, key, value})
+    GenServer.cast(pid, {:put, key, value})
   end
 
   @spec get(GenServer.server(), term()) :: nil | term()
@@ -69,11 +69,41 @@ defmodule ArchethicCache.LRU do
     {:reply, reply, new_state}
   end
 
-  def handle_call({:put, key, value}, _from, state) do
+  def handle_call(:purge, _from, state) do
+    # we call the evict_fn to be able to clean effects (ex: file written to disk)
+    :ets.tab2list(state.table)
+    |> Enum.each(fn {key, value} ->
+      _ = state.evict_fn.(key, value)
+    end)
+
+    :ets.delete_all_objects(state.table)
+    {:reply, :ok, %{state | keys: [], bytes_used: 0}}
+  end
+
+  defp evict_until(state, predicate) do
+    if predicate.(state) do
+      state
+    else
+      oldest_key = List.last(state.keys)
+      [{_, oldest_value}] = :ets.take(state.table, oldest_key)
+      bytes_evicted = state.evict_fn.(oldest_key, oldest_value)
+
+      evict_until(
+        %{
+          state
+          | bytes_used: state.bytes_used - bytes_evicted,
+            keys: List.delete(state.keys, oldest_key)
+        },
+        predicate
+      )
+    end
+  end
+
+  def handle_cast({:put, key, value}, state) do
     size = :erlang.external_size(value)
 
     if size > state.bytes_max do
-      {:reply, false, state}
+      {:noreply, state}
     else
       # maybe evict some keys to make space
       state =
@@ -93,7 +123,7 @@ defmodule ArchethicCache.LRU do
               bytes_used: state.bytes_used + size_to_store
           }
 
-          {:reply, true, new_state}
+          {:noreply, new_state}
 
         [{^key, old_value}] ->
           # this is a replacement, we need to evict to update the bytes_used
@@ -108,41 +138,11 @@ defmodule ArchethicCache.LRU do
               bytes_used: state.bytes_used + size_to_store - bytes_evicted
           }
 
-          {:reply, true, new_state}
+          {:noreply, new_state}
       end
     end
   end
 
-  def handle_call(:purge, _from, state) do
-    # we call the evict_fn to be able to clean effects (ex: file written to disk)
-    :ets.tab2list(state.table)
-    |> Enum.each(fn {key, value} ->
-      _ = state.evict_fn.(key, value)
-    end)
-
-    :ets.delete_all_objects(state.table)
-    {:reply, :ok, %{state | keys: [], bytes_used: 0}}
-  end
-
-  defp evict_until(state, predicate) do
-    if predicate.(state) do
-      state
-    else
-      oldest_key = List.last(state.keys)
-      [{_, oldest_value}] = :ets.take(state.table, oldest_key)
-      bytes_evicted = state.evict_fn.(oldest_key, oldest_value)
-
-      evict_until(
-        %{
-          state
-          | bytes_used: state.bytes_used - bytes_evicted,
-            keys: List.delete(state.keys, oldest_key)
-        },
-        predicate
-      )
-    end
-  end
-
   defp move_front(list, item) do
     [item | List.delete(list, item)]
   end
diff --git a/lib/archethic_cache/lru_disk.ex b/lib/archethic_cache/lru_disk.ex
index 666506ab0..3c8296141 100644
--- a/lib/archethic_cache/lru_disk.ex
+++ b/lib/archethic_cache/lru_disk.ex
@@ -51,7 +51,7 @@ defmodule ArchethicCache.LRUDisk do
     )
   end
 
-  @spec put(GenServer.server(), term(), binary()) :: boolean()
+  @spec put(GenServer.server(), term(), binary()) :: :ok
   defdelegate put(pid, key, value), to: LRU, as: :put
 
   @spec get(GenServer.server(), term()) :: nil | binary()
diff --git a/test/archethic_cache/lru_disk_test.exs b/test/archethic_cache/lru_disk_test.exs
index 569387e0d..3477ee9d1 100644
--- a/test/archethic_cache/lru_disk_test.exs
+++ b/test/archethic_cache/lru_disk_test.exs
@@ -73,7 +73,7 @@ defmodule ArchethicCache.LRUDiskTest do
 
       {:ok, pid} = LRUDisk.start_link(:my_cache, 200, @cache_dir)
 
-      assert false == LRUDisk.put(pid, :key1, binary)
+      assert :ok == LRUDisk.put(pid, :key1, binary)
       assert nil == LRUDisk.get(pid, :key1)
       assert Enum.empty?(File.ls!(cache_dir_for_ls(:my_cache)))
     end
diff --git a/test/archethic_cache/lru_test.exs b/test/archethic_cache/lru_test.exs
index cee94df55..87a2c2eea 100644
--- a/test/archethic_cache/lru_test.exs
+++ b/test/archethic_cache/lru_test.exs
@@ -69,7 +69,7 @@ defmodule ArchethicCache.LRUTest do
 
       {:ok, pid} = LRU.start_link(:my_cache, 200)
 
-      assert false == LRU.put(pid, :key1, binary)
+      assert :ok == LRU.put(pid, :key1, binary)
       assert nil == LRU.get(pid, :key1)
     end
 

From 244645342b0f4c081093ba640c771ae3a9e1fb18 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Tue, 6 Dec 2022 16:38:22 +0100
Subject: [PATCH 06/22] Handle edge case where there is a falsy predicate on a
 empty cache

---
 lib/archethic_cache/lru.ex | 43 +++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex
index f645c89bb..535d005a8 100644
--- a/lib/archethic_cache/lru.ex
+++ b/lib/archethic_cache/lru.ex
@@ -80,25 +80,6 @@ defmodule ArchethicCache.LRU do
     {:reply, :ok, %{state | keys: [], bytes_used: 0}}
   end
 
-  defp evict_until(state, predicate) do
-    if predicate.(state) do
-      state
-    else
-      oldest_key = List.last(state.keys)
-      [{_, oldest_value}] = :ets.take(state.table, oldest_key)
-      bytes_evicted = state.evict_fn.(oldest_key, oldest_value)
-
-      evict_until(
-        %{
-          state
-          | bytes_used: state.bytes_used - bytes_evicted,
-            keys: List.delete(state.keys, oldest_key)
-        },
-        predicate
-      )
-    end
-  end
-
   def handle_cast({:put, key, value}, state) do
     size = :erlang.external_size(value)
 
@@ -143,6 +124,30 @@ defmodule ArchethicCache.LRU do
     end
   end
 
+  defp evict_until(state, predicate) do
+    if predicate.(state) do
+      state
+    else
+      case List.last(state.keys) do
+        nil ->
+          state
+
+        oldest_key ->
+          [{_, oldest_value}] = :ets.take(state.table, oldest_key)
+          bytes_evicted = state.evict_fn.(oldest_key, oldest_value)
+
+          evict_until(
+            %{
+              state
+              | bytes_used: state.bytes_used - bytes_evicted,
+                keys: List.delete(state.keys, oldest_key)
+            },
+            predicate
+          )
+      end
+    end
+  end
+
   defp move_front(list, item) do
     [item | List.delete(list, item)]
   end

From ea4f236c740573c0b1e8a4a185557df07c8f827a Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Thu, 15 Dec 2022 18:23:21 +0100
Subject: [PATCH 07/22] store size in ets table

---
 lib/archethic_cache/lru.ex      | 45 +++++++++++++++------------------
 lib/archethic_cache/lru_disk.ex | 13 +++-------
 2 files changed, 24 insertions(+), 34 deletions(-)

diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex
index 535d005a8..cae96dd07 100644
--- a/lib/archethic_cache/lru.ex
+++ b/lib/archethic_cache/lru.ex
@@ -6,6 +6,10 @@ defmodule ArchethicCache.LRU do
   It keeps track of the order and bytes in the genserver state.
   The `bytes_used` are tracked in here because if we just monitor ETS table size, we would not be able to have a disk cache.
   The `keys` are used to determine the Least Recent Used (first is the most recent used, last is the least recent used).
+
+  We do not store the values directly in ETS but we insert a pair {size, value} instead.
+  Because size can be modified with the hooks
+  (ex: For LRUDisk, we discard the value from the ETS table, but still want to know the size written to disk)
   """
 
   @spec start_link(GenServer.name(), non_neg_integer(), keyword()) ::
@@ -38,25 +42,16 @@ defmodule ArchethicCache.LRU do
        bytes_max: max_bytes,
        bytes_used: 0,
        keys: [],
-       put_fn:
-         Keyword.get(opts, :put_fn, fn _key, value ->
-           {value, :erlang.external_size(value)}
-         end),
-       get_fn:
-         Keyword.get(opts, :get_fn, fn _key, value ->
-           value
-         end),
-       evict_fn:
-         Keyword.get(opts, :evict_fn, fn _key, value ->
-           :erlang.external_size(value)
-         end)
+       put_fn: Keyword.get(opts, :put_fn, fn _key, value -> value end),
+       get_fn: Keyword.get(opts, :get_fn, fn _key, value -> value end),
+       evict_fn: Keyword.get(opts, :evict_fn, fn _key, _value -> :ok end)
      }}
   end
 
   def handle_call({:get, key}, _from, state) do
     {reply, new_state} =
       case :ets.lookup(state.table, key) do
-        [{^key, value}] ->
+        [{^key, {_size, value}}] ->
           {
             state.get_fn.(key, value),
             %{state | keys: state.keys |> move_front(key)}
@@ -72,7 +67,7 @@ defmodule ArchethicCache.LRU do
   def handle_call(:purge, _from, state) do
     # we call the evict_fn to be able to clean effects (ex: file written to disk)
     :ets.tab2list(state.table)
-    |> Enum.each(fn {key, value} ->
+    |> Enum.each(fn {key, {_size, value}} ->
       _ = state.evict_fn.(key, value)
     end)
 
@@ -94,29 +89,29 @@ defmodule ArchethicCache.LRU do
 
       case :ets.lookup(state.table, key) do
         [] ->
-          {value_to_store, size_to_store} = state.put_fn.(key, value)
+          value_to_store = state.put_fn.(key, value)
 
-          :ets.insert(state.table, {key, value_to_store})
+          :ets.insert(state.table, {key, {size, value_to_store}})
 
           new_state = %{
             state
             | keys: [key | state.keys],
-              bytes_used: state.bytes_used + size_to_store
+              bytes_used: state.bytes_used + size
           }
 
           {:noreply, new_state}
 
-        [{^key, old_value}] ->
+        [{^key, {old_size, old_value}}] ->
           # this is a replacement, we need to evict to update the bytes_used
-          bytes_evicted = state.evict_fn.(key, old_value)
-          {value_to_store, size_to_store} = state.put_fn.(key, value)
+          state.evict_fn.(key, old_value)
+          value_to_store = state.put_fn.(key, value)
 
-          :ets.insert(state.table, {key, value_to_store})
+          :ets.insert(state.table, {key, {size, value_to_store}})
 
           new_state = %{
             state
             | keys: state.keys |> move_front(key),
-              bytes_used: state.bytes_used + size_to_store - bytes_evicted
+              bytes_used: state.bytes_used + size - old_size
           }
 
           {:noreply, new_state}
@@ -133,13 +128,13 @@ defmodule ArchethicCache.LRU do
           state
 
         oldest_key ->
-          [{_, oldest_value}] = :ets.take(state.table, oldest_key)
-          bytes_evicted = state.evict_fn.(oldest_key, oldest_value)
+          [{_, {size, oldest_value}}] = :ets.take(state.table, oldest_key)
+          state.evict_fn.(oldest_key, oldest_value)
 
           evict_until(
             %{
               state
-              | bytes_used: state.bytes_used - bytes_evicted,
+              | bytes_used: state.bytes_used - size,
                 keys: List.delete(state.keys, oldest_key)
             },
             predicate
diff --git a/lib/archethic_cache/lru_disk.ex b/lib/archethic_cache/lru_disk.ex
index 3c8296141..c6f04a8d2 100644
--- a/lib/archethic_cache/lru_disk.ex
+++ b/lib/archethic_cache/lru_disk.ex
@@ -21,12 +21,10 @@ defmodule ArchethicCache.LRUDisk do
         # write to disk
         File.write!(key_to_path(cache_dir, key), value, [:exclusive, :binary])
 
-        # return value to store in memory and size
-        # we use the size as value so it's available in the evict fn without doing a File.stat
-        size = byte_size(value)
-        {size, size}
+        # store a nil value in the LRU's ETS table
+        nil
       end,
-      get_fn: fn key, _size ->
+      get_fn: fn key, nil ->
         # called only if the key is already in LRU's ETS table
         case File.read(key_to_path(cache_dir, key)) do
           {:ok, bin} ->
@@ -36,7 +34,7 @@ defmodule ArchethicCache.LRUDisk do
             nil
         end
       end,
-      evict_fn: fn key, size ->
+      evict_fn: fn key, nil ->
         case File.rm(key_to_path(cache_dir, key)) do
           :ok ->
             :ok
@@ -44,9 +42,6 @@ defmodule ArchethicCache.LRUDisk do
           {:error, _} ->
             :ok
         end
-
-        # return size deleted
-        size
       end
     )
   end

From 532f239e152e44393abb2a3e10b059127d999332 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Thu, 15 Dec 2022 18:24:03 +0100
Subject: [PATCH 08/22] lint: naming

---
 lib/archethic_cache/lru.ex      | 12 ++++++------
 lib/archethic_cache/lru_disk.ex |  6 +++---
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex
index cae96dd07..af273465f 100644
--- a/lib/archethic_cache/lru.ex
+++ b/lib/archethic_cache/lru.ex
@@ -19,18 +19,18 @@ defmodule ArchethicCache.LRU do
   end
 
   @spec put(GenServer.server(), term(), term()) :: :ok
-  def put(pid, key, value) do
-    GenServer.cast(pid, {:put, key, value})
+  def put(server, key, value) do
+    GenServer.cast(server, {:put, key, value})
   end
 
   @spec get(GenServer.server(), term()) :: nil | term()
-  def get(pid, key) do
-    GenServer.call(pid, {:get, key})
+  def get(server, key) do
+    GenServer.call(server, {:get, key})
   end
 
   @spec purge(GenServer.server()) :: :ok
-  def purge(pid) do
-    GenServer.call(pid, :purge)
+  def purge(server) do
+    GenServer.call(server, :purge)
   end
 
   def init([name, max_bytes, opts]) do
diff --git a/lib/archethic_cache/lru_disk.ex b/lib/archethic_cache/lru_disk.ex
index c6f04a8d2..15019ffd0 100644
--- a/lib/archethic_cache/lru_disk.ex
+++ b/lib/archethic_cache/lru_disk.ex
@@ -47,13 +47,13 @@ defmodule ArchethicCache.LRUDisk do
   end
 
   @spec put(GenServer.server(), term(), binary()) :: :ok
-  defdelegate put(pid, key, value), to: LRU, as: :put
+  defdelegate put(server, key, value), to: LRU, as: :put
 
   @spec get(GenServer.server(), term()) :: nil | binary()
-  defdelegate get(pid, key), to: LRU, as: :get
+  defdelegate get(server, key), to: LRU, as: :get
 
   @spec purge(GenServer.server()) :: :ok
-  defdelegate purge(pid), to: LRU, as: :purge
+  defdelegate purge(server), to: LRU, as: :purge
 
   defp reset_directory(dir) do
     File.rm_rf!(dir)

From 485a170316e73723ec164096ac5d2d37a79a38be Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Thu, 15 Dec 2022 18:37:29 +0100
Subject: [PATCH 09/22] add comments on config values

---
 config/config.exs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 847f80f39..5e8f147a0 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -152,8 +152,14 @@ config :archethic, Archethic.Networking.IPLookup.RemoteDiscovery,
 # -----End-of-Networking-configs ------
 
 config :archethic_web,
-  tx_cache_bytes: 128 * 1024 * 1024,
-  file_cache_bytes: 512 * 1024 * 1024
+  # The tx_cache is stored on RAM
+  # 750MB should hold a minimum 250 transactions
+  tx_cache_bytes: 750 * 1024 * 1024,
+
+  # The file_cache is stored on DISK
+  # 5GB should hold 2000 average size pages
+  # https://httparchive.org/reports/page-weight
+  file_cache_bytes: 5 * 1024 * 1024 * 1024
 
 config :esbuild,
   version: "0.12.18",

From d0af8f3c25277ba75b1ccf914c8214aa67970519 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Thu, 15 Dec 2022 18:52:53 +0100
Subject: [PATCH 10/22] lint: destructuration convention

---
 lib/archethic_cache/lru.ex | 54 +++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex
index af273465f..6751f152b 100644
--- a/lib/archethic_cache/lru.ex
+++ b/lib/archethic_cache/lru.ex
@@ -48,13 +48,13 @@ defmodule ArchethicCache.LRU do
      }}
   end
 
-  def handle_call({:get, key}, _from, state) do
+  def handle_call({:get, key}, _from, state = %{table: table, keys: keys, get_fn: get_fn}) do
     {reply, new_state} =
-      case :ets.lookup(state.table, key) do
+      case :ets.lookup(table, key) do
         [{^key, {_size, value}}] ->
           {
-            state.get_fn.(key, value),
-            %{state | keys: state.keys |> move_front(key)}
+            get_fn.(key, value),
+            %{state | keys: keys |> move_front(key)}
           }
 
         [] ->
@@ -64,34 +64,37 @@ defmodule ArchethicCache.LRU do
     {:reply, reply, new_state}
   end
 
-  def handle_call(:purge, _from, state) do
+  def handle_call(:purge, _from, state = %{table: table, evict_fn: evict_fn}) do
     # we call the evict_fn to be able to clean effects (ex: file written to disk)
-    :ets.tab2list(state.table)
+    :ets.tab2list(table)
     |> Enum.each(fn {key, {_size, value}} ->
-      _ = state.evict_fn.(key, value)
+      evict_fn.(key, value)
     end)
 
-    :ets.delete_all_objects(state.table)
+    :ets.delete_all_objects(table)
     {:reply, :ok, %{state | keys: [], bytes_used: 0}}
   end
 
-  def handle_cast({:put, key, value}, state) do
+  def handle_cast(
+        {:put, key, value},
+        state = %{table: table, bytes_max: bytes_max, put_fn: put_fn, evict_fn: evict_fn}
+      ) do
     size = :erlang.external_size(value)
 
-    if size > state.bytes_max do
+    if size > bytes_max do
       {:noreply, state}
     else
       # maybe evict some keys to make space
       state =
-        evict_until(state, fn state ->
-          state.bytes_used + size <= state.bytes_max
+        evict_until(state, fn %{bytes_used: bytes_used, bytes_max: bytes_max} ->
+          bytes_used + size <= bytes_max
         end)
 
-      case :ets.lookup(state.table, key) do
+      case :ets.lookup(table, key) do
         [] ->
-          value_to_store = state.put_fn.(key, value)
+          value_to_store = put_fn.(key, value)
 
-          :ets.insert(state.table, {key, {size, value_to_store}})
+          :ets.insert(table, {key, {size, value_to_store}})
 
           new_state = %{
             state
@@ -103,10 +106,10 @@ defmodule ArchethicCache.LRU do
 
         [{^key, {old_size, old_value}}] ->
           # this is a replacement, we need to evict to update the bytes_used
-          state.evict_fn.(key, old_value)
-          value_to_store = state.put_fn.(key, value)
+          evict_fn.(key, old_value)
+          value_to_store = put_fn.(key, value)
 
-          :ets.insert(state.table, {key, {size, value_to_store}})
+          :ets.insert(table, {key, {size, value_to_store}})
 
           new_state = %{
             state
@@ -119,23 +122,26 @@ defmodule ArchethicCache.LRU do
     end
   end
 
-  defp evict_until(state, predicate) do
+  defp evict_until(
+         state = %{table: table, keys: keys, evict_fn: evict_fn, bytes_used: bytes_used},
+         predicate
+       ) do
     if predicate.(state) do
       state
     else
-      case List.last(state.keys) do
+      case List.last(keys) do
         nil ->
           state
 
         oldest_key ->
-          [{_, {size, oldest_value}}] = :ets.take(state.table, oldest_key)
-          state.evict_fn.(oldest_key, oldest_value)
+          [{_, {size, oldest_value}}] = :ets.take(table, oldest_key)
+          evict_fn.(oldest_key, oldest_value)
 
           evict_until(
             %{
               state
-              | bytes_used: state.bytes_used - size,
-                keys: List.delete(state.keys, oldest_key)
+              | bytes_used: bytes_used - size,
+                keys: List.delete(keys, oldest_key)
             },
             predicate
           )

From d32b7dfc246e296cd9f30a5f419d002051fcbd94 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Thu, 15 Dec 2022 18:57:44 +0100
Subject: [PATCH 11/22] remove comment

---
 lib/archethic_cache/lru_disk.ex | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/archethic_cache/lru_disk.ex b/lib/archethic_cache/lru_disk.ex
index 15019ffd0..06a199d21 100644
--- a/lib/archethic_cache/lru_disk.ex
+++ b/lib/archethic_cache/lru_disk.ex
@@ -61,9 +61,9 @@ defmodule ArchethicCache.LRUDisk do
   end
 
   defp key_to_path(cache_dir, key) do
-    # DISCUSS: use a proper hash function
-    # DISCUSS: this is flat and not site/file
-    hash = Base.encode64(:erlang.term_to_binary(key))
-    Path.join(cache_dir, hash)
+    Path.join(
+      cache_dir,
+      Base.encode64(:erlang.term_to_binary(key))
+    )
   end
 end

From 1252b5036cc77d3cd66f6ba25dfa4380735210b9 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Fri, 16 Dec 2022 15:52:48 +0100
Subject: [PATCH 12/22] do not try to recreate dir on every put

---
 lib/archethic_cache/lru_disk.ex        |  3 --
 test/archethic_cache/lru_disk_test.exs | 42 ++++++++++++++++++++------
 2 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/lib/archethic_cache/lru_disk.ex b/lib/archethic_cache/lru_disk.ex
index 06a199d21..fc9e4dba0 100644
--- a/lib/archethic_cache/lru_disk.ex
+++ b/lib/archethic_cache/lru_disk.ex
@@ -15,9 +15,6 @@ defmodule ArchethicCache.LRUDisk do
 
     LRU.start_link(name, max_bytes,
       put_fn: fn key, value ->
-        # retry to create the dir everytime in case someones delete it
-        File.mkdir_p!(cache_dir)
-
         # write to disk
         File.write!(key_to_path(cache_dir, key), value, [:exclusive, :binary])
 
diff --git a/test/archethic_cache/lru_disk_test.exs b/test/archethic_cache/lru_disk_test.exs
index 3477ee9d1..bfa6b4af4 100644
--- a/test/archethic_cache/lru_disk_test.exs
+++ b/test/archethic_cache/lru_disk_test.exs
@@ -81,21 +81,45 @@ defmodule ArchethicCache.LRUDiskTest do
     test "should not crash if an external intervention deletes the file or folder" do
       binary = get_a_binary_of_bytes(400)
 
-      {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir)
+      server = :my_cache
 
-      LRUDisk.put(pid, :key1, binary)
+      start_supervised!(%{
+        id: ArchethicCache.LRUDisk,
+        start: {ArchethicCache.LRUDisk, :start_link, [server, 500, @cache_dir]}
+      })
 
-      assert ^binary = LRUDisk.get(pid, :key1)
+      LRUDisk.put(server, :key1, binary)
+
+      assert ^binary = LRUDisk.get(server, :key1)
 
       # example of external intervention
-      File.rm_rf!(cache_dir_for_ls(:my_cache))
+      File.rm_rf!(cache_dir_for_ls(server))
 
       # we loose the cached value
-      assert nil == LRUDisk.get(pid, :key1)
-
-      # but if we try to add new values, it should resume
-      LRUDisk.put(pid, :key1, binary)
-      assert ^binary = LRUDisk.get(pid, :key1)
+      assert nil == LRUDisk.get(server, :key1)
+
+      pid_before_crash = Process.whereis(server)
+
+      # capture_log is used to hide the LRU process terminating
+      # because we don't want red in our logs when it's expected
+      # ps: only use it with async: false
+      ExUnit.CaptureLog.capture_log(fn ->
+        # if we try to add new values, it will crash the LRU process (write to a non existing dir)
+        # the cache is restarted from a blank state (recreate dir) by the supervisor
+        # the caller will not crash (it's a genserver.cast)
+        LRUDisk.put(server, :key1, binary)
+
+        # allow some time for supervisor to restart the LRU
+        Process.sleep(100)
+      end)
+
+      pid_after_crash = Process.whereis(server)
+      assert Process.alive?(pid_after_crash)
+      refute Process.alive?(pid_before_crash)
+
+      # cache should automatically restart later
+      LRUDisk.put(server, :key1, binary)
+      assert ^binary = LRUDisk.get(server, :key1)
     end
 
     test "should remove when purged" do

From 177beb0acaed81be1beb746c182fa8eeb7db5787 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Fri, 16 Dec 2022 16:18:31 +0100
Subject: [PATCH 13/22] evict_until optimization

---
 lib/archethic_cache/lru.ex | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex
index 6751f152b..072922249 100644
--- a/lib/archethic_cache/lru.ex
+++ b/lib/archethic_cache/lru.ex
@@ -129,11 +129,11 @@ defmodule ArchethicCache.LRU do
     if predicate.(state) do
       state
     else
-      case List.last(keys) do
-        nil ->
+      case Enum.reverse(keys) do
+        [] ->
           state
 
-        oldest_key ->
+        [oldest_key | rest] ->
           [{_, {size, oldest_value}}] = :ets.take(table, oldest_key)
           evict_fn.(oldest_key, oldest_value)
 
@@ -141,7 +141,7 @@ defmodule ArchethicCache.LRU do
             %{
               state
               | bytes_used: bytes_used - size,
-                keys: List.delete(keys, oldest_key)
+                keys: rest
             },
             predicate
           )

From 47df3c7899b779e2275acaf4968611f99c20603b Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Fri, 16 Dec 2022 16:27:48 +0100
Subject: [PATCH 14/22] purge optimization

---
 lib/archethic_cache/lru.ex             | 12 ++++++++----
 test/archethic_cache/lru_disk_test.exs |  2 ++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/archethic_cache/lru.ex b/lib/archethic_cache/lru.ex
index 072922249..0d63096f5 100644
--- a/lib/archethic_cache/lru.ex
+++ b/lib/archethic_cache/lru.ex
@@ -66,10 +66,14 @@ defmodule ArchethicCache.LRU do
 
   def handle_call(:purge, _from, state = %{table: table, evict_fn: evict_fn}) do
     # we call the evict_fn to be able to clean effects (ex: file written to disk)
-    :ets.tab2list(table)
-    |> Enum.each(fn {key, {_size, value}} ->
-      evict_fn.(key, value)
-    end)
+    :ets.foldr(
+      fn {key, {_size, value}}, acc ->
+        evict_fn.(key, value)
+        acc + 1
+      end,
+      0,
+      table
+    )
 
     :ets.delete_all_objects(table)
     {:reply, :ok, %{state | keys: [], bytes_used: 0}}
diff --git a/test/archethic_cache/lru_disk_test.exs b/test/archethic_cache/lru_disk_test.exs
index bfa6b4af4..cc3a3f095 100644
--- a/test/archethic_cache/lru_disk_test.exs
+++ b/test/archethic_cache/lru_disk_test.exs
@@ -128,8 +128,10 @@ defmodule ArchethicCache.LRUDiskTest do
       {:ok, pid} = LRUDisk.start_link(:my_cache, 500, @cache_dir)
 
       LRUDisk.put(pid, :key1, binary)
+      LRUDisk.put(pid, :key2, binary)
       LRUDisk.purge(pid)
       assert nil == LRUDisk.get(pid, :key1)
+      assert nil == LRUDisk.get(pid, :key2)
 
       assert Enum.empty?(File.ls!(cache_dir_for_ls(:my_cache)))
     end

From 3cd54a81e02f1c3be5be68ac7dfb979beebde978 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Fri, 16 Dec 2022 16:46:41 +0100
Subject: [PATCH 15/22] recreate folder on every test

---
 .../controllers/api/web_hosting_controller_test.exs      | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/archethic_web/controllers/api/web_hosting_controller_test.exs b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
index 69b926bf6..19149e0d7 100644
--- a/test/archethic_web/controllers/api/web_hosting_controller_test.exs
+++ b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
@@ -15,13 +15,22 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
   alias Archethic.TransactionChain.Transaction.ValidationStamp
   alias Archethic.TransactionChain.TransactionData
 
+  alias Archethic.Utils
+
   alias ArchethicCache.LRU
+  alias ArchethicCache.LRUDisk
 
   import Mox
 
   setup do
+    # There is a setup in ArchethicCase that removes the mut_dir()
+    # Since we need it for LRUDisk, we recreate it on every test
+    File.mkdir_p!(Path.join([Utils.mut_dir(), "aeweb", "cache_file"]))
+
     # clear cache on every test because most tests use the same address
+    # and cache is a global state
     :ok = LRU.purge(:cache_tx)
+    :ok = LRUDisk.purge(:cache_file)
 
     P2P.add_and_connect_node(%Node{
       ip: {127, 0, 0, 1},

From 6e4b9770caec621be7b09f04e8daede6a851044b Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Wed, 21 Dec 2022 09:18:38 +0100
Subject: [PATCH 16/22] refactor key_to_path fn

---
 lib/archethic_cache/lru_disk.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/archethic_cache/lru_disk.ex b/lib/archethic_cache/lru_disk.ex
index fc9e4dba0..b7872b5bf 100644
--- a/lib/archethic_cache/lru_disk.ex
+++ b/lib/archethic_cache/lru_disk.ex
@@ -4,6 +4,7 @@ defmodule ArchethicCache.LRUDisk do
   The value is always a binary.
   """
   alias ArchethicCache.LRU
+  alias Archethic.Crypto
 
   require Logger
 
@@ -60,7 +61,7 @@ defmodule ArchethicCache.LRUDisk do
   defp key_to_path(cache_dir, key) do
     Path.join(
       cache_dir,
-      Base.encode64(:erlang.term_to_binary(key))
+      Crypto.hash(:erlang.term_to_binary(key)) |> Base.url_encode64(padding: false)
     )
   end
 end

From 9032e4e53fcd9a44d27b35e85d3a3ef44781e196 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Wed, 4 Jan 2023 17:40:59 +0100
Subject: [PATCH 17/22] Cache the reference transactions (as a triplet) and the
 files content

---
 .../controllers/aeweb_root_controller.ex      |  4 +-
 .../controllers/api/web_hosting_controller.ex | 60 ++++++++-----
 .../directory_listing.ex                      | 39 +++------
 .../api/web_hosting_controller/resources.ex   | 84 ++++++++++---------
 lib/archethic_web/supervisor.ex               | 18 ++--
 .../api/web_hosting_controller_test.exs       | 56 ++++++++++---
 6 files changed, 153 insertions(+), 108 deletions(-)

diff --git a/lib/archethic_web/controllers/aeweb_root_controller.ex b/lib/archethic_web/controllers/aeweb_root_controller.ex
index 19af2217a..88c87455d 100644
--- a/lib/archethic_web/controllers/aeweb_root_controller.ex
+++ b/lib/archethic_web/controllers/aeweb_root_controller.ex
@@ -12,12 +12,12 @@ defmodule ArchethicWeb.AEWebRootController do
       {:ok, file_content, encoding, mime_type, cached?, etag} ->
         WebHostingController.send_response(conn, file_content, encoding, mime_type, cached?, etag)
 
-      {:error, {:is_a_directory, transaction}} ->
+      {:error, {:is_a_directory, reference_transaction}} ->
         {:ok, listing_html, encoding, mime_type, cached?, etag} =
           WebHostingController.DirectoryListing.list(
             conn.request_path,
             params,
-            transaction,
+            reference_transaction,
             cache_headers
           )
 
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex
index 7e3e68c06..d38096ef3 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex
@@ -3,7 +3,14 @@ defmodule ArchethicWeb.API.WebHostingController do
 
   use ArchethicWeb, :controller
 
-  alias Archethic.{Crypto, TransactionChain.Transaction}
+  alias Archethic.{
+    Crypto,
+    TransactionChain.Transaction,
+    TransactionChain.Transaction.ValidationStamp,
+    TransactionChain.TransactionData
+  }
+
+  alias ArchethicCache.LRU
 
   require Logger
 
@@ -44,12 +51,12 @@ defmodule ArchethicWeb.API.WebHostingController do
       {:error, :invalid_encoding} ->
         send_resp(conn, 400, "Invalid file encoding")
 
-      {:error, {:is_a_directory, txn}} ->
+      {:error, {:is_a_directory, reference_transaction}} ->
         {:ok, listing_html, encoding, mime_type, cached?, etag} =
           DirectoryListing.list(
             conn.request_path,
             params,
-            txn,
+            reference_transaction,
             cache_headers
           )
 
@@ -70,9 +77,8 @@ defmodule ArchethicWeb.API.WebHostingController do
           | {:error, :website_not_found}
           | {:error, :invalid_content}
           | {:error, :file_not_found}
-          | {:error, :is_a_directory}
           | {:error, :invalid_encoding}
-          | {:error, {:is_a_directory, any()}}
+          | {:error, {:is_a_directory, {binary(), map(), DateTime.t()}}}
           | {:error, any()}
 
   def get_website(params = %{"address" => address}, cache_headers) do
@@ -80,14 +86,18 @@ defmodule ArchethicWeb.API.WebHostingController do
 
     with {:ok, address} <- Base.decode16(address, case: :mixed),
          true <- Crypto.valid_address?(address),
-         {:ok, txn = %Transaction{}} <- Archethic.get_last_transaction(address),
+         {:ok, last_address} <- Archethic.get_last_transaction_address(address),
+         {:ok, reference_transaction} <- get_reference_transaction(last_address),
          {:ok, file_content, encoding, mime_type, cached?, etag} <-
-           Resources.load(txn, url_path, cache_headers) do
+           Resources.load(reference_transaction, url_path, cache_headers) do
       {:ok, file_content, encoding, mime_type, cached?, etag}
     else
       er when er in [:error, false] ->
         {:error, :invalid_address}
 
+      {:error, %Jason.DecodeError{}} ->
+        {:error, :invalid_content}
+
       {:error, reason} when reason in [:transaction_not_exists, :transaction_invalid] ->
         {:error, :website_not_found}
 
@@ -144,22 +154,32 @@ defmodule ArchethicWeb.API.WebHostingController do
     end
   end
 
-  @spec search_transaction(binary()) :: {:ok, Transaction.t()} | {:error, atom()}
-  defp search_transaction(address) do
-    # :cache_tx is started by the ArchethicWeb.Supervisor
-    case LRU.get(:cache_tx, address) do
+  # Fetch the reference transaction either from cache, or from the network.
+  #
+  # Instead of returning the entire transaction,
+  # we return a triplet with only the formatted data we need
+  @spec get_reference_transaction(binary()) ::
+          {:ok, {binary(), map(), DateTime.t()}} | {:error, atom()}
+  defp get_reference_transaction(address) do
+    # started by ArchethicWeb.Supervisor
+    cache_server = :web_hosting_cache_ref_tx
+    cache_key = address
+
+    case LRU.get(cache_server, cache_key) do
       nil ->
-        case Archethic.search_transaction(address) do
-          {:ok, tx} ->
-            LRU.put(:cache_tx, address, tx)
-            {:ok, tx}
-
-          {:error, reason} ->
-            {:error, reason}
+        with {:ok,
+              %Transaction{
+                data: %TransactionData{content: content},
+                validation_stamp: %ValidationStamp{timestamp: timestamp}
+              }} <- Archethic.search_transaction(address),
+             {:ok, json_content} <- Jason.decode(content) do
+          reference_transaction = {address, json_content, timestamp}
+          LRU.put(cache_server, cache_key, reference_transaction)
+          {:ok, reference_transaction}
         end
 
-      tx ->
-        {:ok, tx}
+      reference_transaction ->
+        {:ok, reference_transaction}
     end
   end
 end
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex b/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
index dfd68a71d..f32ad3707 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
@@ -1,14 +1,12 @@
 defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do
   @moduledoc false
 
-  alias Archethic.TransactionChain.{Transaction, Transaction.ValidationStamp, TransactionData}
-
   require Logger
 
   @spec list(
           request_path :: String.t(),
           params :: map(),
-          transaction :: Transaction.t(),
+          reference_transaction :: {binary(), map(), DateTime.t()},
           cached_headers :: list()
         ) ::
           {:ok, listing_html :: binary() | nil, encoding :: nil | binary(), mime_type :: binary(),
@@ -16,11 +14,7 @@ defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do
   def list(
         request_path,
         params,
-        %Transaction{
-          address: last_address,
-          data: %TransactionData{content: content},
-          validation_stamp: %ValidationStamp{timestamp: timestamp}
-        },
+        {last_address, json_content, timestamp},
         cache_headers
       ) do
     url_path = Map.get(params, "url_path", [])
@@ -31,24 +25,17 @@ defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do
         {:ok, nil, nil, mime_type, cached?, etag}
 
       {cached? = false, etag} ->
-        case Jason.decode(content) do
-          {:error, err = %Jason.DecodeError{}} ->
-            {:error, err}
-
-          {:ok, json_content} ->
-            assigns =
-              do_list(
-                request_path,
-                url_path,
-                elem(get_metadata(json_content), 1),
-                timestamp,
-                last_address
-              )
-
-            {:ok,
-             Phoenix.View.render_to_iodata(ArchethicWeb.DirListingView, "index.html", assigns),
-             nil, mime_type, cached?, etag}
-        end
+        assigns =
+          do_list(
+            request_path,
+            url_path,
+            elem(get_metadata(json_content), 1),
+            timestamp,
+            last_address
+          )
+
+        {:ok, Phoenix.View.render_to_iodata(ArchethicWeb.DirListingView, "index.html", assigns),
+         nil, mime_type, cached?, etag}
     end
   end
 
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
index 090ad0856..7e5ddf79f 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
@@ -2,35 +2,31 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
   @moduledoc false
 
   alias Archethic.TransactionChain.{Transaction, TransactionData}
+  alias ArchethicCache.LRUDisk
 
   require Logger
 
-  @spec load(tx :: Transaction.t(), url_path :: list(), cache_headers :: list()) ::
+  @spec load(tx :: {binary(), map(), DateTime.t()}, url_path :: list(), cache_headers :: list()) ::
           {:ok, file_content :: binary() | nil, encoding :: binary() | nil, mime_type :: binary(),
            cached? :: boolean(), etag :: binary()}
           | {:error,
-             :invalid_content
-             | :file_not_found
-             | {:is_a_directory, tx :: Transaction.t()}
+             :file_not_found
+             | {:is_a_directory, {binary(), map(), DateTime.t()}}
              | :invalid_encoding
              | any()}
   def load(
-        tx = %Transaction{
-          address: last_address,
-          data: %TransactionData{content: content}
-        },
+        reference_transaction = {last_address, json_content, _timestamp},
         url_path,
         cache_headers
       ) do
-    with {:ok, json_content} <- Jason.decode(content),
-         {:ok, metadata, _aeweb_version} <- get_metadata(json_content),
+    with {:ok, metadata, _aeweb_version} <- get_metadata(json_content),
          {:ok, file, mime_type, resource_path} <- get_file(metadata, url_path),
          {cached?, etag} <- get_cache(cache_headers, last_address, url_path),
          {:ok, file_content, encoding} <- get_file_content(file, cached?, resource_path) do
       {:ok, file_content, encoding, mime_type, cached?, etag}
     else
-      {:error, %Jason.DecodeError{}} ->
-        {:error, :invalid_content}
+      {:error, :invalid_encoding} ->
+        {:error, :invalid_encoding}
 
       {:error, :file_not_found} ->
         {:error, :file_not_found}
@@ -40,7 +36,7 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
          "Error: Cant access metadata and aewebversion, Reftx: #{Base.encode16(last_address)}"}
 
       {:error, :is_a_directory} ->
-        {:error, {:is_a_directory, tx}}
+        {:error, {:is_a_directory, reference_transaction}}
 
       error ->
         error
@@ -62,7 +58,7 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
   # index file
   @spec get_file(metadata :: map(), url_path :: list()) ::
           {:ok, file :: map(), mime_type :: binary(), resource_path :: binary()}
-          | {:error, :is_a_directory | :file_not_found | :invalid_encoding}
+          | {:error, :is_a_directory | :file_not_found}
   defp get_file(metadata, []) do
     case Map.get(metadata, "index.html", :error) do
       :error ->
@@ -91,46 +87,56 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
 
   @spec get_file_content(file_metadata :: map(), cached? :: boolean(), resource_path :: binary()) ::
           {:ok, nil | binary(), nil | binary()}
-          | {:error, :encoding_error | :file_not_found | :invalid_encoding | any()}
-  defp get_file_content(_, true, _), do: {:ok, nil, nil}
-
+          | {:error, :file_not_found | :invalid_encoding | any()}
   defp get_file_content(
          file_metadata = %{"addresses" => address_list},
          _cached? = false,
          resource_path
        ) do
-    try do
-      file_content =
-        Enum.reduce(address_list, "", fn address, acc ->
-          {:ok, address_bin} = Base.decode16(address, case: :mixed)
+    with {:ok, file_content} <- do_get_file_content(address_list, resource_path),
+         {:ok, encoding} <- access(file_metadata, "encoding", nil) do
+      {:ok, file_content, encoding}
+    end
+  end
+
+  defp get_file_content(_, true, _), do: {:ok, nil, nil}
+  defp get_file_content(_, _, _), do: {:error, :file_not_found}
 
-          {:ok, %Transaction{data: %TransactionData{content: tx_content}}} =
-            Archethic.search_transaction(address_bin)
+  defp do_get_file_content(address_list, resource_path) do
+    # started by ArchethicWeb.Supervisor
+    cache_server = :web_hosting_cache_file
+    cache_key = {address_list, resource_path}
 
-          {:ok, decoded_content} = Jason.decode(tx_content)
+    case LRUDisk.get(cache_server, cache_key) do
+      nil ->
+        encoded_file_content =
+          Enum.reduce(address_list, "", fn address, acc ->
+            {:ok, address_bin} = Base.decode16(address, case: :mixed)
 
-          {:ok, res_content} = access(decoded_content, resource_path)
+            {:ok, %Transaction{data: %TransactionData{content: tx_content}}} =
+              Archethic.search_transaction(address_bin)
 
-          acc <> res_content
-        end)
+            {:ok, decoded_content} = Jason.decode(tx_content)
 
-      {:ok, file_content} = Base.url_decode64(file_content, padding: false)
-      {:ok, encoding} = access(file_metadata, "encoding", nil)
-      {:ok, file_content, encoding}
-    rescue
-      MatchError ->
-        {:error, :invalid_encoding}
+            {:ok, res_content} = access(decoded_content, resource_path)
 
-      ArgumentError ->
-        {:error, :encoding_error}
+            acc <> res_content
+          end)
 
-      error ->
-        {:error, error}
+        case Base.url_decode64(encoded_file_content, padding: false) do
+          {:ok, decoded_file_content} ->
+            LRUDisk.put(cache_server, cache_key, decoded_file_content)
+            {:ok, decoded_file_content}
+
+          :error ->
+            {:error, :invalid_encoding}
+        end
+
+      decoded_file_content ->
+        {:ok, decoded_file_content}
     end
   end
 
-  defp get_file_content(_, false, _), do: {:error, :file_not_found}
-
   @spec access(map(), key :: binary(), any()) :: {:error, :file_not_found} | {:ok, any()}
   defp access(map, key, default \\ :file_not_found) do
     case Map.get(map, key, default) do
diff --git a/lib/archethic_web/supervisor.ex b/lib/archethic_web/supervisor.ex
index aced1914b..bff0cf733 100644
--- a/lib/archethic_web/supervisor.ex
+++ b/lib/archethic_web/supervisor.ex
@@ -33,8 +33,8 @@ defmodule ArchethicWeb.Supervisor do
         Endpoint,
         {Absinthe.Subscription, Endpoint},
         TransactionSubscriber,
-        cache_tx(),
-        cache_file()
+        web_hosting_cache_ref_tx(),
+        web_hosting_cache_file()
       ]
       |> add_faucet_rate_limit_child()
 
@@ -59,25 +59,27 @@ defmodule ArchethicWeb.Supervisor do
     end
   end
 
-  defp cache_tx() do
+  # this is used in web_hosting_controller.ex
+  # it does not store an entire transaction, but a triplet {address, json_content, timestamp}
+  defp web_hosting_cache_ref_tx() do
     %{
-      id: :cache_tx,
+      id: :web_hosting_cache_ref_tx,
       start:
         {LRU, :start_link,
          [
-           :cache_tx,
+           :web_hosting_cache_ref_tx,
            Application.fetch_env!(:archethic_web, :tx_cache_bytes)
          ]}
     }
   end
 
-  defp cache_file() do
+  defp web_hosting_cache_file() do
     %{
-      id: :cache_file,
+      id: :web_hosting_cache_file,
       start:
         {LRUDisk, :start_link,
          [
-           :cache_file,
+           :web_hosting_cache_file,
            Application.fetch_env!(:archethic_web, :file_cache_bytes),
            Path.join(Utils.mut_dir(), "aeweb")
          ]}
diff --git a/test/archethic_web/controllers/api/web_hosting_controller_test.exs b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
index 19149e0d7..e67358ab9 100644
--- a/test/archethic_web/controllers/api/web_hosting_controller_test.exs
+++ b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
@@ -25,12 +25,12 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
   setup do
     # There is a setup in ArchethicCase that removes the mut_dir()
     # Since we need it for LRUDisk, we recreate it on every test
-    File.mkdir_p!(Path.join([Utils.mut_dir(), "aeweb", "cache_file"]))
+    File.mkdir_p!(Path.join([Utils.mut_dir(), "aeweb", "web_hosting_cache_file"]))
 
     # clear cache on every test because most tests use the same address
     # and cache is a global state
-    :ok = LRU.purge(:cache_tx)
-    :ok = LRUDisk.purge(:cache_file)
+    :ok = LRU.purge(:web_hosting_cache_ref_tx)
+    :ok = LRUDisk.purge(:web_hosting_cache_file)
 
     P2P.add_and_connect_node(%Node{
       ip: {127, 0, 0, 1},
@@ -77,7 +77,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
              address:
                <<0, 0, 34, 84, 150, 163, 128, 213, 0, 92, 182, 131, 116, 233, 184, 180, 93, 126,
                  15, 80, 90, 66, 248, 205, 97, 203, 212, 60, 54, 132, 197, 203, 172, 186>>,
-             data: %TransactionData{content: "invalid"}
+             data: %TransactionData{content: "invalid"},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
       end)
 
@@ -136,7 +139,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
           {:ok,
            %Transaction{
              address: address,
-             data: %TransactionData{content: content}
+             data: %TransactionData{content: content},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
 
         _,
@@ -148,7 +154,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
         _ ->
           {:ok,
            %Transaction{
-             data: %TransactionData{content: content2}
+             data: %TransactionData{content: content2},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
       end)
 
@@ -266,7 +275,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
           {:ok,
            %Transaction{
              address: address,
-             data: %TransactionData{content: content}
+             data: %TransactionData{content: content},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
 
         _,
@@ -278,7 +290,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
         _ ->
           {:ok,
            %Transaction{
-             data: %TransactionData{content: content2}
+             data: %TransactionData{content: content2},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
       end)
 
@@ -539,7 +554,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
           {:ok,
            %Transaction{
              address: address,
-             data: %TransactionData{content: content}
+             data: %TransactionData{content: content},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
 
         _,
@@ -551,7 +569,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
         _ ->
           {:ok,
            %Transaction{
-             data: %TransactionData{content: content2}
+             data: %TransactionData{content: content2},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
 
         _,
@@ -563,7 +584,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
         _ ->
           {:ok,
            %Transaction{
-             data: %TransactionData{content: content3}
+             data: %TransactionData{content: content3},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
       end)
 
@@ -630,7 +654,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
           {:ok,
            %Transaction{
              address: address,
-             data: %TransactionData{content: content}
+             data: %TransactionData{content: content},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
 
         _,
@@ -642,7 +669,10 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
         _ ->
           {:ok,
            %Transaction{
-             data: %TransactionData{content: content2}
+             data: %TransactionData{content: content2},
+             validation_stamp: %ValidationStamp{
+               timestamp: DateTime.utc_now()
+             }
            }}
       end)
 

From b5acfc7cc720ba401c15bb048b63ad70e8e954f7 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Wed, 4 Jan 2023 18:00:30 +0100
Subject: [PATCH 18/22] Refactor AEWebRootController

---
 .../controllers/aeweb_root_controller.ex      | 37 +------------------
 .../controllers/api/web_hosting_controller.ex |  2 +-
 .../api/web_hosting_controller/resources.ex   | 18 ++++++---
 .../api/web_hosting_controller_test.exs       |  4 +-
 4 files changed, 18 insertions(+), 43 deletions(-)

diff --git a/lib/archethic_web/controllers/aeweb_root_controller.ex b/lib/archethic_web/controllers/aeweb_root_controller.ex
index 88c87455d..c88c3764f 100644
--- a/lib/archethic_web/controllers/aeweb_root_controller.ex
+++ b/lib/archethic_web/controllers/aeweb_root_controller.ex
@@ -5,40 +5,7 @@ defmodule ArchethicWeb.AEWebRootController do
 
   use ArchethicWeb, :controller
 
-  def index(conn, params = %{"url_path" => url_path}) do
-    cache_headers = WebHostingController.get_cache_headers(conn)
-
-    case WebHostingController.get_website(params, cache_headers) do
-      {:ok, file_content, encoding, mime_type, cached?, etag} ->
-        WebHostingController.send_response(conn, file_content, encoding, mime_type, cached?, etag)
-
-      {:error, {:is_a_directory, reference_transaction}} ->
-        {:ok, listing_html, encoding, mime_type, cached?, etag} =
-          WebHostingController.DirectoryListing.list(
-            conn.request_path,
-            params,
-            reference_transaction,
-            cache_headers
-          )
-
-        WebHostingController.send_response(conn, listing_html, encoding, mime_type, cached?, etag)
-
-      {:error, :file_not_found} ->
-        # If file is not found, returning default file (url can be handled by index file)
-        case url_path do
-          [] ->
-            send_resp(conn, 404, "Not Found")
-
-          ["index.html"] ->
-            send_resp(conn, 400, "Not Found")
-
-          _path ->
-            params = Map.put(params, "url_path", ["index.html"])
-            index(conn, params)
-        end
-
-      _ ->
-        send_resp(conn, 404, "Not Found")
-    end
+  def index(conn, params) do
+    WebHostingController.web_hosting(conn, params)
   end
 end
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex
index d38096ef3..42bce6c14 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex
@@ -159,7 +159,7 @@ defmodule ArchethicWeb.API.WebHostingController do
   # Instead of returning the entire transaction,
   # we return a triplet with only the formatted data we need
   @spec get_reference_transaction(binary()) ::
-          {:ok, {binary(), map(), DateTime.t()}} | {:error, atom()}
+          {:ok, {binary(), map(), DateTime.t()}} | {:error, term()}
   defp get_reference_transaction(address) do
     # started by ArchethicWeb.Supervisor
     cache_server = :web_hosting_cache_ref_tx
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
index 7e5ddf79f..ab7970ad5 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
@@ -60,12 +60,12 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
           {:ok, file :: map(), mime_type :: binary(), resource_path :: binary()}
           | {:error, :is_a_directory | :file_not_found}
   defp get_file(metadata, []) do
-    case Map.get(metadata, "index.html", :error) do
-      :error ->
+    case Map.get(metadata, "index.html") do
+      nil ->
         {:error, :is_a_directory}
 
-      value ->
-        {:ok, value, MIME.from_path("index.html"), "index.html"}
+      file ->
+        {:ok, file, MIME.from_path("index.html"), "index.html"}
     end
   end
 
@@ -77,7 +77,15 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
         if is_a_directory?(metadata, resource_path) do
           {:error, :is_a_directory}
         else
-          {:error, :file_not_found}
+          # Handle JS History API by serving index.html instead of a 404
+          # We loose the ability to return real 404 errors
+          case Map.get(metadata, "index.html") do
+            nil ->
+              {:error, :file_not_found}
+
+            file ->
+              {:ok, file, MIME.from_path("index.html"), "index.html"}
+          end
         end
 
       file ->
diff --git a/test/archethic_web/controllers/api/web_hosting_controller_test.exs b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
index e67358ab9..44e317bf8 100644
--- a/test/archethic_web/controllers/api/web_hosting_controller_test.exs
+++ b/test/archethic_web/controllers/api/web_hosting_controller_test.exs
@@ -164,14 +164,14 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do
       :ok
     end
 
-    test "should return file does not exist", %{conn: conn} do
+    test "should return index.html on file not found (handle JS routing)", %{conn: conn} do
       conn =
         get(
           conn,
           "/api/web_hosting/0000225496a380d5005cb68374e9b8b45d7e0f505a42f8cd61cbd43c3684c5cbacba/file.html"
         )
 
-      assert "Cannot find file content" = response(conn, 404)
+      assert "<h1>Archethic</h1>" = response(conn, 200)
     end
 
     test "should return default index.html file", %{conn: conn} do

From ff8a6a2437c8f6bc39c9a7dc2f76269309a91db8 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Thu, 5 Jan 2023 11:21:45 +0100
Subject: [PATCH 19/22] ReferenceTransaction struct and also fetch the
 transaction from cache in the SNI

---
 .../controllers/api/web_hosting_controller.ex | 44 ++----------
 .../directory_listing.ex                      | 14 ++--
 .../api/web_hosting_controller/resources.ex   | 19 +++--
 lib/archethic_web/domain.ex                   | 18 ++---
 lib/archethic_web/reference_transaction.ex    | 69 +++++++++++++++++++
 5 files changed, 102 insertions(+), 62 deletions(-)
 create mode 100644 lib/archethic_web/reference_transaction.ex

diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex
index 42bce6c14..452b4f289 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex
@@ -3,18 +3,12 @@ defmodule ArchethicWeb.API.WebHostingController do
 
   use ArchethicWeb, :controller
 
-  alias Archethic.{
-    Crypto,
-    TransactionChain.Transaction,
-    TransactionChain.Transaction.ValidationStamp,
-    TransactionChain.TransactionData
-  }
-
-  alias ArchethicCache.LRU
+  alias Archethic.Crypto
 
   require Logger
 
   alias ArchethicWeb.API.WebHostingController.{Resources, DirectoryListing}
+  alias ArchethicWeb.ReferenceTransaction
 
   @spec web_hosting(Plug.Conn.t(), params :: map()) :: Plug.Conn.t()
   def web_hosting(conn, params = %{"url_path" => []}) do
@@ -78,7 +72,7 @@ defmodule ArchethicWeb.API.WebHostingController do
           | {:error, :invalid_content}
           | {:error, :file_not_found}
           | {:error, :invalid_encoding}
-          | {:error, {:is_a_directory, {binary(), map(), DateTime.t()}}}
+          | {:error, {:is_a_directory, ReferenceTransaction.t()}}
           | {:error, any()}
 
   def get_website(params = %{"address" => address}, cache_headers) do
@@ -86,8 +80,7 @@ defmodule ArchethicWeb.API.WebHostingController do
 
     with {:ok, address} <- Base.decode16(address, case: :mixed),
          true <- Crypto.valid_address?(address),
-         {:ok, last_address} <- Archethic.get_last_transaction_address(address),
-         {:ok, reference_transaction} <- get_reference_transaction(last_address),
+         {:ok, reference_transaction} <- ReferenceTransaction.fetch_last(address),
          {:ok, file_content, encoding, mime_type, cached?, etag} <-
            Resources.load(reference_transaction, url_path, cache_headers) do
       {:ok, file_content, encoding, mime_type, cached?, etag}
@@ -153,33 +146,4 @@ defmodule ArchethicWeb.API.WebHostingController do
         else: {conn, file_content}
     end
   end
-
-  # Fetch the reference transaction either from cache, or from the network.
-  #
-  # Instead of returning the entire transaction,
-  # we return a triplet with only the formatted data we need
-  @spec get_reference_transaction(binary()) ::
-          {:ok, {binary(), map(), DateTime.t()}} | {:error, term()}
-  defp get_reference_transaction(address) do
-    # started by ArchethicWeb.Supervisor
-    cache_server = :web_hosting_cache_ref_tx
-    cache_key = address
-
-    case LRU.get(cache_server, cache_key) do
-      nil ->
-        with {:ok,
-              %Transaction{
-                data: %TransactionData{content: content},
-                validation_stamp: %ValidationStamp{timestamp: timestamp}
-              }} <- Archethic.search_transaction(address),
-             {:ok, json_content} <- Jason.decode(content) do
-          reference_transaction = {address, json_content, timestamp}
-          LRU.put(cache_server, cache_key, reference_transaction)
-          {:ok, reference_transaction}
-        end
-
-      reference_transaction ->
-        {:ok, reference_transaction}
-    end
-  end
 end
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex b/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
index f32ad3707..70db9aca5 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
@@ -1,12 +1,14 @@
 defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do
   @moduledoc false
 
+  alias ArchethicWeb.ReferenceTransaction
+
   require Logger
 
   @spec list(
           request_path :: String.t(),
           params :: map(),
-          reference_transaction :: {binary(), map(), DateTime.t()},
+          reference_transaction :: ReferenceTransaction.t(),
           cached_headers :: list()
         ) ::
           {:ok, listing_html :: binary() | nil, encoding :: nil | binary(), mime_type :: binary(),
@@ -14,13 +16,17 @@ defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do
   def list(
         request_path,
         params,
-        {last_address, json_content, timestamp},
+        %ReferenceTransaction{
+          address: address,
+          timestamp: timestamp,
+          json_content: json_content
+        },
         cache_headers
       ) do
     url_path = Map.get(params, "url_path", [])
     mime_type = "text/html"
 
-    case get_cache(cache_headers, last_address, url_path) do
+    case get_cache(cache_headers, address, url_path) do
       {cached? = true, etag} ->
         {:ok, nil, nil, mime_type, cached?, etag}
 
@@ -31,7 +37,7 @@ defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do
             url_path,
             elem(get_metadata(json_content), 1),
             timestamp,
-            last_address
+            address
           )
 
         {:ok, Phoenix.View.render_to_iodata(ArchethicWeb.DirListingView, "index.html", assigns),
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
index ab7970ad5..f7ba2448b 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
@@ -3,25 +3,33 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
 
   alias Archethic.TransactionChain.{Transaction, TransactionData}
   alias ArchethicCache.LRUDisk
+  alias ArchethicWeb.ReferenceTransaction
 
   require Logger
 
-  @spec load(tx :: {binary(), map(), DateTime.t()}, url_path :: list(), cache_headers :: list()) ::
+  @spec load(
+          reference_transaction :: ReferenceTransaction.t(),
+          url_path :: list(),
+          cache_headers :: list()
+        ) ::
           {:ok, file_content :: binary() | nil, encoding :: binary() | nil, mime_type :: binary(),
            cached? :: boolean(), etag :: binary()}
           | {:error,
              :file_not_found
-             | {:is_a_directory, {binary(), map(), DateTime.t()}}
+             | {:is_a_directory, ReferenceTransaction.t()}
              | :invalid_encoding
              | any()}
   def load(
-        reference_transaction = {last_address, json_content, _timestamp},
+        reference_transaction = %ReferenceTransaction{
+          address: address,
+          json_content: json_content
+        },
         url_path,
         cache_headers
       ) do
     with {:ok, metadata, _aeweb_version} <- get_metadata(json_content),
          {:ok, file, mime_type, resource_path} <- get_file(metadata, url_path),
-         {cached?, etag} <- get_cache(cache_headers, last_address, url_path),
+         {cached?, etag} <- get_cache(cache_headers, address, url_path),
          {:ok, file_content, encoding} <- get_file_content(file, cached?, resource_path) do
       {:ok, file_content, encoding, mime_type, cached?, etag}
     else
@@ -32,8 +40,7 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
         {:error, :file_not_found}
 
       {:error, :get_metadata} ->
-        {:error,
-         "Error: Cant access metadata and aewebversion, Reftx: #{Base.encode16(last_address)}"}
+        {:error, "Error: Cant access metadata and aewebversion, Reftx: #{Base.encode16(address)}"}
 
       {:error, :is_a_directory} ->
         {:error, {:is_a_directory, reference_transaction}}
diff --git a/lib/archethic_web/domain.ex b/lib/archethic_web/domain.ex
index 89baa706d..5e7459836 100644
--- a/lib/archethic_web/domain.ex
+++ b/lib/archethic_web/domain.ex
@@ -5,9 +5,8 @@ defmodule ArchethicWeb.Domain do
 
   alias Archethic
   alias Archethic.Crypto
-  alias Archethic.TransactionChain.Transaction
-  alias Archethic.TransactionChain.TransactionData
   alias Archethic.TransactionChain.TransactionData.Ownership
+  alias ArchethicWeb.ReferenceTransaction
 
   require Logger
 
@@ -51,16 +50,11 @@ defmodule ArchethicWeb.Domain do
     with {:ok, tx_address} <- lookup_dnslink_address(domain),
          {:ok, tx_address} <- Base.decode16(tx_address, case: :mixed),
          {:ok,
-          %Transaction{
-            type: :hosting,
-            data: %TransactionData{
-              content: content,
-              ownerships: [ownership = %Ownership{secret: secret} | _]
-            }
-          }} <-
-           Archethic.get_last_transaction(tx_address),
-         {:ok, json} <- Jason.decode(content),
-         {:ok, cert_pem} <- Map.fetch(json, "sslCertificate"),
+          %ReferenceTransaction{
+            json_content: json_content,
+            ownerships: [ownership = %Ownership{secret: secret} | _]
+          }} <- ReferenceTransaction.fetch_last(tx_address),
+         {:ok, cert_pem} <- Map.fetch(json_content, "sslCertificate"),
          %{extensions: extensions} <- EasySSL.parse_pem(cert_pem),
          {:ok, san} <- Map.fetch(extensions, :subjectAltName),
          ^domain <- String.split(san, ":") |> List.last(),
diff --git a/lib/archethic_web/reference_transaction.ex b/lib/archethic_web/reference_transaction.ex
new file mode 100644
index 000000000..a27742f27
--- /dev/null
+++ b/lib/archethic_web/reference_transaction.ex
@@ -0,0 +1,69 @@
+defmodule ArchethicWeb.ReferenceTransaction do
+  @moduledoc """
+  ReferenceTransaction is a subset of a transaction
+  It is meant to be cached so we use a lighter structure
+  """
+  alias Archethic.TransactionChain.Transaction
+  alias Archethic.TransactionChain.Transaction.ValidationStamp
+  alias Archethic.TransactionChain.TransactionData
+  alias Archethic.TransactionChain.TransactionData.Ownership
+  alias ArchethicCache.LRU
+
+  @enforce_keys [:address, :json_content, :timestamp, :ownerships]
+  defstruct [:address, :json_content, :timestamp, :ownerships]
+
+  @type t() :: %__MODULE__{
+          address: binary(),
+          json_content: map(),
+          timestamp: DateTime.t(),
+          ownerships: list(Ownership.t())
+        }
+
+  @doc """
+  Fetch the reference transaction either from cache, or from the network.
+  """
+  @spec fetch(binary()) :: {:ok, t()} | {:error, term()}
+  def fetch(address) do
+    # started by ArchethicWeb.Supervisor
+    cache_server = :web_hosting_cache_ref_tx
+    cache_key = address
+
+    case LRU.get(cache_server, cache_key) do
+      nil ->
+        with {:ok, transaction} <- Archethic.search_transaction(address),
+             {:ok, reference_transaction} <- from_transaction(transaction) do
+          LRU.put(cache_server, cache_key, reference_transaction)
+          {:ok, reference_transaction}
+        end
+
+      reference_transaction ->
+        {:ok, reference_transaction}
+    end
+  end
+
+  @doc """
+  Fetch the latest reference transaction of the chain, either from cache, or from the network.
+  """
+  @spec fetch_last(binary()) :: {:ok, t()} | {:error, term()}
+  def fetch_last(address) do
+    with {:ok, last_address} <- Archethic.get_last_transaction_address(address) do
+      fetch(last_address)
+    end
+  end
+
+  defp from_transaction(%Transaction{
+         address: address,
+         data: %TransactionData{content: content, ownerships: ownerships},
+         validation_stamp: %ValidationStamp{timestamp: timestamp}
+       }) do
+    with {:ok, json_content} <- Jason.decode(content) do
+      {:ok,
+       %__MODULE__{
+         address: address,
+         json_content: json_content,
+         timestamp: timestamp,
+         ownerships: ownerships
+       }}
+    end
+  end
+end

From 8a67770c2f58b15c909e5ad10eeb0092a7c744fc Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Mon, 9 Jan 2023 12:09:19 +0100
Subject: [PATCH 20/22] add cache hit/miss metrics

---
 lib/archethic/metrics/parser.ex                      | 12 ++++++++++++
 lib/archethic/telemetry.ex                           |  8 +++++++-
 .../api/web_hosting_controller/resources.ex          |  2 ++
 lib/archethic_web/reference_transaction.ex           |  2 ++
 4 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/lib/archethic/metrics/parser.ex b/lib/archethic/metrics/parser.ex
index bb1bd260f..ae027e4ca 100755
--- a/lib/archethic/metrics/parser.ex
+++ b/lib/archethic/metrics/parser.ex
@@ -287,9 +287,21 @@ defmodule Archethic.Metrics.Parser do
 
       metric = %{type: "gauge"}, acc ->
         Map.merge(acc, map_gauge(metric))
+
+      metric = %{type: "counter"}, acc ->
+        Map.merge(acc, map_counter(metric))
     end)
   end
 
+  defp map_counter(%{name: name, metrics: metrics}) do
+    metrics =
+      Enum.reduce(metrics, 0, fn _, acc ->
+        acc + 1
+      end)
+
+    %{name => metrics}
+  end
+
   defp map_gauge(%{name: name, metrics: [metric | _]}) do
     value = Map.get(metric, :value, 0)
     %{name => value}
diff --git a/lib/archethic/telemetry.ex b/lib/archethic/telemetry.ex
index e83d9286d..1112e01f4 100644
--- a/lib/archethic/telemetry.ex
+++ b/lib/archethic/telemetry.ex
@@ -189,7 +189,13 @@ defmodule Archethic.Telemetry do
         reporter_options: [buckets: [0.01, 0.025, 0.05, 0.1, 0.3, 0.5, 0.8, 1, 1.5, 2, 5, 10]],
         measurement: :duration,
         tags: [:nb_summaries]
-      )
+      ),
+
+      # Archethic Web
+      counter("archethic_web.hosting.cache_file.hit.count"),
+      counter("archethic_web.hosting.cache_file.miss.count"),
+      counter("archethic_web.hosting.cache_ref_tx.hit.count"),
+      counter("archethic_web.hosting.cache_ref_tx.miss.count")
     ]
   end
 end
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
index f7ba2448b..a620c0d15 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
@@ -140,6 +140,7 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
 
         case Base.url_decode64(encoded_file_content, padding: false) do
           {:ok, decoded_file_content} ->
+            :telemetry.execute([:archethic_web, :hosting, :cache_file, :miss], %{count: 1})
             LRUDisk.put(cache_server, cache_key, decoded_file_content)
             {:ok, decoded_file_content}
 
@@ -148,6 +149,7 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
         end
 
       decoded_file_content ->
+        :telemetry.execute([:archethic_web, :hosting, :cache_file, :hit], %{count: 1})
         {:ok, decoded_file_content}
     end
   end
diff --git a/lib/archethic_web/reference_transaction.ex b/lib/archethic_web/reference_transaction.ex
index a27742f27..c06f2a426 100644
--- a/lib/archethic_web/reference_transaction.ex
+++ b/lib/archethic_web/reference_transaction.ex
@@ -32,11 +32,13 @@ defmodule ArchethicWeb.ReferenceTransaction do
       nil ->
         with {:ok, transaction} <- Archethic.search_transaction(address),
              {:ok, reference_transaction} <- from_transaction(transaction) do
+          :telemetry.execute([:archethic_web, :hosting, :cache_ref_tx, :miss], %{count: 1})
           LRU.put(cache_server, cache_key, reference_transaction)
           {:ok, reference_transaction}
         end
 
       reference_transaction ->
+        :telemetry.execute([:archethic_web, :hosting, :cache_ref_tx, :hit], %{count: 1})
         {:ok, reference_transaction}
     end
   end

From f89a21b127ece8de3a5b483b08297aebdb275724 Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Mon, 9 Jan 2023 14:28:59 +0100
Subject: [PATCH 21/22] fix parser

---
 lib/archethic/metrics/parser.ex | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/archethic/metrics/parser.ex b/lib/archethic/metrics/parser.ex
index ae027e4ca..da3e25b96 100755
--- a/lib/archethic/metrics/parser.ex
+++ b/lib/archethic/metrics/parser.ex
@@ -272,10 +272,16 @@ defmodule Archethic.Metrics.Parser do
       ...>       name: "vm_memory_atom",
       ...>       type: "gauge"
       ...>     },
+      ...>    %{
+      ...>      metrics: [%{value: 4}, %{value: 1}],
+      ...>      name: "cache_hit",
+      ...>      type: "counter"
+      ...>     }
       ...> ] |> Parser.reduce_metrics()
       %{
         "archethic_contract_parsing_duration" => %{count: 2, sum: 10.0},
-        "vm_memory_atom" => 1589609
+        "vm_memory_atom" => 1589609,
+        "cache_hit" => 5
       }
   """
   @spec reduce_metrics(list(metric())) ::
@@ -295,8 +301,8 @@ defmodule Archethic.Metrics.Parser do
 
   defp map_counter(%{name: name, metrics: metrics}) do
     metrics =
-      Enum.reduce(metrics, 0, fn _, acc ->
-        acc + 1
+      Enum.reduce(metrics, 0, fn %{value: value}, acc ->
+        acc + value
       end)
 
     %{name => metrics}

From 443b65d7e3a81c832b3509b722fbd8dcbb170ccc Mon Sep 17 00:00:00 2001
From: Bastien CHAMAGNE <bastien@chamagne.fr>
Date: Tue, 10 Jan 2023 09:45:06 +0100
Subject: [PATCH 22/22] move ReferenceTransaction into WebHosting

---
 .../controllers/api/web_hosting_controller.ex              | 7 +++++--
 .../api/web_hosting_controller/directory_listing.ex        | 2 +-
 .../api/web_hosting_controller}/reference_transaction.ex   | 2 +-
 .../controllers/api/web_hosting_controller/resources.ex    | 2 +-
 lib/archethic_web/domain.ex                                | 2 +-
 5 files changed, 9 insertions(+), 6 deletions(-)
 rename lib/archethic_web/{ => controllers/api/web_hosting_controller}/reference_transaction.ex (97%)

diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex
index 452b4f289..c7a193250 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex
@@ -7,8 +7,11 @@ defmodule ArchethicWeb.API.WebHostingController do
 
   require Logger
 
-  alias ArchethicWeb.API.WebHostingController.{Resources, DirectoryListing}
-  alias ArchethicWeb.ReferenceTransaction
+  alias ArchethicWeb.API.WebHostingController.{
+    Resources,
+    DirectoryListing,
+    ReferenceTransaction
+  }
 
   @spec web_hosting(Plug.Conn.t(), params :: map()) :: Plug.Conn.t()
   def web_hosting(conn, params = %{"url_path" => []}) do
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex b/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
index 70db9aca5..b3a6469ca 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/directory_listing.ex
@@ -1,7 +1,7 @@
 defmodule ArchethicWeb.API.WebHostingController.DirectoryListing do
   @moduledoc false
 
-  alias ArchethicWeb.ReferenceTransaction
+  alias ArchethicWeb.API.WebHostingController.ReferenceTransaction
 
   require Logger
 
diff --git a/lib/archethic_web/reference_transaction.ex b/lib/archethic_web/controllers/api/web_hosting_controller/reference_transaction.ex
similarity index 97%
rename from lib/archethic_web/reference_transaction.ex
rename to lib/archethic_web/controllers/api/web_hosting_controller/reference_transaction.ex
index c06f2a426..35e081384 100644
--- a/lib/archethic_web/reference_transaction.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/reference_transaction.ex
@@ -1,4 +1,4 @@
-defmodule ArchethicWeb.ReferenceTransaction do
+defmodule ArchethicWeb.API.WebHostingController.ReferenceTransaction do
   @moduledoc """
   ReferenceTransaction is a subset of a transaction
   It is meant to be cached so we use a lighter structure
diff --git a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
index a620c0d15..33716d0a2 100644
--- a/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
+++ b/lib/archethic_web/controllers/api/web_hosting_controller/resources.ex
@@ -3,7 +3,7 @@ defmodule ArchethicWeb.API.WebHostingController.Resources do
 
   alias Archethic.TransactionChain.{Transaction, TransactionData}
   alias ArchethicCache.LRUDisk
-  alias ArchethicWeb.ReferenceTransaction
+  alias ArchethicWeb.API.WebHostingController.ReferenceTransaction
 
   require Logger
 
diff --git a/lib/archethic_web/domain.ex b/lib/archethic_web/domain.ex
index 5e7459836..f6ade4e51 100644
--- a/lib/archethic_web/domain.ex
+++ b/lib/archethic_web/domain.ex
@@ -6,7 +6,7 @@ defmodule ArchethicWeb.Domain do
   alias Archethic
   alias Archethic.Crypto
   alias Archethic.TransactionChain.TransactionData.Ownership
-  alias ArchethicWeb.ReferenceTransaction
+  alias ArchethicWeb.API.WebHostingController.ReferenceTransaction
 
   require Logger