From 8f79ce9c209cd96e6fc67e3b0ce686f2ac3f71b0 Mon Sep 17 00:00:00 2001 From: bchamagne <74045243+bchamagne@users.noreply.github.com> Date: Wed, 30 Nov 2022 15:00:21 +0100 Subject: [PATCH] Supports directory listing for the AEWeb's explorer (#705) * AEWeb directory listing if there is no index.html * Put explorer_url in config because we need to link to it from other domains in AEWeb --- config/dev.exs | 7 + config/prod.exs | 13 + config/test.exs | 1 + .../controllers/aeweb_root_controller.ex | 13 +- .../controllers/api/web_hosting_controller.ex | 258 +++++++++++++----- .../templates/dir_listing/index.html.heex | 165 +++++++++++ lib/archethic_web/views/dir_listing_view.ex | 28 ++ .../api/web_hosting_controller_test.exs | 106 +++++++ 8 files changed, 524 insertions(+), 67 deletions(-) create mode 100644 lib/archethic_web/templates/dir_listing/index.html.heex create mode 100644 lib/archethic_web/views/dir_listing_view.ex diff --git a/config/dev.exs b/config/dev.exs index 4b170dfe5..7bd5ce8f7 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -129,6 +129,13 @@ config :archethic, ArchethicWeb.FaucetController, enabled: true # watchers to your application. For example, we use it # with webpack to recompile .js and .css sources. config :archethic, ArchethicWeb.Endpoint, + explorer_url: + URI.to_string(%URI{ + scheme: "https", + host: System.get_env("ARCHETHIC_DOMAIN_NAME", "localhost"), + port: System.get_env("ARCHETHIC_HTTPS_PORT", "5000") |> String.to_integer(), + path: "/explorer" + }), http: [port: System.get_env("ARCHETHIC_HTTP_PORT", "4000") |> String.to_integer()], server: true, debug_errors: true, diff --git a/config/prod.exs b/config/prod.exs index 9b13f4a7f..b4e79ca96 100755 --- a/config/prod.exs +++ b/config/prod.exs @@ -228,6 +228,19 @@ config :archethic, ArchethicWeb.FaucetController, # which you should run after static files are built and # before starting your production server. config :archethic, ArchethicWeb.Endpoint, + explorer_url: + URI.to_string(%URI{ + scheme: "https", + host: + case(System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet") do + true -> + System.get_env("ARCHETHIC_DOMAIN_NAME", "testnet.archethic.net") + + false -> + System.get_env("ARCHETHIC_DOMAIN_NAME", "mainnet.archethic.net") + end, + path: "/explorer" + }), http: [:inet6, port: System.get_env("ARCHETHIC_HTTP_PORT", "40000") |> String.to_integer()], url: [host: nil, port: System.get_env("ARCHETHIC_HTTP_PORT", "40000") |> String.to_integer()], cache_static_manifest: "priv/static/cache_manifest.json", diff --git a/config/test.exs b/config/test.exs index 981dc2819..9e643ee57 100755 --- a/config/test.exs +++ b/config/test.exs @@ -156,5 +156,6 @@ config :archethic, Archethic.Utils.DetectNodeResponsiveness, timeout: 1_000 # We don't run a server during test. If one is required, # you can enable the server option below. config :archethic, ArchethicWeb.Endpoint, + explorer_url: "", http: [port: 4002], server: false diff --git a/lib/archethic_web/controllers/aeweb_root_controller.ex b/lib/archethic_web/controllers/aeweb_root_controller.ex index db608fe8f..cfcce9578 100644 --- a/lib/archethic_web/controllers/aeweb_root_controller.ex +++ b/lib/archethic_web/controllers/aeweb_root_controller.ex @@ -12,14 +12,23 @@ defmodule ArchethicWeb.AEWebRootController do {:ok, file_content, encodage, mime_type, cached?, etag} -> WebHostingController.send_response(conn, file_content, encodage, mime_type, cached?, etag) - {:error, :not_found} -> + {:error, {:is_a_directory, transaction}} -> + {:ok, listing_html, encodage, mime_type, cached?, etag} = + WebHostingController.dir_listing(conn.request_path, params, transaction, cache_headers) + + WebHostingController.send_response(conn, listing_html, encodage, mime_type, cached?, etag) + + {:error, :file_not_found} -> # If file is not found, returning default file (url can be handled by index file) case url_path do [] -> send_resp(conn, 404, "Not Found") + ["index.html"] -> + send_resp(conn, 400, "Not Found") + _path -> - params = Map.put(params, "url_path", []) + params = Map.put(params, "url_path", ["index.html"]) index(conn, params) end diff --git a/lib/archethic_web/controllers/api/web_hosting_controller.ex b/lib/archethic_web/controllers/api/web_hosting_controller.ex index e1cd4c9c3..ab1dc9427 100644 --- a/lib/archethic_web/controllers/api/web_hosting_controller.ex +++ b/lib/archethic_web/controllers/api/web_hosting_controller.ex @@ -6,6 +6,7 @@ defmodule ArchethicWeb.API.WebHostingController do alias Archethic alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData alias Archethic.Crypto @@ -15,6 +16,7 @@ defmodule ArchethicWeb.API.WebHostingController do require Logger def web_hosting(conn, params = %{"url_path" => []}) do + # /web_hosting/:addr redirects to /web_hosting/:addr/ if String.last(conn.request_path) != "/" do redirect(conn, to: conn.request_path <> "/") else @@ -38,12 +40,18 @@ defmodule ArchethicWeb.API.WebHostingController do {:error, :website_not_found} -> send_resp(conn, 404, "Cannot find website content") - {:error, :not_found} -> + {:error, :file_not_found} -> send_resp(conn, 404, "Cannot find file content") {:error, :invalid_encodage} -> send_resp(conn, 400, "Invalid file encodage") + {:error, {:is_a_directory, transaction}} -> + {:ok, listing_html, encodage, mime_type, cached?, etag} = + dir_listing(conn.request_path, params, transaction, get_cache_headers(conn)) + + send_response(conn, listing_html, encodage, mime_type, cached?, etag) + {:error, _} -> send_resp(conn, 404, "Not Found") end @@ -57,7 +65,8 @@ defmodule ArchethicWeb.API.WebHostingController do cached? :: boolean(), etag :: binary()} | {:error, :invalid_address} | {:error, :invalid_content} - | {:error, :not_found} + | {:error, :file_not_found} + | {:error, :is_a_directory} | {:error, :invalid_encodage} | {:error, any()} def get_website(params = %{"address" => address}, cache_headers) do @@ -65,38 +74,167 @@ defmodule ArchethicWeb.API.WebHostingController do with {:ok, address} <- Base.decode16(address, case: :mixed), true <- Crypto.valid_address?(address), - {:ok, %Transaction{address: last_address, data: %TransactionData{content: content}}} <- - Archethic.get_last_transaction(address), - {:ok, json_content} <- Jason.decode(content), - {:ok, file, mime_type} <- get_file(json_content, url_path), - {cached?, etag} <- - get_cache(cache_headers, last_address, url_path), - {:ok, file_content, encodage} <- get_file_content(file, cached?, url_path) do - {:ok, file_content, encodage, mime_type, cached?, etag} + {:ok, + transaction = %Transaction{ + address: last_address, + data: %TransactionData{content: content} + }} <- + Archethic.get_last_transaction(address) do + with {:ok, json_content} <- Jason.decode(content), + {:ok, file, mime_type} <- get_file(json_content, url_path), + {cached?, etag} <- get_cache(cache_headers, last_address, url_path), + {:ok, file_content, encodage} <- get_file_content(file, cached?, url_path) do + {:ok, file_content, encodage, mime_type, cached?, etag} + else + :encodage_error -> + {:error, :invalid_encodage} + + :file_error -> + {:error, :file_not_found} + + {:error, %Jason.DecodeError{}} -> + {:error, :invalid_content} + + {:error, :file_not_found} -> + {:error, :file_not_found} + + {:error, :malformed} -> + # malformed file will return 404 as described in test "should return Cannot find file content" + {:error, :file_not_found} + + {:error, :is_a_directory} -> + # return the transaction so the dir_listing function do not need to do the I/O + {:error, {:is_a_directory, transaction}} + end else er when er in [:error, false] -> {:error, :invalid_address} - {:error, %Jason.DecodeError{}} -> - {:error, :invalid_content} - {:error, reason} when reason in [:transaction_not_exists, :transaction_invalid] -> {:error, :website_not_found} - {:file_not_found, _url} -> - {:error, :not_found} + error -> + error + end + end + + @spec dir_listing( + request_path :: String.t(), + params :: map(), + transaction :: Transaction.t(), + cached_headers :: list() + ) :: + {:ok, listing_html :: binary() | nil, encodage :: nil | binary(), mime_type :: binary(), + cached? :: boolean(), etag :: binary()} + def dir_listing( + request_path, + params, + %Transaction{ + address: last_address, + data: %TransactionData{content: content}, + validation_stamp: %ValidationStamp{timestamp: timestamp} + }, + cache_headers + ) do + url_path = Map.get(params, "url_path", []) + mime_type = "text/html" - :encodage_error -> - {:error, :invalid_encodage} + case get_cache(cache_headers, last_address, url_path) do + {cached? = true, etag} -> + {:ok, nil, nil, mime_type, cached?, etag} - :file_error -> - {:error, :not_found} + {cached? = false, etag} -> + case Jason.decode(content) do + {:error, err = %Jason.DecodeError{}} -> + {:error, err} - error -> - error + {:ok, json_content} -> + assigns = + do_dir_listing(request_path, url_path, json_content, timestamp, last_address) + + {:ok, + Phoenix.View.render_to_iodata(ArchethicWeb.DirListingView, "index.html", assigns), + nil, mime_type, cached?, etag} + end end end + defp do_dir_listing(request_path, url_path, json_content, timestamp, last_address) do + {json_content_subset, current_working_dir, parent_dir_href} = + case url_path do + [] -> + {json_content, "/", nil} + + _ -> + {:ok, subset} = Pathex.view(json_content, get_json_path(url_path)) + + { + subset, + Path.join(["/" | url_path]), + %{href: request_path |> Path.join("..") |> Path.expand()} + } + end + + json_content_subset + |> Enum.map(fn + {key, %{"address" => address}} -> + {:file, key, address} + + {key, _} -> + {:dir, key} + end) + # sort directory last, then DESC order (it will be accumulated in reverse order below) + |> Enum.sort(fn + {:file, a, _}, {:file, b, _} -> + a > b + + {:dir, a}, {:dir, b} -> + a > b + + {:file, _, _}, {:dir, _} -> + true + + {:dir, _}, {:file, _, _} -> + false + end) + |> Enum.reduce(%{dirs: [], files: []}, fn + {:file, name, addresses}, %{dirs: dirs_acc, files: files_acc} -> + item = %{ + href: %{href: Path.join(request_path, name)}, + last_modified: timestamp, + addresses: addresses, + name: name + } + + %{dirs: dirs_acc, files: [item | files_acc]} + + {:dir, name}, %{dirs: dirs_acc, files: files_acc} -> + # directories url end with a slash for relative url in website to work + item = %{ + href: %{href: Path.join([request_path, name]) <> "/"}, + last_modified: timestamp, + name: name + } + + %{files: files_acc, dirs: [item | dirs_acc]} + end) + |> Enum.into(%{ + cwd: current_working_dir, + parent_dir_href: parent_dir_href, + reference_transaction_href: %{ + href: + Path.join([ + Keyword.fetch!( + Application.get_env(:archethic, ArchethicWeb.Endpoint), + :explorer_url + ), + "transaction", + Base.encode16(last_address) + ]) + } + }) + end + @doc """ Return the list of headers for caching """ @@ -145,33 +283,43 @@ defmodule ArchethicWeb.API.WebHostingController do end end - # API without path returns default index.html file - # or the only file if there is only one - defp get_file(json_content, url_path) do - {json_path, url} = - case Enum.count(url_path) do - 0 -> - file_name = get_single_file_name(json_content) - json_path = path(file_name) - {json_path, file_name} - - 1 -> - file_name = Enum.at(url_path, 0) - json_path = path(file_name) - {json_path, file_name} + defp get_file(json_content, path), do: get_file(json_content, path, nil) - _ -> - json_path = get_json_path(url_path) - url = Path.join(url_path) - {json_path, url} - end + # case when we're parsing a reference tx + defp get_file(file = %{"address" => _}, [], previous_path_item) do + {:ok, file, MIME.from_path(previous_path_item)} + end - case Pathex.view(json_content, json_path) do - {:ok, file} -> - {:ok, file, MIME.from_path(url)} + # case when we're parsing a storage tx + defp get_file(file, [], previous_path_item) when is_binary(file) do + {:ok, file, MIME.from_path(previous_path_item)} + end - :error -> - {:file_not_found, url} + # case when we're on a directory + defp get_file(json_content, [], _previous_path_item) when is_map(json_content) do + case Map.get(json_content, "index.html") do + nil -> + # make sure it is a directory instead of a malformed file + if Enum.all?(Map.values(json_content), &is_map/1) do + {:error, :is_a_directory} + else + {:error, :malformed} + end + + file -> + {:ok, file, "text/html"} + end + end + + # recurse until we are on the end of path + defp get_file(json_content, [path_item | rest], _previous_path_item) do + case Map.get(json_content, path_item) do + nil -> + # + {:error, :file_not_found} + + json_content_subset -> + get_file(json_content_subset, rest, path_item) end end @@ -185,26 +333,6 @@ defmodule ArchethicWeb.API.WebHostingController do end) end - defp get_single_file_name(json_content) do - keys = Map.keys(json_content) - - case Enum.count(keys) do - 1 -> - # Control if it is a file or a folder - file_name = Enum.at(keys, 0) - file_content = Map.get(json_content, file_name) - - if !is_map(file_content) or Map.has_key?(file_content, "address") do - file_name - else - "index.html" - end - - _ -> - "index.html" - end - end - defp get_cache(cache_headers, last_address, url_path) do etag = case Enum.empty?(url_path) do diff --git a/lib/archethic_web/templates/dir_listing/index.html.heex b/lib/archethic_web/templates/dir_listing/index.html.heex new file mode 100644 index 000000000..d588d69bc --- /dev/null +++ b/lib/archethic_web/templates/dir_listing/index.html.heex @@ -0,0 +1,165 @@ + + + + Index of <%=@cwd %> + + + + +

+ + + Hosted on the Archethic public blockchain + + + + + + + + + + + + + + + + + + + + Index of <%=@cwd %> +

+ + <%= if @parent_dir_href do %> +

Up to higher level directory

+ <% else %> +

 

+ <% end %> + + + + + + + + + + + <%= for dir <- @dirs do %> + + + + + + <% end %> + <%= for file <- @files do %> + + + + + + <% end %> + + +
NameTransactionsLast Modified
+ + + + + + +
<%= dir.name %>
+ 		<%= format_date(dir.last_modified) %>
+ + + + + + +
<%= file.name %>
+ 		+ <%= for address <- prepare_addresses(file.addresses) do %> + <%= address.text %> + <% end %> + <%= format_date(file.last_modified) %>
+ + + + + diff --git a/lib/archethic_web/views/dir_listing_view.ex b/lib/archethic_web/views/dir_listing_view.ex new file mode 100644 index 000000000..749fd5736 --- /dev/null +++ b/lib/archethic_web/views/dir_listing_view.ex @@ -0,0 +1,28 @@ +defmodule ArchethicWeb.DirListingView do + @moduledoc false + use ArchethicWeb, :view + + @spec prepare_addresses(list(String.t())) :: list(map()) + def prepare_addresses(addresses) do + explorer_url = + Keyword.fetch!( + Application.get_env(:archethic, ArchethicWeb.Endpoint), + :explorer_url + ) + + addresses + |> Enum.map(fn address -> + %{ + href: %{ + href: Path.join([explorer_url, "transaction", address]) + }, + text: shorten_address(address) + } + end) + end + + @spec shorten_address(String.t()) :: String.t() + def shorten_address(address) do + String.slice(address, 4..7) + end +end diff --git a/test/archethic_web/controllers/api/web_hosting_controller_test.exs b/test/archethic_web/controllers/api/web_hosting_controller_test.exs index 7946e5255..930209dee 100644 --- a/test/archethic_web/controllers/api/web_hosting_controller_test.exs +++ b/test/archethic_web/controllers/api/web_hosting_controller_test.exs @@ -12,6 +12,7 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do alias Archethic.P2P.Message.LastTransactionAddress alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData import Mox @@ -368,6 +369,111 @@ defmodule ArchethicWeb.API.WebHostingControllerTest do end end + describe "should return a directory listing if there is no index.html file and more than 1 file" do + setup do + content = """ + { + "dir1": { + "file10.txt":{ + "encodage":"gzip", + "address":[ + "000071fbc2205f3eba39d310baf15bd89a019b0929be76b7864852cb68c9cd6502de" + ] + }, + "file11.txt":{ + "encodage":"gzip", + "address":[ + "000071fbc2205f3eba39d310baf15bd89a019b0929be76b7864852cb68c9cd6502de" + ] + } + }, + "dir2": { + "hello.txt":{ + "encodage":"gzip", + "address":[ + "000071fbc2205f3eba39d310baf15bd89a019b0929be76b7864852cb68c9cd6502de" + ] + } + }, + "dir3": { + "index.html":{ + "encodage":"gzip", + "address":[ + "000071fbc2205f3eba39d310baf15bd89a019b0929be76b7864852cb68c9cd6502de" + ] + } + }, + "file1.txt":{ + "encodage":"gzip", + "address":[ + "000071fbc2205f3eba39d310baf15bd89a019b0929be76b7864852cb68c9cd6502de" + ] + }, + "file2.txt":{ + "encodage":"gzip", + "address":[ + "000071fbc2205f3eba39d310baf15bd89a019b0929be76b7864852cb68c9cd6502de" + ] + }, + "file3.txt":{ + "encodage":"gzip", + "address":[ + "000071fbc2205f3eba39d310baf15bd89a019b0929be76b7864852cb68c9cd6502de" + ] + } + } + """ + + MockClient + |> stub(:send_message, fn + _, %GetLastTransactionAddress{address: address}, _ -> + {:ok, %LastTransactionAddress{address: address}} + + _, %GetTransaction{address: address}, _ -> + {:ok, + %Transaction{ + address: address, + data: %TransactionData{content: content}, + validation_stamp: %ValidationStamp{ + timestamp: DateTime.utc_now() + } + }} + end) + + :ok + end + + test "should return a directory listing", %{conn: conn} do + # directory listing at root + conn1 = + get( + conn, + "/api/web_hosting/0000225496a380d5005cb68374e9b8b45d7e0f505a42f8cd61cbd43c3684c5cbacba/" + ) + + # directory listing in a sub folder with trailing / + conn2 = + get( + conn, + "/api/web_hosting/0000225496a380d5005cb68374e9b8b45d7e0f505a42f8cd61cbd43c3684c5cbacba/dir1/" + ) + + # directory listing in a sub folder w/o trailing / + conn3 = + get( + conn, + "/api/web_hosting/0000225496a380d5005cb68374e9b8b45d7e0f505a42f8cd61cbd43c3684c5cbacba/dir1" + ) + + html1 = response(conn1, 200) + html2 = response(conn2, 200) + html3 = response(conn3, 200) + assert String.contains?(html1, "Index of") + assert String.contains?(html2, "Index of") + assert String.contains?(html3, "Index of") + end + end + describe "get_file_content/3 with address_content" do setup do content = """