Introduce file system abstraction and an S3 implementation

assets/css/components.css

@@ -69,6 +69,12 @@
     @apply rounded-full border-2;
   }
 
+  /* Links */
+
+  .link {
+    @apply font-medium underline text-gray-900 hover:no-underline;
+  }
+
   /* Form fields */
 
   .input {
@@ -185,10 +191,15 @@
   /* Toggleable menu */
 
   .menu {
-    @apply absolute right-0 z-30 rounded-lg bg-white flex flex-col py-2;
+    @apply absolute right-0 z-30 rounded-lg bg-white flex flex-col py-2 mt-1;
     box-shadow: 0px 15px 99px rgba(13, 24, 41, 0.15);
   }
 
+  .menu.left {
+    right: auto;
+    @apply left-0;
+  }
+
   .menu__item {
     @apply flex space-x-3 px-5 py-2 items-center hover:bg-gray-50 whitespace-nowrap;
   }

config/runtime.exs

@@ -6,8 +6,6 @@ config :livebook, LivebookWeb.Endpoint,
     Livebook.Config.secret!("LIVEBOOK_SECRET_KEY_BASE") ||
       Base.encode64(:crypto.strong_rand_bytes(48))
 
-config :livebook, :root_path, Livebook.Config.root_path!("LIVEBOOK_ROOT_PATH")
-
 if password = Livebook.Config.password!("LIVEBOOK_PASSWORD") do
   config :livebook, authentication_mode: :password, password: password
 else
@@ -32,3 +30,10 @@ config :livebook,
        :default_runtime,
        Livebook.Config.default_runtime!("LIVEBOOK_DEFAULT_RUNTIME") ||
          {Livebook.Runtime.ElixirStandalone, []}
+
+root_path =
+  Livebook.Config.root_path!("LIVEBOOK_ROOT_PATH")
+  |> Livebook.FileSystem.Utils.ensure_dir_path()
+
+local_file_system = Livebook.FileSystem.Local.new(default_path: root_path)
+config :livebook, :file_systems, [local_file_system]

lib/livebook/config.ex

@@ -1,6 +1,8 @@
 defmodule Livebook.Config do
   @moduledoc false
 
+  alias Livebook.FileSystem
+
   @type auth_mode() :: :token | :password | :disabled
 
   @doc """
@@ -33,11 +35,40 @@ defmodule Livebook.Config do
   end
 
   @doc """
-  Return the root path for persisting notebooks.
+  Returns the list of currently available file systems.
+  """
+  @spec file_systems() :: list(FileSystem.t())
+  def file_systems() do
+    Application.fetch_env!(:livebook, :file_systems)
+  end
+
+  @doc """
+  Appends a new file system to the configured ones.
+  """
+  @spec append_file_system(FileSystem.t()) :: list(FileSystem.t())
+  def append_file_system(file_system) do
+    file_systems = Enum.uniq(file_systems() ++ [file_system])
+    Application.put_env(:livebook, :file_systems, file_systems, persistent: true)
+    file_systems
+  end
+
+  @doc """
+  Removes the given file system from the configured ones.
+  """
+  @spec remove_file_system(FileSystem.t()) :: list(FileSystem.t())
+  def remove_file_system(file_system) do
+    file_systems = List.delete(file_systems(), file_system)
+    Application.put_env(:livebook, :file_systems, file_systems, persistent: true)
+    file_systems
+  end
+
+  @doc """
+  Returns the default directory.
   """
-  @spec root_path() :: binary()
-  def root_path() do
-    Application.fetch_env!(:livebook, :root_path)
+  @spec default_dir() :: FileSystem.File.t()
+  def default_dir() do
+    [file_system | _] = Livebook.Config.file_systems()
+    FileSystem.File.new(file_system)
   end
 
   ## Parsing

lib/livebook/content_loader.ex

@@ -1,6 +1,8 @@
 defmodule Livebook.ContentLoader do
   @moduledoc false
 
+  alias Livebook.Utils.HTTP
+
   @doc """
   Rewrite known URLs, so that they point to plain text file rather than HTML.
 
@@ -55,28 +57,13 @@ defmodule Livebook.ContentLoader do
 
   @doc """
   Loads binary content from the given URl and validates if its plain text.
-
-  Supports local file:// URLs and remote http(s):// URLs.
   """
   @spec fetch_content(String.t()) :: {:ok, String.t()} | {:error, String.t()}
-  def fetch_content(url)
-
-  def fetch_content("file://" <> path) do
-    case File.read(path) do
-      {:ok, content} ->
-        {:ok, content}
-
-      {:error, error} ->
-        message = :file.format_error(error)
-        {:error, "failed to read #{path}, reason: #{message}"}
-    end
-  end
-
   def fetch_content(url) do
-    case :httpc.request(:get, {url, []}, http_opts(), body_format: :binary) do
-      {:ok, {{_, 200, _}, headers, body}} ->
+    case HTTP.request(:get, url) do
+      {:ok, 200, headers, body} ->
         valid_content? =
-          case fetch_content_type(headers) do
+          case HTTP.fetch_content_type(headers) do
             {:ok, content_type} -> content_type in ["text/plain", "text/markdown"]
             :error -> false
           end
@@ -91,40 +78,4 @@ defmodule Livebook.ContentLoader do
         {:error, "failed to download notebook from the given URL"}
     end
   end
-
-  defp fetch_content_type(headers) do
-    case Enum.find(headers, fn {key, _} -> key == 'content-type' end) do
-      {_, value} ->
-        {:ok,
-         value
-         |> List.to_string()
-         |> String.split(";")
-         |> hd()}
-
-      _ ->
-        :error
-    end
-  end
-
-  crt_file = CAStore.file_path()
-  crt = File.read!(crt_file)
-  pems = :public_key.pem_decode(crt)
-  ders = Enum.map(pems, fn {:Certificate, der, _} -> der end)
-
-  # Note: we need to load the certificates at compilation time,
-  # as we don't have access to package files in Escript.
-  @cacerts ders
-
-  defp http_opts() do
-    [
-      # Use secure options, see https://gist.github.com/jonatanklosko/5e20ca84127f6b31bbe3906498e1a1d7
-      ssl: [
-        verify: :verify_peer,
-        cacerts: @cacerts,
-        customize_hostname_check: [
-          match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
-        ]
-      ]
-    ]
-  end
 end

lib/livebook/file_system.ex

@@ -0,0 +1,149 @@
+defprotocol Livebook.FileSystem do
+  @moduledoc false
+
+  # This protocol defines an interface for file systems
+  # that can be plugged into Livebook.
+
+  @typedoc """
+  A path uniquely idenfies file in the file system.
+
+  Path has most of the semantics of regular file paths,
+  with the following exceptions:
+
+    * path must be be absolute for consistency
+
+    * directory path must have a trailing slash, whereas
+      regular file path must not have a trailing slash.
+      Rationale: some file systems allow a directory and
+      a file with the same name to co-exist, while path
+      needs to distinguish between them
+  """
+  @type path :: String.t()
+
+  @typedoc """
+  A human-readable error message clarifying the operation
+  failure reason.
+  """
+  @type error :: String.t()
+
+  @type access :: :read | :write | :read_write | :none
+
+  @doc """
+  Returns the default directory path.
+
+  To some extent this is similar to current working directory
+  in a regular file system. For most file systems this
+  will just be the root path.
+  """
+  @spec default_path(t()) :: path()
+  def default_path(file_system)
+
+  @doc """
+  Returns a list of files located in the given directory.
+
+  When `recursive` is set to `true`, nested directories
+  are traversed and the final list includes all the paths.
+  """
+  @spec list(t(), path(), boolean()) :: {:ok, list(path())} | {:error, error()}
+  def list(file_system, path, recursive)
+
+  @doc """
+  Returns binary content of the given file.
+  """
+  @spec read(t(), path()) :: {:ok, binary()} | {:error, error()}
+  def read(file_system, path)
+
+  @doc """
+  Writes the given binary content to the given file.
+
+  If the file exists, it gets overridden.
+
+  If the file doesn't exist, it gets created along with
+  all the necessary directories.
+  """
+  @spec write(t(), path(), binary()) :: :ok | {:error, error()}
+  def write(file_system, path, content)
+
+  @doc """
+  Returns the current access level to the given file.
+
+  If determining the access is costly, then this function may
+  always return the most liberal access, since all access
+  functions return error on an invalid attempt.
+  """
+  @spec access(t(), path()) :: {:ok, access()} | {:error, error()}
+  def access(file_system, path)
+
+  @doc """
+  Creates the given directory unless it already exists.
+
+  All necessary parent directories are created as well.
+  """
+  @spec create_dir(t(), path()) :: :ok | {:error, error()}
+  def create_dir(file_system, path)
+
+  @doc """
+  Removes the given file.
+
+  If a directory is given, all of its contents are removed
+  recursively.
+
+  If the file doesn't exist, no error is returned.
+  """
+  @spec remove(t(), path()) :: :ok | {:error, error()}
+  def remove(file_system, path)
+
+  @doc """
+  Copies the given file.
+
+  The given files must be of the same type.
+
+  If regular files are given, the contents are copied,
+  potentially overriding the destination if it already exists.
+
+  If directories are given, the directory contents are copied
+  recursively.
+  """
+  @spec copy(t(), path(), path()) :: :ok | {:error, error()}
+  def copy(file_system, source_path, destination_path)
+
+  @doc """
+  Renames the given file.
+
+  If a directory is given, it gets renamed as expected and
+  consequently all of the child paths change.
+
+  If the destination exists, an error is returned.
+  """
+  @spec rename(t(), path(), path()) :: :ok | {:error, error()}
+  def rename(file_system, source_path, destination_path)
+
+  @doc """
+  Returns a version identifier for the given file.
+
+  The resulting value must be a string of ASCII characters
+  placed between double quotes, suitable for use as the
+  value of the ETag HTTP header.
+  """
+  @spec etag_for(t(), path()) :: {:ok, String.t()} | {:error, error()}
+  def etag_for(file_system, path)
+
+  @doc """
+  Checks if the given path exists in the file system.
+  """
+  @spec exists?(t(), path()) :: {:ok, boolean()} | {:error, error()}
+  def exists?(file_system, path)
+
+  @doc """
+  Resolves `subject` against a valid directory path.
+
+  The `subject` may be either relative or absolute,
+  contain special sequences such as ".." and ".",
+  but the interpretation is left up to the file system.
+
+  In other words, this has the semantics of path join
+  followed by expand.
+  """
+  @spec resolve_path(t(), path(), String.t()) :: path()
+  def resolve_path(file_system, dir_path, subject)
+end