Update migration workflow

config/config.exs

@@ -192,6 +192,8 @@ config :ex_cldr,
   default_backend: Archethic.Cldr,
   json_library: Jason
 
+config :archethic, env: config_env()
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config("#{Mix.env()}.exs")

lib/mix/tasks/migrate.ex

@@ -1,33 +1,85 @@
 defmodule Mix.Tasks.Archethic.Migrate do
   @moduledoc "Handle data migration"
 
-  use Mix.Task
-
   alias Archethic.DB.EmbeddedImpl
   alias Archethic.DB.EmbeddedImpl.ChainWriter
 
-  def run(_arg) do
-    version =
-      :archethic
-      |> Application.spec(:vsn)
-      |> List.to_string()
-
-    file_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path()
+  @doc """
+  Run migration available migration scripts since last updated version
+  """
+  # Called by migrate.sh scripts
+  def run(new_version) do
+    migration_file_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path()
 
-    migration_done? =
-      if File.exists?(file_path) do
-        file_path |> File.read!() |> String.split(";") |> Enum.member?(version)
+    migrations_to_run =
+      if File.exists?(migration_file_path) do
+        read_file(migration_file_path) |> filter_migrations_to_run()
       else
-        File.write(file_path, "#{version};", [:append])
-        true
+        # File does not exist when it's the first time the node is started
+        # We create the folder to write the migration file on first start
+        migration_file_path |> Path.dirname() |> File.mkdir_p!()
+        []
       end
 
-    unless migration_done? do
-      migrate(version)
+    Enum.each(migrations_to_run, fn module ->
+      :erlang.apply(module, :run, [])
+      unload_module(module)
+    end)
 
-      File.write!(file_path, "#{version};", [:append])
-    end
+    File.write(migration_file_path, new_version)
+  end
+
+  defp filter_migrations_to_run(last_version) do
+    # List migration files, name must be [version]-description.exs
+    # Then filter version higher than the last one runned
+    # Eval the migration code and filter migration with function to call
+    get_migrations_path()
+    |> Enum.map(fn migration_path ->
+      file_name = Path.basename(migration_path)
+      migration_version = Regex.run(~r/[0-9\.]*(?=-)/, file_name) |> List.first()
+      {migration_version, migration_path}
+    end)
+    |> Enum.filter(fn {migration_version, _} -> last_version < migration_version end)
+    |> Enum.map(fn {_version, path} -> Code.eval_file(path) end)
+    |> Enum.filter(fn
+      {{:module, module, _, _}, _} ->
+        if function_exported?(module, :run, 0) do
+          true
+        else
+          unload_module(module)
+          false
+        end
+
+      _ ->
+        false
+    end)
+    |> Enum.map(fn {{_, module, _, _}, _} -> module end)
+  end
+
+  defp get_migrations_path() do
+    env = Application.fetch_env!(:archethic, :env)
+
+    Application.app_dir(:archethic)
+    |> Path.join("priv/migration_tasks/#{env}/*")
+    |> Path.wildcard()
+  end
+
+  defp unload_module(module) do
+    # Unload the module from code memory
+    :code.delete(module)
+    :code.purge(module)
   end
 
-  defp migrate(_), do: :ok
+  defp read_file(path) do
+    # handle old migration file format
+    file_content = File.read!(path)
+
+    if String.contains?(file_content, ";") do
+      last_version = file_content |> String.split(";") |> Enum.reject(&(&1 == "")) |> List.last()
+      File.write(path, last_version)
+      last_version
+    else
+      file_content
+    end
+  end
 end

lib/release/call_migrate_script.ex

@@ -0,0 +1,20 @@
+defmodule Archethic.Release.CallMigrateScript do
+  @moduledoc false
+
+  alias Mix.Tasks.Archethic.Migrate
+
+  use Distillery.Releases.Appup.Transform
+
+  def up(:archethic, _v1, v2, instructions, _opts),
+    do: add_migrate_script_call(v2, instructions)
+
+  def up(_, _, _, instructions, _), do: instructions
+
+  def down(_, _, _, instructions, _), do: instructions
+
+  defp add_migrate_script_call(new_version, instructions) do
+    call_instruction = {:apply, {Migrate, :run, [new_version]}}
+
+    instructions ++ [call_instruction]
+  end
+end

priv/migration_tasks/test/0.0.1-migration_test.exs

@@ -0,0 +1,9 @@
+defmodule Migration_0_0_1 do
+  @moduledoc "DB.transaction_exists? used to catch it in MigrateTest mock"
+
+  alias Archethic.DB
+
+  def run() do
+    DB.transaction_exists?("0.0.1", :storage)
+  end
+end

priv/migration_tasks/test/0.0.2-migration_test.exs

@@ -0,0 +1,9 @@
+defmodule Migration_0_0_2 do
+  @moduledoc "DB.transaction_exists? used to catch it in MigrateTest mock"
+
+  alias Archethic.DB
+
+  def run() do
+    DB.transaction_exists?("0.0.2", :storage)
+  end
+end

rel/config.exs

@@ -26,7 +26,6 @@ environment Mix.env() do
   set include_src: false
   set vm_args: "rel/vm.args"
   set pre_configure_hooks: "rel/pre_configure"
-  set pre_start_hooks: "rel/pre_start"
 
   set config_providers: [
     {Distillery.Releases.Config.Providers.Elixir, ["${REL_DIR}/runtime_config.exs"]}
@@ -58,6 +57,7 @@ release :archethic_node do
   ]
 
   set appup_transforms: [
-    {Archethic.Release.TransformPurge, []}
+    {Archethic.Release.TransformPurge, []},
+    {Archethic.Release.CallMigrateScript, []}
   ]
 end

test/mix/tasks/migrate_test.exs

@@ -0,0 +1,86 @@
+defmodule Mix.Tasks.Archethic.MigrateTest do
+  use ArchethicCase
+
+  alias Archethic.Crypto
+  alias Archethic.DB.EmbeddedImpl
+  alias Archethic.DB.EmbeddedImpl.ChainWriter
+  alias Archethic.P2P
+  alias Archethic.P2P.Node
+
+  alias Mix.Tasks.Archethic.Migrate
+
+  import Mox
+
+  describe "run/1" do
+    setup do
+      EmbeddedImpl.Supervisor.start_link()
+      migration_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path()
+
+      P2P.add_and_connect_node(%Node{
+        ip: {127, 0, 0, 1},
+        port: 3001,
+        first_public_key: Crypto.first_node_public_key(),
+        last_public_key: Crypto.last_node_public_key(),
+        available?: true,
+        geo_patch: "AAA",
+        network_patch: "AAA",
+        authorized?: true,
+        authorization_date: DateTime.utc_now()
+      })
+
+      on_exit(fn -> Process.sleep(50) end)
+
+      %{migration_path: migration_path}
+    end
+
+    test "should create migration file with current version", %{
+      migration_path: migration_path
+    } do
+      refute File.exists?(migration_path)
+
+      Migrate.run("0.0.1")
+
+      assert File.exists?(migration_path)
+      assert "0.0.1" = File.read!(migration_path)
+    end
+
+    test "should update version number even without migration", %{
+      migration_path: migration_path
+    } do
+      Migrate.run("0.0.2")
+      assert "0.0.2" = File.read!(migration_path)
+
+      Migrate.run("0.0.3")
+      assert "0.0.3" = File.read!(migration_path)
+    end
+
+    test "should run all missed upgrade", %{
+      migration_path: migration_path
+    } do
+      File.write!(migration_path, "0.0.0")
+
+      MockDB
+      |> expect(:transaction_exists?, fn "0.0.1", _ -> true end)
+      |> expect(:transaction_exists?, fn "0.0.2", _ -> true end)
+
+      Migrate.run("0.0.3")
+      assert "0.0.3" = File.read!(migration_path)
+    end
+
+    test "should not run migration already done", %{
+      migration_path: migration_path
+    } do
+      File.write!(migration_path, "0.0.1")
+
+      me = self()
+
+      MockDB
+      |> stub(:transaction_exists?, fn version, _ -> send(me, version) end)
+
+      Migrate.run("0.0.2")
+
+      refute_receive "0.0.1"
+      assert_receive "0.0.2"
+    end
+  end
+end