Update migration workflow

config/config.exs

@@ -190,6 +190,8 @@ config :ex_cldr,
   default_backend: Archethic.Cldr,
   json_library: Jason
 
+config :archethic, env: config_env()
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config("#{Mix.env()}.exs")

lib/mix/tasks/migrate.ex

@@ -6,28 +6,97 @@ defmodule Mix.Tasks.Archethic.Migrate do
   alias Archethic.DB.EmbeddedImpl
   alias Archethic.DB.EmbeddedImpl.ChainWriter
 
-  def run(_arg) do
-    version =
-      :archethic
-      |> Application.spec(:vsn)
-      |> List.to_string()
+  @doc """
+  Pre start is triggered before the node is started.
+  In this case we check the last migration version done and run the migration scripts
+  from the last migration done to the last migration script available
 
-    file_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path()
+  Pre upgrade is triggered before starting a hot reload
 
-    migration_done? =
-      if File.exists?(file_path) do
-        file_path |> File.read!() |> String.split(";") |> Enum.member?(version)
+  Post upgrade is triggerd after a succesfull hot reload
+  """
+  # Called by migrate.sh scripts
+  def run([function]) when function in ["pre_start", "pre_upgrade", "post_upgrade"] do
+    Application.spec(:archethic, :vsn)
+    |> List.to_string()
+    |> do_run(String.to_existing_atom(function))
+  end
+
+  # Called by MigrateTest
+  def run([function, version])
+      when function in [:pre_start, :pre_upgrade, :post_upgrade] and is_binary(version),
+      do: do_run(version, function)
+
+  def run(_), do: :ok
+
+  defp do_run(current_version, function) do
+    migration_file_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path()
+
+    migrations_to_run =
+      if File.exists?(migration_file_path) do
+        File.read!(migration_file_path) |> filter_migrations_to_run(function)
       else
-        File.write(file_path, "#{version};", [:append])
-        true
+        # File does not exist when it's the first time the node is started
+        # We create the folder to write the migration file on first start
+        migration_file_path |> Path.dirname() |> File.mkdir_p!()
+        []
       end
 
-    unless migration_done? do
-      migrate(version)
+    last_migration_version =
+      Enum.reduce(migrations_to_run, "0.0.0", fn {migration_version, module}, _ ->
+        :erlang.apply(module, function, [])
+        unload_module(module)
+        migration_version
+      end)
 
-      File.write!(file_path, "#{version};", [:append])
-    end
+    if last_migration_version < current_version,
+      do: File.write(migration_file_path, current_version),
+      else: File.write(migration_file_path, last_migration_version)
   end
 
-  defp migrate(_), do: :ok
+  defp filter_migrations_to_run(current_version, function) do
+    # List migration files, name must be [version]-description.exs
+    # Then filter version higher than the last one runned
+    # Eval the migration code and filter migration with function to call
+    env = Application.fetch_env!(:archethic, :env)
+
+    Application.app_dir(:archethic, "/priv/migration_tasks/#{env}/*")
+    |> Path.wildcard()
+    |> Enum.map(fn migration_path ->
+      file_name = Path.basename(migration_path)
+      migration_version = Regex.run(~r/[0-9\.]*(?=-)/, file_name) |> List.first()
+      {migration_version, migration_path}
+    end)
+    |> Enum.filter(fn {migration_version, _} ->
+      keep_migration?(current_version, migration_version, function)
+    end)
+    |> Enum.map(fn {version, path} -> {version, Code.eval_file(path)} end)
+    |> Enum.filter(fn
+      {_version, {{:module, module, _, _}, _}} ->
+        case module.__info__(:functions) |> Keyword.has_key?(function) do
+          true ->
+            true
+
+          false ->
+            unload_module(module)
+            false
+        end
+
+      _ ->
+        false
+    end)
+    |> Enum.map(fn {version, {{_, module, _, _}, _}} -> {version, module} end)
+  end
+
+  defp keep_migration?(current_version, migration_version, :post_upgrade),
+    do: current_version == migration_version
+
+  defp keep_migration?(current_version, migration_version, _function),
+    do: current_version < migration_version
+
+  defp unload_module(module) do
+    # Unload the module from code memory
+    :code.delete(module)
+    :code.purge(module)
+  end
 end

priv/migration_tasks/test/0.0.1-migration_test.exs

@@ -0,0 +1,13 @@
+defmodule Migration_0_0_1 do
+  @moduledoc "DB.transaction_exists? used to catch it in MigrateTest mock"
+
+  alias Archethic.DB
+
+  def pre_start() do
+    DB.transaction_exists?("pre_start 0.0.1", :storage)
+  end
+
+  def pre_upgrade() do
+    raise "error"
+  end
+end

priv/migration_tasks/test/0.0.2-migration_test.exs

@@ -0,0 +1,17 @@
+defmodule Migration_0_0_2 do
+  @moduledoc "DB.transaction_exists? used to catch it in MigrateTest mock"
+
+  alias Archethic.DB
+
+  def pre_start() do
+    DB.transaction_exists?("pre_start 0.0.2", :storage)
+  end
+
+  def pre_upgrade() do
+    DB.transaction_exists?("pre_upgrade 0.0.2", :storage)
+  end
+
+  def post_upgrade() do
+    DB.transaction_exists?("post_upgrade 0.0.2", :storage)
+  end
+end

rel/config.exs

@@ -27,6 +27,8 @@ environment Mix.env() do
   set vm_args: "rel/vm.args"
   set pre_configure_hooks: "rel/pre_configure"
   set pre_start_hooks: "rel/pre_start"
+  set pre_upgrade_hooks: "rel/pre_upgrade"
+  set post_upgrade_hooks: "rel/post_upgrade"
 
   set config_providers: [
     {Distillery.Releases.Config.Providers.Elixir, ["${REL_DIR}/runtime_config.exs"]}

rel/post_upgrade/migrate.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "post_upgrade"

rel/pre_start/migrate.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "$@"
+release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "pre_start"

rel/pre_upgrade/migrate.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "pre_upgrade"

test/mix/tasks/migrate_test.exs

@@ -0,0 +1,122 @@
+defmodule Mix.Tasks.Archethic.MigrateTest do
+  use ArchethicCase
+
+  alias Archethic.Crypto
+  alias Archethic.DB.EmbeddedImpl
+  alias Archethic.DB.EmbeddedImpl.ChainWriter
+  alias Archethic.P2P
+  alias Archethic.P2P.Node
+
+  alias Mix.Tasks.Archethic.Migrate
+
+  import Mox
+
+  describe "run/1" do
+    setup do
+      EmbeddedImpl.Supervisor.start_link()
+      migration_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path()
+
+      P2P.add_and_connect_node(%Node{
+        ip: {127, 0, 0, 1},
+        port: 3001,
+        first_public_key: Crypto.first_node_public_key(),
+        last_public_key: Crypto.last_node_public_key(),
+        available?: true,
+        geo_patch: "AAA",
+        network_patch: "AAA",
+        authorized?: true,
+        authorization_date: DateTime.utc_now()
+      })
+
+      on_exit(fn -> Process.sleep(50) end)
+
+      %{migration_path: migration_path}
+    end
+
+    test "pre_start should create migration file with current version", %{
+      migration_path: migration_path
+    } do
+      refute File.exists?(migration_path)
+
+      Migrate.run([:pre_start, "0.0.1"])
+
+      assert File.exists?(migration_path)
+      assert "0.0.1" = File.read!(migration_path)
+    end
+
+    test "pre_start should update version number even without migration", %{
+      migration_path: migration_path
+    } do
+      Migrate.run([:pre_start, "0.0.2"])
+      assert "0.0.2" = File.read!(migration_path)
+
+      Migrate.run([:pre_start, "0.0.3"])
+      assert "0.0.3" = File.read!(migration_path)
+    end
+
+    test "pre_start should run all missed upgrade", %{
+      migration_path: migration_path
+    } do
+      File.write!(migration_path, "0.0.0")
+
+      MockDB
+      |> expect(:transaction_exists?, fn "pre_start 0.0.1", _ -> true end)
+      |> expect(:transaction_exists?, fn "pre_start 0.0.2", _ -> true end)
+
+      Migrate.run([:pre_start, "0.0.3"])
+      assert "0.0.3" = File.read!(migration_path)
+    end
+
+    test "pre_start should not run migration already done", %{
+      migration_path: migration_path
+    } do
+      File.write!(migration_path, "0.0.1")
+
+      me = self()
+
+      MockDB
+      |> stub(:transaction_exists?, fn version, _ -> send(me, version) end)
+
+      Migrate.run([:pre_start, "0.0.2"])
+
+      refute_receive "pre_start 0.0.1"
+      assert_receive "pre_start 0.0.2"
+    end
+
+    test "post_upgrade should run migration of the current_version if not already done", %{
+      migration_path: migration_path
+    } do
+      File.write!(migration_path, "0.0.2")
+
+      MockDB
+      |> expect(:transaction_exists?, fn "post_upgrade 0.0.2", _ -> true end)
+
+      Migrate.run([:post_upgrade, "0.0.2"])
+
+      assert "0.0.2" = File.read!(migration_path)
+    end
+
+    test "post_upgrade should not try to run function if module does not implement it", %{
+      migration_path: migration_path
+    } do
+      File.write!(migration_path, "0.0.1")
+      Migrate.run([:post_upgrade, "0.0.1"])
+      # No assert, the test should just not crash
+    end
+
+    test "pre_upgrade and post_upgrade should be started", %{
+      migration_path: migration_path
+    } do
+      File.write!(migration_path, "0.0.1")
+
+      MockDB
+      |> expect(:transaction_exists?, fn "pre_upgrade 0.0.2", _ -> true end)
+      |> expect(:transaction_exists?, fn "post_upgrade 0.0.2", _ -> true end)
+
+      Migrate.run([:pre_upgrade, "0.0.1"])
+      Migrate.run([:post_upgrade, "0.0.2"])
+
+      assert "0.0.2" = File.read!(migration_path)
+    end
+  end
+end