From bc9b1b28682623542626e2987c1ba86a7a21938f Mon Sep 17 00:00:00 2001 From: Neylix Date: Thu, 23 Mar 2023 16:03:48 +0100 Subject: [PATCH] Update migration workflow --- config/config.exs | 2 + lib/mix/tasks/migrate.ex | 90 +++++++++++++++---- lib/release/call_migrate_script.ex | 20 +++++ .../test/0.0.1-migration_test.exs | 9 ++ .../test/0.0.2-migration_test.exs | 9 ++ rel/config.exs | 4 +- rel/pre_start/migrate.sh | 3 - test/mix/tasks/migrate_test.exs | 86 ++++++++++++++++++ 8 files changed, 199 insertions(+), 24 deletions(-) create mode 100644 lib/release/call_migrate_script.ex create mode 100644 priv/migration_tasks/test/0.0.1-migration_test.exs create mode 100644 priv/migration_tasks/test/0.0.2-migration_test.exs delete mode 100755 rel/pre_start/migrate.sh create mode 100644 test/mix/tasks/migrate_test.exs diff --git a/config/config.exs b/config/config.exs index 8c883d538..2fa474dde 100644 --- a/config/config.exs +++ b/config/config.exs @@ -192,6 +192,8 @@ config :ex_cldr, default_backend: Archethic.Cldr, json_library: Jason +config :archethic, env: config_env() + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config("#{Mix.env()}.exs") diff --git a/lib/mix/tasks/migrate.ex b/lib/mix/tasks/migrate.ex index 657e12e08..ac6980f30 100644 --- a/lib/mix/tasks/migrate.ex +++ b/lib/mix/tasks/migrate.ex @@ -1,33 +1,85 @@ defmodule Mix.Tasks.Archethic.Migrate do @moduledoc "Handle data migration" - use Mix.Task - alias Archethic.DB.EmbeddedImpl alias Archethic.DB.EmbeddedImpl.ChainWriter - def run(_arg) do - version = - :archethic - |> Application.spec(:vsn) - |> List.to_string() - - file_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path() + @doc """ + Run migration available migration scripts since last updated version + """ + # Called by migrate.sh scripts + def run(new_version) do + migration_file_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path() - migration_done? = - if File.exists?(file_path) do - file_path |> File.read!() |> String.split(";") |> Enum.member?(version) + migrations_to_run = + if File.exists?(migration_file_path) do + read_file(migration_file_path) |> filter_migrations_to_run() else - File.write(file_path, "#{version};", [:append]) - true + # File does not exist when it's the first time the node is started + # We create the folder to write the migration file on first start + migration_file_path |> Path.dirname() |> File.mkdir_p!() + [] end - unless migration_done? do - migrate(version) + Enum.each(migrations_to_run, fn module -> + :erlang.apply(module, :run, []) + unload_module(module) + end) - File.write!(file_path, "#{version};", [:append]) - end + File.write(migration_file_path, new_version) + end + + defp filter_migrations_to_run(last_version) do + # List migration files, name must be [version]-description.exs + # Then filter version higher than the last one runned + # Eval the migration code and filter migration with function to call + get_migrations_path() + |> Enum.map(fn migration_path -> + file_name = Path.basename(migration_path) + migration_version = Regex.run(~r/[0-9\.]*(?=-)/, file_name) |> List.first() + {migration_version, migration_path} + end) + |> Enum.filter(fn {migration_version, _} -> last_version < migration_version end) + |> Enum.map(fn {_version, path} -> Code.eval_file(path) end) + |> Enum.filter(fn + {{:module, module, _, _}, _} -> + if function_exported?(module, :run, 0) do + true + else + unload_module(module) + false + end + + _ -> + false + end) + |> Enum.map(fn {{_, module, _, _}, _} -> module end) + end + + defp get_migrations_path() do + env = Application.fetch_env!(:archethic, :env) + + Application.app_dir(:archethic) + |> Path.join("priv/migration_tasks/#{env}/*") + |> Path.wildcard() + end + + defp unload_module(module) do + # Unload the module from code memory + :code.delete(module) + :code.purge(module) end - defp migrate(_), do: :ok + defp read_file(path) do + # handle old migration file format + file_content = File.read!(path) + + if String.contains?(file_content, ";") do + last_version = file_content |> String.split(";") |> Enum.reject(&(&1 == "")) |> List.last() + File.write(path, last_version) + last_version + else + file_content + end + end end diff --git a/lib/release/call_migrate_script.ex b/lib/release/call_migrate_script.ex new file mode 100644 index 000000000..7bc029493 --- /dev/null +++ b/lib/release/call_migrate_script.ex @@ -0,0 +1,20 @@ +defmodule Archethic.Release.CallMigrateScript do + @moduledoc false + + alias Mix.Tasks.Archethic.Migrate + + use Distillery.Releases.Appup.Transform + + def up(:archethic, _v1, v2, instructions, _opts), + do: add_migrate_script_call(v2, instructions) + + def up(_, _, _, instructions, _), do: instructions + + def down(_, _, _, instructions, _), do: instructions + + defp add_migrate_script_call(new_version, instructions) do + call_instruction = {:apply, {Migrate, :run, [new_version]}} + + instructions ++ [call_instruction] + end +end diff --git a/priv/migration_tasks/test/0.0.1-migration_test.exs b/priv/migration_tasks/test/0.0.1-migration_test.exs new file mode 100644 index 000000000..da4a7424d --- /dev/null +++ b/priv/migration_tasks/test/0.0.1-migration_test.exs @@ -0,0 +1,9 @@ +defmodule Migration_0_0_1 do + @moduledoc "DB.transaction_exists? used to catch it in MigrateTest mock" + + alias Archethic.DB + + def run() do + DB.transaction_exists?("0.0.1", :storage) + end +end diff --git a/priv/migration_tasks/test/0.0.2-migration_test.exs b/priv/migration_tasks/test/0.0.2-migration_test.exs new file mode 100644 index 000000000..c79aa9313 --- /dev/null +++ b/priv/migration_tasks/test/0.0.2-migration_test.exs @@ -0,0 +1,9 @@ +defmodule Migration_0_0_2 do + @moduledoc "DB.transaction_exists? used to catch it in MigrateTest mock" + + alias Archethic.DB + + def run() do + DB.transaction_exists?("0.0.2", :storage) + end +end diff --git a/rel/config.exs b/rel/config.exs index 5d1f37335..8aef01704 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -26,7 +26,6 @@ environment Mix.env() do set include_src: false set vm_args: "rel/vm.args" set pre_configure_hooks: "rel/pre_configure" - set pre_start_hooks: "rel/pre_start" set config_providers: [ {Distillery.Releases.Config.Providers.Elixir, ["${REL_DIR}/runtime_config.exs"]} @@ -58,6 +57,7 @@ release :archethic_node do ] set appup_transforms: [ - {Archethic.Release.TransformPurge, []} + {Archethic.Release.TransformPurge, []}, + {Archethic.Release.CallMigrateScript, []} ] end diff --git a/rel/pre_start/migrate.sh b/rel/pre_start/migrate.sh deleted file mode 100755 index a22d68c92..000000000 --- a/rel/pre_start/migrate.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "$@" diff --git a/test/mix/tasks/migrate_test.exs b/test/mix/tasks/migrate_test.exs new file mode 100644 index 000000000..8a0e46aea --- /dev/null +++ b/test/mix/tasks/migrate_test.exs @@ -0,0 +1,86 @@ +defmodule Mix.Tasks.Archethic.MigrateTest do + use ArchethicCase + + alias Archethic.Crypto + alias Archethic.DB.EmbeddedImpl + alias Archethic.DB.EmbeddedImpl.ChainWriter + alias Archethic.P2P + alias Archethic.P2P.Node + + alias Mix.Tasks.Archethic.Migrate + + import Mox + + describe "run/1" do + setup do + EmbeddedImpl.Supervisor.start_link() + migration_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path() + + P2P.add_and_connect_node(%Node{ + ip: {127, 0, 0, 1}, + port: 3001, + first_public_key: Crypto.first_node_public_key(), + last_public_key: Crypto.last_node_public_key(), + available?: true, + geo_patch: "AAA", + network_patch: "AAA", + authorized?: true, + authorization_date: DateTime.utc_now() + }) + + on_exit(fn -> Process.sleep(50) end) + + %{migration_path: migration_path} + end + + test "should create migration file with current version", %{ + migration_path: migration_path + } do + refute File.exists?(migration_path) + + Migrate.run("0.0.1") + + assert File.exists?(migration_path) + assert "0.0.1" = File.read!(migration_path) + end + + test "should update version number even without migration", %{ + migration_path: migration_path + } do + Migrate.run("0.0.2") + assert "0.0.2" = File.read!(migration_path) + + Migrate.run("0.0.3") + assert "0.0.3" = File.read!(migration_path) + end + + test "should run all missed upgrade", %{ + migration_path: migration_path + } do + File.write!(migration_path, "0.0.0") + + MockDB + |> expect(:transaction_exists?, fn "0.0.1", _ -> true end) + |> expect(:transaction_exists?, fn "0.0.2", _ -> true end) + + Migrate.run("0.0.3") + assert "0.0.3" = File.read!(migration_path) + end + + test "should not run migration already done", %{ + migration_path: migration_path + } do + File.write!(migration_path, "0.0.1") + + me = self() + + MockDB + |> stub(:transaction_exists?, fn version, _ -> send(me, version) end) + + Migrate.run("0.0.2") + + refute_receive "0.0.1" + assert_receive "0.0.2" + end + end +end