From 4887388c061ba7c97ef38868d98ff7a76f40b1f8 Mon Sep 17 00:00:00 2001 From: Neylix Date: Thu, 23 Mar 2023 16:03:48 +0100 Subject: [PATCH] Update migration workflow --- config/config.exs | 2 + lib/mix/tasks/migrate.ex | 101 ++++++++++++--- .../test/0.0.1-migration_test.exs | 13 ++ .../test/0.0.2-migration_test.exs | 17 +++ rel/config.exs | 2 + rel/post_upgrade/migrate.sh | 3 + rel/pre_start/migrate.sh | 2 +- rel/pre_upgrade/migrate.sh | 3 + test/mix/tasks/migrate_test.exs | 122 ++++++++++++++++++ 9 files changed, 248 insertions(+), 17 deletions(-) create mode 100644 priv/migration_tasks/test/0.0.1-migration_test.exs create mode 100644 priv/migration_tasks/test/0.0.2-migration_test.exs create mode 100644 rel/post_upgrade/migrate.sh create mode 100755 rel/pre_upgrade/migrate.sh create mode 100644 test/mix/tasks/migrate_test.exs diff --git a/config/config.exs b/config/config.exs index 3941887879..f385cc8429 100644 --- a/config/config.exs +++ b/config/config.exs @@ -190,6 +190,8 @@ config :ex_cldr, default_backend: Archethic.Cldr, json_library: Jason +config :archethic, env: config_env() + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config("#{Mix.env()}.exs") diff --git a/lib/mix/tasks/migrate.ex b/lib/mix/tasks/migrate.ex index 657e12e087..11d79b688e 100644 --- a/lib/mix/tasks/migrate.ex +++ b/lib/mix/tasks/migrate.ex @@ -6,28 +6,97 @@ defmodule Mix.Tasks.Archethic.Migrate do alias Archethic.DB.EmbeddedImpl alias Archethic.DB.EmbeddedImpl.ChainWriter - def run(_arg) do - version = - :archethic - |> Application.spec(:vsn) - |> List.to_string() + @doc """ + Pre start is triggered before the node is started. + In this case we check the last migration version done and run the migration scripts + from the last migration done to the last migration script available - file_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path() + Pre upgrade is triggered before starting a hot reload - migration_done? = - if File.exists?(file_path) do - file_path |> File.read!() |> String.split(";") |> Enum.member?(version) + Post upgrade is triggerd after a succesfull hot reload + """ + # Called by migrate.sh scripts + def run([function]) when function in ["pre_start", "pre_upgrade", "post_upgrade"] do + Application.spec(:archethic, :vsn) + |> List.to_string() + |> do_run(String.to_existing_atom(function)) + end + + # Called by MigrateTest + def run([function, version]) + when function in [:pre_start, :pre_upgrade, :post_upgrade] and is_binary(version), + do: do_run(version, function) + + def run(_), do: :ok + + defp do_run(current_version, function) do + migration_file_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path() + + migrations_to_run = + if File.exists?(migration_file_path) do + File.read!(migration_file_path) |> filter_migrations_to_run(function) else - File.write(file_path, "#{version};", [:append]) - true + # File does not exist when it's the first time the node is started + # We create the folder to write the migration file on first start + migration_file_path |> Path.dirname() |> File.mkdir_p!() + [] end - unless migration_done? do - migrate(version) + last_migration_version = + Enum.reduce(migrations_to_run, "0.0.0", fn {migration_version, module}, _ -> + :erlang.apply(module, function, []) + unload_module(module) + migration_version + end) - File.write!(file_path, "#{version};", [:append]) - end + if last_migration_version < current_version, + do: File.write(migration_file_path, current_version), + else: File.write(migration_file_path, last_migration_version) end - defp migrate(_), do: :ok + defp filter_migrations_to_run(current_version, function) do + # List migration files, name must be [version]-description.exs + # Then filter version higher than the last one runned + # Eval the migration code and filter migration with function to call + env = Application.fetch_env!(:archethic, :env) + + Application.app_dir(:archethic, "/priv/migration_tasks/#{env}/*") + |> Path.wildcard() + |> Enum.map(fn migration_path -> + file_name = Path.basename(migration_path) + migration_version = Regex.run(~r/[0-9\.]*(?=-)/, file_name) |> List.first() + {migration_version, migration_path} + end) + |> Enum.filter(fn {migration_version, _} -> + keep_migration?(current_version, migration_version, function) + end) + |> Enum.map(fn {version, path} -> {version, Code.eval_file(path)} end) + |> Enum.filter(fn + {_version, {{:module, module, _, _}, _}} -> + case module.__info__(:functions) |> Keyword.has_key?(function) do + true -> + true + + false -> + unload_module(module) + false + end + + _ -> + false + end) + |> Enum.map(fn {version, {{_, module, _, _}, _}} -> {version, module} end) + end + + defp keep_migration?(current_version, migration_version, :post_upgrade), + do: current_version == migration_version + + defp keep_migration?(current_version, migration_version, _function), + do: current_version < migration_version + + defp unload_module(module) do + # Unload the module from code memory + :code.delete(module) + :code.purge(module) + end end diff --git a/priv/migration_tasks/test/0.0.1-migration_test.exs b/priv/migration_tasks/test/0.0.1-migration_test.exs new file mode 100644 index 0000000000..1b5698fa8b --- /dev/null +++ b/priv/migration_tasks/test/0.0.1-migration_test.exs @@ -0,0 +1,13 @@ +defmodule Migration_0_0_1 do + @moduledoc "DB.transaction_exists? used to catch it in MigrateTest mock" + + alias Archethic.DB + + def pre_start() do + DB.transaction_exists?("pre_start 0.0.1", :storage) + end + + def pre_upgrade() do + raise "error" + end +end diff --git a/priv/migration_tasks/test/0.0.2-migration_test.exs b/priv/migration_tasks/test/0.0.2-migration_test.exs new file mode 100644 index 0000000000..60a13f6762 --- /dev/null +++ b/priv/migration_tasks/test/0.0.2-migration_test.exs @@ -0,0 +1,17 @@ +defmodule Migration_0_0_2 do + @moduledoc "DB.transaction_exists? used to catch it in MigrateTest mock" + + alias Archethic.DB + + def pre_start() do + DB.transaction_exists?("pre_start 0.0.2", :storage) + end + + def pre_upgrade() do + DB.transaction_exists?("pre_upgrade 0.0.2", :storage) + end + + def post_upgrade() do + DB.transaction_exists?("post_upgrade 0.0.2", :storage) + end +end diff --git a/rel/config.exs b/rel/config.exs index 5d1f373353..f6077cd562 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -27,6 +27,8 @@ environment Mix.env() do set vm_args: "rel/vm.args" set pre_configure_hooks: "rel/pre_configure" set pre_start_hooks: "rel/pre_start" + set pre_upgrade_hooks: "rel/pre_upgrade" + set post_upgrade_hooks: "rel/post_upgrade" set config_providers: [ {Distillery.Releases.Config.Providers.Elixir, ["${REL_DIR}/runtime_config.exs"]} diff --git a/rel/post_upgrade/migrate.sh b/rel/post_upgrade/migrate.sh new file mode 100644 index 0000000000..1ef5ccea35 --- /dev/null +++ b/rel/post_upgrade/migrate.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "post_upgrade" diff --git a/rel/pre_start/migrate.sh b/rel/pre_start/migrate.sh index a22d68c923..685ebee941 100755 --- a/rel/pre_start/migrate.sh +++ b/rel/pre_start/migrate.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "$@" +release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "pre_start" diff --git a/rel/pre_upgrade/migrate.sh b/rel/pre_upgrade/migrate.sh new file mode 100755 index 0000000000..4dbadee676 --- /dev/null +++ b/rel/pre_upgrade/migrate.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +release_remote_ctl eval --mfa "Mix.Tasks.Archethic.Migrate.run/1" --argv "pre_upgrade" diff --git a/test/mix/tasks/migrate_test.exs b/test/mix/tasks/migrate_test.exs new file mode 100644 index 0000000000..f0e0f5e3a9 --- /dev/null +++ b/test/mix/tasks/migrate_test.exs @@ -0,0 +1,122 @@ +defmodule Mix.Tasks.Archethic.MigrateTest do + use ArchethicCase + + alias Archethic.Crypto + alias Archethic.DB.EmbeddedImpl + alias Archethic.DB.EmbeddedImpl.ChainWriter + alias Archethic.P2P + alias Archethic.P2P.Node + + alias Mix.Tasks.Archethic.Migrate + + import Mox + + describe "run/1" do + setup do + EmbeddedImpl.Supervisor.start_link() + migration_path = EmbeddedImpl.db_path() |> ChainWriter.migration_file_path() + + P2P.add_and_connect_node(%Node{ + ip: {127, 0, 0, 1}, + port: 3001, + first_public_key: Crypto.first_node_public_key(), + last_public_key: Crypto.last_node_public_key(), + available?: true, + geo_patch: "AAA", + network_patch: "AAA", + authorized?: true, + authorization_date: DateTime.utc_now() + }) + + on_exit(fn -> Process.sleep(50) end) + + %{migration_path: migration_path} + end + + test "pre_start should create migration file with current version", %{ + migration_path: migration_path + } do + refute File.exists?(migration_path) + + Migrate.run([:pre_start, "0.0.1"]) + + assert File.exists?(migration_path) + assert "0.0.1" = File.read!(migration_path) + end + + test "pre_start should update version number even without migration", %{ + migration_path: migration_path + } do + Migrate.run([:pre_start, "0.0.2"]) + assert "0.0.2" = File.read!(migration_path) + + Migrate.run([:pre_start, "0.0.3"]) + assert "0.0.3" = File.read!(migration_path) + end + + test "pre_start should run all missed upgrade", %{ + migration_path: migration_path + } do + File.write!(migration_path, "0.0.0") + + MockDB + |> expect(:transaction_exists?, fn "pre_start 0.0.1", _ -> true end) + |> expect(:transaction_exists?, fn "pre_start 0.0.2", _ -> true end) + + Migrate.run([:pre_start, "0.0.3"]) + assert "0.0.3" = File.read!(migration_path) + end + + test "pre_start should not run migration already done", %{ + migration_path: migration_path + } do + File.write!(migration_path, "0.0.1") + + me = self() + + MockDB + |> stub(:transaction_exists?, fn version, _ -> send(me, version) end) + + Migrate.run([:pre_start, "0.0.2"]) + + refute_receive "pre_start 0.0.1" + assert_receive "pre_start 0.0.2" + end + + test "post_upgrade should run migration of the current_version if not already done", %{ + migration_path: migration_path + } do + File.write!(migration_path, "0.0.2") + + MockDB + |> expect(:transaction_exists?, fn "post_upgrade 0.0.2", _ -> true end) + + Migrate.run([:post_upgrade, "0.0.2"]) + + assert "0.0.2" = File.read!(migration_path) + end + + test "post_upgrade should not try to run function if module does not implement it", %{ + migration_path: migration_path + } do + File.write!(migration_path, "0.0.1") + Migrate.run([:post_upgrade, "0.0.1"]) + # No assert, the test should just not crash + end + + test "pre_upgrade and post_upgrade should be started", %{ + migration_path: migration_path + } do + File.write!(migration_path, "0.0.1") + + MockDB + |> expect(:transaction_exists?, fn "pre_upgrade 0.0.2", _ -> true end) + |> expect(:transaction_exists?, fn "post_upgrade 0.0.2", _ -> true end) + + Migrate.run([:pre_upgrade, "0.0.1"]) + Migrate.run([:post_upgrade, "0.0.2"]) + + assert "0.0.2" = File.read!(migration_path) + end + end +end