From 0dc1f204ed9f8c51bb096b13812da97ebc2b89ba Mon Sep 17 00:00:00 2001 From: transhapHigsn Date: Tue, 1 Dec 2020 22:13:00 +0530 Subject: [PATCH 1/3] chore: create migration file name with pattern timestamp_table.json and create table.version.json to track latest file Signed-off-by: transhapHigsn --- .../migration_generator.ex | 37 +++++++++++++++++-- test/migration_generator_test.exs | 15 ++++++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/lib/migration_generator/migration_generator.ex b/lib/migration_generator/migration_generator.ex index 07270f2c..fef962ce 100644 --- a/lib/migration_generator/migration_generator.ex +++ b/lib/migration_generator/migration_generator.ex @@ -309,20 +309,36 @@ defmodule AshPostgres.MigrationGenerator do Enum.each(snapshots, fn snapshot -> snapshot_binary = snapshot_to_binary(snapshot) - snapshot_file = + snapshot_table = "#{timestamp()}_#{snapshot.table}" + + snapshot_folder = if tenant? do opts.snapshot_path |> Path.join(repo_name) |> Path.join("tenants") - |> Path.join(snapshot.table <> ".json") else opts.snapshot_path |> Path.join(repo_name) - |> Path.join(snapshot.table <> ".json") end + snapshot_file = + snapshot_folder + |> Path.join(snapshot_table <> ".json") + File.mkdir_p(Path.dirname(snapshot_file)) File.write!(snapshot_file, snapshot_binary, []) + + # create a new version file to track latest migration file + version_file = + snapshot_folder + |> Path.join(snapshot.table <> ".version.json") + + version_binary = snapshot_to_binary(%{ + latest_version: snapshot_file + }) + File.mkdir_p(Path.dirname(version_file)) + File.write!(version_file, version_binary, []) + end) end @@ -937,7 +953,20 @@ defmodule AshPostgres.MigrationGenerator do Path.join(opts.snapshot_path, repo_name) end - file = Path.join(folder, snapshot.table <> ".json") + # get name of latest version file. + version_file = Path.join(folder, snapshot.table <> ".version.json") + file = + if File.exists?(version_file) do + file_content = + version_file + |> File.read!() + |> Jason.decode!(keys: :atoms!) + + file_content.latest_version + else + version_file = Path.join(folder, snapshot.table <> ".json") + version_file + end if File.exists?(file) do file diff --git a/test/migration_generator_test.exs b/test/migration_generator_test.exs index 68700358..af5a567e 100644 --- a/test/migration_generator_test.exs +++ b/test/migration_generator_test.exs @@ -79,16 +79,25 @@ defmodule AshPostgres.MigrationGeneratorTest do end test "it creates a snapshot for each resource" do - assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.json"])) + assert File.exists?(Path.wildcard("test_snapshots_path/test_repo/*_posts.json")) + assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) end test "the snapshots can be loaded" do - assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.json"])) + assert File.exists?(Path.wildcard("test_snapshots_path/test_repo/*_posts.json")) + assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) end test "the snapshots contain valid json" do - assert File.read!(Path.join(["test_snapshots_path", "test_repo", "posts.json"])) + assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) + assert File.read!(Path.wildcard("test_snapshots_path/test_repo/*_posts.json")) |> Jason.decode!(keys: :atoms!) + + version_file_content = + File.read!(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) + |> Jason.decode!(keys: :atoms!) + + assert Map.get(version_file_content, :latest_version, nil) != nil end test "the migration creates the table" do From 19b3ccf2ea573e171b653e7a3d1a57c4561dbe00 Mon Sep 17 00:00:00 2001 From: transhapHigsn Date: Tue, 1 Dec 2020 22:21:17 +0530 Subject: [PATCH 2/3] chore: format changes Signed-off-by: transhapHigsn --- lib/migration_generator/migration_generator.ex | 10 ++++++---- test/migration_generator_test.exs | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/migration_generator/migration_generator.ex b/lib/migration_generator/migration_generator.ex index fef962ce..ed0e064f 100644 --- a/lib/migration_generator/migration_generator.ex +++ b/lib/migration_generator/migration_generator.ex @@ -333,12 +333,13 @@ defmodule AshPostgres.MigrationGenerator do snapshot_folder |> Path.join(snapshot.table <> ".version.json") - version_binary = snapshot_to_binary(%{ - latest_version: snapshot_file - }) + version_binary = + snapshot_to_binary(%{ + latest_version: snapshot_file + }) + File.mkdir_p(Path.dirname(version_file)) File.write!(version_file, version_binary, []) - end) end @@ -955,6 +956,7 @@ defmodule AshPostgres.MigrationGenerator do # get name of latest version file. version_file = Path.join(folder, snapshot.table <> ".version.json") + file = if File.exists?(version_file) do file_content = diff --git a/test/migration_generator_test.exs b/test/migration_generator_test.exs index af5a567e..63e18e47 100644 --- a/test/migration_generator_test.exs +++ b/test/migration_generator_test.exs @@ -90,6 +90,7 @@ defmodule AshPostgres.MigrationGeneratorTest do test "the snapshots contain valid json" do assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) + assert File.read!(Path.wildcard("test_snapshots_path/test_repo/*_posts.json")) |> Jason.decode!(keys: :atoms!) From 01f60a52e94d54cc9f12dfe488d8b17f1783b04f Mon Sep 17 00:00:00 2001 From: Zach Daniel Date: Thu, 7 Jan 2021 00:34:31 -0500 Subject: [PATCH 3/3] chore: rework how multi-snapshots work --- .../migration_generator.ex | 160 +++++++++++------- .../tasks/ash_postgres.generate_migrations.ex | 7 + test/migration_generator_test.exs | 16 +- 3 files changed, 111 insertions(+), 72 deletions(-) diff --git a/lib/migration_generator/migration_generator.ex b/lib/migration_generator/migration_generator.ex index ed0e064f..72863aeb 100644 --- a/lib/migration_generator/migration_generator.ex +++ b/lib/migration_generator/migration_generator.ex @@ -74,7 +74,7 @@ defmodule AshPostgres.MigrationGenerator do |> group_into_phases() |> comment_out_phases() |> build_up_and_down() - |> write_migration(snapshots, repo, opts, tenant?) + |> write_migration!(snapshots, repo, opts, tenant?) end end) end @@ -302,47 +302,9 @@ defmodule AshPostgres.MigrationGenerator do |> Enum.sort() end - defp write_migration({up, down}, snapshots, repo, opts, tenant?) do + defp write_migration!({up, down}, snapshots, repo, opts, tenant?) do repo_name = repo |> Module.split() |> List.last() |> Macro.underscore() - unless opts.dry_run do - Enum.each(snapshots, fn snapshot -> - snapshot_binary = snapshot_to_binary(snapshot) - - snapshot_table = "#{timestamp()}_#{snapshot.table}" - - snapshot_folder = - if tenant? do - opts.snapshot_path - |> Path.join(repo_name) - |> Path.join("tenants") - else - opts.snapshot_path - |> Path.join(repo_name) - end - - snapshot_file = - snapshot_folder - |> Path.join(snapshot_table <> ".json") - - File.mkdir_p(Path.dirname(snapshot_file)) - File.write!(snapshot_file, snapshot_binary, []) - - # create a new version file to track latest migration file - version_file = - snapshot_folder - |> Path.join(snapshot.table <> ".version.json") - - version_binary = - snapshot_to_binary(%{ - latest_version: snapshot_file - }) - - File.mkdir_p(Path.dirname(version_file)) - File.write!(version_file, version_binary, []) - end) - end - migration_path = if tenant? do if opts.tenant_migration_path do @@ -438,10 +400,73 @@ defmodule AshPostgres.MigrationGenerator do end """ - if opts.dry_run do - Mix.shell().info(format(contents, opts)) - else - create_file(migration_file, format(contents, opts)) + try do + contents = format(contents, opts) + + create_new_snapshot(snapshots, repo_name, opts, tenant?) + + if opts.dry_run do + Mix.shell().info(contents) + else + create_file(migration_file, contents) + end + rescue + exception -> + reraise( + """ + Exception while formatting generated code: + #{Exception.format(:error, exception, __STACKTRACE__)} + + Code: + + #{add_line_numbers(contents)} + + To generate it unformatted anyway, but manually fix it, use the `--no-format` option. + """, + __STACKTRACE__ + ) + end + end + + defp add_line_numbers(contents) do + lines = String.split(contents, "\n") + + digits = String.length(to_string(Enum.count(lines))) + + lines + |> Enum.with_index() + |> Enum.map_join("\n", fn {line, index} -> + "#{String.pad_trailing(to_string(index), digits, " ")} | #{line}" + end) + end + + defp create_new_snapshot(snapshots, repo_name, opts, tenant?) do + unless opts.dry_run do + Enum.each(snapshots, fn snapshot -> + snapshot_binary = snapshot_to_binary(snapshot) + + snapshot_folder = + if tenant? do + opts.snapshot_path + |> Path.join(repo_name) + |> Path.join("tenants") + else + opts.snapshot_path + |> Path.join(repo_name) + end + + snapshot_file = Path.join(snapshot_folder, "#{snapshot.table}/#{timestamp()}.json") + + File.mkdir_p(Path.dirname(snapshot_file)) + File.write!(snapshot_file, snapshot_binary, []) + + old_snapshot_folder = Path.join(snapshot_folder, "#{snapshot.table}.json") + + if File.exists?(old_snapshot_folder) do + new_snapshot_folder = Path.join(snapshot_folder, "#{snapshot.table}/initial.json") + File.rename(old_snapshot_folder, new_snapshot_folder) + end + end) end end @@ -951,27 +976,42 @@ defmodule AshPostgres.MigrationGenerator do |> Path.join(repo_name) |> Path.join("tenants") else - Path.join(opts.snapshot_path, repo_name) + opts.snapshot_path + |> Path.join(repo_name) end - # get name of latest version file. - version_file = Path.join(folder, snapshot.table <> ".version.json") + snapshot_folder = Path.join(folder, snapshot.table) - file = - if File.exists?(version_file) do - file_content = - version_file - |> File.read!() - |> Jason.decode!(keys: :atoms!) + if File.exists?(snapshot_folder) do + snapshot_folder + |> File.ls!() + |> Enum.filter(&String.ends_with?(&1, ".json")) + |> Enum.map(&String.trim_trailing(&1, ".json")) + |> Enum.map(&Integer.parse/1) + |> Enum.filter(fn {_int, remaining} -> remaining == "" end) + |> Enum.map(&elem(&1, 0)) + |> case do + [] -> + get_old_snapshot(folder, snapshot) - file_content.latest_version - else - version_file = Path.join(folder, snapshot.table <> ".json") - version_file + timestamps -> + timestamp = Enum.max(timestamps) + snapshot_file = Path.join(snapshot_folder, "#{timestamp}.json") + + snapshot_file + |> File.read!() + |> load_snapshot() end + else + get_old_snapshot(folder, snapshot) + end + end - if File.exists?(file) do - file + defp get_old_snapshot(folder, snapshot) do + old_snapshot_file = Path.join(folder, "#{snapshot.table}.json") + # This is adapter code for the old version, where migrations were stored in a flat directory + if File.exists?(old_snapshot_file) do + old_snapshot_file |> File.read!() |> load_snapshot() end @@ -1206,6 +1246,8 @@ defmodule AshPostgres.MigrationGenerator do attribute |> Map.update!(:type, &String.to_atom/1) |> Map.update!(:name, &String.to_atom/1) + |> Map.put_new(:default, "nil") + |> Map.update!(:default, &(&1 || "nil")) |> Map.update!(:references, fn nil -> nil diff --git a/lib/mix/tasks/ash_postgres.generate_migrations.ex b/lib/mix/tasks/ash_postgres.generate_migrations.ex index 7ea15ffa..6d2e2480 100644 --- a/lib/mix/tasks/ash_postgres.generate_migrations.ex +++ b/lib/mix/tasks/ash_postgres.generate_migrations.ex @@ -17,6 +17,13 @@ defmodule Mix.Tasks.AshPostgres.GenerateMigrations do * `no_format` - files that are created will not be formatted with the code formatter * `dry_run` - no files are created, instead the new migration is printed + #### Snapshots + + Snapshots are stored in a folder for each table that migrations are generated for. Each snapshot is + stored in a file with a timestamp of when it was generated. + This is important because it allows for simultaneous work to be done on separate branches, and for rolling back + changes more easily, e.g removing a generated migration, and deleting the most recent snapshot, without having to redo + all of it #### Dropping columns diff --git a/test/migration_generator_test.exs b/test/migration_generator_test.exs index 63e18e47..0dc76632 100644 --- a/test/migration_generator_test.exs +++ b/test/migration_generator_test.exs @@ -79,26 +79,16 @@ defmodule AshPostgres.MigrationGeneratorTest do end test "it creates a snapshot for each resource" do - assert File.exists?(Path.wildcard("test_snapshots_path/test_repo/*_posts.json")) - assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) + assert File.exists?(Path.wildcard("test_snapshots_path/test_repo/posts/*.json")) end test "the snapshots can be loaded" do - assert File.exists?(Path.wildcard("test_snapshots_path/test_repo/*_posts.json")) - assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) + assert File.exists?(Path.wildcard("test_snapshots_path/test_repo/posts/*.json")) end test "the snapshots contain valid json" do - assert File.exists?(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) - - assert File.read!(Path.wildcard("test_snapshots_path/test_repo/*_posts.json")) + assert File.read!(Path.wildcard("test_snapshots_path/test_repo/posts/*.json")) |> Jason.decode!(keys: :atoms!) - - version_file_content = - File.read!(Path.join(["test_snapshots_path", "test_repo", "posts.version.json"])) - |> Jason.decode!(keys: :atoms!) - - assert Map.get(version_file_content, :latest_version, nil) != nil end test "the migration creates the table" do