From fce4a53b5123e0ed7231143ba5ab962abff02d45 Mon Sep 17 00:00:00 2001 From: Pavel Fadeev Date: Wed, 6 Feb 2019 02:13:23 +0300 Subject: [PATCH] 8 integrate semiocast psql (#19) * #8 initial integration * #8 first spinning test * #8 first spinning test * #8 one more running test * #8 one more running test * #8 one more running test * #8 enable two more tests * #8 all tests are enabled * #8 fix travis problems * #8 update documentation * #8 update toc * #8 add generic test example --- Makefile | 2 + README.md | 48 +++++ rebar.config | 3 +- test/spgsql_migrations_SUITE.erl | 178 ++++++++++++++++++ .../00_initial_version.sql | 7 + .../00_initial_version.sql | 7 + .../01-two-scripts-test/01_next_version.sql | 5 + .../20_initial_version.sql | 7 + .../03-migration-gap/00_initial_version.sql | 7 + .../03-migration-gap/02_gap_version.sql | 5 + .../00_initial_version.sql | 7 + .../01_faulty_migration.sql | 1 + 12 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 test/spgsql_migrations_SUITE.erl create mode 100644 test/spgsql_migrations_SUITE_data/00-single-script-test/00_initial_version.sql create mode 100644 test/spgsql_migrations_SUITE_data/01-two-scripts-test/00_initial_version.sql create mode 100644 test/spgsql_migrations_SUITE_data/01-two-scripts-test/01_next_version.sql create mode 100644 test/spgsql_migrations_SUITE_data/02-wrong-initial-version/20_initial_version.sql create mode 100644 test/spgsql_migrations_SUITE_data/03-migration-gap/00_initial_version.sql create mode 100644 test/spgsql_migrations_SUITE_data/03-migration-gap/02_gap_version.sql create mode 100644 test/spgsql_migrations_SUITE_data/04-last-migration-fail/00_initial_version.sql create mode 100644 test/spgsql_migrations_SUITE_data/04-last-migration-fail/01_faulty_migration.sql diff --git a/Makefile b/Makefile index 4c6f918..43954e0 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,8 @@ local: format postgres-bounce all postgres-down clean: $(REBAR) clean + rm -rf ./_build/default/lib/pure_migrations + rm -rf ./_build/test/lib/pure_migrations compile: $(REBAR) compile diff --git a/README.md b/README.md index 3d3da38..de68739 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ As an extra - do this in "no side-effects" mode. * [Compatibility table](#compatibility-table) * [Live code samples](#live-code-samples) + [Postgres and epgsql/epgsql sample](#postgres-and--epgsql-epgsql--https---githubcom-epgsql-epgsql--sample) + + [Postgres and semiocast/pgsql sample](#postgres-and--semiocast-pgsql--https---githubcom-semiocast-pgsql--sample) - [No-effects approach and tools used to achieve it](#no-effects-approach-and-tools-used-to-achieve-it) * [Tool #1: effects externalization](#tool--1--effects-externalization) * [Tool #2: make effects explicit](#tool--2--make-effects-explicit) @@ -40,6 +41,8 @@ Just call `engine:migrate/3` (see specification [here](src/engine.erl#L9)), prov | Database dialect | Library | Example | | -------------- | ------ | ------- | | postgres | [epgsql/epgsql:4.2.1](https://github.com/epgsql/epgsql/releases/tag/4.2.1) | [epgsql test](test/epgsql_migrations_SUITE.erl) +| postgres | [semiocast/pgsql:v26.0.2](https://github.com/semiocast/pgsql/releases/tag/v26.0.2) | [spgsql test](test/spgsql_migrations_SUITE.erl) +| postgres | any library with basic postgres functional | [generic test](test/pure_migrations_SUITE.erl) ## Live code samples ### Postgres and [epgsql/epgsql](https://github.com/epgsql/epgsql) sample @@ -78,6 +81,51 @@ Also see examples from live epgsql integration tests [here](test/epgsql_migrations_SUITE.erl) +### Postgres and [semiocast/pgsql](https://github.com/semiocast/pgsql) sample +
+ Click to expand + + ```erlang + Conn = ?config(conn, Opts), + MigrationCall = + engine:migrate( + "scripts/folder/path", + fun(F) -> + pgsql_connection:simple_query("BEGIN", Conn), + try F() of + Res -> + pgsql_connection:simple_query("COMMIT", Conn), + Res + catch + _:Problem -> + pgsql_connection:simple_query("ROLLBACK", Conn), + {rollback, Problem} + end + end, + fun(Q) -> + case pgsql_connection:simple_query(Q, Conn) of + {{select, 0}, []} -> []; + {{select, 1}, Data = [{_V, _F}|_]} -> + [{V, binary_to_list(BinF)} || {V, BinF} <- Data]; + {{select, 1}, [{null}]} -> -1; + {{select, 1}, [{N}]} -> N; + {{insert, 0, 1}, []} -> ok; + {{create, table},[]} -> ok; + {error, Details} -> {error, Details}; + _ -> ok + end + end), + ... + %% more preparation steps + ... + %% migration call + ok = MigrationCall(), + + ``` +Also see examples from live semiocast/pgsql integration tests +[here](test/spgsql_migrations_SUITE.erl) +
+ # No-effects approach and tools used to achieve it Oh, **there is more!** Library implemented in the [way](https://en.wikipedia.org/wiki/Pure_function), that all side-effects either externalized or deferred explicitly. Reasons diff --git a/rebar.config b/rebar.config index 891a9df..dfecfef 100644 --- a/rebar.config +++ b/rebar.config @@ -1,7 +1,8 @@ {profiles, [ {test, [ {deps, [ - {epgsql, {git, "https://github.com/epgsql/epgsql", {tag, "4.2.1"}}} + {epgsql, {git, "https://github.com/epgsql/epgsql", {tag, "4.2.1"}}}, + {spgsql, {git, "https://github.com/semiocast/pgsql", {tag, "v26.0.2"}}} ]}, {eunit_opts , [verbose]}, {ct_opts, [ { diff --git a/test/spgsql_migrations_SUITE.erl b/test/spgsql_migrations_SUITE.erl new file mode 100644 index 0000000..c77c124 --- /dev/null +++ b/test/spgsql_migrations_SUITE.erl @@ -0,0 +1,178 @@ +-module(spgsql_migrations_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). + +all() -> [ migrate_one_script_test + , migrate_few_scripts_test + , incremental_migration_test + , wrong_initial_version_test + , migration_gap_test + , transactional_migration_test + ]. + +migrate_one_script_test(Opts) -> + Conn = ?config(conn, Opts), + PreparedCall = engine:migrate( + filename:join([?config(data_dir, Opts), "00-single-script-test"]), + spgsql_tx_fun(Conn), + spgsql_query_fun(Conn) + ), + ?assertEqual(ok, PreparedCall()), + ?assertMatch( + {{select,1},[{0}]}, + pgsql_connection:simple_query("select max(version) from database_migrations_history", Conn)). + +migrate_few_scripts_test(Opts) -> + Conn = ?config(conn, Opts), + PreparedCall = engine:migrate( + filename:join([?config(data_dir, Opts), "01-two-scripts-test"]), + spgsql_tx_fun(Conn), + spgsql_query_fun(Conn) + ), + ?assertEqual(ok, PreparedCall()), + ?assertMatch( + {{select, 1}, [{1}]}, + pgsql_connection:simple_query("select max(version) from database_migrations_history", Conn)), + ?assertMatch( + {{select, 1}, [{1}]}, + pgsql_connection:simple_query("select count(*) from fruit where color = 'yellow'", Conn)). + +incremental_migration_test(Opts) -> + Conn = ?config(conn, Opts), + TxFun = spgsql_tx_fun(Conn), + QueryFun = spgsql_query_fun(Conn), + MigrationStep1 = engine:migrate( + filename:join([?config(data_dir, Opts), "00-single-script-test"]), + TxFun, QueryFun), + MigrationStep2 = engine:migrate( + filename:join([?config(data_dir, Opts), "01-two-scripts-test"]), + TxFun, QueryFun), + + %% assert migrations table created and nothing done + ?assertMatch( + {{select, 1}, [{null}]}, + pgsql_connection:simple_query("select max(version) from database_migrations_history", Conn)), + ?assertMatch( + {error, {pgsql_error, _}}, + pgsql_connection:simple_query("select count(*) from fruit", Conn)), + + %% assert step 1 migration + ok = MigrationStep1(), + ?assertMatch( + {{select, 1}, [{0}]}, + pgsql_connection:simple_query("select max(version) from database_migrations_history", Conn)), + + %% assert step 2 migration + ok =MigrationStep2(), + ?assertMatch( + {{select, 1}, [{1}]}, + pgsql_connection:simple_query("select max(version) from database_migrations_history", Conn)), + ?assertMatch( + {{select, 1}, [{1}]}, + pgsql_connection:simple_query("select count(*) from fruit where color = 'yellow'", Conn)). + +wrong_initial_version_test(Opts) -> + Conn = ?config(conn, Opts), + PreparedCall = engine:migrate( + filename:join([?config(data_dir, Opts), "02-wrong-initial-version"]), + spgsql_tx_fun(Conn), + spgsql_query_fun(Conn) + ), + ?assertMatch( + {rollback, {badmatch, {error, unexpected_version, {expected, 0, supplied, 20}}}}, + PreparedCall()). + +migration_gap_test(Opts) -> + Conn = ?config(conn, Opts), + MigrationStep1 = engine:migrate( + filename:join([?config(data_dir, Opts), "00-single-script-test"]), + spgsql_tx_fun(Conn), + spgsql_query_fun(Conn) + ), + MigrationStep2 = engine:migrate( + filename:join([?config(data_dir, Opts), "03-migration-gap"]), + spgsql_tx_fun(Conn), + spgsql_query_fun(Conn) + ), + + %% assert step 1 migration + ok = MigrationStep1(), + ?assertMatch( + {{select, 1}, [{0}]}, + pgsql_connection:simple_query("select max(version) from database_migrations_history", Conn)), + + %% assert step 2 failed migration + ?assertMatch( + {rollback, {badmatch, {error, unexpected_version, {expected, 1, supplied, 2}}}}, + MigrationStep2()), + ?assertMatch( + {{select, 1}, [{0}]}, + pgsql_connection:simple_query("select max(version) from database_migrations_history", Conn)), + ?assertMatch( + {error, {pgsql_error, _}}, + pgsql_connection:simple_query("select count(*) from fruit where color = 'yellow'", Conn)). + +transactional_migration_test(Opts) -> + Conn = ?config(conn, Opts), + PreparedCall = engine:migrate( + filename:join([?config(data_dir, Opts), "04-last-migration-fail"]), + spgsql_tx_fun(Conn), + spgsql_query_fun(Conn) + ), + ?assertMatch( + {rollback, {badmatch, {error, {pgsql_error, _}}}}, + PreparedCall()), + ?assertMatch( + {{select, 1}, [{null}]}, + pgsql_connection:simple_query("select max(version) from database_migrations_history", Conn)), + ?assertMatch( + {error, {pgsql_error, _}}, + pgsql_connection:simple_query("select count(*) from fruit", Conn)). + +spgsql_query_fun(Conn) -> + fun(Q) -> + case pgsql_connection:simple_query(Q, Conn) of + {{select, 0}, []} -> []; + {{select, 1}, Data = [{_V, _F}|_]} -> + [{V, binary_to_list(BinF)} || {V, BinF} <- Data]; + {{select, 1}, [{null}]} -> -1; + {{select, 1}, [{N}]} -> N; + {{insert, 0, 1}, []} -> ok; + {{create, table},[]} -> ok; + {error, Details} -> {error, Details}; + _ -> ok + end + end. + +spgsql_tx_fun(Conn) -> + fun(F) -> + pgsql_connection:simple_query("BEGIN", Conn), + try F() of + Res -> + pgsql_connection:simple_query("COMMIT", Conn), + Res + catch + _:Problem -> + pgsql_connection:simple_query("ROLLBACK", Conn), + {rollback, Problem} + end + end. + +init_per_testcase(_TestCase, Opts) -> + {ok, [{host, Host}, + {port, _Port}, + {database, Database}, + {username, Username}, + {secret, Secret}, + {timeout, _Timeout}]} = application:get_env(postgres, config), + {ok, _Pid} = pgsql_connection_sup:start_link(), + C = pgsql_connection:open(Host, Database, Username, Secret), + {{drop, table}, _} = pgsql_connection:simple_query("DROP TABLE IF EXISTS database_migrations_history, fruit", C), + [{conn, C}|Opts]. + +end_per_testcase(_TestCase, Opts) -> + ok = pgsql_connection:close(?config(conn, Opts)). + diff --git a/test/spgsql_migrations_SUITE_data/00-single-script-test/00_initial_version.sql b/test/spgsql_migrations_SUITE_data/00-single-script-test/00_initial_version.sql new file mode 100644 index 0000000..5853b2e --- /dev/null +++ b/test/spgsql_migrations_SUITE_data/00-single-script-test/00_initial_version.sql @@ -0,0 +1,7 @@ +CREATE TABLE fruit ( + id SERIAL, + name TEXT NOT NULL +); +INSERT INTO fruit (name) VALUES ( + 'apple' +); \ No newline at end of file diff --git a/test/spgsql_migrations_SUITE_data/01-two-scripts-test/00_initial_version.sql b/test/spgsql_migrations_SUITE_data/01-two-scripts-test/00_initial_version.sql new file mode 100644 index 0000000..5853b2e --- /dev/null +++ b/test/spgsql_migrations_SUITE_data/01-two-scripts-test/00_initial_version.sql @@ -0,0 +1,7 @@ +CREATE TABLE fruit ( + id SERIAL, + name TEXT NOT NULL +); +INSERT INTO fruit (name) VALUES ( + 'apple' +); \ No newline at end of file diff --git a/test/spgsql_migrations_SUITE_data/01-two-scripts-test/01_next_version.sql b/test/spgsql_migrations_SUITE_data/01-two-scripts-test/01_next_version.sql new file mode 100644 index 0000000..95d3af5 --- /dev/null +++ b/test/spgsql_migrations_SUITE_data/01-two-scripts-test/01_next_version.sql @@ -0,0 +1,5 @@ +ALTER TABLE fruit + ADD COLUMN color TEXT NOT NULL DEFAULT 'green'; +INSERT INTO fruit (name, color) VALUES ( + 'lemon', 'yellow' +); \ No newline at end of file diff --git a/test/spgsql_migrations_SUITE_data/02-wrong-initial-version/20_initial_version.sql b/test/spgsql_migrations_SUITE_data/02-wrong-initial-version/20_initial_version.sql new file mode 100644 index 0000000..5853b2e --- /dev/null +++ b/test/spgsql_migrations_SUITE_data/02-wrong-initial-version/20_initial_version.sql @@ -0,0 +1,7 @@ +CREATE TABLE fruit ( + id SERIAL, + name TEXT NOT NULL +); +INSERT INTO fruit (name) VALUES ( + 'apple' +); \ No newline at end of file diff --git a/test/spgsql_migrations_SUITE_data/03-migration-gap/00_initial_version.sql b/test/spgsql_migrations_SUITE_data/03-migration-gap/00_initial_version.sql new file mode 100644 index 0000000..5853b2e --- /dev/null +++ b/test/spgsql_migrations_SUITE_data/03-migration-gap/00_initial_version.sql @@ -0,0 +1,7 @@ +CREATE TABLE fruit ( + id SERIAL, + name TEXT NOT NULL +); +INSERT INTO fruit (name) VALUES ( + 'apple' +); \ No newline at end of file diff --git a/test/spgsql_migrations_SUITE_data/03-migration-gap/02_gap_version.sql b/test/spgsql_migrations_SUITE_data/03-migration-gap/02_gap_version.sql new file mode 100644 index 0000000..95d3af5 --- /dev/null +++ b/test/spgsql_migrations_SUITE_data/03-migration-gap/02_gap_version.sql @@ -0,0 +1,5 @@ +ALTER TABLE fruit + ADD COLUMN color TEXT NOT NULL DEFAULT 'green'; +INSERT INTO fruit (name, color) VALUES ( + 'lemon', 'yellow' +); \ No newline at end of file diff --git a/test/spgsql_migrations_SUITE_data/04-last-migration-fail/00_initial_version.sql b/test/spgsql_migrations_SUITE_data/04-last-migration-fail/00_initial_version.sql new file mode 100644 index 0000000..5853b2e --- /dev/null +++ b/test/spgsql_migrations_SUITE_data/04-last-migration-fail/00_initial_version.sql @@ -0,0 +1,7 @@ +CREATE TABLE fruit ( + id SERIAL, + name TEXT NOT NULL +); +INSERT INTO fruit (name) VALUES ( + 'apple' +); \ No newline at end of file diff --git a/test/spgsql_migrations_SUITE_data/04-last-migration-fail/01_faulty_migration.sql b/test/spgsql_migrations_SUITE_data/04-last-migration-fail/01_faulty_migration.sql new file mode 100644 index 0000000..951bb93 --- /dev/null +++ b/test/spgsql_migrations_SUITE_data/04-last-migration-fail/01_faulty_migration.sql @@ -0,0 +1 @@ +GARBAGE scrip4 h3r3!@ \ No newline at end of file