6 ensure epgsql integration (#17)

* #6 try to have postgres at travis

* #6 have local dockerized postgres container to run integration tests agains

* #6 enforce local docker operations

* #6 have local container in proper cycle

* #6 instruct travis for postgres setup

* #6 instruct travis for postgres setup

* #6 fix coveralls run

* #6 configure dummy test with connection

* #6 kill redundant dependency

* #6 kill redundant profile

* #6 trigger table cleanup

* #6 first real integration test

* #6 assert single step migration

* #6 few scripts migration

* #6 few scripts migration

* #6 two step migration test

* #6 two step migration test

* #6 have a test plan

* #6 implement negative test

* #6 implement negative test

* #6 implement one more negative test

* #6 implement one more negative test

* #6 little formatting

.travis.yml

@@ -2,6 +2,16 @@ cache:
   directories:
     - ./_build/plt
 
+services:
+  - postgresql
+
+addons:
+  postgresql: "9.4"
+
+before_script:
+  - psql -c "CREATE DATABASE migration;" -U postgres
+  - psql -c "CREATE USER migration WITH PASSWORD 'migration';" -U postgres
+
 language: erlang
 
 otp_release:

Makefile

@@ -1,10 +1,12 @@
 REBAR = ./rebar3
+DOCKER = docker
+CONTAINER_NAME = postgres-migration-test-container
 
 all: clean code-checks test cover
 
 travis: all coveralls
 
-local: format all
+local: format postgres-bounce all postgres-down
 
 clean:
 	$(REBAR) clean
@@ -27,3 +29,16 @@ coveralls:
 
 format:
 	$(REBAR) fmt
+
+postgres-up:
+	$(DOCKER) run --name $(CONTAINER_NAME) \
+	-p 5432:5432 \
+	-e POSTGRES_PASSWORD=migration \
+	-e POSTGRES_USER=migration \
+	-e POSTGRES_DB=migration \
+	-d postgres:9.6-alpine
+
+postgres-down:
+	-$(DOCKER) rm -f $(CONTAINER_NAME)
+
+postgres-bounce: postgres-down postgres-up

rebar.config

@@ -1,9 +1,11 @@
 {profiles, [
             {test, [
                     {deps, [
-                            {erlexec, {git, "https://github.com/saleyn/erlexec.git", {ref, "576fb5d"}}}
+                            {epgsql,    {git, "https://github.com/epgsql/epgsql", {tag, "4.2.1"}}}
                            ]},
                     {eunit_opts             , [verbose]},
+                    {ct_opts, [ {
+                                 sys_config, ["test/config/test.config"]}]},
                     {cover_enabled          , true},
                     {cover_print_enabled    , true},
                     {cover_export_enabled   , true},

src/dialect_postgres.erl

@@ -14,7 +14,7 @@ migrations_names() ->
 
 save_migration(Version, Filename) ->
   lists:flatten(io_lib:format(
-    "INSERT INTO database_migrations_history(version, filename) VALUES (~w, ~s)",
+    "INSERT INTO database_migrations_history(version, filename) VALUES (~w, '~s')",
     [Version, Filename])).
 
 latest_existing_version() ->

src/engine.erl

@@ -20,21 +20,21 @@ migrate(Path, FTx, FQuery) ->
     end.
 
 do_migration(Path, FQuery, {V, F}) ->
-    ExpectedVersion = FQuery(dialect_postgres:latest_existing_version()) + 1,
-    case V of
-        ExpectedVersion ->
-            ScriptPath = filename:join(Path, F),
-            compose(
-              fun() -> file:read_file(ScriptPath) end,
-              fun({ok, ScriptBody}) ->
+    ScriptPath = filename:join(Path, F),
+    compose(
+      fun() -> file:read_file(ScriptPath) end,
+      fun({ok, ScriptBody}) ->
+              ExpectedVersion = FQuery(dialect_postgres:latest_existing_version()) + 1,
+              case V of
+                  ExpectedVersion ->
                       ok = FQuery(ScriptBody),
-                      ok = FQuery(dialect_postgres:save_migration(V, F))
-              end);
-        _ -> {
-              error,
-              unexpected_version,
-              {expected, ExpectedVersion, supplied, V}}
-    end;
+                      ok = FQuery(dialect_postgres:save_migration(V, F));
+                  _ -> {
+                        error,
+                        unexpected_version,
+                        {expected, ExpectedVersion, supplied, V}}
+              end
+      end);
 do_migration(_Path, _FQuery, Error) -> Error.
 
 find_migrations(ScriptsLocation, FQuery) ->
@@ -58,4 +58,4 @@ version_and_filename(F) ->
 
 compose(F1, F2) -> fun() -> F2(F1()) end.
 map(Generate, Map) -> fun() -> [Map(R) || R <- Generate()] end.
-flatten(Generate) -> fun() -> [ok = R() || R <- Generate()] end.
+flatten(Generate) -> fun() -> [ok = R() || R <- Generate()], ok end.

test/config/test.config

@@ -0,0 +1,11 @@
+[{db, [{
+        config, [
+                 {host, "localhost"},
+                 {port, 5432},
+                 {database, "migration"},
+                 {username, "migration"},
+                 {secret, "migration"},
+                 {timeout, 10000}
+                ]
+       }]
+ }].

test/epgsql_migrations_SUITE.erl

@@ -0,0 +1,184 @@
+-module(epgsql_migrations_SUITE).
+
+-include_lib("epgsql/include/epgsql.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-compile(export_all).
+
+all() -> [ migrate_one_script_test
+         , migrate_few_scripts_test
+         , incremental_migration_test
+         , wrong_initial_version_test
+         , migration_gap_test
+         , transactional_migration_test
+         ].
+
+migrate_one_script_test(Opts) ->
+    Conn = ?config(conn, Opts),
+    PreparedCall = engine:migrate(
+                     filename:join([?config(data_dir, Opts), "00-single-script-test"]),
+                     fun(F) ->
+                             epgsql:with_transaction(Conn, fun(_) -> F() end)
+                     end,
+                     epgsql_query_fun(Conn)
+                    ),
+    ?assertEqual(ok, PreparedCall()),
+    ?assertMatch(
+       {ok, _,[{<<"0">>}]},
+       epgsql:squery(Conn, "select max(version) from database_migrations_history")).
+
+migrate_few_scripts_test(Opts) ->
+    Conn = ?config(conn, Opts),
+    PreparedCall = engine:migrate(
+                     filename:join([?config(data_dir, Opts), "01-two-scripts-test"]),
+                     fun(F) ->
+                             epgsql:with_transaction(Conn, fun(_) -> F() end)
+                     end,
+                     epgsql_query_fun(Conn)
+                    ),
+    ?assertEqual(ok, PreparedCall()),
+    ?assertMatch(
+       {ok, _,[{<<"1">>}]},
+       epgsql:squery(Conn, "select max(version) from database_migrations_history")),
+    ?assertMatch(
+       {ok, _,[{<<"1">>}]},
+       epgsql:squery(Conn, "select count(*) from fruit where color = 'yellow'")).
+
+incremental_migration_test(Opts) ->
+    Conn = ?config(conn, Opts),
+    TxFun =
+        fun(F) ->
+                epgsql:with_transaction(Conn, fun(_) -> F() end)
+        end,
+    MigrationStep1 = engine:migrate(
+                       filename:join([?config(data_dir, Opts), "00-single-script-test"]),
+                       TxFun, epgsql_query_fun(Conn)
+                      ),
+    MigrationStep2 = engine:migrate(
+                       filename:join([?config(data_dir, Opts), "01-two-scripts-test"]),
+                       TxFun, epgsql_query_fun(Conn)
+                      ),
+
+    %% assert migrations table created and nothing done
+    ?assertMatch(
+       {ok, _,[{null}]},
+       epgsql:squery(Conn, "select max(version) from database_migrations_history")),
+    ?assertMatch(
+       {error,{error, _, _, undefined_table, <<"relation \"fruit\" does not exist">>, _}},
+       epgsql:squery(Conn, "select count(*) from fruit")),
+
+    %% assert step 1 migration
+    ok = MigrationStep1(),
+    ?assertMatch(
+       {ok, _, [{<<"0">>}]},
+       epgsql:squery(Conn, "select max(version) from database_migrations_history")),
+
+    %% assert step 2 migration
+    ok =MigrationStep2(),
+    ?assertMatch(
+       {ok, _, [{<<"1">>}]},
+       epgsql:squery(Conn, "select max(version) from database_migrations_history")),
+    ?assertMatch(
+       {ok, _, [{<<"1">>}]},
+       epgsql:squery(Conn, "select count(*) from fruit where color = 'yellow'")).
+
+wrong_initial_version_test(Opts) ->
+    Conn = ?config(conn, Opts),
+    PreparedCall = engine:migrate(
+                     filename:join([?config(data_dir, Opts), "02-wrong-initial-version"]),
+                     fun(F) ->
+                             epgsql:with_transaction(Conn, fun(_) -> F() end)
+                     end,
+                     epgsql_query_fun(Conn)
+                    ),
+    ?assertEqual(
+       {rollback, {badmatch, {error, unexpected_version, {expected, 0, supplied, 20}}}},
+       PreparedCall()).
+
+migration_gap_test(Opts) ->
+    Conn = ?config(conn, Opts),
+    TxFun =
+        fun(F) ->
+                epgsql:with_transaction(Conn, fun(_) -> F() end)
+        end,
+    MigrationStep1 = engine:migrate(
+                       filename:join([?config(data_dir, Opts), "00-single-script-test"]),
+                       TxFun, epgsql_query_fun(Conn)
+                      ),
+    MigrationStep2 = engine:migrate(
+                       filename:join([?config(data_dir, Opts), "03-migration-gap"]),
+                       TxFun, epgsql_query_fun(Conn)
+                      ),
+
+    %% assert step 1 migration
+    ok = MigrationStep1(),
+    ?assertMatch(
+       {ok, _, [{<<"0">>}]},
+       epgsql:squery(Conn, "select max(version) from database_migrations_history")),
+
+    %% assert step 2 failed migration
+    ?assertEqual(
+       {rollback, {badmatch, {error, unexpected_version, {expected, 1, supplied, 2}}}},
+       MigrationStep2()),
+    ?assertMatch(
+       {ok, _, [{<<"0">>}]},
+       epgsql:squery(Conn, "select max(version) from database_migrations_history")),
+    ?assertMatch(
+       {error, {error, error, _, undefined_column, <<"column \"color\" does not exist">>, _}},
+       epgsql:squery(Conn, "select count(*) from fruit where color = 'yellow'")).
+
+transactional_migration_test(Opts) ->
+    Conn = ?config(conn, Opts),
+    PreparedCall = engine:migrate(
+                     filename:join([?config(data_dir, Opts), "04-last-migration-fail"]),
+                     fun(F) ->
+                             epgsql:with_transaction(Conn, fun(_) -> F() end)
+                     end,
+                     epgsql_query_fun(Conn)
+                    ),
+    ?assertMatch(
+       {rollback, {badmatch, {error, {error, error, _, syntax_error, _, _}}}},
+       PreparedCall()),
+    ?assertMatch(
+       {ok, _, [{null}]},
+       epgsql:squery(Conn, "select max(version) from database_migrations_history")),
+    ?assertMatch(
+       {error, {error, _, _, undefined_table, <<"relation \"fruit\" does not exist">>, _}},
+       epgsql:squery(Conn, "select count(*) from fruit")).
+
+epgsql_query_fun(Conn) ->
+    fun(Q) ->
+            case epgsql:squery(Conn, Q) of
+                ok -> ok;
+                {ok, [
+                      {column, <<"version">>, _, _, _, _, _},
+                      {column, <<"filename">>, _, _, _, _, _}], Data} ->
+                    [{list_to_integer(binary_to_list(BinV)), binary_to_list(BinF)} || {BinV, BinF} <- Data];
+                {ok, [{column, <<"max">>, _, _, _, _, _}], [{null}]} -> -1;
+                {ok, [{column, <<"max">>, _, _, _, _, _}], [{N}]} ->
+                    list_to_integer(binary_to_list(N));
+                [{ok, _, _}, {ok, _}] -> ok;
+                {ok, _, _} -> ok;
+                {ok, _} -> ok;
+                Default -> Default
+            end
+    end.
+
+init_per_testcase(_TestCase, Opts) ->
+    {ok, [{host, Host},
+          {port, _Port},
+          {database, Database},
+          {username, Username},
+          {secret, Secret},
+          {timeout, Timeout}]} = application:get_env(db, config),
+    {ok, C} = epgsql:connect(Host, Username, Secret,
+                             #{
+                               database => Database,
+                               timeout => Timeout
+                              }),
+    {ok, _, _} = epgsql:squery(C, "DROP TABLE IF EXISTS database_migrations_history, fruit"),
+    [{conn, C}|Opts].
+
+end_per_testcase(_TestCase, Opts) ->
+    ok = epgsql:close(?config(conn, Opts)).

test/epgsql_migrations_SUITE_data/00-single-script-test/00_initial_version.sql

@@ -0,0 +1,7 @@
+CREATE TABLE fruit (
+ id SERIAL,
+ name TEXT NOT NULL
+);
+INSERT INTO fruit (name) VALUES (
+    'apple'
+);

test/epgsql_migrations_SUITE_data/01-two-scripts-test/00_initial_version.sql

@@ -0,0 +1,7 @@
+CREATE TABLE fruit (
+ id SERIAL,
+ name TEXT NOT NULL
+);
+INSERT INTO fruit (name) VALUES (
+    'apple'
+);

test/epgsql_migrations_SUITE_data/01-two-scripts-test/01_next_version.sql

@@ -0,0 +1,5 @@
+ALTER TABLE fruit
+    ADD COLUMN color TEXT NOT NULL DEFAULT 'green';
+INSERT INTO fruit (name, color) VALUES (
+    'lemon', 'yellow'
+);

test/epgsql_migrations_SUITE_data/02-wrong-initial-version/20_initial_version.sql

@@ -0,0 +1,7 @@
+CREATE TABLE fruit (
+ id SERIAL,
+ name TEXT NOT NULL
+);
+INSERT INTO fruit (name) VALUES (
+    'apple'
+);

test/epgsql_migrations_SUITE_data/03-migration-gap/00_initial_version.sql

@@ -0,0 +1,7 @@
+CREATE TABLE fruit (
+ id SERIAL,
+ name TEXT NOT NULL
+);
+INSERT INTO fruit (name) VALUES (
+    'apple'
+);

test/epgsql_migrations_SUITE_data/03-migration-gap/02_gap_version.sql

@@ -0,0 +1,5 @@
+ALTER TABLE fruit
+    ADD COLUMN color TEXT NOT NULL DEFAULT 'green';
+INSERT INTO fruit (name, color) VALUES (
+    'lemon', 'yellow'
+);

test/epgsql_migrations_SUITE_data/04-last-migration-fail/00_initial_version.sql

@@ -0,0 +1,7 @@
+CREATE TABLE fruit (
+ id SERIAL,
+ name TEXT NOT NULL
+);
+INSERT INTO fruit (name) VALUES (
+    'apple'
+);

test/epgsql_migrations_SUITE_data/04-last-migration-fail/01_faulty_migration.sql

@@ -0,0 +1 @@
+GARBAGE scrip4 h3r3!@