Skip to content
Permalink
Browse files

Upgrade tooling / Fix recovery issue / Fix testing env

This repository's been suffering a bit of bit-rot, and when I found a
bug in the migrations system I needed to fix, I took some time to do a
few other things.

The core issue is that if you have a series of migrations 0-3, and 3 is
bad and aborts the migration, the current code would have you start back
at 0 the next run, which is wrong since all the way up to 2 have
completed. An ugly oversight that I hope never returns. Tests were added
for this bug.

Other things: upgrade all go tooling, upgrade to latest (future proofed)
postgres packaging.

Signed-off-by: Erik Hollensbe <github@hollensbe.org>
  • Loading branch information...
erikh committed Nov 10, 2019
1 parent 88eb9a0 commit 70396226fc0eee85a9f5310c92f3640fd51382c6
Showing with 89 additions and 12 deletions.
  1. +1 −1 box.rb
  2. +4 −1 do.go
  3. +18 −0 do_test.go
  4. +1 −1 entrypoint.sh
  5. +13 −7 go.mod
  6. +45 −0 go.sum
  7. +2 −2 run_test.sh
  8. +2 −0 testdata/bad/0.sql
  9. +1 −0 testdata/bad/1.sql
  10. +1 −0 testdata/bad/2.sql
  11. +1 −0 testdata/bad/3.sql
2 box.rb
@@ -6,7 +6,7 @@
run "mkdir /postgresql"
run "chown postgres:postgres /postgresql"

run "su - postgres -c '/usr/lib/postgresql/9.6/bin/initdb -D /postgresql'"
run "su - postgres -c '/usr/lib/postgresql/*/bin/initdb -D /postgresql'"

copy "entrypoint.sh", "/"
run "chmod 755 /entrypoint.sh"
5 do.go
@@ -50,9 +50,12 @@ func doMigrate(conn *pgx.Conn, dir string, quiet bool) (retErr error) {
if err := apply(conn, dir, i, quiet); err != nil {
return err
}
if err := updateMigrationTable(conn, i+1); err != nil {
return err
}
}

return updateMigrationTable(conn, max+1)
return nil
}

func createSchemaMigrations(conn *pgx.Conn) error {
@@ -34,6 +34,7 @@ func mkDB(db string) error {
if err != nil {
return err
}
defer conn.Close()

_, err = conn.Exec(fmt.Sprintf("create database %s", db))
return err
@@ -102,6 +103,23 @@ func (ms *migratorSuite) TearDownTest(c *C) {
c.Assert(clearDBs(), IsNil)
}

func (ms *migratorSuite) TestRecovery(c *C) {
c.Assert(mkDB("bad"), IsNil)

conn, err := getDB("bad")
c.Assert(err, IsNil)
defer conn.Close()

c.Assert(doMigrate(conn, "testdata/bad", true), NotNil)
c.Assert(doMigrate(conn, "testdata/bad", true), NotNil)

row := conn.QueryRow("select max(id) from schema_migrations")
var id int
c.Assert(row.Scan(&id), IsNil)
// the bad dir has 3 good migrations and one bad one at the end. the max value should be 2..
c.Assert(id, Equals, 3)
}

func (ms *migratorSuite) TestBasic(c *C) {
c.Assert(mkDB("one"), IsNil)

@@ -1,6 +1,6 @@
#!/bin/sh

su - postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -D /postgresql start &>/dev/null'
su - postgres -c '/usr/lib/postgresql/*/bin/pg_ctl -D /postgresql start &>/dev/null'

while ! (echo "select 1" | su - postgres sh -c 'psql template1 &>/dev/null')
do
20 go.mod
@@ -1,12 +1,18 @@
module github.com/erikh/migrator

require (
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/fatih/color v1.7.0
github.com/jackc/pgx v3.2.0+incompatible
github.com/mattn/go-colorable v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.4 // indirect
github.com/pkg/errors v0.8.0
github.com/urfave/cli v1.20.0
golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4
golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e // indirect
github.com/jackc/pgx v3.6.0+incompatible
github.com/kr/pretty v0.1.0 // indirect
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.10 // indirect
github.com/pkg/errors v0.8.1
github.com/urfave/cli v1.22.1
golang.org/x/crypto v0.0.0-20191108234033-bd318be0434a
golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd // indirect
golang.org/x/text v0.3.2 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
)

go 1.13
45 go.sum
@@ -1,16 +1,61 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/jackc/pgx v3.2.0+incompatible h1:0Vihzu20St42/UDsvZGdNE6jak7oi/UOeMzwMPHkgFY=
github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
github.com/jackc/pgx v3.6.0+incompatible h1:bJeo4JdVbDAW8KB2m8XkFeo8CPipREoG37BwEoKGz+Q=
github.com/jackc/pgx v3.6.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 h1:Vk3wNqEZwyGyei9yq5ekj7frek2u7HUfffJ1/opblzc=
golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191108234033-bd318be0434a h1:R/qVym5WAxsZWQqZCwDY/8sdVKV1m1WgU4/S5IRQAzc=
golang.org/x/crypto v0.0.0-20191108234033-bd318be0434a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e h1:EfdBzeKbFSvOjoIqSZcfS8wp0FBLokGBEs9lz1OtSg0=
golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd h1:3x5uuvBgE6oaXJjCOvpCC1IpgJogqQ+PqGGU3ZxAgII=
golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -10,8 +10,8 @@ apt-get install postgresql-all -y
mkdir /postgresql
chown postgres:postgres /postgresql

su - postgres -c '/usr/lib/postgresql/9.6/bin/initdb -D /postgresql'
su - postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -D /postgresql start &>/dev/null'
su - postgres -c '/usr/lib/postgresql/*/bin/initdb -D /postgresql'
su - postgres -c '/usr/lib/postgresql/*/bin/pg_ctl -D /postgresql start &>/dev/null'

while ! (echo "select 1" | su - postgres sh -c 'psql template1 &>/dev/null')
do
@@ -0,0 +1,2 @@
create table foo (id int);
create table bar (id int);
@@ -0,0 +1 @@
create table quux (id int);
@@ -0,0 +1 @@
create table another (id int);
@@ -0,0 +1 @@
fart

0 comments on commit 7039622

Please sign in to comment.
You can’t perform that action at this time.