diff --git a/cmd/dump.go b/cmd/dump.go index 329dcf8..678c9a1 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -114,6 +114,10 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er if !v.IsSet("no-database-name") && dumpConfig != nil && dumpConfig.NoDatabaseName != nil { noDatabaseName = *dumpConfig.NoDatabaseName } + skipExtendedInsert := v.GetBool("skip-extended-insert") + if !v.IsSet("skip-extended-insert") && dumpConfig != nil && dumpConfig.SkipExtendedInsert != nil { + skipExtendedInsert = *dumpConfig.SkipExtendedInsert + } compact := v.GetBool("compact") if !v.IsSet("compact") && dumpConfig != nil && dumpConfig.Compact != nil { compact = *dumpConfig.Compact @@ -257,6 +261,7 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er PreBackupScripts: preBackupScripts, PostBackupScripts: postBackupScripts, SuppressUseDatabase: noDatabaseName, + SkipExtendedInsert: skipExtendedInsert, Compact: compact, Triggers: triggers, Routines: routines, @@ -307,6 +312,9 @@ S3: If it is a URL of the format s3://bucketname/path then it will connect via S // single database, do not include `USE database;` in dump flags.Bool("no-database-name", false, "Omit `USE ;` in the dump, so it can be restored easily to a different database.") + // skip extended insert in dump; instead, one INSERT per record in each table + flags.Bool("skip-extended-insert", false, "Skip extended insert in dump; instead, one INSERT per record in each table.") + // frequency flags.Int("frequency", defaultFrequency, "how often to run backups, in minutes") diff --git a/docs/configuration.md b/docs/configuration.md index 69d3584..a3b8eb8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -72,6 +72,7 @@ The following are the environment variables, CLI flags and configuration file op | names of databases to dump, comma-separated | B | `include` | `DB_DUMP_INCLUDE` | `dump.include` | all databases in the server | | names of databases to exclude from the dump | B | `exclude` | `DB_DUMP_EXCLUDE` | `dump.exclude` | | | do not include `USE ;` statement in the dump | B | `no-database-name` | `NO_DATABASE_NAME` | `dump.noDatabaseName` | `false` | +| Replace single long INSERT statement per table with one INSERT statement per line | B | `skip-extended-insert` | `DB_DUMP_SKIP_EXTENDED_INSERT` | `dump.skipExtendedInsert` | `false` | | restore to a specific database | R | `restore --database` | `RESTORE_DATABASE` | `restore.database` | | | how often to do a dump or prune, in minutes | BP | `dump --frequency` | `DB_DUMP_FREQUENCY` | `dump.schedule.frequency` | `1440` (in minutes), i.e. once per day | | what time to do the first dump or prune | BP | `dump --begin` | `DB_DUMP_BEGIN` | `dump.schedule.begin` | `0`, i.e. immediately | diff --git a/examples/configs/local.yaml b/examples/configs/local.yaml index 076f610..7e0dd52 100644 --- a/examples/configs/local.yaml +++ b/examples/configs/local.yaml @@ -20,6 +20,7 @@ spec: - table4 safechars: true # defaults to false noDatabaseName: false # remove the `USE ` statement from backup files, defaults to false + skipExtendedInsert: false # replace single long INSERT statement per table with one INSERT statement per line, defaults to false # schedule to dump, can use one of: cron, frequency, once. If frequency is set, begin will be checked schedule: once: true # run only once and exit; ignores all other scheduling. Defaults to false diff --git a/go.mod b/go.mod index 167f8a4..d8f5b39 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( filippo.io/age v1.2.1 github.com/InfiniteLoopSpace/go_S-MIME v0.0.0-20181221134359-3f58f9a4b2b6 github.com/bramvdbogaerde/go-scp v1.5.0 - github.com/databacker/api/go/api v0.0.0-20250818102239-219c793f2151 + github.com/databacker/api/go/api v0.0.0-20250908082438-7e7e048af692 github.com/gliderlabs/ssh v0.3.8 github.com/google/go-cmp v0.7.0 github.com/kevinburke/ssh_config v1.2.0 @@ -49,12 +49,14 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/go-chi/chi/v5 v5.2.3 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect @@ -63,11 +65,11 @@ require ( github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect + github.com/oapi-codegen/runtime v1.1.2 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/grpc v1.67.0 // indirect @@ -123,6 +125,6 @@ require ( golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect golang.org/x/tools v0.22.0 // indirect - gopkg.in/ini.v1 v1.51.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 49dda43..d99b3f3 100644 --- a/go.sum +++ b/go.sum @@ -15,10 +15,13 @@ github.com/InfiniteLoopSpace/go_S-MIME v0.0.0-20181221134359-3f58f9a4b2b6/go.mod github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -62,6 +65,7 @@ github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM= github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -82,8 +86,8 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/databacker/api/go/api v0.0.0-20250818102239-219c793f2151 h1:WuQNmzJiLSR0d2IpeifwK0E6eOLZQDxzbuHWIEN2/9U= -github.com/databacker/api/go/api v0.0.0-20250818102239-219c793f2151/go.mod h1:bQhbl71Lk1ATni0H+u249hjoQ8ShAdVNcNjnw6z+SbE= +github.com/databacker/api/go/api v0.0.0-20250908082438-7e7e048af692 h1:eIny/Iq0E5Hg7QKXNrjT91oyWjhnY+WVAv5Y8fRC0JY= +github.com/databacker/api/go/api v0.0.0-20250908082438-7e7e048af692/go.mod h1:vXp1gX/diWXW659asaBZ7K3Wgs2OG3igz0dF0UB4yIY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -110,6 +114,8 @@ github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNe github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= +github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -142,7 +148,6 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -161,8 +166,8 @@ github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 h1:O7syWuYG github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -211,6 +216,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oapi-codegen/runtime v1.1.2 h1:P2+CubHq8fO4Q6fV1tqDBZHCwpVpvPg7oKiYzQgXIyI= +github.com/oapi-codegen/runtime v1.1.2/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -248,9 +255,7 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+n github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -269,6 +274,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -418,8 +424,8 @@ golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -457,8 +463,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/pkg/core/dump.go b/pkg/core/dump.go index 9c3f408..f62047f 100644 --- a/pkg/core/dump.go +++ b/pkg/core/dump.go @@ -39,6 +39,7 @@ func (e *Executor) Dump(ctx context.Context, opts DumpOptions) (DumpResults, err triggers := opts.Triggers routines := opts.Routines suppressUseDatabase := opts.SuppressUseDatabase + skipExtendedInsert := opts.SkipExtendedInsert maxAllowedPacket := opts.MaxAllowedPacket filenamePattern := opts.FilenamePattern parallelism := opts.Parallelism @@ -111,6 +112,7 @@ func (e *Executor) Dump(ctx context.Context, opts DumpOptions) (DumpResults, err Triggers: triggers, Routines: routines, SuppressUseDatabase: suppressUseDatabase, + SkipExtendedInsert: skipExtendedInsert, MaxAllowedPacket: maxAllowedPacket, PostDumpDelay: opts.PostDumpDelay, Parallelism: parallelism, diff --git a/pkg/core/dumpoptions.go b/pkg/core/dumpoptions.go index d59414a..408734d 100644 --- a/pkg/core/dumpoptions.go +++ b/pkg/core/dumpoptions.go @@ -24,6 +24,7 @@ type DumpOptions struct { Triggers bool Routines bool SuppressUseDatabase bool + SkipExtendedInsert bool MaxAllowedPacket int Run uuid.UUID FilenamePattern string diff --git a/pkg/database/dump.go b/pkg/database/dump.go index 8ecd09b..0fab972 100644 --- a/pkg/database/dump.go +++ b/pkg/database/dump.go @@ -14,6 +14,7 @@ type DumpOpts struct { Triggers bool Routines bool SuppressUseDatabase bool + SkipExtendedInsert bool MaxAllowedPacket int // PostDumpDelay after each dump is complete, while holding connection open. Do not use outside of tests. PostDumpDelay time.Duration @@ -59,6 +60,7 @@ func Dump(ctx context.Context, dbconn *Connection, opts DumpOpts, writers []Dump Triggers: opts.Triggers, Routines: opts.Routines, SuppressUseDatabase: opts.SuppressUseDatabase, + SkipExtendedInsert: opts.SkipExtendedInsert, MaxAllowedPacket: opts.MaxAllowedPacket, PostDumpDelay: opts.PostDumpDelay, } diff --git a/pkg/database/mysql/dump.go b/pkg/database/mysql/dump.go index af2ceec..9c93a2d 100644 --- a/pkg/database/mysql/dump.go +++ b/pkg/database/mysql/dump.go @@ -50,6 +50,7 @@ type Data struct { Routines bool Host string SuppressUseDatabase bool + SkipExtendedInsert bool Charset string Collation string PostDumpDelay time.Duration diff --git a/pkg/database/mysql/table.go b/pkg/database/mysql/table.go index d63c823..880c1de 100644 --- a/pkg/database/mysql/table.go +++ b/pkg/database/mysql/table.go @@ -312,11 +312,17 @@ func (table *baseTable) Stream() <-chan string { for table.Next() { b := table.RowBuffer() - // Truncate our insert if it won't fit - if insert.Len() != 0 && insert.Len()+b.Len() > table.data.MaxAllowedPacket-1 { + // If we're skipping extended inserts, we write each row individually + if table.data.SkipExtendedInsert { + _, _ = fmt.Fprint(&insert, strings.Join( + // extra "" at the end so we get an extra whitespace as needed + []string{"INSERT", "INTO", esc(table.Name()), "(" + table.columnsList() + ")", "VALUES", ""}, + " ")) + _, _ = b.WriteTo(&insert) _, _ = insert.WriteString(";") valueOut <- insert.String() insert.Reset() + continue } if insert.Len() == 0 { @@ -327,6 +333,14 @@ func (table *baseTable) Stream() <-chan string { } else { _, _ = insert.WriteString(",") } + + // Truncate our insert if it won't fit + if insert.Len() != 0 && insert.Len()+b.Len() > table.data.MaxAllowedPacket-1 { + _, _ = insert.WriteString(";") + valueOut <- insert.String() + insert.Reset() + } + _, _ = b.WriteTo(&insert) } if insert.Len() != 0 { diff --git a/test/backup_test.go b/test/backup_test.go index 5f4c4fd..194e819 100644 --- a/test/backup_test.go +++ b/test/backup_test.go @@ -338,7 +338,7 @@ func (d *dockerContext) makeSMB(smbImage string) error { return nil } -func (d *dockerContext) createBackupFile(mysqlCID, mysqlUser, mysqlPass, outfile, compactOutfile string) error { +func (d *dockerContext) createBackupFiles(mysqlCID, mysqlUser, mysqlPass string, outfilesToCommands map[string][]string) error { ctx := context.Background() // Create and populate the table @@ -424,49 +424,34 @@ DELIMITER ; return fmt.Errorf("failed to create table: %s", bufe.String()) } - // Dump the database - do both compact and non-compact - mysqlDumpCompactCmd := []string{"mysqldump", "-hlocalhost", "--protocol=tcp", "--complete-insert", fmt.Sprintf("-u%s", mysqlUser), fmt.Sprintf("-p%s", mysqlPass), "--compact", "--databases", "--triggers", "--routines", "tester"} - attachResp, exitCode, err = d.execInContainer(ctx, mysqlCID, mysqlDumpCompactCmd) - if err != nil { - return fmt.Errorf("failed to attach to exec: %w", err) - } - defer attachResp.Close() - if exitCode != 0 { - return fmt.Errorf("failed to dump database: %w", err) - } - - fCompact, err := os.Create(compactOutfile) - if err != nil { - return err - } - defer func() { - _ = fCompact.Close() - }() + // Dump the database multiple times, each with different options + for outfile, cmdOpts := range outfilesToCommands { + mysqlDumpCmd := []string{"mysqldump", "-hlocalhost", "--protocol=tcp", "--complete-insert", fmt.Sprintf("-u%s", mysqlUser), fmt.Sprintf("-p%s", mysqlPass), "--databases", "--triggers", "--routines"} + mysqlDumpCmd = append(mysqlDumpCmd, cmdOpts...) + mysqlDumpCmd = append(mysqlDumpCmd, "tester") + attachResp, exitCode, err = d.execInContainer(ctx, mysqlCID, mysqlDumpCmd) + if err != nil { + return fmt.Errorf("failed to attach to exec: %w", err) + } + defer attachResp.Close() + if exitCode != 0 { + return fmt.Errorf("failed to dump database: %w", err) + } - _, _ = stdcopy.StdCopy(fCompact, &bufe, attachResp.Reader) + f, err := os.Create(outfile) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() - bufo.Reset() - bufe.Reset() + _, _ = stdcopy.StdCopy(f, &bufe, attachResp.Reader) - mysqlDumpCmd := []string{"mysqldump", "-hlocalhost", "--protocol=tcp", "--complete-insert", fmt.Sprintf("-u%s", mysqlUser), fmt.Sprintf("-p%s", mysqlPass), "--databases", "--triggers", "--routines", "tester"} - attachResp, exitCode, err = d.execInContainer(ctx, mysqlCID, mysqlDumpCmd) - if err != nil { - return fmt.Errorf("failed to attach to exec: %w", err) + bufo.Reset() + bufe.Reset() } - defer attachResp.Close() - if exitCode != 0 { - return fmt.Errorf("failed to dump database: %w", err) - } - - f, err := os.Create(outfile) - if err != nil { - return err - } - defer func() { - _ = f.Close() - }() - _, _ = stdcopy.StdCopy(f, &bufe, attachResp.Reader) return err } @@ -522,7 +507,7 @@ func (d *dockerContext) rmContainers(cids ...string) error { // - check that the backup now is there in the right format // - clear the target -func setup(dc *dockerContext, base, backupFile, compactBackupFile string) (mysql, smb containerPort, s3url string, s3backend gofakes3.Backend, err error) { +func setup(dc *dockerContext, base string, outfilesToCmdOpts map[string][]string) (mysql, smb containerPort, s3url string, s3backend gofakes3.Backend, err error) { if err := dc.makeSMB(smbImage); err != nil { return mysql, smb, s3url, s3backend, fmt.Errorf("failed to build smb image: %v", err) } @@ -554,8 +539,8 @@ func setup(dc *dockerContext, base, backupFile, compactBackupFile string) (mysql } // create the backup file - log.Debugf("Creating backup file") - if err := dc.createBackupFile(mysql.id, mysqlUser, mysqlPass, backupFile, compactBackupFile); err != nil { + log.Debugf("Creating backup files") + if err := dc.createBackupFiles(mysql.id, mysqlUser, mysqlPass, outfilesToCmdOpts); err != nil { return mysql, smb, s3url, s3backend, fmt.Errorf("failed to create backup file: %v", err) } return @@ -998,7 +983,13 @@ func TestIntegration(t *testing.T) { } backupFile := filepath.Join(base, "backup.sql") compactBackupFile := filepath.Join(base, "backup-compact.sql") - if mysql, smb, s3, s3backend, err = setup(dc, base, backupFile, compactBackupFile); err != nil { + skipExtendedInsertBackupFile := filepath.Join(base, "backup-skip-extended.sql") + outfilesToCmdOpts := map[string][]string{ + backupFile: nil, + compactBackupFile: {"--compact"}, + skipExtendedInsertBackupFile: {"--skip-extended-insert"}, + } + if mysql, smb, s3, s3backend, err = setup(dc, base, outfilesToCmdOpts); err != nil { t.Fatalf("failed to setup test: %v", err) } backupData, err := os.ReadFile(backupFile) @@ -1009,6 +1000,10 @@ func TestIntegration(t *testing.T) { if err != nil { t.Fatalf("failed to read compact backup file %s: %v", compactBackupFile, err) } + skipExtendedBackupData, err := os.ReadFile(skipExtendedInsertBackupFile) + if err != nil { + t.Fatalf("failed to read skip extended insert backup file %s: %v", skipExtendedInsertBackupFile, err) + } defer func() { // log the results before tearing down, if requested if err := logContainers(dc, smb.id, mysql.id); err != nil { @@ -1065,6 +1060,28 @@ func TestIntegration(t *testing.T) { }) }) + // check contents with skip-extended-insert enabled + t.Run("skip-extended-insert", func(t *testing.T) { + dumpOpts := core.DumpOptions{ + Compressor: &compression.GzipCompressor{}, + SkipExtendedInsert: true, + Triggers: true, + Routines: true, + } + runTest(t, testOptions{ + targets: []string{"/compact-backups/"}, + dc: dc, + base: base, + backupData: skipExtendedBackupData, + mysql: mysql, + smb: smb, + s3: s3, + s3backend: s3backend, + dumpOptions: dumpOpts, + checkCommand: checkDumpTest, + }) + }) + t.Run("pattern", func(t *testing.T) { dumpOpts := core.DumpOptions{ Compressor: &compression.GzipCompressor{},