Skip to content

Commit d22c11c

Browse files
meiji163meiji163
authored
Merge branch 'master' into adjust-wait-post-repl-restart
2 parents ea5708e + 690b1e1 commit d22c11c

File tree

1,414 files changed

+231440
-5055
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,414 files changed

+231440
-5055
lines changed

Diff for: ‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
/libexec/
44
/.vendor/
55
.idea/
6+
*.tmp

Diff for: ‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d
9797

9898
`gh-ost` is a Go project; it is built with Go `1.15` and above. To build on your own, use either:
9999
- [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary.
100-
- [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost`
100+
- [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost-release`
101101

102102
Generally speaking, `master` branch is stable, but only [releases](https://github.com/github/gh-ost/releases) are to be used in production.
103103

Diff for: ‎build.sh

+5-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
#!/bin/bash
2-
#
3-
#
42

53
RELEASE_VERSION=
64
buildpath=
@@ -72,11 +70,14 @@ main() {
7270
build macOS osx darwin amd64
7371
build macOS osx darwin arm64
7472

73+
bin_files=$(find $buildpath/gh-ost* -type f -maxdepth 1)
7574
echo "Binaries found in:"
76-
find $buildpath/gh-ost* -type f -maxdepth 1
75+
echo "$bin_files"
7776

7877
echo "Checksums:"
79-
(cd $buildpath && shasum -a256 gh-ost* 2>/dev/null)
78+
(shasum -a256 $bin_files 2>/dev/null)
79+
80+
echo "Build Success!"
8081
}
8182

8283
main "$@"

Diff for: ‎go.mod

+51-3
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,65 @@ require (
88
github.com/go-sql-driver/mysql v1.8.1
99
github.com/google/uuid v1.6.0
1010
github.com/openark/golib v0.0.0-20210531070646-355f37940af8
11+
github.com/stretchr/testify v1.9.0
12+
github.com/testcontainers/testcontainers-go v0.34.0
1113
golang.org/x/net v0.24.0
12-
golang.org/x/term v0.19.0
13-
golang.org/x/text v0.14.0
14+
golang.org/x/sync v0.10.0
15+
golang.org/x/term v0.27.0
16+
golang.org/x/text v0.21.0
1417
)
1518

1619
require (
20+
dario.cat/mergo v1.0.0 // indirect
1721
filippo.io/edwards25519 v1.1.0 // indirect
22+
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
23+
github.com/Microsoft/go-winio v0.6.2 // indirect
24+
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
25+
github.com/containerd/containerd v1.7.18 // indirect
26+
github.com/containerd/log v0.1.0 // indirect
27+
github.com/containerd/platforms v0.2.1 // indirect
28+
github.com/cpuguy83/dockercfg v0.3.2 // indirect
29+
github.com/davecgh/go-spew v1.1.1 // indirect
30+
github.com/distribution/reference v0.6.0 // indirect
31+
github.com/docker/docker v27.1.1+incompatible // indirect
32+
github.com/docker/go-connections v0.5.0 // indirect
33+
github.com/docker/go-units v0.5.0 // indirect
34+
github.com/felixge/httpsnoop v1.0.4 // indirect
35+
github.com/go-logr/logr v1.4.1 // indirect
36+
github.com/go-logr/stdr v1.2.2 // indirect
37+
github.com/go-ole/go-ole v1.2.6 // indirect
38+
github.com/gogo/protobuf v1.3.2 // indirect
39+
github.com/klauspost/compress v1.17.4 // indirect
40+
github.com/kr/text v0.2.0 // indirect
41+
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
42+
github.com/magiconair/properties v1.8.7 // indirect
43+
github.com/moby/docker-image-spec v1.3.1 // indirect
44+
github.com/moby/patternmatcher v0.6.0 // indirect
45+
github.com/moby/sys/sequential v0.5.0 // indirect
46+
github.com/moby/sys/user v0.1.0 // indirect
47+
github.com/moby/term v0.5.0 // indirect
48+
github.com/morikuni/aec v1.0.0 // indirect
49+
github.com/opencontainers/go-digest v1.0.0 // indirect
50+
github.com/opencontainers/image-spec v1.1.0 // indirect
1851
github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 // indirect
52+
github.com/pkg/errors v0.9.1 // indirect
53+
github.com/pmezard/go-difflib v1.0.0 // indirect
54+
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
55+
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
56+
github.com/shoenig/go-m1cpu v0.1.6 // indirect
1957
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
2058
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 // indirect
2159
github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 // indirect
60+
github.com/sirupsen/logrus v1.9.3 // indirect
61+
github.com/tklauser/go-sysconf v0.3.12 // indirect
62+
github.com/tklauser/numcpus v0.6.1 // indirect
63+
github.com/yusufpapurcu/wmi v1.2.3 // indirect
64+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
65+
go.opentelemetry.io/otel v1.24.0 // indirect
66+
go.opentelemetry.io/otel/metric v1.24.0 // indirect
67+
go.opentelemetry.io/otel/trace v1.24.0 // indirect
2268
go.uber.org/atomic v1.7.0 // indirect
23-
golang.org/x/sys v0.19.0 // indirect
69+
golang.org/x/crypto v0.31.0 // indirect
70+
golang.org/x/sys v0.28.0 // indirect
71+
gopkg.in/yaml.v3 v3.0.1 // indirect
2472
)

Diff for: ‎go.sum

+156-7
Large diffs are not rendered by default.

Diff for: ‎go/base/context.go

+5
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,11 @@ type MigrationContext struct {
103103
AzureMySQL bool
104104
AttemptInstantDDL bool
105105

106+
// SkipPortValidation allows skipping the port validation in `ValidateConnection`
107+
// This is useful when connecting to a MySQL instance where the external port
108+
// may not match the internal port.
109+
SkipPortValidation bool
110+
106111
config ContextConfig
107112
configMutex *sync.Mutex
108113
ConfigFile string

Diff for: ‎go/base/context_test.go

+12-12
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"time"
1212

1313
"github.com/openark/golib/log"
14-
test "github.com/openark/golib/tests"
14+
"github.com/stretchr/testify/require"
1515
)
1616

1717
func init() {
@@ -22,22 +22,22 @@ func TestGetTableNames(t *testing.T) {
2222
{
2323
context := NewMigrationContext()
2424
context.OriginalTableName = "some_table"
25-
test.S(t).ExpectEquals(context.GetOldTableName(), "_some_table_del")
26-
test.S(t).ExpectEquals(context.GetGhostTableName(), "_some_table_gho")
27-
test.S(t).ExpectEquals(context.GetChangelogTableName(), "_some_table_ghc")
25+
require.Equal(t, "_some_table_del", context.GetOldTableName())
26+
require.Equal(t, "_some_table_gho", context.GetGhostTableName())
27+
require.Equal(t, "_some_table_ghc", context.GetChangelogTableName(), "_some_table_ghc")
2828
}
2929
{
3030
context := NewMigrationContext()
3131
context.OriginalTableName = "a123456789012345678901234567890123456789012345678901234567890"
32-
test.S(t).ExpectEquals(context.GetOldTableName(), "_a1234567890123456789012345678901234567890123456789012345678_del")
33-
test.S(t).ExpectEquals(context.GetGhostTableName(), "_a1234567890123456789012345678901234567890123456789012345678_gho")
34-
test.S(t).ExpectEquals(context.GetChangelogTableName(), "_a1234567890123456789012345678901234567890123456789012345678_ghc")
32+
require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_del", context.GetOldTableName())
33+
require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_gho", context.GetGhostTableName())
34+
require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_ghc", context.GetChangelogTableName())
3535
}
3636
{
3737
context := NewMigrationContext()
3838
context.OriginalTableName = "a123456789012345678901234567890123456789012345678901234567890123"
3939
oldTableName := context.GetOldTableName()
40-
test.S(t).ExpectEquals(oldTableName, "_a1234567890123456789012345678901234567890123456789012345678_del")
40+
require.Equal(t, "_a1234567890123456789012345678901234567890123456789012345678_del", oldTableName)
4141
}
4242
{
4343
context := NewMigrationContext()
@@ -46,15 +46,15 @@ func TestGetTableNames(t *testing.T) {
4646
longForm := "Jan 2, 2006 at 3:04pm (MST)"
4747
context.StartTime, _ = time.Parse(longForm, "Feb 3, 2013 at 7:54pm (PST)")
4848
oldTableName := context.GetOldTableName()
49-
test.S(t).ExpectEquals(oldTableName, "_a1234567890123456789012345678901234567890123_20130203195400_del")
49+
require.Equal(t, "_a1234567890123456789012345678901234567890123_20130203195400_del", oldTableName)
5050
}
5151
{
5252
context := NewMigrationContext()
5353
context.OriginalTableName = "foo_bar_baz"
5454
context.ForceTmpTableName = "tmp"
55-
test.S(t).ExpectEquals(context.GetOldTableName(), "_tmp_del")
56-
test.S(t).ExpectEquals(context.GetGhostTableName(), "_tmp_gho")
57-
test.S(t).ExpectEquals(context.GetChangelogTableName(), "_tmp_ghc")
55+
require.Equal(t, "_tmp_del", context.GetOldTableName())
56+
require.Equal(t, "_tmp_gho", context.GetGhostTableName())
57+
require.Equal(t, "_tmp_ghc", context.GetChangelogTableName())
5858
}
5959
}
6060

Diff for: ‎go/base/load_map_test.go

+11-11
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"testing"
1010

1111
"github.com/openark/golib/log"
12-
test "github.com/openark/golib/tests"
12+
"github.com/stretchr/testify/require"
1313
)
1414

1515
func init() {
@@ -20,39 +20,39 @@ func TestParseLoadMap(t *testing.T) {
2020
{
2121
loadList := ""
2222
m, err := ParseLoadMap(loadList)
23-
test.S(t).ExpectNil(err)
24-
test.S(t).ExpectEquals(len(m), 0)
23+
require.NoError(t, err)
24+
require.Len(t, m, 0)
2525
}
2626
{
2727
loadList := "threads_running=20,threads_connected=10"
2828
m, err := ParseLoadMap(loadList)
29-
test.S(t).ExpectNil(err)
30-
test.S(t).ExpectEquals(len(m), 2)
31-
test.S(t).ExpectEquals(m["threads_running"], int64(20))
32-
test.S(t).ExpectEquals(m["threads_connected"], int64(10))
29+
require.NoError(t, err)
30+
require.Len(t, m, 2)
31+
require.Equal(t, int64(20), m["threads_running"])
32+
require.Equal(t, int64(10), m["threads_connected"])
3333
}
3434
{
3535
loadList := "threads_running=20=30,threads_connected=10"
3636
_, err := ParseLoadMap(loadList)
37-
test.S(t).ExpectNotNil(err)
37+
require.Error(t, err)
3838
}
3939
{
4040
loadList := "threads_running=20,threads_connected"
4141
_, err := ParseLoadMap(loadList)
42-
test.S(t).ExpectNotNil(err)
42+
require.Error(t, err)
4343
}
4444
}
4545

4646
func TestString(t *testing.T) {
4747
{
4848
m, _ := ParseLoadMap("")
4949
s := m.String()
50-
test.S(t).ExpectEquals(s, "")
50+
require.Equal(t, "", s)
5151
}
5252
{
5353
loadList := "threads_running=20,threads_connected=10"
5454
m, _ := ParseLoadMap(loadList)
5555
s := m.String()
56-
test.S(t).ExpectEquals(s, "threads_connected=10,threads_running=20")
56+
require.Equal(t, "threads_connected=10,threads_running=20", s)
5757
}
5858
}

Diff for: ‎go/base/utils.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -63,18 +63,27 @@ func StringContainsAll(s string, substrings ...string) bool {
6363

6464
func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext, name string) (string, error) {
6565
versionQuery := `select @@global.version`
66-
var port, extraPort int
66+
6767
var version string
6868
if err := db.QueryRow(versionQuery).Scan(&version); err != nil {
6969
return "", err
7070
}
71+
72+
if migrationContext.SkipPortValidation {
73+
return version, nil
74+
}
75+
76+
var extraPort int
77+
7178
extraPortQuery := `select @@global.extra_port`
7279
if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil { //nolint:staticcheck
7380
// swallow this error. not all servers support extra_port
7481
}
82+
7583
// AliyunRDS set users port to "NULL", replace it by gh-ost param
7684
// GCP set users port to "NULL", replace it by gh-ost param
7785
// Azure MySQL set users port to a different value by design, replace it by gh-ost para
86+
var port int
7887
if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL {
7988
port = connectionConfig.Key.Port
8089
} else {

Diff for: ‎go/base/utils_test.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"testing"
1010

1111
"github.com/openark/golib/log"
12-
test "github.com/openark/golib/tests"
12+
"github.com/stretchr/testify/require"
1313
)
1414

1515
func init() {
@@ -19,11 +19,11 @@ func init() {
1919
func TestStringContainsAll(t *testing.T) {
2020
s := `insert,delete,update`
2121

22-
test.S(t).ExpectFalse(StringContainsAll(s))
23-
test.S(t).ExpectFalse(StringContainsAll(s, ""))
24-
test.S(t).ExpectFalse(StringContainsAll(s, "drop"))
25-
test.S(t).ExpectTrue(StringContainsAll(s, "insert"))
26-
test.S(t).ExpectFalse(StringContainsAll(s, "insert", "drop"))
27-
test.S(t).ExpectTrue(StringContainsAll(s, "insert", ""))
28-
test.S(t).ExpectTrue(StringContainsAll(s, "insert", "update", "delete"))
22+
require.False(t, StringContainsAll(s))
23+
require.False(t, StringContainsAll(s, ""))
24+
require.False(t, StringContainsAll(s, "drop"))
25+
require.True(t, StringContainsAll(s, "insert"))
26+
require.False(t, StringContainsAll(s, "insert", "drop"))
27+
require.True(t, StringContainsAll(s, "insert", ""))
28+
require.True(t, StringContainsAll(s, "insert", "update", "delete"))
2929
}

Diff for: ‎go/cmd/gh-ost/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ func main() {
107107
chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 10-100,000)")
108108
dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)")
109109
defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking")
110-
cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout)")
110+
cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout) or attempting instant DDL")
111111
niceRatio := flag.Float64("nice-ratio", 0, "force being 'nice', imply sleep time per chunk time; range: [0.0..100.0]. Example values: 0 is aggressive. 1: for every 1ms spent copying rows, sleep additional 1ms (effectively doubling runtime); 0.7: for every 10ms spend in a rowcopy chunk, spend 7ms sleeping immediately after")
112112

113113
maxLagMillis := flag.Int64("max-lag-millis", 1500, "replication lag at which to throttle operation")

0 commit comments

Comments
 (0)