Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

dumpling: add enable paging for tidb server #39791

Merged
merged 11 commits into from
Dec 13, 2022
Merged

dumpling: add enable paging for tidb server #39791

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
0a0d8f7
add enable paging, parallelize test cases
lichunzhu Dec 9, 2022
f3cd949
Merge branch 'master' into dumplingSupportEnablePaging
lichunzhu Dec 9, 2022
8657cb7
remove ioutil
lichunzhu Dec 9, 2022
0db491b
Merge branch 'dumplingSupportEnablePaging' of https://github.com/lich…
lichunzhu Dec 9, 2022
63b3bee
fix bazel
lichunzhu Dec 9, 2022
74803c2
fix
lichunzhu Dec 9, 2022
fff6631
Merge branch 'master' into dumplingSupportEnablePaging
lichunzhu Dec 9, 2022
9356148
Merge branch 'master' into dumplingSupportEnablePaging
hawkingrei Dec 10, 2022
6dcacc1
Merge branch 'master' into dumplingSupportEnablePaging
hawkingrei Dec 10, 2022
1e199b9
Merge branch 'master' into dumplingSupportEnablePaging
hawkingrei Dec 12, 2022
c739ca6
Merge branch 'master' into dumplingSupportEnablePaging
ti-chi-bot Dec 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions dumpling/export/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ go_test(
"//util/filter",
"//util/promutil",
"//util/table-filter",
"@com_github_coreos_go_semver//semver",
"@com_github_data_dog_go_sqlmock//:go-sqlmock",
"@com_github_go_sql_driver_mysql//:mysql",
"@com_github_pingcap_errors//:errors",
Expand Down
18 changes: 18 additions & 0 deletions dumpling/export/dump.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"sync/atomic"
"time"

"github.com/coreos/go-semver/semver"
// import mysql driver
"github.com/go-sql-driver/mysql"
"github.com/google/uuid"
Expand Down Expand Up @@ -46,6 +47,10 @@ var openDBFunc = openDB

var errEmptyHandleVals = errors.New("empty handleVals for TiDB table")

// After TiDB v6.2.0 we always enable tidb_enable_paging by default.
// see https://docs.pingcap.com/zh/tidb/dev/system-variables#tidb_enable_paging-%E4%BB%8E-v540-%E7%89%88%E6%9C%AC%E5%BC%80%E5%A7%8B%E5%BC%95%E5%85%A5
var enablePagingVersion = semver.New("6.2.0")

// Dumper is the dump progress structure
type Dumper struct {
tctx *tcontext.Context
Expand Down Expand Up @@ -1539,6 +1544,19 @@ func updateServiceSafePoint(tctx *tcontext.Context, pdClient pd.Client, ttl int6
}
}

// setDefaultSessionParams is a step to set default params for session params.
func setDefaultSessionParams(si version.ServerInfo, sessionParams map[string]interface{}) {
defaultSessionParams := map[string]interface{}{}
if si.ServerType == version.ServerTypeTiDB && si.HasTiKV && si.ServerVersion.Compare(*enablePagingVersion) >= 0 {
defaultSessionParams["tidb_enable_paging"] = "ON"
}
for k, v := range defaultSessionParams {
if _, ok := sessionParams[k]; !ok {
sessionParams[k] = v
}
}
}

// setSessionParam is an initialization step of Dumper.
func setSessionParam(d *Dumper) error {
conf, pool := d.conf, d.dbHandle
Expand Down
65 changes: 65 additions & 0 deletions dumpling/export/dump_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

"github.com/DATA-DOG/go-sqlmock"
"github.com/coreos/go-semver/semver"
"github.com/pingcap/errors"
"github.com/pingcap/tidb/br/pkg/version"
tcontext "github.com/pingcap/tidb/dumpling/context"
Expand Down Expand Up @@ -224,3 +225,67 @@ func TestUnregisterMetrics(t *testing.T) {
// should not panic
require.Error(t, err)
}

func TestSetDefaultSessionParams(t *testing.T) {
testCases := []struct {
si version.ServerInfo
sessionParams map[string]interface{}
expectedParams map[string]interface{}
}{
{
si: version.ServerInfo{
ServerType: version.ServerTypeTiDB,
HasTiKV: true,
ServerVersion: semver.New("6.1.0"),
},
sessionParams: map[string]interface{}{
"tidb_snapshot": "2020-01-01 00:00:00",
},
expectedParams: map[string]interface{}{
"tidb_snapshot": "2020-01-01 00:00:00",
},
},
{
si: version.ServerInfo{
ServerType: version.ServerTypeTiDB,
HasTiKV: true,
ServerVersion: semver.New("6.2.0"),
},
sessionParams: map[string]interface{}{
"tidb_snapshot": "2020-01-01 00:00:00",
},
expectedParams: map[string]interface{}{
"tidb_enable_paging": "ON",
"tidb_snapshot": "2020-01-01 00:00:00",
},
},
{
si: version.ServerInfo{
ServerType: version.ServerTypeTiDB,
HasTiKV: true,
ServerVersion: semver.New("6.2.0"),
},
sessionParams: map[string]interface{}{
"tidb_enable_paging": "OFF",
"tidb_snapshot": "2020-01-01 00:00:00",
},
expectedParams: map[string]interface{}{
"tidb_enable_paging": "OFF",
"tidb_snapshot": "2020-01-01 00:00:00",
},
},
{
si: version.ServerInfo{
ServerType: version.ServerTypeMySQL,
ServerVersion: semver.New("8.0.32"),
},
sessionParams: map[string]interface{}{},
expectedParams: map[string]interface{}{},
},
}

for _, testCase := range testCases {
setDefaultSessionParams(testCase.si, testCase.sessionParams)
require.Equal(t, testCase.expectedParams, testCase.sessionParams)
}
}
2 changes: 2 additions & 0 deletions dumpling/export/sql_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1308,6 +1308,8 @@ func buildMockNewRows(mock sqlmock.Sqlmock, columns []string, driverValues [][]d
}

func readRegionCsvDriverValues(t *testing.T) [][]driver.Value {
t.Helper()

csvFilename := "region_results.csv"
file, err := os.Open(csvFilename)
require.NoError(t, err)
Expand Down
25 changes: 13 additions & 12 deletions dumpling/export/writer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package export
import (
"context"
"database/sql/driver"
"io/ioutil"
"os"
"path"
"sync"
Expand All @@ -31,7 +30,7 @@ func TestWriteDatabaseMeta(t *testing.T) {
_, err = os.Stat(p)
require.NoError(t, err)

bytes, err := ioutil.ReadFile(p)
bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE DATABASE `test`;\n", string(bytes))
}
Expand All @@ -50,7 +49,7 @@ func TestWritePolicyMeta(t *testing.T) {
_, err = os.Stat(p)
require.NoError(t, err)

bytes, err := ioutil.ReadFile(p)
bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, "/*!40101 SET NAMES binary*/;\ncreate placement policy `y` followers=2;\n", string(bytes))
}
Expand All @@ -68,7 +67,7 @@ func TestWriteTableMeta(t *testing.T) {
p := path.Join(dir, "test.t-schema.sql")
_, err = os.Stat(p)
require.NoError(t, err)
bytes, err := ioutil.ReadFile(p)
bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes))
}
Expand All @@ -89,14 +88,14 @@ func TestWriteViewMeta(t *testing.T) {
p := path.Join(dir, "test.v-schema.sql")
_, err = os.Stat(p)
require.NoError(t, err)
bytes, err := ioutil.ReadFile(p)
bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, specCmt+createTableSQL, string(bytes))

p = path.Join(dir, "test.v-schema-view.sql")
_, err = os.Stat(p)
require.NoError(t, err)
bytes, err = ioutil.ReadFile(p)
bytes, err = os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, specCmt+createViewSQL, string(bytes))
}
Expand Down Expand Up @@ -126,7 +125,7 @@ func TestWriteTableData(t *testing.T) {
p := path.Join(dir, "test.employee.000000000.sql")
_, err = os.Stat(p)
require.NoError(t, err)
bytes, err := ioutil.ReadFile(p)
bytes, err := os.ReadFile(p)
require.NoError(t, err)

expected := "/*!40101 SET NAMES binary*/;\n" +
Expand Down Expand Up @@ -182,7 +181,7 @@ func TestWriteTableDataWithFileSize(t *testing.T) {
p = path.Join(dir, p)
_, err := os.Stat(p)
require.NoError(t, err)
bytes, err := ioutil.ReadFile(p)
bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, expected, string(bytes))
}
Expand Down Expand Up @@ -232,7 +231,7 @@ func TestWriteTableDataWithFileSizeAndRows(t *testing.T) {
p = path.Join(dir, p)
_, err = os.Stat(p)
require.NoError(t, err)
bytes, err := ioutil.ReadFile(p)
bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, expected, string(bytes))
}
Expand Down Expand Up @@ -281,7 +280,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
p = path.Join(config.OutputDirPath, p)
_, err = os.Stat(p)
require.NoError(t, err)
bytes, err1 := ioutil.ReadFile(p)
bytes, err1 := os.ReadFile(p)
require.NoError(t, err1)
require.Equal(t, expected, string(bytes))
}
Expand All @@ -297,7 +296,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
require.NoError(t, err)
err = os.RemoveAll(config.OutputDirPath)
require.NoError(t, err)
config.OutputDirPath, err = ioutil.TempDir("", "dumpling")
config.OutputDirPath, err = os.MkdirTemp("", "dumpling")

writer = createTestWriter(config, t)

Expand All @@ -322,7 +321,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
p = path.Join(config.OutputDirPath, p)
_, err = os.Stat(p)
require.NoError(t, err)
bytes, err := ioutil.ReadFile(p)
bytes, err := os.ReadFile(p)
require.NoError(t, err)
require.Equal(t, expected, string(bytes))
}
Expand All @@ -331,6 +330,8 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
var mu sync.Mutex

func createTestWriter(conf *Config, t *testing.T) *Writer {
t.Helper()

mu.Lock()
extStore, err := conf.createExternalStorage(context.Background())
mu.Unlock()
Expand Down