Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

test(ticdc): refine test case data col type #6285

Merged
merged 6 commits into from
Jul 15, 2022
Merged

test(ticdc): refine test case data col type #6285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b254fa7
sink(ticdc): refine test case data col type
Rustin170506 Jul 15, 2022
c5db93e
sink(ticdc): make lint happy
Rustin170506 Jul 15, 2022
adba2b7
sink(ticdc): fix test
Rustin170506 Jul 15, 2022
ef9502f
Merge branch 'master' into rustin-patch-sink-test-refine
ti-chi-bot Jul 15, 2022
8454f06
Merge branch 'master' into rustin-patch-sink-test-refine
ti-chi-bot Jul 15, 2022
41ca14b
Merge branch 'master' into rustin-patch-sink-test-refine
ti-chi-bot Jul 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cdc/sink/mq/codec/canal_encoder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"testing"

"github.com/golang/protobuf/proto"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tiflow/cdc/model"
canal "github.com/pingcap/tiflow/proto/canal"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -83,7 +84,11 @@ func TestCanalAppendRowChangedEventWithCallback(t *testing.T) {
row := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

tests := []struct {
Expand Down
7 changes: 6 additions & 1 deletion cdc/sink/mq/codec/canal_json_encoder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"encoding/json"
"testing"

"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tiflow/cdc/model"
"github.com/pingcap/tiflow/pkg/config"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -309,7 +310,11 @@ func TestCanalJSONAppendRowChangedEventWithCallback(t *testing.T) {
row := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

tests := []struct {
Expand Down
12 changes: 10 additions & 2 deletions cdc/sink/mq/codec/canal_test_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,13 +363,21 @@ var (
{{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}},
{
{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
{
CommitTs: 2,
Expand Down
18 changes: 15 additions & 3 deletions cdc/sink/mq/codec/craft_encoder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,11 @@ func TestCraftMaxMessageBytes(t *testing.T) {
testEvent := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: mysql.TypeVarchar, Value: []byte("aa")}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

for i := 0; i < 10000; i++ {
Expand All @@ -54,7 +58,11 @@ func TestCraftMaxBatchSize(t *testing.T) {
testEvent := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: mysql.TypeVarchar, Value: []byte("aa")}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

for i := 0; i < 10000; i++ {
Expand Down Expand Up @@ -211,7 +219,11 @@ func TestCraftAppendRowChangedEventWithCallback(t *testing.T) {
row := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: mysql.TypeVarchar, Value: []byte("aa")}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

tests := []struct {
Expand Down
7 changes: 6 additions & 1 deletion cdc/sink/mq/codec/maxwell_encoder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"context"
"testing"

"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tiflow/cdc/model"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -72,7 +73,11 @@ func TestMaxwellAppendRowChangedEventWithCallback(t *testing.T) {
row := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

tests := []struct {
Expand Down
21 changes: 17 additions & 4 deletions cdc/sink/mq/codec/open_protocol_encoder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"sort"
"testing"

"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tiflow/cdc/model"
"github.com/pingcap/tiflow/pkg/config"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -170,13 +171,17 @@ func TestMaxMessageBytes(t *testing.T) {
testEvent := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

ctx := context.Background()
topic := ""
// for a single message, the overhead is 36(maxRecordOverhead) + 8(versionHea) = 44, just can hold it.
a := 87 + 44
a := 88 + 44
config := NewConfig(config.ProtocolOpen).WithMaxMessageBytes(a)
encoder := newOpenProtocolBatchEncoderBuilder(config).Build()
err := encoder.AppendRowChangedEvent(ctx, topic, testEvent, nil)
Expand Down Expand Up @@ -211,7 +216,11 @@ func TestMaxBatchSize(t *testing.T) {
testEvent := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

for i := 0; i < 10000; i++ {
Expand Down Expand Up @@ -259,7 +268,11 @@ func TestOpenProtocolAppendRowChangedEventWithCallback(t *testing.T) {
row := &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

tests := []struct {
Expand Down
61 changes: 51 additions & 10 deletions cdc/sink/mq/mq_flush_worker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"testing"

"github.com/pingcap/errors"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tiflow/cdc/model"
"github.com/pingcap/tiflow/cdc/sink/metrics"
"github.com/pingcap/tiflow/cdc/sink/mq/codec"
Expand Down Expand Up @@ -125,7 +126,11 @@ func TestBatch(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: key,
},
Expand Down Expand Up @@ -159,7 +164,11 @@ func TestBatch(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: key,
},
Expand Down Expand Up @@ -232,7 +241,11 @@ func TestGroup(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: key1,
},
Expand Down Expand Up @@ -310,7 +323,11 @@ func TestAsyncSend(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: key1,
},
Expand All @@ -334,15 +351,23 @@ func TestAsyncSend(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 2,
Table: &model.TableName{Schema: "aa", Table: "bb"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: key2,
},
{
row: &model.RowChangedEvent{
CommitTs: 2,
Table: &model.TableName{Schema: "aaa", Table: "bbb"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: key3,
},
Expand Down Expand Up @@ -384,7 +409,11 @@ func TestFlush(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: key1,
},
Expand Down Expand Up @@ -489,7 +518,11 @@ func TestProducerError(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: topicPartitionKey{
topic: "test",
Expand Down Expand Up @@ -521,7 +554,11 @@ func TestWorker(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 1,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: topicPartitionKey{
topic: "test",
Expand All @@ -533,7 +570,11 @@ func TestWorker(t *testing.T) {
row: &model.RowChangedEvent{
CommitTs: 300,
Table: &model.TableName{Schema: "a", Table: "b"},
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
},
key: topicPartitionKey{
topic: "test",
Expand Down
25 changes: 21 additions & 4 deletions cdc/sink/mq/mq_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/Shopify/sarama"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/parser/mysql"
"github.com/pingcap/tiflow/cdc/model"
"github.com/pingcap/tiflow/cdc/sink/mq/codec"
kafkap "github.com/pingcap/tiflow/cdc/sink/mq/producer/kafka"
Expand Down Expand Up @@ -106,7 +107,11 @@ func TestKafkaSink(t *testing.T) {
},
StartTs: 100,
CommitTs: 120,
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}
err = sink.EmitRowChangedEvents(ctx, row)
require.Nil(t, err)
Expand Down Expand Up @@ -225,7 +230,11 @@ func TestFlushRowChangedEvents(t *testing.T) {
},
StartTs: 100,
CommitTs: 120,
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}
err = sink.EmitRowChangedEvents(ctx, row1)
require.Nil(t, err)
Expand All @@ -239,7 +248,11 @@ func TestFlushRowChangedEvents(t *testing.T) {
},
StartTs: 90,
CommitTs: 110,
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}
err = sink.EmitRowChangedEvents(ctx, row2)
require.Nil(t, err)
Expand All @@ -253,7 +266,11 @@ func TestFlushRowChangedEvents(t *testing.T) {
},
StartTs: 110,
CommitTs: 130,
Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}},
Columns: []*model.Column{{
Name: "col1",
Type: mysql.TypeVarchar,
Value: []byte("aa"),
}},
}

err = sink.EmitRowChangedEvents(ctx, row3)
Expand Down