From b9846ead647ea8d78403f8ec32efdf4641992a8d Mon Sep 17 00:00:00 2001 From: leoppro Date: Mon, 27 Jul 2020 16:00:44 +0800 Subject: [PATCH 1/5] add some flags --- cdc/model/sink.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cdc/model/sink.go b/cdc/model/sink.go index e106e063af4..82fc6a8cf53 100644 --- a/cdc/model/sink.go +++ b/cdc/model/sink.go @@ -43,6 +43,14 @@ type ColumnFlagType util.Flag const ( // BinaryFlag means col charset is binary BinaryFlag ColumnFlagType = 1 << ColumnFlagType(iota) + // PrimaryKeyFlag means the column is primary key + PrimaryKeyFlag + // UniqueKeyFlag means the column is unique key + UniqueKeyFlag + // MultipleKeyFlag means the column is multiple key + MultipleKeyFlag + // NullableFlag means the column is nullable + NullableFlag ) //SetIsBinary set BinaryFlag From 19edf0afb3aba32d30b265040f73ad5918c777e8 Mon Sep 17 00:00:00 2001 From: leoppro Date: Mon, 27 Jul 2020 16:35:15 +0800 Subject: [PATCH 2/5] support more flag --- cdc/entry/mounter.go | 12 +++++++++ cdc/model/sink.go | 60 +++++++++++++++++++++++++++++++++++++++++ cdc/model/sink_test.go | 15 ++++++++++- cdc/sink/codec/canal.go | 2 +- 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/cdc/entry/mounter.go b/cdc/entry/mounter.go index 271250b8963..b7543acbd96 100644 --- a/cdc/entry/mounter.go +++ b/cdc/entry/mounter.go @@ -650,6 +650,18 @@ func transColumnFlag(col *timodel.ColumnInfo) model.ColumnFlagType { if col.Charset == "binary" { flag.SetIsBinary() } + if mysql.HasPriKeyFlag(col.Flag) { + flag.SetIsPrimaryKey() + } + if mysql.HasUniKeyFlag(col.Flag) { + flag.SetIsUniqueKey() + } + if !mysql.HasNotNullFlag(col.Flag) { + flag.SetIsNullable() + } + if mysql.HasMultipleKeyFlag(col.Flag) { + flag.SetIsMultipleKey() + } return flag } diff --git a/cdc/model/sink.go b/cdc/model/sink.go index 82fc6a8cf53..1eeea4557a7 100644 --- a/cdc/model/sink.go +++ b/cdc/model/sink.go @@ -68,6 +68,66 @@ func (b *ColumnFlagType) IsBinary() bool { return (*util.Flag)(b).HasAll(util.Flag(BinaryFlag)) } +//SetIsPrimaryKey set PrimaryKeyFlag +func (b *ColumnFlagType) SetIsPrimaryKey() { + (*util.Flag)(b).Add(util.Flag(PrimaryKeyFlag)) +} + +//UnsetIsPrimaryKey unset PrimaryKeyFlag +func (b *ColumnFlagType) UnsetIsPrimaryKey() { + (*util.Flag)(b).Remove(util.Flag(PrimaryKeyFlag)) +} + +//PrimaryKeyFlag show whether PrimaryKeyFlag is set +func (b *ColumnFlagType) IsPrimaryKey() bool { + return (*util.Flag)(b).HasAll(util.Flag(PrimaryKeyFlag)) +} + +//SetIsUniqueKey set UniqueKeyFlag +func (b *ColumnFlagType) SetIsUniqueKey() { + (*util.Flag)(b).Add(util.Flag(UniqueKeyFlag)) +} + +//UnsetIsUniqueKey unset UniqueKeyFlag +func (b *ColumnFlagType) UnsetIsUniqueKey() { + (*util.Flag)(b).Remove(util.Flag(UniqueKeyFlag)) +} + +//IsUniqueKey show whether UniqueKeyFlag is set +func (b *ColumnFlagType) IsUniqueKey() bool { + return (*util.Flag)(b).HasAll(util.Flag(UniqueKeyFlag)) +} + +//IsMultipleKey show whether MultipleKeyFlag is set +func (b *ColumnFlagType) IsMultipleKey() bool { + return (*util.Flag)(b).HasAll(util.Flag(MultipleKeyFlag)) +} + +//SetIsMultipleKey set MultipleKeyFlag +func (b *ColumnFlagType) SetIsMultipleKey() { + (*util.Flag)(b).Add(util.Flag(MultipleKeyFlag)) +} + +//UnsetIsMultipleKey unset MultipleKeyFlag +func (b *ColumnFlagType) UnsetIsMultipleKey() { + (*util.Flag)(b).Remove(util.Flag(MultipleKeyFlag)) +} + +//IsNullable show whether NullableFlag is set +func (b *ColumnFlagType) IsNullable() bool { + return (*util.Flag)(b).HasAll(util.Flag(NullableFlag)) +} + +//SetIsNullable set NullableFlag +func (b *ColumnFlagType) SetIsNullable() { + (*util.Flag)(b).Add(util.Flag(NullableFlag)) +} + +//UnsetIsNullable unset NullableFlag +func (b *ColumnFlagType) UnsetIsNullable() { + (*util.Flag)(b).Remove(util.Flag(NullableFlag)) +} + // TableName represents name of a table, includes table name and schema name. type TableName struct { Schema string `toml:"db-name" json:"db-name"` diff --git a/cdc/model/sink_test.go b/cdc/model/sink_test.go index bee71123df4..d7abad0b0dc 100644 --- a/cdc/model/sink_test.go +++ b/cdc/model/sink_test.go @@ -19,11 +19,24 @@ type columnFlagTypeSuite struct{} var _ = check.Suite(&columnFlagTypeSuite{}) -func (s *configSuite) TestBinaryFlag(c *check.C) { +func (s *configSuite) TestSetFlag(c *check.C) { var flag ColumnFlagType c.Assert(flag.IsBinary(), check.IsFalse) flag.SetIsBinary() c.Assert(flag.IsBinary(), check.IsTrue) flag.UnsetIsBinary() c.Assert(flag.IsBinary(), check.IsFalse) + flag.SetIsMultipleKey() + flag.SetIsUniqueKey() + c.Assert(flag.IsMultipleKey() && flag.IsUniqueKey(), check.IsTrue) + flag.UnsetIsUniqueKey() + c.Assert(flag.IsUniqueKey(), check.IsFalse) +} + +func (s *configSuite) TestFlagValue(c *check.C) { + c.Assert(BinaryFlag, check.Equals, ColumnFlagType(0b1)) + c.Assert(PrimaryKeyFlag, check.Equals, ColumnFlagType(0b10)) + c.Assert(UniqueKeyFlag, check.Equals, ColumnFlagType(0b100)) + c.Assert(MultipleKeyFlag, check.Equals, ColumnFlagType(0b1000)) + c.Assert(NullableFlag, check.Equals, ColumnFlagType(0b10000)) } diff --git a/cdc/sink/codec/canal.go b/cdc/sink/codec/canal.go index b39134a65bb..96fe8bdefd3 100644 --- a/cdc/sink/codec/canal.go +++ b/cdc/sink/codec/canal.go @@ -164,7 +164,7 @@ func (b *canalEntryBuilder) buildColumn(c *model.Column, colName string, updated } } - isKey := c.WhereHandle != nil && *c.WhereHandle + isKey := c.Flag.IsPrimaryKey() isNull := c.Value == nil value := "" if !isNull { From f4f2b8570ae2e5c416777d11e439936d58d96588 Mon Sep 17 00:00:00 2001 From: leoppro Date: Tue, 28 Jul 2020 10:31:22 +0800 Subject: [PATCH 3/5] Update cdc/model/sink.go Co-authored-by: Neil Shen --- cdc/model/sink.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdc/model/sink.go b/cdc/model/sink.go index 1eeea4557a7..cc8637380be 100644 --- a/cdc/model/sink.go +++ b/cdc/model/sink.go @@ -42,7 +42,7 @@ type ColumnFlagType util.Flag const ( // BinaryFlag means col charset is binary - BinaryFlag ColumnFlagType = 1 << ColumnFlagType(iota) + BinaryFlag ColumnFlagType = uint64(1) << ColumnFlagType(iota) // PrimaryKeyFlag means the column is primary key PrimaryKeyFlag // UniqueKeyFlag means the column is unique key From 127ce1d3bad1478f8e01c9eba1ae06ce54a37a00 Mon Sep 17 00:00:00 2001 From: leoppro Date: Tue, 4 Aug 2020 11:52:49 +0800 Subject: [PATCH 4/5] fix test --- cdc/model/sink.go | 2 +- cdc/model/sink_test.go | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cdc/model/sink.go b/cdc/model/sink.go index 5d46dffdd4c..cd6b65b6403 100644 --- a/cdc/model/sink.go +++ b/cdc/model/sink.go @@ -112,7 +112,7 @@ func (b *ColumnFlagType) UnsetIsPrimaryKey() { (*util.Flag)(b).Remove(util.Flag(PrimaryKeyFlag)) } -//PrimaryKeyFlag show whether PrimaryKeyFlag is set +//IsPrimaryKey show whether PrimaryKeyFlag is set func (b *ColumnFlagType) IsPrimaryKey() bool { return (*util.Flag)(b).HasAll(util.Flag(PrimaryKeyFlag)) } diff --git a/cdc/model/sink_test.go b/cdc/model/sink_test.go index 2e72028da40..8a7d860b87f 100644 --- a/cdc/model/sink_test.go +++ b/cdc/model/sink_test.go @@ -37,8 +37,10 @@ func (s *configSuite) TestSetFlag(c *check.C) { func (s *configSuite) TestFlagValue(c *check.C) { c.Assert(BinaryFlag, check.Equals, ColumnFlagType(0b1)) - c.Assert(PrimaryKeyFlag, check.Equals, ColumnFlagType(0b10)) - c.Assert(UniqueKeyFlag, check.Equals, ColumnFlagType(0b100)) - c.Assert(MultipleKeyFlag, check.Equals, ColumnFlagType(0b1000)) - c.Assert(NullableFlag, check.Equals, ColumnFlagType(0b10000)) + c.Assert(HandleKeyFlag, check.Equals, ColumnFlagType(0b10)) + c.Assert(GeneratedColumnFlag, check.Equals, ColumnFlagType(0b100)) + c.Assert(PrimaryKeyFlag, check.Equals, ColumnFlagType(0b1000)) + c.Assert(UniqueKeyFlag, check.Equals, ColumnFlagType(0b10000)) + c.Assert(MultipleKeyFlag, check.Equals, ColumnFlagType(0b100000)) + c.Assert(NullableFlag, check.Equals, ColumnFlagType(0b1000000)) } From cfc63de7575a859281810f66794c4ac8088c1b66 Mon Sep 17 00:00:00 2001 From: leoppro Date: Tue, 4 Aug 2020 12:08:42 +0800 Subject: [PATCH 5/5] fix bug --- cdc/sink/codec/canal_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdc/sink/codec/canal_test.go b/cdc/sink/codec/canal_test.go index 645c757d4bd..6a7f143287a 100644 --- a/cdc/sink/codec/canal_test.go +++ b/cdc/sink/codec/canal_test.go @@ -129,7 +129,6 @@ type canalEntrySuite struct{} var _ = check.Suite(&canalEntrySuite{}) func (s *canalEntrySuite) TestConvertEntry(c *check.C) { - trueVar := true testCaseUpdate := &model.RowChangedEvent{ CommitTs: 417318403368288260, Table: &model.TableName{ @@ -138,7 +137,7 @@ func (s *canalEntrySuite) TestConvertEntry(c *check.C) { }, Delete: false, Columns: map[string]*model.Column{ - "id": {Type: mysql.TypeLong, WhereHandle: &trueVar, Value: 1}, + "id": {Type: mysql.TypeLong, Flag: model.PrimaryKeyFlag, Value: 1}, "name": {Type: mysql.TypeVarchar, Value: "Bob"}, "tiny": {Type: mysql.TypeTiny, Value: 255}, "comment": {Type: mysql.TypeBlob, Value: []byte("测试")}, @@ -153,7 +152,7 @@ func (s *canalEntrySuite) TestConvertEntry(c *check.C) { }, Delete: true, PreColumns: map[string]*model.Column{ - "id": {Type: mysql.TypeLong, WhereHandle: &trueVar, Value: 1}, + "id": {Type: mysql.TypeLong, Flag: model.PrimaryKeyFlag, Value: 1}, }, } testCaseDdl := &model.DDLEvent{