Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

*: index codec for common handle #16998

Merged
merged 8 commits into from May 9, 2020
Merged

*: index codec for common handle #16998

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
435052d
*: index codec for common handle
coocood May 6, 2020
87a8ff8
address comment
coocood May 7, 2020
0952f02
fix decode untouched index.
coocood May 7, 2020
2669054
fix check
coocood May 7, 2020
24bab77
update comment
coocood May 7, 2020
1e85ba2
Merge branch 'master' into common-handle-index
coocood May 7, 2020
99a50b6
Merge branch 'master' into common-handle-index
lysu May 9, 2020
55473b9
Merge branch 'master' into common-handle-index
coocood May 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 3 additions & 12 deletions executor/mem_reader.go
View file
Open in desktop
Expand Up @@ -110,7 +110,7 @@ func (m *memIndexReader) getMemRows() ([][]types.Datum, error) {
}

func (m *memIndexReader) decodeIndexKeyValue(key, value []byte, tps []*types.FieldType) ([]types.Datum, error) {
hdStatus := tablecodec.HandleIsSigned
hdStatus := tablecodec.HandleDefault
if mysql.HasUnsignedFlag(tps[len(tps)-1].Flag) {
hdStatus = tablecodec.HandleIsUnsigned
}
Expand Down Expand Up @@ -336,22 +336,13 @@ func reverseDatumSlice(rows [][]types.Datum) {
}

func (m *memIndexReader) getMemRowsHandle() ([]kv.Handle, error) {
pkTp := types.NewFieldType(mysql.TypeLonglong)
if m.table.PKIsHandle {
for _, col := range m.table.Columns {
if mysql.HasPriKeyFlag(col.Flag) {
pkTp = &col.FieldType
break
}
}
}
handles := make([]kv.Handle, 0, m.addedRowsLen)
err := iterTxnMemBuffer(m.ctx, m.kvRanges, func(key, value []byte) error {
handle, err := tablecodec.DecodeIndexHandle(key, value, len(m.index.Columns), pkTp)
handle, err := tablecodec.DecodeIndexHandle(key, value, len(m.index.Columns))
if err != nil {
return err
}
handles = append(handles, kv.IntHandle(handle))
handles = append(handles, handle)
return nil
})
if err != nil {
Expand Down
2 changes: 2 additions & 0 deletions store/mockstore/mocktikv/analyze.go
View file
Open in desktop
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/pingcap/parser/mysql"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/statistics"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/codec"
Expand Down Expand Up @@ -78,6 +79,7 @@ func (h *rpcHandler) handleAnalyzeIndexReq(req *coprocessor.Request, analyzeReq
mvccStore: h.mvccStore,
IndexScan: &tipb.IndexScan{Desc: false},
execDetail: new(execDetail),
hdStatus: tablecodec.HandleNotNeeded,
}
statsBuilder := statistics.NewSortedBuilder(flagsToStatementContext(analyzeReq.Flags), analyzeReq.IdxReq.BucketSize, 0, types.NewFieldType(mysql.TypeBlob))
var cms *statistics.CMSketch
Expand Down
9 changes: 4 additions & 5 deletions store/mockstore/mocktikv/cop_handler_dag.go
View file
Open in desktop
Expand Up @@ -243,17 +243,16 @@ func (h *rpcHandler) buildIndexScan(ctx *dagContext, executor *tipb.Executor) (*
columns := executor.IdxScan.Columns
ctx.evalCtx.setColumnInfo(columns)
length := len(columns)
pkStatus := tablecodec.HandleNotExists
hdStatus := tablecodec.HandleNotNeeded
// The PKHandle column info has been collected in ctx.
if columns[length-1].GetPkHandle() {
if mysql.HasUnsignedFlag(uint(columns[length-1].GetFlag())) {
pkStatus = tablecodec.HandleIsUnsigned
hdStatus = tablecodec.HandleIsUnsigned
} else {
pkStatus = tablecodec.HandleIsSigned
hdStatus = tablecodec.HandleDefault
}
columns = columns[:length-1]
} else if columns[length-1].ColumnId == model.ExtraHandleID {
pkStatus = tablecodec.HandleIsSigned
columns = columns[:length-1]
}
ranges, err := h.extractKVRanges(ctx.keyRanges, executor.IdxScan.Desc)
Expand Down Expand Up @@ -284,7 +283,7 @@ func (h *rpcHandler) buildIndexScan(ctx *dagContext, executor *tipb.Executor) (*
isolationLevel: h.isolationLevel,
resolvedLocks: h.resolvedLocks,
mvccStore: h.mvccStore,
hdStatus: pkStatus,
hdStatus: hdStatus,
execDetail: new(execDetail),
colInfos: colInfos,
}
Expand Down
75 changes: 64 additions & 11 deletions table/tables/index.go
View file
Open in desktop
Expand Up @@ -201,12 +201,16 @@ func (c *index) GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types.
key = c.getIndexKeyBuf(buf, len(c.prefix)+len(indexedValues)*9+9)
key = append(key, []byte(c.prefix)...)
key, err = codec.EncodeKey(sc, key, indexedValues...)
if !distinct && err == nil {
key, err = codec.EncodeKey(sc, key, types.NewDatum(h.IntValue()))
}
if err != nil {
return nil, false, err
}
if !distinct && h != nil {
if h.IsInt() {
key, err = codec.EncodeKey(sc, key, types.NewDatum(h.IntValue()))
} else {
key = append(key, h.Encoded()...)
}
}
return
}

Expand All @@ -232,7 +236,23 @@ func (c *index) GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types.
// | |
// | | The length >= 11 always because of padding.
// | |
// | +--Unique (TailLen = len(Handle) + len(Flag), TailLen == 8 || TailLen == 9)
// | +--Unique Common Handle
// | | |
// | | +--Without Untouched Flag:
// | | |
// | | | Layout: 0x00 | CHandle Flag | CHandle Len | CHandle | RestoreData
// | | | Length: 1 | 1 | 2 | size(CHandle) | size(RestoreData)
// | | |
// | | | The length > 10 always because of CHandle size.
// | | |
// | | +--With Untouched Flag:
// | |
// | | Layout: 0x01 | CHandle Flag | CHandle Len | CHandle | RestoreData | Flag
// | | Length: 1 | 1 | 2 | size(CHandle) | size(RestoreData) | 1
// | |
// | | The length > 10 always because of CHandle size.
// | |
// | +--Unique Integer Handle (TailLen = len(Handle) + len(Flag), TailLen == 8 || TailLen == 9)
// | |
// | +--Without Untouched Flag:
// | |
Expand All @@ -248,7 +268,7 @@ func (c *index) GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types.
// |
// | The length >= 11 always since size(RestoreData) > 0.
// |
// +--Without Restore Data(same with old layout)
// +--Without Restore Data
// |
// +--Non Unique
// | |
Expand All @@ -261,8 +281,19 @@ func (c *index) GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types.
// |
// | Layout: Flag
// | Length: 1
// +--Unique Common Handle
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

L271 Without Restore Data(same with old layout) the comment needs to refine.

// | |
// | +--Without Untouched Flag:
// | |
// | | Layout: 0x00 | CHandle Flag | CHandle Len | CHandle
// | | Length: 1 | 1 | 2 | size(CHandle)
// | |
// | +--With Untouched Flag:
// |
// | Layout: 0x01 | CHandle Flag | CHandle Len | CHandle | Flag
// | Length: 1 | 1 | 2 | size(CHandle) | 1
// |
// +--Unique
// +--Unique Integer Handle
// |
// +--Without Untouched Flag:
// |
Expand Down Expand Up @@ -314,10 +345,15 @@ func (c *index) Create(sctx sessionctx.Context, rm kv.RetrieverMutator, indexedV
if err != nil {
return nil, err
}
idxVal = make([]byte, 1+len(rowRestoredValue))
copy(idxVal[1:], rowRestoredValue)
// tailLen(1) + commonHandleFlag(1) + handleLen(2) + handle + restoredValue
idxValCap := 1 + 1 + 2 + h.Len() + len(rowRestoredValue)
idxVal = make([]byte, 1, idxValCap)
if !h.IsInt() && distinct {
idxVal = encodeCommonHandle(idxVal, h)
}
idxVal = append(idxVal, rowRestoredValue...)
tailLen := 0
if distinct {
if h.IsInt() && distinct {
// The len of the idxVal is always >= 10 since len (restoredValue) > 0.
tailLen += 8
idxVal = append(idxVal, EncodeHandleInUniqueIndexValue(h.IntValue())...)
Copy link
Contributor

@crazycs520 crazycs520 May 7, 2020 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should move this line before L354? The current layout is not consistent with the comment.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No.
It's consistent with the comment.
Integer handle is appended at the tail if it has restored value.

Expand All @@ -338,7 +374,16 @@ func (c *index) Create(sctx sessionctx.Context, rm kv.RetrieverMutator, indexedV
} else {
idxVal = make([]byte, 0)
if distinct {
idxVal = EncodeHandleInUniqueIndexValue(h.IntValue())
if h.IsInt() {
idxVal = EncodeHandleInUniqueIndexValue(h.IntValue())
} else {
if opt.Untouched {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it duplicate with L387?

Copy link
Member Author

@coocood coocood May 7, 2020 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No.
L379 append the tailLen, L387 appends the untouched flag.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add a comment for this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should move L380 ~ 384 before L377?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should move L380 ~ 384 before L377?

No, if the handle is int, there would be no tailLen at the first byte.

idxVal = append(idxVal, 1)
} else {
idxVal = append(idxVal, 0)
}
idxVal = encodeCommonHandle(idxVal, h)
}
}
if opt.Untouched {
// If index is untouched and fetch here means the key is exists in TiKV, but not in txn mem-buffer,
Expand Down Expand Up @@ -382,6 +427,14 @@ func (c *index) Create(sctx sessionctx.Context, rm kv.RetrieverMutator, indexedV
return kv.IntHandle(handle), kv.ErrKeyExists
}

func encodeCommonHandle(idxVal []byte, h kv.Handle) []byte {
idxVal = append(idxVal, tablecodec.CommonHandleFlag)
hLen := uint16(len(h.Encoded()))
idxVal = append(idxVal, byte(hLen>>8), byte(hLen))
idxVal = append(idxVal, h.Encoded()...)
return idxVal
}

// Delete removes the entry for handle h and indexdValues from KV index.
func (c *index) Delete(sc *stmtctx.StatementContext, m kv.Mutator, indexedValues []types.Datum, h kv.Handle) error {
key, _, err := c.GenIndexKey(sc, indexedValues, h, nil)
Expand Down Expand Up @@ -419,7 +472,7 @@ func (c *index) Drop(rm kv.RetrieverMutator) error {

// Seek searches KV index for the entry with indexedValues.
func (c *index) Seek(sc *stmtctx.StatementContext, r kv.Retriever, indexedValues []types.Datum) (iter table.IndexIterator, hit bool, err error) {
key, _, err := c.GenIndexKey(sc, indexedValues, kv.IntHandle(0), nil)
key, _, err := c.GenIndexKey(sc, indexedValues, nil, nil)
if err != nil {
return nil, false, err
}
Expand Down
142 changes: 141 additions & 1 deletion table/tables/index_test.go
View file
Open in desktop
Expand Up @@ -19,17 +19,25 @@ import (
"time"

. "github.com/pingcap/check"
"github.com/pingcap/parser"
"github.com/pingcap/parser/ast"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/parser/terror"
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/store/mockstore"
"github.com/pingcap/tidb/table"
"github.com/pingcap/tidb/table/tables"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/mock"
"github.com/pingcap/tidb/util/rowcodec"
"github.com/pingcap/tidb/util/testleak"
)

Expand Down Expand Up @@ -122,7 +130,7 @@ func (s *testIndexSuite) TestIndex(c *C) {

_, hit, err := index.Seek(sc, txn, values)
c.Assert(err, IsNil)
c.Assert(hit, IsTrue)
c.Assert(hit, IsFalse)
wjhuang2016 marked this conversation as resolved.
Show resolved Hide resolved

err = index.Drop(txn)
c.Assert(err, IsNil)
Expand Down Expand Up @@ -256,3 +264,135 @@ func (s *testIndexSuite) TestCombineIndexSeek(c *C) {
c.Assert(err, IsNil)
c.Assert(h.IntValue(), Equals, int64(1))
}

func (s *testIndexSuite) TestSingleColumnCommonHandle(c *C) {
tblInfo := buildTableInfo(c, "create table t (a varchar(255) primary key, u int unique, nu int, index nu (nu))")
var idxUnique, idxNonUnique table.Index
for _, idxInfo := range tblInfo.Indices {
idx := tables.NewIndex(tblInfo.ID, tblInfo, idxInfo)
if idxInfo.Name.L == "u" {
idxUnique = idx
} else if idxInfo.Name.L == "nu" {
idxNonUnique = idx
}
}
txn, err := s.s.Begin()
c.Assert(err, IsNil)

mockCtx := mock.NewContext()
sc := mockCtx.GetSessionVars().StmtCtx
// create index for "insert t values ('abc', 1, 1)"
idxColVals := types.MakeDatums(1)
handleColVals := types.MakeDatums("abc")
encodedHandle, err := codec.EncodeKey(sc, nil, handleColVals...)
c.Assert(err, IsNil)
commonHandle, err := kv.NewCommonHandle(encodedHandle)
c.Assert(err, IsNil)

for _, idx := range []table.Index{idxUnique, idxNonUnique} {
key, _, err := idx.GenIndexKey(sc, idxColVals, commonHandle, nil)
c.Assert(err, IsNil)
_, err = idx.Create(mockCtx, txn, idxColVals, commonHandle)
c.Assert(err, IsNil)
val, err := txn.Get(context.Background(), key)
c.Assert(err, IsNil)
colVals, err := tablecodec.DecodeIndexKV(key, val, 1, tablecodec.HandleDefault,
createRowcodecColInfo(tblInfo, idx.Meta()))
c.Assert(err, IsNil)
c.Assert(colVals, HasLen, 2)
_, d, err := codec.DecodeOne(colVals[0])
c.Assert(err, IsNil)
c.Assert(d.GetInt64(), Equals, int64(1))
_, d, err = codec.DecodeOne(colVals[1])
c.Assert(err, IsNil)
_, d, err = codec.DecodeOne(d.GetBytes())
c.Assert(err, IsNil)
c.Assert(d.GetString(), Equals, "abc")
handle, err := tablecodec.DecodeIndexHandle(key, val, 1)
c.Assert(err, IsNil)
c.Assert(handle.IsInt(), IsFalse)
c.Assert(handle.Encoded(), BytesEquals, commonHandle.Encoded())

unTouchedVal := append([]byte{1}, val[1:]...)
unTouchedVal = append(unTouchedVal, kv.UnCommitIndexKVFlag)
_, err = tablecodec.DecodeIndexKV(key, unTouchedVal, 1, tablecodec.HandleDefault,
createRowcodecColInfo(tblInfo, idx.Meta()))
c.Assert(err, IsNil)
}
}

func (s *testIndexSuite) TestMultiColumnCommonHandle(c *C) {
collate.SetNewCollationEnabledForTest(true)
defer collate.SetNewCollationEnabledForTest(false)
tblInfo := buildTableInfo(c, "create table t (a int, b int, u varchar(64) unique, nu varchar(64), primary key (a, b), index nu (nu))")
var idxUnique, idxNonUnique table.Index
for _, idxInfo := range tblInfo.Indices {
idx := tables.NewIndex(tblInfo.ID, tblInfo, idxInfo)
if idxInfo.Name.L == "u" {
idxUnique = idx
} else if idxInfo.Name.L == "nu" {
idxNonUnique = idx
}
}

txn, err := s.s.Begin()
c.Assert(err, IsNil)
mockCtx := mock.NewContext()
sc := mockCtx.GetSessionVars().StmtCtx
// create index for "insert t values (3, 2, "abc", "abc")
idxColVals := types.MakeDatums("abc")
handleColVals := types.MakeDatums(3, 2)
encodedHandle, err := codec.EncodeKey(sc, nil, handleColVals...)
c.Assert(err, IsNil)
commonHandle, err := kv.NewCommonHandle(encodedHandle)
c.Assert(err, IsNil)
_ = idxNonUnique
for _, idx := range []table.Index{idxUnique, idxNonUnique} {
key, _, err := idx.GenIndexKey(sc, idxColVals, commonHandle, nil)
c.Assert(err, IsNil)
_, err = idx.Create(mockCtx, txn, idxColVals, commonHandle)
c.Assert(err, IsNil)
val, err := txn.Get(context.Background(), key)
c.Assert(err, IsNil)
colVals, err := tablecodec.DecodeIndexKV(key, val, 1, tablecodec.HandleDefault,
createRowcodecColInfo(tblInfo, idx.Meta()))
c.Assert(err, IsNil)
c.Assert(colVals, HasLen, 2)
_, d, err := codec.DecodeOne(colVals[0])
c.Assert(err, IsNil)
c.Assert(d.GetString(), Equals, "abc")
_, d, err = codec.DecodeOne(colVals[1])
c.Assert(err, IsNil)
handleColVals2, err := codec.Decode(d.GetBytes(), 2)
c.Assert(err, IsNil)
c.Assert(handleColVals2, HasLen, 2)
c.Assert(handleColVals2[0].GetInt64(), Equals, int64(3))
c.Assert(handleColVals2[1].GetInt64(), Equals, int64(2))
handle, err := tablecodec.DecodeIndexHandle(key, val, 1)
c.Assert(err, IsNil)
c.Assert(handle.IsInt(), IsFalse)
c.Assert(handle.Encoded(), BytesEquals, commonHandle.Encoded())
}
}

func buildTableInfo(c *C, sql string) *model.TableInfo {
stmt, err := parser.New().ParseOneStmt(sql, "", "")
c.Assert(err, IsNil)
tblInfo, err := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
c.Assert(err, IsNil)
return tblInfo
}

func createRowcodecColInfo(table *model.TableInfo, index *model.IndexInfo) []rowcodec.ColInfo {
colInfos := make([]rowcodec.ColInfo, 0, len(index.Columns))
for _, idxCol := range index.Columns {
col := table.Columns[idxCol.Offset]
colInfos = append(colInfos, rowcodec.ColInfo{
ID: col.ID,
Tp: int32(col.Tp),
Flag: int32(col.Flag),
IsPKHandle: table.PKIsHandle && mysql.HasPriKeyFlag(col.Flag),
})
}
return colInfos
}