table, executor: let SplitIndexValue work with clustered index tabl…

…e. (#53254)

close #49995

pkg/executor/batch_point_get.go

@@ -297,8 +297,7 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error {
 			if e.tblInfo.Partition != nil {
 				var pid int64
 				if e.idxInfo.Global {
-					segs := tablecodec.SplitIndexValue(handleVal)
-					_, pid, err = codec.DecodeInt(segs.PartitionID)
+					_, pid, err = codec.DecodeInt(tablecodec.SplitIndexValue(handleVal).PartitionID)
 					if err != nil {
 						return err
 					}

pkg/executor/mem_reader.go

@@ -982,8 +982,10 @@ func (iter *memRowsIterForIndex) Next() ([]types.Datum, error) {
 
 		// filter key/value by partitition id
 		if iter.index.Global {
-			seg := tablecodec.SplitIndexValue(value)
-			_, pid, _ := codec.DecodeInt(seg.PartitionID)
+			_, pid, err := codec.DecodeInt(tablecodec.SplitIndexValue(value).PartitionID)
+			if err != nil {
+				return nil, err
+			}
 			if _, exists := iter.partitionIDMap[pid]; !exists {
 				continue
 			}

pkg/executor/point_get.go

@@ -349,8 +349,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error {
 				failpoint.InjectContext(ctx, "pointGetRepeatableReadTest-step2", nil)
 			})
 			if e.idxInfo.Global {
-				segs := tablecodec.SplitIndexValue(e.handleVal)
-				_, pid, err := codec.DecodeInt(segs.PartitionID)
+				_, pid, err := codec.DecodeInt(tablecodec.SplitIndexValue(e.handleVal).PartitionID)
 				if err != nil {
 					return err
 				}

pkg/table/tables/index.go

@@ -28,7 +28,6 @@ import (
 	"github.com/pingcap/tidb/pkg/tablecodec"
 	"github.com/pingcap/tidb/pkg/types"
 	"github.com/pingcap/tidb/pkg/util"
-	"github.com/pingcap/tidb/pkg/util/codec"
 	"github.com/pingcap/tidb/pkg/util/rowcodec"
 	"github.com/pingcap/tidb/pkg/util/tracing"
 )
@@ -406,25 +405,8 @@ func (c *index) Delete(ctx table.MutateContext, txn kv.Transaction, indexedValue
 				return err
 			}
 		}
-		tempValElem := tablecodec.TempIndexValueElem{Handle: h, KeyVer: tempKeyVer, Delete: true, Distinct: distinct}
-
-		// If index is global, decode the pid from value (if exists) and compare with c.physicalID.
-		// Only when pid in value equals to c.physicalID, the key can be deleted.
-		if c.idxInfo.Global {
-			if val, err := txn.GetMemBuffer().Get(context.Background(), key); err == nil {
-				segs := tablecodec.SplitIndexValue(val)
-				if len(segs.PartitionID) != 0 {
-					_, pid, err := codec.DecodeInt(segs.PartitionID)
-					if err != nil {
-						return err
-					}
-					if pid != c.phyTblID {
-						continue
-					}
-				}
-			}
-		}
 
+		tempValElem := tablecodec.TempIndexValueElem{Handle: h, KeyVer: tempKeyVer, Delete: true, Distinct: distinct}
 		if distinct {
 			if len(key) > 0 {
 				okToDelete := true

pkg/tablecodec/tablecodec.go

@@ -978,18 +978,7 @@ func decodeHandleInIndexKey(keySuffix []byte) (kv.Handle, error) {
 }
 
 func decodeHandleInIndexValue(value []byte) (handle kv.Handle, err error) {
-	var seg IndexValueSegments
-	if getIndexVersion(value) == 0 {
-		// For Old Encoding (IntHandle without any others options)
-		if len(value) <= MaxOldEncodeValueLen {
-			return decodeIntHandleInIndexValue(value), nil
-		}
-		// For IndexValueVersion0
-		seg = SplitIndexValue(value)
-	} else {
-		// For IndexValueForClusteredIndexVersion1
-		seg = SplitIndexValueForClusteredIndexVersion1(value)
-	}
+	seg := SplitIndexValue(value)
 	if len(seg.IntHandle) != 0 {
 		handle = decodeIntHandleInIndexValue(seg.IntHandle)
 	}
@@ -1697,7 +1686,7 @@ func DecodeHandleInUniqueIndexValue(data []byte, isCommonHandle bool) (kv.Handle
 		return kv.IntHandle(int64(binary.BigEndian.Uint64(data[dLen-int(data[0]):]))), nil
 	}
 	if getIndexVersion(data) == 1 {
-		seg := SplitIndexValueForClusteredIndexVersion1(data)
+		seg := splitIndexValueForClusteredIndexVersion1(data)
 		h, err := kv.NewCommonHandle(seg.CommonHandle)
 		if err != nil {
 			return nil, err
@@ -1730,8 +1719,23 @@ type IndexValueSegments struct {
 	IntHandle      []byte
 }
 
-// SplitIndexValue splits index value into segments.
+// SplitIndexValue decodes segments in index value for both non-clustered and clustered table.
 func SplitIndexValue(value []byte) (segs IndexValueSegments) {
+	if getIndexVersion(value) == 0 {
+		// For Old Encoding (IntHandle without any others options)
+		if len(value) <= MaxOldEncodeValueLen {
+			segs.IntHandle = value
+			return segs
+		}
+		// For IndexValueVersion0
+		return splitIndexValueForIndexValueVersion0(value)
+	}
+	// For IndexValueForClusteredIndexVersion1
+	return splitIndexValueForClusteredIndexVersion1(value)
+}
+
+// splitIndexValueForIndexValueVersion0 splits index value into segments.
+func splitIndexValueForIndexValueVersion0(value []byte) (segs IndexValueSegments) {
 	tailLen := int(value[0])
 	tail := value[len(value)-tailLen:]
 	value = value[1 : len(value)-tailLen]
@@ -1754,8 +1758,8 @@ func SplitIndexValue(value []byte) (segs IndexValueSegments) {
 	return
 }
 
-// SplitIndexValueForClusteredIndexVersion1 splits index value into segments.
-func SplitIndexValueForClusteredIndexVersion1(value []byte) (segs IndexValueSegments) {
+// splitIndexValueForClusteredIndexVersion1 splits index value into segments.
+func splitIndexValueForClusteredIndexVersion1(value []byte) (segs IndexValueSegments) {
 	tailLen := int(value[0])
 	// Skip the tailLen and version info.
 	value = value[3 : len(value)-tailLen]
@@ -1780,7 +1784,7 @@ func decodeIndexKvForClusteredIndexVersion1(key, value []byte, colsLen int, hdSt
 	var keySuffix []byte
 	var handle kv.Handle
 	var err error
-	segs := SplitIndexValueForClusteredIndexVersion1(value)
+	segs := splitIndexValueForClusteredIndexVersion1(value)
 	resultValues, keySuffix, err = CutIndexKeyNew(key, colsLen)
 	if err != nil {
 		return nil, err
@@ -1830,7 +1834,7 @@ func decodeIndexKvGeneral(key, value []byte, colsLen int, hdStatus HandleStatus,
 	var keySuffix []byte
 	var handle kv.Handle
 	var err error
-	segs := SplitIndexValue(value)
+	segs := splitIndexValueForIndexValueVersion0(value)
 	resultValues, keySuffix, err = CutIndexKeyNew(key, colsLen)
 	if err != nil {
 		return nil, err
@@ -1887,10 +1891,10 @@ func IndexKVIsUnique(value []byte) bool {
 		return len(value) == 8
 	}
 	if getIndexVersion(value) == 1 {
-		segs := SplitIndexValueForClusteredIndexVersion1(value)
+		segs := splitIndexValueForClusteredIndexVersion1(value)
 		return segs.CommonHandle != nil
 	}
-	segs := SplitIndexValue(value)
+	segs := splitIndexValueForIndexValueVersion0(value)
 	return segs.IntHandle != nil || segs.CommonHandle != nil
 }
 

tests/integrationtest/r/globalindex/mem_index_lookup.result

@@ -60,58 +60,58 @@ rollback;
 # CommonHandle
 drop table if exists t;
 CREATE TABLE `t` (
-`a` year(4) primary key,
+`a` year(4) primary key clustered,
 `b` int(11) DEFAULT NULL,
+`c` int(11) DEFAULT NULL,
 UNIQUE KEY `idx1` (`b`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
 PARTITION BY HASH (`a`) PARTITIONS 5;
-insert into t values (2001, 2), (2002, 3), (2003, 4), (2004, 5);
+insert into t(a, b) values (2001, 2), (2002, 3), (2003, 4), (2004, 5);
 begin;
-insert into t values (2005, 1);
+insert into t(a, b) values (2005, 1);
 explain select * from t use index(idx1) where b > 2;
 id	estRows	task	access object	operator info
-Projection_5	3333.33	root		globalindex__mem_index_lookup.t.a, globalindex__mem_index_lookup.t.b
+Projection_5	3333.33	root		globalindex__mem_index_lookup.t.a, globalindex__mem_index_lookup.t.b, globalindex__mem_index_lookup.t.c
 └─UnionScan_6	3333.33	root		gt(globalindex__mem_index_lookup.t.b, 2)
   └─IndexLookUp_10	3333.33	root	partition:all	
     ├─IndexRangeScan_7(Build)	3333.33	cop[tikv]	table:t, index:idx1(b)	range:(2,+inf], keep order:false, stats:pseudo
     └─TableRowIDScan_8(Probe)	3333.33	cop[tikv]	table:t	keep order:false, stats:pseudo
 select * from t use index(idx1) where b > 2;
-a	b
-2002	3
-2003	4
-2004	5
+a	b	c
+2002	3	NULL
+2003	4	NULL
+2004	5	NULL
 explain select * from t partition(p0) use index(idx1) where b <= 2;
 id	estRows	task	access object	operator info
-Projection_5	3323.33	root	NULL	globalindex__mem_index_lookup.t.a, globalindex__mem_index_lookup.t.b
+Projection_5	3323.33	root	NULL	globalindex__mem_index_lookup.t.a, globalindex__mem_index_lookup.t.b, globalindex__mem_index_lookup.t.c
 └─UnionScan_6	3323.33	root	NULL	le(globalindex__mem_index_lookup.t.b, 2)
   └─IndexLookUp_11	3323.33	root	partition:p0	NULL
     ├─Selection_10(Build)	3323.33	cop[tikv]	NULL	in(_tidb_pid, pid0)
     │ └─IndexRangeScan_7	3323.33	cop[tikv]	table:t, index:idx1(b)	range:[-inf,2], keep order:false, stats:pseudo
     └─TableRowIDScan_8(Probe)	3323.33	cop[tikv]	table:t	keep order:false, stats:pseudo
 select * from t partition(p0) use index(idx1) where b <= 2;
-a	b
-2005	1
+a	b	c
+2005	1	NULL
 explain select * from t partition(p1) use index(idx1) where b <= 2 and a = 2010;
 id	estRows	task	access object	operator info
-Projection_5	0.33	root		globalindex__mem_index_lookup.t.a, globalindex__mem_index_lookup.t.b
+Projection_5	0.33	root		globalindex__mem_index_lookup.t.a, globalindex__mem_index_lookup.t.b, globalindex__mem_index_lookup.t.c
 └─UnionScan_6	0.33	root		eq(globalindex__mem_index_lookup.t.a, 2010), le(globalindex__mem_index_lookup.t.b, 2)
-  └─IndexLookUp_12	0.33	root	partition:dual	
-    ├─Selection_10(Build)	3323.33	cop[tikv]		in(_tidb_pid, dual)
+  └─IndexLookUp_11	0.33	root	partition:dual	
+    ├─Selection_10(Build)	0.33	cop[tikv]		eq(globalindex__mem_index_lookup.t.a, 2010), in(_tidb_pid, dual)
     │ └─IndexRangeScan_7	3323.33	cop[tikv]	table:t, index:idx1(b)	range:[-inf,2], keep order:false, stats:pseudo
-    └─Selection_11(Probe)	0.33	cop[tikv]		eq(globalindex__mem_index_lookup.t.a, 2010)
-      └─TableRowIDScan_8	3323.33	cop[tikv]	table:t	keep order:false, stats:pseudo
+    └─TableRowIDScan_8(Probe)	0.33	cop[tikv]	table:t	keep order:false, stats:pseudo
 select * from t partition(p1) use index(idx1) where b <= 2 and a = 2010;
-a	b
+a	b	c
 explain select * from t partition(p0, p1) use index(idx1) where b <= 2;
 id	estRows	task	access object	operator info
-Projection_5	3323.33	root	NULL	globalindex__mem_index_lookup.t.a, globalindex__mem_index_lookup.t.b
+Projection_5	3323.33	root	NULL	globalindex__mem_index_lookup.t.a, globalindex__mem_index_lookup.t.b, globalindex__mem_index_lookup.t.c
 └─UnionScan_6	3323.33	root	NULL	le(globalindex__mem_index_lookup.t.b, 2)
   └─IndexLookUp_11	3323.33	root	partition:p0,p1	NULL
     ├─Selection_10(Build)	3323.33	cop[tikv]	NULL	in(_tidb_pid, pid0, pid1)
     │ └─IndexRangeScan_7	3323.33	cop[tikv]	table:t, index:idx1(b)	range:[-inf,2], keep order:false, stats:pseudo
     └─TableRowIDScan_8(Probe)	3323.33	cop[tikv]	table:t	keep order:false, stats:pseudo
 select * from t partition(p0, p1) use index(idx1) where b <= 2;
-a	b
-2001	2
-2005	1
+a	b	c
+2001	2	NULL
+2005	1	NULL
 rollback;

tests/integrationtest/r/globalindex/mem_index_merge.result

@@ -87,7 +87,7 @@ KEY `idx_bc` (`b`,`c`),
 UNIQUE KEY `uidx_a` (`a`),
 UNIQUE KEY `uidx_ac` (`a`, `c`),
 KEY `idx_c` (`c`),
-PRIMARY KEY(`d`, `c`)
+PRIMARY KEY(`d`, `c`) clustered
 ) PARTITION BY HASH (`d`) PARTITIONS 5;
 insert into tpk2 values (1, 2, 1, 1), (3, 6, 3, 3);
 begin;
@@ -113,10 +113,11 @@ explain select /*+ use_index_merge(tpk2, uidx_a, idx_c) */ * from tpk2 where a >
 id	estRows	task	access object	operator info
 Projection_5	1111.11	root		globalindex__mem_index_merge.tpk2.a, globalindex__mem_index_merge.tpk2.b, globalindex__mem_index_merge.tpk2.c, globalindex__mem_index_merge.tpk2.d
 └─UnionScan_6	1111.11	root		gt(globalindex__mem_index_merge.tpk2.a, 1), gt(globalindex__mem_index_merge.tpk2.c, 1)
-  └─IndexMerge_11	1111.11	root	partition:all	type: intersection
-    ├─IndexRangeScan_7(Build)	3333.33	cop[tikv]	table:tpk2, index:uidx_a(a)	range:(1,+inf], keep order:false, stats:pseudo
-    ├─IndexRangeScan_9(Build)	3333.33	cop[tikv]	table:tpk2, index:idx_c(c)	range:(1,+inf], keep order:false, stats:pseudo
-    └─TableRowIDScan_10(Probe)	1111.11	cop[tikv]	table:tpk2	keep order:false, stats:pseudo
+  └─IndexMerge_12	1111.11	root	partition:all	type: intersection
+    ├─Selection_9(Build)	1111.11	cop[tikv]		gt(globalindex__mem_index_merge.tpk2.c, 1)
+    │ └─IndexRangeScan_7	3333.33	cop[tikv]	table:tpk2, index:uidx_a(a)	range:(1,+inf], keep order:false, stats:pseudo
+    ├─IndexRangeScan_10(Build)	3333.33	cop[tikv]	table:tpk2, index:idx_c(c)	range:(1,+inf], keep order:false, stats:pseudo
+    └─TableRowIDScan_11(Probe)	1111.11	cop[tikv]	table:tpk2	keep order:false, stats:pseudo
 select /*+ use_index_merge(tpk2, uidx_a, idx_c) */ * from tpk2 where a > 1 and c > 1;
 a	b	c	d
 2	4	2	2
@@ -145,7 +146,7 @@ id	estRows	task	access object	operator info
 Projection_5	1111.11	root	NULL	globalindex__mem_index_merge.tpk2.a, globalindex__mem_index_merge.tpk2.b, globalindex__mem_index_merge.tpk2.c, globalindex__mem_index_merge.tpk2.d
 └─UnionScan_6	1111.11	root	NULL	gt(globalindex__mem_index_merge.tpk2.a, 1), gt(globalindex__mem_index_merge.tpk2.c, 1)
   └─IndexMerge_12	1111.11	root	partition:p1	type: intersection
-    ├─Selection_9(Build)	3333.33	cop[tikv]	NULL	in(_tidb_pid, pid1)
+    ├─Selection_9(Build)	1111.11	cop[tikv]	NULL	gt(globalindex__mem_index_merge.tpk2.c, 1), in(_tidb_pid, pid1)
     │ └─IndexRangeScan_7	3333.33	cop[tikv]	table:tpk2, index:uidx_a(a)	range:(1,+inf], keep order:false, stats:pseudo
     ├─IndexRangeScan_10(Build)	3333.33	cop[tikv]	table:tpk2, index:idx_c(c)	range:(1,+inf], keep order:false, stats:pseudo
     └─TableRowIDScan_11(Probe)	1111.11	cop[tikv]	table:tpk2	keep order:false, stats:pseudo

tests/integrationtest/r/globalindex/mem_index_reader.result

@@ -46,7 +46,7 @@ rollback;
 # CommonHandle
 drop table if exists t;
 CREATE TABLE `t` (
-`a` year(4) primary key,
+`a` year(4) primary key CLUSTERED,
 `b` int(11) DEFAULT NULL,
 UNIQUE KEY `idx1` (`b`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin

tests/integrationtest/r/globalindex/point_get.result

@@ -0,0 +1,65 @@
+set tidb_enable_global_index=true;
+drop table if exists pt;
+# Non-clustered index table
+create table pt (a int, b int, c int, d int default 0, primary key (a, b) nonclustered, unique key uidx(c))
+partition by range(a) (
+PARTITION p0 VALUES LESS THAN (3),
+PARTITION p1 VALUES LESS THAN (6),
+PARTITION p2 VALUES LESS THAN (9),
+PARTITION p3 VALUES LESS THAN (20)
+);
+insert into pt(a,b,c) values(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5), (6,6,6), (7,7,7), (8,8,8), (9,9,9), (10,10,10);
+analyze table pt;
+# Test PointGet
+explain select c from pt where c = 1;
+id	estRows	task	access object	operator info
+Point_Get_1	1.00	root	table:pt, index:uidx(c)	
+select c from pt where c = 1;
+c
+1
+explain select c from pt partition(p1) where c = 1;
+id	estRows	task	access object	operator info
+Point_Get_1	1.00	root	table:pt, index:uidx(c)	
+select c from pt partition(p1) where c = 1;
+c
+# Test BatchPointGet
+explain select c from pt where c in (1,2,3);
+id	estRows	task	access object	operator info
+Batch_Point_Get_1	3.00	root	table:pt, index:uidx(c)	keep order:false, desc:false
+select * from pt where c in (1,2,3);
+a	b	c	d
+1	1	1	0
+2	2	2	0
+3	3	3	0
+drop table if exists pt;
+# Clustered index table
+create table pt (a int, b int, c int, d int default 0, primary key (a, b) clustered, unique key uidx(c))
+partition by range(a) (
+PARTITION p0 VALUES LESS THAN (3),
+PARTITION p1 VALUES LESS THAN (6),
+PARTITION p2 VALUES LESS THAN (9),
+PARTITION p3 VALUES LESS THAN (20)
+);
+insert into pt(a,b,c) values(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5), (6,6,6), (7,7,7), (8,8,8), (9,9,9), (10,10,10);
+analyze table pt;
+# Test PointGet
+explain select c from pt where c = 1;
+id	estRows	task	access object	operator info
+Point_Get_1	1.00	root	table:pt, index:uidx(c)	
+select c from pt where c = 1;
+c
+1
+explain select c from pt partition(p1) where c = 1;
+id	estRows	task	access object	operator info
+Point_Get_1	1.00	root	table:pt, index:uidx(c)	
+select c from pt partition(p1) where c = 1;
+c
+# Test BatchPointGet
+explain select c from pt where c in (1,2,3);
+id	estRows	task	access object	operator info
+Batch_Point_Get_1	3.00	root	table:pt, index:uidx(c)	keep order:false, desc:false
+select * from pt where c in (1,2,3);
+a	b	c	d
+1	1	1	0
+2	2	2	0
+3	3	3	0

tests/integrationtest/t/globalindex/mem_index_lookup.test

@@ -37,16 +37,17 @@ rollback;
 --echo # CommonHandle
 drop table if exists t;
 CREATE TABLE `t` (
-  `a` year(4) primary key,
+  `a` year(4) primary key clustered,
   `b` int(11) DEFAULT NULL,
+  `c` int(11) DEFAULT NULL,
   UNIQUE KEY `idx1` (`b`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
 PARTITION BY HASH (`a`) PARTITIONS 5;
 
-insert into t values (2001, 2), (2002, 3), (2003, 4), (2004, 5);
+insert into t(a, b) values (2001, 2), (2002, 3), (2003, 4), (2004, 5);
 
 begin;
-insert into t values (2005, 1);
+insert into t(a, b) values (2005, 1);
 
 explain select * from t use index(idx1) where b > 2;
 --sorted_result

tests/integrationtest/t/globalindex/mem_index_merge.test

@@ -57,7 +57,7 @@ CREATE TABLE `tpk2` (
   UNIQUE KEY `uidx_a` (`a`),
   UNIQUE KEY `uidx_ac` (`a`, `c`),
   KEY `idx_c` (`c`),
-  PRIMARY KEY(`d`, `c`)
+  PRIMARY KEY(`d`, `c`) clustered
 ) PARTITION BY HASH (`d`) PARTITIONS 5;
 
 insert into tpk2 values (1, 2, 1, 1), (3, 6, 3, 3);

tests/integrationtest/t/globalindex/mem_index_reader.test

@@ -33,7 +33,7 @@ rollback;
 --echo # CommonHandle
 drop table if exists t;
 CREATE TABLE `t` (
-  `a` year(4) primary key,
+  `a` year(4) primary key CLUSTERED,
   `b` int(11) DEFAULT NULL,
   UNIQUE KEY `idx1` (`b`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin