Skip to content

Commit b1818cd

Browse files
Defined2014Defined2014
authored
*: global index support admin check table | index (#53156)
ref #52897, close #53019
1 parent 41ce0a5 commit b1818cd

File tree

9 files changed

+277
-10
lines changed

9 files changed

+277
-10
lines changed

‎pkg/ddl/index.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2474,6 +2474,12 @@ func (w *cleanUpIndexWorker) BackfillData(handleRange reorgBackfillTask) (taskCt
24742474
return nil
24752475
})
24762476
logSlowOperations(time.Since(oprStartTime), "cleanUpIndexBackfillDataInTxn", 3000)
2477+
failpoint.Inject("mockDMLExecution", func(val failpoint.Value) {
2478+
//nolint:forcetypeassert
2479+
if val.(bool) && MockDMLExecution != nil {
2480+
MockDMLExecution()
2481+
}
2482+
})
24772483

24782484
return
24792485
}

‎pkg/executor/builder.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,9 @@ func buildIndexLookUpChecker(b *executorBuilder, p *plannercore.PhysicalIndexLoo
451451
if !e.isCommonHandle() {
452452
fullColLen++
453453
}
454+
if e.index.Global {
455+
fullColLen++
456+
}
454457
e.dagPB.OutputOffsets = make([]uint32, fullColLen)
455458
for i := 0; i < fullColLen; i++ {
456459
e.dagPB.OutputOffsets[i] = uint32(i)
@@ -470,6 +473,9 @@ func buildIndexLookUpChecker(b *executorBuilder, p *plannercore.PhysicalIndexLoo
470473
if !e.isCommonHandle() {
471474
tps = append(tps, types.NewFieldType(mysql.TypeLonglong))
472475
}
476+
if e.index.Global {
477+
tps = append(tps, types.NewFieldType(mysql.TypeLonglong))
478+
}
473479

474480
e.checkIndexValue = &checkIndexValue{idxColTps: tps}
475481

‎pkg/executor/distsql.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -594,17 +594,16 @@ func (e *IndexLookUpExecutor) needPartitionHandle(tp getHandleType) (bool, error
594594
outputOffsets := e.tableRequest.OutputOffsets
595595
col = cols[outputOffsets[len(outputOffsets)-1]]
596596

597-
// For TableScan, need partitionHandle in `indexOrder` when e.keepOrder == true
598-
needPartitionHandle = (e.index.Global || e.partitionTableMode) && e.keepOrder
597+
// For TableScan, need partitionHandle in `indexOrder` when e.keepOrder == true or execute `admin check [table|index]` with global index
598+
needPartitionHandle = ((e.index.Global || e.partitionTableMode) && e.keepOrder) || (e.index.Global && e.checkIndexValue != nil)
599599
// no ExtraPidColID here, because TableScan shouldn't contain them.
600600
hasExtraCol = col.ID == model.ExtraPhysTblID
601601
}
602602

603-
// TODO: fix global index related bugs later
604603
// There will be two needPartitionHandle != hasExtraCol situations.
605604
// Only `needPartitionHandle` == true and `hasExtraCol` == false are not allowed.
606605
// `ExtraPhysTblID` will be used in `SelectLock` when `needPartitionHandle` == false and `hasExtraCol` == true.
607-
if needPartitionHandle && !hasExtraCol && !e.index.Global {
606+
if needPartitionHandle && !hasExtraCol {
608607
return needPartitionHandle, errors.Errorf("Internal error, needPartitionHandle != ret, tp(%d)", tp)
609608
}
610609
return needPartitionHandle, nil

‎pkg/executor/test/admintest/BUILD.bazel

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@ go_test(
88
"main_test.go",
99
],
1010
flaky = True,
11-
shard_count = 18,
11+
shard_count = 21,
1212
deps = [
1313
"//pkg/config",
14+
"//pkg/ddl",
15+
"//pkg/ddl/util/callback",
1416
"//pkg/domain",
1517
"//pkg/errno",
1618
"//pkg/executor",
@@ -22,6 +24,7 @@ go_test(
2224
"//pkg/sessionctx/variable",
2325
"//pkg/table",
2426
"//pkg/table/tables",
27+
"//pkg/tablecodec",
2528
"//pkg/testkit",
2629
"//pkg/testkit/testsetup",
2730
"//pkg/testkit/testutil",
@@ -32,6 +35,8 @@ go_test(
3235
"//pkg/util/mock",
3336
"//pkg/util/redact",
3437
"@com_github_pingcap_errors//:errors",
38+
"@com_github_pingcap_failpoint//:failpoint",
39+
"@com_github_stretchr_testify//assert",
3540
"@com_github_stretchr_testify//require",
3641
"@com_github_tikv_client_go_v2//tikv",
3742
"@org_uber_go_goleak//:goleak",

‎pkg/executor/test/admintest/admin_test.go

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ import (
2424
"time"
2525

2626
"github.com/pingcap/errors"
27+
"github.com/pingcap/failpoint"
28+
"github.com/pingcap/tidb/pkg/ddl"
29+
"github.com/pingcap/tidb/pkg/ddl/util/callback"
2730
"github.com/pingcap/tidb/pkg/domain"
2831
mysql "github.com/pingcap/tidb/pkg/errno"
2932
"github.com/pingcap/tidb/pkg/executor"
@@ -34,6 +37,7 @@ import (
3437
"github.com/pingcap/tidb/pkg/sessionctx/variable"
3538
"github.com/pingcap/tidb/pkg/table"
3639
"github.com/pingcap/tidb/pkg/table/tables"
40+
"github.com/pingcap/tidb/pkg/tablecodec"
3741
"github.com/pingcap/tidb/pkg/testkit"
3842
"github.com/pingcap/tidb/pkg/testkit/testutil"
3943
"github.com/pingcap/tidb/pkg/types"
@@ -42,6 +46,7 @@ import (
4246
"github.com/pingcap/tidb/pkg/util/logutil/consistency"
4347
"github.com/pingcap/tidb/pkg/util/mock"
4448
"github.com/pingcap/tidb/pkg/util/redact"
49+
"github.com/stretchr/testify/assert"
4550
"github.com/stretchr/testify/require"
4651
"go.uber.org/zap"
4752
)
@@ -1773,3 +1778,237 @@ func TestAdminCheckTableErrorLocateForClusterIndex(t *testing.T) {
17731778
tk.MustExec("admin check table admin_test")
17741779
}
17751780
}
1781+
1782+
func TestAdminCheckGlobalIndex(t *testing.T) {
1783+
store, domain := testkit.CreateMockStoreAndDomain(t)
1784+
1785+
tk := testkit.NewTestKit(t, store)
1786+
var enableFastCheck = []bool{false, true}
1787+
for _, enabled := range enableFastCheck {
1788+
tk.MustExec("use test")
1789+
tk.MustExec("drop table if exists admin_test")
1790+
1791+
tk.MustExec("set tidb_enable_global_index = true")
1792+
tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
1793+
1794+
tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5")
1795+
tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
1796+
1797+
// Make some corrupted index. Build the index information.
1798+
sctx := mock.NewContext()
1799+
sctx.Store = store
1800+
is := domain.InfoSchema()
1801+
dbName := model.NewCIStr("test")
1802+
tblName := model.NewCIStr("admin_test")
1803+
tbl, err := is.TableByName(dbName, tblName)
1804+
require.NoError(t, err)
1805+
tblInfo := tbl.Meta()
1806+
idxInfo := tblInfo.Indices[0]
1807+
require.True(t, idxInfo.Global)
1808+
idx := tbl.Indices()[0]
1809+
require.NotNil(t, idx)
1810+
1811+
// Reduce one row of table.
1812+
// Index count > table count, (2, 12, 0) is deleted.
1813+
txn, err := store.Begin()
1814+
require.NoError(t, err)
1815+
err = txn.Delete(tablecodec.EncodeRowKey(tblInfo.GetPartitionInfo().Definitions[0].ID, kv.IntHandle(4).Encoded()))
1816+
require.NoError(t, err)
1817+
err = txn.Commit(context.Background())
1818+
require.NoError(t, err)
1819+
err = tk.ExecToErr("admin check table admin_test")
1820+
require.Error(t, err)
1821+
require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
1822+
require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 4, index-values:\"handle: 4, values: [KindInt64 2")
1823+
1824+
indexOpr := tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[0].ID, tblInfo, idxInfo)
1825+
// Remove corresponding index key/value.
1826+
// Admin check table will success.
1827+
txn, err = store.Begin()
1828+
require.NoError(t, err)
1829+
err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(2)}, kv.IntHandle(4))
1830+
require.NoError(t, err)
1831+
err = txn.Commit(context.Background())
1832+
require.NoError(t, err)
1833+
tk.MustExec("admin check table admin_test")
1834+
1835+
indexOpr = tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[2].ID, tblInfo, idxInfo)
1836+
1837+
// Reduce one row of index.
1838+
// Index count < table count, (-1, -10, 2) is deleted.
1839+
txn, err = store.Begin()
1840+
require.NoError(t, err)
1841+
err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(-1)}, kv.IntHandle(2))
1842+
require.NoError(t, err)
1843+
err = txn.Commit(context.Background())
1844+
require.NoError(t, err)
1845+
err = tk.ExecToErr("admin check table admin_test")
1846+
require.Error(t, err)
1847+
require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
1848+
require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 -1]\"")
1849+
1850+
// Add one row of index with inconsistent value.
1851+
// Index count = table count, but data is different.
1852+
txn, err = store.Begin()
1853+
require.NoError(t, err)
1854+
_, err = indexOpr.Create(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(100)}, kv.IntHandle(2), nil)
1855+
require.NoError(t, err)
1856+
err = txn.Commit(context.Background())
1857+
require.NoError(t, err)
1858+
err = tk.ExecToErr("admin check table admin_test")
1859+
require.Error(t, err)
1860+
if !enabled {
1861+
require.True(t, consistency.ErrAdminCheckInconsistentWithColInfo.Equal(err))
1862+
require.EqualError(t, err, "[executor:8134]data inconsistency in table: admin_test, index: uidx_a, col: a, handle: \"2\", index-values:\"KindInt64 100\" != record-values:\"KindInt64 -1\", compare err:<nil>")
1863+
} else {
1864+
require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
1865+
require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"handle: 2, values: [KindInt64 100]\" != record-values:\"handle: 2, values: [KindInt64 -1]\"")
1866+
}
1867+
}
1868+
}
1869+
1870+
func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) {
1871+
store, domain := testkit.CreateMockStoreAndDomain(t)
1872+
1873+
tk := testkit.NewTestKit(t, store)
1874+
1875+
getCommonHandle := func(row int) *kv.CommonHandle {
1876+
h, err := codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.MakeDatums(row)...)
1877+
require.NoError(t, err)
1878+
ch, err := kv.NewCommonHandle(h)
1879+
require.NoError(t, err)
1880+
return ch
1881+
}
1882+
1883+
var enableFastCheck = []bool{false, true}
1884+
for _, enabled := range enableFastCheck {
1885+
tk.MustExec("use test")
1886+
tk.MustExec("drop table if exists admin_test")
1887+
1888+
tk.MustExec("set tidb_enable_global_index = true")
1889+
tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
1890+
1891+
tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5")
1892+
tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
1893+
1894+
// Make some corrupted index. Build the index information.
1895+
sctx := mock.NewContext()
1896+
sctx.Store = store
1897+
is := domain.InfoSchema()
1898+
dbName := model.NewCIStr("test")
1899+
tblName := model.NewCIStr("admin_test")
1900+
tbl, err := is.TableByName(dbName, tblName)
1901+
require.NoError(t, err)
1902+
tblInfo := tbl.Meta()
1903+
idxInfo := tblInfo.Indices[0]
1904+
require.True(t, idxInfo.Global)
1905+
df := tblInfo.GetPartitionInfo().Definitions[0]
1906+
1907+
// Reduce one row of table.
1908+
// Index count > table count, (2, 12, 0) is deleted.
1909+
txn, err := store.Begin()
1910+
require.NoError(t, err)
1911+
txn.Delete(tablecodec.EncodeRowKey(df.ID, kv.IntHandle(0).Encoded()))
1912+
err = txn.Commit(context.Background())
1913+
require.NoError(t, err)
1914+
err = tk.ExecToErr("admin check table admin_test")
1915+
require.Error(t, err)
1916+
require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
1917+
require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 0, index-values:\"handle: 0, values: [KindInt64 2")
1918+
1919+
indexOpr := tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[0].ID, tblInfo, idxInfo)
1920+
// Remove corresponding index key/value.
1921+
// Admin check table will success.
1922+
txn, err = store.Begin()
1923+
require.NoError(t, err)
1924+
err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(2)}, getCommonHandle(0))
1925+
require.NoError(t, err)
1926+
err = txn.Commit(context.Background())
1927+
require.NoError(t, err)
1928+
tk.MustExec("admin check table admin_test")
1929+
1930+
indexOpr = tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[2].ID, tblInfo, idxInfo)
1931+
// Reduce one row of index.
1932+
// Index count < table count, (-1, -10, 2) is deleted.
1933+
txn, err = store.Begin()
1934+
require.NoError(t, err)
1935+
err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(-1)}, getCommonHandle(2))
1936+
require.NoError(t, err)
1937+
err = txn.Commit(context.Background())
1938+
require.NoError(t, err)
1939+
err = tk.ExecToErr("admin check table admin_test")
1940+
require.Error(t, err)
1941+
require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
1942+
require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 -1]\"")
1943+
1944+
// Add one row with inconsistent value.
1945+
// Index count = table count, but data is different.
1946+
txn, err = store.Begin()
1947+
require.NoError(t, err)
1948+
_, err = indexOpr.Create(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(100)}, getCommonHandle(2), nil)
1949+
require.NoError(t, err)
1950+
err = txn.Commit(context.Background())
1951+
require.NoError(t, err)
1952+
err = tk.ExecToErr("admin check table admin_test")
1953+
require.Error(t, err)
1954+
if !enabled {
1955+
require.True(t, consistency.ErrAdminCheckInconsistentWithColInfo.Equal(err))
1956+
require.EqualError(t, err, "[executor:8134]data inconsistency in table: admin_test, index: uidx_a, col: a, handle: \"2\", index-values:\"KindInt64 100\" != record-values:\"KindInt64 -1\", compare err:<nil>")
1957+
} else {
1958+
require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
1959+
require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"handle: 2, values: [KindInt64 100]\" != record-values:\"handle: 2, values: [KindInt64 -1]\"")
1960+
}
1961+
}
1962+
}
1963+
1964+
func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) {
1965+
store, dom := testkit.CreateMockStoreAndDomain(t)
1966+
originalHook := dom.DDL().GetHook()
1967+
tk := testkit.NewTestKit(t, store)
1968+
1969+
var schemaMap = make(map[model.SchemaState]struct{})
1970+
1971+
hook := &callback.TestDDLCallback{Do: dom}
1972+
onJobUpdatedExportedFunc := func(job *model.Job) {
1973+
schemaMap[job.SchemaState] = struct{}{}
1974+
_, err := tk.Exec("admin check table admin_test")
1975+
assert.NoError(t, err)
1976+
}
1977+
hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
1978+
1979+
// check table after delete some index key/value pairs.
1980+
ddl.MockDMLExecution = func() {
1981+
_, err := tk.Exec("admin check table admin_test")
1982+
assert.NoError(t, err)
1983+
}
1984+
1985+
batchSize := 32
1986+
tk.MustExec(fmt.Sprintf("set global tidb_ddl_reorg_batch_size = %d", batchSize))
1987+
1988+
var enableFastCheck = []bool{false, true}
1989+
for _, enabled := range enableFastCheck {
1990+
tk.MustExec("use test")
1991+
tk.MustExec("drop table if exists admin_test")
1992+
1993+
tk.MustExec("set tidb_enable_global_index = true")
1994+
tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
1995+
1996+
tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5")
1997+
tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
1998+
for i := 1; i <= batchSize*2; i++ {
1999+
tk.MustExec(fmt.Sprintf("insert admin_test values (%d, %d, %d)", i*5+1, i, i*5+1))
2000+
}
2001+
2002+
dom.DDL().SetHook(hook)
2003+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution", "1*return(true)->return(false)"))
2004+
tk.MustExec("alter table admin_test truncate partition p1")
2005+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution"))
2006+
dom.DDL().SetHook(originalHook)
2007+
2008+
// Should have 3 different schema states, `none`, `deleteOnly`, `deleteReorg`
2009+
require.Len(t, schemaMap, 3)
2010+
for ss := range schemaMap {
2011+
delete(schemaMap, ss)
2012+
}
2013+
}
2014+
}

‎pkg/planner/core/planbuilder.go

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1457,7 +1457,7 @@ func (b *PlanBuilder) buildAdmin(ctx context.Context, as *ast.AdminStmt) (base.P
14571457

14581458
func (b *PlanBuilder) buildPhysicalIndexLookUpReader(_ context.Context, dbName model.CIStr, tbl table.Table, idx *model.IndexInfo) (base.Plan, error) {
14591459
tblInfo := tbl.Meta()
1460-
physicalID, isPartition := getPhysicalID(tbl)
1460+
physicalID, isPartition := getPhysicalID(tbl, idx.Global)
14611461
fullExprCols, _, err := expression.TableInfo2SchemaAndNames(b.ctx.GetExprCtx(), dbName, tblInfo)
14621462
if err != nil {
14631463
return nil, err
@@ -1531,6 +1531,9 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReader(_ context.Context, dbName m
15311531
}
15321532
}
15331533
}
1534+
if is.Index.Global {
1535+
ts.Columns, ts.schema, _ = AddExtraPhysTblIDColumn(b.ctx, ts.Columns, ts.schema)
1536+
}
15341537

15351538
cop := &CopTask{
15361539
indexPlan: is,
@@ -1567,9 +1570,9 @@ func getIndexColsSchema(tblInfo *model.TableInfo, idx *model.IndexInfo, allColSc
15671570
return schema
15681571
}
15691572

1570-
func getPhysicalID(t table.Table) (physicalID int64, isPartition bool) {
1573+
func getPhysicalID(t table.Table, isGlobalIndex bool) (physicalID int64, isPartition bool) {
15711574
tblInfo := t.Meta()
1572-
if tblInfo.GetPartitionInfo() != nil {
1575+
if !isGlobalIndex && tblInfo.GetPartitionInfo() != nil {
15731576
pid := t.(table.PhysicalTable).GetPhysicalID()
15741577
return pid, true
15751578
}
@@ -1657,8 +1660,8 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReaders(ctx context.Context, dbNam
16571660
}
16581661
}
16591662
indexInfos = append(indexInfos, idxInfo)
1660-
// For partition tables.
1661-
if pi := tbl.Meta().GetPartitionInfo(); pi != nil {
1663+
// For partition tables except global index.
1664+
if pi := tbl.Meta().GetPartitionInfo(); pi != nil && !idxInfo.Global {
16621665
for _, def := range pi.Definitions {
16631666
t := tbl.(table.PartitionedTable).GetPartition(def.ID)
16641667
reader, err := b.buildPhysicalIndexLookUpReader(ctx, dbName, t, idxInfo)

0 commit comments

Comments
 (0)