From 872dabf493f8abdaea4b3b59dc9ff39490edfd98 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Sep 2022 10:56:30 +0200 Subject: [PATCH 01/12] executor: Always add global stats for partitioned tables --- executor/analyze.go | 12 ++++-------- executor/analyze_global_stats.go | 5 +---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 6fccec8a9bf5b..7d82528194d1d 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -107,11 +107,8 @@ func (e *AnalyzeExec) Next(ctx context.Context, _ *chunk.Chunk) error { close(taskCh) e.wg.Wait() close(resultsCh) - pruneMode := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) - // needGlobalStats used to indicate whether we should merge the partition-level stats to global-level stats. - needGlobalStats := pruneMode == variable.Dynamic globalStatsMap := make(map[globalStatsKey]globalStatsInfo) - err = e.handleResultsError(ctx, concurrency, needGlobalStats, globalStatsMap, resultsCh) + err = e.handleResultsError(ctx, concurrency, globalStatsMap, resultsCh) for _, task := range e.tasks { if task.colExec != nil && task.colExec.memTracker != nil { task.colExec.memTracker.Detach() @@ -125,8 +122,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, _ *chunk.Chunk) error { dom.SysProcTracker().KillSysProcess(util.GetAutoAnalyzeProcID(dom.ServerID)) }) - // If we enabled dynamic prune mode, then we need to generate global stats here for partition tables. - err = e.handleGlobalStats(ctx, needGlobalStats, globalStatsMap) + err = e.handleGlobalStats(ctx, globalStatsMap) if err != nil { return err } @@ -215,7 +211,7 @@ func (e *AnalyzeExec) recordHistoricalStats(tableID int64) error { } // handleResultsError will handle the error fetch from resultsCh and record it in log -func (e *AnalyzeExec) handleResultsError(ctx context.Context, concurrency int, needGlobalStats bool, +func (e *AnalyzeExec) handleResultsError(ctx context.Context, concurrency int, globalStatsMap globalStatsMap, resultsCh <-chan *statistics.AnalyzeResults) error { statsHandle := domain.GetDomain(e.ctx).StatsHandle() panicCnt := 0 @@ -235,7 +231,7 @@ func (e *AnalyzeExec) handleResultsError(ctx context.Context, concurrency int, n finishJobWithLog(e.ctx, results.Job, err) continue } - if results.TableID.IsPartitionTable() && needGlobalStats { + if results.TableID.IsPartitionTable() { for _, result := range results.Ars { if result.IsIndex == 0 { // If it does not belong to the statistics of index, we need to set it to -1 to distinguish. diff --git a/executor/analyze_global_stats.go b/executor/analyze_global_stats.go index 85ff647255212..8715c9b4790f0 100644 --- a/executor/analyze_global_stats.go +++ b/executor/analyze_global_stats.go @@ -42,10 +42,7 @@ type globalStatsInfo struct { // The meaning of value in map is some additional information needed to build global-level stats. type globalStatsMap map[globalStatsKey]globalStatsInfo -func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, needGlobalStats bool, globalStatsMap globalStatsMap) error { - if !needGlobalStats { - return nil - } +func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap globalStatsMap) error { statsHandle := domain.GetDomain(e.ctx).StatsHandle() for globalStatsID, info := range globalStatsMap { globalOpts := e.opts From 5bf1ec3dfbfa5014484a44be2b3233c22311b816 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Sep 2022 11:33:30 +0200 Subject: [PATCH 02/12] Updated testcase result --- executor/analyzetest/analyze_test.go | 33 +++++++++++++++++----------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go index b9bfc153fa33c..ec0fb7a2a9201 100644 --- a/executor/analyzetest/analyze_test.go +++ b/executor/analyzetest/analyze_test.go @@ -1785,6 +1785,7 @@ func TestAnalyzeColumnsWithStaticPartitionTable(t *testing.T) { tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) require.NoError(t, err) defs := tbl.Meta().Partition.Definitions + tblID := tbl.Meta().ID p0ID := defs[0].ID p1ID := defs[1].ID @@ -1811,11 +1812,13 @@ func TestAnalyzeColumnsWithStaticPartitionTable(t *testing.T) { } rows := tk.MustQuery("show column_stats_usage where db_name = 'test' and table_name = 't' and last_analyzed_at is not null").Sort().Rows() - require.Equal(t, 4, len(rows)) - require.Equal(t, []interface{}{"test", "t", "p0", "a"}, rows[0][:4]) - require.Equal(t, []interface{}{"test", "t", "p0", "c"}, rows[1][:4]) - require.Equal(t, []interface{}{"test", "t", "p1", "a"}, rows[2][:4]) - require.Equal(t, []interface{}{"test", "t", "p1", "c"}, rows[3][:4]) + require.Equal(t, 6, len(rows)) + require.Equal(t, []interface{}{"test", "t", "global", "a"}, rows[0][:4]) + require.Equal(t, []interface{}{"test", "t", "global", "c"}, rows[1][:4]) + require.Equal(t, []interface{}{"test", "t", "p0", "a"}, rows[2][:4]) + require.Equal(t, []interface{}{"test", "t", "p0", "c"}, rows[3][:4]) + require.Equal(t, []interface{}{"test", "t", "p1", "a"}, rows[4][:4]) + require.Equal(t, []interface{}{"test", "t", "p1", "c"}, rows[5][:4]) rows = tk.MustQuery("show stats_meta where db_name = 'test' and table_name = 't'").Sort().Rows() require.Equal(t, 2, len(rows)) @@ -1859,14 +1862,18 @@ func TestAnalyzeColumnsWithStaticPartitionTable(t *testing.T) { "test t p1 idx 1 1 6 1 11 12 0")) tk.MustQuery("select table_id, is_index, hist_id, distinct_count, null_count, tot_col_size, stats_ver, truncate(correlation,2) from mysql.stats_histograms order by table_id, is_index, hist_id asc").Check( - testkit.Rows(fmt.Sprintf("%d 0 1 5 1 8 2 1", p0ID), // p0, a - fmt.Sprintf("%d 0 2 0 0 8 0 0", p0ID), // p0, b, not analyzed - fmt.Sprintf("%d 0 3 6 0 9 2 1", p0ID), // p0, c - fmt.Sprintf("%d 1 1 6 0 9 2 0", p0ID), // p0, idx - fmt.Sprintf("%d 0 1 7 0 11 2 1", p1ID), // p1, a - fmt.Sprintf("%d 0 2 0 0 11 0 0", p1ID), // p1, b, not analyzed - fmt.Sprintf("%d 0 3 8 0 11 2 1", p1ID), // p1, c - fmt.Sprintf("%d 1 1 8 0 11 2 0", p1ID), // p1, idx + testkit.Rows( + fmt.Sprintf("%d 0 1 12 1 19 2 0", tblID), // tbl, a + fmt.Sprintf("%d 0 3 14 0 20 2 0", tblID), // tbl, b, not analyzed + fmt.Sprintf("%d 1 1 14 0 0 2 0", tblID), // tbl, c + fmt.Sprintf("%d 0 1 5 1 8 2 1", p0ID), // p0, a + fmt.Sprintf("%d 0 2 0 0 8 0 0", p0ID), // p0, b, not analyzed + fmt.Sprintf("%d 0 3 6 0 9 2 1", p0ID), // p0, c + fmt.Sprintf("%d 1 1 6 0 9 2 0", p0ID), // p0, idx + fmt.Sprintf("%d 0 1 7 0 11 2 1", p1ID), // p1, a + fmt.Sprintf("%d 0 2 0 0 11 0 0", p1ID), // p1, b, not analyzed + fmt.Sprintf("%d 0 3 8 0 11 2 1", p1ID), // p1, c + fmt.Sprintf("%d 1 1 8 0 11 2 0", p1ID), // p1, idx )) }(val) } From da913e811df1e67b97b270fc1a9c6281bfefc0cb Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Sep 2022 14:19:58 +0200 Subject: [PATCH 03/12] Updated test testAnalyzeIncremental due to always doing global stats --- executor/analyzetest/analyze_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go index ec0fb7a2a9201..a68eb407b626c 100644 --- a/executor/analyzetest/analyze_test.go +++ b/executor/analyzetest/analyze_test.go @@ -798,13 +798,15 @@ func testAnalyzeIncremental(tk *testkit.TestKit, t *testing.T, dom *domain.Domai tk.MustQuery("show stats_buckets").Check(testkit.Rows()) tk.MustExec("insert into t values (1,1)") tk.MustExec("analyze incremental table t index") - tk.MustQuery("show warnings").Check(testkit.Rows()) // no warning + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 8244 Build table: `t` column: `b` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions")) require.NoError(t, h.LoadNeededHistograms()) tk.MustQuery("show stats_buckets").Check(testkit.Rows("test t p0 a 0 0 1 1 1 1 0", "test t p0 idx 1 0 1 1 1 1 0")) + tk.MustExec(`analyze table t`) tk.MustExec("insert into t values (2,2)") tk.MustExec("analyze incremental table t index") + tk.MustQuery("show warnings").Check(testkit.Rows()) // no warnings require.NoError(t, h.LoadNeededHistograms()) - tk.MustQuery("show stats_buckets").Check(testkit.Rows("test t p0 a 0 0 1 1 1 1 0", "test t p0 a 0 1 2 1 2 2 0", "test t p0 idx 1 0 1 1 1 1 0", "test t p0 idx 1 1 2 1 2 2 0")) + tk.MustQuery("show stats_buckets").Sort().Check(testkit.Rows("test t p0 a 0 0 1 1 1 1 0", "test t p0 a 0 1 2 1 2 2 0", "test t p0 b 0 0 1 1 1 1 0", "test t p0 idx 1 0 1 1 1 1 0", "test t p0 idx 1 1 2 1 2 2 0")) tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic';") tk.MustExec("insert into t values (11,11)") err = tk.ExecToErr("analyze incremental table t index") From 55638fff3c46ee05992a15ee20387d5d959c2b98 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Sep 2022 14:29:34 +0200 Subject: [PATCH 04/12] Updated test case --- planner/core/integration_test.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 1b91cc8106c7e..a1e12cb6412b1 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -7166,29 +7166,23 @@ func TestPartitionTableFallBackStatic(t *testing.T) { tk.MustExec("analyze table t2") tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") + rows = [][]interface{}{ + {"TableReader", "partition:all", "data:TableFullScan"}, + {"└─TableFullScan", "table:t", "keep order:false"}, + } // use static plan in dynamic mode due to having not global stats tk.MustQuery("explain format='brief' select * from t").CheckAt([]int{0, 3, 4}, rows) tk.MustExec("analyze table t") // use dynamic plan in dynamic mode with global stats - rows = [][]interface{}{ - {"TableReader", "partition:all", "data:TableFullScan"}, - {"└─TableFullScan", "table:t", "keep order:false"}, - } tk.MustQuery("explain format='brief' select * from t").CheckAt([]int{0, 3, 4}, rows) rows = [][]interface{}{ {"Union", "", ""}, - {"├─PartitionUnion", "", ""}, - {"│ ├─TableReader", "", "data:TableFullScan"}, - {"│ │ └─TableFullScan", "table:t, partition:p0", "keep order:false"}, - {"│ └─TableReader", "", "data:TableFullScan"}, - {"│ └─TableFullScan", "table:t, partition:p1", "keep order:false"}, - {"└─PartitionUnion", "", ""}, - {" ├─TableReader", "", "data:TableFullScan"}, - {" │ └─TableFullScan", "table:t2, partition:p0", "keep order:false"}, - {" └─TableReader", "", "data:TableFullScan"}, - {" └─TableFullScan", "table:t2, partition:p1", "keep order:false"}, + {"├─TableReader", "partition:all", "data:TableFullScan"}, + {"│ └─TableFullScan", "table:t", "keep order:false"}, + {"└─TableReader", "partition:all", "data:TableFullScan"}, + {" └─TableFullScan", "table:t2", "keep order:false"}, } // use static plan in dynamic mode due to t2 has no global stats tk.MustQuery("explain format='brief' select * from t union all select * from t2;").CheckAt([]int{0, 3, 4}, rows) From 0af1cf5d13165db94ecdc8c1f32a89563225381d Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Sep 2022 14:35:42 +0200 Subject: [PATCH 05/12] Updated testcase --- statistics/handle/gc_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/statistics/handle/gc_test.go b/statistics/handle/gc_test.go index 66c4df5260a9a..7df0763798ba1 100644 --- a/statistics/handle/gc_test.go +++ b/statistics/handle/gc_test.go @@ -70,23 +70,23 @@ func TestGCPartition(t *testing.T) { testKit.MustExec("insert into t values (1,2),(2,3),(3,4),(4,5),(5,6)") testKit.MustExec("analyze table t") - testKit.MustQuery("select count(*) from mysql.stats_histograms").Check(testkit.Rows("6")) - testKit.MustQuery("select count(*) from mysql.stats_buckets").Check(testkit.Rows("15")) + testKit.MustQuery("select count(*) from mysql.stats_histograms").Check(testkit.Rows("9")) + testKit.MustQuery("select count(*) from mysql.stats_buckets").Check(testkit.Rows("30")) h := dom.StatsHandle() ddlLease := time.Duration(0) testKit.MustExec("alter table t drop index idx") require.Nil(t, h.GCStats(dom.InfoSchema(), ddlLease)) - testKit.MustQuery("select count(*) from mysql.stats_histograms").Check(testkit.Rows("4")) - testKit.MustQuery("select count(*) from mysql.stats_buckets").Check(testkit.Rows("10")) + testKit.MustQuery("select count(*) from mysql.stats_histograms").Check(testkit.Rows("6")) + testKit.MustQuery("select count(*) from mysql.stats_buckets").Check(testkit.Rows("20")) testKit.MustExec("alter table t drop column b") require.Nil(t, h.GCStats(dom.InfoSchema(), ddlLease)) - testKit.MustQuery("select count(*) from mysql.stats_histograms").Check(testkit.Rows("2")) - testKit.MustQuery("select count(*) from mysql.stats_buckets").Check(testkit.Rows("5")) + testKit.MustQuery("select count(*) from mysql.stats_histograms").Check(testkit.Rows("3")) + testKit.MustQuery("select count(*) from mysql.stats_buckets").Check(testkit.Rows("10")) testKit.MustExec("drop table t") require.Nil(t, h.GCStats(dom.InfoSchema(), ddlLease)) - testKit.MustQuery("select count(*) from mysql.stats_meta").Check(testkit.Rows("2")) + testKit.MustQuery("select count(*) from mysql.stats_meta").Check(testkit.Rows("3")) testKit.MustQuery("select count(*) from mysql.stats_histograms").Check(testkit.Rows("0")) testKit.MustQuery("select count(*) from mysql.stats_buckets").Check(testkit.Rows("0")) require.Nil(t, h.GCStats(dom.InfoSchema(), ddlLease)) From 2c0b3596cef3d63c8119ea00c766ed2cc2031ee5 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Sep 2022 16:05:24 +0200 Subject: [PATCH 06/12] Always include global stats in SHOW {stats_meta,stats_histograms} --- executor/analyze.go | 3 +- executor/show_stats.go | 102 +++++++++++-------------------- statistics/handle/dump_test.go | 10 ++- statistics/handle/handle_test.go | 8 +-- testkit/testkit.go | 17 ++++++ 5 files changed, 64 insertions(+), 76 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 7d82528194d1d..cf2604251e2af 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -142,10 +142,9 @@ func (e *AnalyzeExec) saveV2AnalyzeOpts() error { return nil } // only to save table options if dynamic prune mode - dynamicPrune := variable.PartitionPruneMode(e.ctx.GetSessionVars().PartitionPruneMode.Load()) == variable.Dynamic toSaveMap := make(map[int64]core.V2AnalyzeOptions) for id, opts := range e.OptionsMap { - if !opts.IsPartition || !dynamicPrune { + if !opts.IsPartition { toSaveMap[id] = opts } } diff --git a/executor/show_stats.go b/executor/show_stats.go index 058b3ed7c62da..3388b90e27b44 100644 --- a/executor/show_stats.go +++ b/executor/show_stats.go @@ -108,18 +108,12 @@ func (e *ShowExec) fetchShowStatsMeta() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { - partitionName := "" - if pi != nil { - partitionName = "global" - } - e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)) - if pi != nil { - for _, def := range pi.Definitions { - e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) - } - } - } else { + partitionName := "" + if pi != nil { + partitionName = "global" + } + e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)) + if pi != nil { for _, def := range pi.Definitions { e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) } @@ -150,18 +144,12 @@ func (e *ShowExec) fetchShowStatsHistogram() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { - partitionName := "" - if pi != nil { - partitionName = "global" - } - e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)) - if pi != nil { - for _, def := range pi.Definitions { - e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) - } - } - } else { + partitionName := "" + if pi != nil { + partitionName = "global" + } + e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)) + if pi != nil { for _, def := range pi.Definitions { e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) } @@ -224,22 +212,14 @@ func (e *ShowExec) fetchShowStatsBuckets() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { - partitionName := "" - if pi != nil { - partitionName = "global" - } - if err := e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)); err != nil { - return err - } - if pi != nil { - for _, def := range pi.Definitions { - if err := e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)); err != nil { - return err - } - } - } - } else { + partitionName := "" + if pi != nil { + partitionName = "global" + } + if err := e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)); err != nil { + return err + } + if pi != nil { for _, def := range pi.Definitions { if err := e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)); err != nil { return err @@ -283,22 +263,14 @@ func (e *ShowExec) fetchShowStatsTopN() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { - partitionName := "" - if pi != nil { - partitionName = "global" - } - if err := e.appendTableForStatsTopN(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)); err != nil { - return err - } - if pi != nil { - for _, def := range pi.Definitions { - if err := e.appendTableForStatsTopN(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)); err != nil { - return err - } - } - } - } else { + partitionName := "" + if pi != nil { + partitionName = "global" + } + if err := e.appendTableForStatsTopN(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)); err != nil { + return err + } + if pi != nil { for _, def := range pi.Definitions { if err := e.appendTableForStatsTopN(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)); err != nil { return err @@ -415,18 +387,12 @@ func (e *ShowExec) fetchShowStatsHealthy() { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { - partitionName := "" - if pi != nil { - partitionName = "global" - } - e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)) - if pi != nil { - for _, def := range pi.Definitions { - e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) - } - } - } else { + partitionName := "" + if pi != nil { + partitionName = "global" + } + e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, partitionName, h.GetTableStats(tbl)) + if pi != nil { for _, def := range pi.Definitions { e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID)) } diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go index 0e3006604e5cc..f4bc176c7b300 100644 --- a/statistics/handle/dump_test.go +++ b/statistics/handle/dump_test.go @@ -133,13 +133,19 @@ func TestDumpGlobalStats(t *testing.T) { tk.MustExec("insert into t values (1), (2)") tk.MustExec("analyze table t") - // global-stats is not existed + // global-stats is also generated even in static prune mode stats := getStatsJSON(t, dom, "test", "t") require.NotNil(t, stats.Partitions["p0"]) require.NotNil(t, stats.Partitions["p1"]) + require.NotNil(t, stats.Partitions["global"]) + + tk.MustExec(`DROP STATS t`) + stats = getStatsJSON(t, dom, "test", "t") + require.Nil(t, stats.Partitions["p0"]) + require.Nil(t, stats.Partitions["p1"]) require.Nil(t, stats.Partitions["global"]) - // global-stats is existed + // global-stats is created also for dynamic prune mode tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") tk.MustExec("analyze table t") stats = getStatsJSON(t, dom, "test", "t") diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index f8f73b933bd9a..9755f30cbddf3 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -2089,16 +2089,16 @@ func TestStaticPartitionPruneMode(t *testing.T) { partition p1 values less than (22))`) tk.MustExec(`insert into t values (1), (2), (3), (10), (11)`) tk.MustExec(`analyze table t`) - require.True(t, tk.MustNoGlobalStats("t")) + require.True(t, tk.MustGlobalStats("t")) tk.MustExec("set @@tidb_partition_prune_mode='" + string(variable.Dynamic) + "'") - require.True(t, tk.MustNoGlobalStats("t")) + require.True(t, tk.MustGlobalStats("t")) tk.MustExec("set @@tidb_partition_prune_mode='" + string(variable.Static) + "'") tk.MustExec(`insert into t values (4), (5), (6)`) tk.MustExec(`analyze table t partition p0`) - require.True(t, tk.MustNoGlobalStats("t")) + require.True(t, tk.MustGlobalStats("t")) tk.MustExec("set @@tidb_partition_prune_mode='" + string(variable.Dynamic) + "'") - require.True(t, tk.MustNoGlobalStats("t")) + require.True(t, tk.MustGlobalStats("t")) tk.MustExec("set @@tidb_partition_prune_mode='" + string(variable.Static) + "'") } diff --git a/testkit/testkit.go b/testkit/testkit.go index 75bfdff68a3bc..f3da15bfe8346 100644 --- a/testkit/testkit.go +++ b/testkit/testkit.go @@ -462,6 +462,23 @@ func (tk *TestKit) MustNoGlobalStats(table string) bool { return true } +// MustGlobalStats checks if there is no global stats. +func (tk *TestKit) MustGlobalStats(table string) bool { + if !containGlobal(tk.MustQuery("show stats_meta where table_name like '" + table + "'")) { + return false + } + // may be empty + /* + if !containGlobal(tk.MustQuery("show stats_buckets where table_name like '" + table + "'")) { + return false + } + */ + if !containGlobal(tk.MustQuery("show stats_histograms where table_name like '" + table + "'")) { + return false + } + return true +} + // CheckLastMessage checks last message after executing MustExec func (tk *TestKit) CheckLastMessage(msg string) { tk.require.Equal(tk.Session().LastMessage(), msg) From 7f0aa55c1b02469869df33d8ccde665bff4db12a Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Sep 2022 16:21:54 +0200 Subject: [PATCH 07/12] Updated a test case --- statistics/integration_test.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/statistics/integration_test.go b/statistics/integration_test.go index c4e4d315c7dbe..49e28d9c538f7 100644 --- a/statistics/integration_test.go +++ b/statistics/integration_test.go @@ -344,10 +344,8 @@ func TestGlobalStats(t *testing.T) { // When we set the mode to `static`, using analyze will not report an error and will not generate global-stats. // In addition, when using explain to view the plan of the related query, it was found that `Union` was used. tk.MustExec("analyze table t;") - result := tk.MustQuery("show stats_meta where table_name = 't'").Sort() - require.Len(t, result.Rows(), 2) - require.Equal(t, "2", result.Rows()[0][5]) - require.Equal(t, "3", result.Rows()[1][5]) + tk.MustQuery("show stats_meta where table_name = 't'").Sort().CheckAt([]int{2, 4, 5}, + [][]interface{}{{"global", "0", "5"}, {"p0", "0", "2"}, {"p1", "0", "3"}}) tk.MustQuery("explain format = 'brief' select a from t where a > 3;").Check(testkit.Rows( "PartitionUnion 2.00 root ", "├─IndexReader 1.00 root index:IndexRangeScan", @@ -355,16 +353,15 @@ func TestGlobalStats(t *testing.T) { "└─IndexReader 1.00 root index:IndexRangeScan", " └─IndexRangeScan 1.00 cop[tikv] table:t, partition:p1, index:a(a) range:(3,+inf], keep order:false")) - // When we turned on the switch, we found that pseudo-stats will be used in the plan instead of `Union`. tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic';") tk.MustQuery("explain format = 'brief' select a from t where a > 3;").Check(testkit.Rows( - "IndexReader 3333.33 root partition:all index:IndexRangeScan", - "└─IndexRangeScan 3333.33 cop[tikv] table:t, index:a(a) range:(3,+inf], keep order:false, stats:pseudo")) + "IndexReader 2.00 root partition:all index:IndexRangeScan", + "└─IndexRangeScan 2.00 cop[tikv] table:t, index:a(a) range:(3,+inf], keep order:false")) // Execute analyze again without error and can generate global-stats. // And when executing related queries, neither Union nor pseudo-stats are used. tk.MustExec("analyze table t;") - result = tk.MustQuery("show stats_meta where table_name = 't'").Sort() + result := tk.MustQuery("show stats_meta where table_name = 't'").Sort() require.Len(t, result.Rows(), 3) require.Equal(t, "5", result.Rows()[0][5]) require.Equal(t, "2", result.Rows()[1][5]) From dc30ed2628f634d36845bf7741ecb20fc12ff900 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Mon, 12 Sep 2022 16:36:14 +0200 Subject: [PATCH 08/12] Updated test cases --- executor/analyzetest/analyze_test.go | 45 ++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go index a68eb407b626c..d82401d285a5b 100644 --- a/executor/analyzetest/analyze_test.go +++ b/executor/analyzetest/analyze_test.go @@ -800,13 +800,23 @@ func testAnalyzeIncremental(tk *testkit.TestKit, t *testing.T, dom *domain.Domai tk.MustExec("analyze incremental table t index") tk.MustQuery("show warnings").Check(testkit.Rows("Warning 8244 Build table: `t` column: `b` global-level stats failed due to missing partition-level column stats, please run analyze table to refresh columns of all partitions")) require.NoError(t, h.LoadNeededHistograms()) - tk.MustQuery("show stats_buckets").Check(testkit.Rows("test t p0 a 0 0 1 1 1 1 0", "test t p0 idx 1 0 1 1 1 1 0")) + tk.MustQuery("show stats_buckets").Check(testkit.Rows("test t global idx 1 0 1 1 1 1 0", "test t p0 a 0 0 1 1 1 1 0", "test t p0 idx 1 0 1 1 1 1 0")) tk.MustExec(`analyze table t`) tk.MustExec("insert into t values (2,2)") tk.MustExec("analyze incremental table t index") tk.MustQuery("show warnings").Check(testkit.Rows()) // no warnings require.NoError(t, h.LoadNeededHistograms()) - tk.MustQuery("show stats_buckets").Sort().Check(testkit.Rows("test t p0 a 0 0 1 1 1 1 0", "test t p0 a 0 1 2 1 2 2 0", "test t p0 b 0 0 1 1 1 1 0", "test t p0 idx 1 0 1 1 1 1 0", "test t p0 idx 1 1 2 1 2 2 0")) + tk.MustQuery("show stats_buckets").Sort().Check(testkit.Rows( + "test t global a 0 0 1 1 1 1 0", + "test t global a 0 1 2 1 1 2 0", + "test t global b 0 0 1 1 1 1 0", + "test t global idx 1 0 1 1 1 1 0", + "test t global idx 1 1 2 1 1 2 0", + "test t p0 a 0 0 1 1 1 1 0", + "test t p0 a 0 1 2 1 2 2 0", + "test t p0 b 0 0 1 1 1 1 0", + "test t p0 idx 1 0 1 1 1 1 0", + "test t p0 idx 1 1 2 1 2 2 0")) tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic';") tk.MustExec("insert into t values (11,11)") err = tk.ExecToErr("analyze incremental table t index") @@ -1823,13 +1833,19 @@ func TestAnalyzeColumnsWithStaticPartitionTable(t *testing.T) { require.Equal(t, []interface{}{"test", "t", "p1", "c"}, rows[5][:4]) rows = tk.MustQuery("show stats_meta where db_name = 'test' and table_name = 't'").Sort().Rows() - require.Equal(t, 2, len(rows)) - require.Equal(t, []interface{}{"test", "t", "p0", "0", "9"}, append(rows[0][:3], rows[0][4:]...)) - require.Equal(t, []interface{}{"test", "t", "p1", "0", "11"}, append(rows[1][:3], rows[1][4:]...)) + require.Equal(t, 3, len(rows)) + require.Equal(t, []interface{}{"test", "t", "global", "0", "20"}, append(rows[0][:3], rows[0][4:]...)) + require.Equal(t, []interface{}{"test", "t", "p0", "0", "9"}, append(rows[1][:3], rows[1][4:]...)) + require.Equal(t, []interface{}{"test", "t", "p1", "0", "11"}, append(rows[2][:3], rows[2][4:]...)) tk.MustQuery("show stats_topn where db_name = 'test' and table_name = 't' and is_index = 0").Sort().Check( // db, tbl, part, col, is_idx, value, count - testkit.Rows("test t p0 a 0 4 2", + testkit.Rows( + "test t global a 0 16 4", + "test t global a 0 5 3", + "test t global c 0 1 3", + "test t global c 0 14 3", + "test t p0 a 0 4 2", "test t p0 a 0 5 3", "test t p0 c 0 1 3", "test t p0 c 0 2 2", @@ -1840,14 +1856,22 @@ func TestAnalyzeColumnsWithStaticPartitionTable(t *testing.T) { tk.MustQuery("show stats_topn where db_name = 'test' and table_name = 't' and is_index = 1").Sort().Check( // db, tbl, part, col, is_idx, value, count - testkit.Rows("test t p0 idx 1 1 3", + testkit.Rows( + "test t global idx 1 1 3", + "test t global idx 1 14 3", + "test t p0 idx 1 1 3", "test t p0 idx 1 2 2", "test t p1 idx 1 13 2", "test t p1 idx 1 14 3")) tk.MustQuery("show stats_buckets where db_name = 'test' and table_name = 't' and is_index = 0").Sort().Check( // db, tbl, part, col, is_index, bucket_id, count, repeats, lower, upper, ndv - testkit.Rows("test t p0 a 0 0 2 1 1 2 0", + testkit.Rows( + "test t global a 0 0 5 2 1 4 0", + "test t global a 0 1 12 2 17 17 0", + "test t global c 0 0 6 1 2 6 0", + "test t global c 0 1 14 2 13 13 0", + "test t p0 a 0 0 2 1 1 2 0", "test t p0 a 0 1 3 1 3 3 0", "test t p0 c 0 0 3 1 3 5 0", "test t p0 c 0 1 4 1 6 6 0", @@ -1858,7 +1882,10 @@ func TestAnalyzeColumnsWithStaticPartitionTable(t *testing.T) { tk.MustQuery("show stats_buckets where db_name = 'test' and table_name = 't' and is_index = 1").Sort().Check( // db, tbl, part, col, is_index, bucket_id, count, repeats, lower, upper, ndv - testkit.Rows("test t p0 idx 1 0 3 1 3 5 0", + testkit.Rows( + "test t global idx 1 0 6 1 2 6 0", + "test t global idx 1 1 14 2 13 13 0", + "test t p0 idx 1 0 3 1 3 5 0", "test t p0 idx 1 1 4 1 6 6 0", "test t p1 idx 1 0 4 1 7 10 0", "test t p1 idx 1 1 6 1 11 12 0")) From 75f073c839dcb91cce3c1550c1d1d7b21ebf9945 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Wed, 14 Sep 2022 16:57:09 +0200 Subject: [PATCH 09/12] Revert "planner: add warning if fallback to static plan from dynamic (#37767)" This reverts commit e68227ca101959818cbf786acdf58b2ee1d79ab1. --- executor/write_test.go | 8 -------- planner/core/integration_test.go | 2 -- planner/core/logical_plan_builder.go | 4 ---- 3 files changed, 14 deletions(-) diff --git a/executor/write_test.go b/executor/write_test.go index ab3ec6813f5c7..a5ef9da73ce00 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -1192,8 +1192,6 @@ func TestGeneratedColumnForInsert(t *testing.T) { } func TestPartitionedTableReplace(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -1452,8 +1450,6 @@ func TestHashPartitionedTableReplace(t *testing.T) { } func TestPartitionedTableUpdate(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -1724,8 +1720,6 @@ func TestDelete(t *testing.T) { } func TestPartitionedTableDelete(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") createTable := `CREATE TABLE test.t (id int not null default 1, name varchar(255), index(id)) PARTITION BY RANGE ( id ) ( PARTITION p0 VALUES LESS THAN (6), @@ -3941,8 +3935,6 @@ func testEqualDatumsAsBinary(t *testing.T, a []interface{}, b []interface{}, sam } func TestUpdate(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 9b7e8be87409a..da230b745d1a5 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5068,8 +5068,6 @@ func TestIncrementalAnalyzeStatsVer2(t *testing.T) { } func TestConflictReadFromStorage(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index e59985c8e1e12..70248a7a20e36 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -4408,10 +4408,6 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as if usePartitionProcessor { b.optFlag = b.optFlag | flagPartitionProcessor b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false - if isDynamicEnabled { - b.ctx.GetSessionVars().StmtCtx.AppendWarning( - fmt.Errorf("disable dynamic pruning due to %s has no global stats", tableInfo.Name.String())) - } } pt := tbl.(table.PartitionedTable) From 0bcbad943d1dd4ac6aedcf38a0591cc793a81cfa Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Wed, 14 Sep 2022 17:09:17 +0200 Subject: [PATCH 10/12] Revert "planner: fallback to static plan for partition table if having no global stats in dynamic prune mode (#37573)" This reverts commit 7eb7ca902a0d8f13e2052a5a497c26e1cc862d09. But keeps the tests --- ddl/db_partition_test.go | 5 ---- executor/analyzetest/analyze_test.go | 4 +-- executor/builder.go | 18 ++++++------- executor/distsql_test.go | 2 -- executor/executor.go | 7 ----- executor/executor_issue_test.go | 2 -- executor/index_lookup_join_test.go | 3 --- executor/partition_table_test.go | 26 ------------------- executor/show_stats.go | 10 +++---- executor/tiflashtest/tiflash_test.go | 2 -- executor/write_test.go | 7 ----- planner/cascades/integration_test.go | 3 --- planner/core/access_object.go | 10 +++---- planner/core/cbo_test.go | 3 --- .../core/collect_column_stats_usage_test.go | 3 --- planner/core/explain.go | 2 +- planner/core/find_best_task.go | 2 +- planner/core/integration_partition_test.go | 7 ----- planner/core/integration_test.go | 11 -------- planner/core/logical_plan_builder.go | 22 +++------------- planner/core/partition_pruner_test.go | 9 ------- planner/core/physical_plan_test.go | 6 ----- planner/core/plan_to_pb.go | 2 +- planner/core/point_get_plan_test.go | 3 --- planner/core/rule_join_reorder_test.go | 3 --- sessionctx/stmtctx/stmtctx.go | 7 ----- sessionctx/variable/session.go | 6 ++--- statistics/handle/handle_test.go | 3 --- statistics/integration_test.go | 2 -- table/tables/partition_test.go | 3 --- 30 files changed, 28 insertions(+), 165 deletions(-) diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 01263fc7e11fa..3c81911d32b97 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -656,8 +656,6 @@ create table log_message_1 ( } func TestPartitionRangeColumnsCollate(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("create schema PartitionRangeColumnsCollate") @@ -3571,9 +3569,6 @@ func TestPartitionListWithTimeType(t *testing.T) { } func TestPartitionListWithNewCollation(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") - store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test;") diff --git a/executor/analyzetest/analyze_test.go b/executor/analyzetest/analyze_test.go index 9274fd62b423a..65c9dd59ef4ad 100644 --- a/executor/analyzetest/analyze_test.go +++ b/executor/analyzetest/analyze_test.go @@ -2849,8 +2849,6 @@ PARTITION BY RANGE ( a ) ( } func TestAnalyzePartitionStaticToDynamic(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) originalVal := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string) @@ -2919,7 +2917,7 @@ PARTITION BY RANGE ( a ) ( tk.MustQuery("select * from t where a > 1 and b > 1 and c > 1 and d > 1") require.NoError(t, h.LoadNeededHistograms()) tbl := h.GetTableStats(tableInfo) - require.Equal(t, 4, len(tbl.Columns)) + require.Equal(t, 0, len(tbl.Columns)) // ignore both p0's 3 buckets, persisted-partition-options' 1 bucket, just use table-level 2 buckets tk.MustExec("analyze table t partition p0") diff --git a/executor/builder.go b/executor/builder.go index 649faeeae3ad5..cbe6462c8316d 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -3321,7 +3321,7 @@ func (b *executorBuilder) buildTableReader(v *plannercore.PhysicalTableReader) E sctx := b.ctx.GetSessionVars().StmtCtx sctx.TableIDs = append(sctx.TableIDs, ts.Table.ID) - if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { return ret } // When isPartition is set, it means the union rewriting is done, so a partition reader is preferred. @@ -3525,7 +3525,7 @@ func buildNoRangeIndexReader(b *executorBuilder, v *plannercore.PhysicalIndexRea e.feedback = statistics.NewQueryFeedback(0, nil, 0, is.Desc) } else { tblID := e.physicalTableID - if b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if b.ctx.GetSessionVars().UseDynamicPartitionPrune() { tblID = e.table.Meta().ID } e.feedback = statistics.NewQueryFeedback(tblID, is.Hist, int64(is.StatsCount()), is.Desc) @@ -3568,7 +3568,7 @@ func (b *executorBuilder) buildIndexReader(v *plannercore.PhysicalIndexReader) E sctx := b.ctx.GetSessionVars().StmtCtx sctx.IndexNames = append(sctx.IndexNames, is.Table.Name.O+":"+is.Index.Name.O) - if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { return ret } // When isPartition is set, it means the union rewriting is done, so a partition reader is preferred. @@ -3744,7 +3744,7 @@ func (b *executorBuilder) buildIndexLookUpReader(v *plannercore.PhysicalIndexLoo sctx.IndexNames = append(sctx.IndexNames, is.Table.Name.O+":"+is.Index.Name.O) sctx.TableIDs = append(sctx.TableIDs, ts.Table.ID) - if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { return ret } @@ -3881,7 +3881,7 @@ func (b *executorBuilder) buildIndexMergeReader(v *plannercore.PhysicalIndexMerg sctx.TableIDs = append(sctx.TableIDs, ts.Table.ID) executorCounterIndexMergeReaderExecutor.Inc() - if !b.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { return ret } @@ -3999,7 +3999,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte return nil, err } tbInfo := e.table.Meta() - if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { if v.IsCommonHandle { kvRanges, err := buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), -1, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { @@ -4247,7 +4247,7 @@ func (builder *dataReaderBuilder) buildIndexReaderForIndexJoin(ctx context.Conte return nil, err } tbInfo := e.table.Meta() - if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { kvRanges, err := buildKvRangesForIndexJoin(e.ctx, e.physicalTableID, e.index.ID, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memoryTracker, interruptSignal) if err != nil { return nil, err @@ -4294,7 +4294,7 @@ func (builder *dataReaderBuilder) buildIndexLookUpReaderForIndexJoin(ctx context } tbInfo := e.table.Meta() - if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if tbInfo.GetPartitionInfo() == nil || !builder.ctx.GetSessionVars().UseDynamicPartitionPrune() { e.kvRanges, err = buildKvRangesForIndexJoin(e.ctx, getPhysicalTableID(e.table), e.index.ID, lookUpContents, indexRanges, keyOff2IdxOff, cwc, memTracker, interruptSignal) if err != nil { return nil, err @@ -4880,7 +4880,7 @@ func getPhysicalTableID(t table.Table) int64 { } func getFeedbackStatsTableID(ctx sessionctx.Context, t table.Table) int64 { - if p, ok := t.(table.PhysicalTable); ok && !ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if p, ok := t.(table.PhysicalTable); ok && !ctx.GetSessionVars().UseDynamicPartitionPrune() { return p.GetPhysicalID() } return t.Meta().ID diff --git a/executor/distsql_test.go b/executor/distsql_test.go index 1f895a6703420..a0fc642a020d7 100644 --- a/executor/distsql_test.go +++ b/executor/distsql_test.go @@ -291,8 +291,6 @@ func TestPushLimitDownIndexLookUpReader(t *testing.T) { } func TestPartitionTableIndexLookUpReader(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/executor/executor.go b/executor/executor.go index 73238ee3cdcb1..e3ac232ba0d53 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1926,13 +1926,6 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.OptimizerCETrace = nil sc.StatsLoadStatus = make(map[model.TableItemID]string) sc.IsSyncStatsFailed = false - // Firstly we assume that UseDynamicPruneMode can be enabled according session variable, then we will check other conditions - // in PlanBuilder.buildDataSource - if ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { - sc.UseDynamicPruneMode = true - } else { - sc.UseDynamicPruneMode = false - } sc.SysdateIsNow = ctx.GetSessionVars().SysdateIsNow diff --git a/executor/executor_issue_test.go b/executor/executor_issue_test.go index febdbd6031823..8e8b4d28f0fa3 100644 --- a/executor/executor_issue_test.go +++ b/executor/executor_issue_test.go @@ -583,8 +583,6 @@ func TestFix31537(t *testing.T) { } func TestIssue30382(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index 24168058353ec..dfd9f04a69140 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -21,7 +21,6 @@ import ( "strings" "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/testkit" "github.com/stretchr/testify/require" ) @@ -392,8 +391,6 @@ func TestIssue24547(t *testing.T) { } func TestIssue27138(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index 1f70c5bc04630..8daef02d4b53e 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" @@ -64,8 +63,6 @@ func TestSetPartitionPruneMode(t *testing.T) { } func TestFourReader(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -562,8 +559,6 @@ func TestView(t *testing.T) { } func TestDirectReadingwithIndexJoin(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -676,8 +671,6 @@ func TestDirectReadingwithIndexJoin(t *testing.T) { } func TestDynamicPruningUnderIndexJoin(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -980,8 +973,6 @@ func TestGlobalStatsAndSQLBinding(t *testing.T) { } func TestPartitionTableWithDifferentJoin(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -1770,8 +1761,6 @@ func TestDynamicPruneModeWithExpression(t *testing.T) { } func TestAddDropPartitions(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -1805,8 +1794,6 @@ func TestAddDropPartitions(t *testing.T) { } func TestMPPQueryExplainInfo(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -1836,8 +1823,6 @@ func TestMPPQueryExplainInfo(t *testing.T) { } func TestPartitionPruningInTransaction(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2071,8 +2056,6 @@ func TestSubqueries(t *testing.T) { } func TestSplitRegion(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2109,9 +2092,6 @@ func TestSplitRegion(t *testing.T) { } func TestParallelApply(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") - store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2619,8 +2599,6 @@ func TestDirectReadingWithAgg(t *testing.T) { } func TestDynamicModeByDefault(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2816,8 +2794,6 @@ func TestIssue25309(t *testing.T) { } func TestGlobalIndexScan(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2838,8 +2814,6 @@ partition p2 values less than (10))`) } func TestGlobalIndexDoubleRead(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/executor/show_stats.go b/executor/show_stats.go index 058b3ed7c62da..597c6923ccc6f 100644 --- a/executor/show_stats.go +++ b/executor/show_stats.go @@ -108,7 +108,7 @@ func (e *ShowExec) fetchShowStatsMeta() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { + if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { partitionName := "" if pi != nil { partitionName = "global" @@ -150,7 +150,7 @@ func (e *ShowExec) fetchShowStatsHistogram() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { + if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { partitionName := "" if pi != nil { partitionName = "global" @@ -224,7 +224,7 @@ func (e *ShowExec) fetchShowStatsBuckets() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { + if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { partitionName := "" if pi != nil { partitionName = "global" @@ -283,7 +283,7 @@ func (e *ShowExec) fetchShowStatsTopN() error { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { + if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { partitionName := "" if pi != nil { partitionName = "global" @@ -415,7 +415,7 @@ func (e *ShowExec) fetchShowStatsHealthy() { for _, db := range dbs { for _, tbl := range db.Tables { pi := tbl.GetPartitionInfo() - if pi == nil || e.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() { + if pi == nil || e.ctx.GetSessionVars().UseDynamicPartitionPrune() { partitionName := "" if pi != nil { partitionName = "global" diff --git a/executor/tiflashtest/tiflash_test.go b/executor/tiflashtest/tiflash_test.go index d18dc2c87132c..5725ca22df4e2 100644 --- a/executor/tiflashtest/tiflash_test.go +++ b/executor/tiflashtest/tiflash_test.go @@ -457,8 +457,6 @@ func TestTiFlashPartitionTableReader(t *testing.T) { } func TestPartitionTable(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t, withMockTiFlash(2)) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/executor/write_test.go b/executor/write_test.go index a5ef9da73ce00..0c28d521c61a1 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -21,7 +21,6 @@ import ( "strconv" "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" @@ -3238,8 +3237,6 @@ func TestWriteListPartitionTable1(t *testing.T) { // TestWriteListPartitionTable2 test for write list partition when the partition expression is complicated and contain generated column. func TestWriteListPartitionTable2(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -3362,8 +3359,6 @@ func TestWriteListPartitionTable2(t *testing.T) { } func TestWriteListColumnsPartitionTable1(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -4205,8 +4200,6 @@ func TestUpdate(t *testing.T) { } func TestListColumnsPartitionWithGlobalIndex(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/planner/cascades/integration_test.go b/planner/cascades/integration_test.go index ebb80b192a383..e3a7b750373ec 100644 --- a/planner/cascades/integration_test.go +++ b/planner/cascades/integration_test.go @@ -18,7 +18,6 @@ import ( "fmt" "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/planner/cascades" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/testkit" @@ -376,8 +375,6 @@ func TestTopN(t *testing.T) { } func TestCascadePlannerHashedPartTable(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/access_object.go b/planner/core/access_object.go index c57f7edc06b90..81296c39c9223 100644 --- a/planner/core/access_object.go +++ b/planner/core/access_object.go @@ -352,7 +352,7 @@ func (p *BatchPointGetPlan) AccessObject() AccessObject { func getDynamicAccessPartition(sctx sessionctx.Context, tblInfo *model.TableInfo, partitionInfo *PartitionInfo, asName string) (res *DynamicPartitionAccessObject) { pi := tblInfo.GetPartitionInfo() - if pi == nil || !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if pi == nil || !sctx.GetSessionVars().UseDynamicPartitionPrune() { return nil } @@ -392,7 +392,7 @@ func getDynamicAccessPartition(sctx sessionctx.Context, tblInfo *model.TableInfo } func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) AccessObject { - if !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if !sctx.GetSessionVars().UseDynamicPartitionPrune() { return DynamicPartitionAccessObjects(nil) } if len(p.PartitionInfos) == 0 { @@ -444,7 +444,7 @@ func (p *PhysicalTableReader) accessObject(sctx sessionctx.Context) AccessObject } func (p *PhysicalIndexReader) accessObject(sctx sessionctx.Context) AccessObject { - if !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if !sctx.GetSessionVars().UseDynamicPartitionPrune() { return DynamicPartitionAccessObjects(nil) } is := p.IndexPlans[0].(*PhysicalIndexScan) @@ -460,7 +460,7 @@ func (p *PhysicalIndexReader) accessObject(sctx sessionctx.Context) AccessObject } func (p *PhysicalIndexLookUpReader) accessObject(sctx sessionctx.Context) AccessObject { - if !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if !sctx.GetSessionVars().UseDynamicPartitionPrune() { return DynamicPartitionAccessObjects(nil) } ts := p.TablePlans[0].(*PhysicalTableScan) @@ -476,7 +476,7 @@ func (p *PhysicalIndexLookUpReader) accessObject(sctx sessionctx.Context) Access } func (p *PhysicalIndexMergeReader) accessObject(sctx sessionctx.Context) AccessObject { - if !sctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if !sctx.GetSessionVars().UseDynamicPartitionPrune() { return DynamicPartitionAccessObjects(nil) } ts := p.TablePlans[0].(*PhysicalTableScan) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 31ba6bfeb3e07..d8d46c04d2aaa 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -23,7 +23,6 @@ import ( "strings" "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/parser/model" @@ -820,8 +819,6 @@ func TestLimitIndexEstimation(t *testing.T) { } func TestBatchPointGetTablePartition(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) testKit := testkit.NewTestKit(t, store) testKit.MustExec("use test") diff --git a/planner/core/collect_column_stats_usage_test.go b/planner/core/collect_column_stats_usage_test.go index 38d246ff8bfd7..d494235bc66fc 100644 --- a/planner/core/collect_column_stats_usage_test.go +++ b/planner/core/collect_column_stats_usage_test.go @@ -20,7 +20,6 @@ import ( "sort" "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/util/hint" @@ -273,8 +272,6 @@ func TestCollectPredicateColumns(t *testing.T) { } func TestCollectHistNeededColumns(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") tests := []struct { pruneMode string sql string diff --git a/planner/core/explain.go b/planner/core/explain.go index 62ece42048433..d056eebc37d60 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -214,7 +214,7 @@ func (p *PhysicalTableScan) OperatorInfo(normalized bool) string { if p.stats.StatsVersion == statistics.PseudoVersion && !normalized { buffer.WriteString(", stats:pseudo") } - if p.StoreType == kv.TiFlash && p.Table.GetPartitionInfo() != nil && p.IsMPPOrBatchCop && p.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if p.StoreType == kv.TiFlash && p.Table.GetPartitionInfo() != nil && p.IsMPPOrBatchCop && p.ctx.GetSessionVars().UseDynamicPartitionPrune() { buffer.WriteString(", PartitionTableScan:true") } return buffer.String() diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index ebd23f271aadd..7c2372f14fa23 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -936,7 +936,7 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter var hashPartColName *ast.ColumnName if tblInfo := ds.table.Meta(); canConvertPointGet && tblInfo.GetPartitionInfo() != nil { // We do not build [batch] point get for dynamic table partitions now. This can be optimized. - if ds.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if ds.ctx.GetSessionVars().UseDynamicPartitionPrune() { canConvertPointGet = false } if canConvertPointGet && len(path.Ranges) > 1 { diff --git a/planner/core/integration_partition_test.go b/planner/core/integration_partition_test.go index 353fc379be794..e8c53739d2819 100644 --- a/planner/core/integration_partition_test.go +++ b/planner/core/integration_partition_test.go @@ -21,7 +21,6 @@ import ( "strings" "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/parser/auth" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" @@ -62,9 +61,6 @@ func TestListPartitionPushDown(t *testing.T) { } func TestListColVariousTypes(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") - store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -106,9 +102,6 @@ func TestListColVariousTypes(t *testing.T) { } func TestListPartitionPruning(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") - store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index da230b745d1a5..17796ca144b86 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1452,8 +1452,6 @@ func TestPartitionTableStats(t *testing.T) { } func TestPartitionPruningForInExpr(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -1479,8 +1477,6 @@ func TestPartitionPruningForInExpr(t *testing.T) { } func TestPartitionPruningWithDateType(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -3413,8 +3409,6 @@ func TestExplainAnalyzeDML2(t *testing.T) { } func TestPartitionExplain(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -3589,8 +3583,6 @@ func TestPartitionUnionWithPPruningColumn(t *testing.T) { } func TestIssue20139(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -6561,9 +6553,6 @@ func TestIssue32632(t *testing.T) { } func TestTiFlashPartitionTableScan(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") - store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 70248a7a20e36..6a9199da2d922 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -26,7 +26,6 @@ import ( "unicode" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" @@ -4173,7 +4172,7 @@ func getStatsTable(ctx sessionctx.Context, tblInfo *model.TableInfo, pid int64) } var statsTbl *statistics.Table - if pid == tblInfo.ID || ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if pid == tblInfo.ID || ctx.GetSessionVars().UseDynamicPartitionPrune() { statsTbl = statsHandle.GetTableStats(tblInfo, handle.WithTableStatsByQuery()) } else { statsTbl = statsHandle.GetPartitionStats(tblInfo, pid, handle.WithTableStatsByQuery()) @@ -4390,24 +4389,9 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as } if tableInfo.GetPartitionInfo() != nil { - h := domain.GetDomain(b.ctx).StatsHandle() - tblStats := h.GetTableStats(tableInfo) - isDynamicEnabled := b.ctx.GetSessionVars().IsDynamicPartitionPruneEnabled() - globalStatsReady := tblStats.IsInitialized() - // If dynamic partition prune isn't enabled or global stats is not ready, we won't enable dynamic prune mode in query - usePartitionProcessor := !isDynamicEnabled || !globalStatsReady - - failpoint.Inject("forceDynamicPrune", func(val failpoint.Value) { - if val.(bool) { - if isDynamicEnabled { - usePartitionProcessor = false - } - } - }) - - if usePartitionProcessor { + // Use the new partition implementation, clean up the code here when it's full implemented. + if !b.ctx.GetSessionVars().UseDynamicPartitionPrune() { b.optFlag = b.optFlag | flagPartitionProcessor - b.ctx.GetSessionVars().StmtCtx.UseDynamicPruneMode = false } pt := tbl.(table.PartitionedTable) diff --git a/planner/core/partition_pruner_test.go b/planner/core/partition_pruner_test.go index fffef54373bd8..d3db641a8efd9 100644 --- a/planner/core/partition_pruner_test.go +++ b/planner/core/partition_pruner_test.go @@ -21,7 +21,6 @@ import ( "strings" "testing" - "github.com/pingcap/failpoint" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/testkit" @@ -30,8 +29,6 @@ import ( ) func TestHashPartitionPruner(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("create database test_partition") @@ -253,8 +250,6 @@ func TestRangeColumnPartitionPruningForInString(t *testing.T) { } func TestListPartitionPruner(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("drop database if exists test_partition;") @@ -326,8 +321,6 @@ func TestListPartitionPruner(t *testing.T) { } func TestListColumnsPartitionPruner(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("set @@session.tidb_enable_list_partition = ON") @@ -539,8 +532,6 @@ func TestListColumnsPartitionPrunerRandom(t *testing.T) { } func TestIssue22635(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("USE test;") diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 792966ff6cebf..13187d28042ce 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -21,7 +21,6 @@ import ( "strings" "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" @@ -1317,8 +1316,6 @@ func doTestPushdownDistinct(t *testing.T, vars, input []string, output []struct } func TestGroupConcatOrderby(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") var ( input []string output []struct { @@ -2126,9 +2123,6 @@ func TestHJBuildAndProbeHint4StaticPartitionTable(t *testing.T) { } func TestHJBuildAndProbeHint4DynamicPartitionTable(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") - var ( input []string output []struct { diff --git a/planner/core/plan_to_pb.go b/planner/core/plan_to_pb.go index 59194e5b37286..0bd07a4415f20 100644 --- a/planner/core/plan_to_pb.go +++ b/planner/core/plan_to_pb.go @@ -185,7 +185,7 @@ func (p *PhysicalLimit) ToPB(ctx sessionctx.Context, storeType kv.StoreType) (*t // ToPB implements PhysicalPlan ToPB interface. func (p *PhysicalTableScan) ToPB(ctx sessionctx.Context, storeType kv.StoreType) (*tipb.Executor, error) { - if storeType == kv.TiFlash && p.Table.GetPartitionInfo() != nil && p.IsMPPOrBatchCop && p.ctx.GetSessionVars().StmtCtx.UseDynamicPartitionPrune() { + if storeType == kv.TiFlash && p.Table.GetPartitionInfo() != nil && p.IsMPPOrBatchCop && p.ctx.GetSessionVars().UseDynamicPartitionPrune() { return p.partitionTableScanToPBForFlash(ctx) } tsExec := tables.BuildTableScanFromInfos(p.Table, p.Columns) diff --git a/planner/core/point_get_plan_test.go b/planner/core/point_get_plan_test.go index 3a92d25719c09..71804ae0fc00e 100644 --- a/planner/core/point_get_plan_test.go +++ b/planner/core/point_get_plan_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/planner" @@ -654,8 +653,6 @@ func TestBatchPointGetPartition(t *testing.T) { } func TestBatchPointGetPartitionForAccessObject(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/rule_join_reorder_test.go b/planner/core/rule_join_reorder_test.go index a0c136d8d4a2f..a981495807d3d 100644 --- a/planner/core/rule_join_reorder_test.go +++ b/planner/core/rule_join_reorder_test.go @@ -17,7 +17,6 @@ package core_test import ( "testing" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/parser/model" plannercore "github.com/pingcap/tidb/planner/core" @@ -234,8 +233,6 @@ func TestJoinOrderHint4StaticPartitionTable(t *testing.T) { } func TestJoinOrderHint4DynamicPartitionTable(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index 470ff975e7724..99e00ae9bbdf5 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -317,8 +317,6 @@ type StatementContext struct { StatsLoadStatus map[model.TableItemID]string // IsSyncStatsFailed indicates whether any failure happened during sync stats IsSyncStatsFailed bool - // UseDynamicPruneMode indicates whether use UseDynamicPruneMode in query stmt - UseDynamicPruneMode bool // ColRefFromPlan mark the column ref used by assignment in update statement. ColRefFromUpdatePlan []int64 @@ -1000,11 +998,6 @@ func (sc *StatementContext) RecordRangeFallback(rangeMaxSize int64) { } } -// UseDynamicPartitionPrune indicates whether dynamic partition is used during the query -func (sc *StatementContext) UseDynamicPartitionPrune() bool { - return sc.UseDynamicPruneMode -} - // CopTasksDetails collects some useful information of cop-tasks during execution. type CopTasksDetails struct { NumCopTasks int diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index ecb4b5b93c638..71d54989d8bf7 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1341,10 +1341,8 @@ func (s *SessionVars) CheckAndGetTxnScope() string { return kv.GlobalTxnScope } -// IsDynamicPartitionPruneEnabled indicates whether dynamic partition prune enabled -// Note that: IsDynamicPartitionPruneEnabled only indicates whether dynamic partition prune mode is enabled according to -// session variable, it isn't guaranteed to be used during query due to other conditions checking. -func (s *SessionVars) IsDynamicPartitionPruneEnabled() bool { +// UseDynamicPartitionPrune indicates whether use new dynamic partition prune. +func (s *SessionVars) UseDynamicPartitionPrune() bool { return PartitionPruneMode(s.PartitionPruneMode.Load()) == Dynamic } diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index f8f73b933bd9a..4243366f4016d 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -2149,9 +2149,6 @@ func TestAnalyzeWithDynamicPartitionPruneMode(t *testing.T) { } func TestPartitionPruneModeSessionVariable(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") - store := testkit.CreateMockStore(t) tk1 := testkit.NewTestKit(t, store) tk1.MustExec("use test") diff --git a/statistics/integration_test.go b/statistics/integration_test.go index c4e4d315c7dbe..5abba85ff9193 100644 --- a/statistics/integration_test.go +++ b/statistics/integration_test.go @@ -300,8 +300,6 @@ func TestExpBackoffEstimation(t *testing.T) { } func TestGlobalStats(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") diff --git a/table/tables/partition_test.go b/table/tables/partition_test.go index aed5ef219ad87..b05cf8f5037bd 100644 --- a/table/tables/partition_test.go +++ b/table/tables/partition_test.go @@ -18,7 +18,6 @@ import ( "context" "testing" - "github.com/pingcap/failpoint" mysql "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/model" @@ -394,8 +393,6 @@ func TestLocatePartitionSingleColumn(t *testing.T) { } func TestLocatePartition(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/planner/core/forceDynamicPrune", `return(true)`) - defer failpoint.Disable("github.com/pingcap/tidb/planner/core/forceDynamicPrune") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") From 67b9cca2c9882ccd0329019811808a73c8fd316c Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Thu, 15 Sep 2022 15:11:52 +0200 Subject: [PATCH 11/12] Updated test results --- executor/show_stats_test.go | 27 +++++++++++++++++---------- statistics/handle/handle_test.go | 13 ++++++++----- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/executor/show_stats_test.go b/executor/show_stats_test.go index cb8bdebcad8ac..bf15298c33fce 100644 --- a/executor/show_stats_test.go +++ b/executor/show_stats_test.go @@ -201,25 +201,32 @@ func TestShowPartitionStats(t *testing.T) { tk.MustExec("analyze table t") result := tk.MustQuery("show stats_meta") - require.Len(t, result.Rows(), 1) + require.Len(t, result.Rows(), 2) require.Equal(t, "test", result.Rows()[0][0]) require.Equal(t, "t", result.Rows()[0][1]) - require.Equal(t, "p0", result.Rows()[0][2]) + require.Equal(t, "global", result.Rows()[0][2]) + require.Equal(t, "test", result.Rows()[1][0]) + require.Equal(t, "t", result.Rows()[1][1]) + require.Equal(t, "p0", result.Rows()[1][2]) result = tk.MustQuery("show stats_histograms").Sort() - require.Len(t, result.Rows(), 3) - require.Equal(t, "p0", result.Rows()[0][2]) - require.Equal(t, "a", result.Rows()[0][3]) - require.Equal(t, "p0", result.Rows()[1][2]) - require.Equal(t, "b", result.Rows()[1][3]) + require.Len(t, result.Rows(), 5) + require.Equal(t, "global", result.Rows()[0][2]) + require.Equal(t, "b", result.Rows()[0][3]) + require.Equal(t, "global", result.Rows()[1][2]) + require.Equal(t, "idx", result.Rows()[1][3]) require.Equal(t, "p0", result.Rows()[2][2]) - require.Equal(t, "idx", result.Rows()[2][3]) + require.Equal(t, "a", result.Rows()[2][3]) + require.Equal(t, "p0", result.Rows()[3][2]) + require.Equal(t, "b", result.Rows()[3][3]) + require.Equal(t, "p0", result.Rows()[4][2]) + require.Equal(t, "idx", result.Rows()[4][3]) result = tk.MustQuery("show stats_buckets").Sort() - result.Check(testkit.Rows("test t p0 a 0 0 1 1 1 1 0", "test t p0 b 0 0 1 1 1 1 0", "test t p0 idx 1 0 1 1 1 1 0")) + result.Check(testkit.Rows("test t global b 0 0 1 1 1 1 0", "test t global idx 1 0 1 1 1 1 0", "test t p0 a 0 0 1 1 1 1 0", "test t p0 b 0 0 1 1 1 1 0", "test t p0 idx 1 0 1 1 1 1 0")) result = tk.MustQuery("show stats_healthy") - result.Check(testkit.Rows("test t p0 100")) + result.Check(testkit.Rows("test t global 100", "test t p0 100")) }) } diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index f618bf7c66490..c613eacf85063 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -828,12 +828,13 @@ func TestBuildGlobalLevelStats(t *testing.T) { testKit.MustExec("create index idx_t_b on t(b);") testKit.MustExec("analyze table t, t1;") result := testKit.MustQuery("show stats_meta where table_name = 't';").Sort() - require.Len(t, result.Rows(), 3) - require.Equal(t, "1", result.Rows()[0][5]) - require.Equal(t, "2", result.Rows()[1][5]) + require.Len(t, result.Rows(), 4) + require.Equal(t, "5", result.Rows()[0][5]) + require.Equal(t, "1", result.Rows()[1][5]) require.Equal(t, "2", result.Rows()[2][5]) + require.Equal(t, "2", result.Rows()[3][5]) result = testKit.MustQuery("show stats_histograms where table_name = 't';").Sort() - require.Len(t, result.Rows(), 15) + require.Len(t, result.Rows(), 20) result = testKit.MustQuery("show stats_meta where table_name = 't1';").Sort() require.Len(t, result.Rows(), 1) @@ -843,6 +844,8 @@ func TestBuildGlobalLevelStats(t *testing.T) { // Test the 'dynamic' mode testKit.MustExec("set @@tidb_partition_prune_mode = 'dynamic';") + testKit.MustExec("drop stats t1") + testKit.MustExec("drop stats t") testKit.MustExec("analyze table t, t1;") result = testKit.MustQuery("show stats_meta where table_name = 't'").Sort() require.Len(t, result.Rows(), 4) @@ -1665,7 +1668,7 @@ partition by range (a) ( tk.MustExec("set @@tidb_partition_prune_mode='static'") tk.MustExec("set @@session.tidb_analyze_version=1") tk.MustExec("analyze table t") // both p0 and p1 are in ver1 - require.Len(t, tk.MustQuery("show stats_meta").Rows(), 2) + require.Len(t, tk.MustQuery("show stats_meta").Rows(), 3) tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") tk.MustExec("set @@session.tidb_analyze_version=1") From 67ccb8b59fe162dcfd5ecff3fef10fea20a87240 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Fri, 16 Sep 2022 14:03:04 +0200 Subject: [PATCH 12/12] Updated test --- statistics/handle/handle_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index c613eacf85063..335d49c943c3f 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -791,17 +791,18 @@ func TestShowGlobalStats(t *testing.T) { tk.MustExec("create table t (a int, key(a)) partition by hash(a) partitions 2") tk.MustExec("insert into t values (1), (2), (3), (4)") tk.MustExec("analyze table t with 1 buckets") - require.Len(t, tk.MustQuery("show stats_meta").Rows(), 2) - require.Len(t, tk.MustQuery("show stats_meta where partition_name='global'").Rows(), 0) - require.Len(t, tk.MustQuery("show stats_buckets").Rows(), 4) // 2 partitions * (1 for the column_a and 1 for the index_a) - require.Len(t, tk.MustQuery("show stats_buckets where partition_name='global'").Rows(), 0) - require.Len(t, tk.MustQuery("show stats_histograms").Rows(), 4) - require.Len(t, tk.MustQuery("show stats_histograms where partition_name='global'").Rows(), 0) - require.Len(t, tk.MustQuery("show stats_healthy").Rows(), 2) - require.Len(t, tk.MustQuery("show stats_healthy where partition_name='global'").Rows(), 0) + require.Len(t, tk.MustQuery("show stats_meta").Rows(), 3) + require.Len(t, tk.MustQuery("show stats_meta where partition_name='global'").Rows(), 1) + require.Len(t, tk.MustQuery("show stats_buckets").Rows(), 6) + require.Len(t, tk.MustQuery("show stats_buckets where partition_name='global'").Rows(), 2) + require.Len(t, tk.MustQuery("show stats_histograms").Rows(), 6) + require.Len(t, tk.MustQuery("show stats_histograms where partition_name='global'").Rows(), 2) + require.Len(t, tk.MustQuery("show stats_healthy").Rows(), 3) + require.Len(t, tk.MustQuery("show stats_healthy where partition_name='global'").Rows(), 1) tk.MustExec("set @@tidb_analyze_version = 2") tk.MustExec("set @@tidb_partition_prune_mode = 'dynamic'") + tk.MustExec(`drop stats t`) tk.MustExec("analyze table t with 0 topn, 1 buckets") require.Len(t, tk.MustQuery("show stats_meta").Rows(), 3) require.Len(t, tk.MustQuery("show stats_meta where partition_name='global'").Rows(), 1)