Skip to content

Commit

Permalink
statstest: split handletest (#44165)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hawkingrei
hawkingrei committed May 25, 2023
1 parent f3931d0 commit d692730
Show file tree
Hide file tree
Showing 4 changed files with 398 additions and 313 deletions.
313 changes: 0 additions & 313 deletions statistics/handle/handletest/handle_test.go
View file
Expand Up @@ -43,136 +43,6 @@ import (
"github.com/tikv/client-go/v2/oracle"
)

func TestStatsCache(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
testKit := testkit.NewTestKit(t, store)
testKit.MustExec("use test")
testKit.MustExec("create table t (c1 int, c2 int)")
testKit.MustExec("insert into t values(1, 2)")
do := dom
is := do.InfoSchema()
tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
tableInfo := tbl.Meta()
statsTbl := do.StatsHandle().GetTableStats(tableInfo)
require.True(t, statsTbl.Pseudo)
testKit.MustExec("analyze table t")
statsTbl = do.StatsHandle().GetTableStats(tableInfo)
require.False(t, statsTbl.Pseudo)
testKit.MustExec("create index idx_t on t(c1)")
do.InfoSchema()
statsTbl = do.StatsHandle().GetTableStats(tableInfo)
// If index is build, but stats is not updated. statsTbl can also work.
require.False(t, statsTbl.Pseudo)
// But the added index will not work.
require.Nil(t, statsTbl.Indices[int64(1)])

testKit.MustExec("analyze table t")
statsTbl = do.StatsHandle().GetTableStats(tableInfo)
require.False(t, statsTbl.Pseudo)
// If the new schema drop a column, the table stats can still work.
testKit.MustExec("alter table t drop column c2")
is = do.InfoSchema()
do.StatsHandle().Clear()
err = do.StatsHandle().Update(is)
require.NoError(t, err)
statsTbl = do.StatsHandle().GetTableStats(tableInfo)
require.False(t, statsTbl.Pseudo)

// If the new schema add a column, the table stats can still work.
testKit.MustExec("alter table t add column c10 int")
is = do.InfoSchema()

do.StatsHandle().Clear()
err = do.StatsHandle().Update(is)
require.NoError(t, err)
statsTbl = do.StatsHandle().GetTableStats(tableInfo)
require.False(t, statsTbl.Pseudo)
}

func TestStatsCacheMemTracker(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
testKit := testkit.NewTestKit(t, store)
testKit.MustExec("use test")
testKit.MustExec("create table t (c1 int, c2 int,c3 int)")
testKit.MustExec("insert into t values(1, 2, 3)")
do := dom
is := do.InfoSchema()
tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
tableInfo := tbl.Meta()
statsTbl := do.StatsHandle().GetTableStats(tableInfo)
require.True(t, statsTbl.MemoryUsage().TotalMemUsage == 0)
require.True(t, statsTbl.Pseudo)

testKit.MustExec("analyze table t")
statsTbl = do.StatsHandle().GetTableStats(tableInfo)

require.False(t, statsTbl.Pseudo)
testKit.MustExec("create index idx_t on t(c1)")
do.InfoSchema()
statsTbl = do.StatsHandle().GetTableStats(tableInfo)

// If index is build, but stats is not updated. statsTbl can also work.
require.False(t, statsTbl.Pseudo)
// But the added index will not work.
require.Nil(t, statsTbl.Indices[int64(1)])

testKit.MustExec("analyze table t")
statsTbl = do.StatsHandle().GetTableStats(tableInfo)

require.False(t, statsTbl.Pseudo)

// If the new schema drop a column, the table stats can still work.
testKit.MustExec("alter table t drop column c2")
is = do.InfoSchema()
do.StatsHandle().Clear()
err = do.StatsHandle().Update(is)
require.NoError(t, err)

statsTbl = do.StatsHandle().GetTableStats(tableInfo)
require.True(t, statsTbl.MemoryUsage().TotalMemUsage > 0)
require.False(t, statsTbl.Pseudo)

// If the new schema add a column, the table stats can still work.
testKit.MustExec("alter table t add column c10 int")
is = do.InfoSchema()

do.StatsHandle().Clear()
err = do.StatsHandle().Update(is)
require.NoError(t, err)
statsTbl = do.StatsHandle().GetTableStats(tableInfo)
require.False(t, statsTbl.Pseudo)
}

func TestStatsStoreAndLoad(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
testKit := testkit.NewTestKit(t, store)
testKit.MustExec("use test")
testKit.MustExec("create table t (c1 int, c2 int)")
recordCount := 1000
for i := 0; i < recordCount; i++ {
testKit.MustExec("insert into t values (?, ?)", i, i+1)
}
testKit.MustExec("create index idx_t on t(c2)")
do := dom
is := do.InfoSchema()
tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
tableInfo := tbl.Meta()

testKit.MustExec("analyze table t")
statsTbl1 := do.StatsHandle().GetTableStats(tableInfo)

do.StatsHandle().Clear()
err = do.StatsHandle().Update(is)
require.NoError(t, err)
statsTbl2 := do.StatsHandle().GetTableStats(tableInfo)
require.False(t, statsTbl2.Pseudo)
require.Equal(t, int64(recordCount), statsTbl2.RealtimeCount)
internal.AssertTableEqual(t, statsTbl1, statsTbl2)
}

func TestEmptyTable(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
testKit := testkit.NewTestKit(t, store)
Expand Down Expand Up @@ -427,93 +297,6 @@ func TestLoadHist(t *testing.T) {
require.Greater(t, newStatsTbl2.Columns[int64(3)].LastUpdateVersion, newStatsTbl2.Columns[int64(1)].LastUpdateVersion)
}

func TestInitStats(t *testing.T) {
originValue := config.GetGlobalConfig().Performance.LiteInitStats
defer func() {
config.GetGlobalConfig().Performance.LiteInitStats = originValue
}()
config.GetGlobalConfig().Performance.LiteInitStats = false
store, dom := testkit.CreateMockStoreAndDomain(t)
testKit := testkit.NewTestKit(t, store)
testKit.MustExec("use test")
testKit.MustExec("set @@session.tidb_analyze_version = 1")
testKit.MustExec("create table t(a int, b int, c int, primary key(a), key idx(b))")
testKit.MustExec("insert into t values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,7,8)")
testKit.MustExec("analyze table t")
h := dom.StatsHandle()
is := dom.InfoSchema()
tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
// `Update` will not use load by need strategy when `Lease` is 0, and `InitStats` is only called when
// `Lease` is not 0, so here we just change it.
h.SetLease(time.Millisecond)

h.Clear()
require.NoError(t, h.InitStats(is))
table0 := h.GetTableStats(tbl.Meta())
cols := table0.Columns
require.Equal(t, uint8(0x36), cols[1].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x37), cols[2].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x38), cols[3].LastAnalyzePos.GetBytes()[0])
h.Clear()
require.NoError(t, h.Update(is))
table1 := h.GetTableStats(tbl.Meta())
internal.AssertTableEqual(t, table0, table1)
h.SetLease(0)
}

func TestInitStatsVer2(t *testing.T) {
originValue := config.GetGlobalConfig().Performance.LiteInitStats
defer func() {
config.GetGlobalConfig().Performance.LiteInitStats = originValue
}()
config.GetGlobalConfig().Performance.LiteInitStats = false
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set @@session.tidb_analyze_version=2")
tk.MustExec("create table t(a int, b int, c int, index idx(a), index idxab(a, b))")
tk.MustExec("insert into t values(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4), (4, 4, 4), (4, 4, 4)")
tk.MustExec("analyze table t with 2 topn, 3 buckets")
h := dom.StatsHandle()
is := dom.InfoSchema()
tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
// `Update` will not use load by need strategy when `Lease` is 0, and `InitStats` is only called when
// `Lease` is not 0, so here we just change it.
h.SetLease(time.Millisecond)

h.Clear()
require.NoError(t, h.InitStats(is))
table0 := h.GetTableStats(tbl.Meta())
cols := table0.Columns
require.Equal(t, uint8(0x33), cols[1].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x33), cols[2].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x33), cols[3].LastAnalyzePos.GetBytes()[0])
h.Clear()
require.NoError(t, h.Update(is))
table1 := h.GetTableStats(tbl.Meta())
internal.AssertTableEqual(t, table0, table1)
h.SetLease(0)
}

func TestInitStatsIssue41938(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set @@global.tidb_analyze_version=1")
tk.MustExec("set @@session.tidb_analyze_version=1")
tk.MustExec("create table t1 (a timestamp primary key)")
tk.MustExec("insert into t1 values ('2023-03-07 14:24:30'), ('2023-03-07 14:24:31'), ('2023-03-07 14:24:32'), ('2023-03-07 14:24:33')")
tk.MustExec("analyze table t1 with 0 topn")
h := dom.StatsHandle()
// `InitStats` is only called when `Lease` is not 0, so here we just change it.
h.SetLease(time.Millisecond)
h.Clear()
require.NoError(t, h.InitStats(dom.InfoSchema()))
h.SetLease(0)
}

func TestReloadExtStatsLockRelease(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
Expand All @@ -531,102 +314,6 @@ func TestReloadExtStatsLockRelease(t *testing.T) {
tk.MustExec("analyze table t")
}

func TestLoadStats(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
testKit := testkit.NewTestKit(t, store)
testKit.MustExec("use test")
testKit.MustExec("drop table if exists t")
testKit.MustExec("set @@session.tidb_analyze_version=1")
testKit.MustExec("create table t(a int, b int, c int, primary key(a), key idx(b))")
testKit.MustExec("insert into t values (1,1,1),(2,2,2),(3,3,3)")

oriLease := dom.StatsHandle().Lease()
dom.StatsHandle().SetLease(1)
defer func() {
dom.StatsHandle().SetLease(oriLease)
}()
testKit.MustExec("analyze table t")

is := dom.InfoSchema()
tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
tableInfo := tbl.Meta()
colAID := tableInfo.Columns[0].ID
colCID := tableInfo.Columns[2].ID
idxBID := tableInfo.Indices[0].ID
h := dom.StatsHandle()

// Index/column stats are not be loaded after analyze.
stat := h.GetTableStats(tableInfo)
require.True(t, stat.Columns[colAID].IsAllEvicted())
hg := stat.Columns[colAID].Histogram
require.Equal(t, hg.Len(), 0)
cms := stat.Columns[colAID].CMSketch
require.Nil(t, cms)
require.True(t, stat.Indices[idxBID].IsAllEvicted())
hg = stat.Indices[idxBID].Histogram
require.Equal(t, hg.Len(), 0)
cms = stat.Indices[idxBID].CMSketch
topN := stat.Indices[idxBID].TopN
require.Equal(t, cms.TotalCount()+topN.TotalCount(), uint64(0))
require.True(t, stat.Columns[colCID].IsAllEvicted())
hg = stat.Columns[colCID].Histogram
require.Equal(t, 0, hg.Len())
cms = stat.Columns[colCID].CMSketch
require.Nil(t, cms)

// Column stats are loaded after they are needed.
_, err = stat.ColumnEqualRowCount(testKit.Session(), types.NewIntDatum(1), colAID)
require.NoError(t, err)
_, err = stat.ColumnEqualRowCount(testKit.Session(), types.NewIntDatum(1), colCID)
require.NoError(t, err)
require.NoError(t, h.LoadNeededHistograms())
stat = h.GetTableStats(tableInfo)
require.True(t, stat.Columns[colAID].IsFullLoad())
hg = stat.Columns[colAID].Histogram
require.Greater(t, hg.Len(), 0)
// We don't maintain cmsketch for pk.
cms = stat.Columns[colAID].CMSketch
require.Nil(t, cms)
require.True(t, stat.Columns[colCID].IsFullLoad())
hg = stat.Columns[colCID].Histogram
require.Greater(t, hg.Len(), 0)
cms = stat.Columns[colCID].CMSketch
require.NotNil(t, cms)

// Index stats are loaded after they are needed.
idx := stat.Indices[idxBID]
hg = idx.Histogram
cms = idx.CMSketch
topN = idx.TopN
require.Equal(t, float64(cms.TotalCount()+topN.TotalCount())+hg.TotalRowCount(), float64(0))
require.False(t, idx.IsEssentialStatsLoaded())
// IsInvalid adds the index to HistogramNeededItems.
idx.IsInvalid(testKit.Session(), false)
require.NoError(t, h.LoadNeededHistograms())
stat = h.GetTableStats(tableInfo)
idx = stat.Indices[tableInfo.Indices[0].ID]
hg = idx.Histogram
cms = idx.CMSketch
topN = idx.TopN
require.Greater(t, float64(cms.TotalCount()+topN.TotalCount())+hg.TotalRowCount(), float64(0))
require.True(t, idx.IsFullLoad())

// Following test tests whether the LoadNeededHistograms would panic.
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/statistics/mockGetStatsReaderFail", `return(true)`))
err = h.LoadNeededHistograms()
require.Error(t, err)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/statistics/mockGetStatsReaderFail"))

require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/statistics/mockGetStatsReaderPanic", "panic"))
err = h.LoadNeededHistograms()
require.Error(t, err)
require.Regexp(t, ".*getStatsReader panic.*", err.Error())
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/statistics/mockGetStatsReaderPanic"))
err = h.LoadNeededHistograms()
require.NoError(t, err)
}

func TestCorrelation(t *testing.T) {
store := testkit.CreateMockStore(t)
testKit := testkit.NewTestKit(t, store)
Expand Down
24 changes: 24 additions & 0 deletions statistics/handle/handletest/statstest/BUILD.bazel
View file
@@ -0,0 +1,24 @@
load("@io_bazel_rules_go//go:def.bzl", "go_test")

go_test(
name = "statstest_test",
timeout = "short",
srcs = [
"main_test.go",
"stats_test.go",
],
flaky = True,
race = "on",
shard_count = 7,
deps = [
"//config",
"//parser/model",
"//statistics/handle/internal",
"//testkit",
"//testkit/testsetup",
"//types",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_stretchr_testify//require",
"@org_uber_go_goleak//:goleak",
],
)
33 changes: 33 additions & 0 deletions statistics/handle/handletest/statstest/main_test.go
View file
@@ -0,0 +1,33 @@
// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package statstest

import (
"testing"

"github.com/pingcap/tidb/testkit/testsetup"
"go.uber.org/goleak"
)

func TestMain(m *testing.M) {
opts := []goleak.Option{
goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"),
goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
}
testsetup.SetupForCommonTest()
goleak.VerifyTestMain(m, opts...)
}

0 comments on commit d692730

Please sign in to comment.