Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

server, statistics: support dump history stats #10291

Merged
merged 16 commits into from May 7, 2019
@@ -396,3 +396,15 @@ timezone.*
Param:

- seconds: profile time(s), default is 10s.

1. Get statistics data of specified table.

```shell
curl http://{TiDBIP}:10080/stats/dump/{db}/{table}
```

1. Get statistics data of specific table and timestamp.

```shell
curl http://{TiDBIP}:10080/stats/dump/{db}/{table}/{yyyyMMddHHmmss}
This conversation was marked as resolved by zz-jason

This comment has been minimized.

Copy link
@zz-jason

zz-jason May 7, 2019

Member

should we use the yyyy-MM-dd HH:mm:ss format?

This comment has been minimized.

Copy link
@XuHuaiyu

XuHuaiyu May 7, 2019

Author Contributor

Both are supported

```
@@ -72,6 +72,7 @@ const (
pColumnFlag = "colFlag"
pColumnLen = "colLen"
pRowBin = "rowBin"
pSnapshot = "snapshot"
)

// For query string
@@ -58,6 +58,7 @@ func (s *Server) startHTTPServer() {

// HTTP path for dump statistics.
router.Handle("/stats/dump/{db}/{table}", s.newStatsHandler()).Name("StatsDump")
router.Handle("/stats/dump/{db}/{table}/{snapshot}", s.newStatsHistoryHandler()).Name("StatsHistoryDump")

router.Handle("/settings", settingsHandler{}).Name("Settings")
router.Handle("/binlog/recover", binlogRecover{}).Name("BinlogRecover")
@@ -15,11 +15,17 @@ package server

import (
"net/http"
"time"

"github.com/gorilla/mux"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/gcutil"
"github.com/pingcap/tidb/util/sqlexec"
)

// StatsHandler is the handler for dumping statistics.
@@ -51,11 +57,77 @@ func (sh StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if err != nil {
writeError(w, err)
} else {
js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta())
js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta(), nil)
if err != nil {
writeError(w, err)
} else {
writeData(w, js)
}
}
}

// StatsHistoryHandler is the handler for dumping statistics.
type StatsHistoryHandler struct {
do *domain.Domain
}

func (s *Server) newStatsHistoryHandler() *StatsHistoryHandler {
store, ok := s.driver.(*TiDBDriver)
if !ok {
panic("Illegal driver")
This conversation was marked as resolved by zz-jason

This comment has been minimized.

Copy link
@zz-jason

zz-jason Apr 29, 2019

Member

Is it proper that a HTTP API could cause tidb server panic?

This comment has been minimized.

Copy link
@qw4990

qw4990 Apr 30, 2019

Contributor

It is called only at start time, so it's ok panic here like an assertion assert(typeof(s.driver) == *TiDBDriver)?

}

do, err := session.GetDomain(store.store)
if err != nil {
panic("Failed to get domain")
}
return &StatsHistoryHandler{do}
}

func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/json")
This conversation was marked as resolved by qw4990

This comment has been minimized.

Copy link
@qw4990

qw4990 Apr 30, 2019

Contributor

Do we need to check the HTTP method and only support GET?


params := mux.Vars(req)
se, err := session.CreateSession(sh.do.Store())
if err != nil {
writeError(w, err)
return
}
se.GetSessionVars().StmtCtx.TimeZone = time.Local
t, err := types.ParseTime(se.GetSessionVars().StmtCtx, params[pSnapshot], mysql.TypeTimestamp, 6)
if err != nil {
writeError(w, err)
return
}
t1, err := t.Time.GoTime(time.Local)
if err != nil {
writeError(w, err)
return
}
snapshot := variable.GoTimeToTS(t1)
err = gcutil.ValidateSnapshot(se, snapshot)
if err != nil {
writeError(w, err)
return
}

is, err := sh.do.GetSnapshotInfoSchema(snapshot)
if err != nil {
writeError(w, err)
return
}
h := sh.do.StatsHandle()
tbl, err := is.TableByName(model.NewCIStr(params[pDBName]), model.NewCIStr(params[pTableName]))
if err != nil {
writeError(w, err)
return
}
se.GetSessionVars().SnapshotInfoschema, se.GetSessionVars().SnapshotTS = is, snapshot
historyStatsExec := se.(sqlexec.RestrictedSQLExecutor)
js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta(), historyStatsExec)
if err != nil {
writeError(w, err)
} else {
writeData(w, js)
}
}
@@ -19,6 +19,7 @@ import (
"io/ioutil"
"net/http"
"os"
"time"

"github.com/go-sql-driver/mysql"
"github.com/gorilla/mux"
@@ -105,6 +106,37 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) {
c.Assert(err, IsNil)
fp.Write(js)
ds.checkData(c, path)

// sleep for 1 seconds to ensure the existence of tidb.test
time.Sleep(time.Second)
timeBeforeDropStats := time.Now()
snapshot := timeBeforeDropStats.Format("20060102150405")
ds.prepare4DumpHistoryStats(c)

// test dump history stats
resp1, err := http.Get("http://127.0.0.1:10090/stats/dump/tidb/test")
c.Assert(err, IsNil)
defer resp1.Body.Close()
js, err = ioutil.ReadAll(resp1.Body)
c.Assert(err, IsNil)
c.Assert(string(js), Equals, "null")

path1 := "/tmp/stats_history.json"
fp1, err := os.Create(path1)
c.Assert(err, IsNil)
c.Assert(fp1, NotNil)
defer func() {
c.Assert(fp1.Close(), IsNil)
c.Assert(os.Remove(path1), IsNil)
}()

resp1, err = http.Get("http://127.0.0.1:10090/stats/dump/tidb/test/" + snapshot)
c.Assert(err, IsNil)

js, err = ioutil.ReadAll(resp1.Body)
c.Assert(err, IsNil)
fp1.Write(js)
ds.checkData(c, path1)
}

func (ds *testDumpStatsSuite) prepareData(c *C) {
@@ -128,20 +160,33 @@ func (ds *testDumpStatsSuite) prepareData(c *C) {
c.Assert(h.Update(is), IsNil)
}

func (ds *testDumpStatsSuite) prepare4DumpHistoryStats(c *C) {
db, err := sql.Open("mysql", getDSN())
c.Assert(err, IsNil, Commentf("Error connecting"))
defer db.Close()

dbt := &DBTest{c, db}

safePointName := "tikv_gc_safe_point"
safePointValue := "20060102-15:04:05 -0700"
safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)"
updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s')
ON DUPLICATE KEY
UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment)
dbt.mustExec(updateSafePoint)

dbt.mustExec("drop table tidb.test")
dbt.mustExec("create table tidb.test (a int, b varchar(20))")
}

func (ds *testDumpStatsSuite) checkData(c *C, path string) {
db, err := sql.Open("mysql", getDSN(func(config *mysql.Config) {
config.AllowAllFiles = true
config.Strict = false
}))
c.Assert(err, IsNil, Commentf("Error connecting"))
dbt := &DBTest{c, db}
defer func() {
dbt.mustExec("drop database tidb")
dbt.mustExec("truncate table mysql.stats_meta")
dbt.mustExec("truncate table mysql.stats_histograms")
dbt.mustExec("truncate table mysql.stats_buckets")
db.Close()
}()
defer db.Close()

dbt.mustExec("use tidb")
dbt.mustExec("drop stats test")
@@ -160,3 +205,15 @@ func (ds *testDumpStatsSuite) checkData(c *C, path string) {
dbt.Check(modifyCount, Equals, int64(3))
dbt.Check(count, Equals, int64(4))
}

func (ds *testDumpStatsSuite) clearData(c *C, path string) {
db, err := sql.Open("mysql", getDSN())
c.Assert(err, IsNil, Commentf("Error connecting"))
defer db.Close()

dbt := &DBTest{c, db}
dbt.mustExec("drop database tidb")
dbt.mustExec("truncate table mysql.stats_meta")
dbt.mustExec("truncate table mysql.stats_histograms")
dbt.mustExec("truncate table mysql.stats_buckets")
}
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/statistics"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tipb/go-tipb"
)

@@ -59,18 +60,18 @@ func dumpJSONCol(hist *statistics.Histogram, CMSketch *statistics.CMSketch) *jso
}

// DumpStatsToJSON dumps statistic to json.
func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JSONTable, error) {
func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo, historyStatsExec sqlexec.RestrictedSQLExecutor) (*JSONTable, error) {
pi := tableInfo.GetPartitionInfo()
if pi == nil {
return h.tableStatsToJSON(dbName, tableInfo, tableInfo.ID)
return h.tableStatsToJSON(dbName, tableInfo, tableInfo.ID, historyStatsExec)
}
jsonTbl := &JSONTable{
DatabaseName: dbName,
TableName: tableInfo.Name.L,
Partitions: make(map[string]*JSONTable, len(pi.Definitions)),
}
for _, def := range pi.Definitions {
tbl, err := h.tableStatsToJSON(dbName, tableInfo, def.ID)
tbl, err := h.tableStatsToJSON(dbName, tableInfo, def.ID, historyStatsExec)
if err != nil {
return nil, errors.Trace(err)
}
@@ -82,8 +83,8 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS
return jsonTbl, nil
}

func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64) (*JSONTable, error) {
tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, true)
func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64, historyStatsExec sqlexec.RestrictedSQLExecutor) (*JSONTable, error) {
tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, true, historyStatsExec)
if err != nil {
return nil, errors.Trace(err)
}
@@ -40,7 +40,7 @@ func (s *testStatsSuite) TestConversion(c *C) {

tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
c.Assert(err, IsNil)
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta())
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
c.Assert(err, IsNil)
loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, jsonTbl)
c.Assert(err, IsNil)
@@ -78,7 +78,7 @@ PARTITION BY RANGE ( a ) (
table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
c.Assert(err, IsNil)
tableInfo := table.Meta()
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo)
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo, nil)
c.Assert(err, IsNil)
pi := tableInfo.GetPartitionInfo()
originTables := make([]*statistics.Table, 0, len(pi.Definitions))
@@ -113,7 +113,7 @@ func (s *testStatsSuite) TestDumpAlteredTable(c *C) {
tk.MustExec("alter table t drop column a")
table, err := s.do.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
c.Assert(err, IsNil)
_, err = h.DumpStatsToJSON("test", table.Meta())
_, err = h.DumpStatsToJSON("test", table.Meta(), nil)
c.Assert(err, IsNil)
}

@@ -150,7 +150,7 @@ func (s *testStatsSuite) TestDumpCMSketchWithTopN(c *C) {
c.Assert(cmsFromStore, NotNil)
c.Check(cms.Equal(cmsFromStore), IsTrue)

jsonTable, err := h.DumpStatsToJSON("test", tableInfo)
jsonTable, err := h.DumpStatsToJSON("test", tableInfo, nil)
c.Check(err, IsNil)
err = h.LoadStatsFromJSON(is, jsonTable)
c.Check(err, IsNil)
@@ -160,7 +160,7 @@ func (h *Handle) Update(is infoschema.InfoSchema) error {
continue
}
tableInfo := table.Meta()
tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, false)
tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, false, nil)
// Error is not nil may mean that there are some ddl changes on this table, we will not update it.
if err != nil {
logutil.Logger(context.Background()).Debug("error occurred when read table stats", zap.String("table", tableInfo.Name.O), zap.Error(err))
@@ -452,7 +452,7 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table,
}

// tableStatsFromStorage loads table stats info from storage.
func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID int64, loadAll bool) (*statistics.Table, error) {
func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID int64, loadAll bool, historyStatsExec sqlexec.RestrictedSQLExecutor) (_ *statistics.Table, err error) {
table, ok := h.StatsCache.Load().(StatsCache)[physicalID]
// If table stats is pseudo, we also need to copy it, since we will use the column stats when
// the average error rate of it is small.
@@ -472,9 +472,14 @@ func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID in
}
table.Pseudo = false
selSQL := fmt.Sprintf("select table_id, is_index, hist_id, distinct_count, version, null_count, tot_col_size, stats_ver, flag, correlation from mysql.stats_histograms where table_id = %d", physicalID)
rows, _, err := h.restrictedExec.ExecRestrictedSQL(nil, selSQL)
var rows []chunk.Row
if historyStatsExec != nil {
rows, _, err = historyStatsExec.ExecRestrictedSQLWithSnapshot(nil, selSQL)
This conversation was marked as resolved by lamxTyler

This comment has been minimized.

Copy link
@lamxTyler

lamxTyler Apr 29, 2019

Member

What about the stats of indices and columns?

} else {
rows, _, err = h.restrictedExec.ExecRestrictedSQL(nil, selSQL)
}
if err != nil {
return nil, errors.Trace(err)
return nil, err
}
// Check deleted table.
if len(rows) == 0 {
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.