Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

server, statistics: support dump history stats #10291

Merged
merged 16 commits into from May 7, 2019
@@ -396,3 +396,18 @@ timezone.*
Param:

- seconds: profile time(s), default is 10s.

1. Get statistics data of specified table.

```shell
curl http://{TiDBIP}:10080/stats/dump/{db}/{table}
```

1. Get statistics data of specific table and timestamp.

```shell
curl http://{TiDBIP}:10080/stats/dump/{db}/{table}/{yyyyMMddHHmmss}
This conversation was marked as resolved by zz-jason

This comment has been minimized.

Copy link
@zz-jason

zz-jason May 7, 2019

Member

should we use the yyyy-MM-dd HH:mm:ss format?

This comment has been minimized.

Copy link
@XuHuaiyu

XuHuaiyu May 7, 2019

Author Contributor

Both are supported

```
```shell
curl http://{TiDBIP}:10080/stats/dump/{db}/{table}/{yyyy-MM-dd HH:mm:ss}
```
@@ -72,6 +72,7 @@ const (
pColumnFlag = "colFlag"
pColumnLen = "colLen"
pRowBin = "rowBin"
pSnapshot = "snapshot"
)

// For query string
@@ -58,6 +58,7 @@ func (s *Server) startHTTPServer() {

// HTTP path for dump statistics.
router.Handle("/stats/dump/{db}/{table}", s.newStatsHandler()).Name("StatsDump")
router.Handle("/stats/dump/{db}/{table}/{snapshot}", s.newStatsHistoryHandler()).Name("StatsHistoryDump")

router.Handle("/settings", settingsHandler{}).Name("Settings")
router.Handle("/reload-config", configReloadHandler{}).Name("ConfigReload")
@@ -15,11 +15,17 @@ package server

import (
"net/http"
"time"

"github.com/gorilla/mux"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/session"
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/gcutil"
"github.com/pingcap/tidb/util/sqlexec"
)

// StatsHandler is the handler for dumping statistics.
@@ -51,11 +57,77 @@ func (sh StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if err != nil {
writeError(w, err)
} else {
js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta())
js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta(), nil)
if err != nil {
writeError(w, err)
} else {
writeData(w, js)
}
}
}

// StatsHistoryHandler is the handler for dumping statistics.
type StatsHistoryHandler struct {
do *domain.Domain
}

func (s *Server) newStatsHistoryHandler() *StatsHistoryHandler {
store, ok := s.driver.(*TiDBDriver)
if !ok {
panic("Illegal driver")
This conversation was marked as resolved by zz-jason

This comment has been minimized.

Copy link
@zz-jason

zz-jason Apr 29, 2019

Member

Is it proper that a HTTP API could cause tidb server panic?

This comment has been minimized.

Copy link
@qw4990

qw4990 Apr 30, 2019

Contributor

It is called only at start time, so it's ok panic here like an assertion assert(typeof(s.driver) == *TiDBDriver)?

}

do, err := session.GetDomain(store.store)
if err != nil {
panic("Failed to get domain")
}
return &StatsHistoryHandler{do}
}

func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/json")
This conversation was marked as resolved by qw4990

This comment has been minimized.

Copy link
@qw4990

qw4990 Apr 30, 2019

Contributor

Do we need to check the HTTP method and only support GET?


params := mux.Vars(req)
se, err := session.CreateSession(sh.do.Store())
if err != nil {
writeError(w, err)
return
}
se.GetSessionVars().StmtCtx.TimeZone = time.Local
t, err := types.ParseTime(se.GetSessionVars().StmtCtx, params[pSnapshot], mysql.TypeTimestamp, 6)
if err != nil {
writeError(w, err)
return
}
t1, err := t.Time.GoTime(time.Local)
if err != nil {
writeError(w, err)
return
}
snapshot := variable.GoTimeToTS(t1)
err = gcutil.ValidateSnapshot(se, snapshot)
if err != nil {
writeError(w, err)
return
}

is, err := sh.do.GetSnapshotInfoSchema(snapshot)
if err != nil {
writeError(w, err)
return
}
h := sh.do.StatsHandle()
tbl, err := is.TableByName(model.NewCIStr(params[pDBName]), model.NewCIStr(params[pTableName]))
if err != nil {
writeError(w, err)
return
}
se.GetSessionVars().SnapshotInfoschema, se.GetSessionVars().SnapshotTS = is, snapshot
historyStatsExec := se.(sqlexec.RestrictedSQLExecutor)
js, err := h.DumpStatsToJSON(params[pDBName], tbl.Meta(), historyStatsExec)
if err != nil {
writeError(w, err)
} else {
writeData(w, js)
}
}
@@ -19,6 +19,7 @@ import (
"io/ioutil"
"net/http"
"os"
"time"

"github.com/go-sql-driver/mysql"
"github.com/gorilla/mux"
@@ -105,6 +106,37 @@ func (ds *testDumpStatsSuite) TestDumpStatsAPI(c *C) {
c.Assert(err, IsNil)
fp.Write(js)
ds.checkData(c, path)

// sleep for 1 seconds to ensure the existence of tidb.test
time.Sleep(time.Second)
timeBeforeDropStats := time.Now()
snapshot := timeBeforeDropStats.Format("20060102150405")
ds.prepare4DumpHistoryStats(c)

// test dump history stats
resp1, err := http.Get("http://127.0.0.1:10090/stats/dump/tidb/test")
c.Assert(err, IsNil)
defer resp1.Body.Close()
js, err = ioutil.ReadAll(resp1.Body)
c.Assert(err, IsNil)
c.Assert(string(js), Equals, "null")

path1 := "/tmp/stats_history.json"
fp1, err := os.Create(path1)
c.Assert(err, IsNil)
c.Assert(fp1, NotNil)
defer func() {
c.Assert(fp1.Close(), IsNil)
c.Assert(os.Remove(path1), IsNil)
}()

resp1, err = http.Get("http://127.0.0.1:10090/stats/dump/tidb/test/" + snapshot)
c.Assert(err, IsNil)

js, err = ioutil.ReadAll(resp1.Body)
c.Assert(err, IsNil)
fp1.Write(js)
ds.checkData(c, path1)
}

func (ds *testDumpStatsSuite) prepareData(c *C) {
@@ -128,20 +160,33 @@ func (ds *testDumpStatsSuite) prepareData(c *C) {
c.Assert(h.Update(is), IsNil)
}

func (ds *testDumpStatsSuite) prepare4DumpHistoryStats(c *C) {
db, err := sql.Open("mysql", getDSN())
c.Assert(err, IsNil, Commentf("Error connecting"))
defer db.Close()

dbt := &DBTest{c, db}

safePointName := "tikv_gc_safe_point"
safePointValue := "20060102-15:04:05 -0700"
safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)"
updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s')
ON DUPLICATE KEY
UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment)
dbt.mustExec(updateSafePoint)

dbt.mustExec("drop table tidb.test")
dbt.mustExec("create table tidb.test (a int, b varchar(20))")
}

func (ds *testDumpStatsSuite) checkData(c *C, path string) {
db, err := sql.Open("mysql", getDSN(func(config *mysql.Config) {
config.AllowAllFiles = true
config.Strict = false
}))
c.Assert(err, IsNil, Commentf("Error connecting"))
dbt := &DBTest{c, db}
defer func() {
dbt.mustExec("drop database tidb")
dbt.mustExec("truncate table mysql.stats_meta")
dbt.mustExec("truncate table mysql.stats_histograms")
dbt.mustExec("truncate table mysql.stats_buckets")
db.Close()
}()
defer db.Close()

dbt.mustExec("use tidb")
dbt.mustExec("drop stats test")
@@ -160,3 +205,15 @@ func (ds *testDumpStatsSuite) checkData(c *C, path string) {
dbt.Check(modifyCount, Equals, int64(3))
dbt.Check(count, Equals, int64(4))
}

func (ds *testDumpStatsSuite) clearData(c *C, path string) {
db, err := sql.Open("mysql", getDSN())
c.Assert(err, IsNil, Commentf("Error connecting"))
defer db.Close()

dbt := &DBTest{c, db}
dbt.mustExec("drop database tidb")
dbt.mustExec("truncate table mysql.stats_meta")
dbt.mustExec("truncate table mysql.stats_histograms")
dbt.mustExec("truncate table mysql.stats_buckets")
}
@@ -23,6 +23,7 @@ import (
"github.com/pingcap/tidb/sessionctx/stmtctx"
"github.com/pingcap/tidb/statistics"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tipb/go-tipb"
)

@@ -59,18 +60,18 @@ func dumpJSONCol(hist *statistics.Histogram, CMSketch *statistics.CMSketch) *jso
}

// DumpStatsToJSON dumps statistic to json.
func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JSONTable, error) {
func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo, historyStatsExec sqlexec.RestrictedSQLExecutor) (*JSONTable, error) {
pi := tableInfo.GetPartitionInfo()
if pi == nil {
return h.tableStatsToJSON(dbName, tableInfo, tableInfo.ID)
return h.tableStatsToJSON(dbName, tableInfo, tableInfo.ID, historyStatsExec)
}
jsonTbl := &JSONTable{
DatabaseName: dbName,
TableName: tableInfo.Name.L,
Partitions: make(map[string]*JSONTable, len(pi.Definitions)),
}
for _, def := range pi.Definitions {
tbl, err := h.tableStatsToJSON(dbName, tableInfo, def.ID)
tbl, err := h.tableStatsToJSON(dbName, tableInfo, def.ID, historyStatsExec)
if err != nil {
return nil, errors.Trace(err)
}
@@ -82,13 +83,14 @@ func (h *Handle) DumpStatsToJSON(dbName string, tableInfo *model.TableInfo) (*JS
return jsonTbl, nil
}

func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64) (*JSONTable, error) {
tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, true)
if err != nil {
return nil, errors.Trace(err)
func (h *Handle) tableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64, historyStatsExec sqlexec.RestrictedSQLExecutor) (*JSONTable, error) {
tbl, err := h.tableStatsFromStorage(tableInfo, physicalID, true, historyStatsExec)
if err != nil || tbl == nil {
This conversation was marked as resolved by zz-jason

This comment has been minimized.

Copy link
@zz-jason

zz-jason May 7, 2019

Member

I think we should return a specific error message for err != nil but tbl == nil

This comment has been minimized.

Copy link
@XuHuaiyu

XuHuaiyu May 7, 2019

Author Contributor

We do not need to return an error for this case.
"null" will be returned when tbl == nil.

return nil, err
}
if tbl == nil {
return nil, nil
tbl.Version, tbl.ModifyCount, tbl.Count, err = h.statsMetaByTableIDFromStorage(physicalID, historyStatsExec)
if err != nil {
return nil, err
}
jsonTbl := &JSONTable{
DatabaseName: dbName,
@@ -40,7 +40,7 @@ func (s *testStatsSuite) TestConversion(c *C) {

tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
c.Assert(err, IsNil)
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta())
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
c.Assert(err, IsNil)
loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, jsonTbl)
c.Assert(err, IsNil)
@@ -78,7 +78,7 @@ PARTITION BY RANGE ( a ) (
table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
c.Assert(err, IsNil)
tableInfo := table.Meta()
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo)
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo, nil)
c.Assert(err, IsNil)
pi := tableInfo.GetPartitionInfo()
originTables := make([]*statistics.Table, 0, len(pi.Definitions))
@@ -113,7 +113,7 @@ func (s *testStatsSuite) TestDumpAlteredTable(c *C) {
tk.MustExec("alter table t drop column a")
table, err := s.do.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
c.Assert(err, IsNil)
_, err = h.DumpStatsToJSON("test", table.Meta())
_, err = h.DumpStatsToJSON("test", table.Meta(), nil)
c.Assert(err, IsNil)
}

@@ -150,7 +150,7 @@ func (s *testStatsSuite) TestDumpCMSketchWithTopN(c *C) {
c.Assert(cmsFromStore, NotNil)
c.Check(cms.Equal(cmsFromStore), IsTrue)

jsonTable, err := h.DumpStatsToJSON("test", tableInfo)
jsonTable, err := h.DumpStatsToJSON("test", tableInfo, nil)
c.Check(err, IsNil)
err = h.LoadStatsFromJSON(is, jsonTable)
c.Check(err, IsNil)
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.