Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

server, statistics: support dump history stats #10291

Merged
merged 16 commits into from May 7, 2019

address comment

  • Loading branch information...
XuHuaiyu committed Apr 29, 2019
commit 83e5e8f90711e064c812e205a3448bed39597f0e
@@ -17,6 +17,7 @@ import (
"net/http"
"time"

"context"
This conversation was marked as resolved by qw4990

This comment has been minimized.

Copy link
@qw4990

qw4990 Apr 30, 2019

Contributor

It seems a mistake 😆

"github.com/gorilla/mux"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
@@ -25,7 +26,9 @@ import (
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/gcutil"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/sqlexec"
"go.uber.org/zap"
)

// StatsHandler is the handler for dumping statistics.
@@ -36,12 +39,12 @@ type StatsHandler struct {
func (s *Server) newStatsHandler() *StatsHandler {
store, ok := s.driver.(*TiDBDriver)
if !ok {
panic("Illegal driver")
logutil.Logger(context.Background()).Error("Illegal driver")
This conversation was marked as resolved by XuHuaiyu

This comment has been minimized.

Copy link
@zz-jason

zz-jason May 7, 2019

Member

I think you should let it panic here. if !ok, store may be nil, it will panic in the next procession.

BTW, newStatsHandler only called when we bootstrap the tidb-server, so it's safe to panic.

}

do, err := session.GetDomain(store.store)
if err != nil {
panic("Failed to get domain")
logutil.Logger(context.Background()).Error("Failed to get domain", zap.Error(err))
}
return &StatsHandler{do}
}
@@ -21,6 +21,7 @@ import (
"time"

"github.com/pingcap/errors"
"github.com/pingcap/parser/ast"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/tidb/ddl/util"
@@ -264,11 +265,11 @@ func (h *Handle) LoadNeededHistograms() error {
statistics.HistogramNeededColumns.Delete(col)
continue
}
hg, err := h.histogramFromStorage(col.TableID, c.ID, &c.Info.FieldType, c.NDV, 0, c.LastUpdateVersion, c.NullCount, c.TotColSize, c.Correlation)
hg, err := h.histogramFromStorage(col.TableID, c.ID, &c.Info.FieldType, c.NDV, 0, c.LastUpdateVersion, c.NullCount, c.TotColSize, c.Correlation, nil)
if err != nil {
return errors.Trace(err)
}
cms, err := h.cmSketchFromStorage(col.TableID, 0, col.ColumnID)
cms, err := h.cmSketchFromStorage(col.TableID, 0, col.ColumnID, nil)
if err != nil {
return errors.Trace(err)
}
@@ -316,9 +317,14 @@ func (h *Handle) FlushStats() {
}
}

func (h *Handle) cmSketchFromStorage(tblID int64, isIndex, histID int64) (*statistics.CMSketch, error) {
func (h *Handle) cmSketchFromStorage(tblID int64, isIndex, histID int64, historyStatsExec sqlexec.RestrictedSQLExecutor) (_ *statistics.CMSketch, err error) {
selSQL := fmt.Sprintf("select cm_sketch from mysql.stats_histograms where table_id = %d and is_index = %d and hist_id = %d", tblID, isIndex, histID)
rows, _, err := h.restrictedExec.ExecRestrictedSQL(nil, selSQL)
var rows []chunk.Row
if historyStatsExec != nil {
rows, _, err = historyStatsExec.ExecRestrictedSQLWithSnapshot(nil, selSQL)
} else {
rows, _, err = h.restrictedExec.ExecRestrictedSQL(nil, selSQL)
}
if err != nil {
return nil, errors.Trace(err)
}
@@ -328,7 +334,7 @@ func (h *Handle) cmSketchFromStorage(tblID int64, isIndex, histID int64) (*stati
return statistics.LoadCMSketchWithTopN(h.restrictedExec, tblID, isIndex, histID, rows[0].GetBytes(0))
}

func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, tableInfo *model.TableInfo) error {
func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, tableInfo *model.TableInfo, historyStatsExec sqlexec.RestrictedSQLExecutor) error {
histID := row.GetInt64(2)
distinct := row.GetInt64(3)
histVer := row.GetUint64(4)
@@ -347,11 +353,11 @@ func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, t
continue
}
if idx == nil || idx.LastUpdateVersion < histVer {
hg, err := h.histogramFromStorage(table.PhysicalID, histID, types.NewFieldType(mysql.TypeBlob), distinct, 1, histVer, nullCount, 0, 0)
hg, err := h.histogramFromStorage(table.PhysicalID, histID, types.NewFieldType(mysql.TypeBlob), distinct, 1, histVer, nullCount, 0, 0, historyStatsExec)
if err != nil {
return errors.Trace(err)
}
cms, err := h.cmSketchFromStorage(table.PhysicalID, 1, idxInfo.ID)
cms, err := h.cmSketchFromStorage(table.PhysicalID, 1, idxInfo.ID, historyStatsExec)
if err != nil {
return errors.Trace(err)
}
@@ -367,7 +373,7 @@ func (h *Handle) indexStatsFromStorage(row chunk.Row, table *statistics.Table, t
return nil
}

func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table, tableInfo *model.TableInfo, loadAll bool) error {
func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table, tableInfo *model.TableInfo, loadAll bool, historyStatsExec sqlexec.RestrictedSQLExecutor) error {
histID := row.GetInt64(2)
distinct := row.GetInt64(3)
histVer := row.GetUint64(4)
@@ -414,11 +420,11 @@ func (h *Handle) columnStatsFromStorage(row chunk.Row, table *statistics.Table,
break
}
if col == nil || col.LastUpdateVersion < histVer || loadAll {
hg, err := h.histogramFromStorage(table.PhysicalID, histID, &colInfo.FieldType, distinct, 0, histVer, nullCount, totColSize, correlation)
hg, err := h.histogramFromStorage(table.PhysicalID, histID, &colInfo.FieldType, distinct, 0, histVer, nullCount, totColSize, correlation, historyStatsExec)
if err != nil {
return errors.Trace(err)
}
cms, err := h.cmSketchFromStorage(table.PhysicalID, 0, colInfo.ID)
cms, err := h.cmSketchFromStorage(table.PhysicalID, 0, colInfo.ID, historyStatsExec)
if err != nil {
return errors.Trace(err)
}
@@ -487,11 +493,11 @@ func (h *Handle) tableStatsFromStorage(tableInfo *model.TableInfo, physicalID in
}
for _, row := range rows {
if row.GetInt64(1) > 0 {
if err := h.indexStatsFromStorage(row, table, tableInfo); err != nil {
if err := h.indexStatsFromStorage(row, table, tableInfo, historyStatsExec); err != nil {
This conversation was marked as resolved by lamxTyler

This comment has been minimized.

Copy link
@lamxTyler

lamxTyler Apr 29, 2019

Member

The stats of index or column will only be updated when idx.LastUpdateVersion < histVer, so it is better to use an empty table, not the one in the stats cache.

return nil, errors.Trace(err)
}
} else {
if err := h.columnStatsFromStorage(row, table, tableInfo, loadAll); err != nil {
if err := h.columnStatsFromStorage(row, table, tableInfo, loadAll, historyStatsExec); err != nil {
return nil, errors.Trace(err)
}
}
@@ -610,9 +616,17 @@ func (h *Handle) SaveMetaToStorage(tableID, count, modifyCount int64) (err error
return
}

func (h *Handle) histogramFromStorage(tableID int64, colID int64, tp *types.FieldType, distinct int64, isIndex int, ver uint64, nullCount int64, totColSize int64, corr float64) (*statistics.Histogram, error) {
func (h *Handle) histogramFromStorage(tableID int64, colID int64, tp *types.FieldType, distinct int64, isIndex int, ver uint64, nullCount int64, totColSize int64, corr float64, historyStatsExec sqlexec.RestrictedSQLExecutor) (_ *statistics.Histogram, err error) {
selSQL := fmt.Sprintf("select count, repeats, lower_bound, upper_bound from mysql.stats_buckets where table_id = %d and is_index = %d and hist_id = %d order by bucket_id", tableID, isIndex, colID)
rows, fields, err := h.restrictedExec.ExecRestrictedSQL(nil, selSQL)
var (
rows []chunk.Row
fields []*ast.ResultField
)
if historyStatsExec != nil {
rows, fields, err = historyStatsExec.ExecRestrictedSQLWithSnapshot(nil, selSQL)
} else {
rows, fields, err = h.restrictedExec.ExecRestrictedSQL(nil, selSQL)
}
if err != nil {
return nil, errors.Trace(err)
}
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.