Skip to content

Commit

Permalink
Merge pull request #3 from IggieWang/newidxadv
Browse files Browse the repository at this point in the history 
finish multiple tables advisor
  • Loading branch information
@zouhuan1215
zouhuan1215 committed Aug 10, 2019
2 parents cdb143f + 9845813 commit 94fb775
Show file tree
Hide file tree
Showing 4 changed files with 309 additions and 133 deletions.
15 changes: 4 additions & 11 deletions executor/compiler.go
View file
Expand Up @@ -34,8 +34,6 @@ import (
"go.uber.org/zap"
)

const tblname string = "orders"

var (
stmtNodeCounterUse = metrics.StmtNodeCounter.WithLabelValues("Use")
stmtNodeCounterShow = metrics.StmtNodeCounter.WithLabelValues("Show")
Expand Down Expand Up @@ -95,13 +93,6 @@ func (c *Compiler) compile(ctx context.Context, stmtNode ast.StmtNode, skipBind

queryInfo := plannercore.NewQueryExprInfo(p)
m := plannercore.NewTableInfoSets(queryInfo)
for _, v := range m {
fmt.Println(v.TblInfo.Name.L)
fmt.Println(v.Eq)
fmt.Println(v.O)
fmt.Println(v.Rg)
fmt.Println(v.Ref)
}

// Get final plan cost.
cost, err := plannercore.GetTaskCost(finalPlan)
Expand All @@ -111,7 +102,9 @@ func (c *Compiler) compile(ctx context.Context, stmtNode ast.StmtNode, skipBind

// Construct virtual infoschema
dbname := c.Ctx.GetSessionVars().CurrentDB
virtualIS := idxadvisor.GetVirtualInfoschema(infoSchema, dbname, tblname)

conn := c.Ctx.GetSessionVars().ConnectionID
virtualIS := idxadvisor.GetVirtualInfoschema(infoSchema, dbname, m)

// Get virtual final plan.
vFinalPlan, err := planner.Optimize(ctx, c.Ctx, stmtNode, virtualIS)
Expand All @@ -135,7 +128,7 @@ func (c *Compiler) compile(ctx context.Context, stmtNode ast.StmtNode, skipBind
// Get virtual indices with cost.
selectedIndices := idxadvisor.FindVirtualIndices(vPhysicalPlan)
iwc := idxadvisor.IndicesWithCost{Indices: selectedIndices, Cost: vcost}
idxadvisor.WriteResult(iwc, c.Ctx.GetSessionVars().ConnectionID, cost)
idxadvisor.SaveVirtualIndices(infoSchema, dbname, iwc, conn, cost)

finalPlan = nil
}
Expand Down
165 changes: 130 additions & 35 deletions idxadvisor/idxadvisor.go
View file
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/pingcap/parser/model"
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/infoschema"
plannercore "github.com/pingcap/tidb/planner/core"
)

type idxAdvPool []*IdxAdvisor
Expand Down Expand Up @@ -44,7 +45,7 @@ type IdxAdvisor struct {

// CandidateIdx includes in index and its benefit.
type CandidateIdx struct {
Index *model.IndexInfo
Index *IdxAndTblInfo
Benefit float64
}

Expand Down Expand Up @@ -82,6 +83,7 @@ func (ia *IdxAdvisor) IsReady() bool {
func (ia *IdxAdvisor) StartTask(query string) {
if ia.IsReady() {
fmt.Printf("********idxadvisor/outline.go: Set variable has done, StartTask starts query\n")

// var err error
sqlFile := "/tmp/queries"
queries := readQuery(&sqlFile)
Expand All @@ -91,54 +93,75 @@ func (ia *IdxAdvisor) StartTask(query string) {
}

}
WritFinaleResult()
}

func GetVirtualInfoschema(is infoschema.InfoSchema, dbName, tblName string) infoschema.InfoSchema {
/*
// StartTask start handling queries in idxadv mode after session variable tidb_enable_index_advisor has been set
func (ia *IdxAdvisor) StartTask(query string) {
if ia.IsReady() {
// var err error
sqlFile := "/tmp/queries"
queries := readQuery(&sqlFile)
for i, query := range queries {
fmt.Printf("$$$$$$$$$$$$$$$$$$$$$$[%v]$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\n", i+1)
ia.dbClient.Exec(query)
}
}
}
*/
func GetVirtualInfoschema(is infoschema.InfoSchema, dbName string, tableInfoSets map[string]*plannercore.TableInfoSets) infoschema.InfoSchema {
// Get a copy of InfoSchema
dbInfos := is.Clone()
ISCopy := infoschema.MockInfoSchemaWithDBInfos(dbInfos, is.SchemaMetaVersion())

dbname := model.NewCIStr(dbName)
tblname := model.NewCIStr(tblName)
tblCopy, err := ISCopy.TableByName(dbname, tblname)
if err != nil {
panic(err)
}
tblInfoCopy := tblCopy.Meta()
for tblname, tblInfoSets := range tableInfoSets {
tblname := model.NewCIStr(tblname)
tblCopy, err := ISCopy.TableByName(dbname, tblname)
if err != nil {
panic(err)
}
tblInfoCopy := tblCopy.Meta()
idxInfo := tblCopy.Meta().Indices

// add virtual indexes to InfoSchemaCopy.TblInfo
virtualIndexes := BuildVirtualIndexes(tblInfoCopy, dbname, tblname)
tblInfoCopy.Indices = append(tblInfoCopy.Indices, virtualIndexes...)
// add virtual indexes to InfoSchemaCopy.TblInfo
virtualIndexes := BuildVirtualIndexes(tblInfoCopy, dbname, tblname, tblInfoSets)
for _, virtualIndex := range virtualIndexes {
if !isExistedInTable(virtualIndex, idxInfo) {
tblInfoCopy.Indices = append(tblInfoCopy.Indices, virtualIndex)
}
}
}
return ISCopy
}

func BuildVirtualIndexes(tblInfo *model.TableInfo, dbname, tblname model.CIStr) []*model.IndexInfo {
indexes := GenVirtualIndexCols(tblInfo, dbname, tblname)
func BuildVirtualIndexes(tblInfo *model.TableInfo, dbname, tblname model.CIStr, tblInfoSets *plannercore.TableInfoSets) []*model.IndexInfo {
indexes := GenVirtualIndexCols(tblInfo, dbname, tblname, tblInfoSets)
result := make([]*model.IndexInfo, 0)
for i, idxColNames := range indexes {
indexName := model.NewCIStr("vIndex" + string(i))
indexinfo, err := ddl.BuildIndexInfo(tblInfo, indexName, idxColNames, model.StatePublic)
if err != nil {
fmt.Printf("***************BuildVirtualIndexes error: %v!\n", err)
fmt.Printf("BuildVirtualIndexes error: %v!\n", err)
panic(err)
}
result = append(result, indexinfo)
}
return result

}

func GenVirtualIndexCols(tblInfo *model.TableInfo, dbname, tblname model.CIStr) [][]*ast.IndexColName {
func GenVirtualIndexCols(tblInfo *model.TableInfo, dbname, tblname model.CIStr, tblInfoSets *plannercore.TableInfoSets) [][]*ast.IndexColName {
columnInfos := tblInfo.Columns
var result [][]*ast.IndexColName

// one column
for _, columnInfo := range columnInfos {
idxCols := make([]*ast.IndexColName, 1, 1)
idxCols[0] = BuildIdxColNameFromColInfo(columnInfo, dbname, tblname)
if !IndexesHasAlreadyExist(idxCols, tblInfo.Indices) {
result = append(result, idxCols)
}
result = append(result, idxCols)
}

// two columns
nCols := len(columnInfos)
for i := 0; i < nCols; i++ {
for j := 0; j < nCols; j++ {
Expand All @@ -151,41 +174,113 @@ func GenVirtualIndexCols(tblInfo *model.TableInfo, dbname, tblname model.CIStr)
}
}

// multi columns
candidateCols := [][]model.CIStr{}
eq := tblInfoSets.Eq
o := tblInfoSets.O
rg := tblInfoSets.Rg
ref := tblInfoSets.Ref

// EQ + O + RANGE + REF
cols := [][]model.CIStr{}
for i, oCols := range o {
cols = append(cols, []model.CIStr{})
addToCandidateCols(eq, &cols[i], &candidateCols)
addToCandidateCols(oCols, &cols[i], &candidateCols)
addToCandidateCols(rg, &cols[i], &candidateCols)
addToCandidateCols(ref, &cols[i], &candidateCols)
}
if len(cols) == 0 {
cols = append(cols, []model.CIStr{})
addToCandidateCols(eq, &cols[0], &candidateCols)
addToCandidateCols(rg, &cols[0], &candidateCols)
addToCandidateCols(ref, &cols[0], &candidateCols)
}

// O + EQ + RANGE + REF
cols = cols[:0]
for i, oCols := range o {
cols = append(cols, []model.CIStr{})
addToCandidateCols(oCols, &cols[i], &candidateCols)
addToCandidateCols(eq, &cols[i], &candidateCols)
addToCandidateCols(rg, &cols[i], &candidateCols)
addToCandidateCols(ref, &cols[i], &candidateCols)
}
if len(cols) == 0 {
cols = append(cols, []model.CIStr{})
addToCandidateCols(eq, &cols[0], &candidateCols)
addToCandidateCols(rg, &cols[0], &candidateCols)
addToCandidateCols(ref, &cols[0], &candidateCols)
}

candidateCols = plannercore.RemoveRepeatedColumnSet(candidateCols)
if len(candidateCols) > 0 {
fmt.Printf("table %s multi candidate index: ", tblname)
fmt.Println(candidateCols)
}
for _, candidateColumns := range candidateCols {
idxCols := make([]*ast.IndexColName, len(candidateColumns), len(candidateColumns))
for i, column := range candidateColumns {
columnInfo := new(model.ColumnInfo)
for _, tmpColumn := range columnInfos {
if tmpColumn.Name.L == column.L {
columnInfo = tmpColumn
break
}
}
idxCols[i] = BuildIdxColNameFromColInfo(columnInfo, dbname, tblname)
}
result = append(result, idxCols)

}

return result
}

func addToCandidateCols(readyCols []model.CIStr, cols *[]model.CIStr, candidateCols *[][]model.CIStr) {
if len(readyCols) == 0 {
return
}

*cols = append(*cols, readyCols...)
*cols = plannercore.RemoveRepeatedColumn(*cols)
if len(*cols) > 2 {
*candidateCols = append(*candidateCols, *cols)
}
}

func BuildIdxColNameFromColInfo(colInfo *model.ColumnInfo, dbname, tblname model.CIStr) *ast.IndexColName {
idxColName := &ast.IndexColName{}
idxColName.Column = &ast.ColumnName{Schema: dbname, Table: tblname, Name: colInfo.Name}
idxColName.Length = -1
return idxColName
}

// TODO: This is only single col index recomendation
func IndexesHasAlreadyExist(idxCols []*ast.IndexColName, indices []*model.IndexInfo) bool {
primaryKey := findPrimaryKey(indices)
if primaryKey == nil {
return false
func GenIndexCols(index *model.IndexInfo) []model.CIStr {
cols := []model.CIStr{}
for _, idxColumn := range index.Columns {
cols = append(cols, idxColumn.Name)
}
return primaryKey.Columns[0].Name.String() == idxCols[0].Column.Name.String()
return cols
}

func findPrimaryKey(indices []*model.IndexInfo) *model.IndexInfo {
if len(indices) == 0 {
return nil
}
for _, indexInfo := range indices {
if indexInfo.Primary {
return indexInfo
func isExistedInTable(virtualIndex *model.IndexInfo, indices []*model.IndexInfo) bool {
is := false
virtualIndexCols := GenIndexCols(virtualIndex)
for _, idx := range indices {
indexCols := GenIndexCols(idx)
if reflect.DeepEqual(virtualIndexCols, indexCols) {
is = true
break
}
}
return nil
return is
}

func (ia *IdxAdvisor) addCandidate(virtualIdx *CandidateIdx) {
in := false
for _, candidateIdx := range ia.Candidate_idx {
if reflect.DeepEqual(candidateIdx.Index.Columns, virtualIdx.Index.Columns) {
if reflect.DeepEqual(candidateIdx.Index.Index.Columns, virtualIdx.Index.Index.Columns) && reflect.DeepEqual(candidateIdx.Index.Table.Name, virtualIdx.Index.Table.Name) {
candidateIdx.Benefit += virtualIdx.Benefit
in = true
break
Expand Down

0 comments on commit 94fb775

Please sign in to comment.