Skip to content

Commit

Permalink
planner: support hint for IndexHashJoin and IndexMergeJoin
Browse files Browse the repository at this point in the history
  • Loading branch information
@XuHuaiyu
XuHuaiyu committed Nov 7, 2019
1 parent 5c5aa10 commit efa1b90
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 52 deletions.
114 changes: 85 additions & 29 deletions planner/core/exhaust_physical_plans.go
View file
Expand Up @@ -1152,18 +1152,37 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
// tryToGetIndexJoin will get index join by hints. If we can generate a valid index join by hint, the second return value
// will be true, which means we force to choose this index join. Otherwise we will select a join algorithm with min-cost.
func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJoins []PhysicalPlan, forced bool) {
rightOuter := (p.preferJoinType & preferLeftAsIndexInner) > 0
leftOuter := (p.preferJoinType & preferRightAsIndexInner) > 0
hasIndexJoinHint := leftOuter || rightOuter
inljRightOuter := (p.preferJoinType & preferLeftAsINLJInner) > 0
inljLeftOuter := (p.preferJoinType & preferRightAsINLJInner) > 0
hasINLJHint := inljLeftOuter || inljRightOuter

inlhjRightOuter := (p.preferJoinType & preferLeftAsINLHJInner) > 0
inlhjLeftOuter := (p.preferJoinType & preferRightAsINLHJInner) > 0
hasINLHJHint := inlhjLeftOuter || inlhjRightOuter

inlmjRightOuter := (p.preferJoinType & preferLeftAsINLMJInner) > 0
inlmjLeftOuter := (p.preferJoinType & preferRightAsINLMJInner) > 0
hasINLMJHint := inlmjLeftOuter || inlmjRightOuter

forceLeftOuter := inljLeftOuter || inlhjLeftOuter || inlmjLeftOuter
forceRightOuter := inljRightOuter || inlhjRightOuter || inlmjRightOuter

defer func() {
if !forced && hasIndexJoinHint {
// refine error message
if !forced && (hasINLJHint || hasINLHJHint || hasINLMJHint) {
// Construct warning message prefix.
errMsg := "Optimizer Hint INL_JOIN or TIDB_INLJ is inapplicable"
if p.hintInfo != nil {
errMsg = fmt.Sprintf("Optimizer Hint %s or %s is inapplicable",
restore2JoinHint(HintINLJ, p.hintInfo.indexNestedLoopJoinTables),
restore2JoinHint(TiDBIndexNestedLoopJoin, p.hintInfo.indexNestedLoopJoinTables))
t := p.hintInfo.indexNestedLoopJoinTables
switch {
case len(t.inljTables) != 0:
errMsg = fmt.Sprintf("Optimizer Hint %s or %s is inapplicable",
restore2JoinHint(HintINLJ, t.inljTables), restore2JoinHint(TiDBIndexNestedLoopJoin, t.inljTables))
case len(t.inlhjTables) != 0:
errMsg = fmt.Sprintf("Optimizer Hint %s is inapplicable", restore2JoinHint(HintINLHJ, t.inlhjTables))
case len(t.inlmjTables) != 0:
errMsg = fmt.Sprintf("Optimizer Hint %s is inapplicable", restore2JoinHint(HintINLMJ, t.inlmjTables))
}
}

// Append inapplicable reason.
Expand All @@ -1177,36 +1196,73 @@ func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJ
}
}()

// canLeftOuter and canRightOuter indicates whether this type of join
// supports the left side or right side to be the outer side.
var canLeftOuter, canRightOuter bool
switch p.JoinType {
case SemiJoin, AntiSemiJoin, LeftOuterSemiJoin, AntiLeftOuterSemiJoin, LeftOuterJoin:
join := p.getIndexJoinByOuterIdx(prop, 0)
return join, join != nil && leftOuter
canLeftOuter = true
case RightOuterJoin:
join := p.getIndexJoinByOuterIdx(prop, 1)
return join, join != nil && rightOuter
canRightOuter = true
case InnerJoin:
lhsCardinality := p.Children()[0].statsInfo().Count()
rhsCardinality := p.Children()[1].statsInfo().Count()

leftJoins := p.getIndexJoinByOuterIdx(prop, 0)
if leftJoins != nil && (leftOuter && !rightOuter || lhsCardinality < rhsCardinality) {
return leftJoins, leftOuter
canLeftOuter, canRightOuter = true, true
}

var allLeftOuterJoins, allRightOuterJoins, forcedLeftOuterJoins, forcedRightOuterJoins []PhysicalPlan
if canLeftOuter {
allLeftOuterJoins = p.getIndexJoinByOuterIdx(prop, 0)
forcedLeftOuterJoins = make([]PhysicalPlan, 0, len(allLeftOuterJoins))
for _, j := range allLeftOuterJoins {
switch j.(type) {
case *PhysicalIndexJoin:
if hasINLJHint {
forcedLeftOuterJoins = append(forcedLeftOuterJoins, j)
}
case *PhysicalIndexHashJoin:
if hasINLHJHint {
forcedLeftOuterJoins = append(forcedLeftOuterJoins, j)
}
case *PhysicalIndexMergeJoin:
if hasINLMJHint {
forcedLeftOuterJoins = append(forcedLeftOuterJoins, j)
}
}
}

rightJoins := p.getIndexJoinByOuterIdx(prop, 1)
if rightJoins != nil && (rightOuter && !leftOuter || rhsCardinality < lhsCardinality) {
return rightJoins, rightOuter
if !canRightOuter || forcedLeftOuterJoins != nil && (forceLeftOuter && !forceRightOuter) {
return forcedLeftOuterJoins, forcedLeftOuterJoins != nil && forceLeftOuter
}
}
if canRightOuter {
allRightOuterJoins = p.getIndexJoinByOuterIdx(prop, 1)
forcedRightOuterJoins = make([]PhysicalPlan, 0, len(allRightOuterJoins))
for _, j := range allRightOuterJoins {
switch j.(type) {
case *PhysicalIndexJoin:
if hasINLJHint {
forcedRightOuterJoins = append(forcedRightOuterJoins, j)
}
case *PhysicalIndexHashJoin:
if hasINLHJHint {
forcedRightOuterJoins = append(forcedRightOuterJoins, j)
}
case *PhysicalIndexMergeJoin:
if hasINLMJHint {
forcedRightOuterJoins = append(forcedRightOuterJoins, j)
}
}
}
if !canLeftOuter || forcedRightOuterJoins != nil && (forceRightOuter && !forceLeftOuter) {
return forcedRightOuterJoins, forcedRightOuterJoins != nil && forceRightOuter
}

canForceLeft := leftJoins != nil && leftOuter
canForceRight := rightJoins != nil && rightOuter
forced = canForceLeft || canForceRight

joins := append(leftJoins, rightJoins...)
return joins, forced
}

return nil, false
canForceLeft := forcedLeftOuterJoins != nil && forceLeftOuter
canForceRight := forcedRightOuterJoins != nil && forceRightOuter
forced = canForceLeft || canForceRight
if forced {
return append(forcedLeftOuterJoins, forcedRightOuterJoins...), forced
}
return append(allLeftOuterJoins, allRightOuterJoins...), forced
}

// LogicalJoin can generates hash join, index join and sort merge join.
Expand Down
4 changes: 3 additions & 1 deletion planner/core/hints.go
View file
Expand Up @@ -315,7 +315,9 @@ func genHintsFromPhysicalPlan(p PhysicalPlan, nodeType nodeType) (res []*ast.Tab
case *PhysicalIndexJoin:
res = append(res, getJoinHints(p.SCtx(), HintINLJ, p.SelectBlockOffset(), nodeType, pp.children[pp.InnerChildIdx])...)
case *PhysicalIndexMergeJoin:
res = append(res, getJoinHints(p.SCtx(), HintINLJ, p.SelectBlockOffset(), nodeType, pp.children[pp.InnerChildIdx])...)
res = append(res, getJoinHints(p.SCtx(), HintINLMJ, p.SelectBlockOffset(), nodeType, pp.children[pp.InnerChildIdx])...)
case *PhysicalIndexHashJoin:
res = append(res, getJoinHints(p.SCtx(), HintINLHJ, p.SelectBlockOffset(), nodeType, pp.children[pp.InnerChildIdx])...)
}
return res
}
61 changes: 48 additions & 13 deletions planner/core/logical_plan_builder.go
View file
Expand Up @@ -55,8 +55,12 @@ const (
HintSMJ = "sm_join"
// TiDBIndexNestedLoopJoin is hint enforce index nested loop join.
TiDBIndexNestedLoopJoin = "tidb_inlj"
// HintINLJ is hint enforce index nested loop join.
// HintINLJ is hint enforce index nested loop hash/sort merge join.
HintINLJ = "inl_join"
// HintINLHJ is hint enforce index nested loop hash join.
HintINLHJ = "inl_hash_join"
// HintINLMJ is hint enforce index nested loop merge join.
HintINLMJ = "inl_merge_join"
// TiDBHashJoin is hint enforce hash join.
TiDBHashJoin = "tidb_hj"
// HintHJ is hint enforce hash join.
Expand Down Expand Up @@ -372,10 +376,22 @@ func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) {
p.preferJoinType |= preferHashJoin
}
if hintInfo.ifPreferINLJ(lhsAlias) {
p.preferJoinType |= preferLeftAsIndexInner
p.preferJoinType |= preferLeftAsINLJInner
}
if hintInfo.ifPreferINLJ(rhsAlias) {
p.preferJoinType |= preferRightAsIndexInner
p.preferJoinType |= preferRightAsINLJInner
}
if hintInfo.ifPreferINLHJ(lhsAlias) {
p.preferJoinType |= preferLeftAsINLHJInner
}
if hintInfo.ifPreferINLHJ(rhsAlias) {
p.preferJoinType |= preferRightAsINLHJInner
}
if hintInfo.ifPreferINLMJ(lhsAlias) {
p.preferJoinType |= preferLeftAsINLMJInner
}
if hintInfo.ifPreferINLMJ(rhsAlias) {
p.preferJoinType |= preferRightAsINLMJInner
}

// set hintInfo for further usage if this hint info can be used.
Expand All @@ -385,7 +401,10 @@ func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) {

// If there're multiple join types and one of them is not index join hint,
// then there is a conflict of join types.
if bits.OnesCount(p.preferJoinType) > 1 && (p.preferJoinType^preferRightAsIndexInner^preferLeftAsIndexInner) > 0 {
containIndexJoin := (p.preferJoinType^preferRightAsINLJInner^preferLeftAsINLJInner) > 0 ||
(p.preferJoinType^preferRightAsINLHJInner^preferLeftAsINLHJInner) > 0 ||
(p.preferJoinType^preferRightAsINLMJInner^preferLeftAsINLMJInner) > 0
if bits.OnesCount(p.preferJoinType) > 1 && containIndexJoin {
errMsg := "Join hints are conflict, you can only specify one type of join"
warning := ErrInternal.GenWithStack(errMsg)
p.ctx.GetSessionVars().StmtCtx.AppendWarning(warning)
Expand Down Expand Up @@ -2076,17 +2095,21 @@ func (b *PlanBuilder) unfoldWildStar(p LogicalPlan, selectFields []*ast.SelectFi
func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType nodeType, currentLevel int) {
hints = b.hintProcessor.getCurrentStmtHints(hints, nodeType, currentLevel)
var (
sortMergeTables, INLJTables, hashJoinTables []hintTableInfo
indexHintList []indexHintInfo
tiflashTables []hintTableInfo
aggHints aggHintInfo
sortMergeTables, INLJTables, INLHJTables, INLMJTables, hashJoinTables []hintTableInfo
indexHintList []indexHintInfo
tiflashTables []hintTableInfo
aggHints aggHintInfo
)
for _, hint := range hints {
switch hint.HintName.L {
case TiDBMergeJoin, HintSMJ:
sortMergeTables = append(sortMergeTables, tableNames2HintTableInfo(b.ctx, hint.Tables, b.hintProcessor, nodeType, currentLevel)...)
case TiDBIndexNestedLoopJoin, HintINLJ:
INLJTables = append(INLJTables, tableNames2HintTableInfo(b.ctx, hint.Tables, b.hintProcessor, nodeType, currentLevel)...)
case HintINLHJ:
INLHJTables = append(INLHJTables, tableNames2HintTableInfo(b.ctx, hint.Tables, b.hintProcessor, nodeType, currentLevel)...)
case HintINLMJ:
INLMJTables = append(INLMJTables, tableNames2HintTableInfo(b.ctx, hint.Tables, b.hintProcessor, nodeType, currentLevel)...)
case TiDBHashJoin, HintHJ:
hashJoinTables = append(hashJoinTables, tableNames2HintTableInfo(b.ctx, hint.Tables, b.hintProcessor, nodeType, currentLevel)...)
case HintHashAgg:
Expand Down Expand Up @@ -2137,7 +2160,7 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType n
}
b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{
sortMergeJoinTables: sortMergeTables,
indexNestedLoopJoinTables: INLJTables,
indexNestedLoopJoinTables: indexNestedLoopJoinTables{INLJTables, INLHJTables, INLMJTables},
hashJoinTables: hashJoinTables,
indexHintList: indexHintList,
flashTables: tiflashTables,
Expand All @@ -2147,7 +2170,9 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, nodeType n

func (b *PlanBuilder) popTableHints() {
hintInfo := b.tableHintInfo[len(b.tableHintInfo)-1]
b.appendUnmatchedJoinHintWarning(HintINLJ, TiDBIndexNestedLoopJoin, hintInfo.indexNestedLoopJoinTables)
b.appendUnmatchedJoinHintWarning(HintINLJ, TiDBIndexNestedLoopJoin, hintInfo.indexNestedLoopJoinTables.inljTables)
b.appendUnmatchedJoinHintWarning(HintINLHJ, "", hintInfo.indexNestedLoopJoinTables.inlhjTables)
b.appendUnmatchedJoinHintWarning(HintINLMJ, "", hintInfo.indexNestedLoopJoinTables.inlmjTables)
b.appendUnmatchedJoinHintWarning(HintSMJ, TiDBMergeJoin, hintInfo.sortMergeJoinTables)
b.appendUnmatchedJoinHintWarning(HintHJ, TiDBHashJoin, hintInfo.hashJoinTables)
b.tableHintInfo = b.tableHintInfo[:len(b.tableHintInfo)-1]
Expand All @@ -2158,8 +2183,12 @@ func (b *PlanBuilder) appendUnmatchedJoinHintWarning(joinType string, joinTypeAl
if len(unMatchedTables) == 0 {
return
}
errMsg := fmt.Sprintf("There are no matching table names for (%s) in optimizer hint %s or %s. Maybe you can use the table alias name",
strings.Join(unMatchedTables, ", "), restore2JoinHint(joinType, hintTables), restore2JoinHint(joinTypeAlias, hintTables))
if len(joinTypeAlias) != 0 {
joinTypeAlias = fmt.Sprintf(" or %s", restore2JoinHint(joinTypeAlias, hintTables))
}

errMsg := fmt.Sprintf("There are no matching table names for (%s) in optimizer hint %s%s. Maybe you can use the table alias name",
strings.Join(unMatchedTables, ", "), restore2JoinHint(joinType, hintTables), joinTypeAlias)
b.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrInternal.GenWithStack(errMsg))
}

Expand Down Expand Up @@ -2791,7 +2820,13 @@ func (b *PlanBuilder) buildSemiJoin(outerPlan, innerPlan LogicalPlan, onConditio
joinPlan.preferJoinType |= preferHashJoin
}
if b.TableHints().ifPreferINLJ(innerAlias) {
joinPlan.preferJoinType = preferRightAsIndexInner
joinPlan.preferJoinType = preferRightAsINLJInner
}
if b.TableHints().ifPreferINLHJ(innerAlias) {
joinPlan.preferJoinType = preferRightAsINLHJInner
}
if b.TableHints().ifPreferINLMJ(innerAlias) {
joinPlan.preferJoinType = preferRightAsINLMJInner
}
// If there're multiple join hints, they're conflict.
if bits.OnesCount(joinPlan.preferJoinType) > 1 {
Expand Down
8 changes: 6 additions & 2 deletions planner/core/logical_plans.go
View file
Expand Up @@ -97,8 +97,12 @@ func (tp JoinType) String() string {
}

const (
preferLeftAsIndexInner = 1 << iota
preferRightAsIndexInner
preferLeftAsINLJInner = 1 << iota
preferRightAsINLJInner
preferLeftAsINLHJInner
preferRightAsINLHJInner
preferLeftAsINLMJInner
preferRightAsINLMJInner
preferHashJoin
preferMergeJoin
preferHashAgg
Expand Down
28 changes: 21 additions & 7 deletions planner/core/planbuilder.go
View file
Expand Up @@ -56,13 +56,19 @@ type visitInfo struct {
err error
}

type indexNestedLoopJoinTables struct {
inljTables []hintTableInfo
inlhjTables []hintTableInfo
inlmjTables []hintTableInfo
}

type tableHintInfo struct {
indexNestedLoopJoinTables []hintTableInfo
sortMergeJoinTables []hintTableInfo
hashJoinTables []hintTableInfo
indexHintList []indexHintInfo
flashTables []hintTableInfo
aggHints aggHintInfo
indexNestedLoopJoinTables
sortMergeJoinTables []hintTableInfo
hashJoinTables []hintTableInfo
indexHintList []indexHintInfo
flashTables []hintTableInfo
aggHints aggHintInfo
}

type hintTableInfo struct {
Expand Down Expand Up @@ -108,7 +114,15 @@ func (info *tableHintInfo) ifPreferHashJoin(tableNames ...*hintTableInfo) bool {
}

func (info *tableHintInfo) ifPreferINLJ(tableNames ...*hintTableInfo) bool {
return info.matchTableName(tableNames, info.indexNestedLoopJoinTables)
return info.matchTableName(tableNames, info.indexNestedLoopJoinTables.inljTables)
}

func (info *tableHintInfo) ifPreferINLHJ(tableNames ...*hintTableInfo) bool {
return info.matchTableName(tableNames, info.indexNestedLoopJoinTables.inlhjTables)
}

func (info *tableHintInfo) ifPreferINLMJ(tableNames ...*hintTableInfo) bool {
return info.matchTableName(tableNames, info.indexNestedLoopJoinTables.inlmjTables)
}

func (info *tableHintInfo) ifPreferTiFlash(tableNames ...*hintTableInfo) bool {
Expand Down

0 comments on commit efa1b90

Please sign in to comment.