Skip to content

Commit

Permalink
feat: support limit without orderby (#170)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dk-lockdown
dk-lockdown committed Jul 1, 2022
1 parent 005ea15 commit dbff926
Show file tree
Hide file tree
Showing 9 changed files with 340 additions and 21 deletions.
16 changes: 7 additions & 9 deletions cmd/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ import (
"github.com/cectc/dbpack/pkg/proto"
"github.com/cectc/dbpack/pkg/resource"
"github.com/cectc/dbpack/pkg/server"
"github.com/cectc/dbpack/pkg/tracing"
"github.com/cectc/dbpack/third_party/pools"
_ "github.com/cectc/dbpack/third_party/types/parser_driver"
)
Expand Down Expand Up @@ -155,13 +154,12 @@ var (
}
}

tracingMgr, err := tracing.NewTracer(Version, "console")
if err != nil {
log.Fatalf("could not setup tracing manager: %s", err.Error())
}
if err != nil {
log.Fatalf("could not setup tracing exporter: %s", err.Error())
}
// temporarily turn off tracer output
//tracingMgr, err := tracing.NewTracer(Version, "console")
//if err != nil {
// log.Fatalf("could not setup tracing manager: %s", err.Error())
//}

ctx, cancel := context.WithCancel(context.Background())
c := make(chan os.Signal, 2)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
Expand All @@ -174,7 +172,7 @@ var (
cancel()
}()
<-c
_ = tracingMgr.Shutdown(ctx)
//_ = tracingMgr.Shutdown(ctx)
os.Exit(1) // second signal. Exit directly.
}()

Expand Down
24 changes: 21 additions & 3 deletions pkg/optimize/optimize_select.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,22 @@ import (
"github.com/pkg/errors"

"github.com/cectc/dbpack/pkg/cond"
"github.com/cectc/dbpack/pkg/dt/schema"
"github.com/cectc/dbpack/pkg/meta"
"github.com/cectc/dbpack/pkg/plan"
"github.com/cectc/dbpack/pkg/proto"
"github.com/cectc/dbpack/pkg/resource"
"github.com/cectc/dbpack/pkg/topo"
"github.com/cectc/dbpack/third_party/parser/ast"
)

func (o Optimizer) optimizeSelect(ctx context.Context, stmt *ast.SelectStmt, args []interface{}) (proto.Plan, error) {
var (
alg cond.ShardingAlgorithm
topology *topo.Topology
exists bool
alg cond.ShardingAlgorithm
topology *topo.Topology
tableMeta schema.TableMeta
exists bool
err error
)
tableName := stmt.From.TableRefs.Left.(*ast.TableSource).Source.(*ast.TableName).Name.String()

Expand All @@ -44,6 +49,17 @@ func (o Optimizer) optimizeSelect(ctx context.Context, stmt *ast.SelectStmt, arg
return nil, errors.New("topology should not be nil")
}

for db, tables := range topology.DBs {
sqlDB := resource.GetDBManager().GetDB(db)
tableMeta, err = meta.GetTableMetaCache().GetTableMeta(ctx, sqlDB, tables[0])
if err != nil {
continue
} else {
break
}
}
pk := tableMeta.GetPKName()

condition, err := cond.ParseCondition(stmt.Where, args...)
if err != nil {
return nil, errors.Wrap(err, "parse condition failed")
Expand All @@ -69,6 +85,7 @@ func (o Optimizer) optimizeSelect(ctx context.Context, stmt *ast.SelectStmt, arg
return &plan.QueryOnSingleDBPlan{
Database: k,
Tables: v,
PK: pk,
Stmt: stmt,
Args: args,
Executor: executor,
Expand All @@ -92,6 +109,7 @@ func (o Optimizer) optimizeSelect(ctx context.Context, stmt *ast.SelectStmt, arg
plans = append(plans, &plan.QueryOnSingleDBPlan{
Database: k,
Tables: shardMap[k],
PK: pk,
Stmt: stmt,
Args: args,
Executor: executor,
Expand Down
178 changes: 178 additions & 0 deletions pkg/optimize/optimizer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,95 @@ func TestOptimizeQueryOnSingleDB(t *testing.T) {
return
}
stmt.Accept(&visitor.ParamVisitor{})

resource.SetDBManager(&resource.DBManager{})
var cache *meta.MysqlTableMetaCache
patches := gomonkey.ApplyMethodFunc(cache, "GetTableMeta", func(ctx context.Context, db proto.DB, tableName string) (schema.TableMeta, error) {
return schema.TableMeta{
SchemaName: "school",
TableName: "student",
Columns: []string{"id", "age"},
AllColumns: map[string]schema.ColumnMeta{
"id": {
TableCat: "def",
TableSchemeName: "school",
TableName: "student",
ColumnName: "id",
DataType: -5,
DataTypeName: "bigint",
ColumnSize: 0,
DecimalDigits: 19,
NumPrecRadix: 0,
Nullable: 0,
Remarks: "",
ColumnDef: "",
SqlDataType: 0,
SqlDatetimeSub: 0,
CharOctetLength: 0,
OrdinalPosition: 1,
IsNullable: "NO",
IsAutoIncrement: "auto_increment",
},
"age": {
TableCat: "def",
TableSchemeName: "school",
TableName: "student",
ColumnName: "age",
DataType: 0,
DataTypeName: "int",
ColumnSize: 0,
DecimalDigits: 10,
NumPrecRadix: 0,
Nullable: 0,
Remarks: "",
ColumnDef: "",
SqlDataType: 0,
SqlDatetimeSub: 0,
CharOctetLength: 0,
OrdinalPosition: 2,
IsNullable: "NO",
IsAutoIncrement: "",
},
},
AllIndexes: map[string]schema.IndexMeta{
"id": {
Values: []schema.ColumnMeta{
{
TableCat: "def",
TableSchemeName: "school",
TableName: "student",
ColumnName: "id",
DataType: -5,
DataTypeName: "bigint",
ColumnSize: 0,
DecimalDigits: 19,
NumPrecRadix: 0,
Nullable: 0,
Remarks: "",
ColumnDef: "",
SqlDataType: 0,
SqlDatetimeSub: 0,
CharOctetLength: 0,
OrdinalPosition: 1,
IsNullable: "NO",
IsAutoIncrement: "auto_increment",
},
},
NonUnique: false,
IndexQualifier: "",
IndexName: "PRIMARY",
ColumnName: "id",
Type: 0,
IndexType: schema.IndexTypePrimary,
AscOrDesc: "A",
Cardinality: 1,
OrdinalPosition: 1,
},
},
}, nil
})
defer patches.Reset()

pl, err := o.Optimize(context.Background(), stmt, args...)
assert.Equal(t, nil, err)
queryPlan, ok := pl.(*plan.QueryOnSingleDBPlan)
Expand All @@ -66,6 +155,95 @@ func TestOptimizeQueryOnMultiDB(t *testing.T) {
return
}
stmt.Accept(&visitor.ParamVisitor{})

resource.SetDBManager(&resource.DBManager{})
var cache *meta.MysqlTableMetaCache
patches := gomonkey.ApplyMethodFunc(cache, "GetTableMeta", func(ctx context.Context, db proto.DB, tableName string) (schema.TableMeta, error) {
return schema.TableMeta{
SchemaName: "school",
TableName: "student",
Columns: []string{"id", "age"},
AllColumns: map[string]schema.ColumnMeta{
"id": {
TableCat: "def",
TableSchemeName: "school",
TableName: "student",
ColumnName: "id",
DataType: -5,
DataTypeName: "bigint",
ColumnSize: 0,
DecimalDigits: 19,
NumPrecRadix: 0,
Nullable: 0,
Remarks: "",
ColumnDef: "",
SqlDataType: 0,
SqlDatetimeSub: 0,
CharOctetLength: 0,
OrdinalPosition: 1,
IsNullable: "NO",
IsAutoIncrement: "auto_increment",
},
"age": {
TableCat: "def",
TableSchemeName: "school",
TableName: "student",
ColumnName: "age",
DataType: 0,
DataTypeName: "int",
ColumnSize: 0,
DecimalDigits: 10,
NumPrecRadix: 0,
Nullable: 0,
Remarks: "",
ColumnDef: "",
SqlDataType: 0,
SqlDatetimeSub: 0,
CharOctetLength: 0,
OrdinalPosition: 2,
IsNullable: "NO",
IsAutoIncrement: "",
},
},
AllIndexes: map[string]schema.IndexMeta{
"id": {
Values: []schema.ColumnMeta{
{
TableCat: "def",
TableSchemeName: "school",
TableName: "student",
ColumnName: "id",
DataType: -5,
DataTypeName: "bigint",
ColumnSize: 0,
DecimalDigits: 19,
NumPrecRadix: 0,
Nullable: 0,
Remarks: "",
ColumnDef: "",
SqlDataType: 0,
SqlDatetimeSub: 0,
CharOctetLength: 0,
OrdinalPosition: 1,
IsNullable: "NO",
IsAutoIncrement: "auto_increment",
},
},
NonUnique: false,
IndexQualifier: "",
IndexName: "PRIMARY",
ColumnName: "id",
Type: 0,
IndexType: schema.IndexTypePrimary,
AscOrDesc: "A",
Cardinality: 1,
OrdinalPosition: 1,
},
},
}, nil
})
defer patches.Reset()

pl, err := o.Optimize(context.Background(), stmt, args...)
assert.Equal(t, nil, err)
queryPlan, ok := pl.(*plan.QueryOnMultiDBPlan)
Expand Down
12 changes: 7 additions & 5 deletions pkg/plan/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
type QueryOnSingleDBPlan struct {
Database string
Tables []string
PK string
Stmt *ast.SelectStmt
Limit *Limit
Args []interface{}
Expand Down Expand Up @@ -83,9 +84,8 @@ func (p *QueryOnSingleDBPlan) generate(sb *strings.Builder, args *[]interface{})
err = generateSelect(p.Tables[0], p.Stmt, sb, p.Limit)
p.appendArgs(args)
default:
if p.Stmt.OrderBy != nil {
sb.WriteString("SELECT * FROM (")
}
sb.WriteString("SELECT * FROM (")

sb.WriteByte('(')
if err = generateSelect(p.Tables[0], p.Stmt, sb, p.Limit); err != nil {
return
Expand All @@ -103,12 +103,14 @@ func (p *QueryOnSingleDBPlan) generate(sb *strings.Builder, args *[]interface{})
sb.WriteByte(')')
p.appendArgs(args)
}
sb.WriteString(") t ")
if p.Stmt.OrderBy != nil {
sb.WriteString(") t ")
restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, sb)
if err := p.Stmt.OrderBy.Restore(restoreCtx); err != nil {
return errors.WithStack(err)
}
} else {
sb.WriteString(fmt.Sprintf("ORDER BY `%s` ASC", p.PK))
}
}
return
Expand Down Expand Up @@ -258,7 +260,7 @@ func generateSelect(table string, stmt *ast.SelectStmt, sb *strings.Builder, lim
if limit != nil {
sb.WriteByte(' ')
limitCount := limit.Offset + limit.Count
sb.WriteString(fmt.Sprintf("limit %d", limitCount))
sb.WriteString(fmt.Sprintf("LIMIT %d", limitCount))
}

return nil
Expand Down
9 changes: 7 additions & 2 deletions pkg/plan/query_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,26 +31,30 @@ func TestQueryOnSingleDBPlan(t *testing.T) {
testCases := []struct {
selectSql string
tables []string
pk string
args []interface{}
expectedGenerateSql string
}{
{
selectSql: "select * from student where id in (?,?)",
tables: []string{"student_1", "student_5"},
pk: "id",
args: []interface{}{1, 5},
expectedGenerateSql: "(SELECT * FROM `student_1` WHERE `id` IN (?,?)) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?))",
expectedGenerateSql: "SELECT * FROM ((SELECT * FROM `student_1` WHERE `id` IN (?,?)) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?))) t ORDER BY `id` ASC",
},
{
selectSql: "select * from student where id in (?,?) order by id desc",
tables: []string{"student_1", "student_5"},
pk: "id",
args: []interface{}{1, 5},
expectedGenerateSql: "SELECT * FROM ((SELECT * FROM `student_1` WHERE `id` IN (?,?) ORDER BY `id` DESC) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?) ORDER BY `id` DESC)) t ORDER BY `id` DESC",
},
{
selectSql: "select * from student where id in (?,?) order by id desc limit ?, ?",
tables: []string{"student_1", "student_5"},
pk: "id",
args: []interface{}{1, 5, 1000, 20},
expectedGenerateSql: "SELECT * FROM ((SELECT * FROM `student_1` WHERE `id` IN (?,?) ORDER BY `id` DESC limit 1020) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?) ORDER BY `id` DESC limit 1020)) t ORDER BY `id` DESC",
expectedGenerateSql: "SELECT * FROM ((SELECT * FROM `student_1` WHERE `id` IN (?,?) ORDER BY `id` DESC LIMIT 1020) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?) ORDER BY `id` DESC LIMIT 1020)) t ORDER BY `id` DESC",
},
}

Expand All @@ -67,6 +71,7 @@ func TestQueryOnSingleDBPlan(t *testing.T) {
plan := &QueryOnSingleDBPlan{
Database: "school_0",
Tables: c.tables,
PK: c.pk,
Stmt: selectStmt,
Args: c.args,
Executor: nil,
Expand Down

0 comments on commit dbff926

Please sign in to comment.