Skip to content

Commit

Permalink
update:
Browse files Browse the repository at this point in the history 
1. update the sql parser to the latest
2. temporarily remove group_concat support
3. fix fail tests
4. add vendor.json to avoid future build failure
  • Loading branch information
@cch123
cch123 committed Sep 21, 2017
1 parent f0142d9 commit 630a383
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 67 deletions.
43 changes: 23 additions & 20 deletions select_agg_handler.go
View file
Expand Up @@ -17,8 +17,8 @@ func handleFuncInSelectAgg(funcExprArr []*sqlparser.FuncExpr) msi {
for _, v := range funcExprArr {
//func expressions will use the same parent bucket

aggName := strings.ToUpper(string(v.Name)) + `(` + sqlparser.String(v.Exprs) + `)`
switch string(v.Name) {
aggName := strings.ToUpper(v.Name.String()) + `(` + sqlparser.String(v.Exprs) + `)`
switch v.Name.Lowered() {
case "count":
//count need to distinguish * and normal field name
if sqlparser.String(v.Exprs) == "*" {
Expand Down Expand Up @@ -47,7 +47,7 @@ func handleFuncInSelectAgg(funcExprArr []*sqlparser.FuncExpr) msi {
// support min/avg/max/stats
// extended_stats/percentiles
innerAggMap[aggName] = msi{
string(v.Name): msi{
v.Name.String(): msi{
"field": sqlparser.String(v.Exprs),
},
}
Expand All @@ -63,20 +63,20 @@ func handleGroupByColName(colName *sqlparser.ColName, index int, child msi) msi
innerMap := make(msi)
if index == 0 {
innerMap["terms"] = msi{
"field": string(colName.Name),
"field": colName.Name.String(),
"size": 200, // this size may need to change ?
}
} else {
innerMap["terms"] = msi{
"field": string(colName.Name),
"field": colName.Name.String(),
"size": 0,
}
}

if len(child) > 0 {
innerMap["aggregations"] = child
}
return msi{string(colName.Name): innerMap}
return msi{colName.Name.String(): innerMap}
}

func handleGroupByFuncExprDateHisto(funcExpr *sqlparser.FuncExpr) (msi, error) {
Expand All @@ -91,10 +91,11 @@ func handleGroupByFuncExprDateHisto(funcExpr *sqlparser.FuncExpr) (msi, error) {
//get field/interval and format
for _, expr := range funcExpr.Exprs {
// the expression in date_histogram must be like a = b format
switch expr.(type) {
case *sqlparser.NonStarExpr:
nonStarExpr := expr.(*sqlparser.NonStarExpr)
comparisonExpr, ok := nonStarExpr.Expr.(*sqlparser.ComparisonExpr)
switch item := expr.(type) {
case *sqlparser.AliasedExpr:
//nonStarExpr := expr.(*sqlparser.NonStarExpr)
comparisonExpr, ok := item.Expr.(*sqlparser.ComparisonExpr)

if !ok {
return nil, errors.New("elasticsql: unsupported expression in date_histogram")
}
Expand All @@ -104,13 +105,13 @@ func handleGroupByFuncExprDateHisto(funcExpr *sqlparser.FuncExpr) (msi, error) {
}
rightStr := sqlparser.String(comparisonExpr.Right)
rightStr = strings.Replace(rightStr, `'`, ``, -1)
if string(left.Name) == "field" {
if left.Name.Lowered() == "field" {
field = rightStr
}
if string(left.Name) == "interval" {
if left.Name.Lowered() == "interval" {
interval = rightStr
}
if string(left.Name) == "format" {
if left.Name.Lowered() == "format" {
format = rightStr
}

Expand Down Expand Up @@ -142,7 +143,7 @@ func handleGroupByFuncExprRange(funcExpr *sqlparser.FuncExpr) (msi, error) {
"to": valTo,
}
}
innerMap[string(funcExpr.Name)] = msi{
innerMap[funcExpr.Name.String()] = msi{
"field": sqlparser.String(funcExpr.Exprs[0]),
"ranges": rangeMapList,
}
Expand All @@ -160,15 +161,16 @@ func handleGroupByFuncExprDateRange(funcExpr *sqlparser.FuncExpr) (msi, error) {
)

for _, expr := range funcExpr.Exprs {
nonStarExpr, ok := expr.(*sqlparser.NonStarExpr)
nonStarExpr, ok := expr.(*sqlparser.AliasedExpr)
if !ok {
return nil, errors.New("elasticsql: unsupported star expression in function date_range")
}

switch item := nonStarExpr.Expr.(type) {
case *sqlparser.ComparisonExpr:
colName := sqlparser.String(item.Left)
equalVal := sqlparser.String(item.Right.(sqlparser.StrVal))
equalVal := sqlparser.String(item.Right.(*sqlparser.SQLVal))
//fmt.Printf("%#v", sqlparser.String(item.Right))
equalVal = strings.Trim(equalVal, `'`)

switch colName {
Expand All @@ -179,8 +181,9 @@ func handleGroupByFuncExprDateRange(funcExpr *sqlparser.FuncExpr) (msi, error) {
default:
return nil, errors.New("elasticsql: unsupported column name " + colName)
}
case sqlparser.StrVal:
rangeList = append(rangeList, sqlparser.String(item))
case *sqlparser.SQLVal:
skippedString := strings.Trim(sqlparser.String(item), "`")
rangeList = append(rangeList, skippedString)
default:
return nil, errors.New("elasticsql: unsupported expression " + sqlparser.String(expr))
}
Expand Down Expand Up @@ -214,7 +217,7 @@ func handleGroupByFuncExpr(funcExpr *sqlparser.FuncExpr, child msi) (msi, error)
var innerMap msi
var err error

switch string(funcExpr.Name) {
switch funcExpr.Name.Lowered() {
case "date_histogram":
innerMap, err = handleGroupByFuncExprDateHisto(funcExpr)
case "range":
Expand Down Expand Up @@ -291,7 +294,7 @@ func extractFuncAndColFromSelect(sqlSelect sqlparser.SelectExprs) ([]*sqlparser.
for _, v := range sqlSelect {
// non star expressioin means column name
// or some aggregation functions
expr, ok := v.(*sqlparser.NonStarExpr)
expr, ok := v.(*sqlparser.AliasedExpr)
if !ok {
// no need to handle, star expression * just skip is ok
continue
Expand Down
49 changes: 16 additions & 33 deletions select_handler.go
View file
Expand Up @@ -14,7 +14,7 @@ func handleSelect(sel *sqlparser.Select) (dsl string, esType string, err error)
// top level node pass in an empty interface
// to tell the children this is root
// is there any better way?
var rootParent sqlparser.BoolExpr
var rootParent sqlparser.Expr
var defaultQueryMapStr = `{"bool" : {"must": [{"match_all" : {}}]}}`
var queryMapStr string

Expand Down Expand Up @@ -72,8 +72,7 @@ func handleSelect(sel *sqlparser.Select) (dsl string, esType string, err error)
var orderByArr []string
if aggFlag == false {
for _, orderByExpr := range sel.OrderBy {
field := strings.Replace(sqlparser.String(orderByExpr.Expr), "`", "", -1)
orderByStr := fmt.Sprintf(`{"%v": "%v"}`, field, orderByExpr.Direction)
orderByStr := fmt.Sprintf(`{"%v": "%v"}`, strings.Replace(sqlparser.String(orderByExpr.Expr), "`", "", -1), orderByExpr.Direction)
orderByArr = append(orderByArr, orderByStr)
}
}
Expand Down Expand Up @@ -106,7 +105,7 @@ func handleSelect(sel *sqlparser.Select) (dsl string, esType string, err error)
// if the where is empty, need to check whether to agg or not
func checkNeedAgg(sqlSelect sqlparser.SelectExprs) bool {
for _, v := range sqlSelect {
expr, ok := v.(*sqlparser.NonStarExpr)
expr, ok := v.(*sqlparser.AliasedExpr)
if !ok {
// no need to handle, star expression * just skip is ok
continue
Expand All @@ -121,26 +120,10 @@ func checkNeedAgg(sqlSelect sqlparser.SelectExprs) bool {
}

func buildNestedFuncStrValue(nestedFunc *sqlparser.FuncExpr) (string, error) {
var result string
switch string(nestedFunc.Name) {
case "group_concat":
for _, nestedExpr := range nestedFunc.Exprs {
switch nestedExpr.(type) {
case *sqlparser.NonStarExpr:
nonStarExpr := nestedExpr.(*sqlparser.NonStarExpr)
result += strings.Trim(sqlparser.String(nonStarExpr), `'`)
default:
return "", errors.New("elasticsql: unsupported expression" + sqlparser.String(nestedExpr))
}
}
//TODO support more functions
default:
return "", errors.New("elasticsql: unsupported function" + string(nestedFunc.Name))
}
return result, nil
return "", errors.New("elasticsql: unsupported function" + nestedFunc.Name.String())
}

func handleSelectWhereAndExpr(expr *sqlparser.BoolExpr, topLevel bool, parent *sqlparser.BoolExpr) (string, error) {
func handleSelectWhereAndExpr(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) (string, error) {
andExpr := (*expr).(*sqlparser.AndExpr)
leftExpr := andExpr.Left
rightExpr := andExpr.Right
Expand Down Expand Up @@ -169,7 +152,7 @@ func handleSelectWhereAndExpr(expr *sqlparser.BoolExpr, topLevel bool, parent *s
return fmt.Sprintf(`{"bool" : {"must" : [%v]}}`, resultStr), nil
}

func handleSelectWhereOrExpr(expr *sqlparser.BoolExpr, topLevel bool, parent *sqlparser.BoolExpr) (string, error) {
func handleSelectWhereOrExpr(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) (string, error) {
orExpr := (*expr).(*sqlparser.OrExpr)
leftExpr := orExpr.Left
rightExpr := orExpr.Right
Expand Down Expand Up @@ -200,16 +183,16 @@ func handleSelectWhereOrExpr(expr *sqlparser.BoolExpr, topLevel bool, parent *sq
return fmt.Sprintf(`{"bool" : {"should" : [%v]}}`, resultStr), nil
}

func buildComparisonExprRightStr(expr sqlparser.ValExpr) (string, bool, error) {
func buildComparisonExprRightStr(expr sqlparser.Expr) (string, bool, error) {
var rightStr string
var err error
var missingCheck = false
switch expr.(type) {
case sqlparser.StrVal:
case *sqlparser.SQLVal:
rightStr = sqlparser.String(expr)
rightStr = strings.Trim(rightStr, `'`)
case sqlparser.NumVal:
rightStr = sqlparser.String(expr)
case *sqlparser.GroupConcatExpr:
return "", missingCheck, errors.New("elasticsql: group_concat not supported")
case *sqlparser.FuncExpr:
// parse nested
funcExpr := expr.(*sqlparser.FuncExpr)
Expand All @@ -232,7 +215,7 @@ func buildComparisonExprRightStr(expr sqlparser.ValExpr) (string, bool, error) {
return rightStr, missingCheck, err
}

func handleSelectWhereComparisonExpr(expr *sqlparser.BoolExpr, topLevel bool, parent *sqlparser.BoolExpr) (string, error) {
func handleSelectWhereComparisonExpr(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) (string, error) {
comparisonExpr := (*expr).(*sqlparser.ComparisonExpr)
colName, ok := comparisonExpr.Left.(*sqlparser.ColName)

Expand Down Expand Up @@ -301,7 +284,7 @@ func handleSelectWhereComparisonExpr(expr *sqlparser.BoolExpr, topLevel bool, pa
return resultStr, nil
}

func handleSelectWhere(expr *sqlparser.BoolExpr, topLevel bool, parent *sqlparser.BoolExpr) (string, error) {
func handleSelectWhere(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) (string, error) {
if expr == nil {
return "", errors.New("elasticsql: error expression cannot be nil here")
}
Expand All @@ -315,8 +298,8 @@ func handleSelectWhere(expr *sqlparser.BoolExpr, topLevel bool, parent *sqlparse
case *sqlparser.ComparisonExpr:
return handleSelectWhereComparisonExpr(expr, topLevel, parent)

case *sqlparser.NullCheck:
return "", errors.New("elasticsql: null check expression currently not supported")
case *sqlparser.IsExpr:
return "", errors.New("elasticsql: is expression currently not supported")
case *sqlparser.RangeCond:
// between a and b
// the meaning is equal to range query
Expand All @@ -338,8 +321,8 @@ func handleSelectWhere(expr *sqlparser.BoolExpr, topLevel bool, parent *sqlparse

return resultStr, nil

case *sqlparser.ParenBoolExpr:
parentBoolExpr := (*expr).(*sqlparser.ParenBoolExpr)
case *sqlparser.ParenExpr:
parentBoolExpr := (*expr).(*sqlparser.ParenExpr)
boolExpr := parentBoolExpr.Expr

// if paren is the top level, bool must is needed
Expand Down
28 changes: 14 additions & 14 deletions select_test.go
View file
Expand Up @@ -9,7 +9,7 @@ import (
)

var badSQLList = []string{
"select aaac fr",
//"select aaac fr",
"delete",
"update x",
"insert ",
Expand Down Expand Up @@ -55,17 +55,17 @@ var selectCaseMap = map[string]string{
"select count(*) from ark group by date_histogram(field='create_time', value='1h')": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"date_histogram(field=create_time,value=1h)":{"aggregations":{"COUNT(*)":{"value_count":{"field":"_index"}}},"date_histogram":{"field":"create_time","format":"yyyy-MM-dd HH:mm:ss","interval":"1h"}}}}`,
"select * from ark group by date_histogram(field='create_time', value='1h')": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"date_histogram(field=create_time,value=1h)":{"date_histogram":{"field":"create_time","format":"yyyy-MM-dd HH:mm:ss","interval":"1h"}}}}`,
"select * from ark group by date_histogram(field='create_time', value='1h'), id": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"date_histogram(field=create_time,value=1h)":{"aggregations":{"id":{"terms":{"field":"id","size":0}}},"date_histogram":{"field":"create_time","format":"yyyy-MM-dd HH:mm:ss","interval":"1h"}}}}`,
"select * from ark where a like group_concat('%', 'abc', '%')": `{"query" : {"bool" : {"must" : [{"match" : {"a" : {"query" : "abc", "type" : "phrase"}}}]}},"from" : 0,"size" : 1}`,
"select * from `order`.abcd where `by` = 1": `{"query" : {"bool" : {"must" : [{"match" : {"by" : {"query" : "1", "type" : "phrase"}}}]}},"from" : 0,"size" : 1}`,
"select * from ark where id not like '%aaa%'": `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"match" : {"id" : {"query" : "aaa", "type" : "phrase"}}}}}]}},"from" : 0,"size" : 1}`,
"select * from ark where id not in (1,2,3)": `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"terms" : {"id" : [1, 2, 3]}}}}]}},"from" : 0,"size" : 1}`,
"select * from abc limit 10,10": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 10,"size" : 10}`,
"select * from abc limit 10": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 10}`,
"select count(*), id from ark group by id": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"id":{"aggregations":{"COUNT(*)":{"value_count":{"field":"_index"}}},"terms":{"field":"id","size":200}}}}`,
"SELECT COUNT(distinct age) FROM bank GROUP BY range(age, 20,25,30,35,40)": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"range(age,20,25,30,35,40)":{"aggregations":{"COUNT(age)":{"cardinality":{"field":"age"}}},"range":{"field":"age","ranges":[{"from":"20","to":"25"},{"from":"25","to":"30"},{"from":"30","to":"35"},{"from":"35","to":"40"}]}}}}`,
"select * from a where id != missing": `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : [{"missing":{"field":"id"}}]}}]}},"from" : 0,"size" : 1}`,
"select * from a where id = missing": `{"query" : {"bool" : {"must" : [{"missing":{"field":"id"}}]}},"from" : 0,"size" : 1}`,
"select count(*) from a": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"COUNT(*)":{"value_count":{"field":"_index"}}}}`,
// "select * from ark where a like group_concat('%', 'abc', '%')": `{"query" : {"bool" : {"must" : [{"match" : {"a" : {"query" : "abc", "type" : "phrase"}}}]}},"from" : 0,"size" : 1}`,
"select * from `order`.abcd where `by` = 1": `{"query" : {"bool" : {"must" : [{"match" : {"by" : {"query" : "1", "type" : "phrase"}}}]}},"from" : 0,"size" : 1}`,
"select * from ark where id not like '%aaa%'": `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"match" : {"id" : {"query" : "aaa", "type" : "phrase"}}}}}]}},"from" : 0,"size" : 1}`,
"select * from ark where id not in (1,2,3)": `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : {"terms" : {"id" : [1, 2, 3]}}}}]}},"from" : 0,"size" : 1}`,
"select * from abc limit 10,10": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 10,"size" : 10}`,
"select * from abc limit 10": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 10}`,
"select count(*), id from ark group by id": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"id":{"aggregations":{"COUNT(*)":{"value_count":{"field":"_index"}}},"terms":{"field":"id","size":200}}}}`,
"SELECT COUNT(distinct age) FROM bank GROUP BY range(age, 20,25,30,35,40)": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"range(age,20,25,30,35,40)":{"aggregations":{"COUNT(age)":{"cardinality":{"field":"age"}}},"range":{"field":"age","ranges":[{"from":"20","to":"25"},{"from":"25","to":"30"},{"from":"30","to":"35"},{"from":"35","to":"40"}]}}}}`,
"select * from a where id != missing": `{"query" : {"bool" : {"must" : [{"bool" : {"must_not" : [{"missing":{"field":"id"}}]}}]}},"from" : 0,"size" : 1}`,
"select * from a where id = missing": `{"query" : {"bool" : {"must" : [{"missing":{"field":"id"}}]}},"from" : 0,"size" : 1}`,
"select count(*) from a": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"COUNT(*)":{"value_count":{"field":"_index"}}}}`,
"SELECT online FROM online GROUP BY date_range(field='insert_time' , format='yyyy-MM-dd', '2014-08-18','2014-08-17','now-8d','now-7d','now-6d','now')": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"date_range(field=insert_time,format=yyyy-MM-dd,2014-08-18,2014-08-17,now-8d,now-7d,now-6d,now)":{"date_range":{"field":"insert_time","format":"yyyy-MM-dd","ranges":[{"from":"2014-08-18","to":"2014-08-17"},{"from":"2014-08-17","to":"now-8d"},{"from":"now-8d","to":"now-7d"},{"from":"now-7d","to":"now-6d"},{"from":"now-6d","to":"now"}]}}}}`,
"select count(id), sum(age) from a group by id": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 0,"aggregations" : {"id":{"aggregations":{"COUNT(id)":{"value_count":{"field":"id"}},"SUM(age)":{"sum":{"field":"age"}}},"terms":{"field":"id","size":200}}}}`,
"select * from a order by `order`.abc": `{"query" : {"bool" : {"must": [{"match_all" : {}}]}},"from" : 0,"size" : 1,"sort" : [{"order.abc": "asc"}]}`,
Expand Down Expand Up @@ -126,10 +126,10 @@ func TestUnsupported(t *testing.T) {

func TestBadSQL(t *testing.T) {
for _, v := range badSQLList {
_, _, err := Convert(v)
dsl, table, err := Convert(v)

if err == nil {
t.Error("can not be true, these cases are not supported!")
t.Error("can not be true, these cases are not supported!", v, dsl, table)
}

_, _, err = ConvertPretty(v)
Expand Down

0 comments on commit 630a383

Please sign in to comment.