Skip to content

Commit

Permalink
sqlparser: zerocopy strings
Browse files Browse the repository at this point in the history 
Signed-off-by: Vicent Marti <vmg@strn.cat>
  • Loading branch information
@vmg
vmg committed Mar 8, 2021
1 parent c8e3eec commit e921494
Show file tree
Hide file tree
Showing 36 changed files with 1,453 additions and 1,514 deletions.
11 changes: 5 additions & 6 deletions go/vt/binlog/event_streamer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ func parsePkNames(tokenizer *sqlparser.Tokenizer) ([]*querypb.Field, error) {
Name: string(val),
})
default:
return nil, fmt.Errorf("syntax error at position: %d", tokenizer.Position)
return nil, fmt.Errorf("syntax error at position: %d", tokenizer.Pos)
}
}
return columns, nil
Expand Down Expand Up @@ -297,15 +297,14 @@ func parsePkTuple(tokenizer *sqlparser.Tokenizer, insertid int64, fields []*quer
return nil, insertid, fmt.Errorf("incompatible string field with type %v", fields[index].Type)
}

decoded := make([]byte, base64.StdEncoding.DecodedLen(len(val)))
numDecoded, err := base64.StdEncoding.Decode(decoded, val)
decoded, err := base64.StdEncoding.DecodeString(val)
if err != nil {
return nil, insertid, err
}
result.Lengths = append(result.Lengths, int64(numDecoded))
result.Values = append(result.Values, decoded[:numDecoded]...)
result.Lengths = append(result.Lengths, int64(len(decoded)))
result.Values = append(result.Values, decoded...)
default:
return nil, insertid, fmt.Errorf("syntax error at position: %d", tokenizer.Position)
return nil, insertid, fmt.Errorf("syntax error at position: %d", tokenizer.Pos)
}
index++
}
Expand Down
4 changes: 2 additions & 2 deletions go/vt/sqlparser/analyzer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,9 +409,9 @@ func NewPlanValue(node Expr) (sqltypes.PlanValue, error) {
}
return sqltypes.PlanValue{Value: n}, nil
case FloatVal:
return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.Float64, node.Val)}, nil
return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.Float64, node.Bytes())}, nil
case StrVal:
return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val)}, nil
return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Bytes())}, nil
case HexVal:
v, err := node.HexDecode()
if err != nil {
Expand Down
56 changes: 20 additions & 36 deletions go/vt/sqlparser/analyzer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func TestIsColName(t *testing.T) {
in: &ColName{},
out: true,
}, {
in: newHexLiteral(""),
in: NewHexLiteral(""),
}}
for _, tc := range testcases {
out := IsColName(tc.in)
Expand All @@ -260,16 +260,16 @@ func TestIsValue(t *testing.T) {
in Expr
out bool
}{{
in: newStrLiteral("aa"),
in: NewStrLiteral("aa"),
out: true,
}, {
in: newHexLiteral("3131"),
in: NewHexLiteral("3131"),
out: true,
}, {
in: newIntLiteral("1"),
in: NewIntLiteral("1"),
out: true,
}, {
in: newArgument(":a"),
in: NewArgument(":a"),
out: true,
}, {
in: &NullVal{},
Expand Down Expand Up @@ -300,7 +300,7 @@ func TestIsNull(t *testing.T) {
in: &NullVal{},
out: true,
}, {
in: newStrLiteral(""),
in: NewStrLiteral(""),
}}
for _, tc := range testcases {
out := IsNull(tc.in)
Expand All @@ -315,7 +315,7 @@ func TestIsSimpleTuple(t *testing.T) {
in Expr
out bool
}{{
in: ValTuple{newStrLiteral("aa")},
in: ValTuple{NewStrLiteral("aa")},
out: true,
}, {
in: ValTuple{&ColName{}},
Expand Down Expand Up @@ -350,37 +350,37 @@ func TestNewPlanValue(t *testing.T) {
}, {
in: &Literal{
Type: IntVal,
Val: []byte("10"),
Val: "10",
},
out: sqltypes.PlanValue{Value: sqltypes.NewInt64(10)},
}, {
in: &Literal{
Type: IntVal,
Val: []byte("1111111111111111111111111111111111111111"),
Val: "1111111111111111111111111111111111111111",
},
err: "value out of range",
}, {
in: &Literal{
Type: StrVal,
Val: []byte("strval"),
Val: "strval",
},
out: sqltypes.PlanValue{Value: sqltypes.NewVarBinary("strval")},
}, {
in: &Literal{
Type: BitVal,
Val: []byte("01100001"),
Val: "01100001",
},
err: "expression is too complex",
}, {
in: &Literal{
Type: HexVal,
Val: []byte("3131"),
Val: "3131",
},
out: sqltypes.PlanValue{Value: sqltypes.NewVarBinary("11")},
}, {
in: &Literal{
Type: HexVal,
Val: []byte("313"),
Val: "313",
},
err: "odd length hex string",
}, {
Expand All @@ -391,7 +391,7 @@ func TestNewPlanValue(t *testing.T) {
Argument(":valarg"),
&Literal{
Type: StrVal,
Val: []byte("strval"),
Val: "strval",
},
},
out: sqltypes.PlanValue{
Expand All @@ -412,15 +412,15 @@ func TestNewPlanValue(t *testing.T) {
}, {
in: &Literal{
Type: FloatVal,
Val: []byte("2.1"),
Val: "2.1",
},
out: sqltypes.PlanValue{Value: sqltypes.NewFloat64(2.1)},
}, {
in: &UnaryExpr{
Operator: Latin1Op,
Expr: &Literal{
Type: StrVal,
Val: []byte("strval"),
Val: "strval",
},
},
out: sqltypes.PlanValue{Value: sqltypes.NewVarBinary("strval")},
Expand All @@ -429,7 +429,7 @@ func TestNewPlanValue(t *testing.T) {
Operator: UBinaryOp,
Expr: &Literal{
Type: StrVal,
Val: []byte("strval"),
Val: "strval",
},
},
out: sqltypes.PlanValue{Value: sqltypes.NewVarBinary("strval")},
Expand All @@ -438,7 +438,7 @@ func TestNewPlanValue(t *testing.T) {
Operator: Utf8mb4Op,
Expr: &Literal{
Type: StrVal,
Val: []byte("strval"),
Val: "strval",
},
},
out: sqltypes.PlanValue{Value: sqltypes.NewVarBinary("strval")},
Expand All @@ -447,7 +447,7 @@ func TestNewPlanValue(t *testing.T) {
Operator: Utf8Op,
Expr: &Literal{
Type: StrVal,
Val: []byte("strval"),
Val: "strval",
},
},
out: sqltypes.PlanValue{Value: sqltypes.NewVarBinary("strval")},
Expand All @@ -456,7 +456,7 @@ func TestNewPlanValue(t *testing.T) {
Operator: UMinusOp,
Expr: &Literal{
Type: FloatVal,
Val: []byte("2.1"),
Val: "2.1",
},
},
err: "expression is too complex",
Expand All @@ -482,19 +482,3 @@ var mustMatch = utils.MustMatchFn(
},
[]string{".Conn"}, // ignored fields
)

func newStrLiteral(in string) *Literal {
return NewStrLiteral([]byte(in))
}

func newIntLiteral(in string) *Literal {
return NewIntLiteral([]byte(in))
}

func newHexLiteral(in string) *Literal {
return NewHexLiteral([]byte(in))
}

func newArgument(in string) Expr {
return NewArgument([]byte(in))
}
8 changes: 4 additions & 4 deletions go/vt/sqlparser/ast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1435,7 +1435,7 @@ type ShowFilter struct {
}

// Comments represents a list of comments.
type Comments [][]byte
type Comments []string

// SelectExprs represents SELECT expressions.
type SelectExprs []SelectExpr
Expand Down Expand Up @@ -1638,11 +1638,11 @@ type (
// Literal represents a fixed value.
Literal struct {
Type ValType
Val []byte
Val string
}

// Argument represents bindvariable expression
Argument []byte
Argument string

// NullVal represents a NULL value.
NullVal struct{}
Expand Down Expand Up @@ -2808,7 +2808,7 @@ func (node *ExistsExpr) Format(buf *TrackedBuffer) {
func (node *Literal) Format(buf *TrackedBuffer) {
switch node.Type {
case StrVal:
sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val).EncodeSQL(buf)
sqltypes.MakeTrusted(sqltypes.VarBinary, node.Bytes()).EncodeSQL(buf)
case IntVal, FloatVal, HexNum:
buf.astPrintf(node, "%s", node.Val)
case HexVal:
Expand Down
29 changes: 14 additions & 15 deletions go/vt/sqlparser/ast_funcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,48 +450,47 @@ func (node *ComparisonExpr) IsImpossible() bool {
}

// NewStrLiteral builds a new StrVal.
func NewStrLiteral(in []byte) *Literal {
func NewStrLiteral(in string) *Literal {
return &Literal{Type: StrVal, Val: in}
}

// NewIntLiteral builds a new IntVal.
func NewIntLiteral(in []byte) *Literal {
func NewIntLiteral(in string) *Literal {
return &Literal{Type: IntVal, Val: in}
}

// NewFloatLiteral builds a new FloatVal.
func NewFloatLiteral(in []byte) *Literal {
func NewFloatLiteral(in string) *Literal {
return &Literal{Type: FloatVal, Val: in}
}

// NewHexNumLiteral builds a new HexNum.
func NewHexNumLiteral(in []byte) *Literal {
func NewHexNumLiteral(in string) *Literal {
return &Literal{Type: HexNum, Val: in}
}

// NewHexLiteral builds a new HexVal.
func NewHexLiteral(in []byte) *Literal {
func NewHexLiteral(in string) *Literal {
return &Literal{Type: HexVal, Val: in}
}

// NewBitLiteral builds a new BitVal containing a bit literal.
func NewBitLiteral(in []byte) *Literal {
func NewBitLiteral(in string) *Literal {
return &Literal{Type: BitVal, Val: in}
}

// NewArgument builds a new ValArg.
func NewArgument(in []byte) Argument {
return in
func NewArgument(in string) Argument {
return Argument(in)
}

func (node *Literal) Bytes() []byte {
return []byte(node.Val)
}

// HexDecode decodes the hexval into bytes.
func (node *Literal) HexDecode() ([]byte, error) {
dst := make([]byte, hex.DecodedLen(len([]byte(node.Val))))
_, err := hex.Decode(dst, []byte(node.Val))
if err != nil {
return nil, err
}
return dst, err
return hex.DecodeString(node.Val)
}

// Equal returns true if the column names match.
Expand Down Expand Up @@ -1347,7 +1346,7 @@ func handleUnaryMinus(expr Expr) Expr {
num.Val = num.Val[1:]
return num
}
return NewIntLiteral(append([]byte("-"), num.Val...))
return NewIntLiteral("-" + num.Val)
}
if unaryExpr, ok := expr.(*UnaryExpr); ok && unaryExpr.Operator == UMinusOp {
return unaryExpr.Expr
Expand Down
2 changes: 1 addition & 1 deletion go/vt/sqlparser/ast_rewriting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ func (er *expressionRewriter) unnestSubQueries(cursor *Cursor, subquery *Subquer
}

func bindVarExpression(name string) Expr {
return NewArgument([]byte(":" + name))
return NewArgument(":" + name)
}

// SystemSchema returns true if the schema passed is system schema
Expand Down

0 comments on commit e921494

Please sign in to comment.