Skip to content

Commit

Permalink
feat(JSON): operators possible in JSON strings
Browse files Browse the repository at this point in the history 
Signed-off-by: Maxime Soulé <btik-git@scoubidou.com>
  • Loading branch information
@maxatome
maxatome committed Oct 15, 2022
1 parent a56657c commit a175c34
Show file tree
Hide file tree
Showing 6 changed files with 600 additions and 271 deletions.
201 changes: 160 additions & 41 deletions internal/json/lex.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2020, 2021, Maxime Soulé
// Copyright (c) 2020-2022, Maxime Soulé
// All rights reserved.
//
// This source code is licensed under the BSD-style license found in the
Expand Down Expand Up @@ -57,7 +57,6 @@ type json struct {
type ParseOpts struct {
Placeholders []any
PlaceholdersByName map[string]any
OpShortcutFn func(string, Position) (any, bool)
OpFn func(Operator, Position) (any, error)
}

Expand All @@ -71,24 +70,27 @@ func Parse(buf []byte, opts ...ParseOpts) (any, error) {
if len(opts) > 0 {
j.opts = opts[0]
}
yyParse(&j)

if len(j.errs) > 0 {
if !j.parse() {
if len(j.errs) == 1 {
return nil, j.errs[0]
}

errStr := bytes.NewBufferString(j.errs[0].Error())
for _, err := range j.errs[1:] {
errStr.WriteByte('\n')
errStr.WriteString(err.Error())
}
return nil, errors.New(errStr.String())
}

return j.value, nil
}

// parse returns true if no errors occurred during parsing.
func (j *json) parse() bool {
yyParse(j)
return len(j.errs) == 0
}

// Lex implements yyLexer interface.
func (j *json) Lex(lval *yySymType) int {
return j.nextToken(lval)
Expand All @@ -114,6 +116,19 @@ func (j *json) Error(s string) {
}
}

func (j *json) newOperator(name string, params []any) any {
if name == "" {
return nil // an operator error is in progress
}
opPos := j.popPos()
op, err := j.getOperator(Operator{Name: name, Params: params}, opPos)
if err != nil {
j.fatal(err.Error(), opPos)
return nil
}
return op
}

func (j *json) pushPos(pos Position) {
j.stackPos = append(j.stackPos, pos)
}
Expand All @@ -130,13 +145,6 @@ func (j *json) moveHoriz(bytes int, runes ...int) {
j.curSize = 0
}

func (j *json) getOperatorShortcut(operator string, opPos Position) (any, bool) {
if j.opts.OpShortcutFn == nil {
return nil, false
}
return j.opts.OpShortcutFn(operator, opPos)
}

func (j *json) getOperator(operator Operator, opPos Position) (any, error) {
if j.opts.OpFn == nil {
return nil, fmt.Errorf("unknown operator %q", operator.Name)
Expand All @@ -160,23 +168,20 @@ func (j *json) nextToken(lval *yySymType) int {
if !ok {
return 0
}
return j.analyzeStringContent(s, firstPos, lval)

// Check for placeholder ($1 or $name) or operator shortcut ($^Nil)
if len(s) <= 1 || !strings.HasPrefix(s, "$") {
lval.string = s
return STRING
}
// Double $$ at start of strings escape a $
if strings.HasPrefix(s[1:], "$") {
lval.string = s[1:]
return STRING
case 'r': // raw string, aka r!str! or r<str> (ws possible bw r & start delim)
if !j.skipWs() {
j.fatal("cannot find r start delimiter")
return 0
}

token, value := j.parseDollarToken(s[1:], firstPos)
if token != 0 {
lval.value = value
firstPos := j.pos.incHoriz(1)
s, ok := j.parseRawString()
if !ok {
return 0
}
return token
return j.analyzeStringContent(s, firstPos, lval)

case 'n': // null
if j.remain() >= 4 && bytes.Equal(j.buf[j.pos.bpos+1:j.pos.bpos+4], []byte(`ull`)) {
Expand Down Expand Up @@ -210,7 +215,7 @@ func (j *json) nextToken(lval *yySymType) int {

case '$':
var dollarToken string
end := bytes.IndexAny(j.buf[j.pos.bpos+1:], " \t\r\n,}])")
end := bytes.IndexAny(j.buf[j.pos.bpos+1:], " \t\r\n,}]()")
if end >= 0 {
dollarToken = string(j.buf[j.pos.bpos+1 : j.pos.bpos+1+end])
} else {
Expand All @@ -221,8 +226,14 @@ func (j *json) nextToken(lval *yySymType) int {
return '$'
}

token, value := j.parseDollarToken(dollarToken, j.pos)
if token != 0 {
token, value := j.parseDollarToken(dollarToken, j.pos, false)
switch token {
case 0:
return yyErrCode
case OPERATOR:
lval.string = value.(string)
return token
default:
lval.value = value
}
j.moveHoriz(1+len(dollarToken), 1+utf8.RuneCountInString(dollarToken))
Expand Down Expand Up @@ -340,6 +351,88 @@ str:
return "", false
}

func (j *json) parseRawString() (string, bool) {
// j.buf[j.pos.bpos] == first non-ws rune after 'r' → caller responsibility

savePos := j.pos
startDelim, _ := j.getRune() // cannot fail, caller called j.skipWs()

var endDelim rune
switch startDelim {
case '(':
endDelim = ')'
case '{':
endDelim = '}'
case '[':
endDelim = ']'
case '<':
endDelim = '>'
default:
if startDelim == '_' ||
(!unicode.IsPunct(startDelim) && !unicode.IsSymbol(startDelim)) {
j.fatal(fmt.Sprintf("invalid r delimiter %q, should be either a punctuation or a symbol rune, excluding '_'",
startDelim))
return "", false
}
endDelim = startDelim
}

from := j.pos.bpos + j.curSize

for innerDelim := 0; ; {
r, ok := j.getRune()
if !ok {
break
}

switch r {
case startDelim:
if startDelim == endDelim {
return string(j.buf[from:j.pos.bpos]), true
}
innerDelim++

case endDelim:
if innerDelim == 0 {
return string(j.buf[from:j.pos.bpos]), true
}
innerDelim--

case '\n', '\r', '\t': // accept these raw bytes
default:
if r < ' ' || r > utf8.MaxRune {
j.fatal("invalid character in raw string")
return "", false
}
}
}

j.fatal("unterminated raw string", savePos)
return "", false
}

// analyzeStringContent checks whether s contains $ prefix or not. If
// yes, it tries to parse it.
func (j *json) analyzeStringContent(s string, strPos Position, lval *yySymType) int {
if len(s) <= 1 || !strings.HasPrefix(s, "$") {
lval.string = s
return STRING
}
// Double $$ at start of strings escape a $
if strings.HasPrefix(s[1:], "$") {
lval.string = s[1:]
return STRING
}

// Check for placeholder ($1 or $name) or operator call as $^Empty
// or $^Re(q<\d+>)
token, value := j.parseDollarToken(s[1:], strPos, true)
if token != 0 {
lval.value = value
}
return token
}

const (
numInt = 1 << iota
numFloat
Expand Down Expand Up @@ -416,10 +509,10 @@ func (j *json) parseNumber() (float64, bool) {
return f, true
}

// parseDollarToken parses a $123 or $tag or $^Shortcut token.
// dollarToken is never empty, does not contain '$' and dollarPos
// is the '$' position.
func (j *json) parseDollarToken(dollarToken string, dollarPos Position) (int, any) {
// parseDollarToken parses a $123 or $tag or $^Shortcut or, if
// inString, $^Operator(PARAMS…) token. dollarToken is never empty,
// does not contain '$' and dollarPos is the '$' position.
func (j *json) parseDollarToken(dollarToken string, dollarPos Position, inString bool) (int, any) {
firstRune, _ := utf8.DecodeRuneInString(dollarToken)

// Test for $123
Expand Down Expand Up @@ -456,16 +549,42 @@ func (j *json) parseDollarToken(dollarToken string, dollarPos Position) (int, an
return PLACEHOLDER, j.opts.Placeholders[np-1]
}

// Test for operator shortcut
// Test for operator call $^Operator or $^Operator(…)
if firstRune == '^' {
op, ok := j.getOperatorShortcut(dollarToken[1:], dollarPos)
nextRune, _ := utf8.DecodeRuneInString(dollarToken[1:])
if nextRune < 'A' || nextRune > 'Z' {
j.error(`$^ must be followed by an operator name`, dollarPos)
if inString {
return SUB_PARSER, nil // continue parsing
}
return OPERATOR, "" // continue parsing
}

if inString {
jr := json{
buf: []byte(dollarToken[1:]),
pos: Position{
Pos: dollarPos.Pos + 2,
Line: dollarPos.Line,
Col: dollarPos.Col + 2,
},
opts: j.opts,
}
if !jr.parse() {
j.errs = append(j.errs, jr.errs...)
return SUB_PARSER, nil // continue parsing
}
return SUB_PARSER, jr.value
}

j.moveHoriz(2)
j.lastTokenPos = j.pos
operator, ok := j.parseOperator()
if !ok {
j.error(
fmt.Sprintf(`bad operator shortcut "$%s"`, dollarToken),
dollarPos)
// continue parsing
return OPERATOR, "" // continue parsing
}
return OPERATOR_SHORTCUT, op
j.pushPos(j.lastTokenPos)
return OPERATOR, operator
}

// Test for $tag
Expand Down Expand Up @@ -494,7 +613,7 @@ func (j *json) parseOperator() (string, bool) {
operator:
for ; i < l; i++ {
switch r := j.buf[i]; r {
case ' ', '\t', '\r', '\n', ',', '}', ']', '(':
case ' ', '\t', '\r', '\n', ',', '}', ']', '(', ')':
break operator

default:
Expand Down

0 comments on commit a175c34

Please sign in to comment.