identifiers can include '-'

osteele · Jul 7, 2017 · 606870e · 606870e
1 parent c67d027
commit 606870e
Show file tree

Hide file tree

Showing 4 changed files with 109 additions and 78 deletions.
diff --git a/expression/parser.go b/expression/parser.go
@@ -1,3 +1,6 @@
+//go:generate ragel -Z scanner.rl
+//go:generate goyacc expressions.y
+
 package expression
 
 import "fmt"

diff --git a/expression/scanner.go b/expression/scanner.go
@@ -18,13 +18,13 @@ var _expression_actions []byte = []byte{
 	2, 2, 11, 2, 2, 12, 
 }
 
-var _expression_key_offsets []byte = []byte{
+var _expression_key_offsets []int16 = []int16{
 	0, 1, 2, 3, 4, 5, 6, 7, 
 	8, 9, 10, 12, 38, 41, 42, 43, 
 	45, 46, 49, 51, 54, 62, 70, 71, 
-	72, 73, 79, 86, 93, 100, 107, 114, 
-	121, 128, 135, 142, 150, 157, 164, 171, 
-	178, 185, 192, 199, 206, 213, 
+	72, 73, 83, 84, 95, 106, 117, 128, 
+	139, 150, 161, 172, 183, 195, 206, 217, 
+	228, 239, 250, 261, 272, 283, 294, 
 }
 
 var _expression_trans_keys []byte = []byte{
@@ -37,52 +37,63 @@ var _expression_trans_keys []byte = []byte{
 	57, 48, 57, 46, 48, 57, 45, 95, 
 	48, 57, 65, 90, 97, 122, 45, 95, 
 	48, 57, 65, 90, 97, 122, 61, 61, 
-	61, 58, 95, 65, 90, 97, 122, 58, 
-	95, 110, 65, 90, 97, 122, 58, 95, 
-	100, 65, 90, 97, 122, 58, 95, 111, 
-	65, 90, 97, 122, 58, 95, 110, 65, 
-	90, 97, 122, 58, 95, 116, 65, 90, 
-	97, 122, 58, 95, 97, 65, 90, 98, 
-	122, 58, 95, 105, 65, 90, 97, 122, 
-	58, 95, 110, 65, 90, 97, 122, 58, 
-	95, 115, 65, 90, 97, 122, 58, 95, 
-	97, 111, 65, 90, 98, 122, 58, 95, 
-	108, 65, 90, 97, 122, 58, 95, 115, 
-	65, 90, 97, 122, 58, 95, 101, 65, 
-	90, 97, 122, 58, 95, 114, 65, 90, 
-	97, 122, 58, 95, 110, 65, 90, 97, 
-	122, 58, 95, 105, 65, 90, 97, 122, 
-	58, 95, 108, 65, 90, 97, 122, 58, 
-	95, 114, 65, 90, 97, 122, 58, 95, 
-	114, 65, 90, 97, 122, 58, 95, 117, 
-	65, 90, 97, 122, 
+	61, 45, 58, 63, 95, 48, 57, 65, 
+	90, 97, 122, 58, 45, 58, 63, 95, 
+	110, 48, 57, 65, 90, 97, 122, 45, 
+	58, 63, 95, 100, 48, 57, 65, 90, 
+	97, 122, 45, 58, 63, 95, 111, 48, 
+	57, 65, 90, 97, 122, 45, 58, 63, 
+	95, 110, 48, 57, 65, 90, 97, 122, 
+	45, 58, 63, 95, 116, 48, 57, 65, 
+	90, 97, 122, 45, 58, 63, 95, 97, 
+	48, 57, 65, 90, 98, 122, 45, 58, 
+	63, 95, 105, 48, 57, 65, 90, 97, 
+	122, 45, 58, 63, 95, 110, 48, 57, 
+	65, 90, 97, 122, 45, 58, 63, 95, 
+	115, 48, 57, 65, 90, 97, 122, 45, 
+	58, 63, 95, 97, 111, 48, 57, 65, 
+	90, 98, 122, 45, 58, 63, 95, 108, 
+	48, 57, 65, 90, 97, 122, 45, 58, 
+	63, 95, 115, 48, 57, 65, 90, 97, 
+	122, 45, 58, 63, 95, 101, 48, 57, 
+	65, 90, 97, 122, 45, 58, 63, 95, 
+	114, 48, 57, 65, 90, 97, 122, 45, 
+	58, 63, 95, 110, 48, 57, 65, 90, 
+	97, 122, 45, 58, 63, 95, 105, 48, 
+	57, 65, 90, 97, 122, 45, 58, 63, 
+	95, 108, 48, 57, 65, 90, 97, 122, 
+	45, 58, 63, 95, 114, 48, 57, 65, 
+	90, 97, 122, 45, 58, 63, 95, 114, 
+	48, 57, 65, 90, 97, 122, 45, 58, 
+	63, 95, 117, 48, 57, 65, 90, 97, 
+	122, 
 }
 
 var _expression_single_lengths []byte = []byte{
 	1, 1, 1, 1, 1, 1, 1, 1, 
 	1, 1, 0, 18, 1, 1, 1, 2, 
 	1, 1, 0, 1, 2, 2, 1, 1, 
-	1, 2, 3, 3, 3, 3, 3, 3, 
-	3, 3, 3, 4, 3, 3, 3, 3, 
-	3, 3, 3, 3, 3, 3, 
+	1, 4, 1, 5, 5, 5, 5, 5, 
+	5, 5, 5, 5, 6, 5, 5, 5, 
+	5, 5, 5, 5, 5, 5, 5, 
 }
 
 var _expression_range_lengths []byte = []byte{
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 1, 4, 1, 0, 0, 0, 
 	0, 1, 1, 1, 3, 3, 0, 0, 
-	0, 2, 2, 2, 2, 2, 2, 2, 
-	2, 2, 2, 2, 2, 2, 2, 2, 
-	2, 2, 2, 2, 2, 2, 
+	0, 3, 0, 3, 3, 3, 3, 3, 
+	3, 3, 3, 3, 3, 3, 3, 3, 
+	3, 3, 3, 3, 3, 3, 3, 
 }
 
-var _expression_index_offsets []byte = []byte{
+var _expression_index_offsets []int16 = []int16{
 	0, 2, 4, 6, 8, 10, 12, 14, 
 	16, 18, 20, 22, 45, 48, 50, 52, 
 	55, 57, 60, 62, 65, 71, 77, 79, 
-	81, 83, 88, 94, 100, 106, 112, 118, 
-	124, 130, 136, 142, 149, 155, 161, 167, 
-	173, 179, 185, 191, 197, 203, 
+	81, 83, 91, 93, 102, 111, 120, 129, 
+	138, 147, 156, 165, 174, 184, 193, 202, 
+	211, 220, 229, 238, 247, 256, 265, 
 }
 
 var _expression_indicies []byte = []byte{
@@ -96,35 +107,43 @@ var _expression_indicies []byte = []byte{
 	11, 38, 21, 34, 12, 39, 12, 21, 
 	40, 42, 42, 43, 42, 42, 41, 42, 
 	42, 42, 42, 42, 44, 45, 34, 46, 
-	34, 47, 34, 48, 25, 25, 25, 41, 
-	48, 25, 50, 25, 25, 49, 48, 25, 
-	51, 25, 25, 49, 48, 25, 52, 25, 
-	25, 49, 48, 25, 53, 25, 25, 49, 
-	48, 25, 54, 25, 25, 49, 48, 25, 
-	55, 25, 25, 49, 48, 25, 56, 25, 
-	25, 49, 48, 25, 57, 25, 25, 49, 
-	48, 25, 58, 25, 25, 49, 48, 25, 
-	59, 60, 25, 25, 49, 48, 25, 61, 
-	25, 25, 49, 48, 25, 62, 25, 25, 
-	49, 48, 25, 63, 25, 25, 49, 48, 
-	25, 64, 25, 25, 49, 48, 25, 65, 
-	25, 25, 49, 48, 25, 66, 25, 25, 
-	49, 48, 25, 67, 25, 25, 49, 48, 
-	25, 68, 25, 25, 49, 48, 25, 69, 
-	25, 25, 49, 48, 25, 62, 25, 25, 
-	49, 
+	34, 47, 34, 25, 48, 49, 25, 25, 
+	25, 25, 41, 48, 50, 25, 48, 49, 
+	25, 51, 25, 25, 25, 50, 25, 48, 
+	49, 25, 52, 25, 25, 25, 50, 25, 
+	48, 49, 25, 53, 25, 25, 25, 50, 
+	25, 48, 49, 25, 54, 25, 25, 25, 
+	50, 25, 48, 49, 25, 55, 25, 25, 
+	25, 50, 25, 48, 49, 25, 56, 25, 
+	25, 25, 50, 25, 48, 49, 25, 57, 
+	25, 25, 25, 50, 25, 48, 49, 25, 
+	58, 25, 25, 25, 50, 25, 48, 49, 
+	25, 59, 25, 25, 25, 50, 25, 48, 
+	49, 25, 60, 61, 25, 25, 25, 50, 
+	25, 48, 49, 25, 62, 25, 25, 25, 
+	50, 25, 48, 49, 25, 63, 25, 25, 
+	25, 50, 25, 48, 49, 25, 64, 25, 
+	25, 25, 50, 25, 48, 49, 25, 65, 
+	25, 25, 25, 50, 25, 48, 49, 25, 
+	66, 25, 25, 25, 50, 25, 48, 49, 
+	25, 67, 25, 25, 25, 50, 25, 48, 
+	49, 25, 68, 25, 25, 25, 50, 25, 
+	48, 49, 25, 69, 25, 25, 25, 50, 
+	25, 48, 49, 25, 70, 25, 25, 25, 
+	50, 25, 48, 49, 25, 63, 25, 25, 
+	25, 50, 
 }
 
 var _expression_trans_targs []byte = []byte{
 	11, 0, 11, 2, 3, 4, 5, 11, 
 	7, 8, 11, 9, 18, 11, 12, 13, 
 	14, 15, 16, 17, 20, 19, 22, 23, 
-	24, 25, 26, 28, 35, 40, 41, 43, 
-	44, 11, 11, 11, 1, 6, 10, 11, 
+	24, 25, 27, 29, 36, 41, 42, 44, 
+	45, 11, 11, 11, 1, 6, 10, 11, 
 	11, 11, 21, 20, 11, 11, 11, 11, 
-	11, 11, 27, 25, 29, 30, 31, 32, 
-	33, 34, 25, 36, 39, 37, 38, 25, 
-	25, 25, 42, 25, 25, 45, 
+	11, 26, 11, 28, 25, 30, 31, 32, 
+	33, 34, 35, 25, 37, 40, 38, 39, 
+	25, 25, 25, 43, 25, 25, 46, 
 }
 
 var _expression_trans_actions []byte = []byte{
@@ -134,9 +153,9 @@ var _expression_trans_actions []byte = []byte{
 	0, 65, 0, 0, 0, 0, 0, 0, 
 	0, 33, 35, 15, 0, 0, 0, 27, 
 	25, 39, 0, 41, 31, 19, 13, 17, 
-	21, 29, 0, 50, 0, 0, 0, 0, 
-	0, 0, 56, 0, 0, 0, 0, 44, 
-	59, 62, 0, 47, 53, 0, 
+	21, 0, 29, 0, 50, 0, 0, 0, 
+	0, 0, 0, 56, 0, 0, 0, 0, 
+	44, 59, 62, 0, 47, 53, 0, 
 }
 
 var _expression_to_state_actions []byte = []byte{
@@ -145,7 +164,7 @@ var _expression_to_state_actions []byte = []byte{
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 
 }
 
 var _expression_from_state_actions []byte = []byte{
@@ -154,16 +173,16 @@ var _expression_from_state_actions []byte = []byte{
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 
 }
 
-var _expression_eof_trans []byte = []byte{
+var _expression_eof_trans []int16 = []int16{
 	1, 1, 1, 1, 1, 1, 1, 1, 
 	1, 1, 1, 0, 34, 35, 35, 35, 
 	35, 35, 40, 41, 42, 45, 35, 35, 
-	35, 42, 50, 50, 50, 50, 50, 50, 
-	50, 50, 50, 50, 50, 50, 50, 50, 
-	50, 50, 50, 50, 50, 50, 
+	35, 42, 51, 51, 51, 51, 51, 51, 
+	51, 51, 51, 51, 51, 51, 51, 51, 
+	51, 51, 51, 51, 51, 51, 51, 
 }
 
 const expression_start int = 11
@@ -193,7 +212,7 @@ func newLexer(data []byte) *lexer {
 			pe: len(data),
 	}
 
-//line scanner.go:197
+//line scanner.go:216
 	{
 	 lex.cs = expression_start
 	 lex.ts = 0
@@ -210,7 +229,7 @@ func (lex *lexer) Lex(out *yySymType) int {
 	tok := 0
 
 
-//line scanner.go:214
+//line scanner.go:233
 	{
 	var _klen int
 	var _trans int
@@ -230,7 +249,7 @@ _resume:
 //line NONE:1
  lex.ts = ( lex.p)
 
-//line scanner.go:234
+//line scanner.go:253
 		}
 	}
 
@@ -505,7 +524,7 @@ _eof_trans:
  }
 	}
 
-//line scanner.go:509
+//line scanner.go:528
 		}
 	}
 
@@ -519,7 +538,7 @@ _again:
 //line NONE:1
  lex.ts = 0
 
-//line scanner.go:523
+//line scanner.go:542
 		}
 	}
 

diff --git a/expression/scanner.rl b/expression/scanner.rl
@@ -71,7 +71,7 @@ func (lex *lexer) Lex(out *yySymType) int {
 		}
 		action Relation { tok = RELATION; out.name = lex.token(); fbreak; }
 
-		identifier = (alpha | '_') + ;
+		identifier = (alpha | '_') . (alnum | '_' | '-')*  '?'? ;
 		# TODO what can a property name contain?
 		property = '.' (alnum | '_' | '-')+ ;
 		int = '-'? digit+ ;

diff --git a/expression/scanner_test.go b/expression/scanner_test.go
@@ -1,24 +1,21 @@
-//go:generate ragel -Z scanner.rl
-//go:generate goyacc expressions.y
-
 package expression
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
-// var lexerTests = []struct{}{
-// 	{"{{var}}", "value"},
-// 	{"{{x}}", "1"},
-// }
-
 type testSymbol struct {
 	tok int
 	typ yySymType
 }
 
+func (s testSymbol) String() string {
+	return fmt.Sprintf("%d:%v", s.tok, s.typ)
+}
+
 func scanExpression(data string) ([]testSymbol, error) {
 	var (
 		lex     = newLexer([]byte(data))
@@ -74,4 +71,16 @@ func TestLex(t *testing.T) {
 	require.Equal(t, 2.3, ts[4].typ.val)
 	require.Equal(t, "abc", ts[5].typ.val)
 	require.Equal(t, "abc", ts[6].typ.val)
+
+	// identifiers
+	ts, _ = scanExpression(`abc ab_c ab-c abc?`)
+	require.Len(t, ts, 4)
+	require.Equal(t, IDENTIFIER, ts[0].tok)
+	require.Equal(t, IDENTIFIER, ts[1].tok)
+	require.Equal(t, IDENTIFIER, ts[2].tok)
+	require.Equal(t, IDENTIFIER, ts[3].tok)
+	require.Equal(t, "abc", ts[0].typ.name)
+	require.Equal(t, "ab_c", ts[1].typ.name)
+	require.Equal(t, "ab-c", ts[2].typ.name)
+	require.Equal(t, "abc?", ts[3].typ.name)
 }