Range expressions (#84)

Add array operators and expression support

cmd/expressions.go

@@ -8,6 +8,8 @@ import (
 	"rare/pkg/expressions/exprofiler"
 	"rare/pkg/expressions/stdlib"
 	"rare/pkg/humanize"
+	"rare/pkg/minijson"
+	"strconv"
 	"strings"
 	"time"
 
@@ -19,7 +21,7 @@ func expressionFunction(c *cli.Context) error {
 		expString  = c.Args().First()
 		noOptimize = c.Bool("no-optimize")
 		data       = c.StringSlice("data")
-		keys       = c.StringSlice("key")
+		keyPairs   = c.StringSlice("key")
 		benchmark  = c.Bool("benchmark")
 		stats      = c.Bool("stats")
 	)
@@ -36,13 +38,26 @@ func expressionFunction(c *cli.Context) error {
 	compiled, err := builder.Compile(expString)
 	expCtx := expressions.KeyBuilderContextArray{
 		Elements: data,
-		Keys:     parseKeyValuesIntoMap(keys...),
+		Keys:     parseKeyValuesIntoMap(keyPairs...),
 	}
 
 	if err != nil {
 		return err
 	}
 
+	// Emulate special keys
+	{
+		keys := parseKeyValuesIntoMap(keyPairs...)
+		expCtx.Keys["src"] = "<args>"
+		expCtx.Keys["line"] = "0"
+		expCtx.Keys["."] = buildSpecialKeyJson(nil, keys)
+		expCtx.Keys["#"] = buildSpecialKeyJson(data, nil)
+		expCtx.Keys[".#"] = buildSpecialKeyJson(data, keys)
+		expCtx.Keys["#."] = expCtx.Keys[".#"]
+		expCtx.Keys["@"] = expressions.MakeArray(data...)
+	}
+
+	// Output results
 	fmt.Printf("Expression: %s\n", color.Wrap(color.BrightWhite, expString))
 	result := compiled.BuildKey(&expCtx)
 	fmt.Printf("Result:     %s\n", color.Wrap(color.BrightYellow, result))
@@ -88,6 +103,19 @@ func parseKeyValue(s string) (string, string) {
 	return s[:idx], s[idx+1:]
 }
 
+func buildSpecialKeyJson(matches []string, values map[string]string) string {
+	var json minijson.JsonObjectBuilder
+	json.Open()
+	for i, val := range matches {
+		json.WriteString(strconv.Itoa(i), val)
+	}
+	for k, v := range values {
+		json.WriteString(k, v)
+	}
+	json.Close()
+	return json.String()
+}
+
 func expressionCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "expression",

docs/usage/expressions.md

@@ -31,6 +31,7 @@ The following are special Keys:
  * `{.}`    Returns all matched values with match names as JSON
  * `{#}`    Returns all matched numbered values as JSON
  * `{.#}`   Returned numbered and named matches as JSON
+ * `{@}`    All extracted matches in array form
 
 ### Testing
 
@@ -136,9 +137,9 @@ filter with `--ignore`
 
 ### Logic
 
-#### If
+#### If, Unless
 
-Syntax: `{if val ifTrue ifFalse}` or `{if val ifTrue}`
+Syntax: `{if val ifTrue ifFalse}`, `{if val ifTrue}`, `{unless val ifFalse}`
 
 If `val` is truthy, then return `ifTrue` else optionally return `ifFalse`
 
@@ -242,6 +243,80 @@ to form arrays that have meaning for a given aggregator.
 
 Specifying multiple expressions is equivalent, eg. `{$ a b}` is the same as `-e a -e b`
 
+### Ranges (Arrays)
+
+Range functions provide the ability to work with arrays in expressions. You
+can create an array either manually with the `{@ ...}` function or
+by `{@split ...}` a string into an array.
+
+#### Array Definition
+
+Syntax: `{@ ele0 ele1 ele2}` (`{$ ele0 ele1 ele2}` is equivalent)
+
+Creates an array with the provided elements. Use `{@}` for an array of all matches.
+
+#### @split
+
+Syntax: `{@split <arr> ["delim"]}`
+
+Splits a string into an array with the separating `delim`.  If `delim` isn't
+specified, `" "` will be used.
+
+#### @join
+
+Syntax: `{@join <arr> ["delim"]}`
+
+Re-joins an array back into a string.  If `delim` is empty, it will be `" "`
+
+#### @map
+
+Syntax: `{@map <arr> <mapfunc>}`
+
+Evaluates `mapfunc` against each element in the array. In `mapfunc`, `{0}`
+is the current element.  The function must be surrounded by quotes.
+
+For example, given the array `[1,2,3]`, and the function
+`{@map {array} "{multi {0} 2}"}` will output [2,4,6].
+
+#### @reduce
+
+Syntax: `{@reduce <arr> <reducefunc>}`
+
+Evaluates `reducefunc` against each element and a memo. `{0}` is the memo, and
+`{1}` is the current value.
+
+For example, given the array `[1,2,3]`, and the function
+`{@reduce {array} "{sumi {0} {1}}"}`, it will return `6`.
+
+#### @filter
+
+Syntax: `{@filter <arr> <filterfunc>}`
+
+Evaluates `filterfunc` for each element.  If *truthy*, item will be in resulting
+array. If false, it will be omitted. `{0}` will be the value examined.
+
+For example, given the array `[1,abc,23,efg]`, and the function
+`{@filter {array} "{isnum {0}}"}` will return `[1,23]`.
+
+#### @select
+
+Syntax: `{@select <arr> "index"}`
+
+Selects a single item at an `index` out of `array`.
+
+#### @slice
+
+Syntax: `{@slice <arr> "begin" ["length"]}`
+
+Gets a slice of an array. If `begin` is a negative number, will start from the end.
+
+Examples: (Array `[1,2,3,4]`)
+
+- `{@slice {array} 1}` - [2,3,4]
+- `{@slice {array} 1 1}` - [2]
+- `{@slice {array} -2}` - [3,4]
+- `{@slice {array} -2 1}` - [3]
+
 
 ### Drawing
 
@@ -351,5 +426,6 @@ const (
 	ErrorConst    = "<CONST>"       // Expected constant value
 	ErrorEnum     = "<ENUM>"        // A given value is not contained within a set
 	ErrorArgName  = "<NAME>"        // A variable accessed by a given name does not exist
+	ErrorEmpty    = "<EMPTY>"       // A value was expected, but was empty
 )
 ```

pkg/expressions/keyBuilder.go

@@ -67,7 +67,9 @@ func (s *KeyBuilder) Compile(template string) (*CompiledKeyBuilder, error) {
 			inStatement--
 			if inStatement == 0 {
 				args := splitTokenizedArguments(sb.String())
-				if len(args) == 1 { // Simple variable keyword like "{1}"
+				if len(args) == 0 {
+					return nil, errors.New("empty statement in expression")
+				} else if len(args) == 1 { // Simple variable keyword like "{1}"
 					kb.stages = append(kb.stages, stageSimpleVariable(args[0]))
 				} else { // Complex function like "{add 1 2}"
 					f := s.functions[args[0]]

pkg/expressions/keyBuilder_test.go

@@ -65,6 +65,12 @@ func TestStringKey(t *testing.T) {
 	assert.Equal(t, "testval  key", key)
 }
 
+func TestEmptyStatement(t *testing.T) {
+	kb, err := NewKeyBuilder().Compile("{} test")
+	assert.Nil(t, kb)
+	assert.Error(t, err)
+}
+
 func BenchmarkSimpleReplacement(b *testing.B) {
 	kb, _ := NewKeyBuilder().Compile("{0} is awesome")
 	for n := 0; n < b.N; n++ {

pkg/expressions/stage.go

@@ -3,6 +3,7 @@ package expressions
 import (
 	"fmt"
 	"strconv"
+	"strings"
 )
 
 const (
@@ -40,3 +41,15 @@ func stageError(msg string) KeyBuilderStage {
 		return errMessage
 	})
 }
+
+// make a delim-separated array
+func MakeArray(args ...string) string {
+	var sb strings.Builder
+	for i := 0; i < len(args); i++ {
+		if i > 0 {
+			sb.WriteRune(ArraySeparator)
+		}
+		sb.WriteString(args[i])
+	}
+	return sb.String()
+}

pkg/expressions/stageAnalysis.go

@@ -1,5 +1,7 @@
 package expressions
 
+import "strconv"
+
 // monitorContext allows monitoring of context use
 //   largely for static analysis of an expression
 type monitorContext struct {
@@ -22,7 +24,6 @@ func EvalStaticStage(stage KeyBuilderStage) (ret string, ok bool) {
 	ok = (monitor.keyLookups == 0)
 	return
 }
-
 func EvalStageOrDefault(stage KeyBuilderStage, dflt string) string {
 	if val, ok := EvalStaticStage(stage); ok {
 		return val
@@ -36,3 +37,12 @@ func EvalStageIndexOrDefault(stages []KeyBuilderStage, idx int, dflt string) str
 	}
 	return dflt
 }
+
+func EvalStageInt(stage KeyBuilderStage, dflt int) int {
+	if s, ok := EvalStaticStage(stage); ok {
+		if v, err := strconv.Atoi(s); err == nil {
+			return v
+		}
+	}
+	return dflt
+}

pkg/expressions/stage_test.go

@@ -0,0 +1,12 @@
+package expressions
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMakeArray(t *testing.T) {
+	assert.Equal(t, "abc", MakeArray("abc"))
+	assert.Equal(t, "abc\x00def", MakeArray("abc", "def"))
+}

pkg/expressions/stdlib/errors.go

@@ -7,4 +7,5 @@ const (
 	ErrorConst    = "<CONST>"       // Expected constant value
 	ErrorEnum     = "<ENUM>"        // A given value is not contained within a set
 	ErrorArgName  = "<NAME>"        // A variable accessed by a given name does not exist
+	ErrorEmpty    = "<EMPTY>"       // A value was expected, but was empty
 )

pkg/expressions/stdlib/funcs.go

@@ -26,7 +26,8 @@ var StandardFunctions = map[string]KeyBuilderFunction{
 	"divf":  arithmaticHelperf(func(a, b float64) float64 { return a / b }),
 
 	// Comparisons
-	"if": KeyBuilderFunction(kfIf),
+	"if":     KeyBuilderFunction(kfIf),
+	"unless": KeyBuilderFunction(kfUnless),
 	"eq": stringComparator(func(a, b string) string {
 		if a == b {
 			return a
@@ -61,6 +62,16 @@ var StandardFunctions = map[string]KeyBuilderFunction{
 	"tab": kfJoin('\t'),
 	"$":   kfJoin(ArraySeparator),
 
+	// Ranges
+	"@":       kfJoin(ArraySeparator),
+	"@map":    kfArrayMap,
+	"@split":  kfArraySplit,
+	"@select": kfArraySelect,
+	"@join":   kfArrayJoin,
+	"@reduce": kfArrayReduce,
+	"@filter": kfArrayFilter,
+	"@slice":  kfArraySlice,
+
 	// Pathing
 	"basename": kfPathBase,
 	"dirname":  kfPathDir,

pkg/expressions/stdlib/funcsComparators.go

@@ -114,3 +114,16 @@ func kfIf(args []KeyBuilderStage) KeyBuilderStage {
 		return FalsyVal
 	})
 }
+
+func kfUnless(args []KeyBuilderStage) KeyBuilderStage {
+	if len(args) != 2 {
+		return stageLiteral(ErrorArgCount)
+	}
+	return func(context KeyBuilderContext) string {
+		ifVal := args[0](context)
+		if !Truthy(ifVal) {
+			return args[1](context)
+		}
+		return ""
+	}
+}

pkg/expressions/stdlib/funcsStrings.go

@@ -123,7 +123,7 @@ func selectField(s string, idx int) string {
 	quoted := false
 
 	for i, c := range s {
-		if (quoted && c == '"') || (!quoted && (c == ' ' || c == '\t' || c == '\n' || c == '\x00')) {
+		if (quoted && c == '"') || (!quoted && (c == ' ' || c == '\t' || c == '\n' || c == ArraySeparator)) {
 			if currIdx == idx {
 				return s[wordStart:i]
 			}

pkg/expressions/stdlib/funcs_test.go

@@ -85,6 +85,10 @@ func TestIfStatement(t *testing.T) {
 	testExpression(t, mockContext("abc efg"), `{if {eq {0} "abc efg"} beq}`, "beq")
 }
 
+func TestUnlessStatement(t *testing.T) {
+	testExpression(t, mockContext("abc"), `{unless {1} {0}} {unless abc efg} {unless "" bob} {unless joe}`, "abc  bob <ARGN>")
+}
+
 func TestComparisonEquality(t *testing.T) {
 	testExpression(t, mockContext("123", "1234"),
 		"{eq {0} 123} {eq {0} 1234} {not {eq {0} abc}} {neq 1 2} {neq 1 1}",