Merge pull request #1516 from projectdiscovery/issue-1497-lexer

Implementing lexer with runtime expression validation

v2/go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/projectdiscovery/rawhttp v0.0.7
 	github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660
 	github.com/projectdiscovery/retryablehttp-go v1.0.2
-	github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
+	github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700
 	github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0 // indirect

v2/go.sum

@@ -457,8 +457,9 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB
 github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
 github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
-github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
+github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 h1:L7Vb5AdzIV1Xs088Nvslfhh/piKP9gjTxjxfiqnd4mk=
+github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8=
 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4=

v2/pkg/protocols/common/expressions/expressions.go

@@ -6,9 +6,9 @@ import (
 	"github.com/Knetic/govaluate"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
+	"github.com/projectdiscovery/stringsutil"
 )
 
 // Evaluate checks if the match contains a dynamic variable, for each
@@ -33,33 +33,125 @@ func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
 }
 
 func evaluate(data string, base map[string]interface{}) (string, error) {
+	// replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value
 	data = replacer.Replace(data, base)
 
+	// expressions can be:
+	// - simple: containing base values keys (variables)
+	// - complex: containing helper functions [ + variables]
+	// literals like {{2+2}} are not considered expressions
+	expressions := findExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, mergeFunctions(dsl.HelperFunctions(), mapToFunctions(base)))
 	dynamicValues := make(map[string]interface{})
-	for _, match := range findMatches(data) {
-		expr := generators.TrimDelimiters(match)
-
-		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
+	for _, expression := range expressions {
+		// replace variable placeholders with base values
+		expression = replacer.Replace(expression, base)
+		// turns expressions (either helper functions+base values or base values)
+		compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions())
 		if err != nil {
 			continue
 		}
 		result, err := compiled.Evaluate(base)
 		if err != nil {
 			continue
 		}
-		dynamicValues[expr] = result
+		dynamicValues[expression] = result
 	}
-	// Replacer dynamic values if any in raw request and parse  it
+	// Replacer dynamic values if any in raw request and parse it
 	return replacer.Replace(data, dynamicValues), nil
 }
 
-func findMatches(data string) []string {
-	var matches []string
-	for _, token := range strings.Split(data, marker.ParenthesisOpen) {
-		closingToken := strings.LastIndex(token, marker.ParenthesisClose)
-		if closingToken > 0 {
-			matches = append(matches, token[:closingToken])
+// maxIterations to avoid infinite loop
+const maxIterations = 250
+
+func findExpressions(data, OpenMarker, CloseMarker string, functions map[string]govaluate.ExpressionFunction) []string {
+	var (
+		iterations int
+		exps       []string
+	)
+	for {
+		// check if we reached the maximum number of iterations
+		if iterations > maxIterations {
+			break
+		}
+		iterations++
+		// attempt to find open markers
+		indexOpenMarker := strings.Index(data, OpenMarker)
+		// exits if not found
+		if indexOpenMarker < 0 {
+			break
+		}
+
+		indexOpenMarkerOffset := indexOpenMarker + len(OpenMarker)
+
+		shouldSearchCloseMarker := true
+		closeMarkerFound := false
+		innerData := data
+		var potentialMatch string
+		var indexCloseMarker, indexCloseMarkerOffset int
+		skip := indexOpenMarkerOffset
+		for shouldSearchCloseMarker {
+			// attempt to find close marker
+			indexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip)
+			// if no close markers are found exit
+			if indexCloseMarker < 0 {
+				shouldSearchCloseMarker = false
+				continue
+			}
+			indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker)
+
+			potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]
+			if isExpression(potentialMatch, functions) {
+				closeMarkerFound = true
+				shouldSearchCloseMarker = false
+				exps = append(exps, potentialMatch)
+			} else {
+				skip = indexCloseMarkerOffset
+			}
+		}
+
+		if closeMarkerFound {
+			// move after the close marker
+			data = data[indexCloseMarkerOffset:]
+		} else {
+			// move after the open marker
+			data = data[indexOpenMarkerOffset:]
 		}
 	}
-	return matches
+	return exps
+}
+
+func isExpression(data string, functions map[string]govaluate.ExpressionFunction) bool {
+	if _, err := govaluate.NewEvaluableExpression(data); err == nil {
+		return stringsutil.ContainsAny(data, getFunctionsNames(functions)...)
+	}
+
+	// check if it's a complex expression
+	_, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions())
+	return err == nil
+}
+
+func mapToFunctions(vars map[string]interface{}) map[string]govaluate.ExpressionFunction {
+	f := make(map[string]govaluate.ExpressionFunction)
+	for k := range vars {
+		f[k] = nil
+	}
+	return f
+}
+
+func mergeFunctions(m ...map[string]govaluate.ExpressionFunction) map[string]govaluate.ExpressionFunction {
+	o := make(map[string]govaluate.ExpressionFunction)
+	for _, mm := range m {
+		for k, v := range mm {
+			o[k] = v
+		}
+	}
+	return o
+}
+
+func getFunctionsNames(m map[string]govaluate.ExpressionFunction) []string {
+	var keys []string
+	for k := range m {
+		keys = append(keys, k)
+	}
+	return keys
 }

v2/pkg/protocols/common/expressions/expressions_test.go

@@ -14,19 +14,19 @@ func TestEvaluate(t *testing.T) {
 	}{
 		{input: "{{url_encode('test}aaa')}}", expected: "test%7Daaa", extra: map[string]interface{}{}},
 		{input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}},
-		// TODO #1501
-		//{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}},
-		//{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
-		//{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
-		//{input: `{{"{{" + '123*123' + "}}"}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+		{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}},
+		{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+		{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}},
+		{input: `{{"{{" + '123*123' + "}}"}}`, expected: `{{"{{" + '123*123' + "}}"}}`, extra: map[string]interface{}{}},
+		{input: `{{a + '123*123' + b}}`, expected: `aa123*123bb`, extra: map[string]interface{}{"a": "aa", "b": "bb"}},
 		{input: `{{concat(123,'*',123)}}`, expected: "123*123", extra: map[string]interface{}{}},
-		{input: `{{1+1}}`, expected: "2", extra: map[string]interface{}{}},
-		{input: `{{"1"+"1"}}`, expected: "11", extra: map[string]interface{}{}},
-		{input: `{{"1" + '*' + "1"}}`, expected: "1*1", extra: map[string]interface{}{}},
-		{input: `{{"a" + 'b' + "c"}}`, expected: "abc", extra: map[string]interface{}{}},
-		{input: `{{10*2}}`, expected: "20", extra: map[string]interface{}{}},
-		{input: `{{10/2}}`, expected: "5", extra: map[string]interface{}{}},
-		{input: `{{10-2}}`, expected: "8", extra: map[string]interface{}{}},
+		{input: `{{1+1}}`, expected: "{{1+1}}", extra: map[string]interface{}{}},
+		{input: `{{"1"+"1"}}`, expected: `{{"1"+"1"}}`, extra: map[string]interface{}{}},
+		{input: `{{"1" + '*' + "1"}}`, expected: `{{"1" + '*' + "1"}}`, extra: map[string]interface{}{}},
+		{input: `{{"a" + 'b' + "c"}}`, expected: `{{"a" + 'b' + "c"}}`, extra: map[string]interface{}{}},
+		{input: `{{10*2}}`, expected: `{{10*2}}`, extra: map[string]interface{}{}},
+		{input: `{{10/2}}`, expected: `{{10/2}}`, extra: map[string]interface{}{}},
+		{input: `{{10-2}}`, expected: `{{10-2}}`, extra: map[string]interface{}{}},
 		{input: "test", expected: "test", extra: map[string]interface{}{}},
 		{input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}},
 		{input: "{{hex_encode(Item)}}\r\n", expected: "50494e47\r\n", extra: map[string]interface{}{"Item": "PING"}},