From 369255a4feecd68816129ceaa274fe53f54259b3 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 19 Jan 2022 14:10:11 +0100 Subject: [PATCH 1/3] Implementing lexer with runtime expression validation --- v2/go.mod | 2 +- v2/go.sum | 3 +- .../common/expressions/expressions.go | 121 +++++++++++++++--- .../common/expressions/expressions_test.go | 24 ++-- 4 files changed, 120 insertions(+), 30 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 5cefa50d15..735efcc609 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -38,7 +38,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 + github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 7f4eb90da4..67e4eda7ce 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -457,8 +457,9 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 h1:L7Vb5AdzIV1Xs088Nvslfhh/piKP9gjTxjxfiqnd4mk= +github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4= diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 742acc9168..b81a408c11 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -6,9 +6,9 @@ import ( "github.com/Knetic/govaluate" "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + "github.com/projectdiscovery/stringsutil" ) // Evaluate checks if the match contains a dynamic variable, for each @@ -33,13 +33,17 @@ func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) { } func evaluate(data string, base map[string]interface{}) (string, error) { - data = replacer.Replace(data, base) - + // expressions can be: + // - simple: containing base values keys (variables) + // - complex: containing helper functions [ + variables] + // literals like {{2+2}} are not considered expressions + expressions := findExpressions(data, mergeFunctions(dsl.HelperFunctions(), mapToFunctions(base))) dynamicValues := make(map[string]interface{}) - for _, match := range findMatches(data) { - expr := generators.TrimDelimiters(match) - - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) + for _, expression := range expressions { + // replace variable placeholders with base values + expression = replacer.Replace(expression, base) + // turns expressions (either helper functions+base values or base values) + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions()) if err != nil { continue } @@ -47,19 +51,104 @@ func evaluate(data string, base map[string]interface{}) (string, error) { if err != nil { continue } - dynamicValues[expr] = result + dynamicValues[expression] = result } - // Replacer dynamic values if any in raw request and parse it + // Replacer dynamic values if any in raw request and parse it return replacer.Replace(data, dynamicValues), nil } -func findMatches(data string) []string { - var matches []string - for _, token := range strings.Split(data, marker.ParenthesisOpen) { - closingToken := strings.LastIndex(token, marker.ParenthesisClose) - if closingToken > 0 { - matches = append(matches, token[:closingToken]) +// maxIterations to avoid infinite loop +const maxIterations = 250 + +func findExpressions(data string, functions map[string]govaluate.ExpressionFunction) []string { + var ( + iterations int + exps []string + ) + for { + // check if we reached the maximum number of iterations + if iterations > maxIterations { + break + } + iterations++ + // attempt to find open markers + indexOpenMarker := strings.Index(data, marker.ParenthesisOpen) + // exits if not found + if indexOpenMarker < 0 { + break + } + + indexOpenMarkerOffset := indexOpenMarker + len(marker.ParenthesisOpen) + + shouldSearchCloseMarker := true + closeMarkerFound := false + innerData := data + var potentialMatch string + var indexCloseMarker, indexCloseMarkerOffset int + skip := indexOpenMarkerOffset + for shouldSearchCloseMarker { + // attempt to find close marker + indexCloseMarker = stringsutil.IndexAt(innerData, marker.ParenthesisClose, skip) + // if no close markers are found exit + if indexCloseMarker < 0 { + shouldSearchCloseMarker = false + continue + } + indexCloseMarkerOffset = indexCloseMarker + len(marker.ParenthesisClose) + + potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker] + if isExpression(potentialMatch, functions) { + closeMarkerFound = true + shouldSearchCloseMarker = false + exps = append(exps, potentialMatch) + } else { + skip = indexCloseMarkerOffset + } + } + + if closeMarkerFound { + // move after the close marker + data = data[indexCloseMarkerOffset:] + } else { + // move after the open marker + data = data[indexOpenMarkerOffset:] + } + } + return exps +} + +func isExpression(data string, functions map[string]govaluate.ExpressionFunction) bool { + if _, err := govaluate.NewEvaluableExpression(data); err == nil { + return stringsutil.ContainsAny(data, getFunctionsNames(functions)...) + } + + // check if it's a complex expression + _, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions()) + return err == nil +} + +func mapToFunctions(vars map[string]interface{}) map[string]govaluate.ExpressionFunction { + f := make(map[string]govaluate.ExpressionFunction) + for k := range vars { + f[k] = nil + } + return f +} + +func mergeFunctions(m ...map[string]govaluate.ExpressionFunction) map[string]govaluate.ExpressionFunction { + o := make(map[string]govaluate.ExpressionFunction) + for _, mm := range m { + for k, v := range mm { + o[k] = v } } - return matches + return o +} + +func getFunctionsNames(m map[string]govaluate.ExpressionFunction) []string { + var keys []string + for k := range m { + keys = append(keys, k) + } + return keys } diff --git a/v2/pkg/protocols/common/expressions/expressions_test.go b/v2/pkg/protocols/common/expressions/expressions_test.go index 668b3b4559..a154eba775 100644 --- a/v2/pkg/protocols/common/expressions/expressions_test.go +++ b/v2/pkg/protocols/common/expressions/expressions_test.go @@ -14,19 +14,19 @@ func TestEvaluate(t *testing.T) { }{ {input: "{{url_encode('test}aaa')}}", expected: "test%7Daaa", extra: map[string]interface{}{}}, {input: "{{hex_encode('PING')}}", expected: "50494e47", extra: map[string]interface{}{}}, - // TODO #1501 - //{input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}}, - //{input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, - //{input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, - //{input: `{{"{{" + '123*123' + "}}"}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, + {input: "{{hex_encode('{{')}}", expected: "7b7b", extra: map[string]interface{}{}}, + {input: `{{concat("{{", 123, "*", 123, "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, + {input: `{{concat("{{", "123*123", "}}")}}`, expected: "{{123*123}}", extra: map[string]interface{}{}}, + {input: `{{"{{" + '123*123' + "}}"}}`, expected: `{{"{{" + '123*123' + "}}"}}`, extra: map[string]interface{}{}}, + {input: `{{a + '123*123' + b}}`, expected: `aa123*123bb`, extra: map[string]interface{}{"a": "aa", "b": "bb"}}, {input: `{{concat(123,'*',123)}}`, expected: "123*123", extra: map[string]interface{}{}}, - {input: `{{1+1}}`, expected: "2", extra: map[string]interface{}{}}, - {input: `{{"1"+"1"}}`, expected: "11", extra: map[string]interface{}{}}, - {input: `{{"1" + '*' + "1"}}`, expected: "1*1", extra: map[string]interface{}{}}, - {input: `{{"a" + 'b' + "c"}}`, expected: "abc", extra: map[string]interface{}{}}, - {input: `{{10*2}}`, expected: "20", extra: map[string]interface{}{}}, - {input: `{{10/2}}`, expected: "5", extra: map[string]interface{}{}}, - {input: `{{10-2}}`, expected: "8", extra: map[string]interface{}{}}, + {input: `{{1+1}}`, expected: "{{1+1}}", extra: map[string]interface{}{}}, + {input: `{{"1"+"1"}}`, expected: `{{"1"+"1"}}`, extra: map[string]interface{}{}}, + {input: `{{"1" + '*' + "1"}}`, expected: `{{"1" + '*' + "1"}}`, extra: map[string]interface{}{}}, + {input: `{{"a" + 'b' + "c"}}`, expected: `{{"a" + 'b' + "c"}}`, extra: map[string]interface{}{}}, + {input: `{{10*2}}`, expected: `{{10*2}}`, extra: map[string]interface{}{}}, + {input: `{{10/2}}`, expected: `{{10/2}}`, extra: map[string]interface{}{}}, + {input: `{{10-2}}`, expected: `{{10-2}}`, extra: map[string]interface{}{}}, {input: "test", expected: "test", extra: map[string]interface{}{}}, {input: "{{hex_encode(Item)}}", expected: "50494e47", extra: map[string]interface{}{"Item": "PING"}}, {input: "{{hex_encode(Item)}}\r\n", expected: "50494e47\r\n", extra: map[string]interface{}{"Item": "PING"}}, From adf4721833907ca9b67ecef63118ec988b547d27 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 19 Jan 2022 14:57:25 +0100 Subject: [PATCH 2/3] making markers parametric + simple match/replace for basic variables --- .../common/expressions/expressions.go | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index b81a408c11..63bc3ce22f 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -33,11 +33,14 @@ func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) { } func evaluate(data string, base map[string]interface{}) (string, error) { + // replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value + data = replacer.Replace(data, base) + // expressions can be: // - simple: containing base values keys (variables) // - complex: containing helper functions [ + variables] // literals like {{2+2}} are not considered expressions - expressions := findExpressions(data, mergeFunctions(dsl.HelperFunctions(), mapToFunctions(base))) + expressions := findExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, mergeFunctions(dsl.HelperFunctions(), mapToFunctions(base))) dynamicValues := make(map[string]interface{}) for _, expression := range expressions { // replace variable placeholders with base values @@ -60,7 +63,7 @@ func evaluate(data string, base map[string]interface{}) (string, error) { // maxIterations to avoid infinite loop const maxIterations = 250 -func findExpressions(data string, functions map[string]govaluate.ExpressionFunction) []string { +func findExpressions(data, OpenMarker, CloseMarker string, functions map[string]govaluate.ExpressionFunction) []string { var ( iterations int exps []string @@ -72,13 +75,13 @@ func findExpressions(data string, functions map[string]govaluate.ExpressionFunct } iterations++ // attempt to find open markers - indexOpenMarker := strings.Index(data, marker.ParenthesisOpen) + indexOpenMarker := strings.Index(data, OpenMarker) // exits if not found if indexOpenMarker < 0 { break } - indexOpenMarkerOffset := indexOpenMarker + len(marker.ParenthesisOpen) + indexOpenMarkerOffset := indexOpenMarker + len(OpenMarker) shouldSearchCloseMarker := true closeMarkerFound := false @@ -88,13 +91,13 @@ func findExpressions(data string, functions map[string]govaluate.ExpressionFunct skip := indexOpenMarkerOffset for shouldSearchCloseMarker { // attempt to find close marker - indexCloseMarker = stringsutil.IndexAt(innerData, marker.ParenthesisClose, skip) + indexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip) // if no close markers are found exit if indexCloseMarker < 0 { shouldSearchCloseMarker = false continue } - indexCloseMarkerOffset = indexCloseMarker + len(marker.ParenthesisClose) + indexCloseMarkerOffset = indexCloseMarker + len(CloseMarker) potentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker] if isExpression(potentialMatch, functions) { @@ -152,3 +155,11 @@ func getFunctionsNames(m map[string]govaluate.ExpressionFunction) []string { } return keys } + +func getMapKeysAsValues(m map[string]interface{}) map[string]interface{} { + mapOut := make(map[string]interface{}) + for k := range m { + mapOut[k] = k + } + return mapOut +} From 8559cfabca1fdcc48fe54f9f1822aac803c5c0e7 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 19 Jan 2022 15:16:47 +0100 Subject: [PATCH 3/3] removing unused code --- v2/pkg/protocols/common/expressions/expressions.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 63bc3ce22f..0eed8976c0 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -155,11 +155,3 @@ func getFunctionsNames(m map[string]govaluate.ExpressionFunction) []string { } return keys } - -func getMapKeysAsValues(m map[string]interface{}) map[string]interface{} { - mapOut := make(map[string]interface{}) - for k := range m { - mapOut[k] = k - } - return mapOut -}