diff --git a/integration_tests/http/dsl-matcher-variable.yaml b/integration_tests/http/dsl-matcher-variable.yaml new file mode 100644 index 0000000000..ecbe5f9e5e --- /dev/null +++ b/integration_tests/http/dsl-matcher-variable.yaml @@ -0,0 +1,23 @@ +id: dsl-matcher-variable + +info: + name: dsl-matcher-variable + author: pd-team + severity: info + +requests: + - + path: + - "{{BaseURL}}" + payloads: + VALUES: + - This + - is + - test + - matcher + - text + matchers: + - + dsl: + - 'contains(body,"{{VALUES}}")' + type: dsl \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 1890048d24..ebf245317f 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -36,6 +36,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{}, + "http/dsl-matcher-variable.yaml": &httpDSLVariable{}, } type httpInteractshRequest struct{} @@ -155,6 +156,27 @@ func (h *httpGet) Execute(filePath string) error { return nil } +type httpDSLVariable struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpDSLVariable) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 5 { + return errIncorrectResultsCount(results) + } + return nil +} + type httpPostBody struct{} // Execute executes a test case and returns an error if occurred diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index cda09a7a86..22fb9e108a 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -3,6 +3,9 @@ package matchers import ( "strings" + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) @@ -39,7 +42,7 @@ func (matcher *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (matcher *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { +func (matcher *Matcher) MatchWords(corpus string, data map[string]interface{}) (bool, []string) { if matcher.CaseInsensitive { corpus = strings.ToLower(corpus) } @@ -47,12 +50,12 @@ func (matcher *Matcher) MatchWords(corpus string, dynamicValues map[string]inter var matchedWords []string // Iterate over all the words accepted as valid for i, word := range matcher.Words { - if dynamicValues == nil { - dynamicValues = make(map[string]interface{}) + if data == nil { + data = make(map[string]interface{}) } var err error - word, err = expressions.Evaluate(word, dynamicValues) + word, err = expressions.Evaluate(word, data) if err != nil { continue } @@ -148,6 +151,18 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) { func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { // Iterate over all the expressions accepted as valid for i, expression := range matcher.dslCompiled { + if varErr := expressions.ContainsUnresolvedVariables(expression.String()); varErr != nil { + resolvedExpression, err := expressions.Evaluate(expression.String(), data) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcher.Name, err.Error()) + return false + } + expression, err = govaluate.NewEvaluableExpressionWithFunctions(resolvedExpression, dsl.HelperFunctions()) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcher.Name, err.Error()) + return false + } + } result, err := expression.Evaluate(data) if err != nil { continue diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index b52adbc6db..68a6d1b019 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -3,6 +3,8 @@ package matchers import ( "testing" + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/stretchr/testify/require" ) @@ -71,3 +73,19 @@ func TestHexEncoding(t *testing.T) { require.True(t, isMatched, "Could not match valid Hex condition") require.Equal(t, m.Words, matched) } + +func TestMatcher_MatchDSL(t *testing.T) { + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("contains(body, \"{{VARIABLE}}\")", dsl.HelperFunctions()) + require.Nil(t, err, "couldn't compile expression") + + m := &Matcher{Type: MatcherTypeHolder{MatcherType: DSLMatcher}, dslCompiled: []*govaluate.EvaluableExpression{compiled}} + err = m.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + values := []string{"PING", "pong"} + + for value := range values { + isMatched := m.MatchDSL(map[string]interface{}{"body": value, "VARIABLE": value}) + require.True(t, isMatched) + } +} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index d303b22cbe..72b7d1589c 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -123,7 +123,6 @@ type Request struct { generator *generators.PayloadGenerator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client - dynamicValues map[string]interface{} // description: | // SelfContained specifies if the request is self-contained. diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 1f319ccff0..70fd4d4c7d 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -32,7 +32,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, request.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, data)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index e2c3f831c1..3bf0e247cc 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -249,8 +249,6 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou if reqURL == "" { reqURL = generatedHttpRequest.URL() } - request.dynamicValues = generatedHttpRequest.dynamicValues - // Check if hosts keep erroring if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { return true, nil @@ -520,7 +518,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } } - event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) if hasInteractMarkers { diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go index 5535e84e1d..6cd10d1087 100644 --- a/v2/pkg/protocols/http/request_test.go +++ b/v2/pkg/protocols/http/request_test.go @@ -48,7 +48,7 @@ func TestHTTPExtractMultipleReuse(t *testing.T) { Extractors: []*extractors.Extractor{{ Part: "body", Name: "endpoint", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"(?m)/([a-zA-Z0-9-_/\\\\]+)"}, Internal: true, }}, diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 64bdd6edac..9acd5cd7ea 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -72,9 +72,8 @@ type Request struct { generator *generators.PayloadGenerator // cache any variables that may be needed for operation. - dialer *fastdialer.Dialer - options *protocols.ExecuterOptions - dynamicValues map[string]interface{} + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions } // RequestPartDefinitions contains a mapping of request part definitions and their diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 010b11d61b..d3083301b5 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -23,7 +23,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, request.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, data)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 1dfe70a376..b3cb5237f8 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -70,6 +70,8 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA } payloads := generators.BuildPayloadFromOptions(request.options.Options) + // add Hostname variable to the payload + payloads = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if request.generator != nil { iterator := request.generator.NewIterator() @@ -100,8 +102,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac err error ) - request.dynamicValues = generators.MergeMaps(payloads, variables) - if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host } @@ -259,7 +259,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac var event *output.InternalWrappedEvent if len(interactshURLs) == 0 { - event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(payloads, outputEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { wrappedEvent.OperatorsResult.PayloadValues = payloads }) callback(event)