Evaluate payload variables (#3503)

* Evaluate payload variables

* Add variables evaluation

* Extend variables test

- to check evaluation of global variables in variables
- to check evaluation of golbal variables in payload

* Add default and cli variables to websocket, whois and dns proto

- use url.Parse with urlutil.Parse

integration_tests/http/variables.yaml

@@ -7,7 +7,7 @@ info:
 
 variables:
   a1: "value"
-  a2: "{{base64('hello')}}"
+  a2: "{{base64('{{Host}}')}}"
 
 requests:
   - raw:
@@ -16,11 +16,16 @@ requests:
         Host: {{FQDN}}
         Test: {{a1}}
         Another: {{a2}}
-
+        Email: {{ username }}
+    payloads:
+      username:
+        - jon.doe@{{ FQDN }}
     stop-at-first-match: true
     matchers-condition: or
     matchers:
       - type: word
+        condition: and
         words: 
           - "value"
-          - "aGVsbG8="
+          - "MTI3LjAuMC4x" # 127.0.0.1
+          - "jon.doe@127.0.0.1"

v2/cmd/integration-test/http.go

@@ -1050,7 +1050,7 @@ type httpVariables struct{}
 func (h *httpVariables) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		fmt.Fprintf(w, "%s\n%s", r.Header.Get("Test"), r.Header.Get("Another"))
+		fmt.Fprintf(w, "%s\n%s\n%s", r.Header.Get("Test"), r.Header.Get("Another"), r.Header.Get("Email"))
 	})
 	ts := httptest.NewServer(router)
 	defer ts.Close()

v2/pkg/protocols/common/expressions/expressions.go

@@ -40,7 +40,7 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
 	// - simple: containing base values keys (variables)
 	// - complex: containing helper functions [ + variables]
 	// literals like {{2+2}} are not considered expressions
-	expressions := findExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)
+	expressions := FindExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)
 	for _, expression := range expressions {
 		// replace variable placeholders with base values
 		expression = replacer.Replace(expression, base)
@@ -62,7 +62,7 @@ func evaluate(data string, base map[string]interface{}) (string, error) {
 // maxIterations to avoid infinite loop
 const maxIterations = 250
 
-func findExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string {
+func FindExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string {
 	var (
 		iterations int
 		exps       []string

v2/pkg/protocols/common/variables/variables.go

@@ -8,13 +8,15 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 )
 
 // Variable is a key-value pair of strings that can be used
 // throughout template.
 type Variable struct {
+	LazyEval                        bool `yaml:"-" json:"-"` // LazyEval is used to evaluate variables lazily if it using any expression or global variables
 	utils.InsertionOrderedStringMap `yaml:"-" json:"-"`
 }
 
@@ -32,6 +34,11 @@ func (variables *Variable) UnmarshalYAML(unmarshal func(interface{}) error) erro
 	if err := unmarshal(&variables.InsertionOrderedStringMap); err != nil {
 		return err
 	}
+
+	if variables.checkForLazyEval() {
+		return nil
+	}
+
 	evaluated := variables.Evaluate(map[string]interface{}{})
 
 	for k, v := range evaluated {
@@ -87,3 +94,16 @@ func evaluateVariableValue(expression string, values, processing map[string]inte
 
 	return result
 }
+
+// checkForLazyEval checks if the variables have any lazy evaluation i.e any dsl function
+// and sets the flag accordingly.
+func (variables *Variable) checkForLazyEval() bool {
+	variables.ForEach(func(key string, value interface{}) {
+		ret := expressions.FindExpressions(types.ToString(value), marker.ParenthesisOpen, marker.ParenthesisClose, nil)
+		if len(ret) > 0 {
+			variables.LazyEval = true
+			return
+		}
+	})
+	return variables.LazyEval
+}

v2/pkg/protocols/dns/request.go

@@ -45,9 +45,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	if err != nil {
 		return errors.Wrap(err, "could not build request")
 	}
+
 	vars := GenerateVariables(domain)
+	// optionvars are vars passed from CLI or env variables
+	optionVars := generators.BuildPayloadFromOptions(request.options.Options)
 	// merge with metadata (eg. from workflow context)
-	vars = generators.MergeMaps(vars, metadata)
+	vars = generators.MergeMaps(vars, metadata, optionVars)
 	variablesMap := request.options.Variables.Evaluate(vars)
 	vars = generators.MergeMaps(variablesMap, vars)
 

v2/pkg/protocols/http/build_request.go

@@ -104,9 +104,22 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 	defaultReqVars := utils.GenerateVariablesWithURL(parsed, hasTrailingSlash, contextargs.GenerateVariables(input))
 	// optionvars are vars passed from CLI or env variables
 	optionVars := generators.BuildPayloadFromOptions(r.request.options.Options)
+
+	variablesMap, interactURLs := r.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(defaultReqVars, optionVars), r.options.Interactsh)
+	if len(interactURLs) > 0 {
+		r.interactshURLs = append(r.interactshURLs, interactURLs...)
+	}
 	// allVars contains all variables from all sources
-	allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars)
+	allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars, variablesMap)
 
+	// Evaluate payload variables
+	// eg: payload variables can be username: jon.doe@{{Hostname}}
+	for payloadName, payloadValue := range payloads {
+		payloads[payloadName], err = expressions.Evaluate(types.ToString(payloadValue), allVars)
+		if err != nil {
+			return nil, ErrEvalExpression.Wrap(err).WithTag("http")
+		}
+	}
 	// finalVars contains allVars and any generator/fuzzing specific payloads
 	// payloads used in generator should be given the most preference
 	finalVars := generators.MergeMaps(allVars, payloads)

v2/pkg/protocols/http/request.go

@@ -345,8 +345,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		// returns two values, error and skip, which skips the execution for the request instance.
 		executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) {
 			hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
-			variablesMap, interactURLs := request.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(dynamicValues, payloads), request.options.Interactsh)
-			dynamicValue = generators.MergeMaps(variablesMap, dynamicValue)
 
 			request.options.RateLimiter.Take()
 
@@ -366,10 +364,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				defer generatedHttpRequest.customCancelFunction()
 			}
 
-			// If the variables contain interactsh urls, use them
-			if len(interactURLs) > 0 {
-				generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...)
-			}
 			hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
 			if input.MetaInput.Input == "" {
 				input.MetaInput.Input = generatedHttpRequest.URL()

v2/pkg/protocols/network/request.go

@@ -51,11 +51,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
 		request.options.Progress.IncrementFailedRequestsBy(1)
 		return errors.Wrap(err, "could not get address from url")
 	}
+	variables := generateNetworkVariables(address)
+	variablesMap := request.options.Variables.Evaluate(variables)
+	variables = generators.MergeMaps(variablesMap, variables)
 
 	for _, kv := range request.addresses {
-		variables := generateNetworkVariables(address)
-		variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(variables, variables))
-		variables = generators.MergeMaps(variablesMap, variables)
 		actualAddress := replacer.Replace(kv.address, variables)
 
 		if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil {

v2/pkg/protocols/websocket/websocket.go

@@ -168,22 +168,14 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 func (request *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	header := http.Header{}
 
-	payloadValues := make(map[string]interface{})
-	for k, v := range dynamicValues {
-		payloadValues[k] = v
-	}
-	parsed, err := url.Parse(input)
+	parsed, err := urlutil.Parse(input)
 	if err != nil {
 		return errors.Wrap(err, parseUrlErrorMessage)
 	}
-	payloadValues["Hostname"] = parsed.Host
-	payloadValues["Host"] = parsed.Hostname()
-	payloadValues["Scheme"] = parsed.Scheme
-	requestPath := parsed.Path
-	if values := urlutil.GetParams(parsed.Query()); len(values) > 0 {
-		requestPath = requestPath + "?" + values.Encode()
-	}
-	payloadValues["Path"] = requestPath
+	defaultVars := getWebsocketVariables(parsed)
+	optionVars := generators.BuildPayloadFromOptions(request.options.Options)
+	variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues))
+	payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues)
 
 	requestOptions := request.options
 	for key, value := range request.Headers {
@@ -415,3 +407,17 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 func (request *Request) Type() templateTypes.ProtocolType {
 	return templateTypes.WebsocketProtocol
 }
+
+func getWebsocketVariables(input *urlutil.URL) map[string]interface{} {
+	websocketVariables := make(map[string]interface{})
+
+	websocketVariables["Hostname"] = input.Host
+	websocketVariables["Host"] = input.Hostname()
+	websocketVariables["Scheme"] = input.Scheme
+	requestPath := input.Path
+	if values := urlutil.GetParams(input.URL.Query()); len(values) > 0 {
+		requestPath = requestPath + "?" + values.Encode()
+	}
+	websocketVariables["Path"] = requestPath
+	return websocketVariables
+}

v2/pkg/protocols/whois/whois.go

@@ -16,13 +16,15 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois/rdapclientpool"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	urlutil "github.com/projectdiscovery/utils/url"
 )
 
 // Request is a request for the WHOIS protocol
@@ -85,7 +87,11 @@ func (request *Request) GetID() string {
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	// generate variables
-	variables := generateVariables(input.MetaInput.Input)
+	defaultVars := generateVariables(input.MetaInput.Input)
+	optionVars := generators.BuildPayloadFromOptions(request.options.Options)
+	vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues))
+
+	variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues)
 
 	if vardump.EnableVarDump {
 		gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables))
@@ -183,7 +189,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
 func generateVariables(input string) map[string]interface{} {
 	var domain string
 
-	parsed, err := url.Parse(input)
+	parsed, err := urlutil.Parse(input)
 	if err != nil {
 		return map[string]interface{}{"Input": input}
 	}