aws signer: fix missing x-content-sha256 header (#3601)

* fix missing x-content-sha256 header

* fix variable priority in self-contained templates

* remove debug statement

* adds generic raw request parser for self-contained req

* more integration tests

* bug fix: 10x faster race requests

* fix failing integration test

integration_tests/http/raw-unsafe-with-params.yaml

@@ -0,0 +1,20 @@
+id: raw-unsafe-with-params
+
+info:
+  name: Test RAW unsafe with params
+  author: pdteam
+  severity: info
+# this test is used to check automerge of params in both unsafe & safe requests
+# key1=value1 is added from inputURL
+
+requests:
+  - raw:
+      - |+
+        GET /?key2=value2 HTTP/1.1
+        Host: {{Hostname}}
+
+    unsafe: true
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-params-matcher text"

integration_tests/http/raw-with-params.yaml

@@ -0,0 +1,20 @@
+id: raw-with-params
+
+info:
+  name: Test RAW Params Template
+  author: pdteam
+  severity: info
+# this test is used to check automerge of params in both unsafe & safe requests
+# key1=value1 is added from inputURL
+
+requests:
+  - raw:
+      - |
+        GET /?key2=value2 HTTP/1.1
+        Host: {{Hostname}}
+        Origin: {{BaseURL}}
+
+    matchers:
+      - type: word
+        words:
+          - "Test is test raw-params-matcher text"

integration_tests/http/self-contained-with-path.yaml

@@ -0,0 +1,18 @@
+id: self-contained-with-path
+
+info:
+  name: self-contained-with-path
+  author: pd-team
+  severity: info
+
+self-contained: true
+requests:
+  - raw:
+      - |
+        GET / HTTP/1.1
+        Host: 127.0.0.1:5431
+
+    matchers:
+      - type: word
+        words:
+          - This is self-contained response

v2/cmd/integration-test/http.go

@@ -42,6 +42,8 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/raw-dynamic-extractor.yaml":               &httpRawDynamicExtractor{},
 	"http/raw-get-query.yaml":                       &httpRawGetQuery{},
 	"http/raw-get.yaml":                             &httpRawGet{},
+	"http/raw-with-params.yaml":                     &httpRawWithParams{},
+	"http/raw-unsafe-with-params.yaml":              &httpRawWithParams{}, // Not a typo, functionality is same as above
 	"http/raw-path-trailing-slash.yaml":             &httpRawPathTrailingSlash{},
 	"http/raw-payload.yaml":                         &httpRawPayload{},
 	"http/raw-post-body.yaml":                       &httpRawPostBody{},
@@ -50,6 +52,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/request-condition.yaml":                   &httpRequestCondition{},
 	"http/request-condition-new.yaml":               &httpRequestCondition{},
 	"http/self-contained.yaml":                      &httpRequestSelfContained{},
+	"http/self-contained-with-path.yaml":            &httpRequestSelfContained{}, // Not a typo, functionality is same as above
 	"http/self-contained-with-params.yaml":          &httpRequestSelfContainedWithParams{},
 	"http/self-contained-file-input.yaml":           &httpRequestSelfContainedFileInput{},
 	"http/get-case-insensitive.yaml":                &httpGetCaseInsensitive{},
@@ -557,6 +560,37 @@ func (h *httpRawGet) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type httpRawWithParams struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpRawWithParams) Execute(filePath string) error {
+	router := httprouter.New()
+	var errx error
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		params := r.URL.Query()
+		// we intentionally use params["test"] instead of params.Get("test") to test the case where
+		// there are multiple parameters with the same name
+		if !reflect.DeepEqual(params["key1"], []string{"value1"}) {
+			errx = errorutil.WrapfWithNil(errx, "expected %v, got %v", []string{"value1"}, params["key1"])
+		}
+		if !reflect.DeepEqual(params["key2"], []string{"value2"}) {
+			errx = errorutil.WrapfWithNil(errx, "expected %v, got %v", []string{"value2"}, params["key2"])
+		}
+		fmt.Fprintf(w, "Test is test raw-params-matcher text")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"/?key1=value1", debug)
+	if err != nil {
+		return err
+	}
+	if errx != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
 type httpRawPathTrailingSlash struct{}
 
 func (h *httpRawPathTrailingSlash) Execute(filepath string) error {
@@ -865,16 +899,16 @@ type httpRequestSelfContainedWithParams struct{}
 // Execute executes a test case and returns an error if occurred
 func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error {
 	router := httprouter.New()
-	var err error
+	var errx error
 	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		params := r.URL.Query()
 		// we intentionally use params["test"] instead of params.Get("test") to test the case where
 		// there are multiple parameters with the same name
 		if !reflect.DeepEqual(params["something"], []string{"here"}) {
-			err = errorutil.WrapfWithNil(err, "expected %v, got %v", []string{"here"}, params["something"])
+			errx = errorutil.WrapfWithNil(errx, "expected %v, got %v", []string{"here"}, params["something"])
 		}
 		if !reflect.DeepEqual(params["key"], []string{"value"}) {
-			err = errorutil.WrapfWithNil(err, "expected %v, got %v", []string{"key"}, params["value"])
+			errx = errorutil.WrapfWithNil(errx, "expected %v, got %v", []string{"value"}, params["key"])
 		}
 		_, _ = w.Write([]byte("This is self-contained response"))
 	})
@@ -891,6 +925,9 @@ func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
+	if errx != nil {
+		return errx
+	}
 
 	return expectResultsCount(results, 1)
 }

v2/pkg/protocols/common/variables/doc.go

@@ -0,0 +1,24 @@
+package variables
+
+// There are total 5 sources of variables
+// 1. VariablesMap - Variables defined in the template  (available at Request.options.Variables in protocols)
+// 2. PayloadsMap  - Payloads defined in the template   (available at Request.generator in protocols)
+// 3. OptionsMap   - Variables passed using CLI Options (+ Env) (available at generators.BuildPayloadFromOptions)
+// 4. DynamicMap   - Variables Obtained by extracting data from templates  (available at Request.ExecuteWithResults + merged with previous internalEvent)
+// 5. ProtocolMap  - Variables generated by Evaluation Request / Responses of xyz protocol (available in Request.Make)
+
+// As we can tell , all variables sources are not linear i.e why they need to re-evaluated
+// consider example
+// variables:
+//  - name: "username@{{Host}}"
+
+// Linear Sources (once obtained no need to re-evaluate)
+// simply put they don't contain references to other variables
+// 1. OptionsMap
+// 2. DynamicMap
+// 3. ProtocolMap
+
+// Non-Linear Sources (need to re-evaluate)
+// 1. VariablesMap
+// 2. PayloadsMap
+// Everytime Linear Sources are updated , Non-Linear Sources need to be re-evaluated

v2/pkg/protocols/http/build_request.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io"
 	"strings"
 	"time"
 
@@ -15,7 +14,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw"
@@ -30,7 +28,10 @@ import (
 )
 
 // ErrEvalExpression
-var ErrEvalExpression = errorutil.NewWithTag("expr", "could not evaluate helper expressions")
+var (
+	ErrEvalExpression = errorutil.NewWithTag("expr", "could not evaluate helper expressions")
+	ErrUnresolvedVars = errorutil.NewWithFmt("unresolved variables `%v` found in request")
+)
 
 // generatedRequest is a single generated request wrapped for a template request
 type generatedRequest struct {
@@ -157,6 +158,27 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
 	isRawRequest := r.request.isRaw()
 
+	values := generators.MergeMaps(
+		generators.BuildPayloadFromOptions(r.request.options.Options),
+		dynamicValues,
+		payloads, // payloads should override other variables in case of duplicate vars
+	)
+	// adds all variables from `variables` section in template
+	variablesMap := r.request.options.Variables.Evaluate(values)
+	values = generators.MergeMaps(variablesMap, values)
+
+	signerVars := GetDefaultSignerVars(r.request.Signature.Value)
+	// this will ensure that default signer variables are overwritten by other variables
+	values = generators.MergeMaps(signerVars, values)
+
+	// priority of variables is as follows (from low to high) for self contained templates
+	// default signer vars < variables <  cli vars  < payload < dynamic values
+
+	// evaluate request
+	data, err := expressions.Evaluate(data, values)
+	if err != nil {
+		return nil, ErrEvalExpression.Wrap(err).WithTag("self-contained")
+	}
 	// If the request is a raw request, get the URL from the request
 	// header and use it to make the request.
 	if isRawRequest {
@@ -177,25 +199,8 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 			return nil, fmt.Errorf("malformed request supplied")
 		}
 
-		// Note: Here the order of payloads matter since payloads passed through file ex: -V "test=numbers.txt"
-		// are stored in payloads and options vars should not override payloads in any case
-		values := generators.MergeMaps(
-			generators.BuildPayloadFromOptions(r.request.options.Options),
-			payloads,
-		)
-
-		parts[1] = replacer.Replace(parts[1], values)
-		if len(dynamicValues) > 0 {
-			parts[1] = replacer.Replace(parts[1], dynamicValues)
-		}
-
-		// the url might contain placeholders with ignore list
-		if ignoreList := GetVariablesNamesSkipList(r.request.Signature.Value); ignoreList != nil {
-			if err := expressions.ContainsVariablesWithIgnoreList(ignoreList, parts[1]); err != nil {
-				return nil, err
-			}
-		} else if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil { // the url might contain placeholders
-			return nil, err
+		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
+			return nil, ErrUnresolvedVars.Msgf(parts[1])
 		}
 
 		parsed, err := urlutil.ParseURL(parts[1], true)
@@ -213,16 +218,12 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 		}
 		return r.generateRawRequest(ctx, data, parsed, values, payloads)
 	}
-	values := generators.MergeMaps(
-		generators.BuildPayloadFromOptions(r.request.options.Options),
-		dynamicValues,
-		payloads, // payloads should override other variables in case of duplicate vars
-	)
-	// Evaluate (replace) variable with final values
-	data, err := expressions.Evaluate(data, values)
-	if err != nil {
-		return nil, ErrEvalExpression.Wrap(err).WithTag("self-contained")
+	if err := expressions.ContainsUnresolvedVariables(data); err != nil {
+		// early exit: if there are any unresolved variables in `path` after evaluation
+		// then return early since this will definitely fail
+		return nil, ErrUnresolvedVars.Msgf(data)
 	}
+
 	urlx, err := urlutil.ParseURL(data, true)
 	if err != nil {
 		return nil, errorutil.NewWithErr(err).Msgf("failed to parse %v in self contained request", data).WithTag("self-contained")
@@ -255,14 +256,17 @@ func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urluti
 // finalVars = contains all variables including generator and protocol specific variables
 // generatorValues = contains variables used in fuzzing or other generator specific values
 func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest string, baseURL *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
-	// Unlike other requests parsedURL/ InputURL in self contained templates is extracted from raw request itself h
-	// and variables are supposed to be given from command line and not from inputURL
-	// ence this cause issues like duplicated params/paths.
-	// TODO: implement a generic raw request parser in rawhttp library (without variables and stuff)
-	baseURL.Params = nil // this fixes issue of duplicated params in self contained templates but not a appropriate fix
-	rawRequestData, err := raw.Parse(rawRequest, baseURL, r.request.Unsafe)
+
+	var rawRequestData *raw.Request
+	var err error
+	if r.request.SelfContained {
+		// in self contained requests baseURL is extracted from raw request itself
+		rawRequestData, err = raw.ParseRawRequest(rawRequest, r.request.Unsafe)
+	} else {
+		rawRequestData, err = raw.Parse(rawRequest, baseURL, r.request.Unsafe)
+	}
 	if err != nil {
-		return nil, err
+		return nil, errorutil.NewWithErr(err).Msgf("failed to parse raw request")
 	}
 
 	// Unsafe option uses rawhttp library
@@ -273,19 +277,12 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
 		unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs}
 		return unsafeReq, nil
 	}
-	var body io.ReadCloser
-	body = io.NopCloser(strings.NewReader(rawRequestData.Data))
-	if r.request.Race && r.request.RaceNumberRequests > 0 {
-		// More or less this ensures that all requests hit the endpoint at the same approximated time
-		// Todo: sync internally upon writing latest request byte
-		body = race.NewOpenGateWithTimeout(body, time.Duration(2)*time.Second)
-	}
 
 	urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
 	if err != nil {
 		return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")
 	}
-	req, err := retryablehttp.NewRequestFromURLWithContext(ctx, rawRequestData.Method, urlx, body)
+	req, err := retryablehttp.NewRequestFromURLWithContext(ctx, rawRequestData.Method, urlx, rawRequestData.Data)
 	if err != nil {
 		return nil, err
 	}

v2/pkg/protocols/http/raw/raw.go

@@ -98,6 +98,35 @@ func Parse(request string, inputURL *urlutil.URL, unsafe bool) (*Request, error)
 	return rawrequest, nil
 }
 
+// ParseRawRequest parses the raw request as supplied by the user
+// this function should only be used for self-contained requests
+func ParseRawRequest(request string, unsafe bool) (*Request, error) {
+	req, err := readRawRequest(request, unsafe)
+	if err != nil {
+		return nil, err
+	}
+	if strings.HasPrefix(req.Path, "http") {
+		urlx, err := urlutil.Parse(req.Path)
+		if err != nil {
+			return nil, errorutil.NewWithErr(err).Msgf("failed to parse url %v", req.Path)
+		}
+		req.Path = urlx.GetRelativePath()
+		req.FullURL = urlx.String()
+	} else {
+
+		if req.Path == "" {
+			return nil, errorutil.NewWithTag("self-contained-raw", "path cannot be empty in self contained request")
+		}
+		// given url is relative construct one using Host Header
+		if _, ok := req.Headers["Host"]; !ok {
+			return nil, errorutil.NewWithTag("self-contained-raw", "host header is required for relative path")
+		}
+		// Review: Current default scheme in self contained templates if relative path is provided is http
+		req.FullURL = fmt.Sprintf("%s://%s%s", urlutil.HTTP, strings.TrimSpace(req.Headers["Host"]), req.Path)
+	}
+	return req, nil
+}
+
 // reads raw request line by line following convention
 func readRawRequest(request string, unsafe bool) (*Request, error) {
 	rawRequest := &Request{
@@ -121,7 +150,7 @@ read_line:
 		goto read_line
 	}
 
-	parts := strings.Split(s, " ")
+	parts := strings.Fields(s)
 	if len(parts) > 0 {
 		rawRequest.Method = parts[0]
 		if len(parts) == 2 && strings.Contains(parts[1], "HTTP") {

v2/pkg/protocols/http/signature.go

@@ -111,3 +111,11 @@ func GetVariablesNamesSkipList(signature SignatureType) map[string]interface{} {
 		return nil
 	}
 }
+
+// GetDefaultSignerVars returns the default signer variables
+func GetDefaultSignerVars(signatureType SignatureType) map[string]interface{} {
+	if signatureType == AWSSignature {
+		return signer.AwsDefaultVars
+	}
+	return map[string]interface{}{}
+}