Merge pull request #1288 from projectdiscovery/dynamic-value-reuse-http

Reusing dynamically extracted values as iterators in http request

v2/Makefile

@@ -18,5 +18,9 @@ docs:
 	./cmd/docgen/docgen docs.md nuclei-jsonschema.json
 test:
 	$(GOTEST) -v ./...
+integration:
+	bash ../integration_tests/run.sh
+functional:
+	bash cmd/functional-tests/run.sh
 tidy:
 	$(GOMOD) tidy

v2/pkg/operators/operators.go

@@ -72,11 +72,64 @@ type Result struct {
 	// OutputExtracts is the list of extracts to be displayed on screen.
 	OutputExtracts []string
 	// DynamicValues contains any dynamic values to be templated
-	DynamicValues map[string]interface{}
+	DynamicValues map[string][]string
 	// PayloadValues contains payload values provided by user. (Optional)
 	PayloadValues map[string]interface{}
 }
 
+// MakeDynamicValuesCallback takes an input dynamic values map and calls
+// the callback function with all variations of the data in input in form
+// of map[string]string (interface{}).
+func MakeDynamicValuesCallback(input map[string][]string, iterateAllValues bool, callback func(map[string]interface{}) bool) {
+	output := make(map[string]interface{}, len(input))
+
+	if !iterateAllValues {
+		for k, v := range input {
+			if len(v) > 0 {
+				output[k] = v[0]
+			}
+		}
+		callback(output)
+		return
+	}
+	inputIndex := make(map[string]int, len(input))
+
+	var maxValue int
+	for _, v := range input {
+		if len(v) > maxValue {
+			maxValue = len(v)
+		}
+	}
+
+	for i := 0; i < maxValue; i++ {
+		for k, v := range input {
+			if len(v) == 0 {
+				continue
+			}
+			if len(v) == 1 {
+				output[k] = v[0]
+				continue
+			}
+			if gotIndex, ok := inputIndex[k]; !ok {
+				inputIndex[k] = 0
+				output[k] = v[0]
+			} else {
+				newIndex := gotIndex + 1
+				if newIndex >= len(v) {
+					output[k] = v[len(v)-1]
+					continue
+				}
+				output[k] = v[newIndex]
+				inputIndex[k] = newIndex
+			}
+		}
+		// skip if the callback says so
+		if callback(output) {
+			return
+		}
+	}
+}
+
 // Merge merges a result structure into the other.
 func (r *Result) Merge(result *Result) {
 	if !r.Matched && result.Matched {
@@ -115,7 +168,7 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
 	result := &Result{
 		Matches:       make(map[string][]string),
 		Extracts:      make(map[string][]string),
-		DynamicValues: make(map[string]interface{}),
+		DynamicValues: make(map[string][]string),
 	}
 
 	// Start with the extractors first and evaluate them.
@@ -126,8 +179,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
 			extractorResults = append(extractorResults, match)
 
 			if extractor.Internal {
-				if _, ok := result.DynamicValues[extractor.Name]; !ok {
-					result.DynamicValues[extractor.Name] = match
+				if data, ok := result.DynamicValues[extractor.Name]; !ok {
+					result.DynamicValues[extractor.Name] = []string{match}
+				} else {
+					result.DynamicValues[extractor.Name] = append(data, match)
 				}
 			} else {
 				result.OutputExtracts = append(result.OutputExtracts, match)

v2/pkg/operators/operators_test.go

@@ -0,0 +1,57 @@
+package operators
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMakeDynamicValuesCallback(t *testing.T) {
+	input := map[string][]string{
+		"a": []string{"1", "2"},
+		"b": []string{"3"},
+		"c": []string{},
+		"d": []string{"A", "B", "C"},
+	}
+
+	count := 0
+	MakeDynamicValuesCallback(input, true, func(data map[string]interface{}) bool {
+		count++
+		require.Len(t, data, 3, "could not get correct output length")
+		return false
+	})
+	require.Equal(t, 3, count, "could not get correct result count")
+
+	t.Run("all", func(t *testing.T) {
+		input := map[string][]string{
+			"a": []string{"1"},
+			"b": []string{"2"},
+			"c": []string{"3"},
+		}
+
+		count := 0
+		MakeDynamicValuesCallback(input, true, func(data map[string]interface{}) bool {
+			count++
+			require.Len(t, data, 3, "could not get correct output length")
+			return false
+		})
+		require.Equal(t, 1, count, "could not get correct result count")
+	})
+
+	t.Run("first", func(t *testing.T) {
+		input := map[string][]string{
+			"a": []string{"1", "2"},
+			"b": []string{"3"},
+			"c": []string{},
+			"d": []string{"A", "B", "C"},
+		}
+
+		count := 0
+		MakeDynamicValuesCallback(input, false, func(data map[string]interface{}) bool {
+			count++
+			require.Len(t, data, 3, "could not get correct output length")
+			return false
+		})
+		require.Equal(t, 1, count, "could not get correct result count")
+	})
+}

v2/pkg/protocols/common/generators/maps.go

@@ -1,9 +1,49 @@
 package generators
 
 import (
+	"reflect"
 	"strings"
 )
 
+// MergeMapsMany merges many maps into a new map
+func MergeMapsMany(maps ...interface{}) map[string][]string {
+	m := make(map[string][]string)
+	for _, gotMap := range maps {
+		val := reflect.ValueOf(gotMap)
+		if val.Kind() != reflect.Map {
+			continue
+		}
+		appendToSlice := func(key, value string) {
+			if values, ok := m[key]; !ok {
+				m[key] = []string{value}
+			} else {
+				m[key] = append(values, value)
+			}
+		}
+		for _, e := range val.MapKeys() {
+			v := val.MapIndex(e)
+			switch v.Kind() {
+			case reflect.Slice, reflect.Array:
+				for i := 0; i < v.Len(); i++ {
+					appendToSlice(e.String(), v.Index(i).String())
+				}
+			case reflect.String:
+				appendToSlice(e.String(), v.String())
+			case reflect.Interface:
+				switch data := v.Interface().(type) {
+				case string:
+					appendToSlice(e.String(), data)
+				case []string:
+					for _, value := range data {
+						appendToSlice(e.String(), value)
+					}
+				}
+			}
+		}
+	}
+	return m
+}
+
 // MergeMaps merges two maps into a new map
 func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} {
 	m := make(map[string]interface{}, len(m1)+len(m2))

v2/pkg/protocols/common/generators/maps_test.go

@@ -0,0 +1,16 @@
+package generators
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeMapsMany(t *testing.T) {
+	got := MergeMapsMany(map[string]interface{}{"a": []string{"1", "2"}, "c": "5"}, map[string][]string{"b": []string{"3", "4"}})
+	require.Equal(t, map[string][]string{
+		"a": []string{"1", "2"},
+		"b": []string{"3", "4"},
+		"c": []string{"5"},
+	}, got, "could not get correct merged map")
+}

v2/pkg/protocols/http/build_request.go

@@ -53,14 +53,9 @@ func (g *generatedRequest) URL() string {
 
 // Make creates a http request for the provided input.
 // It returns io.EOF as error when all the requests have been exhausted.
-func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) {
+func (r *requestGenerator) Make(baseURL, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
 	if r.request.SelfContained {
-		return r.makeSelfContainedRequest(dynamicValues)
-	}
-	// We get the next payload for the request.
-	data, payloads, ok := r.nextValue()
-	if !ok {
-		return nil, io.EOF
+		return r.makeSelfContainedRequest(data, payloads, dynamicValues)
 	}
 	ctx := context.Background()
 
@@ -107,12 +102,7 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	return r.makeHTTPRequestFromModel(ctx, data, values, payloads)
 }
 
-func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}) (*generatedRequest, error) {
-	// We get the next payload for the request.
-	data, payloads, ok := r.nextValue()
-	if !ok {
-		return nil, io.EOF
-	}
+func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
 	ctx := context.Background()
 
 	isRawRequest := r.request.isRaw()

v2/pkg/protocols/http/build_request_test.go

@@ -87,7 +87,8 @@ func TestMakeRequestFromModal(t *testing.T) {
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{})
+	inputData, payloads, _ := generator.nextValue()
+	req, err := generator.Make("https://example.com", inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 
 	bodyBytes, _ := req.request.BodyBytes()
@@ -114,12 +115,14 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com/test.php", map[string]interface{}{})
+	inputData, payloads, _ := generator.nextValue()
+	req, err := generator.Make("https://example.com/test.php", inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
 
 	generator = request.newGenerator()
-	req, err = generator.Make("https://example.com/test/", map[string]interface{}{})
+	inputData, payloads, _ = generator.nextValue()
+	req, err = generator.Make("https://example.com/test/", inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
 }
@@ -152,12 +155,14 @@ Accept-Encoding: gzip`},
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{})
+	inputData, payloads, _ := generator.nextValue()
+	req, err := generator.Make("https://example.com", inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
 
-	req, err = generator.Make("https://example.com", map[string]interface{}{})
+	inputData, payloads, _ = generator.nextValue()
+	req, err = generator.Make("https://example.com", inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
@@ -191,12 +196,14 @@ Accept-Encoding: gzip`},
 	require.Nil(t, err, "could not compile http request")
 
 	generator := request.newGenerator()
-	req, err := generator.Make("https://example.com", map[string]interface{}{})
+	inputData, payloads, _ := generator.nextValue()
+	req, err := generator.Make("https://example.com", inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
 
-	req, err = generator.Make("https://example.com", map[string]interface{}{})
+	inputData, payloads, _ = generator.nextValue()
+	req, err = generator.Make("https://example.com", inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")
@@ -232,7 +239,8 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
 	})
 	require.Nil(t, err, "could not create interactsh client")
 
-	got, err := generator.Make("https://example.com", map[string]interface{}{})
+	inputData, payloads, _ := generator.nextValue()
+	got, err := generator.Make("https://example.com", inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 
 	// check if all the interactsh markers are replaced with unique urls

v2/pkg/protocols/http/http.go

@@ -165,6 +165,9 @@ type Request struct {
 	// description: |
 	//   SkipVariablesCheck skips the check for unresolved variables in request
 	SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"`
+	// description: |
+	//   IterateAll iterates all the values extracted from internal extractors
+	IterateAll bool `yaml:"iterate-all,omitempty" jsonschema:"title=iterate all the values,description=Iterates all the values extracted from internal extractors"`
 }
 
 // RequestPartDefinitions contains a mapping of request part definitions and their