projectdiscovery · ehsandeep · Mar 13, 2024 · Mar 12, 2024 · Mar 12, 2024 · Mar 12, 2024
diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go
@@ -80,6 +80,55 @@ var httpTestcases = []TestCaseInfo{
 	{Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}},
 	{Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}},
 	{Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}},
+	{Path: "protocols/http/http-matcher-extractor-dy-extractor.yaml", TestCase: &httpMatcherExtractorDynamicExtractor{}},
+	{Path: "protocols/http/multi-http-var-sharing.yaml", TestCase: &httpMultiVarSharing{}},
+}
+
+type httpMultiVarSharing struct{}
+
+func (h *httpMultiVarSharing) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
+type httpMatcherExtractorDynamicExtractor struct{}
+
+func (h *httpMatcherExtractorDynamicExtractor) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		html := `<!DOCTYPE html>
+<html lang="en">
+<body>
+    <a href="/domains">Domains</a>
+</body>
+</html>`
+		fmt.Fprint(w, html)
+	})
+	router.GET("/domains", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		html := `<!DOCTYPE html>
+		<html lang="en">
+		<head>
+			<title>Dynamic Extractor Test</title>
+		</head>
+		<body>
+			<!-- The content of the title tag matches the regex pattern for both the extractor and matcher for 'title' -->
+		</body>
+		</html>
+		`
+		fmt.Fprint(w, html)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
 }
 
 type httpInteractshRequest struct{}
@@ -776,8 +825,18 @@ func (h *httpPaths) Execute(filepath string) error {
 		}
 	}
 
-	if !reflect.DeepEqual(expected, actual) {
-		return fmt.Errorf("%8v: %v\n%-8v: %v", "expected", expected, "actual", actual)
+	if len(expected) > len(actual) {
+		actualValuesIndex := len(actual) - 1
+		if actualValuesIndex < 0 {
+			actualValuesIndex = 0
+		}
+		return fmt.Errorf("missing values : %v", expected[actualValuesIndex:])
+	} else if len(expected) < len(actual) {
+		return fmt.Errorf("unexpected values : %v", actual[len(expected)-1:])
+	} else {
+		if !reflect.DeepEqual(expected, actual) {
+			return fmt.Errorf("expected: %v\n\nactual: %v", expected, actual)
+		}
 	}
 	return nil
 }

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
@@ -322,6 +322,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
 		flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
+		flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"),
 	)
 	flagSet.CreateGroup("optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),

diff --git a/go.mod b/go.mod
@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.61
+	github.com/projectdiscovery/fastdialer v0.0.62
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.40
 	github.com/projectdiscovery/retryabledns v1.0.58
-	github.com/projectdiscovery/retryablehttp-go v1.0.50
+	github.com/projectdiscovery/retryablehttp-go v1.0.51
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -91,7 +91,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.82
+	github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca
 	github.com/projectdiscovery/wappalyzergo v0.0.112
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/sashabaranov/go-openai v1.15.3
@@ -274,7 +274,6 @@ require (
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/trivago/tgo v1.0.7
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
 	github.com/ysmood/goob v0.4.0 // indirect

diff --git a/go.sum b/go.sum
@@ -812,8 +812,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
 github.com/projectdiscovery/dsl v0.0.46 h1:zBNNzSBA1aakGY44w6KhnjJQx/zO+oW2Wx7TR8xZm/A=
 github.com/projectdiscovery/dsl v0.0.46/go.mod h1:eoZEJFCIT5emI00xj8HTQwHz4YwwGAD+grL3G7CDlfs=
-github.com/projectdiscovery/fastdialer v0.0.61 h1:z5OzP9lRbn6fSIezgReKC3hkzRh+YX41ST9OgkVEm/s=
-github.com/projectdiscovery/fastdialer v0.0.61/go.mod h1:FyxJ0m1MwB69nLmdXYqK32f3a0Pf+5YpC8wBY73baiE=
+github.com/projectdiscovery/fastdialer v0.0.62 h1:Wyba2hD6ZF3S04MgCn380mC+1RXJ+dq14Yq8u2yk7ps=
+github.com/projectdiscovery/fastdialer v0.0.62/go.mod h1:2baj2TRXTw+hHbKTW9IZR4dhpxCGJkq5AKL1ge5gis8=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -852,8 +852,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
 github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
-github.com/projectdiscovery/retryablehttp-go v1.0.50 h1:QiVqNzpDFoTZm2z1oO68FEtl2VTk2kFscPetxmCK2gc=
-github.com/projectdiscovery/retryablehttp-go v1.0.50/go.mod h1:yY/go76Lx/MgJmR+dSg/Xa8wOiXNjmuEdBDVDODKeDk=
+github.com/projectdiscovery/retryablehttp-go v1.0.51 h1:8XMrNC8JrwvySESe2d+XWF9bq4unWqD4PUPEC4Cai8s=
+github.com/projectdiscovery/retryablehttp-go v1.0.51/go.mod h1:6cdh/acYHpeYWg7+Iblh4xBRb87bC118L4G4mpvCMuA=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -864,8 +864,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.82 h1:U//02floCSFxJluN7MP+rJSwI4Px7o454JL7ukERArI=
-github.com/projectdiscovery/utils v0.0.82/go.mod h1:AbmIvy0TTlsfXxPDEMaNPVrxmqDmYiCnbGqh0TTthE4=
+github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca h1:GY9lUYDlENXPSFPJH01Bm1BfhrUF2jpnUBR+K4VPJIs=
+github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca/go.mod h1:wzMfHBq2I9oy+DEiMfUYV86g1D7eXKaQsgWnqFpmMtI=
 github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA=
 github.com/projectdiscovery/wappalyzergo v0.0.112/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -1034,8 +1034,6 @@ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW
 github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
 github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA=
-github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=

diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml
@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a00473045022100b290a0c40f27573f0de9a950be13457a9bf59ade1ff2f497bf01a3b526e5db750220761942acffd6d27e2714ddaa1c73c699ccd7de48839f08cff1d6a9456bc8ff1f:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a0047304502207e3a5eda5f3207c3c01c820562243281926c1215224a7c80ed7528559b9f52cb022100f6ef99bb45843f481705778630f2cfd8f4d1cc3acb96392ff016f22e06aa91af:4a3eb6b4988d95847d4203be25ed1d46
diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml
@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 490a004630440220335663a6a4db720ee6276ab7179a87a6be0b4030771ec5ee82ecf6982342113602200a2570db7eb9721f6ceb1a89543fc436ee62b30d1b720c75ea3834ed3d2b64f3:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a004730450220069673af9bd6d6677f9529d06f5d8bd46d543089a4731ed18ee806761d75fd60022100913a3e27b0a5809baf710ba9585bf9fe729634c0e19e3e13eef70a6bd100df34:4a3eb6b4988d95847d4203be25ed1d46
diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml
@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 490a004630440220400892730a62fa1bbb1064e4d88eea760dbf8f01c6b630ff0f5b126fd1952839022025a6d52e730c1f1cfcbd440e6269f93489db3a77cb2a27d0f47522c0819dc8d3:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a00463044022003b8d069e3c84412729c43e33013a52ee04eabcf096d511979691d71d8e905f60220011f4475899abed4f86b4bd5e6c2423750759135206e4729826afe1ed8a44f4d:4a3eb6b4988d95847d4203be25ed1d46
diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml
@@ -21,4 +21,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 490a0046304402206b14abdc0d5fc13466f5c292da9fb2a19d1b2c5e683cc052037fe367b372f82b02202c00b9acbd8106a769eb411794c567d3019433671397bf909e16b286105ed69e:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a00473045022100c291615cf2a8005450c17a6554e81a9cdab14743b299f0679c644247929198b502206fdacc8ab173bde2b4015340012637916bf2659f66f320fcc06b97ac639072a1:4a3eb6b4988d95847d4203be25ed1d46
diff --git a/integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml b/integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml
@@ -0,0 +1,36 @@
+id: http-matcher-extractor-dy-extractor
+info:
+  name: HTTP matcher and extractor & dynamic extractor
+  description: >
+    Edgecase to test for a combination of matchers , extractors and dynamic extractors
+  author: pdteam
+  severity: info
+
+http:
+  - raw:
+      - |
+        GET {{BaseURL}} HTTP/1.1
+      - |
+        GET {{absolutePath}} HTTP/1.1
+
+    req-condition: true
+    extractors:
+      - type: regex
+        internal: true
+        part: body_1
+        name: absolutePath
+        regex:
+          - '<a href="(/domains)">'
+        group: 1
+      - type: regex
+        internal: false
+        part: body_2
+        name: title
+        regex:
+          - '<title[^>]*>([^<]+)</title>'
+        group: 1
+    matchers:
+      - type: regex
+        part: body_2
+        regex:
+          - '<title[^>]*>([^<]+)</title>'
diff --git a/integration_tests/protocols/http/http-paths.yaml b/integration_tests/protocols/http/http-paths.yaml
@@ -28,7 +28,7 @@ info:
     - "//CFIDE/wizards/common/utils.cfc"
 
 # Test all templates with FullURLs
-requests:
+http:
   - raw:
     # relative path without leading slash with param
     # If relative path does not have `/` prefix it is autocorrected 

diff --git a/integration_tests/protocols/http/multi-http-var-sharing.yaml b/integration_tests/protocols/http/multi-http-var-sharing.yaml
@@ -0,0 +1,36 @@
+id: multi-http-var-sharing
+
+info:
+  name: Multi HTTP var sharing
+  author: pdteam
+  severity: info
+  description: |
+    A template which has multiple HTTP requests block and variables are shared between them
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"
+        negative: true
+        internal: true
+
+    extractors:
+      - type: dsl
+        name: ffff
+        dsl:
+          - status_code
+        internal: true
+
+  - method: GET
+    path:
+      - "{{BaseURL}}/{{ffff}}"
+
+    matchers:
+      - type: status
+        status: 
+          - 200
diff --git a/lib/config.go b/lib/config.go
@@ -107,10 +107,12 @@ func WithInteractshOptions(opts InteractshOpts) NucleiSDKOptions {
 
 // Concurrency options
 type Concurrency struct {
-	TemplateConcurrency         int // number of templates to run concurrently (per host in host-spray mode)
-	HostConcurrency             int // number of hosts to scan concurrently  (per template in template-spray mode)
-	HeadlessHostConcurrency     int // number of hosts to scan concurrently for headless templates  (per template in template-spray mode)
-	HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode)
+	TemplateConcurrency           int // number of templates to run concurrently (per host in host-spray mode)
+	HostConcurrency               int // number of hosts to scan concurrently  (per template in template-spray mode)
+	HeadlessHostConcurrency       int // number of hosts to scan concurrently for headless templates  (per template in template-spray mode)
+	HeadlessTemplateConcurrency   int // number of templates to run concurrently for headless templates (per host in host-spray mode)
+	JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode)
+	TemplatePayloadConcurrency    int // max concurrent payloads to run for a template (a good default is 25)
 }
 
 // WithConcurrency sets concurrency options
@@ -120,6 +122,8 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 		e.opts.BulkSize = opts.HostConcurrency
 		e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
 		e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
+		e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
+		e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
 		return nil
 	}
 }

diff --git a/pkg/operators/operators.go b/pkg/operators/operators.go
@@ -299,7 +299,7 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
 			if len(result.DynamicValues) > 0 {
 				return result, true
 			}
-			return nil, false
+			return result, false
 		}
 	}
 
@@ -313,13 +313,21 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
 
 	// Don't print if we have matchers, and they have not matched, regardless of extractor
 	if len(operators.Matchers) > 0 && !matches {
+		// if dynamic values are present then it is not a failure
+		if len(result.DynamicValues) > 0 {
+			return result, true
+		}
 		return nil, false
 	}
 	// Write a final string of output if matcher type is
 	// AND or if we have extractors for the mechanism too.
 	if len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches {
 		return result, true
 	}
+	// if dynamic values are present then it is not a failure
+	if len(result.DynamicValues) > 0 {
+		return result, true
+	}
 	return nil, false
 }
 

diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go
@@ -192,6 +192,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	dataOutputString := fmtStdout(gOutput.Stdout.String())
 
 	data := make(output.InternalEvent)
+	// also include all request variables in result event
+	for _, value := range metaSrc.Variables {
+		data[value.Name] = value.Value
+	}
 
 	data["type"] = request.Type().String()
 	data["response"] = dataOutputString // response contains filtered output (eg without trailing \n)

diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go
@@ -97,7 +97,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 	}
 
 	// Parse target url
-	parsed, err := urlutil.Parse(input.MetaInput.Input)
+	parsed, err := urlutil.ParseAbsoluteURL(input.MetaInput.Input, false)
 	if err != nil {
 		return nil, err
 	}
@@ -153,7 +153,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 		return r.generateRawRequest(ctx, reqData, parsed, finalVars, payloads)
 	}
 
-	reqURL, err := urlutil.ParseURL(reqData, true)
+	reqURL, err := urlutil.ParseAbsoluteURL(reqData, true)
 	if err != nil {
 		return nil, errorutil.NewWithTag("http", "failed to parse url %v while creating http request", reqData)
 	}
@@ -291,8 +291,7 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
 		unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs}
 		return unsafeReq, nil
 	}
-
-	urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
+	urlx, err := urlutil.ParseAbsoluteURL(rawRequestData.FullURL, true)
 	if err != nil {
 		return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")
 	}