enable asnmap unit tests (#3880)

* fix showing multiple failure matchers per template add integration test * exclude AS134029 from unit test * Add flag for match status per request * chore(deps): bump golangci/golangci-lint-action from 3.4.0 to 3.5.0 (#3777) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](golangci/golangci-lint-action@v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/xanzy/go-gitlab in /v2 (#3778) Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.83.0 to 0.84.0. - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](xanzy/go-gitlab@v0.83.0...v0.84.0) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/spf13/cast from 1.5.0 to 1.5.1 in /v2 (#3780) Bumps [github.com/spf13/cast](https://github.com/spf13/cast) from 1.5.0 to 1.5.1. - [Release notes](https://github.com/spf13/cast/releases) - [Commits](spf13/cast@v1.5.0...v1.5.1) --- updated-dependencies: - dependency-name: github.com/spf13/cast dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * enable no-httpx when passive scan is launched (#3789) * chore(deps): bump github.com/projectdiscovery/fastdialer from 0.0.26 to 0.0.28 in /v2 (#3779) * chore(deps): bump github.com/projectdiscovery/fastdialer in /v2 Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.26 to 0.0.28. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](projectdiscovery/fastdialer@v0.0.26...v0.0.28) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * Bump retryabledns to 0.28 * Update the retryabledns --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> * deprecatedProtocolNameTemplates concurrent map writes (#3785) * deprecatedProtocolNameTemplates * use syncLock * fix lint error * change version in deprecated warning msg * comment asnmap expand unit test --------- Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> * Issue 3339 headless fuzz (#3790) * Basic headless fuzzing * Remove debug statements * Add integration tests * Update template * Fix recognize payload value in matcher * Update tempalte * use req.SetURL() --------- Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> * Auto Generate Syntax Docs + JSONSchema [Fri Jun 9 00:23:32 UTC 2023] :robot: * Add headless header and status matchers (#3794) * add headless header and status matchers * rename headers as header * add integration test for header+status * fix typo * chore(deps): bump golang from 1.20.4-alpine to 1.20.5-alpine (#3809) Bumps golang from 1.20.4-alpine to 1.20.5-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/go-playground/validator/v10 in /v2 (#3810) Bumps [github.com/go-playground/validator/v10](https://github.com/go-playground/validator) from 10.11.2 to 10.14.1. - [Release notes](https://github.com/go-playground/validator/releases) - [Commits](go-playground/validator@v10.11.2...v10.14.1) --- updated-dependencies: - dependency-name: github.com/go-playground/validator/v10 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/rawhttp in /v2 (#3811) Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.11 to 0.1.13. - [Release notes](https://github.com/projectdiscovery/rawhttp/releases) - [Commits](projectdiscovery/rawhttp@v0.1.11...v0.1.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/rawhttp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/go-git/go-git/v5 from 5.6.1 to 5.7.0 in /v2 (#3812) Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.6.1 to 5.7.0. - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](go-git/go-git@v5.6.1...v5.7.0) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/hmap in /v2 (#3781) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.11 to 0.0.13. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](projectdiscovery/hmap@v0.0.11...v0.0.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Using safe dereferencing * adding comment * fixing and condition * fixing test id * adding integration test * update goflags dependency * update goflags dependency * bump goflags v0.1.9 => v0.1.10 * handle failure matcher flags logic at executor itself * add integration test to matcher status per request * Adding random tls impersonate (#3844) * adding random tls impersonate * dep update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> * Use templateman enhance api to populate CVE info (#3788) * use templateman enhance api to populate cve info * rename cve-annotate => tmc add additional flags to format, lint and enhance template using templateman apis * minior changes * remove duplicate code * misc update * Add validate and error log option * print if updated * print format and enhance only if updated * make max-request optional * fix reference unmarshal error * fix removing self-contained tag --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> * enable asnmap unit test --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Ramana Reddy <ramanaredy.manda@gmail.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> Co-authored-by: 三米前有蕉皮 <kali-team@qq.com> Co-authored-by: Shubham Rasal <shubham@projectdiscovery.io> Co-authored-by: GitHub Action <action@github.com> Co-authored-by: Mzack9999 <mzack9999@protonmail.com> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
projectdiscovery · Jun 27, 2023 · 878580d · 878580d
1 parent c851a6f
commit 878580d
Show file tree

Hide file tree

Showing 9 changed files with 181 additions and 43 deletions.
diff --git a/integration_tests/http/matcher-status-per-request.yaml b/integration_tests/http/matcher-status-per-request.yaml
@@ -0,0 +1,40 @@
+id: matcher-status-per-request
+
+info:
+  name: Test Matcher Status Per Request
+  author: pdteam
+  severity: critical
+
+variables:
+  username: test
+  password: admin
+  date: 2023-05-31
+
+http:
+  - method: GET
+    path:
+      - "{{RootURL}}/login?username={{username}}&password={{password}}"
+      - "{{BaseURL}}/admin-pannel"
+
+  - method: GET
+    path:
+      - "{{BaseURL}}/dashboard?date={{date}}"
+      - "{{BaseURL}}/signup"
+
+  - method: POST
+    path:
+      - "{{BaseURL}}/filemanager/upload.php"
+    body: "fldr=&url=file:///etc/passwd"
+
+
+    stop-at-first-match: true
+    matchers-condition: and
+    matchers:
+      - type: word
+        part: body
+        words:
+          - "matcher status"
+
+      - type: status
+        status:
+          - 200
diff --git a/integration_tests/http/matcher-status.yaml b/integration_tests/http/matcher-status.yaml
@@ -0,0 +1,40 @@
+id: matchet-status
+
+info:
+  name: Test Matcher Status
+  author: pdteam
+  severity: critical
+
+variables:
+  username: test
+  password: admin
+  date: 2023-05-31
+
+http:
+  - method: GET
+    path:
+      - "{{RootURL}}/login?username={{username}}&password={{password}}"
+      - "{{BaseURL}}/admin-pannel"
+
+  - method: GET
+    path:
+      - "{{BaseURL}}/dashboard?date={{date}}"
+      - "{{BaseURL}}/signup"
+
+  - method: POST
+    path:
+      - "{{BaseURL}}/filemanager/upload.php"
+    body: "fldr=&url=file:///etc/passwd"
+
+
+    stop-at-first-match: true
+    matchers-condition: and
+    matchers:
+      - type: word
+        part: body
+        words:
+          - "matcher status"
+
+      - type: status
+        status:
+          - 200
diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go
@@ -78,6 +78,8 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/cl-body-with-header.yaml":                 &httpCLBodyWithHeader{},
 	"http/save-extractor-values-to-file.yaml":       &httpSaveExtractorValuesToFile{},
 	"http/cli-with-constants.yaml":                  &ConstantWithCliVar{},
+	"http/matcher-status.yaml":                      &matcherStatusTest{},
+	"http/matcher-status-per-request.yaml":          &matcherStatusPerRequestTest{},
 	"http/disable-path-automerge.yaml":              &httpDisablePathAutomerge{},
 }
 
@@ -1425,6 +1427,42 @@ func (h *ConstantWithCliVar) Execute(filePath string) error {
 	return expectResultsCount(got, 1)
 }
 
+type matcherStatusTest struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *matcherStatusTest) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-ms")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
+type matcherStatusPerRequestTest struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *matcherStatusPerRequestTest) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.WriteHeader(http.StatusOK)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-msr")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 5)
+}
+
 // disable path automerge in raw request
 type httpDisablePathAutomerge struct{}
 

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
@@ -175,6 +175,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"),
 		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"),
 		flagSet.BoolVarP(&options.MatcherStatus, "matcher-status", "ms", false, "display match failure status"),
+		flagSet.BoolVarP(&options.MatchStatusPerRequest, "matcher-status-request", "msr", false, "display match failure status per request"),
 		flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
 		flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
 		flagSet.StringVarP(&options.JSONExport, "json-export", "je", "", "file to export results in JSON format"),

diff --git a/v2/pkg/core/inputs/hybrid/hmap_test.go b/v2/pkg/core/inputs/hybrid/hmap_test.go
@@ -155,15 +155,14 @@ func Test_expandASNInputValue(t *testing.T) {
 		asn                string
 		expectedOutputFile string
 	}{
-		// {
-		// 	asn:                "AS14421",
-		// 	expectedOutputFile: "tests/AS14421.txt",
-		// },
-		// skipping since there is a issue with ASN lookup for AS134029
-		// {
-		// 	asn:                "AS134029",
-		// 	expectedOutputFile: "tests/AS134029.txt",
-		// },
+		{
+			asn:                "AS14421",
+			expectedOutputFile: "tests/AS14421.txt",
+		},
+		{
+			asn:                "AS134029",
+			expectedOutputFile: "tests/AS134029.txt",
+		},
 	}
 	for _, tt := range tests {
 		hm, err := hybrid.New(hybrid.DefaultDiskOptions)

diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go
@@ -45,19 +45,19 @@ type Writer interface {
 
 // StandardWriter is a writer writing output to file and screen for results.
 type StandardWriter struct {
-	json             bool
-	jsonReqResp      bool
-	timestamp        bool
-	noMetadata       bool
-	matcherStatus    bool
-	mutex            *sync.Mutex
-	aurora           aurora.Aurora
-	outputFile       io.WriteCloser
-	traceFile        io.WriteCloser
-	errorFile        io.WriteCloser
-	severityColors   func(severity.Severity) string
-	storeResponse    bool
-	storeResponseDir string
+	json                  bool
+	jsonReqResp           bool
+	timestamp             bool
+	noMetadata            bool
+	matcherStatus         bool
+	mutex                 *sync.Mutex
+	aurora                aurora.Aurora
+	outputFile            io.WriteCloser
+	traceFile             io.WriteCloser
+	errorFile             io.WriteCloser
+	severityColors        func(severity.Severity) string
+	storeResponse         bool
+	storeResponseDir      string
 }
 
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@@ -187,20 +187,21 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
 			gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", options.StoreResponseDir, err)
 		}
 	}
+
 	writer := &StandardWriter{
-		json:             options.JSONL,
-		jsonReqResp:      options.JSONRequests,
-		noMetadata:       options.NoMeta,
-		matcherStatus:    options.MatcherStatus,
-		timestamp:        options.Timestamp,
-		aurora:           auroraColorizer,
-		mutex:            &sync.Mutex{},
-		outputFile:       outputFile,
-		traceFile:        traceOutput,
-		errorFile:        errorOutput,
-		severityColors:   colorizer.New(auroraColorizer),
-		storeResponse:    options.StoreResponse,
-		storeResponseDir: options.StoreResponseDir,
+		json:                  options.JSONL,
+		jsonReqResp:           options.JSONRequests,
+		noMetadata:            options.NoMeta,
+		matcherStatus:         options.MatcherStatus || options.MatchStatusPerRequest,
+		timestamp:             options.Timestamp,
+		aurora:                auroraColorizer,
+		mutex:                 &sync.Mutex{},
+		outputFile:            outputFile,
+		traceFile:             traceOutput,
+		errorFile:             errorOutput,
+		severityColors:        colorizer.New(auroraColorizer),
+		storeResponse:         options.StoreResponse,
+		storeResponseDir:      options.StoreResponseDir,
 	}
 	return writer, nil
 }

diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go
@@ -70,6 +70,23 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
 		})
 	}
 	previous := make(map[string]interface{})
+
+	var lastMatcherEvent *output.InternalWrappedEvent
+	writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatusPerReq, matcherStatus bool) {
+		if matcherStatusPerReq || matcherStatus {
+			results.CompareAndSwap(false, true)
+		}
+		if matcherStatus {
+			lastMatcherEvent = event
+			return
+		}
+		if matcherStatusPerReq {
+			if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil {
+				gologger.Warning().Msgf("Could not write failure event to output: %s\n", err)
+			}
+		}
+	}
+
 	for _, req := range e.requests {
 		inputItem := input.Clone()
 		if e.options.InputHelper != nil && input.MetaInput.Input != "" {
@@ -93,17 +110,13 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
 			// If no results were found, and also interactsh is not being used
 			// in that case we can skip it, otherwise we've to show failure in
 			// case of matcher-status flag.
-			if !event.HasOperatorResult() && !event.UsesInteractsh {
-				if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil {
-					gologger.Warning().Msgf("Could not write failure event to output: %s\n", err)
-				}
+			if !event.HasOperatorResult() && !event.UsesInteractsh && e.options.Options.MatchStatusPerRequest {
+				writeFailureCallback(event, e.options.Options.MatchStatusPerRequest, e.options.Options.MatcherStatus)
 			} else {
 				if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) {
 					results.CompareAndSwap(false, true)
 				} else {
-					if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil {
-						gologger.Warning().Msgf("Could not write failure event to output: %s\n", err)
-					}
+					writeFailureCallback(event, e.options.Options.MatchStatusPerRequest, e.options.Options.MatcherStatus)
 				}
 			}
 		})
@@ -118,6 +131,10 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
 			break
 		}
 	}
+
+	if lastMatcherEvent != nil {
+		writeFailureCallback(lastMatcherEvent, true, false)
+	}
 	return results.Load(), nil
 }
 

diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go
@@ -63,7 +63,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
 	if debug {
 		fmt.Println(string(data))
 	}
-	if err != nil {
+	if len(data) < 1 && err != nil {
 		return nil, fmt.Errorf("%v: %v", err.Error(), string(data))
 	}
 	var parts []string

diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go
@@ -268,6 +268,8 @@ type Options struct {
 	EnvironmentVariables bool
 	// MatcherStatus displays optional status for the failed matches as well
 	MatcherStatus bool
+	// MatchStatusPerRequest displays optional status for the failed matches per request
+	MatchStatusPerRequest bool
 	// ClientCertFile client certificate file (PEM-encoded) used for authenticating against scanned hosts
 	ClientCertFile string
 	// ClientKeyFile client key file (PEM-encoded) used for authenticating against scanned hosts