Merge branch 'dev' into issue-4016-data-race

README.md

@@ -126,9 +126,9 @@ TEMPLATES:
    -ntv, -new-templates-version string[]  run new templates added in specific version
    -as, -automatic-scan                   automatic web scan using wappalyzer technology detection to tags mapping
    -t, -templates string[]                list of template or template directory to run (comma-separated, file)
-   -tu, -template-url string[]            list of template urls to run (comma-separated, file)
+   -turl, -template-url string[]          template url or list containing template urls to run (comma-separated, file)
    -w, -workflows string[]                list of workflow or workflow directory to run (comma-separated, file)
-   -wu, -workflow-url string[]            list of workflow urls to run (comma-separated, file)
+   -wurl, -workflow-url string[]          workflow url or list containing workflow urls to run (comma-separated, file)
    -validate                              validate the passed templates to nuclei
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content

docs/editor/ai.mdx

@@ -6,14 +6,11 @@ title: 'AI Assistance'
 
 <img src="/images/ai.jpg" alt="AI Prompt" width="800px"/>
 
-[Nuclei Template Editor](https://templates.nuclei.sh/) employs AI to generate templates for vulnerability reports. This document seeks to guide you through the process, offering you usage tips and examples.
+[Nuclei Template Editor](https://templates.nuclei.sh/) has AI to generate templates for vulnerability reports. This document helps to guide you through the process, offering you usage tips and examples.
 
 ## Overview
 
-Powered by public Nuclei templates and a rich CVE data set, the AI understands a broad array of security vulnerabilities. It operates following these steps:
-
-1.  **Interpret a Prompt**: Analyzes a detailed prompt outlining a specific vulnerability.
-2.  **Generate a Template**: Creates a Nuclei template using the AI API.
+Powered by public Nuclei templates and a rich CVE data set, the AI understands a broad array of security vulnerabilities. First, the system interprets the user's prompt to identify a specific vulnerability. Then, it generates a template based on the steps required to reproduce the vulnerability along with all the necessary meta information to reproduce and remediate.
 
 ---
 

docs/template-guide/operators/matchers.mdx

@@ -8,7 +8,7 @@ Matchers allow different type of flexible comparisons on protocol responses. The
 
 ### Types
 
-Multiple matchers can be specified in a request. There are basically 6 types of matchers:
+Multiple matchers can be specified in a request. There are basically 7 types of matchers:
 
 | Matcher Type | Part Matched                |
 |--------------|-----------------------------|
@@ -18,6 +18,7 @@ Multiple matchers can be specified in a request. There are basically 6 types of
 | regex        | Part for a protocol         |
 | binary       | Part for a protocol         |
 | dsl          | Part for a protocol         |
+| xpath        | Part for a protocol         |
 
 To match status codes for responses, you can use the following syntax.
 
@@ -57,6 +58,16 @@ matchers:
 
 **Word** and **Regex** matchers can be further configured depending on the needs of the users.
 
+**XPath** matchers use XPath queries to match XML and HTML responses. If the XPath query returns any results, it's considered a match.
+
+```yaml
+matchers:
+  - type: xpath
+    part: body
+    xpath:
+      - "/html/head/title[contains(text(), 'Example Domain')]"
+```
+
 Complex matchers of type **dsl** allows building more elaborate expressions with helper functions. These function allow access to Protocol Response which contains variety of data based on each protocol. See protocol specific documentation to learn about different returned results.
 
 

integration_tests/headless/file-upload-negative.yaml

@@ -0,0 +1,29 @@
+id: file-upload
+# template for testing when file upload is disabled
+info:
+  name: Basic File Upload
+  author: pdteam
+  severity: info
+
+headless:
+  - steps:
+    - action: navigate
+      args:
+        url: "{{BaseURL}}"
+    - action: waitload
+    - action: files
+      args:
+        by: xpath
+        xpath: /html/body/form/input[1]
+        value: headless/file-upload.yaml
+    - action: sleep
+      args:
+        duration: 2
+    - action: click
+      args:
+        by: x
+        xpath: /html/body/form/input[2]
+    matchers:
+      - type: word
+        words:
+          - "Basic File Upload"

integration_tests/headless/headless-local.yaml

@@ -0,0 +1,15 @@
+id: nuclei-headless-local
+
+info:
+  name: Nuclei Headless Local
+  author: pdteam
+  severity: high
+
+headless:
+  - steps: 
+    - action: navigate
+      args:
+        url: "{{BaseURL}}"
+
+    - action: waitload
+

nuclei-jsonschema.json

@@ -342,6 +342,14 @@
           "title": "dsl expressions to match in response",
           "description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules"
         },
+        "xpath": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "title": "xpath queries to match in response",
+          "description": "xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules"
+        },
         "encoding": {
           "enum": [
             "hex"
@@ -371,7 +379,8 @@
         "binary",
         "status",
         "size",
-        "dsl"
+        "dsl",
+        "xpath"
       ],
       "type": "string",
       "title": "type of the matcher",

v2/cmd/integration-test/headless.go

@@ -16,7 +16,9 @@ var headlessTestcases = []TestCaseInfo{
 	{Path: "headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}},
 	{Path: "headless/headless-payloads.yaml", TestCase: &headlessPayloads{}},
 	{Path: "headless/variables.yaml", TestCase: &headlessVariables{}},
+	{Path: "headless/headless-local.yaml", TestCase: &headlessLocal{}},
 	{Path: "headless/file-upload.yaml", TestCase: &headlessFileUpload{}},
+	{Path: "headless/file-upload-negative.yaml", TestCase: &headlessFileUploadNegative{}},
 	{Path: "headless/headless-header-status-test.yaml", TestCase: &headlessHeaderStatus{}},
 }
 
@@ -39,6 +41,27 @@ func (h *headlessBasic) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type headlessLocal struct{}
+
+// Execute executes a test case and returns an error if occurred
+// in this testcases local network access is disabled
+func (h *headlessLocal) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = w.Write([]byte("<html><body></body></html>"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	args := []string{"-t", filePath, "-u", ts.URL, "-headless", "-lna"}
+
+	results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 0)
+}
+
 type headlessHeaderActions struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -171,3 +194,48 @@ func (h *headlessHeaderStatus) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type headlessFileUploadNegative struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *headlessFileUploadNegative) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = w.Write([]byte(`
+		<!doctype html>
+			<body>
+				<form method=post enctype=multipart/form-data>
+				<input type=file name=file>
+				<input type=submit value=Upload>
+				</form>
+			</body>
+		</html>
+		`))
+	})
+	router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		file, _, err := r.FormFile("file")
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		defer file.Close()
+
+		content, err := io.ReadAll(file)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		_, _ = w.Write(content)
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+	args := []string{"-t", filePath, "-u", ts.URL, "-headless"}
+
+	results, err := testutils.RunNucleiWithArgsAndGetResults(debug, args...)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 0)
+}

v2/cmd/integration-test/interactsh.go

@@ -6,6 +6,6 @@ import osutils "github.com/projectdiscovery/utils/os"
 var interactshTestCases = []TestCaseInfo{
 	{Path: "http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
 	{Path: "http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
-	{Path: "http/default-matcher-condition.yaml", TestCase: &httpDefaultMatcherCondition{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
+	{Path: "http/default-matcher-condition.yaml", TestCase: &httpDefaultMatcherCondition{}, DisableOn: func() bool { return true }}, // disable this test for now
 	{Path: "http/interactsh-requests-mc-and.yaml", TestCase: &httpInteractshRequestsWithMCAnd{}},
 }

v2/cmd/integration-test/loader.go

@@ -55,7 +55,7 @@ func (h *remoteTemplateList) Execute(templateList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -112,7 +112,7 @@ func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
 	if err == nil {
 		return fmt.Errorf("expected error for not allowed remote template list url")
 	}
@@ -154,7 +154,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error {
 	}
 	defer os.Remove("test-config.yaml")
 
-	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list", "-config", "test-config.yaml")
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
 	if err != nil {
 		return err
 	}
@@ -170,7 +170,7 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-template-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}
@@ -186,7 +186,7 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404")
+	_, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-workflow-url", ts.URL+"/404")
 	if err == nil {
 		return fmt.Errorf("expected error for nonexisting workflow url")
 	}

v2/cmd/nuclei/main.go

@@ -137,14 +137,14 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
 		flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", nil, "list of template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", nil, "list of workflow or workflow directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
-		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", nil, "list of workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wurl", nil, "workflow url or list containing workflow urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"),
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
-		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"api.nuclei.sh"}, "allowed domain list to load remote templates from"),
+		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"templates.nuclei.sh"}, "allowed domain list to load remote templates from"),
 	)
 
 	flagSet.CreateGroup("filters", "Filtering",
@@ -341,9 +341,9 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)
 
-	if options.Silent {
+	if options.VerboseVerbose {
 		// hide release notes if silent mode is enabled
-		installer.HideReleaseNotes = true
+		installer.HideReleaseNotes = false
 	}
 
 	if options.LeaveDefaultPorts {