Merge branch 'dev'

cmd/integration-test/code.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"log"
+	"os"
 	"path/filepath"
 
 	osutils "github.com/projectdiscovery/utils/os"
@@ -12,14 +13,16 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
+var isCodeDisabled = func() bool { return osutils.IsWindows() && os.Getenv("CI") == "true" }
+
 var codeTestCases = []TestCaseInfo{
-	{Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}},
-	{Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}},
-	{Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}},
-	{Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}},
-	{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}},
-	{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}},
-	{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() }},
+	{Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
+	{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }},
 }
 
 const (
@@ -30,6 +33,10 @@ const (
 var testcertpath = ""
 
 func init() {
+	if isCodeDisabled() {
+		// skip executing code protocol in CI on windows
+		return
+	}
 	// allow local file access to load content of file references in template
 	// in order to sign them for testing purposes
 	templates.TemplateSignerLFA()

cmd/integration-test/flow.go

@@ -15,6 +15,7 @@ var flowTestcases = []TestCaseInfo{
 	{Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}},
 	{Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}},
 	{Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}},
+	{Path: "flow/flow-hide-matcher.yaml", TestCase: &flowHideMatcher{}},
 }
 
 type conditionalFlow struct{}
@@ -24,7 +25,7 @@ func (t *conditionalFlow) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 2)
+	return expectResultsCount(results, 1)
 }
 
 type conditionalFlowNegative struct{}
@@ -66,7 +67,7 @@ func (t *iterateValuesFlow) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 2)
+	return expectResultsCount(results, 1)
 }
 
 type dnsNsProbe struct{}
@@ -76,9 +77,20 @@ func (t *dnsNsProbe) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 3)
+	return expectResultsCount(results, 1)
 }
 
 func getBase64(input string) string {
 	return base64.StdEncoding.EncodeToString([]byte(input))
 }
+
+type flowHideMatcher struct{}
+
+func (t *flowHideMatcher) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	// this matcher should not return any results
+	return expectResultsCount(results, 0)
+}

cmd/integration-test/headless.go

@@ -12,6 +12,7 @@ import (
 
 var headlessTestcases = []TestCaseInfo{
 	{Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessBasic{}},
+	{Path: "protocols/headless/headless-self-contained.yaml", TestCase: &headlessSelfContained{}},
 	{Path: "protocols/headless/headless-header-action.yaml", TestCase: &headlessHeaderActions{}},
 	{Path: "protocols/headless/headless-extract-values.yaml", TestCase: &headlessExtractValues{}},
 	{Path: "protocols/headless/headless-payloads.yaml", TestCase: &headlessPayloads{}},
@@ -41,6 +42,18 @@ func (h *headlessBasic) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type headlessSelfContained struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *headlessSelfContained) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-headless", "-var query=selfcontained")
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
 type headlessLocal struct{}
 
 // Execute executes a test case and returns an error if occurred

cmd/integration-test/javascript.go

@@ -14,6 +14,7 @@ var jsTestcases = []TestCaseInfo{
 	{Path: "protocols/javascript/redis-pass-brute.yaml", TestCase: &javascriptRedisPassBrute{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
 	{Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
 	{Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}},
+	{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}},
 }
 
 var (
@@ -23,6 +24,16 @@ var (
 	defaultRetry  = 3
 )
 
+type javascriptNetHttps struct{}
+
+func (j *javascriptNetHttps) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
 type javascriptRedisPassBrute struct{}
 
 func (j *javascriptRedisPassBrute) Execute(filePath string) error {

cmd/integration-test/network.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"fmt"
 	"net"
+	"os"
 	"strings"
+	"time"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 	osutils "github.com/projectdiscovery/utils/os"
+	"github.com/projectdiscovery/utils/reader"
 )
 
 var networkTestcases = []TestCaseInfo{
@@ -16,6 +20,8 @@ var networkTestcases = []TestCaseInfo{
 	{Path: "protocols/network/variables.yaml", TestCase: &networkVariables{}},
 	{Path: "protocols/network/same-address.yaml", TestCase: &networkBasic{}},
 	{Path: "protocols/network/network-port.yaml", TestCase: &networkPort{}},
+	{Path: "protocols/network/net-https.yaml", TestCase: &networkhttps{}},
+	{Path: "protocols/network/net-https-timeout.yaml", TestCase: &networkhttps{}},
 }
 
 const defaultStaticPort = 5431
@@ -29,22 +35,26 @@ func (h *networkBasic) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
+		if err != nil {
 			routerErr = err
 			return
 		}
 		if string(data) == "PING" {
 			_, _ = conn.Write([]byte("PONG"))
+		} else {
+			routerErr = fmt.Errorf("invalid data received: %s", string(data))
 		}
 	})
 	defer ts.Close()
 
 	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
 	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not run nuclei: %s\n", err)
 		return err
 	}
 	if routerErr != nil {
+		fmt.Fprintf(os.Stderr, "routerErr: %s\n", routerErr)
 		return routerErr
 	}
 
@@ -60,17 +70,17 @@ func (h *networkMultiStep) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 5)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 5, time.Duration(5)*time.Second)
+		if err != nil {
 			routerErr = err
 			return
 		}
 		if string(data) == "FIRST" {
 			_, _ = conn.Write([]byte("PING"))
 		}
 
-		data = make([]byte, 6)
-		if _, err := conn.Read(data); err != nil {
+		data, err = reader.ConnReadNWithTimeout(conn, 6, time.Duration(5)*time.Second)
+		if err != nil {
 			routerErr = err
 			return
 		}
@@ -126,8 +136,8 @@ func (h *networkVariables) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
+		if err != nil {
 			routerErr = err
 			return
 		}
@@ -154,8 +164,8 @@ func (n *networkPort) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
+		if err != nil {
 			return
 		}
 		if string(data) == "PING" {
@@ -187,8 +197,8 @@ func (n *networkPort) Execute(filePath string) error {
 	ts2 := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
-		if _, err := conn.Read(data); err != nil {
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
+		if err != nil {
 			return
 		}
 		if string(data) == "PING" {
@@ -206,3 +216,14 @@ func (n *networkPort) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type networkhttps struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *networkhttps) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}

cmd/nuclei/main.go

@@ -16,9 +16,9 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/interactsh/pkg/client"
-	"github.com/projectdiscovery/nuclei/v3/internal/installer"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"

docs/getting-started/install.mdx

@@ -9,7 +9,7 @@ title: 'Install'
     go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
     ```
 
-    <Note>Nuclei require latest **GO** version to install successfully.</Note>
+    <Note>Nuclei requires latest **GO** version to install successfully.</Note>
 
   </Tab>
   <Tab title="Brew">
@@ -66,4 +66,4 @@ title: 'Install'
     </Tip>
 
 </Tab>
-</Tabs>
+</Tabs>

docs/template-guide/flow.mdx

@@ -34,9 +34,9 @@ http:
       - "{{BaseURL}}/wp-login.php"
 
     matchers:
-        - type: word
-            words:
-            - "WordPress"
+      - type: word
+        words:
+          - "WordPress"
 
   - method: POST
     path:

go.mod

@@ -21,12 +21,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.19
-	github.com/projectdiscovery/fastdialer v0.0.40
-	github.com/projectdiscovery/hmap v0.0.22
+	github.com/projectdiscovery/fastdialer v0.0.42
+	github.com/projectdiscovery/hmap v0.0.23
 	github.com/projectdiscovery/interactsh v1.1.7
 	github.com/projectdiscovery/rawhttp v0.1.23
-	github.com/projectdiscovery/retryabledns v1.0.39
-	github.com/projectdiscovery/retryablehttp-go v1.0.32
+	github.com/projectdiscovery/retryabledns v1.0.40
+	github.com/projectdiscovery/retryablehttp-go v1.0.33
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -77,7 +77,7 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/praetorian-inc/fingerprintx v1.1.9
-	github.com/projectdiscovery/dsl v0.0.26
+	github.com/projectdiscovery/dsl v0.0.27
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/goflags v0.1.25
 	github.com/projectdiscovery/gologger v1.1.11
@@ -86,12 +86,12 @@ require (
 	github.com/projectdiscovery/httpx v1.3.5
 	github.com/projectdiscovery/mapcidr v1.1.12
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/ratelimit v0.0.12
+	github.com/projectdiscovery/ratelimit v0.0.13
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766
 	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/utils v0.0.58
+	github.com/projectdiscovery/utils v0.0.62
 	github.com/projectdiscovery/wappalyzergo v0.0.109
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02
@@ -166,7 +166,7 @@ require (
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect
-	github.com/minio/selfupdate v0.6.0 // indirect
+	github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/term v0.5.0 // indirect