Merge pull request #3021 from projectdiscovery/dev

Bugfix release

.github/workflows/publish-docs.yaml

@@ -10,12 +10,8 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out code
+      - name: Checkout code
         uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
-          fetch-depth: 0
-          token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: "Set up Go"
         uses: actions/setup-go@v3

README.md

@@ -98,7 +98,7 @@ TARGET:
    -u, -target string[]       target URLs/hosts to scan
    -l, -list string           path to file containing a list of target URLs/hosts to scan (one per line)
    -resume string             resume scan using resume.cfg (clustering will be disabled)
-   -sa, -scan-all-ips         scan all the IPs associated with dns record
+   -sa, -scan-all-ips         scan all the IP's associated with dns record
    -iv, -ip-version string[]  IP version to scan of hostname (4,6) - (default 4)
 
 TEMPLATES:
@@ -126,8 +126,8 @@ FILTERING:
    -em, -exclude-matchers string[]    template matchers to exclude in result
    -s, -severity value[]              templates to run based on severity. Possible values: info, low, medium, high, critical, unknown
    -es, -exclude-severity value[]     templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown
-   -pt, -type value[]                 templates to run based on protocol type. Possible values: dns, file, http, , headless, network, workflow, ssl, websocket, whois
-   -ept, -exclude-type value[]        templates to exclude based on protocol type. Possible values: dns, file, http, , headless, network, workflow, ssl, websocket, whois
+   -pt, -type value[]                 templates to run based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
+   -ept, -exclude-type value[]        templates to exclude based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
    -tc, -template-condition string[]  templates to run based on expression condition
 
 OUTPUT:
@@ -139,7 +139,7 @@ OUTPUT:
    -json                         write output in JSONL(ines) format
    -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
    -nm, -no-meta                 disable printing result metadata in cli output
-   -ts, -timestamp               enable printing timestamp in cli output
+   -ts, -timestamp               enables printing timestamp in cli output
    -rdb, -report-db string       nuclei reporting database (always use this to persist report data)
    -ms, -matcher-status          display match failure status
    -me, -markdown-export string  directory to export results in markdown format
@@ -156,6 +156,7 @@ CONFIGURATIONS:
    -V, -var value                 custom vars in key=value format
    -r, -resolvers string          file containing resolver list for nuclei
    -sr, -system-resolvers         use system DNS resolving as error fallback
+   -dc, -disable-clustering       disable clustering of requests
    -passive                       enable passive HTTP response processing mode
    -fh2, -force-http2             force http2 connection on requests
    -ev, -env-vars                 enable environment variables to be used in template
@@ -171,7 +172,7 @@ CONFIGURATIONS:
    -sip, -source-ip string        source ip address to use for network scan
    -config-directory string       override the default config path ($home/.config)
    -rsr, -response-size-read int  max response size to read in bytes (default 10485760)
-   -rss, -response-size-save int  max response size to save in bytes (default 10485760)
+   -rss, -response-size-save int  max response size to save in bytes (default 1048576)
 
 INTERACTSH:
    -iserver, -interactsh-server string  interactsh server url for self-hosted instance (default: oast.pro,oast.live,oast.site,oast.online,oast.fun,oast.me)

v2/cmd/integration-test/http.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
@@ -19,7 +18,7 @@ import (
 )
 
 var httpTestcases = map[string]testutils.TestCase{
-	"http/raw-unsafe-request.yaml":                  &httpRawUnsafeRequest{},
+	//"http/raw-unsafe-request.yaml":                  &httpRawUnsafeRequest{},
 	"http/get-headers.yaml":                         &httpGetHeaders{},
 	"http/get-query-string.yaml":                    &httpGetQueryString{},
 	"http/get-redirects.yaml":                       &httpGetRedirects{},
@@ -614,28 +613,27 @@ func (h *httpRawCookieReuse) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
-type httpRawUnsafeRequest struct{}
-
+// type httpRawUnsafeRequest struct{
 // Execute executes a test case and returns an error if occurred
-func (h *httpRawUnsafeRequest) Execute(filePath string) error {
-	var routerErr error
-
-	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
-		defer conn.Close()
-		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 36\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nThis is test raw-unsafe-matcher test"))
-	})
-	defer ts.Close()
-
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "http://"+ts.URL, debug)
-	if err != nil {
-		return err
-	}
-	if routerErr != nil {
-		return routerErr
-	}
-
-	return expectResultsCount(results, 1)
-}
+// func (h *httpRawUnsafeRequest) Execute(filePath string) error {
+// 	var routerErr error
+//
+// 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
+// 		defer conn.Close()
+// 		_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 36\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nThis is test raw-unsafe-matcher test"))
+// 	})
+// 	defer ts.Close()
+//
+// 	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "http://"+ts.URL, debug)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	if routerErr != nil {
+// 		return routerErr
+// 	}
+//
+// 	return expectResultsCount(results, 1)
+// }
 
 type httpRequestCondition struct{}
 

v2/cmd/nuclei/main.go

@@ -189,6 +189,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in key=value format"),
 		flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
 		flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"),
+		flagSet.BoolVarP(&options.DisableClustering, "disable-clustering", "dc", false, "disable clustering of requests"),
 		flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"),
 		flagSet.BoolVarP(&options.ForceAttemptHTTP2, "force-http2", "fh2", false, "force http2 connection on requests"),
 		flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"),

v2/internal/runner/templates.go

@@ -2,7 +2,7 @@ package runner
 
 import (
 	"bytes"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -46,7 +46,7 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 				colorize := !r.options.NoColor
 
 				path := tpl.Path
-				tplBody, err := ioutil.ReadFile(path)
+				tplBody, err := os.ReadFile(path)
 				if err != nil {
 					gologger.Error().Msgf("Could not read the template %s: %s", path, err)
 					continue

v2/pkg/catalog/config/config.go

@@ -32,7 +32,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.8.2`
+const Version = `2.8.3`
 
 var customConfigDirectory string
 

v2/pkg/protocols/common/contextargs/variables.go

@@ -0,0 +1,9 @@
+package contextargs
+
+// GenerateVariables from context args
+func GenerateVariables(ctx *Context) map[string]interface{} {
+	vars := map[string]interface{}{
+		"ip": ctx.MetaInput.CustomIP,
+	}
+	return vars
+}

v2/pkg/protocols/common/updatecheck/client.go

@@ -3,7 +3,6 @@ package updatecheck
 import (
 	"context"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"time"
@@ -54,7 +53,7 @@ func GetLatestIgnoreFile() ([]byte, error) {
 	}
 	defer body.Close()
 
-	data, err := ioutil.ReadAll(body)
+	data, err := io.ReadAll(body)
 	if err != nil {
 		return nil, err
 	}

v2/pkg/protocols/headless/request.go

@@ -30,13 +30,13 @@ func (request *Request) Type() templateTypes.ProtocolType {
 }
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
+func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	inputURL := input.MetaInput.Input
 	if request.options.Browser.UserAgent() == "" {
 		request.options.Browser.SetUserAgent(request.compiledUserAgent)
 	}
 
-	vars := GenerateVariables(inputURL)
+	vars := utils.GenerateVariablesWithContextArgs(input, false)
 	payloads := generators.BuildPayloadFromOptions(request.options.Options)
 	values := generators.MergeMaps(vars, metadata, payloads)
 	variablesMap := request.options.Variables.Evaluate(values)
@@ -151,13 +151,3 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
 		gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse)
 	}
 }
-
-// GenerateVariables will create default variables
-func GenerateVariables(URL string) map[string]interface{} {
-	parsed, err := url.Parse(URL)
-	if err != nil {
-		return nil
-	}
-
-	return utils.GenerateVariables(parsed, false)
-}

v2/pkg/protocols/http/build_request.go

@@ -16,6 +16,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
@@ -59,7 +60,7 @@ func (g *generatedRequest) URL() string {
 
 // Make creates a http request for the provided input.
 // It returns io.EOF as error when all the requests have been exhausted.
-func (r *requestGenerator) Make(ctx context.Context, baseURL, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
+func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) {
 	if r.request.SelfContained {
 		return r.makeSelfContainedRequest(ctx, data, payloads, dynamicValues)
 	}
@@ -74,7 +75,7 @@ func (r *requestGenerator) Make(ctx context.Context, baseURL, data string, paylo
 		}
 	}
 
-	parsed, err := url.Parse(baseURL)
+	parsed, err := url.Parse(input.MetaInput.Input)
 	if err != nil {
 		return nil, err
 	}
@@ -94,7 +95,7 @@ func (r *requestGenerator) Make(ctx context.Context, baseURL, data string, paylo
 	}
 
 	values := generators.MergeMaps(
-		generators.MergeMaps(dynamicValues, utils.GenerateVariables(parsed, trailingSlash)),
+		generators.MergeMaps(dynamicValues, utils.GenerateVariablesWithURL(parsed, trailingSlash, contextargs.GenerateVariables(input))),
 		generators.BuildPayloadFromOptions(r.request.options.Options),
 	)
 	if vardump.EnableVarDump {
@@ -152,18 +153,16 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 			if err := expressions.ContainsVariablesWithIgnoreList(ignoreList, parts[1]); err != nil {
 				return nil, err
 			}
-		} else { // the url might contain placeholders
-			if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
-				return nil, err
-			}
+		} else if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil { // the url might contain placeholders
+			return nil, err
 		}
 
 		parsed, err := url.Parse(parts[1])
 		if err != nil {
 			return nil, fmt.Errorf("could not parse request URL: %w", err)
 		}
 		values = generators.MergeMaps(
-			generators.MergeMaps(dynamicValues, utils.GenerateVariables(parsed, false)),
+			generators.MergeMaps(dynamicValues, utils.GenerateVariablesWithURL(parsed, false, nil)),
 			values,
 		)
 

v2/pkg/protocols/http/build_request_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/model"
 	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
@@ -50,7 +51,7 @@ func TestMakeRequestFromModal(t *testing.T) {
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), "https://example.com", inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 
 	bodyBytes, _ := req.request.BodyBytes()
@@ -78,13 +79,13 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), "https://example.com/test.php", inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com/test.php"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
 
 	generator = request.newGenerator(false)
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), "https://example.com/test/", inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com/test/"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
 }
@@ -118,13 +119,13 @@ Accept-Encoding: gzip`},
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), "https://example.com", inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
 
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), "https://example.com", inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
@@ -159,13 +160,13 @@ Accept-Encoding: gzip`},
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), "https://example.com", inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
 
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), "https://example.com", inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")
@@ -203,7 +204,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
 	require.Nil(t, err, "could not create interactsh client")
 
 	inputData, payloads, _ := generator.nextValue()
-	got, err := generator.Make(context.Background(), "https://example.com", inputData, payloads, map[string]interface{}{})
+	got, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 
 	// check if all the interactsh markers are replaced with unique urls