Merge branch 'dev'

.gitignore

@@ -19,9 +19,9 @@ pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
 .gitignore
 pkg/js/devtools/bindgen/cmd/bindgen
 pkg/js/devtools/jsdocgen/jsdocgen
-./bindgen
-./jsdocgen
-./scrapefuncs
 *.DS_Store
 pkg/protocols/headless/engine/.cache
-./nuclei
+/nuclei
+/bindgen
+/jsdocgen
+/scrapefuncs

cmd/integration-test/loader.go

@@ -10,6 +10,7 @@ import (
 	"github.com/julienschmidt/httprouter"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+	errorutil "github.com/projectdiscovery/utils/errors"
 	permissionutil "github.com/projectdiscovery/utils/permission"
 )
 
@@ -20,6 +21,7 @@ var loaderTestcases = []TestCaseInfo{
 	{Path: "loader/nonexistent-template-list.yaml", TestCase: &nonExistentTemplateList{}},
 	{Path: "loader/nonexistent-workflow-list.yaml", TestCase: &nonExistentWorkflowList{}},
 	{Path: "loader/template-list-not-allowed.yaml", TestCase: &remoteTemplateListNotAllowed{}},
+	{Path: "loader/load-template-with-id", TestCase: &loadTemplateWithID{}},
 }
 
 type remoteTemplateList struct{}
@@ -193,3 +195,13 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error
 
 	return nil
 }
+
+type loadTemplateWithID struct{}
+
+func (h *loadTemplateWithID) Execute(nooop string) error {
+	results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", "scanme.sh", "-id", "self-signed-ssl")
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	return expectResultsCount(results, 1)
+}

cmd/nuclei/main.go

@@ -32,6 +32,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/monitor"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
+	updateutils "github.com/projectdiscovery/utils/update"
 )
 
 var (
@@ -158,6 +159,10 @@ func main() {
 }
 
 func readConfig() *goflags.FlagSet {
+
+	// when true updates nuclei binary to latest version
+	var updateNucleiBinary bool
+
 	flagSet := goflags.NewFlagSet()
 	flagSet.CaseSensitive = true
 	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
@@ -342,7 +347,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	)
 
 	flagSet.CreateGroup("update", "Update",
-		flagSet.CallbackVarP(runner.NucleiToolUpdateCallback, "update", "up", "update nuclei engine to the latest released version"),
+		flagSet.BoolVarP(&updateNucleiBinary, "update", "up", false, "update nuclei engine to the latest released version"),
 		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"),
 		flagSet.StringVarP(&options.NewTemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"),
 		flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"),
@@ -412,6 +417,14 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 		installer.HideReleaseNotes = false
 	}
 
+	if options.Timeout > 30 {
+		// default github binary/template download timeout is 30 sec
+		updateutils.DownloadUpdateTimeout = time.Duration(options.Timeout) * time.Second
+	}
+	if updateNucleiBinary {
+		runner.NucleiToolUpdateCallback()
+	}
+
 	if options.LeaveDefaultPorts {
 		http.LeaveDefaultPorts = true
 	}

docs/getting-started/running.mdx

@@ -402,10 +402,15 @@ STATISTICS:
    -stats                    display statistics about the running scan
    -sj, -stats-json          write statistics data to an output file in JSONL(ines) format
    -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
-   -m, -metrics              expose nuclei metrics on a port
    -mp, -metrics-port int    port to expose nuclei metrics on (default 9092)
 ```
 
+<Tip>
+  From Nuclei v3.0.0 `-metrics` port has been removed and merged with `-stats`
+  when using `-stats` flag metrics will be by default available at `localhost:9092/metrics`
+  and metrics-port can be configured by `-metrics-port` flag
+</Tip>
+
 ### Rate **Limits**
 
 Nuclei have multiple rate limit controls for multiple factors, including a number of templates to execute in parallel, a number of hosts to be scanned in parallel for each template, and the global number of request / per second you wanted to make/limit using nuclei, here is an example of each flag with description.

go.mod

@@ -24,9 +24,9 @@ require (
 	github.com/projectdiscovery/fastdialer v0.0.40
 	github.com/projectdiscovery/hmap v0.0.22
 	github.com/projectdiscovery/interactsh v1.1.7
-	github.com/projectdiscovery/rawhttp v0.1.22
+	github.com/projectdiscovery/rawhttp v0.1.23
 	github.com/projectdiscovery/retryabledns v1.0.39
-	github.com/projectdiscovery/retryablehttp-go v1.0.31
+	github.com/projectdiscovery/retryablehttp-go v1.0.32
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -79,7 +79,7 @@ require (
 	github.com/praetorian-inc/fingerprintx v1.1.9
 	github.com/projectdiscovery/dsl v0.0.26
 	github.com/projectdiscovery/fasttemplate v0.0.2
-	github.com/projectdiscovery/goflags v0.1.24
+	github.com/projectdiscovery/goflags v0.1.25
 	github.com/projectdiscovery/gologger v1.1.11
 	github.com/projectdiscovery/gostruct v0.0.1
 	github.com/projectdiscovery/gozero v0.0.1

go.sum

@@ -799,8 +799,8 @@ github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOi
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
 github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
-github.com/projectdiscovery/goflags v0.1.24 h1:eYMUgohu/pJLhA6e8iUYJ9DIiXn61FRzKbywV4iea4s=
-github.com/projectdiscovery/goflags v0.1.24/go.mod h1:qTJZXaPNaJ4jARphm2y09HpYUTYbWaW3gcoQnS6Lrmo=
+github.com/projectdiscovery/goflags v0.1.25 h1:nHXKo464RW7qzyeRMGEtHmsySWw2Cl1tqK5c5nlSnLk=
+github.com/projectdiscovery/goflags v0.1.25/go.mod h1:EhPEodSIWQkoPXtK80o2yY5xjfE+KNSLrKcLM02y9tw=
 github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8=
 github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY=
 github.com/projectdiscovery/gostruct v0.0.1 h1:1KvR6Pn4mDbQqoLEQzhRfHpbreLno2R9xqRCCt5tgmU=
@@ -821,14 +821,14 @@ github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzV
 github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk=
 github.com/projectdiscovery/ratelimit v0.0.12 h1:Hwn9D1PHDz4O4ndzFZCttvTqdtxjBJrifi7n4oa4/Og=
 github.com/projectdiscovery/ratelimit v0.0.12/go.mod h1:mA4775qZGakULYWDI0yswMntTWKQmEbFbb199uuGdzs=
-github.com/projectdiscovery/rawhttp v0.1.22 h1:tJiAUjgM7xtDadiLqpF76w/ddJxGCSFu5FfTFwKqIhk=
-github.com/projectdiscovery/rawhttp v0.1.22/go.mod h1:zZl88fEut97zuZco8gHaEQEOuYKN7F089/REo3C+J3Y=
+github.com/projectdiscovery/rawhttp v0.1.23 h1:j5QwJBR1G7Xx4lw62IsIf5+9F7Ywtt76DB6W2lMRJ+I=
+github.com/projectdiscovery/rawhttp v0.1.23/go.mod h1:VP3RbSitMZCeohVcxFGvW/LFMaX9H4GxZMiuiVwV8cU=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.39 h1:hvh/fNfKAB+JdOtLtJLo1/MO2nihTE/OWanGnIQp8Ds=
 github.com/projectdiscovery/retryabledns v1.0.39/go.mod h1:HaHABQBdiL1KoOQ99XqMRDC0RhkzkGfiV3QnUhYUeRg=
-github.com/projectdiscovery/retryablehttp-go v1.0.31 h1:Z66QM4FkAyIEBje4acu3hdUPaE3NSFYNa54hR69ZlPo=
-github.com/projectdiscovery/retryablehttp-go v1.0.31/go.mod h1:pFBFbxnb7fupJbl99n9M0GkoUZCtbkRZM3xfmTSWjqE=
+github.com/projectdiscovery/retryablehttp-go v1.0.32 h1:lWGi58MUH7LslgE0qc6DEkidoDMFI/nsRmHM/PdXe6Y=
+github.com/projectdiscovery/retryablehttp-go v1.0.32/go.mod h1:3fKRRexRsSmHmewYK1at8vyyLLj1coofDNw3EASFx9c=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=

pkg/catalog/config/constants.go

@@ -17,7 +17,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.0.1`
+	Version = `v3.0.2`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/catalog/disk/catalog.go

@@ -4,6 +4,8 @@ import (
 	"io"
 	"io/fs"
 	"os"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 )
 
 // DiskCatalog is a template catalog helper implementation based on disk
@@ -19,7 +21,7 @@ func NewCatalog(directory string) *DiskCatalog {
 	if directory != "" {
 		catalog.templatesFS = os.DirFS(directory)
 	} else {
-		catalog.templatesFS = os.DirFS("./")
+		catalog.templatesFS = os.DirFS(config.DefaultConfig.GetTemplateDir())
 	}
 	return catalog
 }

pkg/catalog/disk/path.go

@@ -50,7 +50,7 @@ var errNoValidCombination = errors.New("no valid combination found")
 
 // tryResolve attempts to load locate the target by iterating across all the folders tree
 func (c *DiskCatalog) tryResolve(fullPath string) (string, error) {
-	if fileutil.FileExists(fullPath) {
+	if fileutil.FileOrFolderExists(fullPath) {
 		return fullPath, nil
 	}
 	return "", errNoValidCombination

pkg/catalog/loader/loader.go

@@ -159,6 +159,7 @@ func New(config *Config) (*Store, error) {
 		if _, err := urlutil.Parse(v); err == nil {
 			remoteTemplates = append(remoteTemplates, handleTemplatesEditorURLs(v))
 		} else {
+
 			templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file
 		}
 	}
@@ -187,6 +188,7 @@ func New(config *Config) (*Store, error) {
 	if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlBasedTemplatesProvided {
 		store.finalTemplates = []string{cfg.DefaultConfig.TemplatesDirectory}
 	}
+
 	return store, nil
 }
 

pkg/progress/progress.go

@@ -65,6 +65,8 @@ func NewStatsTicker(duration int, active, outputJSON, cloud bool, port int) (Pro
 	if err != nil {
 		return nil, err
 	}
+	// only print in verbose mode
+	gologger.Verbose().Msgf("Started metrics server at localhost:%v", stats.Options.ListenPort)
 	progress.cloud = cloud
 	progress.active = active
 	progress.stats = stats

pkg/protocols/common/generators/generators.go

@@ -28,16 +28,6 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st
 	for name, payload := range payloads {
 		payloadsFinal[name] = payload
 	}
-	for name, payload := range payloads {
-		payloadStr, ok := payload.(string)
-		if ok {
-			final, resolveErr := catalog.ResolvePath(payloadStr, templatePath)
-			if resolveErr != nil {
-				return nil, errors.Wrap(resolveErr, "could not read payload file")
-			}
-			payloadsFinal[name] = final
-		}
-	}
 
 	generator := &PayloadGenerator{catalog: catalog, options: opts}
 	if err := generator.validate(payloadsFinal, templatePath); err != nil {

pkg/protocols/common/generators/validate.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	fileutil "github.com/projectdiscovery/utils/file"
 	folderutil "github.com/projectdiscovery/utils/folder"
@@ -25,7 +26,20 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat
 			if fileutil.FileExists(payloadType) {
 				continue
 			}
+			// if file already exists in nuclei-templates directory, skip any further checks
+			if fileutil.FileExists(filepath.Join(config.DefaultConfig.GetTemplateDir(), payloadType)) {
+				continue
+			}
 
+			// in below code, we calculate all possible paths from root and try to resolve the payload
+			// at each level of the path. if the payload is found, we break the loop and continue
+			// ex: template-path: /home/user/nuclei-templates/cves/2020/CVE-2020-1234.yaml
+			// then we check if helper file "my-payload.txt" exists at below paths:
+			// 1. /home/user/nuclei-templates/cves/2020/my-payload.txt
+			// 2. /home/user/nuclei-templates/cves/my-payload.txt
+			// 3. /home/user/nuclei-templates/my-payload.txt
+			// 4. /home/user/my-payload.txt
+			// 5. /home/my-payload.txt
 			changed := false
 
 			dir, _ := filepath.Split(templatePath)