update logic + config management refactor

v2/cmd/integration-test/code.go

@@ -120,11 +120,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
 	}
 	executerOpts.WorkflowLoader = workflowLoader
 
-	configObject, err := config.ReadConfiguration()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not read configuration file")
-	}
-	store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts))
+	store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create loader")
 	}

v2/cmd/integration-test/template-path.go

@@ -4,13 +4,12 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
-	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 )
 
 func getTemplatePath() string {
-	templatePath, _ := utils.GetDefaultTemplatePath()
-	return templatePath
+	return config.DefaultConfig.TemplatesDirectory
 }
 
 var templatesPathTestCases = map[string]testutils.TestCase{

v2/cmd/nuclei/main.go

@@ -1,14 +1,12 @@
 package main
 
 import (
-	"errors"
 	"fmt"
-	"io"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"runtime"
 	"runtime/pprof"
+	"strings"
 	"time"
 
 	"github.com/projectdiscovery/goflags"
@@ -39,8 +37,7 @@ func main() {
 	if err := runner.ConfigureOptions(); err != nil {
 		gologger.Fatal().Msgf("Could not initialize options: %s\n", err)
 	}
-	flagSet := readConfig()
-	configPath, _ := flagSet.GetConfigFilePath()
+	_ = readConfig()
 
 	if options.ListDslSignatures {
 		gologger.Info().Msgf("The available custom DSL functions are:")
@@ -67,7 +64,6 @@ func main() {
 	}
 
 	runner.ParseOptions(options)
-	options.ConfigPath = configPath
 
 	if options.HangMonitor {
 		cancel := monitor.NewStackMonitor(10 * time.Second)
@@ -117,7 +113,6 @@ func main() {
 }
 
 func readConfig() *goflags.FlagSet {
-
 	flagSet := goflags.NewFlagSet()
 	flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing
 on extensive configurability, massive extensibility and ease of use.`)
@@ -171,7 +166,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"),
 		flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"),
 		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
-		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", isNoColorEnabled(), "disable output content coloring (ANSI escape codes)"),
+		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", isColorNotAvailable(), "disable output content coloring (ANSI escape codes)"),
 		flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"),
 		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
 		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"),
@@ -284,22 +279,22 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.ListDslSignatures, "list-dsl-function", "ldf", false, "list all supported DSL function signatures"),
 		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
 		flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"),
-		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
+		flagSet.CallbackVar(printVersion, "version", "show nuclei version"),
 		flagSet.BoolVarP(&options.HangMonitor, "hang-monitor", "hm", false, "enable nuclei hang monitoring"),
 		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
 		flagSet.StringVar(&memProfile, "profile-mem", "", "optional nuclei memory profile dump file"),
 		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"),
 		flagSet.BoolVarP(&options.ShowVarDump, "show-var-dump", "svd", false, "show variables dump for debugging"),
 		flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"),
-		flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
+		flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"),
 		flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"),
 	)
 
 	flagSet.CreateGroup("update", "Update",
-		flagSet.BoolVarP(&options.UpdateNuclei, "update", "un", false, "update nuclei engine to the latest released version"),
+		flagSet.CallbackVarP(runner.NucleiToolUpdateCallback, "update", "un", "update nuclei engine to the latest released version"),
 		flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"),
-		flagSet.StringVarP(&options.TemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"),
-		flagSet.BoolVarP(&options.NoUpdateTemplates, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"),
+		flagSet.StringVarP(&options.NewTemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"),
+		flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"),
 	)
 
 	flagSet.CreateGroup("stats", "Statistics",
@@ -342,62 +337,97 @@ on extensive configurability, massive extensibility and ease of use.`)
 		http.LeaveDefaultPorts = true
 	}
 	if options.CustomConfigDir != "" {
-		originalIgnorePath := config.GetIgnoreFilePath()
-		config.SetCustomConfigDirectory(options.CustomConfigDir)
-		configPath := filepath.Join(options.CustomConfigDir, "config.yaml")
-		ignoreFile := filepath.Join(options.CustomConfigDir, ".nuclei-ignore")
-		if !fileutil.FileExists(ignoreFile) {
-			if err := fileutil.CopyFile(originalIgnorePath, ignoreFile); err != nil {
-				gologger.Error().Msgf("failed to copy .nuclei-ignore file in custom config directory got %v", err)
-			}
-		}
-		readConfigFile := func() error {
-			if err := flagSet.MergeConfigFile(configPath); err != nil && !errors.Is(err, io.EOF) {
-				defaultConfigPath, _ := flagSet.GetConfigFilePath()
-				err = fileutil.CopyFile(defaultConfigPath, configPath)
-				if err != nil {
-					return err
-				}
-				return errors.New("reload the config file")
-			}
-			return nil
-		}
-		if err := readConfigFile(); err != nil {
-			_ = readConfigFile()
-		}
+		config.DefaultConfig.SetConfigDir(options.CustomConfigDir)
+		readFlagsConfig(flagSet)
 	}
 	if cfgFile != "" {
+		if !fileutil.FileExists(cfgFile) {
+			gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile)
+		}
+		// merge config file with flags
 		if err := flagSet.MergeConfigFile(cfgFile); err != nil {
 			gologger.Fatal().Msgf("Could not read config: %s\n", err)
 		}
-		cfgFileFolder := filepath.Dir(cfgFile)
-		if err := config.OverrideIgnoreFilePath(cfgFileFolder); err != nil {
-			gologger.Warning().Msgf("Could not read ignore file from custom path: %s\n", err)
-		}
 	}
+	if options.NewTemplatesDirectory != "" {
+		config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory)
+	}
+
 	cleanupOldResumeFiles()
 	return flagSet
 }
 
-func isNoColorEnabled() bool {
+// isColorNotAvailable returns true if ascii collored output is not available.
+func isColorNotAvailable() bool {
 	return runtime.GOOS == "windows"
 }
 
+// cleanupOldResumeFiles cleans up resume files older than 10 days.
 func cleanupOldResumeFiles() {
-	root, err := config.GetConfigDir()
-	if err != nil {
-		return
-	}
+	root := config.DefaultConfig.GetConfigDir()
 	filter := fileutil.FileFilters{
 		OlderThan: 24 * time.Hour * 10, // cleanup on the 10th day
 		Prefix:    "resume-",
 	}
 	_ = fileutil.DeleteFilesOlderThan(root, filter)
 }
 
+// readFlagsConfig reads the config file from the default config dir and copies it to the current config dir.
+func readFlagsConfig(flagset *goflags.FlagSet) {
+	// check if config.yaml file exists
+	defaultCfgFile, err := flagset.GetConfigFilePath()
+	if err != nil {
+		// something went wrong either dir is not readable or something else went wrong upstream in `goflags`
+		// warn and exit in this case
+		gologger.Warning().Msgf("Could not read config file: %s\n", err)
+		return
+	}
+	cfgFile := config.DefaultConfig.GetFlagsConfigFilePath()
+	if !fileutil.FileExists(cfgFile) {
+		if !fileutil.FileExists(defaultCfgFile) {
+			// if default config does not exist, warn and exit
+			gologger.Warning().Msgf("missing default config file : %s", defaultCfgFile)
+			return
+		}
+		// if does not exist copy it from the default config
+		if err = fileutil.CopyFile(defaultCfgFile, cfgFile); err != nil {
+			gologger.Warning().Msgf("Could not copy config file: %s\n", err)
+		}
+		return
+	}
+	// if config file exists, merge it with the default config
+	if err = flagset.MergeConfigFile(cfgFile); err != nil {
+		gologger.Warning().Msgf("failed to merge configfile with flags got: %s\n", err)
+	}
+}
+
+// disableUpdatesCallback disables the update check.
+func disableUpdatesCallback() {
+	config.DefaultConfig.DisableUpdateCheck()
+}
+
+// printVersion prints the nuclei version and exits.
+func printVersion() {
+	gologger.Info().Msgf("Nuclei Engine Version: %s", config.Version)
+	os.Exit(0)
+}
+
+// printTemplateVersion prints the nuclei template version and exits.
+func printTemplateVersion() {
+	cfg := config.DefaultConfig
+	gologger.Info().Msgf("Public nuclei-templates version: %s (%s)\n", cfg.TemplateVersion, cfg.TemplatesDirectory)
+	if cfg.CustomS3TemplatesDirectory != "" {
+		gologger.Info().Msgf("Custom S3 templates location: %s\n", cfg.CustomS3TemplatesDirectory)
+	}
+	if cfg.CustomGithubTemplatesDirectory != "" {
+		gologger.Info().Msgf("Custom Github templates location: %s ", cfg.CustomGithubTemplatesDirectory)
+	}
+	os.Exit(0)
+}
+
 func init() {
 	// print stacktrace of errors in debug mode
-	if os.Getenv("DEBUG") != "" {
+	if strings.EqualFold(os.Getenv("DEBUG"), "true") {
 		errorutil.ShowStackTrace = true
 	}
 }

v2/examples/simple.go

@@ -80,11 +80,7 @@ func main() {
 	}
 	executerOpts.WorkflowLoader = workflowLoader
 
-	configObject, err := config.ReadConfiguration()
-	if err != nil {
-		log.Fatalf("Could not read config: %s\n", err)
-	}
-	store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts))
+	store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts))
 	if err != nil {
 		log.Fatalf("Could not create loader client: %s\n", err)
 	}

v2/go.mod

@@ -7,8 +7,6 @@ require (
 	github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725
 	github.com/andygrunwald/go-jira v1.16.0
 	github.com/antchfx/htmlquery v1.3.0
-	github.com/apex/log v1.9.0
-	github.com/blang/semver v3.5.1+incompatible
 	github.com/bluele/gcache v0.0.2
 	github.com/corpix/uarand v0.2.0
 	github.com/go-playground/validator/v10 v10.11.2
@@ -25,7 +23,7 @@ require (
 	github.com/projectdiscovery/clistats v0.0.12
 	github.com/projectdiscovery/fastdialer v0.0.24
 	github.com/projectdiscovery/hmap v0.0.11
-	github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d
+	github.com/projectdiscovery/interactsh v1.1.1-0.20230417162754-2cd861b12467
 	github.com/projectdiscovery/rawhttp v0.1.11
 	github.com/projectdiscovery/retryabledns v1.0.21
 	github.com/projectdiscovery/retryablehttp-go v1.0.14
@@ -38,7 +36,6 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/cast v1.5.0
 	github.com/syndtr/goleveldb v1.0.0
-	github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
 	github.com/valyala/fasttemplate v1.2.2
 	github.com/weppos/publicsuffix-go v0.30.0
 	github.com/xanzy/go-gitlab v0.82.0
@@ -54,6 +51,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
 	github.com/DataDog/gostackparse v0.6.0
+	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
 	github.com/antchfx/xmlquery v1.3.15
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
@@ -69,7 +67,6 @@ require (
 	github.com/klauspost/compress v1.16.4
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/mholt/archiver v3.1.1+incompatible
-	github.com/mitchellh/go-homedir v1.1.0
 	github.com/projectdiscovery/dsl v0.0.5-0.20230328190851-15d12ab4c5e4
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/goflags v0.1.8
@@ -82,7 +79,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.0.7
 	github.com/projectdiscovery/uncover v1.0.2
-	github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35
+	github.com/projectdiscovery/utils v0.0.21-0.20230419140949-a6527b072e4a
 	github.com/projectdiscovery/wappalyzergo v0.0.88
 	github.com/stretchr/testify v1.8.2
 	gopkg.in/src-d/go-git.v4 v4.13.1
@@ -94,7 +91,6 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
-	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24 // indirect
@@ -121,6 +117,7 @@ require (
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect
 	github.com/minio/selfupdate v0.6.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.15.1 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
@@ -130,6 +127,7 @@ require (
 	github.com/projectdiscovery/freeport v0.0.4 // indirect
 	github.com/shoenig/go-m1cpu v0.1.4 // indirect
 	github.com/skeema/knownhosts v1.1.0 // indirect
+	github.com/smartystreets/assertions v1.0.0 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tidwall/buntdb v1.2.10 // indirect
 	github.com/tidwall/gjson v1.14.4 // indirect
@@ -155,7 +153,6 @@ require (
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/antchfx/xpath v1.2.3 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/c4milo/unpackit v0.1.0 // indirect
 	github.com/caddyserver/certmagic v0.17.2 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -174,15 +171,12 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
-	github.com/gosuri/uilive v0.0.4 // indirect
-	github.com/gosuri/uiprogress v0.0.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
 	github.com/hdm/jarm-go v0.0.7 // indirect
 	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/libdns/libdns v0.2.1 // indirect
 	github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect