diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index eb53be5c69..408ccbf5ac 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -120,11 +120,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) { } executerOpts.WorkflowLoader = workflowLoader - configObject, err := config.ReadConfiguration() - if err != nil { - return nil, errors.Wrap(err, "could not read configuration file") - } - store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts)) + store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts)) if err != nil { return nil, errors.Wrap(err, "could not create loader") } diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 984fe29642..9633558d8d 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -4,13 +4,12 @@ import ( "fmt" "strings" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) func getTemplatePath() string { - templatePath, _ := utils.GetDefaultTemplatePath() - return templatePath + return config.DefaultConfig.TemplatesDirectory } var templatesPathTestCases = map[string]testutils.TestCase{ diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 60f2d6b8f1..4f7c4316ad 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -1,14 +1,12 @@ package main import ( - "errors" "fmt" - "io" "os" "os/signal" - "path/filepath" "runtime" "runtime/pprof" + "strings" "time" "github.com/projectdiscovery/goflags" @@ -39,8 +37,7 @@ func main() { if err := runner.ConfigureOptions(); err != nil { gologger.Fatal().Msgf("Could not initialize options: %s\n", err) } - flagSet := readConfig() - configPath, _ := flagSet.GetConfigFilePath() + _ = readConfig() if options.ListDslSignatures { gologger.Info().Msgf("The available custom DSL functions are:") @@ -67,7 +64,6 @@ func main() { } runner.ParseOptions(options) - options.ConfigPath = configPath if options.HangMonitor { cancel := monitor.NewStackMonitor(10 * time.Second) @@ -117,7 +113,6 @@ func main() { } func readConfig() *goflags.FlagSet { - flagSet := goflags.NewFlagSet() flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing on extensive configurability, massive extensibility and ease of use.`) @@ -171,7 +166,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.StoreResponse, "store-resp", "sresp", false, "store all request/response passed through nuclei to output directory"), flagSet.StringVarP(&options.StoreResponseDir, "store-resp-dir", "srd", runner.DefaultDumpTrafficOutputFolder, "store all request/response passed through nuclei to custom directory"), flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), - flagSet.BoolVarP(&options.NoColor, "no-color", "nc", isNoColorEnabled(), "disable output content coloring (ANSI escape codes)"), + flagSet.BoolVarP(&options.NoColor, "no-color", "nc", isColorNotAvailable(), "disable output content coloring (ANSI escape codes)"), flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"), @@ -284,22 +279,22 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.ListDslSignatures, "list-dsl-function", "ldf", false, "list all supported DSL function signatures"), flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"), - flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), + flagSet.CallbackVar(printVersion, "version", "show nuclei version"), flagSet.BoolVarP(&options.HangMonitor, "hang-monitor", "hm", false, "enable nuclei hang monitoring"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.StringVar(&memProfile, "profile-mem", "", "optional nuclei memory profile dump file"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), flagSet.BoolVarP(&options.ShowVarDump, "show-var-dump", "svd", false, "show variables dump for debugging"), flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"), - flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"), + flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"), flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"), ) flagSet.CreateGroup("update", "Update", - flagSet.BoolVarP(&options.UpdateNuclei, "update", "un", false, "update nuclei engine to the latest released version"), + flagSet.CallbackVarP(runner.NucleiToolUpdateCallback, "update", "un", "update nuclei engine to the latest released version"), flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"), - flagSet.StringVarP(&options.TemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"), - flagSet.BoolVarP(&options.NoUpdateTemplates, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"), + flagSet.StringVarP(&options.NewTemplatesDirectory, "update-template-dir", "ud", "", "custom directory to install / update nuclei-templates"), + flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"), ) flagSet.CreateGroup("stats", "Statistics", @@ -342,52 +337,34 @@ on extensive configurability, massive extensibility and ease of use.`) http.LeaveDefaultPorts = true } if options.CustomConfigDir != "" { - originalIgnorePath := config.GetIgnoreFilePath() - config.SetCustomConfigDirectory(options.CustomConfigDir) - configPath := filepath.Join(options.CustomConfigDir, "config.yaml") - ignoreFile := filepath.Join(options.CustomConfigDir, ".nuclei-ignore") - if !fileutil.FileExists(ignoreFile) { - if err := fileutil.CopyFile(originalIgnorePath, ignoreFile); err != nil { - gologger.Error().Msgf("failed to copy .nuclei-ignore file in custom config directory got %v", err) - } - } - readConfigFile := func() error { - if err := flagSet.MergeConfigFile(configPath); err != nil && !errors.Is(err, io.EOF) { - defaultConfigPath, _ := flagSet.GetConfigFilePath() - err = fileutil.CopyFile(defaultConfigPath, configPath) - if err != nil { - return err - } - return errors.New("reload the config file") - } - return nil - } - if err := readConfigFile(); err != nil { - _ = readConfigFile() - } + config.DefaultConfig.SetConfigDir(options.CustomConfigDir) + readFlagsConfig(flagSet) } if cfgFile != "" { + if !fileutil.FileExists(cfgFile) { + gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile) + } + // merge config file with flags if err := flagSet.MergeConfigFile(cfgFile); err != nil { gologger.Fatal().Msgf("Could not read config: %s\n", err) } - cfgFileFolder := filepath.Dir(cfgFile) - if err := config.OverrideIgnoreFilePath(cfgFileFolder); err != nil { - gologger.Warning().Msgf("Could not read ignore file from custom path: %s\n", err) - } } + if options.NewTemplatesDirectory != "" { + config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory) + } + cleanupOldResumeFiles() return flagSet } -func isNoColorEnabled() bool { +// isColorNotAvailable returns true if ascii collored output is not available. +func isColorNotAvailable() bool { return runtime.GOOS == "windows" } +// cleanupOldResumeFiles cleans up resume files older than 10 days. func cleanupOldResumeFiles() { - root, err := config.GetConfigDir() - if err != nil { - return - } + root := config.DefaultConfig.GetConfigDir() filter := fileutil.FileFilters{ OlderThan: 24 * time.Hour * 10, // cleanup on the 10th day Prefix: "resume-", @@ -395,9 +372,62 @@ func cleanupOldResumeFiles() { _ = fileutil.DeleteFilesOlderThan(root, filter) } +// readFlagsConfig reads the config file from the default config dir and copies it to the current config dir. +func readFlagsConfig(flagset *goflags.FlagSet) { + // check if config.yaml file exists + defaultCfgFile, err := flagset.GetConfigFilePath() + if err != nil { + // something went wrong either dir is not readable or something else went wrong upstream in `goflags` + // warn and exit in this case + gologger.Warning().Msgf("Could not read config file: %s\n", err) + return + } + cfgFile := config.DefaultConfig.GetFlagsConfigFilePath() + if !fileutil.FileExists(cfgFile) { + if !fileutil.FileExists(defaultCfgFile) { + // if default config does not exist, warn and exit + gologger.Warning().Msgf("missing default config file : %s", defaultCfgFile) + return + } + // if does not exist copy it from the default config + if err = fileutil.CopyFile(defaultCfgFile, cfgFile); err != nil { + gologger.Warning().Msgf("Could not copy config file: %s\n", err) + } + return + } + // if config file exists, merge it with the default config + if err = flagset.MergeConfigFile(cfgFile); err != nil { + gologger.Warning().Msgf("failed to merge configfile with flags got: %s\n", err) + } +} + +// disableUpdatesCallback disables the update check. +func disableUpdatesCallback() { + config.DefaultConfig.DisableUpdateCheck() +} + +// printVersion prints the nuclei version and exits. +func printVersion() { + gologger.Info().Msgf("Nuclei Engine Version: %s", config.Version) + os.Exit(0) +} + +// printTemplateVersion prints the nuclei template version and exits. +func printTemplateVersion() { + cfg := config.DefaultConfig + gologger.Info().Msgf("Public nuclei-templates version: %s (%s)\n", cfg.TemplateVersion, cfg.TemplatesDirectory) + if cfg.CustomS3TemplatesDirectory != "" { + gologger.Info().Msgf("Custom S3 templates location: %s\n", cfg.CustomS3TemplatesDirectory) + } + if cfg.CustomGithubTemplatesDirectory != "" { + gologger.Info().Msgf("Custom Github templates location: %s ", cfg.CustomGithubTemplatesDirectory) + } + os.Exit(0) +} + func init() { // print stacktrace of errors in debug mode - if os.Getenv("DEBUG") != "" { + if strings.EqualFold(os.Getenv("DEBUG"), "true") { errorutil.ShowStackTrace = true } } diff --git a/v2/examples/simple.go b/v2/examples/simple.go index 5b80d22821..1550fbbc7b 100644 --- a/v2/examples/simple.go +++ b/v2/examples/simple.go @@ -80,11 +80,7 @@ func main() { } executerOpts.WorkflowLoader = workflowLoader - configObject, err := config.ReadConfiguration() - if err != nil { - log.Fatalf("Could not read config: %s\n", err) - } - store, err := loader.New(loader.NewConfig(defaultOpts, configObject, catalog, executerOpts)) + store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts)) if err != nil { log.Fatalf("Could not create loader client: %s\n", err) } diff --git a/v2/go.mod b/v2/go.mod index 2789afc8b3..88d839acf8 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -7,8 +7,6 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 github.com/andygrunwald/go-jira v1.16.0 github.com/antchfx/htmlquery v1.3.0 - github.com/apex/log v1.9.0 - github.com/blang/semver v3.5.1+incompatible github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.2.0 github.com/go-playground/validator/v10 v10.11.2 @@ -25,7 +23,7 @@ require ( github.com/projectdiscovery/clistats v0.0.12 github.com/projectdiscovery/fastdialer v0.0.24 github.com/projectdiscovery/hmap v0.0.11 - github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d + github.com/projectdiscovery/interactsh v1.1.1-0.20230417162754-2cd861b12467 github.com/projectdiscovery/rawhttp v0.1.11 github.com/projectdiscovery/retryabledns v1.0.21 github.com/projectdiscovery/retryablehttp-go v1.0.14 @@ -38,7 +36,6 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.0 github.com/syndtr/goleveldb v1.0.0 - github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.2 github.com/weppos/publicsuffix-go v0.30.0 github.com/xanzy/go-gitlab v0.82.0 @@ -54,6 +51,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/DataDog/gostackparse v0.6.0 + github.com/Masterminds/semver/v3 v3.2.1 github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/antchfx/xmlquery v1.3.15 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 @@ -69,7 +67,6 @@ require ( github.com/klauspost/compress v1.16.4 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/mitchellh/go-homedir v1.1.0 github.com/projectdiscovery/dsl v0.0.5-0.20230328190851-15d12ab4c5e4 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.8 @@ -82,7 +79,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.0.7 github.com/projectdiscovery/uncover v1.0.2 - github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 + github.com/projectdiscovery/utils v0.0.21-0.20230419140949-a6527b072e4a github.com/projectdiscovery/wappalyzergo v0.0.88 github.com/stretchr/testify v1.8.2 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -94,7 +91,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.24 // indirect @@ -121,6 +117,7 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect github.com/minio/selfupdate v0.6.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -130,6 +127,7 @@ require ( github.com/projectdiscovery/freeport v0.0.4 // indirect github.com/shoenig/go-m1cpu v0.1.4 // indirect github.com/skeema/knownhosts v1.1.0 // indirect + github.com/smartystreets/assertions v1.0.0 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.2.10 // indirect github.com/tidwall/gjson v1.14.4 // indirect @@ -155,7 +153,6 @@ require ( github.com/andybalholm/cascadia v1.3.1 // indirect github.com/antchfx/xpath v1.2.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/c4milo/unpackit v0.1.0 // indirect github.com/caddyserver/certmagic v0.17.2 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -174,15 +171,12 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/gosuri/uilive v0.0.4 // indirect - github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hdm/jarm-go v0.0.7 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/libdns/libdns v0.2.1 // indirect github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect diff --git a/v2/go.sum b/v2/go.sum index d22f6270a7..feb7aef8e5 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -16,8 +16,8 @@ github.com/DataDog/gostackparse v0.6.0 h1:egCGQviIabPwsyoWpGvIBGrEnNWez35aEO7OJ1 github.com/DataDog/gostackparse v0.6.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= -github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= @@ -63,17 +63,11 @@ github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8 github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= -github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= -github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= -github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= -github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.17.8 h1:GMupCNNI7FARX27L7GjCJM8NgivWbRgpjNI/hOQjFS8= github.com/aws/aws-sdk-go-v2 v1.17.8/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= @@ -116,7 +110,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 h1:Qf1aWwnsNkyAoqDqmdM3nHwN78XQ github.com/aws/aws-sdk-go-v2/service/sts v1.18.9/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= @@ -128,15 +121,9 @@ github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMf github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= -github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= -github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= -github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= github.com/caddyserver/certmagic v0.17.2 h1:o30seC1T/dBqBCNNGNHWwj2i5/I/FMjBbTAhjADP3nE= github.com/caddyserver/certmagic v0.17.2/go.mod h1:ouWUuC490GOLJzkyN35eXfV8bSbwMwSf4bdhkIxtdQE= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= @@ -171,7 +158,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= @@ -193,7 +179,6 @@ github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlK github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -249,17 +234,12 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= -github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= -github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= -github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -278,8 +258,6 @@ github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= -github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= -github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= @@ -296,12 +274,10 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -320,12 +296,8 @@ github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -355,13 +327,9 @@ github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZj github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= @@ -370,7 +338,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/acmez v1.0.4 h1:N3cE4Pek+dSolbsofIkAYz6H1d3pE+2G0os7QHslf80= github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DAYQY= github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= @@ -411,7 +378,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= @@ -453,8 +419,8 @@ github.com/projectdiscovery/hmap v0.0.11 h1:nA3qCFzWPcOw27T8PII5IWI3ZP0ys7TGCi2n github.com/projectdiscovery/hmap v0.0.11/go.mod h1:5sbLn2OHexvpVupStNOhusWO9jLCyEm5jcHwWB2nOkI= github.com/projectdiscovery/httpx v1.2.9 h1:bSXXjPHIDywokASSXLaWScyIRTFT6Avr9JJS7lV96c0= github.com/projectdiscovery/httpx v1.2.9/go.mod h1:99+TTGdMDKQgWRLzzWj3zGffQ6f4NCAWQHqHWfPn0Uc= -github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d h1:V/TpFiLbpSsgZgxBXSTXSgwGD8uKayImAwqz9etAzK8= -github.com/projectdiscovery/interactsh v1.1.1-0.20230403200718-191eb81c4b0d/go.mod h1:6qYsmXQ//CPXib21ugKYXuLwr43zLPlO+UYyx+o+4Mw= +github.com/projectdiscovery/interactsh v1.1.1-0.20230417162754-2cd861b12467 h1:tMpLB8FzcgAhVOZT9V8G4IYhWufNk/7U5ZqgaVYPeI4= +github.com/projectdiscovery/interactsh v1.1.1-0.20230417162754-2cd861b12467/go.mod h1:Sk5aRjJk3lYN+MCVIJNxOA6Y+UHgtpyRjKcVqgumFbs= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/mapcidr v1.1.1 h1:68Xvw9cKugNeAVxHE3Nl1Ej26nm1taWq6e1WPXpluc0= @@ -482,8 +448,8 @@ github.com/projectdiscovery/tlsx v1.0.7/go.mod h1:bMz1JMJf1sSBLpk7Y6vLwRZrijxPUv github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo= github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= -github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 h1:UBOE9Eob1wj7YZ1MGBbtHvc3ptqBHvXxNKRVxTg21Rc= -github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o= +github.com/projectdiscovery/utils v0.0.21-0.20230419140949-a6527b072e4a h1:h9ceITnnFLJ0qucXCrI3WOXtNRK2oAtcCuFsMAiuIc0= +github.com/projectdiscovery/utils v0.0.21-0.20230419140949-a6527b072e4a/go.mod h1:954dxg9AWmNmcNQdc5BpucghibSvC76prWLQFrv14FQ= github.com/projectdiscovery/wappalyzergo v0.0.88 h1:N/1vFlKmc3GJco9rANJdHrxg8jdav/xmnICo8rztmH8= github.com/projectdiscovery/wappalyzergo v0.0.88/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -494,7 +460,6 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -521,10 +486,8 @@ github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXi github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -569,15 +532,6 @@ github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= -github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= -github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= -github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= -github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= -github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= -github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= -github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs= -github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -588,7 +542,6 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -660,7 +613,6 @@ golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -727,7 +679,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -845,7 +796,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/v2/internal/installer/doc.go b/v2/internal/installer/doc.go new file mode 100644 index 0000000000..7259db1d62 --- /dev/null +++ b/v2/internal/installer/doc.go @@ -0,0 +1,6 @@ +package installer + +// package install provides helper functions for all download and installation of following tasks: +// 1. downloading and install nuclei templates +// 2. download and update nuclei binary (i.e self update) +// 3. version check for nuclei binary & templates diff --git a/v2/internal/installer/template.go b/v2/internal/installer/template.go new file mode 100644 index 0000000000..8c0f671a0e --- /dev/null +++ b/v2/internal/installer/template.go @@ -0,0 +1,328 @@ +package installer + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/olekukonko/tablewriter" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" + stringsutil "github.com/projectdiscovery/utils/strings" + updateutils "github.com/projectdiscovery/utils/update" +) + +const ( + checkSumFilePerm = 0644 +) + +var ( + HideProgressBar = true + HideUpdateChangesTable = false +) + +// TemplateUpdateResults contains the results of template update +type templateUpdateResults struct { + additions []string + deletions []string + modifications []string + totalCount int +} + +// String returns markdown table of template update results +func (t *templateUpdateResults) String() string { + var buff bytes.Buffer + data := [][]string{ + {strconv.Itoa(t.totalCount), strconv.Itoa(len(t.additions)), strconv.Itoa(len(t.deletions))}, + } + table := tablewriter.NewWriter(&buff) + table.SetHeader([]string{"Total", "Added", "Removed"}) + for _, v := range data { + table.Append(v) + } + table.Render() + return buff.String() +} + +// TemplateManager is a manager for templates. +// It downloads / updates / installs templates. +type TemplateManager struct{} + +// FreshInstallIfNotExists installs templates if they are not already installed +// if templates directory already exists, it does nothing +func (t *TemplateManager) FreshInstallIfNotExists() error { + if fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) { + return nil + } + gologger.Info().Msgf("nuclei-templates are not installed, installing...") + return t.installTemplatesAt(config.DefaultConfig.TemplatesDirectory) +} + +// UpdateIfOutdated updates templates if they are outdated +func (t *TemplateManager) UpdateIfOutdated() error { + // if folder does not exist, its a fresh install and not update + if !fileutil.FolderExists(config.DefaultConfig.TemplatesDirectory) { + return t.FreshInstallIfNotExists() + } + if config.DefaultConfig.NeedsTemplateUpdate() { + return t.updateTemplatesAt(config.DefaultConfig.TemplatesDirectory) + } + return nil +} + +// installTemplatesAt installs templates at given directory +func (t *TemplateManager) installTemplatesAt(dir string) error { + if !fileutil.FolderExists(dir) { + if err := fileutil.CreateFolder(dir); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to create directory at %s", dir) + } + } + ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTeamplatesRepoName) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir) + } + // write templates to disk + if err := t.writeTemplatestoDisk(ghrd, dir); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to write templates to disk at %s", dir) + } + gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir) + return nil +} + +// updateTemplatesAt updates templates at given directory +func (t *TemplateManager) updateTemplatesAt(dir string) error { + // firstly read checksums from .checksum file these are used to generate stats + oldchecksums, err := t.getChecksumFromDir(dir) + if err != nil { + // if something went wrong overwrite all files + oldchecksums = make(map[string]string) + } + + ghrd, err := updateutils.NewghReleaseDownloader(config.OfficialNucleiTeamplatesRepoName) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to install templates at %s", dir) + } + + gologger.Info().Msgf("Your current nuclei-templates %s are outdated. Latest is %s\n", config.DefaultConfig.TemplateVersion, ghrd.Latest.GetTagName()) + + // write templates to disk + if err := t.writeTemplatestoDisk(ghrd, dir); err != nil { + return err + } + + // get checksums from new templates + newchecksums, err := t.getChecksumFromDir(dir) + if err != nil { + // unlikely this case will happen + return errorutil.NewWithErr(err).Msgf("failed to get checksums from %s after update", dir) + } + + // summarize all changes + results := t.summarizeChanges(oldchecksums, newchecksums) + + // print summary + if results.totalCount > 0 { + gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir) + if !HideUpdateChangesTable { + // print summary table + gologger.Print().Msgf("\nNuclei Templates %s Changelog\n", ghrd.Latest.GetTagName()) + gologger.DefaultLogger.Print().Msg(results.String()) + } + } else { + gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir) + } + return nil +} + +// summarizeChanges summarizes changes between old and new checksums +func (t *TemplateManager) summarizeChanges(old, new map[string]string) *templateUpdateResults { + results := &templateUpdateResults{} + for k, v := range new { + if oldv, ok := old[k]; ok { + if oldv != v { + results.modifications = append(results.modifications, k) + } + } else { + results.additions = append(results.additions, k) + } + } + for k := range old { + if _, ok := new[k]; !ok { + results.deletions = append(results.deletions, k) + } + } + results.totalCount = len(results.additions) + len(results.deletions) + len(results.modifications) + return results +} + +// getAbsoluteFilePath returns absolute path where a file should be written based on given uri(i.e files in zip) +// if returned path is empty, it means that file should not be written and skipped +func (t *TemplateManager) getAbsoluteFilePath(templatedir, uri string, f fs.FileInfo) string { + // overwrite .nuclei-ignore everytime nuclei-templates are downloaded + if f.Name() == config.NucleiIgnoreFileName { + return config.DefaultConfig.GetIgnoreFilePath() + } + // skip all meta files + if !strings.EqualFold(f.Name(), config.NewTemplateAdditionsFileName) { + if strings.TrimSpace(f.Name()) == "" || strings.HasPrefix(f.Name(), ".") || strings.EqualFold(f.Name(), "README.md") { + return "" + } + } + + // get root or leftmost directory name from path + // this is in format `projectdiscovery-nuclei-templates-commithash` + + index := strings.Index(uri, "/") + if index == -1 { + // zip files does not have directory at all , in this case log error but continue + gologger.Warning().Msgf("failed to get directory name from uri: %s", uri) + return filepath.Join(templatedir, uri) + } + // seperator is also included in rootDir + rootDirectory := uri[:index+1] + relPath := strings.TrimPrefix(uri, rootDirectory) + + // if it is a github meta directory skip it + if stringsutil.HasPrefixAny(relPath, ".github", ".git") { + return "" + } + + newPath := filepath.Clean(filepath.Join(templatedir, relPath)) + + if !strings.HasPrefix(newPath, templatedir) { + // we don't allow LFI + return "" + } + + if newPath == templatedir || newPath == templatedir+string(os.PathSeparator) { + // skip writing the folder itself since it already exists + return "" + } + + if relPath != "" && f.IsDir() { + // if uri is a directory, create it + if err := fileutil.CreateFolder(newPath); err != nil { + gologger.Warning().Msgf("uri %v: got %s while installing templates", uri, err) + } + return "" + } + return newPath +} + +// writeChecksumFileInDir is actual method responsible for writing all templates to directory +func (t *TemplateManager) writeTemplatestoDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error { + callbackFunc := func(uri string, f fs.FileInfo, r io.Reader) error { + writePath := t.getAbsoluteFilePath(dir, uri, f) + if writePath == "" { + // skip writing file + return nil + } + bin, err := io.ReadAll(r) + if err != nil { + // if error occurs, iteration also stops + return errorutil.NewWithErr(err).Msgf("failed to read file %s", uri) + } + return os.WriteFile(writePath, bin, f.Mode()) + } + err := ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to download templates") + } + if err := config.DefaultConfig.WriteTemplatesConfig(); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to write templates config") + } + // update ignore hash after writing new templates + if err := config.DefaultConfig.UpdateNucleiIgnoreHash(); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to update nuclei ignore hash") + } + + // update templates version in config file + if err := config.DefaultConfig.SetTemplatesVersion(ghrd.Latest.GetTagName()); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to update templates version") + } + + // after installation create and write checksums to .checksum file + return t.writeChecksumFileInDir(dir) +} + +// getChecksumFromDir returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension) +// if .checksum file does not exist checksums are calculated and returned +func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, error) { + checksumFilePath := config.DefaultConfig.GetChecksumFilePath() + if fileutil.FileExists(checksumFilePath) { + checksums, err := os.ReadFile(checksumFilePath) + if err == nil { + allChecksums := make(map[string]string) + for _, v := range strings.Split(string(checksums), "\n") { + v = strings.TrimSpace(v) + tmparr := strings.Split(v, ",") + if len(tmparr) != 2 { + continue + } + allChecksums[tmparr[0]] = tmparr[1] + } + return allChecksums, nil + } + } + return t.calculateChecksumMap(dir) +} + +// writeChecksumFileInDir creates checksums of all yaml files in given directory +// and writes them to a file named .checksum +func (t *TemplateManager) writeChecksumFileInDir(dir string) error { + checksumMap, err := t.calculateChecksumMap(dir) + if err != nil { + return err + } + var buff bytes.Buffer + for k, v := range checksumMap { + buff.WriteString(k + "," + v) + } + return os.WriteFile(config.DefaultConfig.GetChecksumFilePath(), buff.Bytes(), checkSumFilePerm) +} + +// getChecksumMap returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension) +func (t *TemplateManager) calculateChecksumMap(dir string) (map[string]string, error) { + // getchecksumMap walks given directory `dir` and returns a map containing + // checksums (md5 hash) of all yaml files (with .yaml extension) and the + // format is map[filePath]checksum + checksumMap := map[string]string{} + + getChecksum := func(filepath string) (string, error) { + // return md5 hash of the file + bin, err := os.ReadFile(filepath) + if err != nil { + return "", err + } + return fmt.Sprintf("%x", md5.Sum(bin)), nil + } + + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // skip checksums of custom templates i.e github and s3 + if stringsutil.HasPrefixAny(path, config.DefaultConfig.CustomGithubTemplatesDirectory, config.DefaultConfig.CustomS3TemplatesDirectory) { + return nil + } + + // current implementations calculates checksums of all files (including .yaml,.txt,.md,.json etc) + if !d.IsDir() { + checksum, err := getChecksum(path) + if err != nil { + return err + } + checksumMap[path] = checksum + } + return nil + }) + return checksumMap, errorutil.WrapfWithNil(err, "failed to calculate checksums of templates") +} diff --git a/v2/internal/installer/template_test.go b/v2/internal/installer/template_test.go new file mode 100644 index 0000000000..e6b0345e24 --- /dev/null +++ b/v2/internal/installer/template_test.go @@ -0,0 +1,59 @@ +package installer + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/stretchr/testify/require" +) + +func TestTemplateInstallation(t *testing.T) { + // test that the templates are installed correctly + // along with necessary changes that are made + HideProgressBar = true + + tm := &TemplateManager{} + dir, err := os.MkdirTemp("", "nuclei-templates-*") + require.Nil(t, err) + defer os.RemoveAll(dir) + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.Nil(t, err) + defer os.RemoveAll(cfgdir) + + // set the config directory to a temporary directory + config.DefaultConfig.SetConfigDir(cfgdir) + // set the templates directory to a temporary directory + templatesTempDir := filepath.Join(dir, "templates") + config.DefaultConfig.SetTemplatesDir(templatesTempDir) + + err = tm.FreshInstallIfNotExists() + if err != nil { + if strings.Contains(err.Error(), "rate limit") { + t.Skip("Skipping test due to github rate limit") + } + require.Nil(t, err) + } + + // we should switch to more fine granular tests for template + // integrity, but for now, we just check that the templates are installed + counter := 0 + err = filepath.Walk(templatesTempDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + counter++ + } + return nil + }) + require.Nil(t, err) + + // we should have at least 1000 templates + require.Greater(t, counter, 1000) + // everytime we install templates, it should override the ignore file with latest one + require.FileExists(t, config.DefaultConfig.GetIgnoreFilePath()) + t.Logf("Installed %d templates", counter) +} diff --git a/v2/internal/installer/util.go b/v2/internal/installer/util.go new file mode 100644 index 0000000000..7285626687 --- /dev/null +++ b/v2/internal/installer/util.go @@ -0,0 +1,69 @@ +package installer + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/http" + + "github.com/Masterminds/semver/v3" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + errorutil "github.com/projectdiscovery/utils/errors" +) + +// GetNewTemplatesInVersions returns templates path of all newly added templates +// in these versions +func GetNewTemplatesInVersions(versions ...string) []string { + allTemplates := []string{} + for _, v := range versions { + if v == config.DefaultConfig.TemplateVersion { + allTemplates = append(allTemplates, config.DefaultConfig.GetNewAdditions()...) + } + _, err := semver.NewVersion(v) + if err != nil { + gologger.Error().Msgf("%v is not a valid semver version. skipping", v) + continue + } + if config.IsOutdatedVersion(v, "v8.8.4") { + // .new-additions was added in v8.8.4 any version before that is not supported + gologger.Error().Msgf(".new-additions support was added in v8.8.4 older versions are not supported") + continue + } + + arr, err := getNewAdditionsFileFromGithub(v) + if err != nil { + gologger.Error().Msgf("failed to fetch new additions for %v got: %v", v, err) + continue + } + allTemplates = append(allTemplates, arr...) + } + return allTemplates +} + +func getNewAdditionsFileFromGithub(version string) ([]string, error) { + resp, err := retryableHttpClient.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, errorutil.New("version not found") + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + templatesList := []string{} + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + text := scanner.Text() + if text == "" { + continue + } + if config.IsTemplate(text) { + templatesList = append(templatesList, text) + } + } + return templatesList, nil +} diff --git a/v2/internal/installer/versioncheck.go b/v2/internal/installer/versioncheck.go new file mode 100644 index 0000000000..12dd2aed51 --- /dev/null +++ b/v2/internal/installer/versioncheck.go @@ -0,0 +1,91 @@ +package installer + +import ( + "encoding/json" + "io" + "net/url" + "os" + "runtime" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/retryablehttp-go" + updateutils "github.com/projectdiscovery/utils/update" +) + +const ( + pdtmNucleiVersionEndpoint = "https://api.pdtm.sh/api/v1/tools/nuclei" + pdtmNucleiIgnoreFileEndpoint = "https://api.pdtm.sh/api/v1/tools/nuclei/ignore" +) + +// defaultHttpClient is http client that is only meant to be used for version check +// if proxy env variables are set those are reflected in this client +var retryableHttpClient = retryablehttp.NewClient(retryablehttp.Options{HttpClient: updateutils.DefaultHttpClient, RetryMax: 2}) + +// PdtmAPIResponse is the response from pdtm API for nuclei endpoint +type PdtmAPIResponse struct { + IgnoreHash string `json:"ignore-hash"` + Tools []struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"tools"` +} + +// NucleiVersionCheck checks for the latest version of nuclei and nuclei templates +// and returns an error if it fails to check on success it returns nil and changes are +// made to the default config in config.DefaultConfig +func NucleiVersionCheck() error { + resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams()) + if err != nil { + return err + } + defer resp.Body.Close() + bin, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + var pdtmResp PdtmAPIResponse + if err := json.Unmarshal(bin, &pdtmResp); err != nil { + return err + } + var nucleiversion, templateversion string + for _, tool := range pdtmResp.Tools { + switch tool.Name { + case "nuclei": + if tool.Version != "" { + nucleiversion = "v" + tool.Version + } + + case "nuclei-templates": + if tool.Version != "" { + templateversion = "v" + tool.Version + } + } + } + return config.DefaultConfig.WriteVersionCheckData(pdtmResp.IgnoreHash, nucleiversion, templateversion) +} + +// getpdtmParams returns encoded query parameters sent to update check endpoint +func getpdtmParams() string { + params := &url.Values{} + params.Add("os", runtime.GOOS) + params.Add("arch", runtime.GOARCH) + params.Add("go_version", runtime.Version()) + params.Add("v", config.Version) + return params.Encode() +} + +// UpdateIgnoreFile updates default ignore file by downloading latest ignore file +func UpdateIgnoreFile() error { + resp, err := retryableHttpClient.Get(pdtmNucleiIgnoreFileEndpoint + "?" + getpdtmParams()) + if err != nil { + return err + } + bin, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if err := os.WriteFile(config.DefaultConfig.GetIgnoreFilePath(), bin, 0644); err != nil { + return err + } + return config.DefaultConfig.UpdateNucleiIgnoreHash() +} diff --git a/v2/internal/installer/versioncheck_test.go b/v2/internal/installer/versioncheck_test.go new file mode 100644 index 0000000000..dca1665bf7 --- /dev/null +++ b/v2/internal/installer/versioncheck_test.go @@ -0,0 +1,19 @@ +package installer + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/utils/generic" + "github.com/stretchr/testify/require" +) + +func TestVersionCheck(t *testing.T) { + err := NucleiVersionCheck() + require.Nil(t, err) + cfg := config.DefaultConfig + if generic.EqualsAny("", cfg.LatestNucleiIgnoreHash, cfg.LatestNucleiVersion, cfg.LatestNucleiTemplatesVersion) { + // all above values cannot be empty + t.Errorf("something went wrong got empty response nuclei-version=%v templates-version=%v ignore-hash=%v", cfg.LatestNucleiVersion, cfg.LatestNucleiTemplatesVersion, cfg.LatestNucleiIgnoreHash) + } +} diff --git a/v2/internal/installer/zipslip_unix_test.go b/v2/internal/installer/zipslip_unix_test.go new file mode 100644 index 0000000000..7e9eab94a5 --- /dev/null +++ b/v2/internal/installer/zipslip_unix_test.go @@ -0,0 +1,68 @@ +package installer + +import ( + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var _ fs.FileInfo = &tempFileInfo{} + +type tempFileInfo struct { + name string +} + +func (t *tempFileInfo) Name() string { + return t.name +} + +func (t *tempFileInfo) ModTime() time.Time { + return time.Now() +} + +func (t *tempFileInfo) Mode() fs.FileMode { + return fs.ModePerm +} + +func (t tempFileInfo) IsDir() bool { + return false +} + +func (t *tempFileInfo) Size() int64 { + return 100 +} + +func (t *tempFileInfo) Sys() any { + return nil +} + +func TestZipSlip(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping Unix Zip LFI Check") + } + + configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") + defer os.RemoveAll(configuredTemplateDirectory) + + t.Run("negative scenarios", func(t *testing.T) { + filePathsFromZip := []string{ + "./../nuclei-templates/../cve/test.yaml", + "nuclei-templates/../cve/test.yaml", + "nuclei-templates/././../cve/test.yaml", + "nuclei-templates/.././../cve/test.yaml", + "nuclei-templates/.././../cve/../test.yaml", + } + tm := TemplateManager{} + + for _, filePathFromZip := range filePathsFromZip { + var tmp fs.FileInfo = &tempFileInfo{name: filePathFromZip} + writePath := tm.getAbsoluteFilePath(configuredTemplateDirectory, filePathFromZip, tmp) + require.Equal(t, "", writePath, filePathFromZip) + } + }) +} diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 6054fcea97..03725ec180 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -5,6 +5,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + updateutils "github.com/projectdiscovery/utils/update" ) var banner = fmt.Sprintf(` @@ -12,7 +13,7 @@ var banner = fmt.Sprintf(` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / -/_/ /_/\__,_/\___/_/\___/_/ v%s +/_/ /_/\__,_/\___/_/\___/_/ %s `, config.Version) // showBanner is used to show the banner to the user @@ -20,3 +21,9 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") } + +// NucleiToolUpdateCallback updates nuclei binary/tool to latest version +func NucleiToolUpdateCallback() { + showBanner() + updateutils.GetUpdateToolCallback("nuclei", config.Version)() +} diff --git a/v2/internal/runner/healthcheck.go b/v2/internal/runner/healthcheck.go index 7c10b6df98..abeb51ecc0 100644 --- a/v2/internal/runner/healthcheck.go +++ b/v2/internal/runner/healthcheck.go @@ -3,7 +3,6 @@ package runner import ( "fmt" "net" - "path/filepath" "runtime" "strings" @@ -23,15 +22,8 @@ func DoHealthCheck(options *types.Options) string { test.WriteString(fmt.Sprintf("Compiler: %s\n", runtime.Compiler)) var testResult string - - nucleiIgnorePath := config.GetIgnoreFilePath() - cf, _ := config.ReadConfiguration() - templatePath := "" - if cf != nil { - templatePath = cf.TemplatesDirectory - } - nucleiTemplatePath := filepath.Join(templatePath, "/", ".checksum") - for _, filename := range []string{options.ConfigPath, nucleiIgnorePath, nucleiTemplatePath} { + cfg := config.DefaultConfig + for _, filename := range []string{cfg.GetFlagsConfigFilePath(), cfg.GetIgnoreFilePath(), cfg.GetChecksumFilePath()} { ok, err := fileutil.IsReadable(filename) if ok { testResult = "Ok" diff --git a/v2/internal/runner/nucleicloud/utils.go b/v2/internal/runner/nucleicloud/utils.go index 77a87573d5..5586937cac 100644 --- a/v2/internal/runner/nucleicloud/utils.go +++ b/v2/internal/runner/nucleicloud/utils.go @@ -14,10 +14,7 @@ const DDMMYYYYhhmmss = "2006-01-02 15:04:05" // ReadCatalogChecksum reads catalog checksum from nuclei-templates repository func ReadCatalogChecksum() map[string]string { - config, _ := config.ReadConfiguration() - if config == nil { - return nil - } + config := config.DefaultConfig checksumFile := filepath.Join(config.TemplatesDirectory, "templates-checksum.txt") file, err := os.Open(checksumFile) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 3d880e125a..7a89f7de35 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -26,8 +26,10 @@ import ( ) func ConfigureOptions() error { + // with FileStringSliceOptions, FileNormalizedStringSliceOptions, FileCommaSeparatedStringSliceOptions + // if file has extension `.yaml,.json` we consider those as strings and not files to be read isFromFileFunc := func(s string) bool { - return !isTemplate(s) + return !config.IsTemplate(s) } goflags.FileNormalizedStringSliceOptions.IsFromFile = isFromFileFunc goflags.FileStringSliceOptions.IsFromFile = isFromFileFunc @@ -48,31 +50,9 @@ func ParseOptions(options *types.Options) { // Show the user the banner showBanner() - if options.TemplatesDirectory != "" && !filepath.IsAbs(options.TemplatesDirectory) { - cwd, _ := os.Getwd() - options.TemplatesDirectory = filepath.Join(cwd, options.TemplatesDirectory) - } - if options.Version { - gologger.Info().Msgf("Current Version: %s\n", config.Version) - os.Exit(0) - } if options.ShowVarDump { vardump.EnableVarDump = true } - if options.TemplatesVersion { - configuration, err := config.ReadConfiguration() - if err != nil { - gologger.Fatal().Msgf("Could not read template configuration: %s\n", err) - } - gologger.Info().Msgf("Public nuclei-templates version: %s (%s)\n", configuration.TemplateVersion, configuration.TemplatesDirectory) - if configuration.CustomS3TemplatesDirectory != "" { - gologger.Info().Msgf("Custom S3 templates location: %s\n", configuration.CustomS3TemplatesDirectory) - } - if configuration.CustomGithubTemplatesDirectory != "" { - gologger.Info().Msgf("Custom Github templates location: %s ", configuration.CustomGithubTemplatesDirectory) - } - os.Exit(0) - } if options.ShowActions { gologger.Info().Msgf("Showing available headless actions: ") for action := range engine.ActionStringToAction { @@ -139,7 +119,7 @@ func validateOptions(options *types.Options) error { return err } if options.Validate { - validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) + validateTemplatePaths(config.DefaultConfig.TemplatesDirectory, options.Templates, options.Workflows) } // Verify if any of the client certificate options were set since it requires all three to work properly diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index ba467d159a..54039e909d 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -1,25 +1,21 @@ package runner import ( - "bufio" - "bytes" "context" "encoding/json" - "fmt" - "io" "net/http" _ "net/http/pprof" "os" - "path/filepath" "reflect" "strconv" "strings" "sync/atomic" "time" + "github.com/projectdiscovery/nuclei/v2/internal/installer" "github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud" + updateutils "github.com/projectdiscovery/utils/update" - "github.com/blang/semver" "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/ratelimit" @@ -59,14 +55,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml" "github.com/projectdiscovery/retryablehttp-go" - stringsutil "github.com/projectdiscovery/utils/strings" ) // Runner is a client for running the enumeration process. type Runner struct { output output.Writer interactsh *interactsh.Client - templatesConfig *config.Config options *types.Options projectFile *projectfile.ProjectFile catalog catalog.Catalog @@ -101,16 +95,31 @@ func New(options *types.Options) (*Runner, error) { runner.cloudClient = nucleicloud.New(options.CloudURL, options.CloudAPIKey) } - if options.UpdateNuclei { - if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil { - return nil, err + // Version check by default + if config.DefaultConfig.CanCheckForUpdates() { + if err := installer.NucleiVersionCheck(); err != nil { + if options.Verbose || options.Debug { + gologger.Error().Msgf("nuclei version check failed got: %s\n", err) + } + } + // Check for template updates and update if available + tm := &installer.TemplateManager{} + if err := tm.FreshInstallIfNotExists(); err != nil { + gologger.Warning().Msgf("failed to install nuclei templates: %s\n", err) + } + if err := tm.UpdateIfOutdated(); err != nil { + gologger.Warning().Msgf("failed to update nuclei templates: %s\n", err) + } + + if config.DefaultConfig.NeedsIgnoreFileUpdate() { + if err := installer.UpdateIgnoreFile(); err != nil { + gologger.Warning().Msgf("failed to update nuclei ignore file: %s\n", err) + } } - return nil, nil } + if options.Validate { parsers.ShouldValidate = true - // Does not update the templates when validate flag is used - options.NoUpdateTemplates = true } // TODO: refactor to pass options reference globally without cycles @@ -119,9 +128,6 @@ func New(options *types.Options) (*Runner, error) { // parse the runner.options.GithubTemplateRepo and store the valid repos in runner.customTemplateRepos runner.customTemplates = customtemplates.ParseCustomTemplates(runner.options) - if err := runner.updateTemplates(); err != nil { - gologger.Error().Msgf("Could not update templates: %s\n", err) - } if options.Headless { if engine.MustDisableSandbox() { gologger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox\n") @@ -133,7 +139,7 @@ func New(options *types.Options) (*Runner, error) { runner.browser = browser } - runner.catalog = disk.NewCatalog(runner.options.TemplatesDirectory) + runner.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory) var httpclient *retryablehttp.Client if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" { @@ -374,34 +380,13 @@ func (r *Runner) Close() { func (r *Runner) RunEnumeration() error { // If user asked for new templates to be executed, collect the list from the templates' directory. if r.options.NewTemplates { - templatesLoaded, err := r.readNewTemplatesFile() - if err != nil { - return errors.Wrap(err, "could not get newly added templates") + if arr := config.DefaultConfig.GetNewAdditions(); len(arr) > 0 { + r.options.Templates = append(r.options.Templates, arr...) } - r.options.Templates = append(r.options.Templates, templatesLoaded...) } if len(r.options.NewTemplatesWithVersion) > 0 { - minVersion, err := semver.Parse("8.8.4") - if err != nil { - return errors.Wrap(err, "could not parse minimum version") - } - latestVersion, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return errors.Wrap(err, "could not get latest version") - } - for _, version := range r.options.NewTemplatesWithVersion { - current, err := semver.Parse(strings.Trim(version, "v")) - if err != nil { - return errors.Wrap(err, "could not parse current version") - } - if !(current.GT(minVersion) && current.LTE(latestVersion)) { - return fmt.Errorf("version should be greater than %s and less than %s", minVersion, latestVersion) - } - templatesLoaded, err := r.readNewTemplatesWithVersionFile(fmt.Sprintf("v%s", current)) - if err != nil { - return errors.Wrap(err, "could not get newly added templates for "+current.String()) - } - r.options.Templates = append(r.options.Templates, templatesLoaded...) + if arr := installer.GetNewTemplatesInVersions(r.options.NewTemplatesWithVersion...); len(arr) > 0 { + r.options.Templates = append(r.options.Templates, arr...) } } // Exclude ignored file for validation @@ -445,12 +430,7 @@ func (r *Runner) RunEnumeration() error { } executerOpts.WorkflowLoader = workflowLoader - templateConfig := r.templatesConfig - if templateConfig == nil { - templateConfig = &config.Config{} - } - - store, err := loader.New(loader.NewConfig(r.options, templateConfig, r.catalog, executerOpts)) + store, err := loader.New(loader.NewConfig(r.options, r.catalog, executerOpts)) if err != nil { return errors.Wrap(err, "could not load templates from config") } @@ -499,6 +479,8 @@ func (r *Runner) RunEnumeration() error { r.listAvailableStoreTemplates(store) os.Exit(0) } + + // display execution info like version , templates used etc r.displayExecutionInfo(store) // If not explicitly disabled, check if http based protocols @@ -685,136 +667,21 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { stats.Display(parsers.SyntaxErrorStats) stats.Display(parsers.RuntimeWarningsStats) - builder := &strings.Builder{} - if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { - builder.WriteString(" (") + cfg := config.DefaultConfig - if strings.Contains(config.Version, "-dev") { - builder.WriteString(r.colorizer.Blue("development").String()) - } else if config.Version == r.templatesConfig.NucleiLatestVersion { - builder.WriteString(r.colorizer.Green("latest").String()) - } else { - builder.WriteString(r.colorizer.Red("outdated").String()) - } - builder.WriteString(")") - } - messageStr := builder.String() - builder.Reset() - - gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr) + gologger.Info().Msgf("Current nuclei version: %v %v", config.Version, updateutils.GetVersionDescription(config.Version, cfg.LatestNucleiVersion)) + gologger.Info().Msgf("Current nuclei-templates version: %v %v", cfg.TemplateVersion, updateutils.GetVersionDescription(cfg.TemplateVersion, cfg.LatestNucleiTemplatesVersion)) - if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic - builder.WriteString(" (") - - if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion { - builder.WriteString(r.colorizer.Green("latest").String()) - } else { - builder.WriteString(r.colorizer.Red("outdated").String()) - } - builder.WriteString(")") - } - messageStr = builder.String() - builder.Reset() - - if r.templatesConfig != nil { - gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr) - } if len(store.Templates()) > 0 { - gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates()) - gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates())) + gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions())) + gologger.Info().Msgf("Templates loaded for current scan: %d", len(store.Templates())) } if len(store.Workflows()) > 0 { - gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) + gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows())) } if r.hmapInputProvider.Count() > 0 { - gologger.Info().Msgf("Targets loaded for scan: %d", r.hmapInputProvider.Count()) - } -} - -func (r *Runner) readNewTemplatesWithVersionFile(version string) ([]string, error) { - resp, err := retryablehttp.DefaultClient().Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version)) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, errors.New("version not found") - } - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - templatesList := []string{} - scanner := bufio.NewScanner(bytes.NewReader(data)) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - if isTemplate(text) { - templatesList = append(templatesList, text) - } - } - return templatesList, nil -} - -// readNewTemplatesFile reads newly added templates from directory if it exists -func (r *Runner) readNewTemplatesFile() ([]string, error) { - if r.templatesConfig == nil { - return nil, nil - } - additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions") - file, err := os.Open(additionsFile) - if err != nil { - return nil, err - } - defer file.Close() - - templatesList := []string{} - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - if isTemplate(text) { - templatesList = append(templatesList, text) - } + gologger.Info().Msgf("Targets loaded for current scan: %d", r.hmapInputProvider.Count()) } - return templatesList, nil -} - -// countNewTemplates returns the number of newly added templates -func (r *Runner) countNewTemplates() int { - if r.templatesConfig == nil { - return 0 - } - additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions") - file, err := os.Open(additionsFile) - if err != nil { - return 0 - } - defer file.Close() - - count := 0 - scanner := bufio.NewScanner(file) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - - if isTemplate(text) { - count++ - } - - } - return count -} - -// isTemplate is a callback function used by goflags to decide if given file should be read -// if it is not a nuclei-template file only then file is read -func isTemplate(filename string) bool { - return stringsutil.EqualFoldAny(filepath.Ext(filename), config.GetSupportTemplateFileExtensions()...) } // SaveResumeConfig to file diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 4c61439079..de667f7452 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -8,6 +8,7 @@ import ( "github.com/alecthomas/chroma/quick" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/gologger" @@ -36,9 +37,9 @@ func (r *Runner) verboseTemplate(tpl *templates.Template) { func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { gologger.Print().Msgf( - "\nListing available v.%s nuclei templates for %s", - r.templatesConfig.TemplateVersion, - r.templatesConfig.TemplatesDirectory, + "\nListing available %v nuclei templates for %v", + config.DefaultConfig.TemplateVersion, + config.DefaultConfig.TemplatesDirectory, ) for _, tpl := range store.Templates() { if hasExtraFlags(r.options) { @@ -63,7 +64,7 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) { } gologger.Silent().Msgf("Template: %s\n\n%s", path, tplBody) } else { - gologger.Silent().Msgf("%s\n", strings.TrimPrefix(tpl.Path, r.templatesConfig.TemplatesDirectory+string(filepath.Separator))) + gologger.Silent().Msgf("%s\n", strings.TrimPrefix(tpl.Path, config.DefaultConfig.TemplatesDirectory+string(filepath.Separator))) } } else { r.verboseTemplate(tpl) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go deleted file mode 100644 index 8cf463b7c8..0000000000 --- a/v2/internal/runner/update.go +++ /dev/null @@ -1,646 +0,0 @@ -package runner - -import ( - "archive/zip" - "bufio" - "bytes" - "context" - "crypto/md5" - "encoding/hex" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "regexp" - "runtime" - "strconv" - "strings" - - "github.com/apex/log" - "github.com/blang/semver" - "github.com/google/go-github/github" - "github.com/olekukonko/tablewriter" - "github.com/pkg/errors" - "golang.org/x/oauth2" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v2/pkg/external/customtemplates" - client "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/updatecheck" - "github.com/projectdiscovery/nuclei/v2/pkg/utils" - "github.com/projectdiscovery/retryablehttp-go" - fileutil "github.com/projectdiscovery/utils/file" - folderutil "github.com/projectdiscovery/utils/folder" - - "github.com/tj/go-update" - "github.com/tj/go-update/progress" - githubUpdateStore "github.com/tj/go-update/stores/github" -) - -const ( - userName = "projectdiscovery" - repoName = "nuclei-templates" - nucleiIgnoreFile = ".nuclei-ignore" - nucleiConfigFilename = ".templates-config.json" -) - -var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) - -// updateTemplates checks if the default list of nuclei-templates -// exist in the user's home directory, if not the latest revision -// is downloaded from GitHub. -// -// If the path exists but does not contain the latest version of public templates, -// the new version is downloaded from GitHub to the templates' directory, overwriting the old content. -func (r *Runner) updateTemplates() error { // TODO this method does more than just update templates. Should be refactored. - configDir, err := config.GetConfigDir() - if err != nil { - return err - } - _ = os.MkdirAll(configDir, 0755) - - if err := r.readInternalConfigurationFile(configDir); err != nil { - return errors.Wrap(err, "could not read configuration file") - } - - // If the config doesn't exist, create it now. - defaultTemplatesDirectory, err := utils.GetDefaultTemplatePath() - if err != nil { - return err - } - err = r.createDefaultConfig(defaultTemplatesDirectory) - if err != nil { - return err - } - - if r.options.TemplatesDirectory == "" { - // if no -tud flag passed then read from template config - if r.templatesConfig.TemplatesDirectory != "" { - r.options.TemplatesDirectory = r.templatesConfig.TemplatesDirectory - } else { - r.options.TemplatesDirectory = defaultTemplatesDirectory - } - } else if r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory { - // if -tud pass then update the templateConfig & it is diff then template config - r.templatesConfig.TemplatesDirectory, _ = filepath.Abs(r.options.TemplatesDirectory) - } - - // if disable update check flag is passed and no update template flag is passed - if (r.options.NoUpdateTemplates && !r.options.UpdateTemplates) || r.options.Cloud { - return nil - } - - client.InitNucleiVersion(config.Version) - r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions - - ctx := context.Background() - - var noTemplatesFound bool - if !fileutil.FolderExists(r.templatesConfig.TemplatesDirectory) { - noTemplatesFound = true - } - if r.templatesConfig.TemplateVersion == "" || noTemplatesFound { - return r.freshTemplateInstallation(configDir, ctx) - } - - // download | update the custom templates repos - if r.options.UpdateTemplates { - for _, ct := range r.customTemplates { - ct.Update(r.templatesConfig.TemplatesDirectory, ctx) - } - } - - latestVersion, currentVersion, err := getVersions(r) - if err != nil { - return err - } - - if latestVersion.EQ(currentVersion) { - if r.options.UpdateTemplates { - gologger.Info().Msgf("No new updates found for nuclei templates") - } - return config.WriteConfiguration(r.templatesConfig) - } - - if err := r.updateTemplatesWithVersion(latestVersion, currentVersion, r, ctx); err != nil { - return err - } - return nil -} - -// createDefaultConfig create template config file is template config is not found -func (r *Runner) createDefaultConfig(defaultTemplatesDirectory string) error { - // TODO remove customTemplate check in next version. - if r.templatesConfig == nil || r.templatesConfig.CustomGithubTemplatesDirectory == "" || r.templatesConfig.CustomS3TemplatesDirectory == "" { - currentConfig := &config.Config{ - TemplatesDirectory: defaultTemplatesDirectory, - NucleiVersion: config.Version, - CustomS3TemplatesDirectory: filepath.Join(defaultTemplatesDirectory, customtemplates.CustomS3TemplateDirectory), - CustomGithubTemplatesDirectory: filepath.Join(defaultTemplatesDirectory, customtemplates.CustomGithubTemplateDirectory), - } - r.templatesConfig = currentConfig - if writeErr := config.WriteConfiguration(currentConfig); writeErr != nil { - return errors.Wrap(writeErr, "could not write template configuration") - } - } - return nil -} - -// freshTemplateInstallation downloads the nuclei template and custom templates if new directory passed -func (r *Runner) freshTemplateInstallation(configDir string, ctx context.Context) error { - gologger.Info().Msgf("nuclei-templates are not installed, installing...\n") - - r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions - - version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return err - } - - // Download the repository and write the revision to a HEAD file. - asset, getErr := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion) - if getErr != nil { - return getErr - } - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) - - if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil { - return err - } - r.templatesConfig.TemplateVersion = version.String() - - if err := config.WriteConfiguration(r.templatesConfig); err != nil { - return err - } - gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s) to %s. GoodLuck!\n", version.String(), r.templatesConfig.TemplatesDirectory) - - // case where -gtr flag is passed for the first time installation - for _, ct := range r.customTemplates { - ct.Download(r.templatesConfig.TemplatesDirectory, ctx) - } - return nil -} - -func (r *Runner) updateTemplatesWithVersion(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error { - if latestVersion.GT(currentVersion) { - gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String()) - gologger.Info().Msgf("Downloading latest release...") - - if runner.options.TemplatesDirectory != "" { - runner.templatesConfig.TemplatesDirectory = runner.options.TemplatesDirectory - } - runner.templatesConfig.TemplateVersion = latestVersion.String() - - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", latestVersion.String(), runner.templatesConfig.TemplatesDirectory) - - asset, err := runner.getLatestReleaseFromGithub(runner.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return err - } - if _, err := runner.downloadReleaseAndUnzip(ctx, latestVersion.String(), asset.GetZipballURL()); err != nil { - return err - } - if err := config.WriteConfiguration(runner.templatesConfig); err != nil { - return err - } - gologger.Info().Msgf("Successfully updated nuclei-templates (v%s) to %s. GoodLuck!\n", latestVersion.String(), r.templatesConfig.TemplatesDirectory) - } - return nil -} - -func getVersions(runner *Runner) (semver.Version, semver.Version, error) { - // Get the configuration currently on disk. - verText := runner.templatesConfig.TemplateVersion - indices := reVersion.FindStringIndex(verText) - if indices == nil { - return semver.Version{}, semver.Version{}, fmt.Errorf("invalid release found with tag %s", verText) - } - if indices[0] > 0 { - verText = verText[indices[0]:] - } - - currentVersion, err := semver.Make(verText) - if err != nil { - return semver.Version{}, semver.Version{}, err - } - - latestVersion, err := semver.Parse(runner.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return semver.Version{}, semver.Version{}, err - } - return latestVersion, currentVersion, nil -} - -// readInternalConfigurationFile reads the internal configuration file for nuclei -func (r *Runner) readInternalConfigurationFile(configDir string) error { - templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) - if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { - configuration, readErr := config.ReadConfiguration() - if readErr != nil { - return readErr - } - r.templatesConfig = configuration - } - return nil -} - -// checkNucleiIgnoreFileUpdates checks .nuclei-ignore file for updates from GitHub -func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { - data, err := client.GetLatestIgnoreFile() - if err != nil { - return false - } - if len(data) > 0 { - _ = os.WriteFile(filepath.Join(configDir, nucleiIgnoreFile), data, 0644) - } - if r.templatesConfig != nil { - if err := config.WriteConfiguration(r.templatesConfig); err != nil { - gologger.Warning().Msgf("Could not get ignore-file from server: %s", err) - } - } - return true -} - -func getGHClientIncognito() *github.Client { - var tc *http.Client - return github.NewClient(tc) -} - -func getGHClientWithToken() *github.Client { - if token, ok := os.LookupEnv("GITHUB_TOKEN"); ok { - ctx := context.Background() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - oauthClient := oauth2.NewClient(ctx, ts) - return github.NewClient(oauthClient) - } - return nil -} - -// getLatestReleaseFromGithub returns the latest release from GitHub -func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) { - var ( - gitHubClient *github.Client - retried bool - ) - gitHubClient = getGHClientIncognito() -getRelease: - release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) - if err != nil { - // retry with authentication - if gitHubClient = getGHClientWithToken(); gitHubClient != nil && !retried { - retried = true - goto getRelease - } - return nil, err - } - if release == nil { - return nil, errors.New("no version found for the templates") - } - return release, nil -} - -// downloadReleaseAndUnzip downloads and unzips the release in a directory -func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create HTTP request to %s: %w", downloadURL, err) - } - - res, err := retryablehttp.DefaultClient().Do(req) - if err != nil { - return nil, fmt.Errorf("failed to download a release file from %s: %w", downloadURL, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode) - } - - buf, err := io.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("failed to create buffer for zip file: %w", err) - } - - reader := bytes.NewReader(buf) - zipReader, err := zip.NewReader(reader, reader.Size()) - if err != nil { - return nil, fmt.Errorf("failed to uncompress zip file: %w", err) - } - - // Create the template folder if it doesn't exist - if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, 0755); err != nil { - return nil, fmt.Errorf("failed to create template base folder: %w", err) - } - - results, err := r.compareAndWriteTemplates(zipReader) - if err != nil { - return nil, fmt.Errorf("failed to write templates: %w", err) - } - - if r.options.Verbose { - r.printUpdateChangelog(results, version) - } - checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum") - if err := writeTemplatesChecksum(checksumFile, results.checksums); err != nil { - return nil, errors.Wrap(err, "could not write checksum") - } - - return results, err -} - -type templateUpdateResults struct { - additions []string - deletions []string - modifications []string - totalCount int - checksums map[string]string -} - -// compareAndWriteTemplates compares and returns the stats of a template update operations. -func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdateResults, error) { - results := &templateUpdateResults{ - checksums: make(map[string]string), - } - - // We use file-checksums that are md5 hashes to store the list of files->hashes - // that have been downloaded previously. - // If the path isn't found in new update after being read from the previous checksum, - // it is removed. This allows us fine-grained control over the download process - // as well as solves a long problem with nuclei-template updates. - configuredTemplateDirectory := r.templatesConfig.TemplatesDirectory - checksumFile := filepath.Join(configuredTemplateDirectory, ".checksum") - templateChecksumsMap, _ := createTemplateChecksumsMap(checksumFile) - for _, zipTemplateFile := range zipReader.File { - templateAbsolutePath, skipFile, err := calculateTemplateAbsolutePath(zipTemplateFile.Name, configuredTemplateDirectory) - if err != nil { - return nil, err - } - if skipFile { - continue - } - - newTemplateChecksum, err := writeUnZippedTemplateFile(templateAbsolutePath, zipTemplateFile) - if err != nil { - return nil, err - } - - oldTemplateChecksum, checksumOk := templateChecksumsMap[templateAbsolutePath] - - relativeTemplatePath, err := filepath.Rel(configuredTemplateDirectory, templateAbsolutePath) - if err != nil { - return nil, fmt.Errorf("could not calculate relative path for template: %s. %w", templateAbsolutePath, err) - } - - if checksumOk && oldTemplateChecksum[0] != newTemplateChecksum { - results.modifications = append(results.modifications, relativeTemplatePath) - } - results.checksums[templateAbsolutePath] = newTemplateChecksum - results.totalCount++ - } - - var err error - results.additions, err = r.readNewTemplatesFile() - if err != nil { - results.additions = []string{} - } - - // If we don't find the previous file in the newly downloaded list, - // and it hasn't been changed on the disk, delete it. - for templatePath, templateChecksums := range templateChecksumsMap { - _, ok := results.checksums[templatePath] - if !ok && templateChecksums[0] == templateChecksums[1] { - _ = os.Remove(templatePath) - results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, configuredTemplateDirectory), string(os.PathSeparator))) - } - } - return results, nil -} - -func writeUnZippedTemplateFile(templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { - templateFile, err := os.OpenFile(templateAbsolutePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return "", fmt.Errorf("could not create template file: %w", err) - } - - zipTemplateFileReader, err := zipTemplateFile.Open() - if err != nil { - _ = templateFile.Close() - return "", fmt.Errorf("could not open archive to extract file: %w", err) - } - - md5Hash := md5.New() - - // Save file and also read into hash.Hash for md5 - if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, md5Hash)); err != nil { - _ = templateFile.Close() - return "", fmt.Errorf("could not write template file: %w", err) - } - - if err := templateFile.Close(); err != nil { - return "", fmt.Errorf("could not close file newly created template file: %w", err) - } - - checksum := hex.EncodeToString(md5Hash.Sum(nil)) - return checksum, nil -} - -func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory string) (string, bool, error) { - directory, fileName := filepath.Split(zipFilePath) - - // overwrite .nuclei-ignore everytime nuclei-templates are downloaded - if fileName == ".nuclei-ignore" { - return config.GetIgnoreFilePath(), false, nil - } - - if !strings.EqualFold(fileName, ".new-additions") { - if strings.TrimSpace(fileName) == "" || strings.HasPrefix(fileName, ".") || strings.EqualFold(fileName, "README.md") { - return "", true, nil - } - } - - var ( - directoryPathChunks []string - relativeDirectoryPathWithoutZipRoot string - ) - if folderutil.IsUnixOS() { - directoryPathChunks = strings.Split(directory, string(os.PathSeparator)) - } else if folderutil.IsWindowsOS() { - pathInfo, _ := folderutil.NewPathInfo(directory) - directoryPathChunks = pathInfo.Parts - } - relativeDirectoryPathWithoutZipRoot = filepath.Join(directoryPathChunks[1:]...) - - if strings.HasPrefix(relativeDirectoryPathWithoutZipRoot, ".") { - return "", true, nil - } - - templateDirectory := filepath.Join(configuredTemplateDirectory, relativeDirectoryPathWithoutZipRoot) - - if err := os.MkdirAll(templateDirectory, 0755); err != nil { - return "", false, fmt.Errorf("failed to create template folder: %s. %w", templateDirectory, err) - } - - return filepath.Join(templateDirectory, fileName), false, nil -} - -// createTemplateChecksumsMap reads the previous checksum file from the disk. -// Creates a map of template paths and their previous and currently calculated checksums as values. -func createTemplateChecksumsMap(checksumsFilePath string) (map[string][2]string, error) { - checksumFile, err := os.Open(checksumsFilePath) - if err != nil { - return nil, err - } - defer checksumFile.Close() - scanner := bufio.NewScanner(checksumFile) - - templatePathChecksumsMap := make(map[string][2]string) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - - parts := strings.Split(text, ",") - if len(parts) < 2 { - continue - } - templatePath := parts[0] - expectedTemplateChecksum := parts[1] - - templateFile, err := os.Open(templatePath) - if err != nil { - return nil, err - } - - hasher := md5.New() - if _, err := io.Copy(hasher, templateFile); err != nil { - return nil, err - } - templateFile.Close() - - values := [2]string{expectedTemplateChecksum} - values[1] = hex.EncodeToString(hasher.Sum(nil)) - templatePathChecksumsMap[templatePath] = values - } - return templatePathChecksumsMap, nil -} - -// writeTemplatesChecksum writes the nuclei-templates checksum data to disk. -func writeTemplatesChecksum(file string, checksum map[string]string) error { - f, err := os.Create(file) - if err != nil { - return err - } - defer f.Close() - - builder := &strings.Builder{} - for k, v := range checksum { - builder.WriteString(k) - builder.WriteString(",") - builder.WriteString(v) - builder.WriteString("\n") - - if _, checksumErr := f.WriteString(builder.String()); checksumErr != nil { - return err - } - builder.Reset() - } - return nil -} - -func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version string) { - if len(results.additions) > 0 && r.options.Verbose { - gologger.Print().Msgf("\nNewly added templates: \n\n") - - for _, addition := range results.additions { - gologger.Print().Msgf("%s", addition) - } - } - - gologger.Print().Msgf("\nNuclei Templates v%s Changelog\n", version) - data := [][]string{ - {strconv.Itoa(results.totalCount), strconv.Itoa(len(results.additions)), strconv.Itoa(len(results.deletions))}, - } - table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Total", "Added", "Removed"}) - for _, v := range data { - table.Append(v) - } - table.Render() -} - -// fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub -// -// This fetches the latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions -// If you want to disable this automatic update check, use -nut flag. -func (r *Runner) fetchLatestVersionsFromGithub(configDir string) { - versions, err := client.GetLatestNucleiTemplatesVersion() - if err != nil { - gologger.Warning().Msgf("Could not fetch latest releases: %s", err) - return - } - if r.templatesConfig != nil { - r.templatesConfig.NucleiLatestVersion = versions.Nuclei - r.templatesConfig.NucleiTemplatesLatestVersion = versions.Templates - - // If the fetch has resulted in new version of ignore file, update. - if r.templatesConfig.NucleiIgnoreHash == "" || r.templatesConfig.NucleiIgnoreHash != versions.IgnoreHash { - r.templatesConfig.NucleiIgnoreHash = versions.IgnoreHash - r.checkNucleiIgnoreFileUpdates(configDir) - } - } -} - -// updateNucleiVersionToLatest implements nuclei auto-update using GitHub Releases. -func updateNucleiVersionToLatest(verbose bool) error { - if verbose { - log.SetLevel(log.DebugLevel) - } - var command string - switch runtime.GOOS { - case "windows": - command = "nuclei.exe" - default: - command = "nuclei" - } - m := &update.Manager{ - Command: command, - Store: &githubUpdateStore.Store{ - Owner: "projectdiscovery", - Repo: "nuclei", - Version: config.Version, - }, - } - releases, err := m.LatestReleases() - if err != nil { - return errors.Wrap(err, "could not fetch latest release") - } - if len(releases) == 0 { - gologger.Info().Msgf("No new updates found for nuclei engine!") - return nil - } - - latest := releases[0] - var currentOS string - switch runtime.GOOS { - case "darwin": - currentOS = "macOS" - default: - currentOS = runtime.GOOS - } - final := latest.FindZip(currentOS, runtime.GOARCH) - if final == nil { - return fmt.Errorf("no compatible binary found for %s/%s", currentOS, runtime.GOARCH) - } - tarball, err := final.DownloadProxy(progress.Reader) - if err != nil { - return errors.Wrap(err, "could not download latest release") - } - if err := m.Install(tarball); err != nil { - return errors.Wrap(err, "could not install latest release") - } - gologger.Info().Msgf("Successfully updated to Nuclei %s\n", latest.Version) - return nil -} diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go deleted file mode 100644 index 1a8191de36..0000000000 --- a/v2/internal/runner/update_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package runner - -import ( - "archive/zip" - "context" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v2/pkg/testutils" -) - -func TestDownloadReleaseAndUnzipAddition(t *testing.T) { - gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{}) - - templatesDirectory, err := os.MkdirTemp("", "template-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(templatesDirectory) - - r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions} - - newTempDir, err := os.MkdirTemp("", "new-tmp-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(newTempDir) - - err = os.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), os.ModePerm) - require.Nil(t, err, "could not create base file") - err = os.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), os.ModePerm) - require.Nil(t, err, "could not create new file") - err = os.WriteFile(filepath.Join(newTempDir, ".new-additions"), []byte("new.yaml"), os.ModePerm) - require.Nil(t, err, "could not create new file") - - err = zipFromDirectory("new.zip", newTempDir) - require.Nil(t, err, "could not create new zip from directory") - defer os.Remove("new.zip") - - ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, "new.zip") - })) - defer ts2.Close() - - results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL) - require.Nil(t, err, "could not download release and unzip") - - require.Equal(t, "new.yaml", results.additions[0], "could not get correct new addition") -} - -func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { - gologger.DefaultLogger.SetWriter(&testutils.NoopWriter{}) - - baseTemplates, err := os.MkdirTemp("", "old-temp-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(baseTemplates) - - err = os.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), os.ModePerm) - require.Nil(t, err, "could not create write base file") - err = os.WriteFile(filepath.Join(baseTemplates, ".new-additions"), []byte("base.yaml"), os.ModePerm) - require.Nil(t, err, "could not create new file") - - err = zipFromDirectory("base.zip", baseTemplates) - require.Nil(t, err, "could not create zip from directory") - defer os.Remove("base.zip") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, "base.zip") - })) - defer ts.Close() - - templatesDirectory, err := os.MkdirTemp("", "template-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(templatesDirectory) - - r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: testutils.DefaultOptions} - - results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL) - require.Nil(t, err, "could not download release and unzip") - require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition") - - newTempDir, err := os.MkdirTemp("", "new-tmp-*") - require.Nil(t, err, "could not create temp directory") - defer os.RemoveAll(newTempDir) - - err = os.WriteFile(filepath.Join(newTempDir, ".new-additions"), []byte(""), os.ModePerm) - require.Nil(t, err, "could not create new file") - - err = zipFromDirectory("new.zip", newTempDir) - require.Nil(t, err, "could not create new zip from directory") - defer os.Remove("new.zip") - - ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, "new.zip") - })) - defer ts2.Close() - - results, err = r.downloadReleaseAndUnzip(context.Background(), "1.0.1", ts2.URL) - require.Nil(t, err, "could not download release and unzip") - - require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions") -} - -func TestCalculateTemplateAbsolutePathPositiveScenario(t *testing.T) { - configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") - defer os.RemoveAll(configuredTemplateDirectory) - - t.Run("positive scenarios", func(t *testing.T) { - zipFilePathsExpectedPathsMap := map[string]string{ - "nuclei-templates/cve/test.yaml": filepath.Join(configuredTemplateDirectory, "cve/test.yaml"), - "nuclei-templates/cve/test/test.yaml": filepath.Join(configuredTemplateDirectory, "cve/test/test.yaml"), - } - - for filePathFromZip, expectedTemplateAbsPath := range zipFilePathsExpectedPathsMap { - calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) - require.Nil(t, err) - require.Equal(t, expectedTemplateAbsPath, calculatedTemplateAbsPath) - require.False(t, skipFile) - } - }) -} - -func zipFromDirectory(zipPath, directory string) error { - file, err := os.Create(zipPath) - if err != nil { - return err - } - defer file.Close() - - w := zip.NewWriter(file) - defer w.Close() - - walker := func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - f, err := w.Create(strings.TrimPrefix(path, directory)) - if err != nil { - return err - } - _, err = io.Copy(f, file) - if err != nil { - return err - } - return nil - } - return filepath.Walk(directory, walker) -} diff --git a/v2/internal/runner/update_unix_test.go b/v2/internal/runner/update_unix_test.go deleted file mode 100644 index c9e1a9a2d3..0000000000 --- a/v2/internal/runner/update_unix_test.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build !windows - -package runner - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCalculateTemplateAbsolutePathNegativeScenario(t *testing.T) { - configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") - defer os.RemoveAll(configuredTemplateDirectory) - - t.Run("negative scenarios", func(t *testing.T) { - filePathsFromZip := []string{ - "./../nuclei-templates/../cve/test.yaml", - "nuclei-templates/../cve/test.yaml", - "nuclei-templates/cve/../test.yaml", - "nuclei-templates/././../cve/test.yaml", - "nuclei-templates/.././../cve/test.yaml", - "nuclei-templates/.././../cve/../test.yaml", - } - - for _, filePathFromZip := range filePathsFromZip { - calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) - require.Nil(t, err) - require.True(t, skipFile) - require.Equal(t, "", calculatedTemplateAbsPath) - } - }) -} diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go deleted file mode 100644 index 44e3141acb..0000000000 --- a/v2/pkg/catalog/config/config.go +++ /dev/null @@ -1,183 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - - jsoniter "github.com/json-iterator/go" - "github.com/mitchellh/go-homedir" - "github.com/pkg/errors" - "gopkg.in/yaml.v2" - - "github.com/projectdiscovery/gologger" - fileutil "github.com/projectdiscovery/utils/file" -) - -// Config contains the internal nuclei engine configuration -type Config struct { - TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"` - - CustomS3TemplatesDirectory string `json:"custom-s3-templates-directory"` - CustomGithubTemplatesDirectory string `json:"custom-github-templates-directory"` - CustomAzureTemplatesDirectory string `json:"custom-azure-templates-directory"` - - TemplateVersion string `json:"nuclei-templates-version,omitempty"` - NucleiVersion string `json:"nuclei-version,omitempty"` - NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` - - NucleiLatestVersion string `json:"nuclei-latest-version"` - NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"` -} - -// nucleiConfigFilename is the filename of nuclei configuration file. -const nucleiConfigFilename = ".templates-config.json" - -// Version is the current version of nuclei -const Version = `2.9.2-dev` - -var customConfigDirectory string - -func SetCustomConfigDirectory(dir string) { - customConfigDirectory = dir - if !fileutil.FolderExists(dir) { - _ = fileutil.CreateFolder(dir) - } -} -func getConfigDetails() (string, error) { - configDir, err := GetConfigDir() - if err != nil { - return "", errors.Wrap(err, "could not get home directory") - } - _ = os.MkdirAll(configDir, 0755) - templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) - return templatesConfigFile, nil -} - -// GetConfigDir returns the nuclei configuration directory -func GetConfigDir() (string, error) { - var ( - home string - err error - ) - if customConfigDirectory != "" { - home = customConfigDirectory - return home, nil - } - home, err = homedir.Dir() - if err != nil { - return "", err - } - return filepath.Join(home, ".config", "nuclei"), nil -} - -// ReadConfiguration reads the nuclei configuration file from disk. -func ReadConfiguration() (*Config, error) { - templatesConfigFile, err := getConfigDetails() - if err != nil { - return nil, err - } - file, err := os.Open(templatesConfigFile) - if err != nil { - return nil, err - } - defer file.Close() - - config := &Config{} - if err := jsoniter.NewDecoder(file).Decode(config); err != nil { - return nil, err - } - return config, nil -} - -// WriteConfiguration writes the updated nuclei configuration to disk -func WriteConfiguration(config *Config) error { - config.NucleiVersion = Version - - templatesConfigFile, err := getConfigDetails() - if err != nil { - return err - } - file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - defer file.Close() - - err = jsoniter.NewEncoder(file).Encode(config) - if err != nil { - return err - } - return nil -} - -const nucleiIgnoreFile = ".nuclei-ignore" - -// IgnoreFile is an internal nuclei template blocking configuration file -type IgnoreFile struct { - Tags []string `yaml:"tags"` - Files []string `yaml:"files"` -} - -// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths -func ReadIgnoreFile() IgnoreFile { - file, err := os.Open(GetIgnoreFilePath()) - if err != nil { - gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err) - return IgnoreFile{} - } - defer file.Close() - - ignore := IgnoreFile{} - if err := yaml.NewDecoder(file).Decode(&ignore); err != nil { - gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) - return IgnoreFile{} - } - return ignore -} - -var ( - // customIgnoreFilePath contains a custom path for the ignore file - customIgnoreFilePath string - // ErrCustomIgnoreFilePathNotExist is raised when the ignore file doesn't exist in the custom path - ErrCustomIgnoreFilePathNotExist = errors.New("Ignore file doesn't exist in custom path") - // ErrCustomFolderNotExist is raised when the custom ignore folder doesn't exist - ErrCustomFolderNotExist = errors.New("The custom ignore path doesn't exist") -) - -// OverrideIgnoreFilePath with a custom existing folder -func OverrideIgnoreFilePath(customPath string) error { - // custom path does not exist - if !fileutil.FolderExists(customPath) { - return ErrCustomFolderNotExist - } - // ignore file within the custom path does not exist - if !fileutil.FileExists(filepath.Join(customPath, nucleiIgnoreFile)) { - return ErrCustomIgnoreFilePathNotExist - } - customIgnoreFilePath = customPath - return nil -} - -// GetIgnoreFilePath returns the ignore file path for the runner -func GetIgnoreFilePath() string { - var defIgnoreFilePath string - - if customIgnoreFilePath != "" { - defIgnoreFilePath = filepath.Join(customIgnoreFilePath, nucleiIgnoreFile) - return defIgnoreFilePath - } - - configDir, err := GetConfigDir() - if err == nil { - _ = os.MkdirAll(configDir, 0755) - - defIgnoreFilePath = filepath.Join(configDir, nucleiIgnoreFile) - return defIgnoreFilePath - } - cwd, err := os.Getwd() - if err != nil { - return defIgnoreFilePath - } - cwdIgnoreFilePath := filepath.Join(cwd, nucleiIgnoreFile) - return cwdIgnoreFilePath -} diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go new file mode 100644 index 0000000000..86cdf4e714 --- /dev/null +++ b/v2/pkg/catalog/config/constants.go @@ -0,0 +1,48 @@ +package config + +import ( + "strings" + + "github.com/Masterminds/semver/v3" +) + +const ( + TemplateConfigFileName = ".templates-config.json" + NucleiTemplatesDirName = "nuclei-templates" + CustomS3TemplatesDirName = "s3" + CustomGithubTemplatesDirName = "github" + OfficialNucleiTeamplatesRepoName = "nuclei-templates" + NucleiIgnoreFileName = ".nuclei-ignore" + NucleiTemplatesCheckSumFileName = ".checksum" + NewTemplateAdditionsFileName = ".new-additions" + CLIConifgFileName = "config.yaml" + ReportingConfigFilename = "reporting-config.yaml" + // Version is the current version of nuclei + Version = `v2.9.2-dev` +) + +// IsOutdatedVersion compares two versions and returns true +// if current version is outdated +func IsOutdatedVersion(current, latest string) bool { + if latest == "" { + // if pdtm api call failed it's assumed that current version is outdated + // and it will be confirmed while updating from github + // this fixes `version string empty` errors + return true + } + current = trimDevIfExists(current) + currentVer, _ := semver.NewVersion(current) + newVer, _ := semver.NewVersion(latest) + if currentVer == nil || newVer == nil { + // fallback to naive comparison + return current == latest + } + return newVer.GreaterThan(currentVer) +} + +func trimDevIfExists(version string) string { + if strings.HasSuffix(version, "-dev") { + return strings.TrimSuffix(version, "-dev") + } + return version +} diff --git a/v2/pkg/catalog/config/ignorefile.go b/v2/pkg/catalog/config/ignorefile.go new file mode 100644 index 0000000000..b8a03544f9 --- /dev/null +++ b/v2/pkg/catalog/config/ignorefile.go @@ -0,0 +1,31 @@ +package config + +import ( + "os" + + "github.com/projectdiscovery/gologger" + "gopkg.in/yaml.v2" +) + +// IgnoreFile is an internal nuclei template blocking configuration file +type IgnoreFile struct { + Tags []string `yaml:"tags"` + Files []string `yaml:"files"` +} + +// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths +func ReadIgnoreFile() IgnoreFile { + file, err := os.Open(DefaultConfig.GetIgnoreFilePath()) + if err != nil { + gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err) + return IgnoreFile{} + } + defer file.Close() + + ignore := IgnoreFile{} + if err := yaml.NewDecoder(file).Decode(&ignore); err != nil { + gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) + return IgnoreFile{} + } + return ignore +} diff --git a/v2/pkg/catalog/config/nucleiconfig.go b/v2/pkg/catalog/config/nucleiconfig.go new file mode 100644 index 0000000000..ad5f0ea17e --- /dev/null +++ b/v2/pkg/catalog/config/nucleiconfig.go @@ -0,0 +1,302 @@ +package config + +import ( + "crypto/md5" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/projectdiscovery/gologger" + errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" + folderutil "github.com/projectdiscovery/utils/folder" +) + +// DefaultConfig is the default nuclei configuration +// all config values and default are centralized here +var DefaultConfig *Config + +type Config struct { + TemplatesDirectory string `json:"nuclei-templates-directory,omitempty"` + + CustomS3TemplatesDirectory string `json:"custom-s3-templates-directory"` + CustomGithubTemplatesDirectory string `json:"custom-github-templates-directory"` + + TemplateVersion string `json:"nuclei-templates-version,omitempty"` + NucleiIgnoreHash string `json:"nuclei-ignore-hash,omitempty"` + + // Latestxxx are not meant to be used directly and is used as + // local cache of nuclei version check endpoint + // these fields are only update during nuclei version check + // TODO: move these fields to a separate unexported struct as they are not meant to be used directly + LatestNucleiVersion string `json:"nuclei-latest-version"` + LatestNucleiTemplatesVersion string `json:"nuclei-templates-latest-version"` + LatestNucleiIgnoreHash string `json:"nuclei-latest-ignore-hash,omitempty"` + + // internal / unexported fields + disableUpdates bool `json:"-"` // disable updates both version check and template updates + homeDir string `json:"-"` // User Home Directory + configDir string `json:"-"` // Nuclei Global Config Directory +} + +// WriteVersionCheckData writes version check data to config file +func (c *Config) WriteVersionCheckData(ignorehash, nucleiVersion, templatesVersion string) error { + updated := false + if ignorehash != "" && c.LatestNucleiIgnoreHash != ignorehash { + c.LatestNucleiIgnoreHash = ignorehash + updated = true + } + if nucleiVersion != "" && c.LatestNucleiVersion != nucleiVersion { + c.LatestNucleiVersion = nucleiVersion + updated = true + } + if templatesVersion != "" && c.LatestNucleiTemplatesVersion != templatesVersion { + c.LatestNucleiTemplatesVersion = templatesVersion + updated = true + } + // write config to disk if any of the fields are updated + if updated { + return c.WriteTemplatesConfig() + } + return nil +} + +// DisableUpdateCheck disables update check and template updates +func (c *Config) DisableUpdateCheck() { + c.disableUpdates = true +} + +// CanCheckForUpdates returns true if update check is enabled +func (c *Config) CanCheckForUpdates() bool { + return !c.disableUpdates +} + +// NeedsTemplateUpdate returns true if template installation/update is required +func (c *Config) NeedsTemplateUpdate() bool { + return !c.disableUpdates && (c.TemplateVersion == "" || IsOutdatedVersion(c.TemplateVersion, c.LatestNucleiTemplatesVersion) || !fileutil.FolderExists(c.TemplatesDirectory)) +} + +// NeedsIngoreFileUpdate returns true if Ignore file hash is different (aka ignore file is outdated) +func (c *Config) NeedsIgnoreFileUpdate() bool { + return c.NucleiIgnoreHash == "" || c.NucleiIgnoreHash != c.LatestNucleiIgnoreHash +} + +// UpdateNucleiIgnoreHash updates the nuclei ignore hash in config +func (c *Config) UpdateNucleiIgnoreHash() error { + // calculate hash of ignore file and update config + ignoreFilePath := c.GetIgnoreFilePath() + if fileutil.FileExists(ignoreFilePath) { + bin, err := os.ReadFile(ignoreFilePath) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not read nuclei ignore file") + } + c.NucleiIgnoreHash = fmt.Sprintf("%x", md5.Sum(bin)) + // write config to disk + return c.WriteTemplatesConfig() + } + return errorutil.NewWithTag("config", "ignore file not found: could not update nuclei ignore hash") +} + +// GetConfigDir returns the nuclei configuration directory +func (c *Config) GetConfigDir() string { + return c.configDir +} + +// GetReportingConfigFilePath returns the nuclei reporting config file path +func (c *Config) GetReportingConfigFilePath() string { + return filepath.Join(c.configDir, ReportingConfigFilename) +} + +// GetIgnoreFilePath returns the nuclei ignore file path +func (c *Config) GetIgnoreFilePath() string { + return filepath.Join(c.configDir, NucleiIgnoreFileName) +} + +// GetTemplatesConfigFilePath returns checksum file path of nuclei templates +func (c *Config) GetChecksumFilePath() string { + return filepath.Join(c.TemplatesDirectory, NucleiTemplatesCheckSumFileName) +} + +// GetCLIOptsConfigFilePath returns the nuclei cli config file path +func (c *Config) GetFlagsConfigFilePath() string { + return filepath.Join(c.configDir, CLIConifgFileName) +} + +// GetNewAdditions returns new template additions in current template release +// if .new-additions file is not present empty slice is returned +func (c *Config) GetNewAdditions() []string { + arr := []string{} + newAdditionsPath := filepath.Join(c.TemplatesDirectory, NewTemplateAdditionsFileName) + if !fileutil.FileExists(newAdditionsPath) { + return arr + } + bin, err := os.ReadFile(newAdditionsPath) + if err != nil { + return arr + } + for _, v := range strings.Fields(string(bin)) { + if IsTemplate(v) { + arr = append(arr, v) + } + } + return arr +} + +// SetConfigDir sets the nuclei configuration directory +// and appropriate changes are made to the config +func (c *Config) SetConfigDir(dir string) { + c.configDir = dir + if err := c.createConfigDirIfNotExists(); err != nil { + gologger.Fatal().Msgf("Could not create nuclei config directory at %s: %s", c.configDir, err) + } + + // if folder already exists read config or create new + if err := c.ReadTemplatesConfig(); err != nil { + // create new config + applyDefaultConfig() + if err2 := c.WriteTemplatesConfig(); err2 != nil { + gologger.Fatal().Msgf("Could not create nuclei config file at %s: %s", c.getTemplatesConfigFilePath(), err2) + } + } + + // while other config files are optional, ignore file is mandatory + // since it is used to ignore templates with weak matchers + c.copyIgnoreFile() +} + +// SetTemplatesDir sets the new nuclei templates directory +func (c *Config) SetTemplatesDir(dirPath string) { + if dirPath != "" && !filepath.IsAbs(dirPath) { + cwd, _ := os.Getwd() + dirPath = filepath.Join(cwd, dirPath) + } + c.TemplatesDirectory = dirPath + // Update the custom templates directory + c.CustomGithubTemplatesDirectory = filepath.Join(dirPath, CustomGithubTemplatesDirName) + c.CustomS3TemplatesDirectory = filepath.Join(dirPath, CustomGithubTemplatesDirName) +} + +// SetTemplatesVersion sets the new nuclei templates version +func (c *Config) SetTemplatesVersion(version string) error { + c.TemplateVersion = version + // write config to disk + if err := c.WriteTemplatesConfig(); err != nil { + return errorutil.NewWithErr(err).Msgf("could not write nuclei config file at %s", c.getTemplatesConfigFilePath()) + } + return nil +} + +// ReadTemplatesConfig reads the nuclei templates config file +func (c *Config) ReadTemplatesConfig() error { + if !fileutil.FileExists(c.getTemplatesConfigFilePath()) { + return errorutil.NewWithTag("config", "nuclei config file at %s does not exist", c.getTemplatesConfigFilePath()) + } + var cfg *Config + bin, err := os.ReadFile(c.getTemplatesConfigFilePath()) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not read nuclei config file at %s", c.getTemplatesConfigFilePath()) + } + if err := json.Unmarshal(bin, &cfg); err != nil { + return errorutil.NewWithErr(err).Msgf("could not unmarshal nuclei config file at %s", c.getTemplatesConfigFilePath()) + } + // apply config + c.CustomGithubTemplatesDirectory = cfg.CustomGithubTemplatesDirectory + c.CustomS3TemplatesDirectory = cfg.CustomS3TemplatesDirectory + c.TemplatesDirectory = cfg.TemplatesDirectory + c.TemplateVersion = cfg.TemplateVersion + c.NucleiIgnoreHash = cfg.NucleiIgnoreHash + c.LatestNucleiIgnoreHash = cfg.LatestNucleiIgnoreHash + c.LatestNucleiTemplatesVersion = cfg.LatestNucleiTemplatesVersion + return nil +} + +// WriteTemplatesConfig writes the nuclei templates config file +func (c *Config) WriteTemplatesConfig() error { + // check if config folder exists if not create one + if err := c.createConfigDirIfNotExists(); err != nil { + return err + } + bin, err := json.Marshal(c) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to marshal nuclei config") + } + if err = os.WriteFile(c.getTemplatesConfigFilePath(), bin, 0600); err != nil { + return errorutil.NewWithErr(err).Msgf("failed to write nuclei config file at %s", c.getTemplatesConfigFilePath()) + } + return nil +} + +// getTemplatesConfigFilePath returns configDir/.templates-config.json file path +func (c *Config) getTemplatesConfigFilePath() string { + return filepath.Join(c.configDir, TemplateConfigFileName) +} + +// createConfigDirIfNotExists creates the nuclei config directory if not exists +func (c *Config) createConfigDirIfNotExists() error { + if !fileutil.FolderExists(c.configDir) { + if err := fileutil.CreateFolder(c.configDir); err != nil { + return errorutil.NewWithErr(err).Msgf("could not create nuclei config directory at %s", c.configDir) + } + } + return nil +} + +// copyIgnoreFile copies the nuclei ignore file default config directory +// to the current config directory +func (c *Config) copyIgnoreFile() { + if err := c.createConfigDirIfNotExists(); err != nil { + gologger.Error().Msgf("Could not create nuclei config directory at %s: %s", c.configDir, err) + return + } + ignoreFilePath := c.GetIgnoreFilePath() + if !fileutil.FileExists(ignoreFilePath) { + // copy ignore file + if err := fileutil.CopyFile(filepath.Join(getDefaultConfigDir(), NucleiIgnoreFileName), ignoreFilePath); err != nil { + gologger.Error().Msgf("Could not copy nuclei ignore file at %s: %s", ignoreFilePath, err) + } + } +} + +func init() { + ConfigDir := getDefaultConfigDir() + if !fileutil.FolderExists(ConfigDir) { + if err := fileutil.CreateFolder(ConfigDir); err != nil { + gologger.Error().Msgf("failed to create config directory at %v got: %s", ConfigDir, err) + } + } + DefaultConfig = &Config{ + homeDir: folderutil.HomeDirOrDefault(""), + configDir: ConfigDir, + } + // try to read config from file + if err := DefaultConfig.ReadTemplatesConfig(); err != nil { + gologger.Verbose().Msgf("config file not found, creating new config file at %s", DefaultConfig.getTemplatesConfigFilePath()) + applyDefaultConfig() + // write config to file + if err := DefaultConfig.WriteTemplatesConfig(); err != nil { + gologger.Error().Msgf("failed to write config file at %s got: %s", DefaultConfig.getTemplatesConfigFilePath(), err) + } + } +} + +func getDefaultConfigDir() string { + // Review Needed: Earlier a dependency was used to locate home dir + // i.e "github.com/mitchellh/go-homedir" not sure if it is needed + // Even if such case exists it should be abstracted via below function call in utils/folder + homedir := folderutil.HomeDirOrDefault("") + // TBD: we should probably stick to specification and use config directories provided by distro + // instead of manually creating one since $HOME/.config/ is config directory of Linux desktops + // Ref: https://pkg.go.dev/os#UserConfigDir + // some distros like NixOS or others have totally different config directories this causes issues for us (since we are not using os.UserConfigDir) + userCfgDir := filepath.Join(homedir, ".config") + return filepath.Join(userCfgDir, "nuclei") +} + +// Add Default Config adds default when .templates-config.json file is not present +func applyDefaultConfig() { + DefaultConfig.TemplatesDirectory = filepath.Join(DefaultConfig.homeDir, NucleiTemplatesDirName) + DefaultConfig.CustomGithubTemplatesDirectory = filepath.Join(DefaultConfig.TemplatesDirectory, CustomGithubTemplatesDirName) + DefaultConfig.CustomS3TemplatesDirectory = filepath.Join(DefaultConfig.TemplatesDirectory, CustomS3TemplatesDirName) +} diff --git a/v2/pkg/catalog/config/template.go b/v2/pkg/catalog/config/template.go index b36bd777bf..806a67f014 100644 --- a/v2/pkg/catalog/config/template.go +++ b/v2/pkg/catalog/config/template.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v2/pkg/templates/extensions" + stringsutil "github.com/projectdiscovery/utils/strings" ) // TemplateFormat @@ -33,3 +34,9 @@ func GetTemplateFormatFromExt(filePath string) TemplateFormat { func GetSupportTemplateFileExtensions() []string { return []string{extensions.YAML, extensions.JSON} } + +// isTemplate is a callback function used by goflags to decide if given file should be read +// if it is not a nuclei-template file only then file is read +func IsTemplate(filename string) bool { + return stringsutil.EqualFoldAny(filepath.Ext(filename), GetSupportTemplateFileExtensions()...) +} diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 9b963d3037..773f32492a 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -7,7 +7,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + cfg "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" @@ -41,9 +41,8 @@ type Config struct { ExcludeIds []string IncludeConditions []string - Catalog catalog.Catalog - ExecutorOptions protocols.ExecuterOptions - TemplatesDirectory string + Catalog catalog.Catalog + ExecutorOptions protocols.ExecuterOptions } // Store is a storage for loaded nuclei templates @@ -65,7 +64,7 @@ type Store struct { } // NewConfig returns a new loader config -func NewConfig(options *types.Options, templateConfig *config.Config, catalog catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { +func NewConfig(options *types.Options, catalog catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { loaderConfig := Config{ Templates: options.Templates, Workflows: options.Workflows, @@ -82,7 +81,6 @@ func NewConfig(options *types.Options, templateConfig *config.Config, catalog ca IncludeTags: options.IncludeTags, IncludeIds: options.IncludeIds, ExcludeIds: options.ExcludeIds, - TemplatesDirectory: templateConfig.TemplatesDirectory, Protocols: options.Protocols, ExcludeProtocols: options.ExcludeProtocols, IncludeConditions: options.IncludeConditions, @@ -142,7 +140,7 @@ func New(config *Config) (*Store, error) { } // Handle a case with no templates or workflows, where we use base directory if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlBasedTemplatesProvided { - store.finalTemplates = []string{config.TemplatesDirectory} + store.finalTemplates = []string{cfg.DefaultConfig.TemplatesDirectory} } return store, nil } diff --git a/v2/pkg/catalog/loader/loader_test.go b/v2/pkg/catalog/loader/loader_test.go index 69d1f7d647..9c3f4f561c 100644 --- a/v2/pkg/catalog/loader/loader_test.go +++ b/v2/pkg/catalog/loader/loader_test.go @@ -4,6 +4,7 @@ import ( "reflect" "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/stretchr/testify/require" ) @@ -19,28 +20,26 @@ func TestLoadTemplates(t *testing.T) { require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates") templatesDirectory := "/test" + config.DefaultConfig.TemplatesDirectory = templatesDirectory t.Run("blank", func(t *testing.T) { store, err := New(&Config{ - TemplatesDirectory: templatesDirectory, - Catalog: catalog, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") }) t.Run("only-tags", func(t *testing.T) { store, err := New(&Config{ - Tags: []string{"cves"}, - TemplatesDirectory: templatesDirectory, - Catalog: catalog, + Tags: []string{"cves"}, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") }) t.Run("tags-with-path", func(t *testing.T) { store, err := New(&Config{ - Tags: []string{"cves"}, - TemplatesDirectory: templatesDirectory, - Catalog: catalog, + Tags: []string{"cves"}, + Catalog: catalog, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") diff --git a/v2/pkg/parsers/workflow_loader.go b/v2/pkg/parsers/workflow_loader.go index d450b34d12..a2fc123b82 100644 --- a/v2/pkg/parsers/workflow_loader.go +++ b/v2/pkg/parsers/workflow_loader.go @@ -2,6 +2,7 @@ package parsers import ( "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -40,7 +41,7 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) } func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string { - includedTemplates, errs := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory}) + includedTemplates, errs := w.options.Catalog.GetTemplatesPath([]string{config.DefaultConfig.TemplatesDirectory}) for template, err := range errs { gologger.Error().Msgf("Could not find template '%s': %s", template, err) } diff --git a/v2/pkg/protocols/common/automaticscan/automaticscan.go b/v2/pkg/protocols/common/automaticscan/automaticscan.go index 10bd7fe8db..780314edcb 100644 --- a/v2/pkg/protocols/common/automaticscan/automaticscan.go +++ b/v2/pkg/protocols/common/automaticscan/automaticscan.go @@ -57,7 +57,7 @@ func New(opts Options) (*Service, error) { } var mappingData map[string]string - config, err := config.ReadConfiguration() + config := config.DefaultConfig if err == nil { mappingFile := filepath.Join(config.TemplatesDirectory, mappingFilename) if file, err := os.Open(mappingFile); err == nil { diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 58c2419b88..9cc8a97612 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" ) // PayloadGenerator is the generator struct for generating payloads @@ -16,7 +17,7 @@ type PayloadGenerator struct { } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath, templateDirectory string, sandbox bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, sandbox bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -42,7 +43,7 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath, t return nil, err } - compiled, err := generator.loadPayloads(payloadsFinal, templatePath, templateDirectory, sandbox) + compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, sandbox) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 00a1a6212d..7187808904 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -12,7 +12,7 @@ func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -32,7 +32,7 @@ func TestPitchforkGenerator(t *testing.T) { passwords := []string{"password1", "password2", "password3"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -54,7 +54,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 4f6241e4f5..7056305ea6 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -98,7 +98,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { if len(request.Payloads) > 0 { var err error - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.TemplatesDirectory, options.Options.Sandbox, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.Sandbox, options.Catalog, options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index a6a15b898e..7cfe74aab8 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -350,7 +350,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.TemplatesDirectory, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index b30a5cd110..89447b9936 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -34,7 +34,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := disk.NewCatalog("") - req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", "", false, catalogInstance, "") + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") generator := req.newGenerator(false) @@ -58,7 +58,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := disk.NewCatalog("") - req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", "", false, catalogInstance, "") + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "") require.Nil(t, err, "could not create generator") generator := req.newGenerator(false) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 0ac06248dc..0d8995b5ac 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.TemplatesDirectory, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, request.options.Catalog, request.options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 324194043f..bd5b655c41 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -105,7 +105,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.dialer = client if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.TemplatesDirectory, request.options.Options.Sandbox, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.Sandbox, options.Catalog, options.Options.AttackType) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index 23197ad1e9..7ecd4e4216 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -2,8 +2,8 @@ package reporting import ( "os" - "path/filepath" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" json_exporter "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonexporter" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/jsonl" @@ -12,7 +12,6 @@ import ( "errors" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -177,11 +176,7 @@ func New(options *Options, db string) (Client, error) { // CreateConfigIfNotExists creates report-config if it doesn't exists func CreateConfigIfNotExists() error { - config, err := config.GetConfigDir() - if err != nil { - return errorutil.NewWithErr(err).Msgf("could not get config directory") - } - reportingConfig := filepath.Join(config, "report-config.yaml") + reportingConfig := config.DefaultConfig.GetReportingConfigFilePath() if fileutil.FileExists(reportingConfig) { return nil diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index a2ed7d6153..3b5fdad798 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -310,7 +310,7 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th builder.WriteString(event.CURLCommand) builder.WriteString("\n{code}") } - builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei v%s](https://github.com/projectdiscovery/nuclei)", config.Version)) + builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei %s](https://github.com/projectdiscovery/nuclei)", config.Version)) data := builder.String() return data } diff --git a/v2/pkg/templates/compile_test.go b/v2/pkg/templates/compile_test.go index 91d3720e6c..575d039332 100644 --- a/v2/pkg/templates/compile_test.go +++ b/v2/pkg/templates/compile_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" @@ -44,7 +45,7 @@ func setup() { ProjectFile: nil, IssuesClient: nil, Browser: nil, - Catalog: disk.NewCatalog(options.TemplatesDirectory), + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), } workflowLoader, err := parsers.NewLoader(&executerOpts) diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index e11a439622..5d08d06d40 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -9,6 +9,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" @@ -31,14 +32,12 @@ var DefaultOptions = &types.Options{ DebugRequests: false, DebugResponse: false, Silent: false, - Version: false, Verbose: false, NoColor: true, UpdateTemplates: false, JSONL: false, JSONRequests: false, EnableProgressBar: false, - TemplatesVersion: false, TemplateList: false, Stdin: false, StopAtFirstMatch: false, @@ -56,7 +55,6 @@ var DefaultOptions = &types.Options{ TargetsFilePath: "", Output: "", Proxy: []string{}, - TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, ExcludedTemplates: []string{}, @@ -90,7 +88,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco ProjectFile: nil, IssuesClient: nil, Browser: nil, - Catalog: disk.NewCatalog(options.TemplatesDirectory), + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), } return executerOpts diff --git a/v2/pkg/types/resume.go b/v2/pkg/types/resume.go index 198631ad73..b65a930145 100644 --- a/v2/pkg/types/resume.go +++ b/v2/pkg/types/resume.go @@ -14,10 +14,7 @@ import ( const DefaultResumeFileName = "resume-%s.cfg" func DefaultResumeFilePath() string { - configDir, err := config.GetConfigDir() - if err != nil { - return fmt.Sprintf(DefaultResumeFileName, xid.New().String()) - } + configDir := config.DefaultConfig.GetConfigDir() resumeFile := filepath.Join(configDir, fmt.Sprintf(DefaultResumeFileName, xid.New().String())) return resumeFile } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 10ea6d8972..0cae038945 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -76,7 +76,7 @@ type Options struct { // List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input) Proxy goflags.StringSlice // TemplatesDirectory is the directory to use for storing templates - TemplatesDirectory string + NewTemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests TraceLogFile string // ErrorLogFile specifies a file to write with the errors of all requests @@ -215,8 +215,6 @@ type Options struct { AutomaticScan bool // Silent suppresses any extra text and only writes found URLs on screen. Silent bool - // Version specifies if we should just show version and exit - Version bool // Validate validates the templates passed to nuclei. Validate bool // NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs). @@ -228,7 +226,7 @@ type Options struct { ShowVarDump bool // No-Color disables the colored output. NoColor bool - // UpdateTemplates updates the templates installed at startup + // UpdateTemplates updates the templates installed at startup (also used by cloud to update datasources) UpdateTemplates bool // JSON writes json line output to files JSONL bool @@ -242,8 +240,6 @@ type Options struct { Cloud bool // EnableProgressBar enables progress bar EnableProgressBar bool - // TemplatesVersion shows the templates installed version - TemplatesVersion bool // TemplateDisplay displays the template contents TemplateDisplay bool // TemplateList lists available templates @@ -268,10 +264,6 @@ type Options struct { NewTemplatesWithVersion goflags.StringSlice // NoInteractsh disables use of interactsh server for interaction polling NoInteractsh bool - // UpdateNuclei checks for an update for the nuclei engine - UpdateNuclei bool - // NoUpdateTemplates disables checking for nuclei templates updates - NoUpdateTemplates bool // EnvironmentVariables enables support for environment variables EnvironmentVariables bool // MatcherStatus displays optional status for the failed matches as well @@ -330,8 +322,6 @@ type Options struct { UncoverLimit int // Uncover search delay UncoverDelay int - // ConfigPath contains the config path (used by healthcheck) - ConfigPath string // ScanAllIPs associated to a dns record ScanAllIPs bool // IPVersion to scan (4,6) diff --git a/v2/pkg/utils/template_path.go b/v2/pkg/utils/template_path.go index 7fac7e4a44..0e54d9904c 100644 --- a/v2/pkg/utils/template_path.go +++ b/v2/pkg/utils/template_path.go @@ -1,10 +1,8 @@ package utils import ( - "path/filepath" "strings" - "github.com/mitchellh/go-homedir" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" ) @@ -13,16 +11,11 @@ const ( TemplatesRepoURL = "https://github.com/projectdiscovery/nuclei-templates/blob/main/" ) -var configData *config.Config - -func init() { - configData, _ = config.ReadConfiguration() -} - // TemplatePathURL returns the Path and URL for the provided template func TemplatePathURL(fullPath string) (string, string) { var templateDirectory string - if configData != nil && configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { + configData := config.DefaultConfig + if configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { templateDirectory = configData.TemplatesDirectory } else { return "", "" @@ -32,12 +25,3 @@ func TemplatePathURL(fullPath string) (string, string) { templateURL := TemplatesRepoURL + finalPath return finalPath, templateURL } - -// GetDefaultTemplatePath on default settings -func GetDefaultTemplatePath() (string, error) { - home, err := homedir.Dir() - if err != nil { - return "", err - } - return filepath.Join(home, "nuclei-templates"), nil -}