Merge pull request #638 from projectdiscovery/fix-bugs

v2.3.1 bug fixes

v2/cmd/nuclei/main.go

@@ -24,9 +24,7 @@ func main() {
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
 	}
-
 	nucleiRunner.RunEnumeration()
-	nucleiRunner.Close()
 }
 
 func readConfig() {
@@ -77,7 +75,7 @@ based on templates offering massive extensibility and ease of use.`)
 	set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
 	set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
 	set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file")
-	set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database")
+	set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)")
 	set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
 	set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags")
 	set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
@@ -87,6 +85,7 @@ based on templates offering massive extensibility and ease of use.`)
 	set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
 	set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
 	set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
+	set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
 	_ = set.Parse()
 
 	if cfgFile != "" {

v2/go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/projectdiscovery/clistats v0.0.8
 	github.com/projectdiscovery/collaborator v0.0.2
 	github.com/projectdiscovery/fastdialer v0.0.8
-	github.com/projectdiscovery/goflags v0.0.3
+	github.com/projectdiscovery/goflags v0.0.4
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
 	github.com/projectdiscovery/rawhttp v0.0.6

v2/go.sum

@@ -201,8 +201,8 @@ github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5
 github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
 github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
 github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
-github.com/projectdiscovery/goflags v0.0.3 h1:5s9qblVIP/dQt7Mr3PMvZLvekEyioOS5oZtZ6ncLQHA=
-github.com/projectdiscovery/goflags v0.0.3/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY=
+github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=

v2/internal/runner/config.go

@@ -16,13 +16,15 @@ type nucleiConfig struct {
 	TemplatesDirectory string    `json:"templates-directory,omitempty"`
 	CurrentVersion     string    `json:"current-version,omitempty"`
 	LastChecked        time.Time `json:"last-checked,omitempty"`
-
+	IgnoreURL          string    `json:"ignore-url,omitempty"`
+	NucleiVersion      string    `json:"nuclei-version,omitempty"`
+	LastCheckedIgnore  time.Time `json:"last-checked-ignore,omitempty"`
 	// IgnorePaths ignores all the paths listed unless specified manually
 	IgnorePaths []string `json:"ignore-paths,omitempty"`
 }
 
 // nucleiConfigFilename is the filename of nuclei configuration file.
-const nucleiConfigFilename = ".nuclei-config.json"
+const nucleiConfigFilename = ".templates-config.json"
 
 var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
 
@@ -32,8 +34,10 @@ func readConfiguration() (*nucleiConfig, error) {
 	if err != nil {
 		return nil, err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
 	file, err := os.Open(templatesConfigFile)
 	if err != nil {
 		return nil, err
@@ -54,9 +58,16 @@ func (r *Runner) writeConfiguration(config *nucleiConfig) error {
 	if err != nil {
 		return err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
+	if config.IgnoreURL == "" {
+		config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	}
 	config.LastChecked = time.Now()
-	templatesConfigFile := path.Join(home, nucleiConfigFilename)
+	config.LastCheckedIgnore = time.Now()
+	config.NucleiVersion = Version
+	templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
 	file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
 	if err != nil {
 		return err
@@ -95,7 +106,15 @@ func (r *Runner) readNucleiIgnoreFile() {
 
 // getIgnoreFilePath returns the ignore file path for the runner
 func (r *Runner) getIgnoreFilePath() string {
-	defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile)
+	var defIgnoreFilePath string
+
+	home, err := os.UserHomeDir()
+	if err == nil {
+		configDir := path.Join(home, "/.config", "/nuclei")
+		_ = os.MkdirAll(configDir, os.ModePerm)
+
+		defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
+	}
 
 	cwd, err := os.Getwd()
 	if err != nil {

v2/internal/runner/runner.go

@@ -19,13 +19,15 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/remeh/sizedwaitgroup"
 	"github.com/rs/xid"
 	"go.uber.org/atomic"
 	"go.uber.org/ratelimit"
+	"gopkg.in/yaml.v2"
 )
 
 // Runner is a client for running the enumeration process.
@@ -39,7 +41,7 @@ type Runner struct {
 	catalog         *catalog.Catalog
 	progress        progress.Progress
 	colorizer       aurora.Aurora
-	issuesClient    *issues.Client
+	issuesClient    *reporting.Client
 	severityColors  *colorizer.Colorizer
 	browser         *engine.Browser
 	ratelimiter     ratelimit.Limiter
@@ -66,13 +68,36 @@ func New(options *types.Options) (*Runner, error) {
 	}
 	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
 
+	var reportingOptions *reporting.Options
 	if options.ReportingConfig != "" {
-		if client, err := issues.New(options.ReportingConfig, options.ReportingDB); err != nil {
+		file, err := os.Open(options.ReportingConfig)
+		if err != nil {
+			gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err)
+		}
+
+		reportingOptions = &reporting.Options{}
+		if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
+			file.Close()
+			gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr)
+		}
+		file.Close()
+	}
+	if options.DiskExportDirectory != "" {
+		if reportingOptions != nil {
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		}
+	}
+	if reportingOptions != nil {
+		if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
 			gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
 		} else {
 			runner.issuesClient = client
 		}
 	}
+
 	// output coloring
 	useColor := !options.NoColor
 	runner.colorizer = aurora.NewAurora(useColor)
@@ -197,6 +222,8 @@ func (r *Runner) Close() {
 // RunEnumeration sets up the input layer for giving input nuclei.
 // binary and runs the actual enumeration
 func (r *Runner) RunEnumeration() {
+	defer r.Close()
+
 	// If we have no templates, run on whole template directory with provided tags
 	if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) {
 		r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)

v2/internal/runner/update.go

@@ -41,6 +41,8 @@ func (r *Runner) updateTemplates() error {
 	if err != nil {
 		return err
 	}
+	configDir := path.Join(home, "/.config", "/nuclei")
+	_ = os.MkdirAll(configDir, os.ModePerm)
 
 	templatesConfigFile := path.Join(home, nucleiConfigFilename)
 	if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
@@ -51,15 +53,61 @@ func (r *Runner) updateTemplates() error {
 		r.templatesConfig = config
 	}
 
+	ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
+	if r.templatesConfig == nil {
+		currentConfig := &nucleiConfig{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+			IgnoreURL:          ignoreURL,
+			NucleiVersion:      Version,
+		}
+		if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
+			return errors.Wrap(writeErr, "could not write template configuration")
+		}
+		r.templatesConfig = currentConfig
+	}
+	// Check if last checked for nuclei-ignore is more than 1 hours.
+	// and if true, run the check.
+	if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
+		if r.templatesConfig != nil && r.templatesConfig.IgnoreURL == "" {
+			ignoreURL = r.templatesConfig.IgnoreURL
+		}
+		gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
+		if reqErr == nil {
+			resp, httpGet := http.DefaultClient.Do(req)
+			if httpGet != nil {
+				if resp != nil && resp.Body != nil {
+					resp.Body.Close()
+				}
+				gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
+			} else {
+				data, _ := ioutil.ReadAll(resp.Body)
+				resp.Body.Close()
+
+				if len(data) > 0 {
+					_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
+				}
+				if r.templatesConfig != nil {
+					r.templatesConfig.LastCheckedIgnore = time.Now()
+				}
+			}
+		}
+		cancel()
+	}
+
 	ctx := context.Background()
-	if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
+	if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
 		if !r.options.UpdateTemplates {
-			gologger.Warning().Msgf("nuclei-templates are not installed, use update-templates flag.\n")
+			gologger.Warning().Msgf("nuclei-templates are not installed (or indexed), use update-templates flag.\n")
 			return nil
 		}
 
 		// Use custom location if user has given a template directory
-		r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")}
+		r.templatesConfig = &nucleiConfig{
+			TemplatesDirectory: path.Join(home, "nuclei-templates"),
+		}
 		if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
 			r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
 		}
@@ -132,7 +180,6 @@ func (r *Runner) updateTemplates() error {
 		if err != nil {
 			return err
 		}
-
 		err = r.writeConfiguration(r.templatesConfig)
 		if err != nil {
 			return err

v2/pkg/progress/progress.go

@@ -183,7 +183,7 @@ func (p *StatsTicker) getMetrics() map[string]interface{} {
 	results["templates"] = clistats.String(templates)
 	hosts, _ := p.stats.GetStatic("hosts")
 	results["hosts"] = clistats.String(hosts)
-	matched, _ := p.stats.GetStatic("matched")
+	matched, _ := p.stats.GetCounter("matched")
 	results["matched"] = clistats.String(matched)
 	requests, _ := p.stats.GetCounter("requests")
 	results["requests"] = clistats.String(requests)

v2/pkg/protocols/protocols.go

@@ -9,7 +9,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"go.uber.org/ratelimit"
 )
@@ -39,7 +39,7 @@ type ExecuterOptions struct {
 	// Options contains configuration options for the executer.
 	Options *types.Options
 	// IssuesClient is a client for nuclei issue tracker reporting
-	IssuesClient *issues.Client
+	IssuesClient *reporting.Client
 	// Progress is a progress client for scan reporting
 	Progress progress.Progress
 	// RateLimiter is a rate-limiter for limiting sent number of requests.

v2/pkg/reporting/exporters/disk/disk.go

@@ -0,0 +1,60 @@
+package disk
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
+)
+
+type Exporter struct {
+	directory string
+	options   *Options
+}
+
+// Options contains the configuration options for github issue tracker client
+type Options struct {
+	// Directory is the directory to export found results to
+	Directory string `yaml:"directory"`
+}
+
+// New creates a new disk exporter integration client based on options.
+func New(options *Options) (*Exporter, error) {
+	directory := options.Directory
+	if options.Directory == "" {
+		dir, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		directory = dir
+	}
+	_ = os.MkdirAll(directory, os.ModePerm)
+	return &Exporter{options: options, directory: directory}, nil
+}
+
+// Export exports a passed result event to disk
+func (i *Exporter) Export(event *output.ResultEvent) error {
+	summary := format.Summary(event)
+	description := format.MarkdownDescription(event)
+
+	filenameBuilder := &strings.Builder{}
+	filenameBuilder.WriteString(event.TemplateID)
+	filenameBuilder.WriteString("-")
+	filenameBuilder.WriteString(strings.ReplaceAll(strings.ReplaceAll(event.Matched, "/", "_"), ":", "_"))
+	filenameBuilder.WriteString(".md")
+	finalFilename := filenameBuilder.String()
+
+	dataBuilder := &bytes.Buffer{}
+	dataBuilder.WriteString("### ")
+	dataBuilder.WriteString(summary)
+	dataBuilder.WriteString("\n---\n")
+	dataBuilder.WriteString(description)
+	data := dataBuilder.Bytes()
+
+	err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644)
+	return err
+}

v2/pkg/reporting/issues/format/format.go → v2/pkg/reporting/format/format.go

@@ -46,11 +46,22 @@ func MarkdownDescription(event *output.ResultEvent) string {
 	for k, v := range event.Info {
 		builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
 	}
-	builder.WriteString("\n**Request**\n\n```\n")
-	builder.WriteString(event.Request)
-	builder.WriteString("\n```\n\n<details><summary>**Response**</summary>\n\n```\n")
-	builder.WriteString(event.Response)
-	builder.WriteString("\n```\n\n")
+	if event.Request != "" {
+		builder.WriteString("\n**Request**\n\n```http\n")
+		builder.WriteString(event.Request)
+		builder.WriteString("\n```\n")
+	}
+	if event.Response != "" {
+		builder.WriteString("\n**Response**\n\n```http\n")
+		// If the response is larger than 5 kb, truncate it before writing.
+		if len(event.Response) > 5*1024 {
+			builder.WriteString(event.Response[:5*1024])
+			builder.WriteString(".... Truncated ....")
+		} else {
+			builder.WriteString(event.Response)
+		}
+		builder.WriteString("\n```\n\n")
+	}
 
 	if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
 		builder.WriteString("**Extra Information**\n\n")
@@ -75,6 +86,7 @@ func MarkdownDescription(event *output.ResultEvent) string {
 			builder.WriteString("\n")
 		}
 	}
+	builder.WriteString("\n---\nGenerated by [Nuclei](https://github.com/projectdiscovery/nuclei)")
 	data := builder.String()
 	return data
 }