diff --git a/Dockerfile b/Dockerfile index f64550068a..244dd80384 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,19 @@ -FROM golang:1.14-alpine AS build-env +FROM golang:alpine as builder -RUN apk add --no-cache --upgrade git openssh-client ca-certificates -RUN go get -u github.com/golang/dep/cmd/dep -WORKDIR /go/src/app +RUN mkdir -p /app +WORKDIR /app +COPY ./go.mod . +RUN go mod download -# Install -RUN GO111MODULE=on go get -u github.com/projectdiscovery/nuclei/v2/cmd/nuclei +COPY . . +RUN cd ./cmd/nuclei && go build -o nuclei . -ENTRYPOINT ["nuclei"] +FROM alpine + +RUN mkdir /app +RUN adduser -S -D -H -h /app appuser +USER appuser +COPY --from=builder /app/cmd/nuclei/nuclei /app + +WORKDIR /app +CMD ["./nuclei"] diff --git a/README.md b/README.md index c771d38d58..0f41c7aa0a 100644 --- a/README.md +++ b/README.md @@ -22,19 +22,19 @@ We have also [open-sourced a template repository](https://github.com/projectdisc
Resources -- [Features](#features) -- [Usage](#usage) -- [Installation Instructions](#installation-instructions) - - [From Binary](#from-binary) - - [From Source](#from-source) - - [From Github](#from-github) -- [Nuclei templates](#nuclei-templates) -- [Running nuclei](#running-nuclei) - - [Running with a single template.](#running-with-single-template) - - [Running with multiple templates.](#running-with-multiple-templates) - - [Running with subfinder](#running-with-subfinder) - - [Running in Docker](#running-in-docker-container) -- [Thanks](#thanks) +- [Features](#features) +- [Usage](#usage) +- [Installation Instructions](#installation-instructions) + - [From Binary](#from-binary) + - [From Source](#from-source) + - [From Github](#from-github) +- [Nuclei templates](#nuclei-templates) +- [Running nuclei](#running-nuclei) + - [Running with a single template.](#running-with-single-template) + - [Running with multiple templates.](#running-with-multiple-templates) + - [Running with subfinder](#running-with-subfinder) + - [Running in Docker](#running-in-docker-container) +- [Thanks](#thanks)
@@ -71,12 +71,13 @@ This will display help for the tool. Here are all the switches it supports. | -o | File to save output result (optional) | nuclei -o output.txt | | -pbar | Enable the progress bar (optional) | nuclei -pbar | | -silent | Show only found results in output | nuclei -silent | +| | (except when using with pbar) | | | -retries | Number of times to retry a failed request (default 1) | nuclei -retries 1 | | -timeout | Seconds to wait before timeout (default 5) | nuclei -timeout 5 | | -debug | Allow debugging of request/responses. | nuclei -debug | | -update-templates | Download and updates nuclei templates | nuclei -update-templates | | -update-directory | Directory for storing nuclei-templates(optional) | nuclei -update-directory templates | -| -lt | List available templates | nuclei -lt | +| -tl | List available templates | nuclei -tl | | -v | Shows verbose output of all sent requests | nuclei -v | | -version | Show version of nuclei | nuclei -version | | -proxy-url | Proxy URL | nuclei -proxy-url hxxp://127.0.0.1:8080 | @@ -102,7 +103,7 @@ Download latest binary from https://github.com/projectdiscovery/nuclei/releases nuclei requires **go1.14+** to install successfully. Run the following command to get the repo - ```sh -▶ GO111MODULE=auto go get -u -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +▶ GO111MODULE=on go get -u -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei ``` ### From Github diff --git a/go.mod b/go.mod index 2a0d0774c8..527b09803b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.14 require ( github.com/Knetic/govaluate v3.0.0+incompatible - github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 github.com/blang/semver v3.5.1+incompatible github.com/d5/tengo/v2 v2.6.0 github.com/google/go-github/v32 v32.1.0 diff --git a/go.sum b/go.sum index 291954f385..acc2417d1e 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,17 @@ -github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI= github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= @@ -33,7 +27,6 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= @@ -46,9 +39,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= -github.com/projectdiscovery/nuclei v1.1.7 h1:5Z1fBHcjyAuuI89xcCzv8tYK7b6ucqLxs+mCC/nJjno= -github.com/projectdiscovery/nuclei/v2 v2.1.0 h1:TUr9lwJ8lulBmEfz6AQHoKPiBQocl4PrPXSu3ekXsXY= -github.com/projectdiscovery/nuclei/v2 v2.1.0/go.mod h1:iufrjj0m0PCZc8J8eVLZqZMyiCaMMv4R5d6QKBdqpZA= github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60= github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= @@ -57,8 +47,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/vbauerster/mpb v1.1.3 h1:IRgic8VFaURXkW0VxDLkNOiNaAgtw0okB2YIaVvJDI4= -github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw= github.com/vbauerster/mpb/v5 v5.2.4 h1:PLP8vv75RcEgxGoJVtKaRD2FHSxEmIV/u4ZuOrfO8Qg= github.com/vbauerster/mpb/v5 v5.2.4/go.mod h1:K4iCHQp5sWnmAgEn+uW1sAxSilctb4JPAGXx49jV+Aw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -73,6 +61,7 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -84,6 +73,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= diff --git a/internal/bufwriter/bufwriter.go b/internal/bufwriter/bufwriter.go new file mode 100644 index 0000000000..32592d4825 --- /dev/null +++ b/internal/bufwriter/bufwriter.go @@ -0,0 +1,74 @@ +package bufwriter + +import ( + "bufio" + "os" + "sync" +) + +// Writer is a mutex protected buffered writer +type Writer struct { + file *os.File + writer *bufio.Writer + mutex *sync.Mutex +} + +// New creates a new mutex protected buffered writer for a file +func New(file string) (*Writer, error) { + output, err := os.Create(file) + if err != nil { + return nil, err + } + return &Writer{file: output, writer: bufio.NewWriter(output), mutex: &sync.Mutex{}}, nil +} + +// Write writes a byte slice to the underlying file +// +// It also writes a newline if the last byte isn't a newline character. +func (w *Writer) Write(data []byte) error { + if len(data) == 0 { + return nil + } + w.mutex.Lock() + defer w.mutex.Unlock() + + _, err := w.writer.Write(data) + if err != nil { + return err + } + if data[len(data)-1] != '\n' { + _, err = w.writer.WriteRune('\n') + } + return err +} + +// WriteString writes a string to the underlying file +// +// It also writes a newline if the last byte isn't a newline character. +func (w *Writer) WriteString(data string) error { + if data == "" { + return nil + } + w.mutex.Lock() + defer w.mutex.Unlock() + + _, err := w.writer.WriteString(data) + if err != nil { + return err + } + if data[len(data)-1] != '\n' { + _, err = w.writer.WriteRune('\n') + } + return err +} + +// Close closes the underlying writer flushing everything to disk +func (w *Writer) Close() error { + w.mutex.Lock() + defer w.mutex.Unlock() + + w.writer.Flush() + //nolint:errcheck // we don't care whether sync failed or succeeded. + w.file.Sync() + return w.file.Close() +} diff --git a/internal/progress/progress.go b/internal/progress/progress.go index e14224efce..77c1acf812 100644 --- a/internal/progress/progress.go +++ b/internal/progress/progress.go @@ -21,7 +21,7 @@ const ( mili = 1000. ) -// Encapsulates progress tracking. +// IProgress encapsulates progress tracking. type IProgress interface { InitProgressbar(hostCount int64, templateCount int, requestCount int64) AddToTotal(delta int64) @@ -37,7 +37,7 @@ type Progress struct { initialTotal int64 totalMutex *sync.Mutex - colorizer aurora.Aurora + colorizer *aurora.Aurora renderChan chan time.Time captureData *captureData @@ -49,8 +49,8 @@ type Progress struct { stdRenderWaitGroup *sync.WaitGroup } -// Creates and returns a new progress tracking object. -func NewProgress(noColor, active bool) IProgress { +// NewProgress creates and returns a new progress tracking object. +func NewProgress(colorizer aurora.Aurora, active bool) IProgress { if !active { return &NoOpProgress{} } @@ -65,7 +65,7 @@ func NewProgress(noColor, active bool) IProgress { mpb.WithManualRefresh(renderChan), ), totalMutex: &sync.Mutex{}, - colorizer: aurora.NewAurora(!noColor), + colorizer: &colorizer, renderChan: renderChan, stdCaptureMutex: &sync.Mutex{}, @@ -85,7 +85,7 @@ func (p *Progress) InitProgressbar(hostCount int64, rulesCount int, requestCount panic("A global progressbar is already present.") } - color := p.colorizer + color := *p.colorizer barName := color.Sprintf( color.Cyan("%d %s, %d %s"), @@ -193,7 +193,7 @@ func (p *Progress) renderStdData() { // Creates and returns a progress bar. func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb.Bar { - color := p.colorizer + color := *p.colorizer p.total = total p.initialTotal = total diff --git a/internal/runner/banner.go b/internal/runner/banner.go index e3c4b4277d..abfb6f0557 100644 --- a/internal/runner/banner.go +++ b/internal/runner/banner.go @@ -7,11 +7,11 @@ const banner = ` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.1 + /_/ /_/\__,_/\___/_/\___/_/ v2.1.1 ` // Version is the current version of nuclei -const Version = `2.1.0` +const Version = `2.1.1` // showBanner is used to show the banner to the user func showBanner() { diff --git a/internal/runner/processor.go b/internal/runner/processor.go index 30ac76a09d..01ffab4075 100644 --- a/internal/runner/processor.go +++ b/internal/runner/processor.go @@ -31,16 +31,8 @@ type workflowTemplates struct { // processTemplateWithList processes a template and runs the enumeration on all the targets func (r *Runner) processTemplateWithList(ctx context.Context, p progress.IProgress, template *templates.Template, request interface{}) bool { - var writer *bufio.Writer - if r.output != nil { - writer = bufio.NewWriter(r.output) - defer writer.Flush() - } - var httpExecuter *executer.HTTPExecuter - var dnsExecuter *executer.DNSExecuter - var err error // Create an executer based on the request type. @@ -50,7 +42,7 @@ func (r *Runner) processTemplateWithList(ctx context.Context, p progress.IProgre Debug: r.options.Debug, Template: template, DNSRequest: value, - Writer: writer, + Writer: r.output, JSON: r.options.JSON, JSONRequests: r.options.JSONRequests, ColoredOutput: !r.options.NoColor, @@ -62,7 +54,7 @@ func (r *Runner) processTemplateWithList(ctx context.Context, p progress.IProgre Debug: r.options.Debug, Template: template, BulkHTTPRequest: value, - Writer: writer, + Writer: r.output, Timeout: r.options.Timeout, Retries: r.options.Retries, ProxyURL: r.options.ProxyURL, @@ -72,7 +64,7 @@ func (r *Runner) processTemplateWithList(ctx context.Context, p progress.IProgre JSONRequests: r.options.JSONRequests, CookieReuse: value.CookieReuse, ColoredOutput: !r.options.NoColor, - Colorizer: r.colorizer, + Colorizer: &r.colorizer, Decolorizer: r.decolorizer, }) } @@ -126,12 +118,14 @@ func (r *Runner) processTemplateWithList(ctx context.Context, p progress.IProgre } // ProcessWorkflowWithList coming from stdin or list of targets -func (r *Runner) processWorkflowWithList(p progress.IProgress, workflow *workflows.Workflow) { +func (r *Runner) processWorkflowWithList(p progress.IProgress, workflow *workflows.Workflow) bool { + result := false + workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow) if err != nil { gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err) - return + return result } logicBytes := []byte(workflow.Logic) @@ -151,13 +145,18 @@ func (r *Runner) processWorkflowWithList(p progress.IProgress, workflow *workflo script := tengo.NewScript(logicBytes) script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) + variables := make(map[string]*workflows.NucleiVar) + for _, workflowTemplate := range *workflowTemplatesList { - err := script.Add(workflowTemplate.Name, &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL}) + name := workflowTemplate.Name + variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL} + err := script.Add(name, variable) if err != nil { gologger.Errorf("Could not initialize script for workflow '%s': %s\n", workflow.ID, err) continue } + variables[name] = variable } _, err := script.RunContext(context.Background()) @@ -165,11 +164,20 @@ func (r *Runner) processWorkflowWithList(p progress.IProgress, workflow *workflo gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err) } + for _, variable := range variables { + result = !variable.IsFalsy() + if result { + break + } + } + <-r.limiter }(targetURL) } wg.Wait() + + return result } func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workflows.Workflow) (*[]workflowTemplates, error) { @@ -188,12 +196,6 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl var wflTemplatesList []workflowTemplates for name, value := range workflow.Variables { - var writer *bufio.Writer - if r.output != nil { - writer = bufio.NewWriter(r.output) - defer writer.Flush() - } - // Check if the template is an absolute path or relative path. // If the path is absolute, use it. Otherwise, if isRelative(value) { @@ -220,23 +222,27 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl if len(t.BulkRequestsHTTP) > 0 { template.HTTPOptions = &executer.HTTPOptions{ Debug: r.options.Debug, - Writer: writer, + Writer: r.output, Template: t, Timeout: r.options.Timeout, Retries: r.options.Retries, ProxyURL: r.options.ProxyURL, ProxySocksURL: r.options.ProxySocksURL, CustomHeaders: r.options.CustomHeaders, + JSON: r.options.JSON, + JSONRequests: r.options.JSONRequests, CookieJar: jar, ColoredOutput: !r.options.NoColor, - Colorizer: r.colorizer, + Colorizer: &r.colorizer, Decolorizer: r.decolorizer, } } else if len(t.RequestsDNS) > 0 { template.DNSOptions = &executer.DNSOptions{ Debug: r.options.Debug, Template: t, - Writer: writer, + Writer: r.output, + JSON: r.options.JSON, + JSONRequests: r.options.JSONRequests, ColoredOutput: !r.options.NoColor, Colorizer: r.colorizer, Decolorizer: r.decolorizer, @@ -281,7 +287,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl if len(t.BulkRequestsHTTP) > 0 { template.HTTPOptions = &executer.HTTPOptions{ Debug: r.options.Debug, - Writer: writer, + Writer: r.output, Template: t, Timeout: r.options.Timeout, Retries: r.options.Retries, @@ -294,7 +300,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl template.DNSOptions = &executer.DNSOptions{ Debug: r.options.Debug, Template: t, - Writer: writer, + Writer: r.output, } } if template.DNSOptions != nil || template.HTTPOptions != nil { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 670fbcac72..b6f33059dc 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -13,8 +13,10 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" + "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -25,8 +27,7 @@ type Runner struct { inputCount int64 // output is the output file to write if any - output *os.File - outputMutex *sync.Mutex + output *bufwriter.Writer tempFile string templatesConfig *nucleiConfig @@ -38,21 +39,29 @@ type Runner struct { progress progress.IProgress // output coloring - colorizer aurora.Aurora + colorizer colorizer.NucleiColorizer decolorizer *regexp.Regexp } // New creates a new client for running enumeration process. func New(options *Options) (*Runner, error) { runner := &Runner{ - outputMutex: &sync.Mutex{}, - options: options, + options: options, } if err := runner.updateTemplates(); err != nil { gologger.Labelf("Could not update templates: %s\n", err) } + // output coloring + useColor := !options.NoColor + runner.colorizer = *colorizer.NewNucleiColorizer(aurora.NewAurora(useColor)) + + if useColor { + // compile a decolorization regex to cleanup file output messages + runner.decolorizer = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) + } + if options.TemplateList { runner.listAvailableTemplates() os.Exit(0) @@ -66,15 +75,6 @@ func New(options *Options) (*Runner, error) { runner.readNucleiIgnoreFile() } - // output coloring - useColor := !options.NoColor - runner.colorizer = aurora.NewAurora(useColor) - - if useColor { - // compile a decolorization regex to cleanup file output messages - runner.decolorizer = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) - } - // If we have stdin, write it to a new file if options.Stdin { tempInput, err := ioutil.TempFile("", "stdin-input-*") @@ -152,16 +152,15 @@ func New(options *Options) (*Runner, error) { // Create the output file if asked if options.Output != "" { - output, err := os.Create(options.Output) + output, err := bufwriter.New(options.Output) if err != nil { gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err) } - runner.output = output } // Creates the progress tracking object - runner.progress = progress.NewProgress(runner.options.NoColor, !options.Silent && options.EnableProgressBar) + runner.progress = progress.NewProgress(runner.colorizer.Colorizer, options.EnableProgressBar) runner.limiter = make(chan struct{}, options.Threads) @@ -170,7 +169,9 @@ func New(options *Options) (*Runner, error) { // Close releases all the resources and cleans up func (r *Runner) Close() { - r.output.Close() + if r.output != nil { + r.output.Close() + } os.Remove(r.tempFile) } @@ -211,9 +212,9 @@ func (r *Runner) RunEnumeration() { } gologger.Infof("Using %s rules (%s templates, %s workflows)", - r.colorizer.Bold(templateCount).String(), - r.colorizer.Bold(templateCount-workflowCount).String(), - r.colorizer.Bold(workflowCount).String()) + r.colorizer.Colorizer.Bold(templateCount).String(), + r.colorizer.Colorizer.Bold(templateCount-workflowCount).String(), + r.colorizer.Colorizer.Bold(workflowCount).String()) // precompute total request count var totalRequests int64 = 0 @@ -254,8 +255,7 @@ func (r *Runner) RunEnumeration() { results.Or(r.processTemplateWithList(ctx, p, tt, request)) } case *workflows.Workflow: - workflow := template.(*workflows.Workflow) - r.processWorkflowWithList(p, workflow) + results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow))) } }(t) } @@ -266,9 +266,8 @@ func (r *Runner) RunEnumeration() { if !results.Get() { if r.output != nil { - outputFile := r.output.Name() r.output.Close() - os.Remove(outputFile) + os.Remove(r.options.Output) } gologger.Infof("No results found. Happy hacking!") diff --git a/internal/runner/templates.go b/internal/runner/templates.go index fd1326a40b..9b7d4badae 100644 --- a/internal/runner/templates.go +++ b/internal/runner/templates.go @@ -8,19 +8,11 @@ import ( "strings" "github.com/karrick/godirwalk" - "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) -var severityMap = map[string]string{ - "info": aurora.Cyan("info").String(), - "low": aurora.Green("low").String(), - "medium": aurora.Yellow("medium").String(), - "high": aurora.Red("high").String(), -} - // getTemplatesFor parses the specified input template definitions and returns a list of unique, absolute template paths. func (r *Runner) getTemplatesFor(definitions []string) []string { // keeps track of processed dirs and files @@ -190,12 +182,12 @@ func (r *Runner) parseTemplateFile(file string) (interface{}, error) { func (r *Runner) templateLogMsg(id, name, author, severity string) string { // Display the message for the template message := fmt.Sprintf("[%s] %s (%s)", - r.colorizer.BrightBlue(id).String(), - r.colorizer.Bold(name).String(), - r.colorizer.BrightYellow("@"+author).String()) + r.colorizer.Colorizer.BrightBlue(id).String(), + r.colorizer.Colorizer.Bold(name).String(), + r.colorizer.Colorizer.BrightYellow("@"+author).String()) if severity != "" { - message += " [" + severityMap[severity] + "]" + message += " [" + r.colorizer.GetColorizedSeverity(severity) + "]" } return message @@ -231,12 +223,11 @@ func (r *Runner) listAvailableTemplates() { r.templatesConfig.CurrentVersion, r.templatesConfig.TemplatesDirectory, ) - r.colorizer = aurora.NewAurora(true) err := directoryWalker( r.templatesConfig.TemplatesDirectory, func(path string, d *godirwalk.Dirent) error { if d.IsDir() && path != r.templatesConfig.TemplatesDirectory { - gologger.Silentf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(strings.Title(d.Name()))).String()) + gologger.Silentf("\n%s:\n\n", r.colorizer.Colorizer.Bold(r.colorizer.Colorizer.BgBrightBlue(d.Name())).String()) } else if strings.HasSuffix(path, ".yaml") { r.logAvailableTemplate(path) } diff --git a/pkg/colorizer/colorizer.go b/pkg/colorizer/colorizer.go new file mode 100644 index 0000000000..61937bbf69 --- /dev/null +++ b/pkg/colorizer/colorizer.go @@ -0,0 +1,42 @@ +package colorizer + +import ( + "strings" + + "github.com/logrusorgru/aurora" +) + +const ( + fgOrange uint8 = 208 + undefined string = "undefined" +) + +// NucleiColorizer contains the severity color mapping +type NucleiColorizer struct { + Colorizer aurora.Aurora + SeverityMap map[string]string +} + +// NewNucleiColorizer initializes the new nuclei colorizer +func NewNucleiColorizer(colorizer aurora.Aurora) *NucleiColorizer { + return &NucleiColorizer{ + Colorizer: colorizer, + SeverityMap: map[string]string{ + "info": colorizer.Blue("info").String(), + "low": colorizer.Green("low").String(), + "medium": colorizer.Yellow("medium").String(), + "high": colorizer.Index(fgOrange, "high").String(), + "critical": colorizer.Red("critical").String(), + }, + } +} + +// GetColorizedSeverity returns the colorized severity string +func (r *NucleiColorizer) GetColorizedSeverity(severity string) string { + sev := r.SeverityMap[strings.ToLower(severity)] + if sev == "" { + return undefined + } + + return sev +} diff --git a/pkg/executer/executer_dns.go b/pkg/executer/executer_dns.go index 458c3cb36c..4a8baf00f0 100644 --- a/pkg/executer/executer_dns.go +++ b/pkg/executer/executer_dns.go @@ -1,16 +1,15 @@ package executer import ( - "bufio" "fmt" "os" "regexp" - "sync" - "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -28,10 +27,9 @@ type DNSExecuter struct { dnsClient *retryabledns.Client template *templates.Template dnsRequest *requests.DNSRequest - writer *bufio.Writer - outputMutex *sync.Mutex + writer *bufwriter.Writer - colorizer aurora.Aurora + colorizer colorizer.NucleiColorizer decolorizer *regexp.Regexp } @@ -51,9 +49,9 @@ type DNSOptions struct { JSONRequests bool Template *templates.Template DNSRequest *requests.DNSRequest - Writer *bufio.Writer + Writer *bufwriter.Writer - Colorizer aurora.Aurora + Colorizer colorizer.NucleiColorizer Decolorizer *regexp.Regexp } @@ -70,7 +68,6 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter { template: options.Template, dnsRequest: options.DNSRequest, writer: options.Writer, - outputMutex: &sync.Mutex{}, coloredOutput: options.ColoredOutput, colorizer: options.Colorizer, decolorizer: options.Decolorizer, @@ -166,8 +163,4 @@ func (e *DNSExecuter) ExecuteDNS(p progress.IProgress, reqURL string) (result Re } // Close closes the dns executer for a template. -func (e *DNSExecuter) Close() { - e.outputMutex.Lock() - defer e.outputMutex.Unlock() - e.writer.Flush() -} +func (e *DNSExecuter) Close() {} diff --git a/pkg/executer/executer_http.go b/pkg/executer/executer_http.go index 0a93e68b5d..ce450b1395 100644 --- a/pkg/executer/executer_http.go +++ b/pkg/executer/executer_http.go @@ -1,7 +1,6 @@ package executer import ( - "bufio" "context" "crypto/tls" "fmt" @@ -15,14 +14,13 @@ import ( "os" "regexp" "strings" - "sync" "time" - "github.com/logrusorgru/aurora" - "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -46,12 +44,11 @@ type HTTPExecuter struct { httpClient *retryablehttp.Client template *templates.Template bulkHTTPRequest *requests.BulkHTTPRequest - writer *bufio.Writer - outputMutex *sync.Mutex + writer *bufwriter.Writer customHeaders requests.CustomHeaders CookieJar *cookiejar.Jar - colorizer aurora.Aurora + colorizer colorizer.NucleiColorizer decolorizer *regexp.Regexp } @@ -64,14 +61,14 @@ type HTTPOptions struct { ColoredOutput bool Template *templates.Template BulkHTTPRequest *requests.BulkHTTPRequest - Writer *bufio.Writer + Writer *bufwriter.Writer Timeout int Retries int ProxyURL string ProxySocksURL string CustomHeaders requests.CustomHeaders CookieJar *cookiejar.Jar - Colorizer aurora.Aurora + Colorizer *colorizer.NucleiColorizer Decolorizer *regexp.Regexp } @@ -112,12 +109,11 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { httpClient: client, template: options.Template, bulkHTTPRequest: options.BulkHTTPRequest, - outputMutex: &sync.Mutex{}, writer: options.Writer, customHeaders: options.CustomHeaders, CookieJar: options.CookieJar, coloredOutput: options.ColoredOutput, - colorizer: options.Colorizer, + colorizer: *options.Colorizer, decolorizer: options.Decolorizer, } @@ -282,11 +278,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, } // Close closes the http executer for a template. -func (e *HTTPExecuter) Close() { - e.outputMutex.Lock() - defer e.outputMutex.Unlock() - e.writer.Flush() -} +func (e *HTTPExecuter) Close() {} // makeHTTPClient creates a http client func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client { @@ -369,7 +361,7 @@ func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirect func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { for _, customHeader := range e.customHeaders { // This should be pre-computed somewhere and done only once - tokens := strings.Split(customHeader, ":") + tokens := strings.SplitN(customHeader, ":", 2) // if it's an invalid header skip it if len(tokens) < two { continue diff --git a/pkg/executer/http_utils.go b/pkg/executer/http_utils.go index 5c0416cdf5..ee30c6a21f 100644 --- a/pkg/executer/http_utils.go +++ b/pkg/executer/http_utils.go @@ -37,7 +37,9 @@ func headersToString(headers http.Header) string { builder.WriteString(value) if i != len(values)-1 { - builder.WriteRune(',') + builder.WriteRune('\n') + builder.WriteString(header) + builder.WriteString(": ") } } diff --git a/pkg/executer/output_dns.go b/pkg/executer/output_dns.go index 5bbbe4b61d..70e694ff27 100644 --- a/pkg/executer/output_dns.go +++ b/pkg/executer/output_dns.go @@ -45,27 +45,11 @@ func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher gologger.Silentf("%s", string(data)) if e.writer != nil { - e.outputMutex.Lock() - _, err := e.writer.Write(data) - - if err != nil { - e.outputMutex.Unlock() - gologger.Errorf("Could not write output data: %s\n", err) - - return - } - - _, err = e.writer.WriteRune('\n') - - if err != nil { - e.outputMutex.Unlock() + if err := e.writer.Write(data); err != nil { gologger.Errorf("Could not write output data: %s\n", err) - return } - e.outputMutex.Unlock() } - return } @@ -73,16 +57,23 @@ func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher colorizer := e.colorizer builder.WriteRune('[') - builder.WriteString(colorizer.BrightGreen(e.template.ID).String()) + builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String()) if matcher != nil && len(matcher.Name) > 0 { builder.WriteString(":") - builder.WriteString(colorizer.BrightGreen(matcher.Name).Bold().String()) + builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String()) } builder.WriteString("] [") - builder.WriteString(colorizer.BrightBlue("dns").String()) + builder.WriteString(colorizer.Colorizer.BrightBlue("dns").String()) builder.WriteString("] ") + + if e.template.Info.Severity != "" { + builder.WriteString("[") + builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info.Severity)) + builder.WriteString("] ") + } + builder.WriteString(domain) // If any extractors, write the results @@ -90,7 +81,7 @@ func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher builder.WriteString(" [") for i, result := range extractorResults { - builder.WriteString(colorizer.BrightCyan(result).String()) + builder.WriteString(colorizer.Colorizer.BrightCyan(result).String()) if i != len(extractorResults)-1 { builder.WriteRune(',') @@ -107,19 +98,13 @@ func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher gologger.Silentf("%s", message) if e.writer != nil { - e.outputMutex.Lock() if e.coloredOutput { message = e.decolorizer.ReplaceAllString(message, "") } - _, err := e.writer.WriteString(message) - - if err != nil { - e.outputMutex.Unlock() + if err := e.writer.WriteString(message); err != nil { gologger.Errorf("Could not write output data: %s\n", err) - return } - e.outputMutex.Unlock() } } diff --git a/pkg/executer/output_http.go b/pkg/executer/output_http.go index 8da2576b88..423f0217c8 100644 --- a/pkg/executer/output_http.go +++ b/pkg/executer/output_http.go @@ -60,25 +60,10 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res gologger.Silentf("%s", string(data)) if e.writer != nil { - e.outputMutex.Lock() - _, err := e.writer.Write(data) - - if err != nil { - e.outputMutex.Unlock() + if err := e.writer.Write(data); err != nil { gologger.Errorf("Could not write output data: %s\n", err) - return } - - _, err = e.writer.WriteRune('\n') - - if err != nil { - e.outputMutex.Unlock() - gologger.Errorf("Could not write output data: %s\n", err) - - return - } - e.outputMutex.Unlock() } return @@ -88,17 +73,23 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res colorizer := e.colorizer builder.WriteRune('[') - builder.WriteString(colorizer.BrightGreen(e.template.ID).String()) + builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String()) if matcher != nil && len(matcher.Name) > 0 { builder.WriteString(":") - builder.WriteString(colorizer.BrightGreen(matcher.Name).Bold().String()) + builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String()) } builder.WriteString("] [") - builder.WriteString(colorizer.BrightBlue("http").String()) + builder.WriteString(colorizer.Colorizer.BrightBlue("http").String()) builder.WriteString("] ") + if e.template.Info.Severity != "" { + builder.WriteString("[") + builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info.Severity)) + builder.WriteString("] ") + } + // Escape the URL by replacing all % with %% escapedURL := strings.ReplaceAll(URL, "%", "%%") builder.WriteString(escapedURL) @@ -108,7 +99,7 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res builder.WriteString(" [") for i, result := range extractorResults { - builder.WriteString(colorizer.BrightCyan(result).String()) + builder.WriteString(colorizer.Colorizer.BrightCyan(result).String()) if i != len(extractorResults)-1 { builder.WriteRune(',') @@ -125,7 +116,7 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res var metas []string for name, value := range req.Meta { - metas = append(metas, colorizer.BrightYellow(name).Bold().String()+"="+colorizer.BrightYellow(value.(string)).String()) + metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(value.(string)).String()) } builder.WriteString(strings.Join(metas, ",")) @@ -139,19 +130,13 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res gologger.Silentf("%s", message) if e.writer != nil { - e.outputMutex.Lock() if e.coloredOutput { message = e.decolorizer.ReplaceAllString(message, "") } - _, err := e.writer.WriteString(message) - - if err != nil { - e.outputMutex.Unlock() + if err := e.writer.WriteString(message); err != nil { gologger.Errorf("Could not write output data: %s\n", err) - return } - e.outputMutex.Unlock() } } diff --git a/pkg/extractors/extract.go b/pkg/extractors/extract.go index 10a8794043..f6e2df4003 100644 --- a/pkg/extractors/extract.go +++ b/pkg/extractors/extract.go @@ -54,13 +54,15 @@ func (e *Extractor) ExtractDNS(msg *dns.Msg) map[string]struct{} { func (e *Extractor) extractRegex(corpus string) map[string]struct{} { results := make(map[string]struct{}) + groupPlusOne := e.RegexGroup + 1 for _, regex := range e.regexCompiled { - matches := regex.FindAllString(corpus, -1) + matches := regex.FindAllStringSubmatch(corpus, -1) for _, match := range matches { - results[match] = struct{}{} + if len(match) >= groupPlusOne { + results[match[e.RegexGroup]] = struct{}{} + } } } - return results } diff --git a/pkg/extractors/extractors.go b/pkg/extractors/extractors.go index ddfd07990e..4f78d50429 100644 --- a/pkg/extractors/extractors.go +++ b/pkg/extractors/extractors.go @@ -13,6 +13,8 @@ type Extractor struct { // Regex are the regex pattern required to be present in the response Regex []string `yaml:"regex"` + // RegexGroup specifies a group to extract from the regex + RegexGroup int `yaml:"group"` // regexCompiled is the compiled variant regexCompiled []*regexp.Regexp @@ -25,7 +27,6 @@ type Extractor struct { Part string `yaml:"part,omitempty"` // part is the part of the request to match part Part - // Internal defines if this is used internally Internal bool `yaml:"internal,omitempty"` } diff --git a/pkg/workflows/var.go b/pkg/workflows/var.go index 6d33efe0e5..830ff45309 100644 --- a/pkg/workflows/var.go +++ b/pkg/workflows/var.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" + "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/executer" "github.com/projectdiscovery/nuclei/v2/pkg/generators" ) @@ -76,7 +77,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { template.HTTPOptions.BulkHTTPRequest = request if template.HTTPOptions.Colorizer == nil { - template.HTTPOptions.Colorizer = aurora.NewAurora(true) + template.HTTPOptions.Colorizer = colorizer.NewNucleiColorizer(aurora.NewAurora(true)) } httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions)