Skip to content

Commit

Permalink
Merge pull request #378 from projectdiscovery/iceman-add-trace-log
Browse files Browse the repository at this point in the history
Added trace log feature to write execution log
  • Loading branch information
ehsandeep committed Oct 23, 2020
2 parents 88d187b + 4aa2d49 commit 9b7de41
Show file tree
Hide file tree
Showing 6 changed files with 123 additions and 4 deletions.
2 changes: 2 additions & 0 deletions v2/internal/runner/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type Options struct {
ProxyURL string // ProxyURL is the URL for the proxy server
ProxySocksURL string // ProxySocksURL is the URL for the proxy socks server
TemplatesDirectory string // TemplatesDirectory is the directory to use for storing templates
TraceLogFile string // TraceLogFile specifies a file to write with the trace of all requests
Templates multiStringFlag // Signature specifies the template/templates to use
ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude
CustomHeaders requests.CustomHeaders // Custom global headers
Expand Down Expand Up @@ -76,6 +77,7 @@ func ParseOptions() *Options {
flag.Var(&options.CustomHeaders, "H", "Custom Header.")
flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses")
flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)")
flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log")
flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates")
flag.BoolVar(&options.JSON, "json", false, "Write json output to files")
flag.BoolVar(&options.JSONRequests, "json-requests", false, "Write requests/responses for matches in JSON output")
Expand Down
4 changes: 4 additions & 0 deletions v2/internal/runner/processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat
switch value := request.(type) {
case *requests.DNSRequest:
dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{
TraceLog: r.traceLog,
Debug: r.options.Debug,
Template: template,
DNSRequest: value,
Expand All @@ -52,6 +53,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat
})
case *requests.BulkHTTPRequest:
httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
TraceLog: r.traceLog,
Debug: r.options.Debug,
Template: template,
BulkHTTPRequest: value,
Expand Down Expand Up @@ -214,6 +216,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl
template := &workflows.Template{Progress: p}
if len(t.BulkRequestsHTTP) > 0 {
template.HTTPOptions = &executer.HTTPOptions{
TraceLog: r.traceLog,
Debug: r.options.Debug,
Writer: r.output,
Template: t,
Expand All @@ -231,6 +234,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl
}
} else if len(t.RequestsDNS) > 0 {
template.DNSOptions = &executer.DNSOptions{
TraceLog: r.traceLog,
Debug: r.options.Debug,
Template: t,
Writer: r.output,
Expand Down
16 changes: 14 additions & 2 deletions v2/internal/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@ import (
"sync"

"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter"
Expand All @@ -25,11 +27,13 @@ import (
type Runner struct {
input string
inputCount int64
tempFile string

traceLog tracelog.Log

// output is the output file to write if any
output *bufwriter.Writer

tempFile string
templatesConfig *nucleiConfig
// options contains configuration options for runner
options *Options
Expand All @@ -45,7 +49,15 @@ type Runner struct {
// New creates a new client for running enumeration process.
func New(options *Options) (*Runner, error) {
runner := &Runner{
options: options,
traceLog: &tracelog.NoopLogger{},
options: options,
}
if options.TraceLogFile != "" {
fileLog, err := tracelog.NewFileLogger(options.TraceLogFile)
if err != nil {
return nil, errors.Wrap(err, "could not create file trace logger")
}
runner.traceLog = fileLog
}

if err := runner.updateTemplates(); err != nil {
Expand Down
76 changes: 76 additions & 0 deletions v2/internal/tracelog/tracelog.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package tracelog

import (
"os"
"sync"

jsoniter "github.com/json-iterator/go"
)

// Log is an interface for logging trace log of all the requests
type Log interface {
// Close closes the log interface flushing data
Close()
// Request writes a log the requests trace log
Request(templateID, url, requestType string, err error)
}

// NoopLogger is a noop logger that simply does nothing
type NoopLogger struct{}

// Close closes the log interface flushing data
func (n *NoopLogger) Close() {}

// Request writes a log the requests trace log
func (n *NoopLogger) Request(templateID, url, requestType string, err error) {}

// FileLogger is a trace logger that writes request logs to a file.
type FileLogger struct {
encoder *jsoniter.Encoder
file *os.File
mutex *sync.Mutex
}

// NewFileLogger creates a new file logger structure
func NewFileLogger(path string) (*FileLogger, error) {
file, err := os.Create(path)
if err != nil {
return nil, err
}
return &FileLogger{file: file, encoder: jsoniter.NewEncoder(file), mutex: &sync.Mutex{}}, nil
}

// Close closes the log interface flushing data
func (f *FileLogger) Close() {
f.mutex.Lock()
defer f.mutex.Unlock()

f.file.Close()
}

// JSONRequest is a trace log request written to file
type JSONRequest struct {
ID string `json:"id"`
URL string `json:"url"`
Error string `json:"error"`
Type string `json:"type"`
}

// Request writes a log the requests trace log
func (f *FileLogger) Request(templateID, url, requestType string, err error) {
request := &JSONRequest{
ID: templateID,
URL: url,
Type: requestType,
}
if err != nil {
request.Error = err.Error()
} else {
request.Error = "none"
}

f.mutex.Lock()
defer f.mutex.Unlock()
//nolint:errcheck // We don't need to do anything here
f.encoder.Encode(request)
}
6 changes: 6 additions & 0 deletions v2/pkg/executer/executer_dns.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
Expand All @@ -25,6 +26,7 @@ type DNSExecuter struct {
jsonRequest bool
noMeta bool
Results bool
traceLog tracelog.Log
dnsClient *retryabledns.Client
template *templates.Template
dnsRequest *requests.DNSRequest
Expand All @@ -49,6 +51,7 @@ type DNSOptions struct {
JSON bool
JSONRequests bool
NoMeta bool
TraceLog tracelog.Log
Template *templates.Template
DNSRequest *requests.DNSRequest
Writer *bufwriter.Writer
Expand All @@ -66,6 +69,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
debug: options.Debug,
noMeta: options.NoMeta,
jsonOutput: options.JSON,
traceLog: options.TraceLog,
jsonRequest: options.JSONRequests,
dnsClient: dnsClient,
template: options.Template,
Expand Down Expand Up @@ -94,12 +98,14 @@ func (e *DNSExecuter) ExecuteDNS(p progress.IProgress, reqURL string) *Result {
// Compile each request for the template based on the URL
compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
if err != nil {
e.traceLog.Request(e.template.ID, domain, "dns", err)
result.Error = errors.Wrap(err, "could not make dns request")

p.Drop(1)

return result
}
e.traceLog.Request(e.template.ID, domain, "dns", nil)

if e.debug {
gologger.Infof("Dumped DNS request for %s (%s)\n\n", reqURL, e.template.ID)
Expand Down
23 changes: 21 additions & 2 deletions v2/pkg/executer/executer_http.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/projectdiscovery/httpx/common/cache"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
Expand Down Expand Up @@ -49,6 +50,7 @@ type HTTPExecuter struct {
bulkHTTPRequest *requests.BulkHTTPRequest
writer *bufwriter.Writer
CookieJar *cookiejar.Jar
traceLog tracelog.Log
decolorizer *regexp.Regexp
coloredOutput bool
debug bool
Expand All @@ -72,6 +74,7 @@ type HTTPOptions struct {
CookieJar *cookiejar.Jar
Colorizer *colorizer.NucleiColorizer
Decolorizer *regexp.Regexp
TraceLog tracelog.Log
Debug bool
JSON bool
JSONRequests bool
Expand Down Expand Up @@ -124,6 +127,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
noMeta: options.NoMeta,
httpClient: client,
rawHTTPClient: rawClient,
traceLog: options.TraceLog,
template: options.Template,
bulkHTTPRequest: options.BulkHTTPRequest,
writer: options.Writer,
Expand Down Expand Up @@ -170,10 +174,13 @@ func (e *HTTPExecuter) ExecuteParallelHTTP(p progress.IProgress, reqURL string)
globalratelimiter.Take(reqURL)

// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
err := e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
if err != nil {
e.traceLog.Request(e.template.ID, reqURL, "http", err)
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
}(request)
}
Expand Down Expand Up @@ -241,8 +248,11 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(p progress.IProgress, reqURL string) *Re
request.PipelineClient = pipeclient
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
if err != nil {
e.traceLog.Request(e.template.ID, reqURL, "http", err)
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
request.PipelineClient = nil
}(request)
Expand Down Expand Up @@ -291,10 +301,13 @@ func (e *HTTPExecuter) ExecuteHTTP(p progress.IProgress, reqURL string) *Result
} else {
globalratelimiter.Take(reqURL)
// If the request was built correctly then execute it
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
err := e.handleHTTP(reqURL, httpRequest, dynamicvalues, result)
if err != nil {
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
e.traceLog.Request(e.template.ID, reqURL, "http", err)
} else {
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}
}

Expand Down Expand Up @@ -340,8 +353,10 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
} else if request.Unsafe {
// rawhttp
// burp uses "\r\n" as new line character
Expand All @@ -355,17 +370,21 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
} else {
// retryablehttp
resp, err = e.httpClient.Do(request.Request)
if err != nil {
if resp != nil {
resp.Body.Close()
}
e.traceLog.Request(e.template.ID, reqURL, "http", err)
return err
}
e.traceLog.Request(e.template.ID, reqURL, "http", nil)
}

duration := time.Since(timeStart)
Expand Down

0 comments on commit 9b7de41

Please sign in to comment.