-
Notifications
You must be signed in to change notification settings - Fork 2.3k
/
output.go
454 lines (409 loc) · 14.8 KB
/
output.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
package output
import (
"encoding/base64"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pkg/errors"
"go.uber.org/multierr"
jsoniter "github.com/json-iterator/go"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/interactsh/pkg/server"
"github.com/projectdiscovery/nuclei/v3/internal/colorizer"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
fileutil "github.com/projectdiscovery/utils/file"
osutils "github.com/projectdiscovery/utils/os"
)
// Writer is an interface which writes output to somewhere for nuclei events.
type Writer interface {
// Close closes the output writer interface
Close()
// Colorizer returns the colorizer instance for writer
Colorizer() aurora.Aurora
// Write writes the event to file and/or screen.
Write(*ResultEvent) error
// WriteFailure writes the optional failure event for template to file and/or screen.
WriteFailure(*InternalWrappedEvent) error
// Request logs a request in the trace log
Request(templateID, url, requestType string, err error)
// WriteStoreDebugData writes the request/response debug data to file
WriteStoreDebugData(host, templateID, eventType string, data string)
}
// StandardWriter is a writer writing output to file and screen for results.
type StandardWriter struct {
json bool
jsonReqResp bool
timestamp bool
noMetadata bool
matcherStatus bool
mutex *sync.Mutex
aurora aurora.Aurora
outputFile io.WriteCloser
traceFile io.WriteCloser
errorFile io.WriteCloser
severityColors func(severity.Severity) string
storeResponse bool
storeResponseDir string
omitTemplate bool
DisableStdout bool
AddNewLinesOutputFile bool // by default this is only done for stdout
}
var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
// InternalEvent is an internal output generation structure for nuclei.
type InternalEvent map[string]interface{}
func (ie InternalEvent) Set(k string, v interface{}) {
ie[k] = v
}
// InternalWrappedEvent is a wrapped event with operators result added to it.
type InternalWrappedEvent struct {
// Mutex is internal field which is implicitly used
// to synchronize callback(event) and interactsh polling updates
// Refer protocols/http.Request.ExecuteWithResults for more details
sync.RWMutex
InternalEvent InternalEvent
Results []*ResultEvent
OperatorsResult *operators.Result
UsesInteractsh bool
// Only applicable if interactsh is used
// This is used to avoid duplicate successful interactsh events
InteractshMatched atomic.Bool
}
func (iwe *InternalWrappedEvent) HasOperatorResult() bool {
iwe.RLock()
defer iwe.RUnlock()
return iwe.OperatorsResult != nil
}
func (iwe *InternalWrappedEvent) HasResults() bool {
iwe.RLock()
defer iwe.RUnlock()
return len(iwe.Results) > 0
}
func (iwe *InternalWrappedEvent) SetOperatorResult(operatorResult *operators.Result) {
iwe.Lock()
defer iwe.Unlock()
iwe.OperatorsResult = operatorResult
}
// ResultEvent is a wrapped result event for a single nuclei output.
type ResultEvent struct {
// Template is the relative filename for the template
Template string `json:"template,omitempty"`
// TemplateURL is the URL of the template for the result inside the nuclei
// templates repository if it belongs to the repository.
TemplateURL string `json:"template-url,omitempty"`
// TemplateID is the ID of the template for the result.
TemplateID string `json:"template-id"`
// TemplatePath is the path of template
TemplatePath string `json:"template-path,omitempty"`
// TemplateEncoded is the base64 encoded template
TemplateEncoded string `json:"template-encoded,omitempty"`
// Info contains information block of the template for the result.
Info model.Info `json:"info,inline"`
// MatcherName is the name of the matcher matched if any.
MatcherName string `json:"matcher-name,omitempty"`
// ExtractorName is the name of the extractor matched if any.
ExtractorName string `json:"extractor-name,omitempty"`
// Type is the type of the result event.
Type string `json:"type"`
// Host is the host input on which match was found.
Host string `json:"host,omitempty"`
// Port is port of the host input on which match was found (if applicable).
Port string `json:"port,omitempty"`
// Scheme is the scheme of the host input on which match was found (if applicable).
Scheme string `json:"scheme,omitempty"`
// URL is the Base URL of the host input on which match was found (if applicable).
URL string `json:"url,omitempty"`
// Path is the path input on which match was found.
Path string `json:"path,omitempty"`
// Matched contains the matched input in its transformed form.
Matched string `json:"matched-at,omitempty"`
// ExtractedResults contains the extraction result from the inputs.
ExtractedResults []string `json:"extracted-results,omitempty"`
// Request is the optional, dumped request for the match.
Request string `json:"request,omitempty"`
// Response is the optional, dumped response for the match.
Response string `json:"response,omitempty"`
// Metadata contains any optional metadata for the event
Metadata map[string]interface{} `json:"meta,omitempty"`
// IP is the IP address for the found result event.
IP string `json:"ip,omitempty"`
// Timestamp is the time the result was found at.
Timestamp time.Time `json:"timestamp"`
// Interaction is the full details of interactsh interaction.
Interaction *server.Interaction `json:"interaction,omitempty"`
// CURLCommand is an optional curl command to reproduce the request
// Only applicable if the report is for HTTP.
CURLCommand string `json:"curl-command,omitempty"`
// MatcherStatus is the status of the match
MatcherStatus bool `json:"matcher-status"`
// Lines is the line count for the specified match
Lines []int `json:"matched-line,omitempty"`
// IssueTrackers is the metadata for issue trackers
IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"`
// ReqURLPattern when enabled contains base URL pattern that was used to generate the request
// must be enabled by setting protocols.ExecuterOptions.ExportReqURLPattern to true
ReqURLPattern string `json:"req_url_pattern,omitempty"`
// Fields related to HTTP Fuzzing functionality of nuclei.
// The output contains additional fields when the result is
// for a fuzzing template.
IsFuzzingResult bool `json:"is_fuzzing_result,omitempty"`
FuzzingMethod string `json:"fuzzing_method,omitempty"`
FuzzingParameter string `json:"fuzzing_parameter,omitempty"`
FuzzingPosition string `json:"fuzzing_position,omitempty"`
FileToIndexPosition map[string]int `json:"-"`
Error string `json:"error,omitempty"`
}
type IssueTrackerMetadata struct {
// IssueID is the ID of the issue created
IssueID string `json:"id,omitempty"`
// IssueURL is the URL of the issue created
IssueURL string `json:"url,omitempty"`
}
// NewStandardWriter creates a new output writer based on user configurations
func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
resumeBool := false
if options.Resume != "" {
resumeBool = true
}
auroraColorizer := aurora.NewAurora(!options.NoColor)
var outputFile io.WriteCloser
if options.Output != "" {
output, err := newFileOutputWriter(options.Output, resumeBool)
if err != nil {
return nil, errors.Wrap(err, "could not create output file")
}
outputFile = output
}
var traceOutput io.WriteCloser
if options.TraceLogFile != "" {
output, err := newFileOutputWriter(options.TraceLogFile, resumeBool)
if err != nil {
return nil, errors.Wrap(err, "could not create output file")
}
traceOutput = output
}
var errorOutput io.WriteCloser
if options.ErrorLogFile != "" {
output, err := newFileOutputWriter(options.ErrorLogFile, resumeBool)
if err != nil {
return nil, errors.Wrap(err, "could not create error file")
}
errorOutput = output
}
// Try to create output folder if it doesn't exist
if options.StoreResponse && !fileutil.FolderExists(options.StoreResponseDir) {
if err := fileutil.CreateFolder(options.StoreResponseDir); err != nil {
gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", options.StoreResponseDir, err)
}
}
writer := &StandardWriter{
json: options.JSONL,
jsonReqResp: !options.OmitRawRequests,
noMetadata: options.NoMeta,
matcherStatus: options.MatcherStatus,
timestamp: options.Timestamp,
aurora: auroraColorizer,
mutex: &sync.Mutex{},
outputFile: outputFile,
traceFile: traceOutput,
errorFile: errorOutput,
severityColors: colorizer.New(auroraColorizer),
storeResponse: options.StoreResponse,
storeResponseDir: options.StoreResponseDir,
omitTemplate: options.OmitTemplate,
}
return writer, nil
}
// Write writes the event to file and/or screen.
func (w *StandardWriter) Write(event *ResultEvent) error {
// Enrich the result event with extra metadata on the template-path and url.
if event.TemplatePath != "" {
event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID))
}
event.Timestamp = time.Now()
var data []byte
var err error
if w.json {
data, err = w.formatJSON(event)
} else {
data = w.formatScreen(event)
}
if err != nil {
return errors.Wrap(err, "could not format output")
}
if len(data) == 0 {
return nil
}
w.mutex.Lock()
defer w.mutex.Unlock()
if !w.DisableStdout {
_, _ = os.Stdout.Write(data)
_, _ = os.Stdout.Write([]byte("\n"))
}
if w.outputFile != nil {
if !w.json {
data = decolorizerRegex.ReplaceAll(data, []byte(""))
}
if _, writeErr := w.outputFile.Write(data); writeErr != nil {
return errors.Wrap(err, "could not write to output")
}
if w.AddNewLinesOutputFile && w.json {
_, _ = w.outputFile.Write([]byte("\n"))
}
}
return nil
}
// JSONLogRequest is a trace/error log request written to file
type JSONLogRequest struct {
Template string `json:"template"`
Input string `json:"input"`
Error string `json:"error"`
Type string `json:"type"`
}
// Request writes a log the requests trace log
func (w *StandardWriter) Request(templatePath, input, requestType string, requestErr error) {
if w.traceFile == nil && w.errorFile == nil {
return
}
request := &JSONLogRequest{
Template: templatePath,
Input: input,
Type: requestType,
}
if unwrappedErr := utils.UnwrapError(requestErr); unwrappedErr != nil {
request.Error = unwrappedErr.Error()
} else {
request.Error = "none"
}
data, err := jsoniter.Marshal(request)
if err != nil {
return
}
if w.traceFile != nil {
_, _ = w.traceFile.Write(data)
}
if requestErr != nil && w.errorFile != nil {
_, _ = w.errorFile.Write(data)
}
}
// Colorizer returns the colorizer instance for writer
func (w *StandardWriter) Colorizer() aurora.Aurora {
return w.aurora
}
// Close closes the output writing interface
func (w *StandardWriter) Close() {
if w.outputFile != nil {
w.outputFile.Close()
}
if w.traceFile != nil {
w.traceFile.Close()
}
if w.errorFile != nil {
w.errorFile.Close()
}
}
// WriteFailure writes the failure event for template to file and/or screen.
func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error {
if !w.matcherStatus {
return nil
}
if len(wrappedEvent.Results) > 0 {
errs := []error{}
for _, result := range wrappedEvent.Results {
result.MatcherStatus = false // just in case
if err := w.Write(result); err != nil {
errs = append(errs, err)
}
}
if len(errs) > 0 {
return multierr.Combine(errs...)
}
return nil
}
// if no results were found, manually create a failure event
event := wrappedEvent.InternalEvent
templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]))
var templateInfo model.Info
if event["template-info"] != nil {
templateInfo = event["template-info"].(model.Info)
}
fields := protocolUtils.GetJsonFieldsFromURL(types.ToString(event["host"]))
if types.ToString(event["ip"]) != "" {
fields.Ip = types.ToString(event["ip"])
}
if types.ToString(event["path"]) != "" {
fields.Path = types.ToString(event["path"])
}
data := &ResultEvent{
Template: templatePath,
TemplateURL: templateURL,
TemplateID: types.ToString(event["template-id"]),
TemplatePath: types.ToString(event["template-path"]),
Info: templateInfo,
Type: types.ToString(event["type"]),
Host: fields.Host,
Path: fields.Path,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
IP: fields.Ip,
Request: types.ToString(event["request"]),
Response: types.ToString(event["response"]),
MatcherStatus: false,
Timestamp: time.Now(),
//FIXME: this is workaround to encode the template when no results were found
TemplateEncoded: w.encodeTemplate(types.ToString(event["template-path"])),
Error: types.ToString(event["error"]),
}
return w.Write(data)
}
var maxTemplateFileSizeForEncoding = 1024 * 1024
func (w *StandardWriter) encodeTemplate(templatePath string) string {
data, err := os.ReadFile(templatePath)
if err == nil && !w.omitTemplate && len(data) <= maxTemplateFileSizeForEncoding && config.DefaultConfig.IsCustomTemplate(templatePath) {
return base64.StdEncoding.EncodeToString(data)
}
return ""
}
func sanitizeFileName(fileName string) string {
fileName = strings.ReplaceAll(fileName, "http:", "")
fileName = strings.ReplaceAll(fileName, "https:", "")
fileName = strings.ReplaceAll(fileName, "/", "_")
fileName = strings.ReplaceAll(fileName, "\\", "_")
fileName = strings.ReplaceAll(fileName, "-", "_")
fileName = strings.ReplaceAll(fileName, ".", "_")
if osutils.IsWindows() {
fileName = strings.ReplaceAll(fileName, ":", "_")
}
fileName = strings.TrimPrefix(fileName, "__")
return fileName
}
func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {
if w.storeResponse {
filename := sanitizeFileName(fmt.Sprintf("%s_%s", host, templateID))
subFolder := filepath.Join(w.storeResponseDir, sanitizeFileName(eventType))
if !fileutil.FolderExists(subFolder) {
_ = fileutil.CreateFolder(subFolder)
}
filename = filepath.Join(subFolder, fmt.Sprintf("%s.txt", filename))
f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
fmt.Print(err)
return
}
_, _ = f.WriteString(fmt.Sprintln(data))
f.Close()
}
}