From 371b58d20b0e6fb95d950d85883212d01e5dd5b2 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 10 Oct 2020 17:53:17 +0200 Subject: [PATCH 1/9] reducing locking --- v2/pkg/globalratelimiter/ratelimit.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/v2/pkg/globalratelimiter/ratelimit.go b/v2/pkg/globalratelimiter/ratelimit.go index ea292878f9..f760706fe3 100644 --- a/v2/pkg/globalratelimiter/ratelimit.go +++ b/v2/pkg/globalratelimiter/ratelimit.go @@ -26,10 +26,16 @@ func Add(k string, rateLimit int) { } func Take(k string) { + rl := take(k) + + rl.Take() +} + +func take(k string) ratelimit.Limiter { defaultrwmutex.RLock() defer defaultrwmutex.RUnlock() - defaultGlobalRateLimiter.ratesLimiters[k].Take() + return defaultGlobalRateLimiter.ratesLimiters[k] } func Del(k string, rateLimit int) { @@ -56,11 +62,16 @@ func (grl *GlobalRateLimiter) Add(k string, rateLimit int) { } } -func (grl *GlobalRateLimiter) Take(k string) { +func (grl *GlobalRateLimiter) take(k string) ratelimit.Limiter { grl.RLock() defer grl.RUnlock() - grl.ratesLimiters[k].Take() + return grl.ratesLimiters[k] +} + +func (grl *GlobalRateLimiter) Take(k string) { + rl := grl.take(k) + rl.Take() } func (grl *GlobalRateLimiter) Del(k string, rateLimit int) { From 583b065ea4f4a00c9671dd14a480f1c3d58bfce7 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 10 Oct 2020 20:59:19 +0200 Subject: [PATCH 2/9] small logic change --- v2/pkg/requests/bulk-http-request.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 9c5ebd27c9..ead57881df 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -245,31 +245,23 @@ func (r *BulkHTTPRequest) handleRawWithPaylods(raw, baseURL string, values, genV } func (r *BulkHTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { - // In case of multiple threads the underlying connection should remain open to allow reuse - if r.Threads <= 0 { - setHeader(req, "Connection", "close") - req.Close = true - } - replacer := newReplacer(values) - - // Check if the user requested a request body - if r.Body != "" { - req.Body = ioutil.NopCloser(strings.NewReader(r.Body)) - } - // Set the header values requested for header, value := range r.Headers { req.Header[header] = []string{replacer.Replace(value)} } - // if the user specified a Connection header we don't alter it - if req.Header.Get("Connection") == "" { - // Otherwise we set it to "Connection: close" - The instruction is redundant, but it ensures that internally net/http don't miss the header/internal flag + // In case of multiple threads the underlying connection should remain open to allow reuse + if r.Threads <= 0 && req.Header.Get("Connection") == "" { setHeader(req, "Connection", "close") req.Close = true } + // Check if the user requested a request body + if r.Body != "" { + req.Body = ioutil.NopCloser(strings.NewReader(r.Body)) + } + setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)") // raw requests are left untouched From 83aef7e468fa98677a12142f41f20d3df5100eaa Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 11 Oct 2020 01:41:45 +0200 Subject: [PATCH 3/9] added missing pipeline flag --- v2/pkg/executer/executer_http.go | 17 ++++++++++------- v2/pkg/requests/bulk-http-request.go | 6 +++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 7f6c87d062..cb5a4e6704 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -198,17 +198,20 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(p progress.IProgress, reqURL string) (re pipeOptions := rawhttp.DefaultPipelineOptions pipeOptions.Host = URL.Host pipeOptions.MaxConnections = 1 - if e.bulkHTTPRequest.PipelineMaxWorkers > 0 { - pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineMaxWorkers + if e.bulkHTTPRequest.PipelineConcurrentConnections > 0 { + pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineConcurrentConnections + } + if e.bulkHTTPRequest.PipelineRequestsPerConnection > 0 { + pipeOptions.MaxPendingRequests = e.bulkHTTPRequest.PipelineRequestsPerConnection } pipeclient := rawhttp.NewPipelineClient(pipeOptions) - // Workers that keeps enqueuing new requests + // 150 should be a sufficient value to keep queues always full maxWorkers := 150 - if e.bulkHTTPRequest.PipelineMaxWorkers > 0 { - maxWorkers = e.bulkHTTPRequest.PipelineMaxWorkers + // in case the queue is bigger increase the workers + if pipeOptions.MaxPendingRequests > maxWorkers { + maxWorkers = pipeOptions.MaxPendingRequests } - swg := sizedwaitgroup.New(maxWorkers) for e.bulkHTTPRequest.Next(reqURL) && !result.Done { request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) @@ -221,8 +224,8 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(p progress.IProgress, reqURL string) (re defer swg.Done() // HTTP pipelining ignores rate limit - // If the request was built correctly then execute it + request.Pipeline = true request.PipelineClient = pipeclient err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, &result) if err != nil { diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index ead57881df..451b025073 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -64,9 +64,9 @@ type BulkHTTPRequest struct { Raw []string `yaml:"raw,omitempty"` // Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining (race conditions/billions requests) // All requests must be indempotent (GET/POST) - Pipeline bool `yaml:"pipeline,omitempty"` - PipelineMaxConnections int `yaml:"pipeline-max-connections,omitempty"` - PipelineMaxWorkers int `yaml:"pipeline-max-workers,omitempty"` + Pipeline bool `yaml:"pipeline,omitempty"` + PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections,omitempty"` + PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection,omitempty"` // Specify in order to skip request RFC normalization Unsafe bool `yaml:"unsafe,omitempty"` // DisableAutoHostname Enable/Disable Host header for unsafe raw requests From 236f3b2dfbc95a36039f94498eb4e357ead91b20 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 11 Oct 2020 19:37:34 +0200 Subject: [PATCH 4/9] =?UTF-8?q?using=20=C2=A7=20marker=20-=20Closes=20#347?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- v2/pkg/requests/util.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/requests/util.go b/v2/pkg/requests/util.go index a59deaee95..5bdc406e80 100644 --- a/v2/pkg/requests/util.go +++ b/v2/pkg/requests/util.go @@ -8,10 +8,17 @@ import ( "strings" ) +const ( + MARKER_PARENTHESIS_OPEN = "{{" + MARKER_PARENTHESIS_CLOSE = "}}" + MARKER_GENERAL = "ยง" +) + func newReplacer(values map[string]interface{}) *strings.Replacer { var replacerItems []string for k, v := range values { - replacerItems = append(replacerItems, fmt.Sprintf("{{%s}}", k), fmt.Sprintf("%s", v), k, fmt.Sprintf("%s", v)) + replacerItems = append(replacerItems, fmt.Sprintf("%s%s%s", MARKER_PARENTHESIS_OPEN, k, MARKER_PARENTHESIS_CLOSE), fmt.Sprintf("%s", v)) + replacerItems = append(replacerItems, fmt.Sprintf("%s%s%s", MARKER_GENERAL, k, MARKER_GENERAL), fmt.Sprintf("%s", v)) } return strings.NewReplacer(replacerItems...) From ed855c9224afc350a21e8dd7177be508483b5a7c Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 11 Oct 2020 20:26:27 +0200 Subject: [PATCH 5/9] adding random generators - close #234 --- v2/pkg/generators/dsl.go | 106 ++++++++++++++++++++++++++++++++++++++ v2/pkg/generators/util.go | 26 ++++++++++ 2 files changed, 132 insertions(+) diff --git a/v2/pkg/generators/dsl.go b/v2/pkg/generators/dsl.go index 8115adf699..89449c3737 100644 --- a/v2/pkg/generators/dsl.go +++ b/v2/pkg/generators/dsl.go @@ -7,6 +7,8 @@ import ( "encoding/base64" "encoding/hex" "html" + "math" + "math/rand" "net/url" "regexp" "strings" @@ -14,6 +16,9 @@ import ( "github.com/Knetic/govaluate" ) +var letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +var numbers = "1234567890" + // HelperFunctions contains the dsl functions func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { functions = make(map[string]govaluate.ExpressionFunction) @@ -144,5 +149,106 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { return compiled.MatchString(args[1].(string)), nil } + // random generators + functions["rand_char"] = func(args ...interface{}) (interface{}, error) { + chars := letters + numbers + bad := "" + if len(args) >= 1 { + chars = args[0].(string) + } + if len(args) >= 2 { + bad = args[1].(string) + } + + chars = TrimAll(chars, bad) + + return chars[rand.Intn(len(chars))], nil + } + + functions["rand_base"] = func(args ...interface{}) (interface{}, error) { + l := 0 + bad := "" + base := letters + numbers + + if len(args) >= 1 { + l = args[0].(int) + } + if len(args) >= 2 { + bad = args[1].(string) + } + if len(args) >= 3 { + base = args[2].(string) + } + + base = TrimAll(base, bad) + + return RandSeq(base, l), nil + } + + functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) { + l := 0 + bad := "" + chars := letters + numbers + + if len(args) >= 1 { + l = args[0].(int) + } + if len(args) >= 2 { + bad = args[1].(string) + } + + chars = TrimAll(chars, bad) + + return RandSeq(chars, l), nil + } + + functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) { + l := 0 + bad := "" + chars := letters + + if len(args) >= 1 { + l = args[0].(int) + } + if len(args) >= 2 { + bad = args[1].(string) + } + + chars = TrimAll(chars, bad) + + return RandSeq(chars, l), nil + } + + functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) { + l := 0 + bad := "" + chars := numbers + + if len(args) >= 1 { + l = args[0].(int) + } + if len(args) >= 2 { + bad = args[1].(string) + } + + chars = TrimAll(chars, bad) + + return RandSeq(chars, l), nil + } + + functions["rand_int"] = func(args ...interface{}) (interface{}, error) { + min := 0 + max := math.MaxInt32 + + if len(args) >= 1 { + min = args[0].(int) + } + if len(args) >= 2 { + max = args[1].(int) + } + + return rand.Intn(max-min) + min, nil + } + return functions } diff --git a/v2/pkg/generators/util.go b/v2/pkg/generators/util.go index 0b542e6bca..20c491c7b5 100644 --- a/v2/pkg/generators/util.go +++ b/v2/pkg/generators/util.go @@ -3,6 +3,7 @@ package generators import ( "bufio" "fmt" + "math/rand" "os" "strings" ) @@ -158,3 +159,28 @@ func FileExists(filename string) bool { return !info.IsDir() } + +// TrimDelimiters removes trailing brackets +func SliceContins(s []string, k string) bool { + for _, a := range s { + if a == k { + return true + } + } + return false +} + +func TrimAll(s string, cutset string) string { + for _, c := range cutset { + s = strings.ReplaceAll(s, string(c), "") + } + return s +} + +func RandSeq(base string, n int) string { + b := make([]rune, n) + for i := range b { + b[i] = rune(base[rand.Intn(len(base))]) + } + return string(b) +} From 39fc9736d0976c97ca09a97daba736c0461edecc Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 11 Oct 2020 20:59:37 +0200 Subject: [PATCH 6/9] removed formatting dirctive as string builder is used - Closes #220 --- v2/pkg/executer/output_http.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/v2/pkg/executer/output_http.go b/v2/pkg/executer/output_http.go index 12aeb75978..387b7a9ff5 100644 --- a/v2/pkg/executer/output_http.go +++ b/v2/pkg/executer/output_http.go @@ -99,9 +99,7 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res builder.WriteString("] ") } - // Escape the URL by replacing all % with %% - escapedURL := strings.ReplaceAll(URL, "%", "%%") - builder.WriteString(escapedURL) + builder.WriteString(URL) // If any extractors, write the results if len(extractorResults) > 0 { From 695c2a2768d7ee31d08afacc149f79db64e553c9 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 11 Oct 2020 21:18:10 +0200 Subject: [PATCH 7/9] adding payloads values to json output --- v2/pkg/executer/executer_http.go | 4 ++-- v2/pkg/executer/http_utils.go | 23 ++++++++++++----------- v2/pkg/executer/output_http.go | 3 ++- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index cb5a4e6704..6407958e5a 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -402,7 +402,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, result.Meta = request.Meta result.GotResults = true result.Unlock() - e.writeOutputHTTP(request, resp, body, matcher, nil) + e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta) } } } @@ -433,7 +433,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition { - e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults) + e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta) result.Lock() result.GotResults = true result.Unlock() diff --git a/v2/pkg/executer/http_utils.go b/v2/pkg/executer/http_utils.go index ee30c6a21f..8937367aed 100644 --- a/v2/pkg/executer/http_utils.go +++ b/v2/pkg/executer/http_utils.go @@ -7,17 +7,18 @@ import ( ) type jsonOutput struct { - Template string `json:"template"` - Type string `json:"type"` - Matched string `json:"matched"` - MatcherName string `json:"matcher_name,omitempty"` - ExtractedResults []string `json:"extracted_results,omitempty"` - Name string `json:"name"` - Severity string `json:"severity"` - Author string `json:"author"` - Description string `json:"description"` - Request string `json:"request,omitempty"` - Response string `json:"response,omitempty"` + Template string `json:"template"` + Type string `json:"type"` + Matched string `json:"matched"` + MatcherName string `json:"matcher_name,omitempty"` + ExtractedResults []string `json:"extracted_results,omitempty"` + Name string `json:"name"` + Severity string `json:"severity"` + Author string `json:"author"` + Description string `json:"description"` + Request string `json:"request,omitempty"` + Response string `json:"response,omitempty"` + Meta map[string]interface{} `json:"meta,omitempty"` } // unsafeToString converts byte slice to string with zero allocations diff --git a/v2/pkg/executer/output_http.go b/v2/pkg/executer/output_http.go index 387b7a9ff5..a6f17bfbdb 100644 --- a/v2/pkg/executer/output_http.go +++ b/v2/pkg/executer/output_http.go @@ -12,7 +12,7 @@ import ( ) // writeOutputHTTP writes http output to streams -func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string) { +func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}) { var URL string // rawhttp if req.RawRequest != nil { @@ -32,6 +32,7 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res Severity: e.template.Info.Severity, Author: e.template.Info.Author, Description: e.template.Info.Description, + Meta: meta, } if matcher != nil && len(matcher.Name) > 0 { From 500d7bb99761e18008d70b1d408f37aebc11f3fa Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 11 Oct 2020 22:44:15 +0200 Subject: [PATCH 8/9] solving logical bug with input without port --- v2/pkg/requests/bulk-http-request.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 451b025073..f9e49e1455 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -303,10 +303,13 @@ func setHeader(req *http.Request, name, value string) { // the template port and path preference func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { // template port preference over input URL port + // template has port hasPort := len(urlWithPortRgx.FindStringSubmatch(data)) > 0 if hasPort { - hostname, _, _ := net.SplitHostPort(parsedURL.Host) - parsedURL.Host = hostname + // check if also the input contains port, in this case extracts the url + if hostname, _, err := net.SplitHostPort(parsedURL.Host); err == nil { + parsedURL.Host = hostname + } } return parsedURL.String() From 84c9373b71d6dda734bbd769e0106938b7dc8f75 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 12 Oct 2020 08:14:07 +0200 Subject: [PATCH 9/9] implemented request count estimation --- v2/pkg/requests/bulk-http-request.go | 5 ++--- v2/pkg/requests/generator.go | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index f9e49e1455..c2185a901f 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -74,7 +74,6 @@ type BulkHTTPRequest struct { // DisableAutoContentLength Enable/Disable Content-Length header for unsafe raw requests DisableAutoContentLength bool `yaml:"disable-automatic-content-length-header,omitempty"` Threads int `yaml:"threads,omitempty"` - RateLimit int `yaml:"rate-limit,omitempty"` // Internal Finite State Machine keeping track of scan process gsfm *GeneratorFSM @@ -102,7 +101,7 @@ func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) { // GetRequestCount returns the total number of requests the YAML rule will perform func (r *BulkHTTPRequest) GetRequestCount() int64 { - return int64(len(r.Raw) | len(r.Path)) + return int64(r.gsfm.Total()) } // MakeHTTPRequest makes the HTTP request @@ -450,7 +449,7 @@ func (r *BulkHTTPRequest) Current(reqURL string) string { // Total is the total number of requests func (r *BulkHTTPRequest) Total() int { - return len(r.Path) + len(r.Raw) + return r.gsfm.Total() } // Increment increments the processed request diff --git a/v2/pkg/requests/generator.go b/v2/pkg/requests/generator.go index faba9e02fe..fa0e4f8379 100644 --- a/v2/pkg/requests/generator.go +++ b/v2/pkg/requests/generator.go @@ -41,6 +41,7 @@ func NewGeneratorFSM(typ generators.Type, payloads map[string]interface{}, paths gsfm.payloads = payloads gsfm.Paths = paths gsfm.Raws = raws + gsfm.Type = typ if len(gsfm.payloads) > 0 { // load payloads if not already done @@ -231,7 +232,30 @@ func (gfsm *GeneratorFSM) Current(key string) string { return gfsm.Raws[g.positionRaw] } func (gfsm *GeneratorFSM) Total() int { - return len(gfsm.Paths) + len(gfsm.Raws) + estimatedRequestsWithPayload := 0 + if len(gfsm.basePayloads) > 0 { + switch gfsm.Type { + case generators.Sniper: + for _, kv := range gfsm.basePayloads { + estimatedRequestsWithPayload += len(kv) + } + case generators.PitchFork: + // Positional so it's equal to the length of one list + for _, kv := range gfsm.basePayloads { + estimatedRequestsWithPayload += len(kv) + break + } + case generators.ClusterBomb: + // Total of combinations => rule of product + prod := 1 + for _, kv := range gfsm.basePayloads { + prod = prod * len(kv) + } + estimatedRequestsWithPayload += prod + } + } + + return len(gfsm.Paths) + len(gfsm.Raws) + estimatedRequestsWithPayload } func (gfsm *GeneratorFSM) Increment(key string) {