Merge pull request #348 from projectdiscovery/bugfix-various

reducing locking

v2/pkg/executer/executer_http.go

@@ -198,17 +198,20 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(p progress.IProgress, reqURL string) (re
 	pipeOptions := rawhttp.DefaultPipelineOptions
 	pipeOptions.Host = URL.Host
 	pipeOptions.MaxConnections = 1
-	if e.bulkHTTPRequest.PipelineMaxWorkers > 0 {
-		pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineMaxWorkers
+	if e.bulkHTTPRequest.PipelineConcurrentConnections > 0 {
+		pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineConcurrentConnections
+	}
+	if e.bulkHTTPRequest.PipelineRequestsPerConnection > 0 {
+		pipeOptions.MaxPendingRequests = e.bulkHTTPRequest.PipelineRequestsPerConnection
 	}
 	pipeclient := rawhttp.NewPipelineClient(pipeOptions)
 
-	// Workers that keeps enqueuing new requests
+	// 150 should be a sufficient value to keep queues always full
 	maxWorkers := 150
-	if e.bulkHTTPRequest.PipelineMaxWorkers > 0 {
-		maxWorkers = e.bulkHTTPRequest.PipelineMaxWorkers
+	// in case the queue is bigger increase the workers
+	if pipeOptions.MaxPendingRequests > maxWorkers {
+		maxWorkers = pipeOptions.MaxPendingRequests
 	}
-
 	swg := sizedwaitgroup.New(maxWorkers)
 	for e.bulkHTTPRequest.Next(reqURL) && !result.Done {
 		request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL))
@@ -221,8 +224,8 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(p progress.IProgress, reqURL string) (re
 				defer swg.Done()
 
 				// HTTP pipelining ignores rate limit
-
 				// If the request was built correctly then execute it
+				request.Pipeline = true
 				request.PipelineClient = pipeclient
 				err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, &result)
 				if err != nil {
@@ -399,7 +402,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 				result.Meta = request.Meta
 				result.GotResults = true
 				result.Unlock()
-				e.writeOutputHTTP(request, resp, body, matcher, nil)
+				e.writeOutputHTTP(request, resp, body, matcher, nil, result.Meta)
 			}
 		}
 	}
@@ -430,7 +433,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
 	// Write a final string of output if matcher type is
 	// AND or if we have extractors for the mechanism too.
 	if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
-		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults)
+		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, result.Meta)
 		result.Lock()
 		result.GotResults = true
 		result.Unlock()

v2/pkg/executer/http_utils.go

@@ -7,17 +7,18 @@ import (
 )
 
 type jsonOutput struct {
-	Template         string   `json:"template"`
-	Type             string   `json:"type"`
-	Matched          string   `json:"matched"`
-	MatcherName      string   `json:"matcher_name,omitempty"`
-	ExtractedResults []string `json:"extracted_results,omitempty"`
-	Name             string   `json:"name"`
-	Severity         string   `json:"severity"`
-	Author           string   `json:"author"`
-	Description      string   `json:"description"`
-	Request          string   `json:"request,omitempty"`
-	Response         string   `json:"response,omitempty"`
+	Template         string                 `json:"template"`
+	Type             string                 `json:"type"`
+	Matched          string                 `json:"matched"`
+	MatcherName      string                 `json:"matcher_name,omitempty"`
+	ExtractedResults []string               `json:"extracted_results,omitempty"`
+	Name             string                 `json:"name"`
+	Severity         string                 `json:"severity"`
+	Author           string                 `json:"author"`
+	Description      string                 `json:"description"`
+	Request          string                 `json:"request,omitempty"`
+	Response         string                 `json:"response,omitempty"`
+	Meta             map[string]interface{} `json:"meta,omitempty"`
 }
 
 // unsafeToString converts byte slice to string with zero allocations

v2/pkg/executer/output_http.go

@@ -12,7 +12,7 @@ import (
 )
 
 // writeOutputHTTP writes http output to streams
-func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string) {
+func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}) {
 	var URL string
 	// rawhttp
 	if req.RawRequest != nil {
@@ -32,6 +32,7 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res
 			Severity:    e.template.Info.Severity,
 			Author:      e.template.Info.Author,
 			Description: e.template.Info.Description,
+			Meta:        meta,
 		}
 
 		if matcher != nil && len(matcher.Name) > 0 {
@@ -99,9 +100,7 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res
 		builder.WriteString("] ")
 	}
 
-	// Escape the URL by replacing all % with %%
-	escapedURL := strings.ReplaceAll(URL, "%", "%%")
-	builder.WriteString(escapedURL)
+	builder.WriteString(URL)
 
 	// If any extractors, write the results
 	if len(extractorResults) > 0 {

v2/pkg/generators/dsl.go

@@ -7,13 +7,18 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"html"
+	"math"
+	"math/rand"
 	"net/url"
 	"regexp"
 	"strings"
 
 	"github.com/Knetic/govaluate"
 )
 
+var letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+var numbers = "1234567890"
+
 // HelperFunctions contains the dsl functions
 func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 	functions = make(map[string]govaluate.ExpressionFunction)
@@ -144,5 +149,106 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) {
 		return compiled.MatchString(args[1].(string)), nil
 	}
 
+	// random generators
+	functions["rand_char"] = func(args ...interface{}) (interface{}, error) {
+		chars := letters + numbers
+		bad := ""
+		if len(args) >= 1 {
+			chars = args[0].(string)
+		}
+		if len(args) >= 2 {
+			bad = args[1].(string)
+		}
+
+		chars = TrimAll(chars, bad)
+
+		return chars[rand.Intn(len(chars))], nil
+	}
+
+	functions["rand_base"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		base := letters + numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= 2 {
+			bad = args[1].(string)
+		}
+		if len(args) >= 3 {
+			base = args[2].(string)
+		}
+
+		base = TrimAll(base, bad)
+
+		return RandSeq(base, l), nil
+	}
+
+	functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := letters + numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= 2 {
+			bad = args[1].(string)
+		}
+
+		chars = TrimAll(chars, bad)
+
+		return RandSeq(chars, l), nil
+	}
+
+	functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := letters
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= 2 {
+			bad = args[1].(string)
+		}
+
+		chars = TrimAll(chars, bad)
+
+		return RandSeq(chars, l), nil
+	}
+
+	functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) {
+		l := 0
+		bad := ""
+		chars := numbers
+
+		if len(args) >= 1 {
+			l = args[0].(int)
+		}
+		if len(args) >= 2 {
+			bad = args[1].(string)
+		}
+
+		chars = TrimAll(chars, bad)
+
+		return RandSeq(chars, l), nil
+	}
+
+	functions["rand_int"] = func(args ...interface{}) (interface{}, error) {
+		min := 0
+		max := math.MaxInt32
+
+		if len(args) >= 1 {
+			min = args[0].(int)
+		}
+		if len(args) >= 2 {
+			max = args[1].(int)
+		}
+
+		return rand.Intn(max-min) + min, nil
+	}
+
 	return functions
 }

v2/pkg/generators/util.go

@@ -3,6 +3,7 @@ package generators
 import (
 	"bufio"
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
 )
@@ -158,3 +159,28 @@ func FileExists(filename string) bool {
 
 	return !info.IsDir()
 }
+
+// TrimDelimiters removes trailing brackets
+func SliceContins(s []string, k string) bool {
+	for _, a := range s {
+		if a == k {
+			return true
+		}
+	}
+	return false
+}
+
+func TrimAll(s string, cutset string) string {
+	for _, c := range cutset {
+		s = strings.ReplaceAll(s, string(c), "")
+	}
+	return s
+}
+
+func RandSeq(base string, n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = rune(base[rand.Intn(len(base))])
+	}
+	return string(b)
+}

v2/pkg/globalratelimiter/ratelimit.go

@@ -26,10 +26,16 @@ func Add(k string, rateLimit int) {
 }
 
 func Take(k string) {
+	rl := take(k)
+
+	rl.Take()
+}
+
+func take(k string) ratelimit.Limiter {
 	defaultrwmutex.RLock()
 	defer defaultrwmutex.RUnlock()
 
-	defaultGlobalRateLimiter.ratesLimiters[k].Take()
+	return defaultGlobalRateLimiter.ratesLimiters[k]
 }
 
 func Del(k string, rateLimit int) {
@@ -56,11 +62,16 @@ func (grl *GlobalRateLimiter) Add(k string, rateLimit int) {
 	}
 }
 
-func (grl *GlobalRateLimiter) Take(k string) {
+func (grl *GlobalRateLimiter) take(k string) ratelimit.Limiter {
 	grl.RLock()
 	defer grl.RUnlock()
 
-	grl.ratesLimiters[k].Take()
+	return grl.ratesLimiters[k]
+}
+
+func (grl *GlobalRateLimiter) Take(k string) {
+	rl := grl.take(k)
+	rl.Take()
 }
 
 func (grl *GlobalRateLimiter) Del(k string, rateLimit int) {