Skip to content

Commit

Permalink
minor refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@Mzack9999
Mzack9999 committed Oct 9, 2020
1 parent eb73df7 commit 7ab1933
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 128 deletions.
127 changes: 9 additions & 118 deletions v2/pkg/executer/executer_http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,11 +175,13 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(ctx context.Context, p progress.IProgres
defer swg.Done()

// If the request was built correctly then execute it
err = e.handleHTTPPipeline(pipeclient, reqURL, httpRequest, dynamicvalues, &result)
request.PipelineClient = pipeclient
err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, &result)
if err != nil {
result.Error = errors.Wrap(err, "could not handle http request")
p.Drop(remaining)
}
request.PipelineClient = nil

}(request)
}
Expand Down Expand Up @@ -236,8 +238,7 @@ func (e *HTTPExecuter) ExecuteHTTP(ctx context.Context, p progress.IProgress, re
return result
}

func (e *HTTPExecuter) handleHTTPPipeline(pipeline *rawhttp.PipelineClient, reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result) error {
// TODO: for now just a copy+paste for testing purposes => Needs to be fully async
func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result) error {
e.setCustomHeaders(request)

var (
Expand All @@ -256,128 +257,18 @@ func (e *HTTPExecuter) handleHTTPPipeline(pipeline *rawhttp.PipelineClient, reqU
}

timeStart := time.Now()
resp, err = pipeline.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)))
if err != nil {
return err
}
duration := time.Since(timeStart)

if e.debug {
dumpedResponse, dumpErr := httputil.DumpResponse(resp, true)
if dumpErr != nil {
return errors.Wrap(dumpErr, "could not dump http response")
}

gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID)
fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
}

data, err := ioutil.ReadAll(resp.Body)
if err != nil {
_, copyErr := io.Copy(ioutil.Discard, resp.Body)
if copyErr != nil {
resp.Body.Close()
return copyErr
}

resp.Body.Close()

return errors.Wrap(err, "could not read http body")
}

resp.Body.Close()

// net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request
// so in case we have to manually do it
data, err = requests.HandleDecompression(request, data)
if err != nil {
return errors.Wrap(err, "could not decompress http body")
}

// Convert response body from []byte to string with zero copy
body := unsafeToString(data)

headers := headersToString(resp.Header)
matcherCondition := e.bulkHTTPRequest.GetMatchersCondition()

for _, matcher := range e.bulkHTTPRequest.Matchers {
// Check if the matcher matched
if !matcher.Match(resp, body, headers, duration) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
return nil
}
} else {
// If the matcher has matched, and its an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition {
result.Matches[matcher.Name] = nil
// probably redundant but ensures we snapshot current payload values when matchers are valid
result.Meta = request.Meta
e.writeOutputHTTP(request, resp, body, matcher, nil)
result.GotResults = true
}
}
}

// All matchers have successfully completed so now start with the
// next task which is extraction of input from matchers.
var extractorResults, outputExtractorResults []string

for _, extractor := range e.bulkHTTPRequest.Extractors {
for match := range extractor.Extract(resp, body, headers) {
if _, ok := dynamicvalues[extractor.Name]; !ok {
dynamicvalues[extractor.Name] = match
}

extractorResults = append(extractorResults, match)

if !extractor.Internal {
outputExtractorResults = append(outputExtractorResults, match)
}
}
// probably redundant but ensures we snapshot current payload values when extractors are valid
result.Meta = request.Meta
result.Extractions[extractor.Name] = extractorResults
}

// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults)

result.GotResults = true
}

return nil
}

func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result) error {
e.setCustomHeaders(request)

var (
resp *http.Response
err error
)

if e.debug {
dumpedRequest, err := requests.Dump(request, reqURL)
if request.Pipeline {
resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)))
if err != nil {
return err
}

gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID)
fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
}

timeStart := time.Now()
if request.RawRequest != nil {
} else if request.Unsafe {
// rawhttp
// burp uses "\r\n" as new line character
request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n")
options := e.rawHttpClient.Options
options.AutomaticContentLength = request.RawRequest.AutomaticContentLength
options.AutomaticHostHeader = request.RawRequest.AutomaticHostHeader
options.AutomaticContentLength = request.AutomaticContentLengthHeader
options.AutomaticHostHeader = request.AutomaticHostHeader
resp, err = e.rawHttpClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options)
if err != nil {
return err
Expand Down
27 changes: 17 additions & 10 deletions v2/pkg/requests/bulk-http-request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/rawhttp"
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
)

Expand Down Expand Up @@ -220,9 +221,7 @@ func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL

// rawhttp
if r.Unsafe {
rawRequest.AutomaticContentLength = !r.DisableAutoContentLength
rawRequest.AutomaticHostHeader = !r.DisableAutoHostname
return &HTTPRequest{RawRequest: rawRequest, Meta: genValues}, nil
return &HTTPRequest{RawRequest: rawRequest, Meta: genValues, AutomaticHostHeader: !r.DisableAutoHostname, AutomaticContentLengthHeader: !r.DisableAutoContentLength, Unsafe: true}, nil
}

// retryablehttp
Expand Down Expand Up @@ -277,6 +276,16 @@ type HTTPRequest struct {
Request *retryablehttp.Request
RawRequest *RawRequest
Meta map[string]interface{}

// flags
Unsafe bool
Pipeline bool
AutomaticHostHeader bool
AutomaticContentLengthHeader bool
AutomaticConnectionHeader bool
Rawclient *rawhttp.Client
Httpclient *retryablehttp.Client
PipelineClient *rawhttp.PipelineClient
}

func setHeader(req *http.Request, name, value string) {
Expand Down Expand Up @@ -321,13 +330,11 @@ func (c *CustomHeaders) Set(value string) error {

// RawRequest defines a basic HTTP raw request
type RawRequest struct {
FullURL string
Method string
Path string
Data string
Headers map[string]string
AutomaticHostHeader bool
AutomaticContentLength bool
FullURL string
Method string
Path string
Data string
Headers map[string]string
}

// parseRawRequest parses the raw request as supplied by the user
Expand Down

0 comments on commit 7ab1933

Please sign in to comment.