diff --git a/v2/go.mod b/v2/go.mod index e597560420..89ea9b17cc 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053 github.com/projectdiscovery/rawhttp v0.1.7 github.com/projectdiscovery/retryabledns v1.0.20 - github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a + github.com/projectdiscovery/retryablehttp-go v1.0.11 github.com/projectdiscovery/stringsutil v0.0.2 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 github.com/remeh/sizedwaitgroup v1.0.0 @@ -44,9 +44,9 @@ require ( github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 github.com/xanzy/go-gitlab v0.79.0 go.uber.org/multierr v1.9.0 - golang.org/x/net v0.5.0 + golang.org/x/net v0.6.0 golang.org/x/oauth2 v0.4.0 - golang.org/x/text v0.6.0 + golang.org/x/text v0.7.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) @@ -80,7 +80,7 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.0.2 github.com/projectdiscovery/uncover v1.0.2 - github.com/projectdiscovery/utils v0.0.8 + github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b github.com/projectdiscovery/wappalyzergo v0.0.79 github.com/stretchr/testify v1.8.1 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -185,7 +185,7 @@ require ( github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mholt/acmez v1.0.4 // indirect - github.com/microcosm-cc/bluemonday v1.0.21 // indirect + github.com/microcosm-cc/bluemonday v1.0.22 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -215,7 +215,7 @@ require ( golang.org/x/crypto v0.5.0 // indirect golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 golang.org/x/mod v0.7.0 // indirect - golang.org/x/sys v0.4.0 // indirect + golang.org/x/sys v0.5.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.5.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index ac223a8358..8c427988e8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -485,8 +485,9 @@ github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DA github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= +github.com/microcosm-cc/bluemonday v1.0.22 h1:p2tT7RNzRdCi0qmwxG+HbqD6ILkmwter1ZwVZn1oTxA= +github.com/microcosm-cc/bluemonday v1.0.22/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= @@ -592,8 +593,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.20 h1:grRyh4EzuyqsaK07iNkJKgrGLu/qDJwfDJ+83SBo6yo= github.com/projectdiscovery/retryabledns v1.0.20/go.mod h1:97Et22Kw2iPyvz/Vn41/i3dSbhLMHfeWP/S7EaLgmtg= -github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a h1:KUHx4Yxx7S+qX94TtCegLj/01obmohdVDeiG86FCHjM= -github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU= +github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7EegvkB8JfaoKCyoz6zo= +github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg= @@ -603,8 +604,8 @@ github.com/projectdiscovery/tlsx v1.0.2 h1:2bbfPQLuMIhs6FPmGsIcAo3uJaB2E+9ssJtZ8 github.com/projectdiscovery/tlsx v1.0.2/go.mod h1:WW+PdBImrqnMl18v4Brp3OsbnO4A1tqYPUcfiVtjNLM= github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo= github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY= -github.com/projectdiscovery/utils v0.0.8 h1:yPl/DwhW0IGnWNjapcw03g97ria8ZM8fH5PbcX4QFUo= -github.com/projectdiscovery/utils v0.0.8/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw= +github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b h1:7a4pnoEny9vrn9mmoBxo1yRP1RPMKCgFWkUaGKyGdAM= +github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw= github.com/projectdiscovery/wappalyzergo v0.0.79 h1:hWMxNysxC/P6fxnu6c+opqf5L27hHQ9wD1QzPRCb+I8= github.com/projectdiscovery/wappalyzergo v0.0.79/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= @@ -883,8 +884,9 @@ golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= @@ -954,8 +956,9 @@ golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -963,8 +966,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -972,8 +975,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 7fadd41c7e..0ec891ae7b 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -133,8 +133,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, finalparams := parsed.Params finalparams.Merge(reqURL.Params) reqURL.Params = finalparams - - return r.generateHttpRequest(ctx, reqURL.String(), finalVars, payloads) + return r.generateHttpRequest(ctx, reqURL, finalVars, payloads) } // selfContained templates do not need/use target data and all values i.e {{Hostname}} , {{BaseURL}} etc are already available @@ -205,19 +204,23 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st if err != nil { return nil, ErrEvalExpression.Wrap(err).WithTag("self-contained") } - return r.generateHttpRequest(ctx, data, values, payloads) + urlx, err := urlutil.ParseURL(data, true) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("failed to parse %v in self contained request", data).WithTag("self-contained") + } + return r.generateHttpRequest(ctx, urlx, values, payloads) } // generateHttpRequest generates http request from request data from template and variables // finalVars = contains all variables including generator and protocol specific variables // generatorValues = contains variables used in fuzzing or other generator specific values -func (r *requestGenerator) generateHttpRequest(ctx context.Context, data string, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) { +func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) { method, err := expressions.Evaluate(r.request.Method.String(), finalVars) if err != nil { return nil, ErrEvalExpression.Wrap(err).Msgf("failed to evaluate while generating http request") } // Build a request on the specified URL - req, err := retryablehttp.NewRequestWithContext(ctx, method, data, nil) + req, err := retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil) if err != nil { return nil, err } @@ -254,8 +257,11 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st // Todo: sync internally upon writing latest request byte body = race.NewOpenGateWithTimeout(body, time.Duration(2)*time.Second) } - - req, err := retryablehttp.NewRequestWithContext(ctx, rawRequestData.Method, rawRequestData.FullURL, body) + urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw") + } + req, err := retryablehttp.NewRequestFromURLWithContext(ctx, rawRequestData.Method, urlx, body) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/http/fuzz/parts.go b/v2/pkg/protocols/http/fuzz/parts.go index ccf95dbf50..83d9992b60 100644 --- a/v2/pkg/protocols/http/fuzz/parts.go +++ b/v2/pkg/protocols/http/fuzz/parts.go @@ -66,7 +66,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL, var req *retryablehttp.Request var err error if input.BaseRequest == nil { - req, err = retryablehttp.NewRequest(http.MethodGet, parsed.String(), nil) + req, err = retryablehttp.NewRequestFromURL(http.MethodGet, parsed, nil) if err != nil { return err } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1bd348840b..97c4936665 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -512,6 +512,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) } else if generatedRequest.request != nil { + // hot fix to avoid double url encoding (should only be called once) + generatedRequest.request.Prepare() resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) } } else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil { @@ -562,6 +564,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } httpclient = client } + generatedRequest.request.Prepare() resp, err = httpclient.Do(generatedRequest.request) } } @@ -570,6 +573,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ formedURL = input.MetaInput.Input } + // converts whitespace and other chars that cannot be printed to url encoded values + formedURL = urlutil.URLEncodeWithEscapes(formedURL) + // Dump the requests containing all headers if !generatedRequest.original.Race { var dumpError error