Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix url re-encoding issues #3294

Merged
merged 4 commits into from
Feb 10, 2023
Merged

fix url re-encoding issues #3294

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1a5323c
fix double url encoding in urls
tarunKoyalwar Feb 9, 2023
d01cdfa
remove extra slash
tarunKoyalwar Feb 9, 2023
cc5e0fe
url encode matchedURL
tarunKoyalwar Feb 9, 2023
f5bf396
resolve merge conflicts
tarunKoyalwar Feb 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions v2/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ require (
github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053
github.com/projectdiscovery/rawhttp v0.1.7
github.com/projectdiscovery/retryabledns v1.0.20
github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a
github.com/projectdiscovery/retryablehttp-go v1.0.11
github.com/projectdiscovery/stringsutil v0.0.2
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
github.com/remeh/sizedwaitgroup v1.0.0
Expand All @@ -44,9 +44,9 @@ require (
github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37
github.com/xanzy/go-gitlab v0.79.0
go.uber.org/multierr v1.9.0
golang.org/x/net v0.5.0
golang.org/x/net v0.6.0
golang.org/x/oauth2 v0.4.0
golang.org/x/text v0.6.0
golang.org/x/text v0.7.0
gopkg.in/yaml.v2 v2.4.0
moul.io/http2curl v1.0.0
)
Expand Down Expand Up @@ -80,7 +80,7 @@ require (
github.com/projectdiscovery/sarif v0.0.1
github.com/projectdiscovery/tlsx v1.0.2
github.com/projectdiscovery/uncover v1.0.2
github.com/projectdiscovery/utils v0.0.8
github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b
github.com/projectdiscovery/wappalyzergo v0.0.79
github.com/stretchr/testify v1.8.1
gopkg.in/src-d/go-git.v4 v4.13.1
Expand Down Expand Up @@ -185,7 +185,7 @@ require (
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mholt/acmez v1.0.4 // indirect
github.com/microcosm-cc/bluemonday v1.0.21 // indirect
github.com/microcosm-cc/bluemonday v1.0.22 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand Down Expand Up @@ -215,7 +215,7 @@ require (
golang.org/x/crypto v0.5.0 // indirect
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30
golang.org/x/mod v0.7.0 // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.5.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
22 changes: 13 additions & 9 deletions v2/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,8 +485,9 @@ github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DA
github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU=
github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
github.com/microcosm-cc/bluemonday v1.0.22 h1:p2tT7RNzRdCi0qmwxG+HbqD6ILkmwter1ZwVZn1oTxA=
github.com/microcosm-cc/bluemonday v1.0.22/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
Expand Down Expand Up @@ -592,8 +593,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.20 h1:grRyh4EzuyqsaK07iNkJKgrGLu/qDJwfDJ+83SBo6yo=
github.com/projectdiscovery/retryabledns v1.0.20/go.mod h1:97Et22Kw2iPyvz/Vn41/i3dSbhLMHfeWP/S7EaLgmtg=
github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a h1:KUHx4Yxx7S+qX94TtCegLj/01obmohdVDeiG86FCHjM=
github.com/projectdiscovery/retryablehttp-go v1.0.10-0.20230123170312-75b58f90739a/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU=
github.com/projectdiscovery/retryablehttp-go v1.0.11 h1:dxJy/qR+4uOQ7th4rq8nIrW7EegvkB8JfaoKCyoz6zo=
github.com/projectdiscovery/retryablehttp-go v1.0.11/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg=
Expand All @@ -603,8 +604,8 @@ github.com/projectdiscovery/tlsx v1.0.2 h1:2bbfPQLuMIhs6FPmGsIcAo3uJaB2E+9ssJtZ8
github.com/projectdiscovery/tlsx v1.0.2/go.mod h1:WW+PdBImrqnMl18v4Brp3OsbnO4A1tqYPUcfiVtjNLM=
github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo=
github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY=
github.com/projectdiscovery/utils v0.0.8 h1:yPl/DwhW0IGnWNjapcw03g97ria8ZM8fH5PbcX4QFUo=
github.com/projectdiscovery/utils v0.0.8/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw=
github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b h1:7a4pnoEny9vrn9mmoBxo1yRP1RPMKCgFWkUaGKyGdAM=
github.com/projectdiscovery/utils v0.0.9-0.20230209185915-234ad5ea272b/go.mod h1:dZqlayNwgCGn2HgYfKrI71RjBEyKsEPovrU+UDfpQWw=
github.com/projectdiscovery/wappalyzergo v0.0.79 h1:hWMxNysxC/P6fxnu6c+opqf5L27hHQ9wD1QzPRCb+I8=
github.com/projectdiscovery/wappalyzergo v0.0.79/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8=
Expand Down Expand Up @@ -883,8 +884,9 @@ golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfS
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
Expand Down Expand Up @@ -954,26 +956,28 @@ golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
20 changes: 13 additions & 7 deletions v2/pkg/protocols/http/build_request.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
finalparams := parsed.Params
finalparams.Merge(reqURL.Params)
reqURL.Params = finalparams

return r.generateHttpRequest(ctx, reqURL.String(), finalVars, payloads)
return r.generateHttpRequest(ctx, reqURL, finalVars, payloads)
}

// selfContained templates do not need/use target data and all values i.e {{Hostname}} , {{BaseURL}} etc are already available
Expand Down Expand Up @@ -205,19 +204,23 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
if err != nil {
return nil, ErrEvalExpression.Wrap(err).WithTag("self-contained")
}
return r.generateHttpRequest(ctx, data, values, payloads)
urlx, err := urlutil.ParseURL(data, true)
if err != nil {
return nil, errorutil.NewWithErr(err).Msgf("failed to parse %v in self contained request", data).WithTag("self-contained")
}
return r.generateHttpRequest(ctx, urlx, values, payloads)
}

// generateHttpRequest generates http request from request data from template and variables
// finalVars = contains all variables including generator and protocol specific variables
// generatorValues = contains variables used in fuzzing or other generator specific values
func (r *requestGenerator) generateHttpRequest(ctx context.Context, data string, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
func (r *requestGenerator) generateHttpRequest(ctx context.Context, urlx *urlutil.URL, finalVars, generatorValues map[string]interface{}) (*generatedRequest, error) {
method, err := expressions.Evaluate(r.request.Method.String(), finalVars)
if err != nil {
return nil, ErrEvalExpression.Wrap(err).Msgf("failed to evaluate while generating http request")
}
// Build a request on the specified URL
req, err := retryablehttp.NewRequestWithContext(ctx, method, data, nil)
req, err := retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -254,8 +257,11 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
// Todo: sync internally upon writing latest request byte
body = race.NewOpenGateWithTimeout(body, time.Duration(2)*time.Second)
}

req, err := retryablehttp.NewRequestWithContext(ctx, rawRequestData.Method, rawRequestData.FullURL, body)
urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
if err != nil {
return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")
}
req, err := retryablehttp.NewRequestFromURLWithContext(ctx, rawRequestData.Method, urlx, body)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion v2/pkg/protocols/http/fuzz/parts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL,
var req *retryablehttp.Request
var err error
if input.BaseRequest == nil {
req, err = retryablehttp.NewRequest(http.MethodGet, parsed.String(), nil)
req, err = retryablehttp.NewRequestFromURL(http.MethodGet, parsed, nil)
if err != nil {
return err
}
Expand Down
6 changes: 6 additions & 0 deletions v2/pkg/protocols/http/request.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
}
resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
} else if generatedRequest.request != nil {
// hot fix to avoid double url encoding (should only be called once)
generatedRequest.request.Prepare()
resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
}
} else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil {
Expand Down Expand Up @@ -562,6 +564,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
}
httpclient = client
}
generatedRequest.request.Prepare()
resp, err = httpclient.Do(generatedRequest.request)
}
}
Expand All @@ -570,6 +573,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
formedURL = input.MetaInput.Input
}

// converts whitespace and other chars that cannot be printed to url encoded values
formedURL = urlutil.URLEncodeWithEscapes(formedURL)

// Dump the requests containing all headers
if !generatedRequest.original.Race {
var dumpError error
Expand Down