Add -raw cli flag (#721)

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Changelog
 - master
   - New
+    - New cli flag `-raw` to omit urlencoding for URIs
     - Integration with `github.com/ffuf/pencode` library, added `-enc` cli flag to do various in-fly encodings for input data
   - Changed
     - Explicitly allow TLS1.0 

ffufrc.example

@@ -14,6 +14,7 @@
     ignorebody = false
     method = "GET"
     proxyurl = "http://127.0.0.1:8080"
+    raw = false
     recursion = false
     recursion_depth = 0
     recursion_strategy = "default"

help.go

@@ -54,7 +54,7 @@ func Usage() {
 		Description:   "Options controlling the HTTP request and its parts.",
 		Flags:         make([]UsageFlag, 0),
 		Hidden:        false,
-		ExpectedFlags: []string{"cc", "ck", "H", "X", "b", "d", "r", "u", "recursion", "recursion-depth", "recursion-strategy", "replay-proxy", "timeout", "ignore-body", "x", "sni", "http2"},
+		ExpectedFlags: []string{"cc", "ck", "H", "X", "b", "d", "r", "u", "raw", "recursion", "recursion-depth", "recursion-strategy", "replay-proxy", "timeout", "ignore-body", "x", "sni", "http2"},
 	}
 	u_general := UsageSection{
 		Name:          "GENERAL OPTIONS",

main.go

@@ -77,6 +77,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	flag.BoolVar(&opts.General.Verbose, "v", opts.General.Verbose, "Verbose output, printing full URL and redirect location (if any) with the results.")
 	flag.BoolVar(&opts.HTTP.FollowRedirects, "r", opts.HTTP.FollowRedirects, "Follow redirects")
 	flag.BoolVar(&opts.HTTP.IgnoreBody, "ignore-body", opts.HTTP.IgnoreBody, "Do not fetch the response content.")
+	flag.BoolVar(&opts.HTTP.Raw, "raw", opts.HTTP.Raw, "Do not encode URI")
 	flag.BoolVar(&opts.HTTP.Recursion, "recursion", opts.HTTP.Recursion, "Scan recursively. Only FUZZ keyword is supported, and URL (-u) has to end in it.")
 	flag.BoolVar(&opts.HTTP.Http2, "http2", opts.HTTP.Http2, "Use HTTP2 protocol")
 	flag.BoolVar(&opts.Input.DirSearchCompat, "D", opts.Input.DirSearchCompat, "DirSearch wordlist compatibility mode. Used in conjunction with -e flag.")

pkg/ffuf/config.go

@@ -46,6 +46,7 @@ type Config struct {
 	ProxyURL                string                `json:"proxyurl"`
 	Quiet                   bool                  `json:"quiet"`
 	Rate                    int64                 `json:"rate"`
+	Raw                     bool                  `json:"raw"`
 	Recursion               bool                  `json:"recursion"`
 	RecursionDepth          int                   `json:"recursion_depth"`
 	RecursionStrategy       string                `json:"recursion_strategy"`
@@ -108,6 +109,7 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
 	conf.ProxyURL = ""
 	conf.Quiet = false
 	conf.Rate = 0
+	conf.Raw = false
 	conf.Recursion = false
 	conf.RecursionDepth = 0
 	conf.RecursionStrategy = "default"

pkg/ffuf/configmarshaller.go

@@ -18,6 +18,7 @@ func (c *Config) ToOptions() ConfigOptions {
 	o.HTTP.IgnoreBody = c.IgnoreBody
 	o.HTTP.Method = c.Method
 	o.HTTP.ProxyURL = c.ProxyURL
+	o.HTTP.Raw = c.Raw
 	o.HTTP.Recursion = c.Recursion
 	o.HTTP.RecursionDepth = c.RecursionDepth
 	o.HTTP.RecursionStrategy = c.RecursionStrategy

pkg/ffuf/optionsparser.go

@@ -33,6 +33,7 @@ type HTTPOptions struct {
 	IgnoreBody        bool     `json:"ignore_body"`
 	Method            string   `json:"method"`
 	ProxyURL          string   `json:"proxy_url"`
+	Raw               bool     `json:"raw"`
 	Recursion         bool     `json:"recursion"`
 	RecursionDepth    int      `json:"recursion_depth"`
 	RecursionStrategy string   `json:"recursion_strategy"`
@@ -148,6 +149,7 @@ func NewConfigOptions() *ConfigOptions {
 	c.HTTP.IgnoreBody = false
 	c.HTTP.Method = ""
 	c.HTTP.ProxyURL = ""
+	c.HTTP.Raw = false
 	c.HTTP.Recursion = false
 	c.HTTP.RecursionDepth = 0
 	c.HTTP.RecursionStrategy = "default"
@@ -514,6 +516,7 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 	conf.StopOnAll = parseOpts.General.StopOnAll
 	conf.StopOnErrors = parseOpts.General.StopOnErrors
 	conf.FollowRedirects = parseOpts.HTTP.FollowRedirects
+	conf.Raw = parseOpts.HTTP.Raw
 	conf.Recursion = parseOpts.HTTP.Recursion
 	conf.RecursionDepth = parseOpts.HTTP.RecursionDepth
 	conf.RecursionStrategy = parseOpts.HTTP.RecursionStrategy

pkg/runner/simple.go

@@ -137,13 +137,19 @@ func (r *SimpleRunner) Execute(req *ffuf.Request) (ffuf.Response, error) {
 
 	req.Host = httpreq.Host
 	httpreq = httpreq.WithContext(httptrace.WithClientTrace(r.config.Context, trace))
+
+	if r.config.Raw {
+		httpreq.URL.Opaque = req.Url
+	}
+
 	for k, v := range req.Headers {
 		httpreq.Header.Set(k, v)
 	}
 
 	if len(r.config.OutputDirectory) > 0 {
 		rawreq, _ = httputil.DumpRequestOut(httpreq, true)
 	}
+
 	httpresp, err := r.client.Do(httpreq)
 	if err != nil {
 		return ffuf.Response{}, err