New functionality to map fired blind payloads back to the initial req…

…uest (#632)

* Fix ioutil deprecation and use xdg paths instead (wip)

* Clean up deprecated ioutil references, add config directory structure creation and run entry creation

* Add wordlist position setting and FFUFHASH variable

* Save full wordlist paths and print out a raw request when searched

* Cast from string to 32bit integer, 2billion should be enough for a position

* Use correct format strings for float

CHANGELOG.md

@@ -1,7 +1,11 @@
 ## Changelog
 - master
   - New
+    - Added a new, dynamic keyword `FFUFHASH` that generates hash from job configuration and wordlist position to map blind payloads back to the initial request.
+    - New command line parameter for searching a hash: `-search FFUFHASH`
   - Changed
+    - Multiline output prints out alphabetically sorted by keyword
+    - Default configuration directories now follow `XDG_CONFIG_HOME` variable (less spam in your home directory)
     - Fixed issue with autocalibration of line & words filter
     - Fixed issue with `-json` when used in conjunction with silent mode
 

go.mod

@@ -2,4 +2,7 @@ module github.com/ffuf/ffuf
 
 go 1.13
 
-require github.com/pelletier/go-toml v1.8.1
+require (
+	github.com/adrg/xdg v0.4.0
+	github.com/pelletier/go-toml v1.8.1
+)

go.sum

@@ -1,3 +1,18 @@
+github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
+github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

help.go

@@ -16,7 +16,7 @@ type UsageSection struct {
 	ExpectedFlags []string
 }
 
-//PrintSection prints out the section name, description and each of the flags
+// PrintSection prints out the section name, description and each of the flags
 func (u *UsageSection) PrintSection(max_length int, extended bool) {
 	// Do not print if extended usage not requested and section marked as hidden
 	if !extended && u.Hidden {
@@ -35,7 +35,7 @@ type UsageFlag struct {
 	Default     string
 }
 
-//PrintFlag prints out the flag name, usage string and default value
+// PrintFlag prints out the flag name, usage string and default value
 func (f *UsageFlag) PrintFlag(max_length int) {
 	// Create format string, used for padding
 	format := fmt.Sprintf("  -%%-%ds %%s", max_length)
@@ -61,7 +61,7 @@ func Usage() {
 		Description:   "",
 		Flags:         make([]UsageFlag, 0),
 		Hidden:        false,
-		ExpectedFlags: []string{"ac", "acc", "ack", "ach", "acs", "c", "config", "json", "maxtime", "maxtime-job", "noninteractive", "p", "rate", "s", "sa", "se", "sf", "t", "v", "V"},
+		ExpectedFlags: []string{"ac", "acc", "ack", "ach", "acs", "c", "config", "json", "maxtime", "maxtime-job", "noninteractive", "p", "rate", "search", "s", "sa", "se", "sf", "t", "v", "V"},
 	}
 	u_compat := UsageSection{
 		Name:          "COMPATIBILITY OPTIONS",

main.go

@@ -4,17 +4,17 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"github.com/ffuf/ffuf/pkg/filter"
-	"io/ioutil"
-	"log"
-	"os"
-	"strings"
-
 	"github.com/ffuf/ffuf/pkg/ffuf"
+	"github.com/ffuf/ffuf/pkg/filter"
 	"github.com/ffuf/ffuf/pkg/input"
 	"github.com/ffuf/ffuf/pkg/interactive"
 	"github.com/ffuf/ffuf/pkg/output"
 	"github.com/ffuf/ffuf/pkg/runner"
+	"io"
+	"log"
+	"os"
+	"strings"
+	"time"
 )
 
 type multiStringFlag []string
@@ -45,7 +45,7 @@ func (m *wordlistFlag) Set(value string) error {
 	return nil
 }
 
-//ParseFlags parses the command line flags and (re)populates the ConfigOptions struct
+// ParseFlags parses the command line flags and (re)populates the ConfigOptions struct
 func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	var ignored bool
 	var cookies, autocalibrationstrings, headers, inputcommands multiStringFlag
@@ -96,6 +96,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	flag.StringVar(&opts.Filter.Time, "ft", opts.Filter.Time, "Filter by number of milliseconds to the first response byte, either greater or less than. EG: >100 or <100")
 	flag.StringVar(&opts.Filter.Words, "fw", opts.Filter.Words, "Filter by amount of words in response. Comma separated list of word counts and ranges")
 	flag.StringVar(&opts.General.Delay, "p", opts.General.Delay, "Seconds of `delay` between requests, or a range of random delay. For example \"0.1\" or \"0.1-2.0\"")
+	flag.StringVar(&opts.General.Searchhash, "search", opts.General.Searchhash, "Search for a FFUFHASH payload from ffuf history")
 	flag.StringVar(&opts.HTTP.Data, "d", opts.HTTP.Data, "POST data")
 	flag.StringVar(&opts.HTTP.Data, "data", opts.HTTP.Data, "POST data (alias of -d)")
 	flag.StringVar(&opts.HTTP.Data, "data-ascii", opts.HTTP.Data, "POST data (alias of -d)")
@@ -142,13 +143,37 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 func main() {
 
 	var err, optserr error
-
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 	// prepare the default config options from default config file
 	var opts *ffuf.ConfigOptions
 	opts, optserr = ffuf.ReadDefaultConfig()
 
 	opts = ParseFlags(opts)
 
+	// Handle searchhash functionality and exit
+	if opts.General.Searchhash != "" {
+		coptions, pos, err := ffuf.SearchHash(opts.General.Searchhash)
+		if err != nil {
+			fmt.Printf("[ERR] %s\n", err)
+			os.Exit(1)
+		}
+		if len(coptions) > 0 {
+			fmt.Printf("Request candidate(s) for hash %s\n", opts.General.Searchhash)
+		}
+		for _, copt := range coptions {
+			conf, err := ffuf.ConfigFromOptions(&copt.ConfigOptions, ctx, cancel)
+			if err != nil {
+				continue
+			}
+			printSearchResults(conf, pos, copt.Time, opts.General.Searchhash)
+		}
+		if err != nil {
+			fmt.Printf("[ERR] %s\n", err)
+		}
+		os.Exit(0)
+	}
+
 	if opts.General.ShowVersion {
 		fmt.Printf("ffuf version: %s\n", ffuf.Version())
 		os.Exit(0)
@@ -157,13 +182,13 @@ func main() {
 		f, err := os.OpenFile(opts.Output.DebugLog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Disabling logging, encountered error(s): %s\n", err)
-			log.SetOutput(ioutil.Discard)
+			log.SetOutput(io.Discard)
 		} else {
 			log.SetOutput(f)
 			defer f.Close()
 		}
 	} else {
-		log.SetOutput(ioutil.Discard)
+		log.SetOutput(io.Discard)
 	}
 	if optserr != nil {
 		log.Printf("Error while opening default config file: %s", optserr)
@@ -183,16 +208,15 @@ func main() {
 		opts = ParseFlags(opts)
 	}
 
-	// Prepare context and set up Config struct
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	// Set up Config struct
 	conf, err := ffuf.ConfigFromOptions(opts, ctx, cancel)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
 		Usage()
 		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
 		os.Exit(1)
 	}
+
 	job, err := prepareJob(conf)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
@@ -335,3 +359,23 @@ func SetupFilters(parseOpts *ffuf.ConfigOptions, conf *ffuf.Config) error {
 	}
 	return errs.ErrorOrNil()
 }
+
+func printSearchResults(conf *ffuf.Config, pos int, exectime time.Time, hash string) {
+	inp, err := input.NewInputProvider(conf)
+	if err.ErrorOrNil() != nil {
+		fmt.Printf("-------------------------------------------\n")
+		fmt.Println("Encountered error that prevents reproduction of the request:")
+		fmt.Println(err.ErrorOrNil())
+		return
+	}
+	inp.SetPosition(pos)
+	inputdata := inp.Value()
+	inputdata["FFUFHASH"] = []byte(hash)
+	basereq := ffuf.BaseRequest(conf)
+	dummyrunner := runner.NewRunnerByName("simple", conf, false)
+	ffufreq, _ := dummyrunner.Prepare(inputdata, &basereq)
+	rawreq, _ := dummyrunner.Dump(&ffufreq)
+	fmt.Printf("-------------------------------------------\n")
+	fmt.Printf("ffuf job started at: %s\n\n", exectime.Format(time.RFC3339))
+	fmt.Printf("%s\n", string(rawreq))
+}

pkg/ffuf/config.go

@@ -17,6 +17,7 @@ type Config struct {
 	ConfigFile              string                `json:"configfile"`
 	Context                 context.Context       `json:"-"`
 	Data                    string                `json:"postdata"`
+	Debuglog                string                `json:"debuglog"`
 	Delay                   optRange              `json:"delay"`
 	DirSearchCompat         bool                  `json:"dirsearch_compatibility"`
 	Extensions              []string              `json:"extensions"`
@@ -48,6 +49,8 @@ type Config struct {
 	RecursionDepth          int                   `json:"recursion_depth"`
 	RecursionStrategy       string                `json:"recursion_strategy"`
 	ReplayProxyURL          string                `json:"replayproxyurl"`
+	RequestFile             string                `json:"requestfile"`
+	RequestProto            string                `json:"requestproto"`
 	SNI                     string                `json:"sni"`
 	StopOn403               bool                  `json:"stop_403"`
 	StopOnAll               bool                  `json:"stop_all"`
@@ -56,6 +59,7 @@ type Config struct {
 	Timeout                 int                   `json:"timeout"`
 	Url                     string                `json:"url"`
 	Verbose                 bool                  `json:"verbose"`
+	Wordlists               []string              `json:"wordlists"`
 	Http2                   bool                  `json:"http2"`
 }
 
@@ -75,6 +79,7 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
 	conf.Context = ctx
 	conf.Cancel = cancel
 	conf.Data = ""
+	conf.Debuglog = ""
 	conf.Delay = optRange{0, 0, false, false}
 	conf.DirSearchCompat = false
 	conf.Extensions = make([]string, 0)
@@ -99,13 +104,16 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
 	conf.Recursion = false
 	conf.RecursionDepth = 0
 	conf.RecursionStrategy = "default"
+	conf.RequestFile = ""
+	conf.RequestProto = "https"
 	conf.SNI = ""
 	conf.StopOn403 = false
 	conf.StopOnAll = false
 	conf.StopOnErrors = false
 	conf.Timeout = 10
 	conf.Url = ""
 	conf.Verbose = false
+	conf.Wordlists = []string{}
 	conf.Http2 = false
 	return conf
 }

pkg/ffuf/configmarshaller.go

@@ -0,0 +1,127 @@
+package ffuf
+
+import (
+	"fmt"
+	"strings"
+)
+
+func (c *Config) ToOptions() ConfigOptions {
+	o := ConfigOptions{}
+	// HTTP options
+	o.HTTP.Cookies = []string{}
+	o.HTTP.Data = c.Data
+	o.HTTP.FollowRedirects = c.FollowRedirects
+	o.HTTP.Headers = make([]string, 0)
+	for k, v := range c.Headers {
+		o.HTTP.Headers = append(o.HTTP.Headers, fmt.Sprintf("%s: %s", k, v))
+	}
+	o.HTTP.IgnoreBody = c.IgnoreBody
+	o.HTTP.Method = c.Method
+	o.HTTP.ProxyURL = c.ProxyURL
+	o.HTTP.Recursion = c.Recursion
+	o.HTTP.RecursionDepth = c.RecursionDepth
+	o.HTTP.RecursionStrategy = c.RecursionStrategy
+	o.HTTP.ReplayProxyURL = c.ReplayProxyURL
+	o.HTTP.SNI = c.SNI
+	o.HTTP.Timeout = c.Timeout
+	o.HTTP.URL = c.Url
+	o.HTTP.Http2 = c.Http2
+
+	o.General.AutoCalibration = c.AutoCalibration
+	o.General.AutoCalibrationKeyword = c.AutoCalibrationKeyword
+	o.General.AutoCalibrationPerHost = c.AutoCalibrationPerHost
+	o.General.AutoCalibrationStrategy = c.AutoCalibrationStrategy
+	o.General.AutoCalibrationStrings = c.AutoCalibrationStrings
+	o.General.Colors = c.Colors
+	o.General.ConfigFile = ""
+	if c.Delay.HasDelay {
+		if c.Delay.IsRange {
+			o.General.Delay = fmt.Sprintf("%.2f-%.2f", c.Delay.Min, c.Delay.Max)
+		} else {
+			o.General.Delay = fmt.Sprintf("%.2f", c.Delay.Min)
+		}
+	} else {
+		o.General.Delay = ""
+	}
+	o.General.Json = c.Json
+	o.General.MaxTime = c.MaxTime
+	o.General.MaxTimeJob = c.MaxTimeJob
+	o.General.Noninteractive = c.Noninteractive
+	o.General.Quiet = c.Quiet
+	o.General.Rate = int(c.Rate)
+	o.General.StopOn403 = c.StopOn403
+	o.General.StopOnAll = c.StopOnAll
+	o.General.StopOnErrors = c.StopOnErrors
+	o.General.Threads = c.Threads
+	o.General.Verbose = c.Verbose
+
+	o.Input.DirSearchCompat = c.DirSearchCompat
+	o.Input.Extensions = strings.Join(c.Extensions, ",")
+	o.Input.IgnoreWordlistComments = c.IgnoreWordlistComments
+	o.Input.InputMode = c.InputMode
+	o.Input.InputNum = c.InputNum
+	o.Input.InputShell = c.InputShell
+	o.Input.Inputcommands = []string{}
+	for _, v := range c.InputProviders {
+		if v.Name == "command" {
+			o.Input.Inputcommands = append(o.Input.Inputcommands, fmt.Sprintf("%s:%s", v.Value, v.Keyword))
+		}
+	}
+	o.Input.Request = c.RequestFile
+	o.Input.RequestProto = c.RequestProto
+	o.Input.Wordlists = c.Wordlists
+
+	o.Output.DebugLog = c.Debuglog
+	o.Output.OutputDirectory = c.OutputDirectory
+	o.Output.OutputFile = c.OutputFile
+	o.Output.OutputFormat = c.OutputFormat
+	o.Output.OutputSkipEmptyFile = c.OutputSkipEmptyFile
+
+	o.Filter.Mode = c.FilterMode
+	o.Filter.Lines = ""
+	o.Filter.Regexp = ""
+	o.Filter.Size = ""
+	o.Filter.Status = ""
+	o.Filter.Time = ""
+	o.Filter.Words = ""
+	for name, filter := range c.MatcherManager.GetFilters() {
+		switch name {
+		case "line":
+			o.Filter.Lines = filter.Repr()
+		case "regexp":
+			o.Filter.Regexp = filter.Repr()
+		case "size":
+			o.Filter.Size = filter.Repr()
+		case "status":
+			o.Filter.Status = filter.Repr()
+		case "time":
+			o.Filter.Time = filter.Repr()
+		case "words":
+			o.Filter.Words = filter.Repr()
+		}
+	}
+	o.Matcher.Mode = c.MatcherMode
+	o.Matcher.Lines = ""
+	o.Matcher.Regexp = ""
+	o.Matcher.Size = ""
+	o.Matcher.Status = ""
+	o.Matcher.Time = ""
+	o.Matcher.Words = ""
+	for name, filter := range c.MatcherManager.GetMatchers() {
+		switch name {
+		case "line":
+			o.Matcher.Lines = filter.Repr()
+		case "regexp":
+			o.Matcher.Regexp = filter.Repr()
+		case "size":
+			o.Matcher.Size = filter.Repr()
+		case "status":
+			o.Matcher.Status = filter.Repr()
+		case "time":
+			o.Matcher.Time = filter.Repr()
+		case "words":
+			o.Matcher.Words = filter.Repr()
+		}
+	}
+	return o
+}

pkg/ffuf/version.go → pkg/ffuf/constants.go

@@ -1,8 +1,15 @@
 package ffuf
 
+import (
+	"github.com/adrg/xdg"
+	"path/filepath"
+)
+
 var (
 	//VERSION holds the current version number
 	VERSION = "1.5.0"
 	//VERSION_APPENDIX holds additional version definition
 	VERSION_APPENDIX = "-dev"
+	CONFIGDIR        = filepath.Join(xdg.ConfigHome, "ffuf")
+	HISTORYDIR       = filepath.Join(CONFIGDIR, "history")
 )