Skip to content

Commit

Permalink
[FEATURE] Extensible Auto-calibration strategies (#694)
Browse files Browse the repository at this point in the history 
* blacklist detection

* added option to help.go

* refactored -blacklist-detection to autocalibrationstrategy extra

* "No common filtering values found" fixed

* added wildcard not found detection

* custom auto-calibration strategies

* Make linter happy

---------

Co-authored-by: Joona Hoikkala <5235109+joohoi@users.noreply.github.com>
  • Loading branch information
@aristosMiliaressis @joohoi
aristosMiliaressis and joohoi committed Sep 15, 2023
1 parent a7dea16 commit e80fdc4
Show file tree
Hide file tree
Showing 9 changed files with 192 additions and 116 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
## Changelog
- master
- New
- autocalibration-strategy refactored to support extensible strategy configuration
- New cli flag `-raw` to omit urlencoding for URIs
- Integration with `github.com/ffuf/pencode` library, added `-enc` cli flag to do various in-fly encodings for input data
- Changed
Expand Down
11 changes: 9 additions & 2 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ func (m *wordlistFlag) Set(value string) error {
// ParseFlags parses the command line flags and (re)populates the ConfigOptions struct
func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
var ignored bool
var cookies, autocalibrationstrings, headers, inputcommands multiStringFlag

var cookies, autocalibrationstrings, autocalibrationstrategies, headers, inputcommands multiStringFlag
var wordlists, encoders wordlistFlag

cookies = opts.HTTP.Cookies
Expand Down Expand Up @@ -92,7 +93,6 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
flag.StringVar(&opts.General.AutoCalibrationKeyword, "ack", opts.General.AutoCalibrationKeyword, "Autocalibration keyword")
flag.StringVar(&opts.HTTP.ClientCert, "cc", "", "Client cert for authentication. Client key needs to be defined as well for this to work")
flag.StringVar(&opts.HTTP.ClientKey, "ck", "", "Client key for authentication. Client certificate needs to be defined as well for this to work")
flag.StringVar(&opts.General.AutoCalibrationStrategy, "acs", opts.General.AutoCalibrationStrategy, "Autocalibration strategy: \"basic\" or \"advanced\"")
flag.StringVar(&opts.General.ConfigFile, "config", "", "Load configuration from a file")
flag.StringVar(&opts.General.ScraperFile, "scraperfile", "", "Custom scraper file path")
flag.StringVar(&opts.General.Scrapers, "scrapers", opts.General.Scrapers, "Active scraper groups")
Expand Down Expand Up @@ -132,6 +132,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
flag.StringVar(&opts.Output.OutputFile, "o", opts.Output.OutputFile, "Write output to file")
flag.StringVar(&opts.Output.OutputFormat, "of", opts.Output.OutputFormat, "Output file format. Available formats: json, ejson, html, md, csv, ecsv (or, 'all' for all formats)")
flag.Var(&autocalibrationstrings, "acc", "Custom auto-calibration string. Can be used multiple times. Implies -ac")
flag.Var(&autocalibrationstrategies, "acs", "Custom auto-calibration strategies. Can be used multiple times. Implies -ac")
flag.Var(&cookies, "b", "Cookie data `\"NAME1=VALUE1; NAME2=VALUE2\"` for copy as curl functionality.")
flag.Var(&cookies, "cookie", "Cookie data (alias of -b)")
flag.Var(&headers, "H", "Header `\"Name: Value\"`, separated by colon. Multiple -H flags are accepted.")
Expand All @@ -142,6 +143,12 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
flag.Parse()

opts.General.AutoCalibrationStrings = autocalibrationstrings
if len(autocalibrationstrategies) > 0 {
opts.General.AutoCalibrationStrategies = []string {}
for _, strategy := range autocalibrationstrategies {
opts.General.AutoCalibrationStrategies = append(opts.General.AutoCalibrationStrategies, strings.Split(strategy, ",")...)
}
}
opts.HTTP.Cookies = cookies
opts.HTTP.Headers = headers
opts.Input.Inputcommands = inputcommands
Expand Down
79 changes: 64 additions & 15 deletions pkg/ffuf/autocalibration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,80 @@ import (
"math/rand"
"strconv"
"time"
"encoding/json"
"path/filepath"
"os"
)

type AutocalibrationStrategy map[string][]string

func (j *Job) autoCalibrationStrings() map[string][]string {
rand.Seed(time.Now().UnixNano())
cInputs := make(map[string][]string)
if len(j.Config.AutoCalibrationStrings) < 1 {
cInputs["basic_admin"] = append(cInputs["basic_admin"], "admin"+RandomString(16))
cInputs["basic_admin"] = append(cInputs["basic_admin"], "admin"+RandomString(8))
cInputs["htaccess"] = append(cInputs["htaccess"], ".htaccess"+RandomString(16))
cInputs["htaccess"] = append(cInputs["htaccess"], ".htaccess"+RandomString(8))
cInputs["basic_random"] = append(cInputs["basic_random"], RandomString(16))
cInputs["basic_random"] = append(cInputs["basic_random"], RandomString(8))
if j.Config.AutoCalibrationStrategy == "advanced" {
// Add directory tests and .htaccess too
cInputs["admin_dir"] = append(cInputs["admin_dir"], "admin"+RandomString(16)+"/")
cInputs["admin_dir"] = append(cInputs["admin_dir"], "admin"+RandomString(8)+"/")
cInputs["random_dir"] = append(cInputs["random_dir"], RandomString(16)+"/")
cInputs["random_dir"] = append(cInputs["random_dir"], RandomString(8)+"/")
}
} else {

if len(j.Config.AutoCalibrationStrings) > 0 {
cInputs["custom"] = append(cInputs["custom"], j.Config.AutoCalibrationStrings...)
return cInputs

}

for _, strategy := range j.Config.AutoCalibrationStrategies {
jsonStrategy, err := os.ReadFile(filepath.Join(AUTOCALIBDIR, strategy+".json"))
if err != nil {
j.Output.Warning(fmt.Sprintf("Skipping strategy \"%s\" because of error: %s\n", strategy, err))
continue
}

tmpStrategy := AutocalibrationStrategy{}
err = json.Unmarshal(jsonStrategy, &tmpStrategy)
if err != nil {
j.Output.Warning(fmt.Sprintf("Skipping strategy \"%s\" because of error: %s\n", strategy, err))
continue
}

cInputs = mergeMaps(cInputs, tmpStrategy)
}

return cInputs
}

func setupDefaultAutocalibrationStrategies() error {
basic_strategy := AutocalibrationStrategy {
"basic_admin": []string{"admin"+RandomString(16), "admin"+RandomString(8)},
"htaccess": []string{".htaccess"+RandomString(16), ".htaccess"+RandomString(8)},
"basic_random": []string{RandomString(16), RandomString(8)},
}
basic_strategy_json, err := json.Marshal(basic_strategy)
if err != nil {
return err
}

advanced_strategy := AutocalibrationStrategy {
"basic_admin": []string{"admin"+RandomString(16), "admin"+RandomString(8)},
"htaccess": []string{".htaccess"+RandomString(16), ".htaccess"+RandomString(8)},
"basic_random": []string{RandomString(16), RandomString(8)},
"admin_dir": []string{"admin"+RandomString(16)+"/", "admin"+RandomString(8)+"/"},
"random_dir": []string{RandomString(16)+"/", RandomString(8)+"/"},
}
advanced_strategy_json, err := json.Marshal(advanced_strategy)
if err != nil {
return err
}

basic_strategy_file := filepath.Join(AUTOCALIBDIR, "basic.json")
if !FileExists(basic_strategy_file) {
err = os.WriteFile(filepath.Join(AUTOCALIBDIR, "basic.json"), basic_strategy_json, 0640)
return err
}
advanced_strategy_file := filepath.Join(AUTOCALIBDIR, "advanced.json")
if !FileExists(advanced_strategy_file) {
err = os.WriteFile(filepath.Join(AUTOCALIBDIR, "advanced.json"), advanced_strategy_json, 0640)
return err
}

return nil
}

func (j *Job) calibrationRequest(inputs map[string][]byte) (Response, error) {
basereq := BaseRequest(j.Config)
req, err := j.Runner.Prepare(inputs, &basereq)
Expand Down
126 changes: 63 additions & 63 deletions pkg/ffuf/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,68 +5,68 @@ import (
)

type Config struct {
AutoCalibration bool `json:"autocalibration"`
AutoCalibrationKeyword string `json:"autocalibration_keyword"`
AutoCalibrationPerHost bool `json:"autocalibration_perhost"`
AutoCalibrationStrategy string `json:"autocalibration_strategy"`
AutoCalibrationStrings []string `json:"autocalibration_strings"`
Cancel context.CancelFunc `json:"-"`
Colors bool `json:"colors"`
CommandKeywords []string `json:"-"`
CommandLine string `json:"cmdline"`
ConfigFile string `json:"configfile"`
Context context.Context `json:"-"`
Data string `json:"postdata"`
Debuglog string `json:"debuglog"`
Delay optRange `json:"delay"`
DirSearchCompat bool `json:"dirsearch_compatibility"`
Encoders []string `json:"encoders"`
Extensions []string `json:"extensions"`
FilterMode string `json:"fmode"`
FollowRedirects bool `json:"follow_redirects"`
Headers map[string]string `json:"headers"`
IgnoreBody bool `json:"ignorebody"`
IgnoreWordlistComments bool `json:"ignore_wordlist_comments"`
InputMode string `json:"inputmode"`
InputNum int `json:"cmd_inputnum"`
InputProviders []InputProviderConfig `json:"inputproviders"`
InputShell string `json:"inputshell"`
Json bool `json:"json"`
MatcherManager MatcherManager `json:"matchers"`
MatcherMode string `json:"mmode"`
MaxTime int `json:"maxtime"`
MaxTimeJob int `json:"maxtime_job"`
Method string `json:"method"`
Noninteractive bool `json:"noninteractive"`
OutputDirectory string `json:"outputdirectory"`
OutputFile string `json:"outputfile"`
OutputFormat string `json:"outputformat"`
OutputSkipEmptyFile bool `json:"OutputSkipEmptyFile"`
ProgressFrequency int `json:"-"`
ProxyURL string `json:"proxyurl"`
Quiet bool `json:"quiet"`
Rate int64 `json:"rate"`
Raw bool `json:"raw"`
Recursion bool `json:"recursion"`
RecursionDepth int `json:"recursion_depth"`
RecursionStrategy string `json:"recursion_strategy"`
ReplayProxyURL string `json:"replayproxyurl"`
RequestFile string `json:"requestfile"`
RequestProto string `json:"requestproto"`
ScraperFile string `json:"scraperfile"`
Scrapers string `json:"scrapers"`
SNI string `json:"sni"`
StopOn403 bool `json:"stop_403"`
StopOnAll bool `json:"stop_all"`
StopOnErrors bool `json:"stop_errors"`
Threads int `json:"threads"`
Timeout int `json:"timeout"`
Url string `json:"url"`
Verbose bool `json:"verbose"`
Wordlists []string `json:"wordlists"`
Http2 bool `json:"http2"`
ClientCert string `json:"client-cert"`
ClientKey string `json:"client-key"`
AutoCalibration bool `json:"autocalibration"`
AutoCalibrationKeyword string `json:"autocalibration_keyword"`
AutoCalibrationPerHost bool `json:"autocalibration_perhost"`
AutoCalibrationStrategies []string `json:"autocalibration_strategies"`
AutoCalibrationStrings []string `json:"autocalibration_strings"`
Cancel context.CancelFunc `json:"-"`
Colors bool `json:"colors"`
CommandKeywords []string `json:"-"`
CommandLine string `json:"cmdline"`
ConfigFile string `json:"configfile"`
Context context.Context `json:"-"`
Data string `json:"postdata"`
Debuglog string `json:"debuglog"`
Delay optRange `json:"delay"`
DirSearchCompat bool `json:"dirsearch_compatibility"`
Encoders []string `json:"encoders"`
Extensions []string `json:"extensions"`
FilterMode string `json:"fmode"`
FollowRedirects bool `json:"follow_redirects"`
Headers map[string]string `json:"headers"`
IgnoreBody bool `json:"ignorebody"`
IgnoreWordlistComments bool `json:"ignore_wordlist_comments"`
InputMode string `json:"inputmode"`
InputNum int `json:"cmd_inputnum"`
InputProviders []InputProviderConfig `json:"inputproviders"`
InputShell string `json:"inputshell"`
Json bool `json:"json"`
MatcherManager MatcherManager `json:"matchers"`
MatcherMode string `json:"mmode"`
MaxTime int `json:"maxtime"`
MaxTimeJob int `json:"maxtime_job"`
Method string `json:"method"`
Noninteractive bool `json:"noninteractive"`
OutputDirectory string `json:"outputdirectory"`
OutputFile string `json:"outputfile"`
OutputFormat string `json:"outputformat"`
OutputSkipEmptyFile bool `json:"OutputSkipEmptyFile"`
ProgressFrequency int `json:"-"`
ProxyURL string `json:"proxyurl"`
Quiet bool `json:"quiet"`
Rate int64 `json:"rate"`
Raw bool `json:"raw"`
Recursion bool `json:"recursion"`
RecursionDepth int `json:"recursion_depth"`
RecursionStrategy string `json:"recursion_strategy"`
ReplayProxyURL string `json:"replayproxyurl"`
RequestFile string `json:"requestfile"`
RequestProto string `json:"requestproto"`
ScraperFile string `json:"scraperfile"`
Scrapers string `json:"scrapers"`
SNI string `json:"sni"`
StopOn403 bool `json:"stop_403"`
StopOnAll bool `json:"stop_all"`
StopOnErrors bool `json:"stop_errors"`
Threads int `json:"threads"`
Timeout int `json:"timeout"`
Url string `json:"url"`
Verbose bool `json:"verbose"`
Wordlists []string `json:"wordlists"`
Http2 bool `json:"http2"`
ClientCert string `json:"client-cert"`
ClientKey string `json:"client-key"`
}

type InputProviderConfig struct {
Expand All @@ -80,7 +80,7 @@ type InputProviderConfig struct {
func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
var conf Config
conf.AutoCalibrationKeyword = "FUZZ"
conf.AutoCalibrationStrategy = "basic"
conf.AutoCalibrationStrategies = []string{"basic"}
conf.AutoCalibrationStrings = make([]string, 0)
conf.CommandKeywords = make([]string, 0)
conf.Context = ctx
Expand Down
2 changes: 1 addition & 1 deletion pkg/ffuf/configmarshaller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func (c *Config) ToOptions() ConfigOptions {
o.General.AutoCalibration = c.AutoCalibration
o.General.AutoCalibrationKeyword = c.AutoCalibrationKeyword
o.General.AutoCalibrationPerHost = c.AutoCalibrationPerHost
o.General.AutoCalibrationStrategy = c.AutoCalibrationStrategy
o.General.AutoCalibrationStrategies = c.AutoCalibrationStrategies
o.General.AutoCalibrationStrings = c.AutoCalibrationStrings
o.General.Colors = c.Colors
o.General.ConfigFile = ""
Expand Down
1 change: 1 addition & 0 deletions pkg/ffuf/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@ var (
CONFIGDIR = filepath.Join(xdg.ConfigHome, "ffuf")
HISTORYDIR = filepath.Join(CONFIGDIR, "history")
SCRAPERDIR = filepath.Join(CONFIGDIR, "scraper")
AUTOCALIBDIR = filepath.Join(CONFIGDIR, "autocalibration")
)
Loading

0 comments on commit e80fdc4

Please sign in to comment.