Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SDK Improvements #1147

Merged
merged 3 commits into from
May 6, 2023
Merged

SDK Improvements #1147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0dc39fe
Added exported ScanOptions + Raw + Err methods
Ice3man543 Apr 12, 2023
318f9cf
Merge branch 'dev' into sdk-code-improvements
Mzack9999 May 6, 2023
b3b14b6
fixing tests
Mzack9999 May 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions runner/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const (
// OnResultCallback (hostResult)
type OnResultCallback func(Result)

type scanOptions struct {
type ScanOptions struct {
Methods []string
StoreResponseDirectory string
RequestURI string
Expand Down Expand Up @@ -87,8 +87,8 @@ type scanOptions struct {
UseInstalledChrome bool
}

func (s *scanOptions) Clone() *scanOptions {
return &scanOptions{
func (s *ScanOptions) Clone() *ScanOptions {
return &ScanOptions{
Methods: s.Methods,
StoreResponseDirectory: s.StoreResponseDirectory,
RequestURI: s.RequestURI,
Expand Down
52 changes: 26 additions & 26 deletions runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ type Runner struct {
hp *httpx.HTTPX
wappalyzer *wappalyzer.Wappalyze
fastdialer *fastdialer.Dialer
scanopts scanOptions
scanopts ScanOptions
hm *hybrid.HybridMap
stats clistats.StatisticsClient
ratelimiter ratelimit.Limiter
Expand Down Expand Up @@ -162,7 +162,7 @@ func New(options *Options) (*Runner, error) {
gologger.Fatal().Msgf("Could not create httpx instance: %s\n", err)
}

var scanopts scanOptions
var scanopts ScanOptions

if options.InputRawRequest != "" {
var rawRequest []byte
Expand Down Expand Up @@ -677,12 +677,12 @@ func (r *Runner) RunEnumeration() {
}

for resp := range output {
if resp.err != nil {
if resp.Err != nil {
// Change the error message if any port value passed explicitly
if url, err := r.parseURL(resp.URL); err == nil && url.Port() != "" {
resp.err = errors.New(strings.ReplaceAll(resp.err.Error(), "address", "port"))
resp.Err = errors.New(strings.ReplaceAll(resp.Err.Error(), "address", "port"))
}
gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.err)
gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.Err)
}
if resp.str == "" {
continue
Expand Down Expand Up @@ -746,10 +746,10 @@ func (r *Runner) RunEnumeration() {
if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) {
continue
}
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) {
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.Raw) {
continue
}
if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputFilterString)) {
if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.Raw), strings.ToLower(r.options.OutputFilterString)) {
continue
}
if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) {
Expand All @@ -761,10 +761,10 @@ func (r *Runner) RunEnumeration() {
if len(r.options.matchContentLength) > 0 && !slice.IntSliceContains(r.options.matchContentLength, resp.ContentLength) {
continue
}
if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.raw) {
if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.Raw) {
continue
}
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.Raw), strings.ToLower(r.options.OutputMatchString)) {
continue
}
if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) {
Expand Down Expand Up @@ -911,15 +911,15 @@ func (r *Runner) RunEnumeration() {
wgoutput.Wait()
}

func (r *Runner) GetScanOpts() scanOptions {
func (r *Runner) GetScanOpts() ScanOptions {
return r.scanopts
}

func (r *Runner) Process(t string, wg *sizedwaitgroup.SizedWaitGroup, protocol string, scanopts *scanOptions, output chan Result) {
func (r *Runner) Process(t string, wg *sizedwaitgroup.SizedWaitGroup, protocol string, scanopts *ScanOptions, output chan Result) {
r.process(t, wg, r.hp, protocol, scanopts, output)
}

func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) {
func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) {
protocols := []string{protocol}
if scanopts.NoFallback || protocol == httpx.HTTPandHTTPS {
protocols = []string{httpx.HTTPS, httpx.HTTP}
Expand Down Expand Up @@ -1062,7 +1062,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target {
return results
}

func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *scanOptions) Result {
func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *ScanOptions) Result {
origProtocol := protocol
if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS {
protocol = httpx.HTTPS
Expand All @@ -1074,22 +1074,22 @@ retry:
}
URL, err := r.parseURL(target.Host)
if err != nil {
return Result{URL: target.Host, Input: origInput, err: err}
return Result{URL: target.Host, Input: origInput, Err: err}
}

// check if we have to skip the host:port as a result of a previous failure
hostPort := net.JoinHostPort(URL.Host, URL.Port())
if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) {
numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort)
if err == nil && numberOfErrors >= r.options.HostMaxErrors {
return Result{URL: target.Host, err: errors.New("skipping as previously unresponsive")}
return Result{URL: target.Host, Err: errors.New("skipping as previously unresponsive")}
}
}

// check if the combination host:port should be skipped if belonging to a cdn
if r.skipCDNPort(URL.Host, URL.Port()) {
gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port())
return Result{URL: target.Host, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")}
return Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")}
}

URL.Scheme = protocol
Expand Down Expand Up @@ -1117,7 +1117,7 @@ retry:
req, err = hp.NewRequest(method, URL.String())
}
if err != nil {
return Result{URL: URL.String(), Input: origInput, err: err}
return Result{URL: URL.String(), Input: origInput, Err: err}
}

if target.CustomHost != "" {
Expand Down Expand Up @@ -1158,7 +1158,7 @@ retry:
var errDump error
requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
if errDump != nil {
return Result{URL: URL.String(), Input: origInput, err: errDump}
return Result{URL: URL.String(), Input: origInput, Err: errDump}
}
} else {
// Create a copy on the fly of the request body
Expand All @@ -1169,7 +1169,7 @@ retry:
var errDump error
requestDump, errDump = httputil.DumpRequestOut(req.Request, true)
if errDump != nil {
return Result{URL: URL.String(), Input: origInput, err: errDump}
return Result{URL: URL.String(), Input: origInput, Err: errDump}
}
// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
Expand All @@ -1181,7 +1181,7 @@ retry:
// fix the final output url
fullURL := req.URL.String()
if parsedURL, errParse := r.parseURL(fullURL); errParse != nil {
return Result{URL: URL.String(), Input: origInput, err: errParse}
return Result{URL: URL.String(), Input: origInput, Err: errParse}
} else {
if r.options.Unsafe {
parsedURL.Path = reqURI
Expand Down Expand Up @@ -1249,9 +1249,9 @@ retry:
}

if r.options.Probe {
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, str: builder.String()}
} else {
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err}
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err}
}
}

Expand Down Expand Up @@ -1652,7 +1652,7 @@ retry:

parsed, err := r.parseURL(fullURL)
if err != nil {
return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")}
return Result{URL: fullURL, Input: origInput, Err: errors.Wrap(err, "could not parse url")}
}

finalPort := parsed.Port()
Expand Down Expand Up @@ -1704,7 +1704,7 @@ retry:
Scheme: parsed.Scheme,
Port: finalPort,
Path: finalPath,
raw: resp.Raw,
Raw: resp.Raw,
URL: fullURL,
Input: origInput,
ContentLength: resp.ContentLength,
Expand Down Expand Up @@ -1829,7 +1829,7 @@ func (r *Runner) SaveResumeConfig() error {
}

// JSON the result
func (r Result) JSON(scanopts *scanOptions) string { //nolint
func (r Result) JSON(scanopts *ScanOptions) string { //nolint
if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave {
r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave]
}
Expand Down Expand Up @@ -1864,7 +1864,7 @@ func (r Result) CSVHeader() string { //nolint
}

// CSVRow the CSV Row
func (r Result) CSVRow(scanopts *scanOptions) string { //nolint
func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint
if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave {
r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave]
}
Expand Down
18 changes: 9 additions & 9 deletions runner/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,20 @@ func (o AsnResponse) String() string {

// Result of a scan
type Result struct {
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"`
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
err error
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"`
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
Err error `json:"-" csv:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
Port string `json:"port,omitempty" csv:"port"`
raw string
URL string `json:"url,omitempty" csv:"url"`
Input string `json:"input,omitempty" csv:"input"`
Location string `json:"location,omitempty" csv:"location"`
Title string `json:"title,omitempty" csv:"title"`
Raw string `json:"-" csv:"-"`
URL string `json:"url,omitempty" csv:"url"`
Input string `json:"input,omitempty" csv:"input"`
Location string `json:"location,omitempty" csv:"location"`
Title string `json:"title,omitempty" csv:"title"`
str string
Scheme string `json:"scheme,omitempty" csv:"scheme"`
Error string `json:"error,omitempty" csv:"error"`
Expand Down Expand Up @@ -81,7 +81,7 @@ type Result struct {
// function to get dsl variables from result struct
func dslVariables() ([]string, error) {
fakeResult := Result{}
fieldsToIgnore := []string{"Hashes", "ResponseHeader"}
fieldsToIgnore := []string{"Hashes", "ResponseHeader", "Err"}
if err := faker.FakeData(&fakeResult, options.WithFieldsToIgnore(fieldsToIgnore...)); err != nil {
return nil, err
}
Expand Down