Skip to content

Commit

Permalink
Merge pull request #335 from projectdiscovery/feature-turbo-intruder
Browse files Browse the repository at this point in the history 
Turbo intruder/pipelining
  • Loading branch information
@Mzack9999
Mzack9999 authored Oct 9, 2020
2 parents 12cfece + e91fd95 commit 9ff0ce8
Show file tree
Hide file tree
Showing 9 changed files with 303 additions and 85 deletions.
5 changes: 3 additions & 2 deletions v2/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@ require (
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1
github.com/projectdiscovery/retryabledns v1.0.4
github.com/projectdiscovery/retryablehttp-go v1.0.1
github.com/stretchr/testify v1.5.1
github.com/remeh/sizedwaitgroup v1.0.0
github.com/vbauerster/mpb/v5 v5.3.0
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c
go.uber.org/ratelimit v0.1.0
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
gopkg.in/yaml.v2 v2.3.0
)
30 changes: 15 additions & 15 deletions v2/go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,20 @@
github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc=
github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
Expand All @@ -35,31 +38,28 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/rawhttp v0.0.2-0.20200929200351-394a0e5b5a8a h1:+MysXksoug4+a2q739DVhrtoKkOGwNoen0ux5swMGZw=
github.com/projectdiscovery/rawhttp v0.0.2-0.20200929200351-394a0e5b5a8a/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005001033-5c3b518c4288 h1:0gWt2HFAIj1KP/aflQ0wjxRnSMAD0eECd/FGX+aA7F8=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005001033-5c3b518c4288/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005194701-94f961939048 h1:o3ohMDTWYgTRQb7nivDZyk0acxEf7F2cG+qj8hdpCzc=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005194701-94f961939048/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200243-8cd4c43f84dc h1:Igp422RKczdlWm7XOEUpH3OCJxDGZ5L2oiAglbxtX6k=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200243-8cd4c43f84dc/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/rawhttp v0.0.1 h1:g9lzZPNBZsIAscTS2VLgv8RGQJcFXsOOTlzU49mDeBk=
github.com/projectdiscovery/rawhttp v0.0.1/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1 h1:I3aE8ta92M2XbrYKNYOTlXhodxyH+zQOt1jIatorhQA=
github.com/projectdiscovery/rawhttp v0.0.2-0.20201005200949-0a5c878e6ee1/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60=
github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/vbauerster/mpb v1.1.3 h1:IRgic8VFaURXkW0VxDLkNOiNaAgtw0okB2YIaVvJDI4=
github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw=
github.com/vbauerster/mpb/v5 v5.3.0 h1:vgrEJjUzHaSZKDRRxul5Oh4C72Yy/5VEMb0em+9M0mQ=
github.com/vbauerster/mpb/v5 v5.3.0/go.mod h1:4yTkvAb8Cm4eylAp6t0JRq6pXDkFJ4krUlDqWYkakAs=
go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
Expand All @@ -69,10 +69,9 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c h1:dk0ukUIHmGHqASjP0iue2261isepFCC6XRCSd1nHgDw=
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c/go.mod h1:iQL9McJNjoIa5mjH6nYTCTZXUN6RP+XW3eib7Ya3XcI=
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M=
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand All @@ -81,12 +80,13 @@ golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
Expand Down
3 changes: 1 addition & 2 deletions v2/internal/runner/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ func ParseOptions() *Options {
flag.BoolVar(&options.Version, "version", false, "Show version of nuclei")
flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output")
flag.IntVar(&options.Threads, "c", 50, "Number of concurrent requests to make")
flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout")
flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request")
flag.Var(&options.CustomHeaders, "H", "Custom Header.")
Expand All @@ -80,7 +79,7 @@ func ParseOptions() *Options {
flag.BoolVar(&options.JSONRequests, "json-requests", false, "Write requests/responses for matches in JSON output")
flag.BoolVar(&options.EnableProgressBar, "pbar", false, "Enable the progress bar")
flag.BoolVar(&options.TemplateList, "tl", false, "List available templates")
flag.IntVar(&options.RateLimit, "rl", 9999999, "Rate-Limit of requests per specified target") // 9999999 to avoid limiting
flag.IntVar(&options.RateLimit, "rate-limit", -1, "Per Target Rate-Limit")
flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)")

flag.Parse()
Expand Down
18 changes: 4 additions & 14 deletions v2/internal/runner/processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ type workflowTemplates struct {
}

// processTemplateWithList processes a template and runs the enumeration on all the targets
func (r *Runner) processTemplateWithList(ctx context.Context, p progress.IProgress, template *templates.Template, request interface{}) bool {
func (r *Runner) processTemplateWithList(p progress.IProgress, template *templates.Template, request interface{}) bool {
var httpExecuter *executer.HTTPExecuter
var dnsExecuter *executer.DNSExecuter
var err error
Expand Down Expand Up @@ -83,19 +83,15 @@ func (r *Runner) processTemplateWithList(ctx context.Context, p progress.IProgre

scanner := bufio.NewScanner(strings.NewReader(r.input))
for scanner.Scan() {
text := scanner.Text()

r.limiter <- struct{}{}

URL := scanner.Text()
wg.Add(1)

go func(URL string) {
defer wg.Done()

var result executer.Result

if httpExecuter != nil {
result = httpExecuter.ExecuteHTTP(ctx, p, URL)
result = httpExecuter.ExecuteHTTP(p, URL)
globalresult.Or(result.GotResults)
}

Expand All @@ -107,9 +103,7 @@ func (r *Runner) processTemplateWithList(ctx context.Context, p progress.IProgre
if result.Error != nil {
gologger.Warningf("[%s] Could not execute step: %s\n", r.colorizer.Colorizer.BrightBlue(template.ID), result.Error)
}

<-r.limiter
}(text)
}(URL)
}

wg.Wait()
Expand All @@ -136,8 +130,6 @@ func (r *Runner) processWorkflowWithList(p progress.IProgress, workflow *workflo
scanner := bufio.NewScanner(strings.NewReader(r.input))
for scanner.Scan() {
targetURL := scanner.Text()
r.limiter <- struct{}{}

wg.Add(1)

go func(targetURL string) {
Expand Down Expand Up @@ -171,8 +163,6 @@ func (r *Runner) processWorkflowWithList(p progress.IProgress, workflow *workflo
break
}
}

<-r.limiter
}(targetURL)
}

Expand Down
26 changes: 8 additions & 18 deletions v2/internal/runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,21 @@ package runner

import (
"bufio"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"regexp"
"strings"
"sync"
"time"

"golang.org/x/time/rate"

"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
)
Expand All @@ -36,7 +33,6 @@ type Runner struct {
templatesConfig *nucleiConfig
// options contains configuration options for runner
options *Options
limiter chan struct{}

// progress tracking
progress progress.IProgress
Expand Down Expand Up @@ -121,7 +117,7 @@ func New(options *Options) (*Runner, error) {
}

// Sanitize input and pre-compute total number of targets
var usedInput = make(map[string]bool)
var usedInput = make(map[string]struct{})

dupeCount := 0
sb := strings.Builder{}
Expand All @@ -136,9 +132,12 @@ func New(options *Options) (*Runner, error) {
}
// deduplication
if _, ok := usedInput[url]; !ok {
usedInput[url] = true
usedInput[url] = struct{}{}
runner.inputCount++

// allocate global rate limiters
globalratelimiter.Add(url, options.RateLimit)

sb.WriteString(url)
sb.WriteString("\n")
} else {
Expand All @@ -165,8 +164,6 @@ func New(options *Options) (*Runner, error) {
// Creates the progress tracking object
runner.progress = progress.NewProgress(runner.colorizer.Colorizer, options.EnableProgressBar)

runner.limiter = make(chan struct{}, options.Threads)

return runner, nil
}

Expand Down Expand Up @@ -240,9 +237,6 @@ func (r *Runner) RunEnumeration() {
if r.inputCount == 0 {
gologger.Errorf("Could not find any valid input URLs.")
} else if totalRequests > 0 || hasWorkflows {
ctx := context.Background()
// Limiter that will add to the tokenbucket every second and set the max size to -rl flag
rateLimit := rate.NewLimiter(rate.Every(1*time.Second), r.options.RateLimit)
// tracks global progress and captures stdout/stderr until p.Wait finishes
p := r.progress
p.InitProgressbar(r.inputCount, templateCount, totalRequests)
Expand All @@ -251,17 +245,13 @@ func (r *Runner) RunEnumeration() {
wgtemplates.Add(1)
go func(template interface{}) {
defer wgtemplates.Done()
err := rateLimit.Wait(ctx)
if err != nil {
gologger.Errorf("Issue with rate-limit")
}
switch tt := template.(type) {
case *templates.Template:
for _, request := range tt.RequestsDNS {
results.Or(r.processTemplateWithList(ctx, p, tt, request))
results.Or(r.processTemplateWithList(p, tt, request))
}
for _, request := range tt.BulkRequestsHTTP {
results.Or(r.processTemplateWithList(ctx, p, tt, request))
results.Or(r.processTemplateWithList(p, tt, request))
}
case *workflows.Workflow:
results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow)))
Expand Down
Loading

0 comments on commit 9ff0ce8

Please sign in to comment.