From 98dad656526798362f970b24a74fd93928f9e2f1 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 15 Jul 2020 00:47:01 +0200 Subject: [PATCH 1/6] prototyping --- v2/pkg/extractors/compile.go | 5 +++-- v2/pkg/extractors/extract.go | 35 +++++++++++++++++++++------------ v2/pkg/extractors/extractors.go | 7 ++++++- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/v2/pkg/extractors/compile.go b/v2/pkg/extractors/compile.go index b4fd420aa6..6040580b2b 100644 --- a/v2/pkg/extractors/compile.go +++ b/v2/pkg/extractors/compile.go @@ -7,8 +7,9 @@ import ( // CompileExtractors performs the initial setup operation on a extractor func (e *Extractor) CompileExtractors() error { - // Setup the matcher type - _, ok := ExtractorTypes[e.Type] + var ok bool + // Setup the extractor type + e.extractorType, ok = ExtractorTypes[e.Type] if !ok { return fmt.Errorf("unknown extractor type specified: %s", e.Type) } diff --git a/v2/pkg/extractors/extract.go b/v2/pkg/extractors/extract.go index 2ff17cd8c4..a0502f5667 100644 --- a/v2/pkg/extractors/extract.go +++ b/v2/pkg/extractors/extract.go @@ -2,30 +2,39 @@ package extractors // Extract extracts response from the parts of request using a regex func (e *Extractor) Extract(body, headers string) map[string]struct{} { - // Match the parts as required for regex check - if e.part == BodyPart { - return e.extractRegex(body) - } else if e.part == HeaderPart { - return e.extractRegex(headers) - } else { - matches := e.extractRegex(headers) - if len(matches) > 0 { - return matches + switch e.extractorType { + case RegexExtractor: + if e.part == BodyPart { + return e.extractRegex(body) + } else if e.part == HeaderPart { + return e.extractRegex(headers) + } else { + matches := e.extractRegex(headers) + if len(matches) > 0 { + return matches + } + return e.extractRegex(body) } - return e.extractRegex(body) + case KValExtractor: } + + return nil } // ExtractDNS extracts response from dns message using a regex func (e *Extractor) ExtractDNS(msg string) map[string]struct{} { - // Match the parts as required for regex check - return e.extractRegex(msg) + switch e.extractorType { + case RegexExtractor: + return e.extractRegex(msg) + case KValExtractor: + } + + return nil } // extractRegex extracts text from a corpus and returns it func (e *Extractor) extractRegex(corpus string) map[string]struct{} { results := make(map[string]struct{}) - for _, regex := range e.regexCompiled { matches := regex.FindAllString(corpus, -1) for _, match := range matches { diff --git a/v2/pkg/extractors/extractors.go b/v2/pkg/extractors/extractors.go index 9be2093080..15de3ec96e 100644 --- a/v2/pkg/extractors/extractors.go +++ b/v2/pkg/extractors/extractors.go @@ -6,8 +6,10 @@ import "regexp" type Extractor struct { // Name is the extractor's name Name string `yaml:"name,omitempty"` - // Type is the type of the matcher + // Type is the type of the extractor Type string `yaml:"type"` + // extractorType is the internal type of the extractor + extractorType ExtractorType // Regex are the regex pattern required to be present in the response Regex []string `yaml:"regex"` @@ -28,11 +30,14 @@ type ExtractorType = int const ( // RegexExtractor extracts responses with regexes RegexExtractor ExtractorType = iota + 1 + // KValExtractor extracts responses with key:value + KValExtractor ) // ExtractorTypes is an table for conversion of extractor type from string. var ExtractorTypes = map[string]ExtractorType{ "regex": RegexExtractor, + "kval": KValExtractor, } // Part is the part of the request to match From 075509f91f0e3fcc872c65cced7788995a47058f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 16 Jul 2020 10:32:00 +0200 Subject: [PATCH 2/6] finalized + corrected typo --- v2/pkg/{executor => executer}/dns_utils.go | 0 .../executer_dns.go} | 2 +- .../{executor => executer}/executer_http.go | 2 +- v2/pkg/{executor => executer}/http_utils.go | 0 v2/pkg/{executor => executer}/output_dns.go | 0 v2/pkg/{executor => executer}/output_http.go | 0 v2/pkg/extractors/extract.go | 50 +++++++++++++++++-- v2/pkg/extractors/extractors.go | 3 ++ 8 files changed, 52 insertions(+), 5 deletions(-) rename v2/pkg/{executor => executer}/dns_utils.go (100%) rename v2/pkg/{executor/executor_dns.go => executer/executer_dns.go} (98%) rename v2/pkg/{executor => executer}/executer_http.go (99%) rename v2/pkg/{executor => executer}/http_utils.go (100%) rename v2/pkg/{executor => executer}/output_dns.go (100%) rename v2/pkg/{executor => executer}/output_http.go (100%) diff --git a/v2/pkg/executor/dns_utils.go b/v2/pkg/executer/dns_utils.go similarity index 100% rename from v2/pkg/executor/dns_utils.go rename to v2/pkg/executer/dns_utils.go diff --git a/v2/pkg/executor/executor_dns.go b/v2/pkg/executer/executer_dns.go similarity index 98% rename from v2/pkg/executor/executor_dns.go rename to v2/pkg/executer/executer_dns.go index 2561ab02bf..ea67231c28 100644 --- a/v2/pkg/executor/executor_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -129,7 +129,7 @@ func (e *DNSExecutor) ExecuteDNS(URL string) (result Result) { // next task which is extraction of input from matchers. var extractorResults []string for _, extractor := range e.dnsRequest.Extractors { - for match := range extractor.ExtractDNS(resp.String()) { + for match := range extractor.ExtractDNS(resp) { extractorResults = append(extractorResults, match) } } diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executer/executer_http.go similarity index 99% rename from v2/pkg/executor/executer_http.go rename to v2/pkg/executer/executer_http.go index 500f41e163..7e02b07078 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -187,7 +187,7 @@ mainLoop: // next task which is extraction of input from matchers. var extractorResults []string for _, extractor := range e.httpRequest.Extractors { - for match := range extractor.Extract(body, headers) { + for match := range extractor.Extract(resp, body, headers) { extractorResults = append(extractorResults, match) } // probably redundant but ensures we snapshot current payload values when extractors are valid diff --git a/v2/pkg/executor/http_utils.go b/v2/pkg/executer/http_utils.go similarity index 100% rename from v2/pkg/executor/http_utils.go rename to v2/pkg/executer/http_utils.go diff --git a/v2/pkg/executor/output_dns.go b/v2/pkg/executer/output_dns.go similarity index 100% rename from v2/pkg/executor/output_dns.go rename to v2/pkg/executer/output_dns.go diff --git a/v2/pkg/executor/output_http.go b/v2/pkg/executer/output_http.go similarity index 100% rename from v2/pkg/executor/output_http.go rename to v2/pkg/executer/output_http.go diff --git a/v2/pkg/extractors/extract.go b/v2/pkg/extractors/extract.go index a0502f5667..3030b1237a 100644 --- a/v2/pkg/extractors/extract.go +++ b/v2/pkg/extractors/extract.go @@ -1,7 +1,14 @@ package extractors +import ( + "net/http" + "strings" + + "github.com/miekg/dns" +) + // Extract extracts response from the parts of request using a regex -func (e *Extractor) Extract(body, headers string) map[string]struct{} { +func (e *Extractor) Extract(resp *http.Response, body, headers string) map[string]struct{} { switch e.extractorType { case RegexExtractor: if e.part == BodyPart { @@ -16,16 +23,25 @@ func (e *Extractor) Extract(body, headers string) map[string]struct{} { return e.extractRegex(body) } case KValExtractor: + if e.part == HeaderPart { + return e.extractKVal(resp) + } else { + matches := e.extractKVal(resp) + if len(matches) > 0 { + return matches + } + return e.extractInlineKVal(resp, "cookies") + } } return nil } // ExtractDNS extracts response from dns message using a regex -func (e *Extractor) ExtractDNS(msg string) map[string]struct{} { +func (e *Extractor) ExtractDNS(msg *dns.Msg) map[string]struct{} { switch e.extractorType { case RegexExtractor: - return e.extractRegex(msg) + return e.extractRegex(msg.String()) case KValExtractor: } @@ -43,3 +59,31 @@ func (e *Extractor) extractRegex(corpus string) map[string]struct{} { } return results } + +// extractKVal extracts text from http response +func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} { + results := make(map[string]struct{}) + for _, k := range e.KVal { + for _, v := range r.Header.Values(k) { + results[v] = struct{}{} + } + } + return results +} + +// extractInlineKVal extracts text from inline corpus +func (e *Extractor) extractInlineKVal(r *http.Response, key string) map[string]struct{} { + results := make(map[string]struct{}) + for _, v := range r.Header.Values(key) { + for _, token := range strings.Split(v, " ") { + semicolon := strings.Index(token, ":") + key, value := token[:semicolon], token[semicolon:] + for _, k := range e.KVal { + if key == k { + results[value] = struct{}{} + } + } + } + } + return results +} diff --git a/v2/pkg/extractors/extractors.go b/v2/pkg/extractors/extractors.go index 15de3ec96e..3a61e76bb3 100644 --- a/v2/pkg/extractors/extractors.go +++ b/v2/pkg/extractors/extractors.go @@ -16,6 +16,9 @@ type Extractor struct { // regexCompiled is the compiled variant regexCompiled []*regexp.Regexp + // KVal are the kval to be present in the response headers/cookies + KVal []string `yaml:"kval,omitempty"` + // Part is the part of the request to match // // By default, matching is performed in request body. From ef8146cfd55f450f059551caaf40e26bfd529c6f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 16 Jul 2020 10:57:28 +0200 Subject: [PATCH 3/6] typo global correction --- go.mod | 5 --- go.sum | 75 -------------------------------- v2/go.mod | 1 - v2/go.sum | 1 - v2/internal/runner/runner.go | 40 ++++++++--------- v2/pkg/executer/dns_utils.go | 2 +- v2/pkg/executer/executer_dns.go | 24 +++++----- v2/pkg/executer/executer_http.go | 26 +++++------ v2/pkg/executer/http_utils.go | 2 +- v2/pkg/executer/output_dns.go | 16 +++---- v2/pkg/executer/output_http.go | 16 +++---- v2/pkg/workflows/var.go | 20 ++++----- 12 files changed, 73 insertions(+), 155 deletions(-) delete mode 100644 go.sum diff --git a/go.mod b/go.mod index 7352132ebb..14e48f74b9 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,3 @@ module github.com/projectdiscovery/nuclei go 1.14 - -require ( - github.com/projectdiscovery/gologger v1.0.0 // indirect - github.com/projectdiscovery/nuclei/v2 v2.0.2 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 8347c0f353..0000000000 --- a/go.sum +++ /dev/null @@ -1,75 +0,0 @@ -github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= -github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= -github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ= -github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI= -github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM= -github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA= -github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= -github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s= -github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= -github.com/projectdiscovery/nuclei/v2 v2.0.2 h1:cB76N50hJv2G+EJpm3JFH5txHiyl7s9ZDYa2wuOoc5w= -github.com/projectdiscovery/nuclei/v2 v2.0.2/go.mod h1:mkj8PlAZ/c03i+ZTs+JD7X6iZeHwdbVuHpHFOkjjxMw= -github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60= -github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= -github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= -github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs= -golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/v2/go.mod b/v2/go.mod index 4ad4411413..ebed3c4216 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -6,7 +6,6 @@ require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 github.com/blang/semver v3.5.1+incompatible - github.com/d5/tengo v1.24.8 github.com/d5/tengo/v2 v2.6.0 github.com/google/go-github/v32 v32.0.0 github.com/json-iterator/go v1.1.10 diff --git a/v2/go.sum b/v2/go.sum index da00a4302d..bb6e2e38ff 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -7,7 +7,6 @@ github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= -github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ= github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI= github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index ee4ca914e9..fbd2a1bd0e 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -15,7 +15,7 @@ import ( "github.com/d5/tengo/v2/stdlib" "github.com/karrick/godirwalk" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/executor" + "github.com/projectdiscovery/nuclei/v2/pkg/executer" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" @@ -233,14 +233,14 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i defer writer.Flush() } - var httpExecutor *executor.HTTPExecutor - var dnsExecutor *executor.DNSExecutor + var httpExecuter *executer.HTTPExecuter + var dnsExecuter *executer.DNSExecuter var err error - // Create an executor based on the request type. + // Create an executer based on the request type. switch value := request.(type) { case *requests.DNSRequest: - dnsExecutor = executor.NewDNSExecutor(&executor.DNSOptions{ + dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{ Debug: r.options.Debug, Template: template, DNSRequest: value, @@ -248,7 +248,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i JSON: r.options.JSON, }) case *requests.HTTPRequest: - httpExecutor, err = executor.NewHTTPExecutor(&executor.HTTPOptions{ + httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{ Debug: r.options.Debug, Template: template, HTTPRequest: value, @@ -279,13 +279,13 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i wg.Add(1) go func(URL string) { - var result executor.Result + var result executer.Result - if httpExecutor != nil { - result = httpExecutor.ExecuteHTTP(URL) + if httpExecuter != nil { + result = httpExecuter.ExecuteHTTP(URL) } - if dnsExecutor != nil { - result = dnsExecutor.ExecuteDNS(URL) + if dnsExecuter != nil { + result = dnsExecuter.ExecuteDNS(URL) } if result.Error != nil { gologger.Warningf("Could not execute step: %s\n", result.Error) @@ -297,14 +297,14 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i close(limiter) wg.Wait() - // See if we got any results from the executors + // See if we got any results from the executers var results bool - if httpExecutor != nil { - results = httpExecutor.GotResults() + if httpExecuter != nil { + results = httpExecuter.GotResults() } - if dnsExecutor != nil { + if dnsExecuter != nil { if !results { - results = dnsExecutor.GotResults() + results = dnsExecuter.GotResults() } } return results @@ -367,7 +367,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error } template := &workflows.Template{} if len(t.RequestsHTTP) > 0 { - template.HTTPOptions = &executor.HTTPOptions{ + template.HTTPOptions = &executer.HTTPOptions{ Debug: r.options.Debug, Writer: writer, Template: t, @@ -378,7 +378,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error CustomHeaders: r.options.CustomHeaders, } } else if len(t.RequestsDNS) > 0 { - template.DNSOptions = &executor.DNSOptions{ + template.DNSOptions = &executer.DNSOptions{ Debug: r.options.Debug, Template: t, Writer: writer, @@ -417,7 +417,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error } template := &workflows.Template{} if len(t.RequestsHTTP) > 0 { - template.HTTPOptions = &executor.HTTPOptions{ + template.HTTPOptions = &executer.HTTPOptions{ Debug: r.options.Debug, Writer: writer, Template: t, @@ -428,7 +428,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error CustomHeaders: r.options.CustomHeaders, } } else if len(t.RequestsDNS) > 0 { - template.DNSOptions = &executor.DNSOptions{ + template.DNSOptions = &executer.DNSOptions{ Debug: r.options.Debug, Template: t, Writer: writer, diff --git a/v2/pkg/executer/dns_utils.go b/v2/pkg/executer/dns_utils.go index 6009e5011d..8c31939d37 100644 --- a/v2/pkg/executer/dns_utils.go +++ b/v2/pkg/executer/dns_utils.go @@ -1,4 +1,4 @@ -package executor +package executer import ( "net/url" diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index ea67231c28..8dda4e1c1b 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -1,4 +1,4 @@ -package executor +package executer import ( "bufio" @@ -15,9 +15,9 @@ import ( retryabledns "github.com/projectdiscovery/retryabledns" ) -// DNSExecutor is a client for performing a DNS request +// DNSExecuter is a client for performing a DNS request // for a template. -type DNSExecutor struct { +type DNSExecuter struct { debug bool jsonOutput bool results uint32 @@ -36,7 +36,7 @@ var DefaultResolvers = []string{ "8.8.4.4:53", // Google } -// DNSOptions contains configuration options for the DNS executor. +// DNSOptions contains configuration options for the DNS executer. type DNSOptions struct { Debug bool JSON bool @@ -45,12 +45,12 @@ type DNSOptions struct { Writer *bufio.Writer } -// NewDNSExecutor creates a new DNS executor from a template +// NewDNSExecuter creates a new DNS executer from a template // and a DNS request query. -func NewDNSExecutor(options *DNSOptions) *DNSExecutor { +func NewDNSExecuter(options *DNSOptions) *DNSExecuter { dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries) - executer := &DNSExecutor{ + executer := &DNSExecuter{ debug: options.Debug, jsonOutput: options.JSON, results: 0, @@ -63,8 +63,8 @@ func NewDNSExecutor(options *DNSOptions) *DNSExecutor { return executer } -// GotResults returns true if there were any results for the executor -func (e *DNSExecutor) GotResults() bool { +// GotResults returns true if there were any results for the executer +func (e *DNSExecuter) GotResults() bool { if atomic.LoadUint32(&e.results) == 0 { return false } @@ -72,7 +72,7 @@ func (e *DNSExecutor) GotResults() bool { } // ExecuteDNS executes the DNS request on a URL -func (e *DNSExecutor) ExecuteDNS(URL string) (result Result) { +func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) { // Parse the URL and return domain if URL. var domain string if isURL(URL) { @@ -144,8 +144,8 @@ func (e *DNSExecutor) ExecuteDNS(URL string) (result Result) { return } -// Close closes the dns executor for a template. -func (e *DNSExecutor) Close() { +// Close closes the dns executer for a template. +func (e *DNSExecuter) Close() { e.outputMutex.Lock() e.writer.Flush() e.outputMutex.Unlock() diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 7e02b07078..d1f1fe9eac 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -1,4 +1,4 @@ -package executor +package executer import ( "bufio" @@ -24,9 +24,9 @@ import ( "golang.org/x/net/proxy" ) -// HTTPExecutor is client for performing HTTP requests +// HTTPExecuter is client for performing HTTP requests // for a template. -type HTTPExecutor struct { +type HTTPExecuter struct { debug bool results uint32 jsonOutput bool @@ -38,7 +38,7 @@ type HTTPExecutor struct { customHeaders requests.CustomHeaders } -// HTTPOptions contains configuration options for the HTTP executor. +// HTTPOptions contains configuration options for the HTTP executer. type HTTPOptions struct { Template *templates.Template HTTPRequest *requests.HTTPRequest @@ -52,9 +52,9 @@ type HTTPOptions struct { CustomHeaders requests.CustomHeaders } -// NewHTTPExecutor creates a new HTTP executor from a template +// NewHTTPExecuter creates a new HTTP executer from a template // and a HTTP request query. -func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) { +func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { var proxyURL *url.URL var err error @@ -69,7 +69,7 @@ func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) { client := makeHTTPClient(proxyURL, options) client.CheckRetry = retryablehttp.HostSprayRetryPolicy() - executer := &HTTPExecutor{ + executer := &HTTPExecuter{ debug: options.Debug, jsonOutput: options.JSON, results: 0, @@ -83,8 +83,8 @@ func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) { return executer, nil } -// GotResults returns true if there were any results for the executor -func (e *HTTPExecutor) GotResults() bool { +// GotResults returns true if there were any results for the executer +func (e *HTTPExecuter) GotResults() bool { if atomic.LoadUint32(&e.results) == 0 { return false } @@ -92,7 +92,7 @@ func (e *HTTPExecutor) GotResults() bool { } // ExecuteHTTP executes the HTTP request on a URL -func (e *HTTPExecutor) ExecuteHTTP(URL string) (result Result) { +func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) { result.Matches = make(map[string]interface{}) result.Extractions = make(map[string]interface{}) // Compile each request for the template based on the URL @@ -208,8 +208,8 @@ mainLoop: return } -// Close closes the http executor for a template. -func (e *HTTPExecutor) Close() { +// Close closes the http executer for a template. +func (e *HTTPExecuter) Close() { e.outputMutex.Lock() e.writer.Flush() e.outputMutex.Unlock() @@ -277,7 +277,7 @@ func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirect } } -func (e *HTTPExecutor) setCustomHeaders(r *requests.CompiledHTTP) { +func (e *HTTPExecuter) setCustomHeaders(r *requests.CompiledHTTP) { for _, customHeader := range e.customHeaders { // This should be pre-computed somewhere and done only once tokens := strings.Split(customHeader, ":") diff --git a/v2/pkg/executer/http_utils.go b/v2/pkg/executer/http_utils.go index 39c4336405..8863d13205 100644 --- a/v2/pkg/executer/http_utils.go +++ b/v2/pkg/executer/http_utils.go @@ -1,4 +1,4 @@ -package executor +package executer import ( "net/http" diff --git a/v2/pkg/executer/output_dns.go b/v2/pkg/executer/output_dns.go index d32973e381..dc9f69b629 100644 --- a/v2/pkg/executer/output_dns.go +++ b/v2/pkg/executer/output_dns.go @@ -1,4 +1,4 @@ -package executor +package executer import ( "strings" @@ -9,15 +9,15 @@ import ( ) // writeOutputDNS writes dns output to streams -func (e *DNSExecutor) writeOutputDNS(domain string, matcher *matchers.Matcher, extractorResults []string) { +func (e *DNSExecuter) writeOutputDNS(domain string, matcher *matchers.Matcher, extractorResults []string) { if e.jsonOutput { output := jsonOutput{ - Template: e.template.ID, - Type: "dns", - Matched: domain, - Severity: e.template.Info.Severity, - Author: e.template.Info.Author, - Description: e.template.Info.Description, + Template: e.template.ID, + Type: "dns", + Matched: domain, + Severity: e.template.Info.Severity, + Author: e.template.Info.Author, + Description: e.template.Info.Description, } if matcher != nil && len(matcher.Name) > 0 { output.MatcherName = matcher.Name diff --git a/v2/pkg/executer/output_http.go b/v2/pkg/executer/output_http.go index 3bd7ec485f..cf04f5d004 100644 --- a/v2/pkg/executer/output_http.go +++ b/v2/pkg/executer/output_http.go @@ -1,4 +1,4 @@ -package executor +package executer import ( "strings" @@ -10,17 +10,17 @@ import ( ) // writeOutputHTTP writes http output to streams -func (e *HTTPExecutor) writeOutputHTTP(req *requests.CompiledHTTP, matcher *matchers.Matcher, extractorResults []string) { +func (e *HTTPExecuter) writeOutputHTTP(req *requests.CompiledHTTP, matcher *matchers.Matcher, extractorResults []string) { URL := req.Request.URL.String() if e.jsonOutput { output := jsonOutput{ - Template: e.template.ID, - Type: "http", - Matched: URL, - Severity: e.template.Info.Severity, - Author: e.template.Info.Author, - Description: e.template.Info.Description, + Template: e.template.ID, + Type: "http", + Matched: URL, + Severity: e.template.Info.Severity, + Author: e.template.Info.Author, + Description: e.template.Info.Description, } if matcher != nil && len(matcher.Name) > 0 { output.MatcherName = matcher.Name diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 98bb2a66fe..54d353201e 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -5,7 +5,7 @@ import ( tengo "github.com/d5/tengo/v2" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/executor" + "github.com/projectdiscovery/nuclei/v2/pkg/executer" "github.com/projectdiscovery/nuclei/v2/pkg/generators" ) @@ -20,8 +20,8 @@ type NucleiVar struct { // Template contains HTTPOptions and DNSOptions for a single template type Template struct { - HTTPOptions *executor.HTTPOptions - DNSOptions *executor.DNSOptions + HTTPOptions *executer.HTTPOptions + DNSOptions *executer.DNSOptions } // TypeName of the variable @@ -61,18 +61,18 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { for _, request := range template.HTTPOptions.Template.RequestsHTTP { request.Payloads = generators.MergeMaps(request.Payloads, externalVars) template.HTTPOptions.HTTPRequest = request - httpExecutor, err := executor.NewHTTPExecutor(template.HTTPOptions) + httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions) if err != nil { gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err) continue } - result := httpExecutor.ExecuteHTTP(n.URL) + result := httpExecuter.ExecuteHTTP(n.URL) if result.Error != nil { gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) continue } - if httpExecutor.GotResults() { + if httpExecuter.GotResults() { gotResult = true n.addResults(&result) } @@ -82,14 +82,14 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { if template.DNSOptions != nil { for _, request := range template.DNSOptions.Template.RequestsDNS { template.DNSOptions.DNSRequest = request - dnsExecutor := executor.NewDNSExecutor(template.DNSOptions) - result := dnsExecutor.ExecuteDNS(n.URL) + dnsExecuter := executer.NewDNSExecuter(template.DNSOptions) + result := dnsExecuter.ExecuteDNS(n.URL) if result.Error != nil { gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) continue } - if dnsExecutor.GotResults() { + if dnsExecuter.GotResults() { gotResult = true n.addResults(&result) } @@ -110,7 +110,7 @@ func (n *NucleiVar) IsFalsy() bool { return len(n.InternalVars) == 0 } -func (n *NucleiVar) addResults(r *executor.Result) { +func (n *NucleiVar) addResults(r *executer.Result) { n.RLock() defer n.RUnlock() From 15eb60c102e231ca980d2d0cd6eb90190c616b77 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 16 Jul 2020 11:55:57 +0200 Subject: [PATCH 4/6] removed outdated go module --- go.mod | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 go.mod diff --git a/go.mod b/go.mod deleted file mode 100644 index 14e48f74b9..0000000000 --- a/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/projectdiscovery/nuclei - -go 1.14 From 037aa356a65c72ba451d7d99c1d6bb07ef8bc998 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 16 Jul 2020 12:47:38 +0200 Subject: [PATCH 5/6] small correction --- v2/pkg/extractors/extract.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/extractors/extract.go b/v2/pkg/extractors/extract.go index 3030b1237a..2be5aed85c 100644 --- a/v2/pkg/extractors/extract.go +++ b/v2/pkg/extractors/extract.go @@ -30,7 +30,7 @@ func (e *Extractor) Extract(resp *http.Response, body, headers string) map[strin if len(matches) > 0 { return matches } - return e.extractInlineKVal(resp, "cookies") + return e.extractInlineKVal(resp, "set-cookie") } } @@ -75,7 +75,7 @@ func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} { func (e *Extractor) extractInlineKVal(r *http.Response, key string) map[string]struct{} { results := make(map[string]struct{}) for _, v := range r.Header.Values(key) { - for _, token := range strings.Split(v, " ") { + for _, token := range strings.Split(v, ";") { semicolon := strings.Index(token, ":") key, value := token[:semicolon], token[semicolon:] for _, k := range e.KVal { From a23031a2e6cf6144919bdf0a26e6fc27c13dcb66 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 16 Jul 2020 12:58:56 +0200 Subject: [PATCH 6/6] using native go for cookies --- v2/pkg/extractors/extract.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/v2/pkg/extractors/extract.go b/v2/pkg/extractors/extract.go index 2be5aed85c..c18b607416 100644 --- a/v2/pkg/extractors/extract.go +++ b/v2/pkg/extractors/extract.go @@ -2,7 +2,6 @@ package extractors import ( "net/http" - "strings" "github.com/miekg/dns" ) @@ -30,7 +29,7 @@ func (e *Extractor) Extract(resp *http.Response, body, headers string) map[strin if len(matches) > 0 { return matches } - return e.extractInlineKVal(resp, "set-cookie") + return e.extractCookieKVal(resp, "set-cookie") } } @@ -71,17 +70,13 @@ func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} { return results } -// extractInlineKVal extracts text from inline corpus -func (e *Extractor) extractInlineKVal(r *http.Response, key string) map[string]struct{} { +// extractCookieKVal extracts text from cookies +func (e *Extractor) extractCookieKVal(r *http.Response, key string) map[string]struct{} { results := make(map[string]struct{}) - for _, v := range r.Header.Values(key) { - for _, token := range strings.Split(v, ";") { - semicolon := strings.Index(token, ":") - key, value := token[:semicolon], token[semicolon:] - for _, k := range e.KVal { - if key == k { - results[value] = struct{}{} - } + for _, k := range e.KVal { + for _, cookie := range r.Cookies() { + if cookie.Name == k { + results[cookie.Value] = struct{}{} } } }