Merge pull request #158 from projectdiscovery/feature-151-kval-extractor

kval extractor

v2/go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/d5/tengo v1.24.8
 	github.com/d5/tengo/v2 v2.6.0
 	github.com/google/go-github/v32 v32.0.0
 	github.com/json-iterator/go v1.1.10

v2/go.sum

@@ -7,7 +7,6 @@ github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
-github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ=
 github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI=
 github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

v2/internal/runner/runner.go

@@ -15,7 +15,7 @@ import (
 	"github.com/d5/tengo/v2/stdlib"
 	"github.com/karrick/godirwalk"
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v2/pkg/executor"
+	"github.com/projectdiscovery/nuclei/v2/pkg/executer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
@@ -233,22 +233,22 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 		defer writer.Flush()
 	}
 
-	var httpExecutor *executor.HTTPExecutor
-	var dnsExecutor *executor.DNSExecutor
+	var httpExecuter *executer.HTTPExecuter
+	var dnsExecuter *executer.DNSExecuter
 	var err error
 
-	// Create an executor based on the request type.
+	// Create an executer based on the request type.
 	switch value := request.(type) {
 	case *requests.DNSRequest:
-		dnsExecutor = executor.NewDNSExecutor(&executor.DNSOptions{
+		dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{
 			Debug:      r.options.Debug,
 			Template:   template,
 			DNSRequest: value,
 			Writer:     writer,
 			JSON:       r.options.JSON,
 		})
 	case *requests.HTTPRequest:
-		httpExecutor, err = executor.NewHTTPExecutor(&executor.HTTPOptions{
+		httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
 			Debug:         r.options.Debug,
 			Template:      template,
 			HTTPRequest:   value,
@@ -279,13 +279,13 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 		wg.Add(1)
 
 		go func(URL string) {
-			var result executor.Result
+			var result executer.Result
 
-			if httpExecutor != nil {
-				result = httpExecutor.ExecuteHTTP(URL)
+			if httpExecuter != nil {
+				result = httpExecuter.ExecuteHTTP(URL)
 			}
-			if dnsExecutor != nil {
-				result = dnsExecutor.ExecuteDNS(URL)
+			if dnsExecuter != nil {
+				result = dnsExecuter.ExecuteDNS(URL)
 			}
 			if result.Error != nil {
 				gologger.Warningf("Could not execute step: %s\n", result.Error)
@@ -297,14 +297,14 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 	close(limiter)
 	wg.Wait()
 
-	// See if we got any results from the executors
+	// See if we got any results from the executers
 	var results bool
-	if httpExecutor != nil {
-		results = httpExecutor.GotResults()
+	if httpExecuter != nil {
+		results = httpExecuter.GotResults()
 	}
-	if dnsExecutor != nil {
+	if dnsExecuter != nil {
 		if !results {
-			results = dnsExecutor.GotResults()
+			results = dnsExecuter.GotResults()
 		}
 	}
 	return results
@@ -367,7 +367,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
 			}
 			template := &workflows.Template{}
 			if len(t.RequestsHTTP) > 0 {
-				template.HTTPOptions = &executor.HTTPOptions{
+				template.HTTPOptions = &executer.HTTPOptions{
 					Debug:         r.options.Debug,
 					Writer:        writer,
 					Template:      t,
@@ -378,7 +378,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
 					CustomHeaders: r.options.CustomHeaders,
 				}
 			} else if len(t.RequestsDNS) > 0 {
-				template.DNSOptions = &executor.DNSOptions{
+				template.DNSOptions = &executer.DNSOptions{
 					Debug:    r.options.Debug,
 					Template: t,
 					Writer:   writer,
@@ -417,7 +417,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
 				}
 				template := &workflows.Template{}
 				if len(t.RequestsHTTP) > 0 {
-					template.HTTPOptions = &executor.HTTPOptions{
+					template.HTTPOptions = &executer.HTTPOptions{
 						Debug:         r.options.Debug,
 						Writer:        writer,
 						Template:      t,
@@ -428,7 +428,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
 						CustomHeaders: r.options.CustomHeaders,
 					}
 				} else if len(t.RequestsDNS) > 0 {
-					template.DNSOptions = &executor.DNSOptions{
+					template.DNSOptions = &executer.DNSOptions{
 						Debug:    r.options.Debug,
 						Template: t,
 						Writer:   writer,

v2/pkg/executor/dns_utils.go → v2/pkg/executer/dns_utils.go

@@ -1,4 +1,4 @@
-package executor
+package executer
 
 import (
 	"net/url"

v2/pkg/executor/executor_dns.go → v2/pkg/executer/executer_dns.go

@@ -1,4 +1,4 @@
-package executor
+package executer
 
 import (
 	"bufio"
@@ -15,9 +15,9 @@ import (
 	retryabledns "github.com/projectdiscovery/retryabledns"
 )
 
-// DNSExecutor is a client for performing a DNS request
+// DNSExecuter is a client for performing a DNS request
 // for a template.
-type DNSExecutor struct {
+type DNSExecuter struct {
 	debug       bool
 	jsonOutput  bool
 	results     uint32
@@ -36,7 +36,7 @@ var DefaultResolvers = []string{
 	"8.8.4.4:53", // Google
 }
 
-// DNSOptions contains configuration options for the DNS executor.
+// DNSOptions contains configuration options for the DNS executer.
 type DNSOptions struct {
 	Debug      bool
 	JSON       bool
@@ -45,12 +45,12 @@ type DNSOptions struct {
 	Writer     *bufio.Writer
 }
 
-// NewDNSExecutor creates a new DNS executor from a template
+// NewDNSExecuter creates a new DNS executer from a template
 // and a DNS request query.
-func NewDNSExecutor(options *DNSOptions) *DNSExecutor {
+func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
 	dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
 
-	executer := &DNSExecutor{
+	executer := &DNSExecuter{
 		debug:       options.Debug,
 		jsonOutput:  options.JSON,
 		results:     0,
@@ -63,16 +63,16 @@ func NewDNSExecutor(options *DNSOptions) *DNSExecutor {
 	return executer
 }
 
-// GotResults returns true if there were any results for the executor
-func (e *DNSExecutor) GotResults() bool {
+// GotResults returns true if there were any results for the executer
+func (e *DNSExecuter) GotResults() bool {
 	if atomic.LoadUint32(&e.results) == 0 {
 		return false
 	}
 	return true
 }
 
 // ExecuteDNS executes the DNS request on a URL
-func (e *DNSExecutor) ExecuteDNS(URL string) (result Result) {
+func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
 	// Parse the URL and return domain if URL.
 	var domain string
 	if isURL(URL) {
@@ -129,7 +129,7 @@ func (e *DNSExecutor) ExecuteDNS(URL string) (result Result) {
 	// next task which is extraction of input from matchers.
 	var extractorResults []string
 	for _, extractor := range e.dnsRequest.Extractors {
-		for match := range extractor.ExtractDNS(resp.String()) {
+		for match := range extractor.ExtractDNS(resp) {
 			extractorResults = append(extractorResults, match)
 		}
 	}
@@ -144,8 +144,8 @@ func (e *DNSExecutor) ExecuteDNS(URL string) (result Result) {
 	return
 }
 
-// Close closes the dns executor for a template.
-func (e *DNSExecutor) Close() {
+// Close closes the dns executer for a template.
+func (e *DNSExecuter) Close() {
 	e.outputMutex.Lock()
 	e.writer.Flush()
 	e.outputMutex.Unlock()

v2/pkg/executor/executer_http.go → v2/pkg/executer/executer_http.go

@@ -1,4 +1,4 @@
-package executor
+package executer
 
 import (
 	"bufio"
@@ -24,9 +24,9 @@ import (
 	"golang.org/x/net/proxy"
 )
 
-// HTTPExecutor is client for performing HTTP requests
+// HTTPExecuter is client for performing HTTP requests
 // for a template.
-type HTTPExecutor struct {
+type HTTPExecuter struct {
 	debug         bool
 	results       uint32
 	jsonOutput    bool
@@ -38,7 +38,7 @@ type HTTPExecutor struct {
 	customHeaders requests.CustomHeaders
 }
 
-// HTTPOptions contains configuration options for the HTTP executor.
+// HTTPOptions contains configuration options for the HTTP executer.
 type HTTPOptions struct {
 	Template      *templates.Template
 	HTTPRequest   *requests.HTTPRequest
@@ -52,9 +52,9 @@ type HTTPOptions struct {
 	CustomHeaders requests.CustomHeaders
 }
 
-// NewHTTPExecutor creates a new HTTP executor from a template
+// NewHTTPExecuter creates a new HTTP executer from a template
 // and a HTTP request query.
-func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) {
+func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
 	var proxyURL *url.URL
 	var err error
 
@@ -69,7 +69,7 @@ func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) {
 	client := makeHTTPClient(proxyURL, options)
 	client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
 
-	executer := &HTTPExecutor{
+	executer := &HTTPExecuter{
 		debug:         options.Debug,
 		jsonOutput:    options.JSON,
 		results:       0,
@@ -83,16 +83,16 @@ func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) {
 	return executer, nil
 }
 
-// GotResults returns true if there were any results for the executor
-func (e *HTTPExecutor) GotResults() bool {
+// GotResults returns true if there were any results for the executer
+func (e *HTTPExecuter) GotResults() bool {
 	if atomic.LoadUint32(&e.results) == 0 {
 		return false
 	}
 	return true
 }
 
 // ExecuteHTTP executes the HTTP request on a URL
-func (e *HTTPExecutor) ExecuteHTTP(URL string) (result Result) {
+func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) {
 	result.Matches = make(map[string]interface{})
 	result.Extractions = make(map[string]interface{})
 	// Compile each request for the template based on the URL
@@ -187,7 +187,7 @@ mainLoop:
 		// next task which is extraction of input from matchers.
 		var extractorResults []string
 		for _, extractor := range e.httpRequest.Extractors {
-			for match := range extractor.Extract(body, headers) {
+			for match := range extractor.Extract(resp, body, headers) {
 				extractorResults = append(extractorResults, match)
 			}
 			// probably redundant but ensures we snapshot current payload values when extractors are valid
@@ -208,8 +208,8 @@ mainLoop:
 	return
 }
 
-// Close closes the http executor for a template.
-func (e *HTTPExecutor) Close() {
+// Close closes the http executer for a template.
+func (e *HTTPExecuter) Close() {
 	e.outputMutex.Lock()
 	e.writer.Flush()
 	e.outputMutex.Unlock()
@@ -277,7 +277,7 @@ func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirect
 	}
 }
 
-func (e *HTTPExecutor) setCustomHeaders(r *requests.CompiledHTTP) {
+func (e *HTTPExecuter) setCustomHeaders(r *requests.CompiledHTTP) {
 	for _, customHeader := range e.customHeaders {
 		// This should be pre-computed somewhere and done only once
 		tokens := strings.Split(customHeader, ":")