Issue 3033 deny list

v2/pkg/input/input.go

@@ -3,12 +3,13 @@ package input
 import (
 	"net"
 	"net/url"
-	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/projectdiscovery/hmap/store/hybrid"
 	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+	fileutil "github.com/projectdiscovery/utils/file"
+	"github.com/projectdiscovery/utils/ports"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
@@ -77,8 +78,8 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 	}
 
 	hasHost := host != ""
-	hasPort := port != ""
-	hasDefaultPort := defaultPort != ""
+	hasPort := ports.IsValid(port)
+	hasDefaultPort := ports.IsValid(defaultPort)
 
 	switch inputType {
 	case typeFilepath:
@@ -89,7 +90,7 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 		if filepath.IsAbs(input) {
 			return input
 		}
-		if absPath, _ := filepath.Abs(input); absPath != "" && fileOrFolderExists(absPath) {
+		if absPath, _ := filepath.Abs(input); absPath != "" && fileutil.FileOrFolderExists(absPath) {
 			return input
 		}
 		if _, err := filepath.Match(input, ""); err != filepath.ErrBadPattern && !isURL {
@@ -132,8 +133,3 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 	}
 	return ""
 }
-
-func fileOrFolderExists(filename string) bool {
-	_, err := os.Stat(filename)
-	return !os.IsNotExist(err)
-}

v2/pkg/operators/operators.go

@@ -329,3 +329,13 @@ func (operators *Operators) ExecuteInternalExtractors(data map[string]interface{
 	}
 	return dynamicValues
 }
+
+// IsEmpty determines if the operator has matchers or extractors
+func (operators *Operators) IsEmpty() bool {
+	return operators.Len() == 0
+}
+
+// Len calculates the sum of the number of matchers and extractors
+func (operators *Operators) Len() int {
+	return len(operators.Matchers) + len(operators.Extractors)
+}

v2/pkg/protocols/common/executer/executer.go

@@ -72,7 +72,7 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
 	for _, req := range e.requests {
 		inputItem := input.Clone()
 		if e.options.InputHelper != nil && input.MetaInput.Input != "" {
-			if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" {
+			if inputItem.MetaInput.Input = e.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" {
 				return false, nil
 			}
 		}

v2/pkg/protocols/common/generators/maps.go

@@ -2,9 +2,6 @@ package generators
 
 import (
 	"reflect"
-	"strings"
-
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/marker"
 )
 
 // MergeMapsMany merges many maps into a new map
@@ -65,26 +62,3 @@ func ExpandMapValues(m map[string]string) map[string][]string {
 	}
 	return m1
 }
-
-// CopyMap creates a new copy of an existing map
-func CopyMap(originalMap map[string]interface{}) map[string]interface{} {
-	newMap := make(map[string]interface{})
-	for key, value := range originalMap {
-		newMap[key] = value
-	}
-	return newMap
-}
-
-// CopyMapWithDefaultValue creates a new copy of an existing map and set a default value
-func CopyMapWithDefaultValue(originalMap map[string][]string, defaultValue interface{}) map[string]interface{} {
-	newMap := make(map[string]interface{})
-	for key := range originalMap {
-		newMap[key] = defaultValue
-	}
-	return newMap
-}
-
-// TrimDelimiters removes trailing brackets
-func TrimDelimiters(s string) string {
-	return strings.TrimSuffix(strings.TrimPrefix(s, marker.ParenthesisOpen), marker.ParenthesisClose)
-}

v2/pkg/protocols/common/generators/validate.go

@@ -3,10 +3,10 @@ package generators
 import (
 	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/projectdiscovery/fileutil"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	folderutil "github.com/projectdiscovery/utils/folder"
 )
@@ -22,7 +22,7 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat
 			}
 
 			// check if it's a file and try to load it
-			if fileExists(payloadType) {
+			if fileutil.FileExists(payloadType) {
 				continue
 			}
 
@@ -33,7 +33,7 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat
 			payloadPathsToProbe, _ := templatePathInfo.MeshWith(payloadType)
 
 			for _, payloadPath := range payloadPathsToProbe {
-				if fileExists(payloadPath) {
+				if fileutil.FileExists(payloadPath) {
 					payloads[name] = payloadPath
 					changed = true
 					break
@@ -53,15 +53,3 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat
 	}
 	return nil
 }
-
-// fileExists checks if a file exists and is not a directory
-func fileExists(filename string) bool {
-	info, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		return false
-	}
-	if info == nil {
-		return false
-	}
-	return !info.IsDir()
-}

v2/pkg/protocols/file/file.go

@@ -99,15 +99,17 @@ func (request *Request) GetID() string {
 
 // Compile compiles the protocol request for further execution.
 func (request *Request) Compile(options *protocols.ExecuterOptions) error {
-	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
-		compiled := &request.Operators
-		compiled.ExcludeMatchers = options.ExcludeMatchers
-		compiled.TemplateID = options.TemplateID
-		if err := compiled.Compile(); err != nil {
-			return errors.Wrap(err, "could not compile operators")
-		}
-		request.CompiledOperators = compiled
+	// if there are no matchers/extractors, we trigger an error as no operation would be performed on the template
+	if request.Operators.IsEmpty() {
+		return errors.New("empty operators")
+	}
+	compiled := &request.Operators
+	compiled.ExcludeMatchers = options.ExcludeMatchers
+	compiled.TemplateID = options.TemplateID
+	if err := compiled.Compile(); err != nil {
+		return errors.Wrap(err, "could not compile operators")
 	}
+	request.CompiledOperators = compiled
 
 	// By default, use default max size if not defined
 	switch {

v2/pkg/protocols/file/find_test.go

@@ -23,6 +23,7 @@ func TestFindInputPaths(t *testing.T) {
 		NoRecursive: false,
 		Extensions:  []string{"all", ".lock"},
 		DenyList:    []string{".go"},
+		Operators:   newMockOperator(),
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,

v2/pkg/protocols/file/operators_test.go

@@ -14,6 +14,19 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
 )
 
+func newMockOperator() operators.Operators {
+	operators := operators.Operators{
+		Matchers: []*matchers.Matcher{
+			{
+				Type: matchers.MatcherTypeHolder{
+					MatcherType: matchers.WordsMatcher,
+				},
+			},
+		},
+	}
+	return operators
+}
+
 func TestResponseToDSLMap(t *testing.T) {
 	options := testutils.DefaultOptions
 
@@ -25,6 +38,7 @@ func TestResponseToDSLMap(t *testing.T) {
 		NoRecursive: false,
 		Extensions:  []string{"*", ".lock"},
 		DenyList:    []string{".go"},
+		Operators:   newMockOperator(),
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -50,6 +64,7 @@ func TestFileOperatorMatch(t *testing.T) {
 		NoRecursive: false,
 		Extensions:  []string{"*", ".lock"},
 		DenyList:    []string{".go"},
+		Operators:   newMockOperator(),
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -138,6 +153,7 @@ func TestFileOperatorExtract(t *testing.T) {
 		NoRecursive: false,
 		Extensions:  []string{"*", ".lock"},
 		DenyList:    []string{".go"},
+		Operators:   newMockOperator(),
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,

v2/pkg/protocols/file/request.go

@@ -42,7 +42,7 @@ type FileMatch struct {
 	Raw       string
 }
 
-var emptyResultErr = errors.New("Empty result")
+var errEmptyResult = errors.New("Empty result")
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
@@ -65,7 +65,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 						archiveFileName := filepath.Join(filePath, file.Name())
 						event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.MetaInput.Input, file.Size(), previous)
 						if err != nil {
-							if errors.Is(err, emptyResultErr) {
+							if errors.Is(err, errEmptyResult) {
 								// no matches but one file elaborated
 								request.options.Progress.IncrementRequests()
 								return nil
@@ -118,7 +118,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 					_, _ = tmpFileOut.Seek(0, 0)
 					event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.MetaInput.Input, fileStat.Size(), previous)
 					if err != nil {
-						if errors.Is(err, emptyResultErr) {
+						if errors.Is(err, errEmptyResult) {
 							// no matches but one file elaborated
 							request.options.Progress.IncrementRequests()
 							return
@@ -138,7 +138,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 				request.options.Progress.AddToTotal(1)
 				event, fileMatches, err := request.processFile(filePath, input.MetaInput.Input, previous)
 				if err != nil {
-					if errors.Is(err, emptyResultErr) {
+					if errors.Is(err, errEmptyResult) {
 						// no matches but one file elaborated
 						request.options.Progress.IncrementRequests()
 						return
@@ -188,7 +188,7 @@ func (request *Request) processReader(reader io.Reader, filePath, input string,
 	fileReader := io.LimitReader(reader, request.maxSize)
 	fileMatches, opResult := request.findMatchesWithReader(fileReader, input, filePath, totalBytes, previousInternalEvent)
 	if opResult == nil && len(fileMatches) == 0 {
-		return nil, nil, emptyResultErr
+		return nil, nil, errEmptyResult
 	}
 
 	// build event structure to interface with internal logic

v2/pkg/protocols/headless/request.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"golang.org/x/exp/maps"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
@@ -67,7 +68,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 			}
 		}
 	} else {
-		value := generators.CopyMap(payloads)
+		value := maps.Clone(payloads)
 		if err := request.executeRequestWithPayloads(inputURL, value, previous, wrappedCallback); err != nil {
 			return err
 		}

v2/pkg/protocols/network/request.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"golang.org/x/exp/maps"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
@@ -91,7 +92,7 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 			}
 		}
 	} else {
-		value := generators.CopyMap(payloads)
+		value := maps.Clone(payloads)
 		if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
 			return err
 		}