Refactoring variables logic with map + get public ip (#3853)

* refactoring variables logic with map + get public ip

* moving to dsl package

* updating dep

* updating dsl with new ip endpoint

* deps bump

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>

v2/go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/miekg/dns v1.1.54
+	github.com/miekg/dns v1.1.55
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.12
@@ -66,7 +66,7 @@ require (
 	github.com/klauspost/compress v1.16.6
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/mholt/archiver v3.1.1+incompatible
-	github.com/projectdiscovery/dsl v0.0.9
+	github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/goflags v0.1.10
 	github.com/projectdiscovery/gologger v1.1.10
@@ -77,7 +77,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.0
 	github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1
-	github.com/projectdiscovery/utils v0.0.38
+	github.com/projectdiscovery/utils v0.0.39-0.20230621170112-8dd2c290d962
 	github.com/projectdiscovery/wappalyzergo v0.0.94
 	github.com/stretchr/testify v1.8.4
 	gopkg.in/src-d/go-git.v4 v4.13.1
@@ -128,7 +128,7 @@ require (
 	github.com/projectdiscovery/cdncheck v1.0.6 // indirect
 	github.com/projectdiscovery/freeport v0.0.4 // indirect
 	github.com/refraction-networking/utls v1.3.2 // indirect
-	github.com/sashabaranov/go-openai v1.9.1 // indirect
+	github.com/sashabaranov/go-openai v1.11.2 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skeema/knownhosts v1.1.1 // indirect
 	github.com/smartystreets/assertions v1.0.0 // indirect

v2/go.sum

@@ -352,8 +352,8 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm
 github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw=
 github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8=
 github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI=
-github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
+github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo=
+github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
 github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
 github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
@@ -405,8 +405,8 @@ github.com/projectdiscovery/cdncheck v1.0.6 h1:bjo4oxCD1Y5972ow0LWCjUpO8KOO12j6u
 github.com/projectdiscovery/cdncheck v1.0.6/go.mod h1:NN0QRfxBzUVZJoS0lN37spElCOXHzFuvq1yg5RhTxCE=
 github.com/projectdiscovery/clistats v0.0.12 h1:KLYJxpiwEFidduU4PbcwEcCQ2L7c5wrf7DI5IN5fZ+8=
 github.com/projectdiscovery/clistats v0.0.12/go.mod h1:9luKJj+7Hjq3+a7g129sKWRYx4SbTdkUWZQxabn3H5Y=
-github.com/projectdiscovery/dsl v0.0.9 h1:VfznBxpbNKMn2amQd9gtRnMfK1/Sf9MwsJD9x2Et/fY=
-github.com/projectdiscovery/dsl v0.0.9/go.mod h1:kdPdbbqceWxkSedXm99z0Hzh9z/DFj42A9L95GJjybo=
+github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd h1:16DMjd4HeACrC9CkWJkkLeSh+LYPDorwNx11BlTbonU=
+github.com/projectdiscovery/dsl v0.0.11-0.20230621170216-97e70ffb7efd/go.mod h1:S72Cq/lfxzkldf64Sul1G2KFbGKNgpRFFCF/FazpznM=
 github.com/projectdiscovery/fastdialer v0.0.31 h1:eu0wTBCWjT8dXChmBtnQaAxoFpkLdvq0VroRxZoe/M8=
 github.com/projectdiscovery/fastdialer v0.0.31/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -444,8 +444,8 @@ github.com/projectdiscovery/tlsx v1.1.0 h1:6L5VKpHaoqvIHN6lH9zi7jIvph1JwYMYZOIpW
 github.com/projectdiscovery/tlsx v1.1.0/go.mod h1:C9xTbU2t54Anmvuq+4jxevR5rzqpp6XUUtV7G9J5CTE=
 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak=
 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8=
-github.com/projectdiscovery/utils v0.0.38 h1:EIAgaP3imfcQY+laxNOU9LXh7VZNAbmiwXsQN0mAxdQ=
-github.com/projectdiscovery/utils v0.0.38/go.mod h1:5+WAxSV7yGl6SDCtR1qiOyiEMCIo3jIff+A5OiYTCgM=
+github.com/projectdiscovery/utils v0.0.39-0.20230621170112-8dd2c290d962 h1:qQnIsYB72MmuaM9orhKpDzY0ddJKHf9Nuih0FnyV6x8=
+github.com/projectdiscovery/utils v0.0.39-0.20230621170112-8dd2c290d962/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo=
 github.com/projectdiscovery/wappalyzergo v0.0.94 h1:IVRskuU95MajWCKYgvH5L67+MXDOWJDWSeBD61OsS/A=
 github.com/projectdiscovery/wappalyzergo v0.0.94/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -463,8 +463,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
-github.com/sashabaranov/go-openai v1.9.1 h1:3N52HkJKo9Zlo/oe1AVv5ZkCOny0ra58/ACvAxkN3MM=
-github.com/sashabaranov/go-openai v1.9.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/sashabaranov/go-openai v1.11.2 h1:HuMf+18eldSKbqVblyeCQbtcqSpGVfqTshvi8Bn6zes=
+github.com/sashabaranov/go-openai v1.11.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
 github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=

v2/pkg/protocols/common/variables/variables.go

@@ -109,11 +109,12 @@ func evaluateVariableValue(expression string, values, processing map[string]inte
 // checkForLazyEval checks if the variables have any lazy evaluation i.e any dsl function
 // and sets the flag accordingly.
 func (variables *Variable) checkForLazyEval() bool {
-
 	variables.ForEach(func(key string, value interface{}) {
-		if stringsutil.ContainsAny(types.ToString(value), protocolutils.KnownVariables...) {
-			variables.LazyEval = true
-			return
+		for _, v := range protocolutils.KnownVariables {
+			if stringsutil.ContainsAny(types.ToString(value), v) {
+				variables.LazyEval = true
+				return
+			}
 		}
 	})
 	return variables.LazyEval

v2/pkg/protocols/utils/variables.go

@@ -8,12 +8,51 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	maputil "github.com/projectdiscovery/utils/maps"
 	urlutil "github.com/projectdiscovery/utils/url"
 	"github.com/weppos/publicsuffix-go/publicsuffix"
 )
 
 // KnownVariables are the variables that are known to input requests
-var KnownVariables = []string{"BaseURL", "RootURL", "Hostname", "Host", "Port", "Path", "File", "Scheme", "Input", "FQDN", "RDN", "DN", "TLD", "SD"}
+var KnownVariables maputil.Map[KnownVariable, string]
+
+func init() {
+	KnownVariables = maputil.Map[KnownVariable, string]{
+		BaseURL:  "BaseURL",
+		RootURL:  "RootURL",
+		Hostname: "Hostname",
+		Host:     "Host",
+		Port:     "Port",
+		Path:     "Path",
+		File:     "File",
+		Scheme:   "Scheme",
+		Input:    "Input",
+		Fqdn:     "FQDN",
+		Rdn:      "RDN",
+		Dn:       "DN",
+		Tld:      "TLD",
+		Sd:       "SD",
+	}
+}
+
+type KnownVariable uint16
+
+const (
+	BaseURL KnownVariable = iota
+	RootURL
+	Hostname
+	Host
+	Port
+	Path
+	File
+	Scheme
+	Input
+	Fqdn
+	Rdn
+	Dn
+	Tld
+	Sd
+)
 
 // GenerateVariables will create default variables with context args
 func GenerateVariablesWithContextArgs(input *contextargs.Context, trailingSlash bool) map[string]interface{} {
@@ -34,18 +73,18 @@ func GenerateDNSVariables(domain string) map[string]interface{} {
 
 	domainName := strings.Join([]string{parsed.SLD, parsed.TLD}, ".")
 	dnsVariables := make(map[string]interface{})
-	for _, k := range KnownVariables {
+	for k, v := range KnownVariables {
 		switch k {
-		case "FQDN":
-			dnsVariables[k] = domain
-		case "RDN":
-			dnsVariables[k] = domainName
-		case "DN":
-			dnsVariables[k] = parsed.SLD
-		case "TLD":
-			dnsVariables[k] = parsed.TLD
-		case "SD":
-			dnsVariables[k] = parsed.TRD
+		case Fqdn:
+			dnsVariables[v] = domain
+		case Rdn:
+			dnsVariables[v] = domainName
+		case Dn:
+			dnsVariables[v] = parsed.SLD
+		case Tld:
+			dnsVariables[v] = parsed.TLD
+		case Sd:
+			dnsVariables[v] = parsed.TRD
 		}
 	}
 	return dnsVariables
@@ -55,13 +94,12 @@ func GenerateDNSVariables(domain string) map[string]interface{} {
 // Returns the map of KnownVariables keys
 // This function is used by http, headless, websocket, network and whois protocols to generate protocol variables
 func GenerateVariables(input interface{}, removeTrailingSlash bool, additionalVars map[string]interface{}) map[string]interface{} {
-
 	var vars = make(map[string]interface{})
 	switch input := input.(type) {
 	case string:
 		parsed, err := urlutil.Parse(input)
 		if err != nil {
-			return map[string]interface{}{"Input": input, "Hostname": input}
+			return map[string]interface{}{KnownVariables[Input]: input, KnownVariables[Hostname]: input}
 		}
 		vars = generateVariables(parsed, removeTrailingSlash)
 	case *urlutil.URL:
@@ -106,26 +144,26 @@ func generateVariables(inputURL *urlutil.URL, removeTrailingSlash bool) map[stri
 		}
 	}
 	knownVariables := make(map[string]interface{})
-	for _, k := range KnownVariables {
+	for k, v := range KnownVariables {
 		switch k {
-		case "BaseURL":
-			knownVariables[k] = parsed.String()
-		case "RootURL":
-			knownVariables[k] = fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
-		case "Hostname":
-			knownVariables[k] = parsed.Host
-		case "Host":
-			knownVariables[k] = parsed.Hostname()
-		case "Port":
-			knownVariables[k] = port
-		case "Path":
-			knownVariables[k] = requestPath
-		case "File":
-			knownVariables[k] = base
-		case "Scheme":
-			knownVariables[k] = parsed.Scheme
-		case "Input":
-			knownVariables[k] = parsed.String()
+		case BaseURL:
+			knownVariables[v] = parsed.String()
+		case RootURL:
+			knownVariables[v] = fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
+		case Hostname:
+			knownVariables[v] = parsed.Host
+		case Host:
+			knownVariables[v] = parsed.Hostname()
+		case Port:
+			knownVariables[v] = port
+		case Path:
+			knownVariables[v] = requestPath
+		case File:
+			knownVariables[v] = base
+		case Scheme:
+			knownVariables[v] = parsed.Scheme
+		case Input:
+			knownVariables[v] = parsed.String()
 		}
 	}
 	return generators.MergeMaps(knownVariables, GenerateDNSVariables(parsed.Hostname()))