use proxyutils + fix proxy panic (#3526)

v2/go.mod

@@ -80,7 +80,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.0.6
 	github.com/projectdiscovery/uncover v1.0.2
-	github.com/projectdiscovery/utils v0.0.18
+	github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35
 	github.com/projectdiscovery/wappalyzergo v0.0.88
 	github.com/stretchr/testify v1.8.2
 	gopkg.in/src-d/go-git.v4 v4.13.1
@@ -211,7 +211,7 @@ require (
 	golang.org/x/crypto v0.7.0
 	golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0
 	golang.org/x/mod v0.9.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

v2/go.sum

@@ -461,8 +461,10 @@ github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMK
 github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY=
 github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo=
 github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY=
-github.com/projectdiscovery/utils v0.0.18 h1:gyBMnA4y2ryui0G98iFqKAXuNdoSy6Z6K0/1KHB0czU=
-github.com/projectdiscovery/utils v0.0.18/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs=
+github.com/projectdiscovery/utils v0.0.20-0.20230410124851-595261704707 h1:usIIWZ/didvXCAhJ39ruuUKH1po4gTV7O097B+YGXDM=
+github.com/projectdiscovery/utils v0.0.20-0.20230410124851-595261704707/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o=
+github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 h1:UBOE9Eob1wj7YZ1MGBbtHvc3ptqBHvXxNKRVxTg21Rc=
+github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o=
 github.com/projectdiscovery/wappalyzergo v0.0.88 h1:N/1vFlKmc3GJco9rANJdHrxg8jdav/xmnICo8rztmH8=
 github.com/projectdiscovery/wappalyzergo v0.0.88/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -738,8 +740,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

v2/internal/runner/proxy.go

@@ -2,30 +2,26 @@ package runner
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
-	"net"
 	"net/url"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
+	proxyutils "github.com/projectdiscovery/utils/proxy"
 )
 
-var proxyURLList []url.URL
-
 // loadProxyServers load list of proxy servers from file or comma seperated
 func loadProxyServers(options *types.Options) error {
 	if len(options.Proxy) == 0 {
 		return nil
 	}
+	proxyList := []string{}
 	for _, p := range options.Proxy {
-		if proxyURL, err := validateProxyURL(p); err == nil {
-			proxyURLList = append(proxyURLList, proxyURL)
-		} else if fileutil.FileExists(p) {
+		if fileutil.FileExists(p) {
 			file, err := os.Open(p)
 			if err != nil {
 				return fmt.Errorf("could not open proxy file: %w", err)
@@ -37,91 +33,31 @@ func loadProxyServers(options *types.Options) error {
 				if strings.TrimSpace(proxy) == "" {
 					continue
 				}
-				if proxyURL, err := validateProxyURL(proxy); err != nil {
-					return err
-				} else {
-					proxyURLList = append(proxyURLList, proxyURL)
-				}
+				proxyList = append(proxyList, proxy)
 			}
 		} else {
-			return fmt.Errorf("invalid proxy file or URL provided for %s", p)
-		}
-	}
-	return processProxyList(options)
-}
-
-func processProxyList(options *types.Options) error {
-	if len(proxyURLList) == 0 {
-		return fmt.Errorf("could not find any valid proxy")
-	} else {
-		done := make(chan bool)
-		exitCounter := make(chan bool)
-		counter := 0
-		for _, url := range proxyURLList {
-			go runProxyConnectivity(url, options, done, exitCounter)
-		}
-		for {
-			select {
-			case <-done:
-				{
-					close(done)
-					return nil
-				}
-			case <-exitCounter:
-				{
-					if counter += 1; counter == len(proxyURLList) {
-						return errors.New("no reachable proxy found")
-					}
-				}
-			}
-		}
-	}
-}
-
-func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) {
-	if err := testProxyConnection(proxyURL, options.Timeout); err == nil {
-		if types.ProxyURL == "" && types.ProxySocksURL == "" {
-			assignProxyURL(proxyURL, options)
-			done <- true
+			proxyList = append(proxyList, p)
 		}
-	} else {
-		gologger.Debug().Msgf("Proxy validation failed for '%s': %s", proxyURL.String(), err)
 	}
-	exitCounter <- true
-}
-
-func testProxyConnection(proxyURL url.URL, timeoutDelay int) error {
-	timeout := time.Duration(timeoutDelay) * time.Second
-	_, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout)
+	aliveProxy, err := proxyutils.GetAnyAliveProxy(options.Timeout, proxyList...)
 	if err != nil {
 		return err
 	}
-	return nil
-}
-
-func assignProxyURL(proxyURL url.URL, options *types.Options) {
+	proxyURL, err := url.Parse(aliveProxy)
+	if err != nil {
+		return errorutil.WrapfWithNil(err, "failed to parse proxy got %v", err)
+	}
 	if options.ProxyInternal {
 		os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String())
 	}
-	if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS {
+	if proxyURL.Scheme == proxyutils.HTTP || proxyURL.Scheme == proxyutils.HTTPS {
 		types.ProxyURL = proxyURL.String()
 		types.ProxySocksURL = ""
 		gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String())
-	} else if proxyURL.Scheme == types.SOCKS5 {
+	} else if proxyURL.Scheme == proxyutils.SOCKS5 {
 		types.ProxyURL = ""
 		types.ProxySocksURL = proxyURL.String()
 		gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String())
 	}
-}
-
-func validateProxyURL(proxy string) (url.URL, error) {
-	if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) {
-		return *url, nil
-	}
-	return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)")
-}
-
-// isSupportedProtocol checks given protocols are supported
-func isSupportedProtocol(value string) bool {
-	return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5
+	return nil
 }

v2/pkg/types/proxy.go

@@ -2,9 +2,6 @@ package types
 
 const (
 	HTTP_PROXY_ENV = "HTTP_PROXY"
-	SOCKS5         = "socks5"
-	HTTP           = "http"
-	HTTPS          = "https"
 )
 
 var (