adds proxy utils

errors/enriched.go

@@ -50,11 +50,11 @@ func (e *enrichedError) Error() string {
 	}()
 	var buff bytes.Buffer
 	label := fmt.Sprintf("[%v:%v]", strings.Join(e.Tags, ","), e.Level.String())
-	buff.WriteString(fmt.Sprintf("%v %v\n", label, e.errString))
+	buff.WriteString(fmt.Sprintf("%v %v", label, e.errString))
 
 	if ShowStackTrace {
 		e.captureStack()
-		buff.WriteString(fmt.Sprintf("Stacktrace:\n%v\n", e.StackTrace))
+		buff.WriteString(fmt.Sprintf("Stacktrace:\n%v", e.StackTrace))
 	}
 	return buff.String()
 }

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/miekg/dns v1.1.53
 	github.com/minio/selfupdate v0.6.0
 	github.com/pkg/errors v0.9.1
+	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
 	github.com/stretchr/testify v1.8.1
 	github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be

go.sum

@@ -109,6 +109,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE=
 github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw=
+github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
+github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=

proxy/README.md

@@ -0,0 +1,2 @@
+## proxy utils
+

proxy/proxy.go

@@ -0,0 +1,105 @@
+package proxyutils
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+	"time"
+
+	errorutil "github.com/projectdiscovery/utils/errors"
+	"github.com/remeh/sizedwaitgroup"
+)
+
+type proxyResult struct {
+	AliveProxy string
+	Error      error
+}
+
+// ProxyProbeConcurrency (default 8)
+var ProxyProbeConcurrency = 8
+
+const (
+	SOCKS5 = "socks5"
+	HTTP   = "http"
+	HTTPS  = "https"
+)
+
+// GetAnyAliveProxy takes proxies as input and returns the first alive proxy
+// or returns error if all of them not alive
+func GetAnyAliveProxy(timeoutInSec int, proxies ...string) (string, error) {
+	sg := sizedwaitgroup.New(ProxyProbeConcurrency)
+	resChan := make(chan proxyResult, 4)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	go func() {
+		for _, v := range proxies {
+			// skip iterating if alive proxy is found
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				proxy, err := GetProxyURL(v)
+				if err != nil {
+					resChan <- proxyResult{Error: err}
+					continue
+				}
+				sg.Add()
+				go func(proxyAddr url.URL) {
+					defer sg.Done()
+					select {
+					case <-ctx.Done():
+						return
+					case resChan <- testProxyConn(proxyAddr, timeoutInSec):
+						cancel()
+					}
+				}(proxy)
+			}
+		}
+		sg.Wait()
+		close(resChan)
+	}()
+
+	errstack := []string{}
+	for {
+		result, ok := <-resChan
+		if !ok {
+			break
+		}
+		if result.AliveProxy != "" {
+			// found alive proxy return now
+			return result.AliveProxy, nil
+		} else if result.Error != nil {
+			errstack = append(errstack, result.Error.Error())
+		}
+	}
+
+	// all proxies are dead
+	return "", errorutil.NewWithTag("proxyutils", "all proxies are dead got : %v", strings.Join(errstack, " : "))
+}
+
+// dial and test if proxy is open
+func testProxyConn(proxyAddr url.URL, timeoutInSec int) proxyResult {
+	p := proxyResult{}
+	if Conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyAddr.Hostname(), proxyAddr.Port()), time.Duration(timeoutInSec)*time.Second); err == nil {
+		_ = Conn.Close()
+		p.AliveProxy = proxyAddr.String()
+	} else {
+		p.Error = err
+	}
+	return p
+}
+
+// GetProxyURL returns a Proxy URL after validating if given proxy url is valid
+func GetProxyURL(proxyAddr string) (url.URL, error) {
+	if url, err := url.Parse(proxyAddr); err == nil && isSupportedProtocol(url.Scheme) {
+		return *url, nil
+	}
+	return url.URL{}, errorutil.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)").WithTag("proxyutils")
+}
+
+// isSupportedProtocol checks given protocols are supported
+func isSupportedProtocol(value string) bool {
+	return value == HTTP || value == HTTPS || value == SOCKS5
+}