Adding support for tls impersonate (#126)

* Adding support for tls impersonate

* fixing gh action

* removing types - helpers should be used

* running examples

* fixing command

.github/workflows/build-test.yml

@@ -22,5 +22,7 @@ jobs:
       - name: Test
         run: go test ./...
 
-      - name: Build
-        run: go build example/main.go
+      - name: Testing Examples
+        run: |
+          go run -race example/simple/main.go
+          go run -race example/impersonate/main.go

example/impersonate/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
+)
+
+func main() {
+	options := fastdialer.DefaultOptions
+
+	// Create new dialer using NewDialer(opts fastdialer.options)
+	fd, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Configure Cache if required
+	// memory based (also support Hybrid and Disk Cache)
+	options.CacheType = fastdialer.Memory
+	options.CacheMemoryMaxItems = 100
+
+	ctx := context.Background()
+
+	target := "www.projectdiscovery.io"
+
+	conn, err := fd.DialTLSWithConfigImpersonate(ctx, "tcp", target+":443", &tls.Config{InsecureSkipVerify: true}, impersonate.Random, nil)
+	if err != nil || conn == nil {
+		log.Fatalf("couldn't connect to target: %s", err)
+	}
+	defer conn.Close()
+	log.Println("connected to the target")
+
+	// To look up Host/ Get DNS details use
+	data, err := fd.GetDNSData(target)
+	if err != nil || data == nil {
+		log.Fatalf("couldn't retrieve dns data: %s", err)
+	}
+
+	// To Print All Type of DNS Data use
+	jsonData, err := data.JSON()
+	if err != nil {
+		log.Fatalf("failed to marshal json: %s", err)
+	}
+	log.Println(jsonData)
+}

example/main.go → example/simple/main.go

@@ -9,14 +9,12 @@ import (
 )
 
 func main() {
-
-	// refer fastdialer/options.go for options and customization
 	options := fastdialer.DefaultOptions
 
 	// Create new dialer using NewDialer(opts fastdialer.options)
 	fd, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
 	if err != nil {
-		panic(err)
+		log.Fatal(err)
 	}
 
 	// Configure Cache if required
@@ -26,19 +24,17 @@ func main() {
 
 	ctx := context.Background()
 
-	// To dial and create connection use
-	// To create connection over TLS or older versions use
-	// fd.DialTLS() or fd.DialZTLS()
-	conn, err := fd.Dial(ctx, "tcp", "www.projectdiscovery.io:80")
+	target := "www.projectdiscovery.io"
+
+	conn, err := fd.DialTLS(ctx, "tcp", target+":443")
 	if err != nil || conn == nil {
 		log.Fatalf("couldn't connect to target: %s", err)
-	} else {
-		fmt.Println("Connected: TCP stream created with www.projectdiscovery.io:80")
 	}
-	conn.Close()
+	defer conn.Close()
+	log.Println("connected to the target")
 
 	// To look up Host/ Get DNS details use
-	data, err := fd.GetDNSData("www.projectdiscovery.io")
+	data, err := fd.GetDNSData(target)
 	if err != nil || data == nil {
 		log.Fatalf("couldn't retrieve dns data: %s", err)
 	}
@@ -48,6 +44,5 @@ func main() {
 	if err != nil {
 		log.Fatalf("failed to marshal json: %s", err)
 	}
-	fmt.Println(jsonData)
-
+	fmt.Print(jsonData)
 }

fastdialer/dialer.go

@@ -10,11 +10,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
 	"github.com/projectdiscovery/hmap/store/hybrid"
 	"github.com/projectdiscovery/networkpolicy"
 	retryabledns "github.com/projectdiscovery/retryabledns"
 	cryptoutil "github.com/projectdiscovery/utils/crypto"
 	iputil "github.com/projectdiscovery/utils/ip"
+	ptrutil "github.com/projectdiscovery/utils/ptr"
+	utls "github.com/refraction-networking/utls"
 	"github.com/zmap/zcrypto/encoding/asn1"
 	ztls "github.com/zmap/zcrypto/tls"
 	"golang.org/x/net/proxy"
@@ -109,7 +112,7 @@ func NewDialer(options Options) (*Dialer, error) {
 
 // Dial function compatible with net/http
 func (d *Dialer) Dial(ctx context.Context, network, address string) (conn net.Conn, err error) {
-	conn, err = d.dial(ctx, network, address, false, false, nil, nil)
+	conn, err = d.dial(ctx, network, address, false, false, nil, nil, impersonate.None, nil)
 	return
 }
 
@@ -130,7 +133,12 @@ func (d *Dialer) DialZTLS(ctx context.Context, network, address string) (conn ne
 
 // DialTLS with encrypted connection
 func (d *Dialer) DialTLSWithConfig(ctx context.Context, network, address string, config *tls.Config) (conn net.Conn, err error) {
-	conn, err = d.dial(ctx, network, address, true, false, config, nil)
+	conn, err = d.dial(ctx, network, address, true, false, config, nil, impersonate.None, nil)
+	return
+}
+
+func (d *Dialer) DialTLSWithConfigImpersonate(ctx context.Context, network, address string, config *tls.Config, impersonate impersonate.Strategy, identity *impersonate.Identity) (conn net.Conn, err error) {
+	conn, err = d.dial(ctx, network, address, true, false, config, nil, impersonate, identity)
 	return
 }
 
@@ -141,12 +149,12 @@ func (d *Dialer) DialZTLSWithConfig(ctx context.Context, network, address string
 		if err != nil {
 			return nil, err
 		}
-		return d.dial(ctx, network, address, true, false, stdTLSConfig, nil)
+		return d.dial(ctx, network, address, true, false, stdTLSConfig, nil, impersonate.None, nil)
 	}
-	return d.dial(ctx, network, address, false, true, nil, config)
+	return d.dial(ctx, network, address, false, true, nil, config, impersonate.None, nil)
 }
 
-func (d *Dialer) dial(ctx context.Context, network, address string, shouldUseTLS, shouldUseZTLS bool, tlsconfig *tls.Config, ztlsconfig *ztls.Config) (conn net.Conn, err error) {
+func (d *Dialer) dial(ctx context.Context, network, address string, shouldUseTLS, shouldUseZTLS bool, tlsconfig *tls.Config, ztlsconfig *ztls.Config, impersonateStrategy impersonate.Strategy, impersonateIdentity *impersonate.Identity) (conn net.Conn, err error) {
 	var hostname, port, fixedIP string
 
 	if strings.HasPrefix(address, "[") {
@@ -225,7 +233,36 @@ func (d *Dialer) dial(ctx context.Context, network, address string, shouldUseTLS
 			case !iputil.IsIP(hostname):
 				tlsconfigCopy.ServerName = hostname
 			}
-			conn, err = tls.DialWithDialer(d.dialer, network, hostPort, tlsconfigCopy)
+			if impersonateStrategy == impersonate.None {
+				conn, err = tls.DialWithDialer(d.dialer, network, hostPort, tlsconfigCopy)
+			} else {
+				conn, err := d.dialer.DialContext(ctx, network, hostPort)
+				if err != nil {
+					return conn, err
+				}
+				// clone existing tls config
+				uTLSConfig := &utls.Config{
+					InsecureSkipVerify: tlsconfigCopy.InsecureSkipVerify,
+					ServerName:         tlsconfigCopy.ServerName,
+					MinVersion:         tlsconfigCopy.MinVersion,
+					MaxVersion:         tlsconfigCopy.MaxVersion,
+					CipherSuites:       tlsconfigCopy.CipherSuites,
+				}
+				var uTLSConn *utls.UConn
+				if impersonateStrategy == impersonate.Random {
+					uTLSConn = utls.UClient(conn, uTLSConfig, utls.HelloRandomized)
+				} else if impersonateStrategy == impersonate.Custom {
+					uTLSConn = utls.UClient(conn, uTLSConfig, utls.HelloCustom)
+					clientHelloSpec := utls.ClientHelloSpec(ptrutil.Safe(impersonateIdentity))
+					if err := uTLSConn.ApplyPreset(&clientHelloSpec); err != nil {
+						return nil, err
+					}
+				}
+				if err := uTLSConn.Handshake(); err != nil {
+					return nil, err
+				}
+				return uTLSConn, nil
+			}
 		} else if shouldUseZTLS {
 			ztlsconfigCopy := ztlsconfig.Clone()
 			switch {

fastdialer/ja3/error.go

@@ -0,0 +1,11 @@
+package ja3
+
+import "fmt"
+
+// ErrExtensionNotExist is returned when an extension is not supported by the library
+type ErrExtensionNotExist string
+
+// Error is the error value which contains the extension that does not exist
+func (e ErrExtensionNotExist) Error() string {
+	return fmt.Sprintf("Extension does not exist: %s\n", string(e))
+}

fastdialer/ja3/extmap.go

@@ -0,0 +1,62 @@
+// ja3 is a package for creating JA3 fingerprints from TLS clients.
+// The original extension map and numeric id=>tls extension mapping is from https://github.com/CUCyber
+package ja3
+
+import (
+	utls "github.com/refraction-networking/utls"
+	"golang.org/x/exp/maps"
+)
+
+// extMap maps extension values to the TLSExtension object associated with the
+// number. Some values are not put in here because they must be applied in a
+// special way. For example, "10" is the SupportedCurves extension which is also
+// used to calculate the JA3 signature. These JA3-dependent values are applied
+// after the instantiation of the map.
+var defaultExtensionMap = map[string]utls.TLSExtension{
+	"0": &utls.SNIExtension{},
+	"5": &utls.StatusRequestExtension{},
+	// These are applied later
+	// "10": &tls.SupportedCurvesExtension{...}
+	// "11": &tls.SupportedPointsExtension{...}
+	"13": &utls.SignatureAlgorithmsExtension{
+		SupportedSignatureAlgorithms: []utls.SignatureScheme{
+			utls.ECDSAWithP256AndSHA256,
+			utls.PSSWithSHA256,
+			utls.PKCS1WithSHA256,
+			utls.ECDSAWithP384AndSHA384,
+			utls.PSSWithSHA384,
+			utls.PKCS1WithSHA384,
+			utls.PSSWithSHA512,
+			utls.PKCS1WithSHA512,
+			utls.PKCS1WithSHA1,
+		},
+	},
+	"16": &utls.ALPNExtension{
+		AlpnProtocols: []string{"h2", "http/1.1"},
+	},
+	"18": &utls.SCTExtension{},
+	"21": &utls.UtlsPaddingExtension{GetPaddingLen: utls.BoringPaddingStyle},
+	"23": &utls.UtlsExtendedMasterSecretExtension{},
+	"28": &utls.FakeRecordSizeLimitExtension{},
+	"35": &utls.SessionTicketExtension{},
+	"43": &utls.SupportedVersionsExtension{Versions: []uint16{
+		utls.GREASE_PLACEHOLDER,
+		utls.VersionTLS13,
+		utls.VersionTLS12,
+		utls.VersionTLS11,
+		utls.VersionTLS10}},
+	"44": &utls.CookieExtension{},
+	"45": &utls.PSKKeyExchangeModesExtension{
+		Modes: []uint8{
+			utls.PskModeDHE,
+		}},
+	"51":    &utls.KeyShareExtension{KeyShares: []utls.KeyShare{}},
+	"13172": &utls.NPNExtension{},
+	"65281": &utls.RenegotiationInfoExtension{
+		Renegotiation: utls.RenegotiateOnceAsClient,
+	},
+}
+
+func getExtensionMap() map[string]utls.TLSExtension {
+	return maps.Clone(defaultExtensionMap)
+}

fastdialer/ja3/impersonate/doc.go

@@ -0,0 +1,3 @@
+// impersonate package contains strategy to impersonate a client and define an alias for the internal
+// client tls spefications
+package impersonate

fastdialer/ja3/impersonate/types.go

@@ -0,0 +1,20 @@
+package impersonate
+
+import (
+	utls "github.com/refraction-networking/utls"
+)
+
+// Strategy is the type of strategy to use for impersonation
+type Strategy uint8
+
+const (
+	// None is the default strategy which use the default client hello spec
+	None Strategy = iota
+	// Random is the strategy which use a random client hello spec
+	Random
+	// JA3 or Raw is the strategy which parses a client hello spec from ja3 full string
+	Custom
+)
+
+// Identity contains the structured client hello spec
+type Identity utls.ClientHelloSpec