cli: utls for TLS parroting

internal/engine/experiment/urlgetter/configurer.go

@@ -12,6 +12,7 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/engine/netx"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	utls "gitlab.com/yawning/utls.git"
 )
 
 // The Configurer job is to construct a Configuration that can
@@ -36,12 +37,28 @@ func (c Configuration) CloseIdleConnections() {
 
 // NewConfiguration builds a new measurement configuration.
 func (c Configurer) NewConfiguration() (Configuration, error) {
+	// select ClientHelloID
+	// TODO(kelmenhorst) this is used for testing different fingerprints
+	var clientHelloID *utls.ClientHelloID
+	switch strings.ToLower(c.Config.ClientHelloID) {
+	case "chrome":
+		clientHelloID = &utls.HelloChrome_Auto
+	case "firefox":
+		clientHelloID = &utls.HelloFirefox_Auto
+	case "ios":
+		clientHelloID = &utls.HelloIOS_Auto
+	case "golang":
+		clientHelloID = &utls.HelloGolang
+	default:
+		clientHelloID = nil
+	}
 	// set up defaults
 	configuration := Configuration{
 		HTTPConfig: netx.Config{
 			BogonIsError:        c.Config.RejectDNSBogons,
 			CacheResolutions:    true,
 			CertPool:            c.Config.CertPool,
+			ClientHelloID:       clientHelloID,
 			ContextByteCounting: true,
 			DialSaver:           c.Saver,
 			HTTP3Enabled:        c.Config.HTTP3Enabled,

internal/engine/experiment/urlgetter/urlgetter.go

@@ -24,6 +24,7 @@ type Config struct {
 	Timeout  time.Duration
 
 	// settable from command line
+	ClientHelloID     string `ooni:"Parrot specific TLS ClientHello fingerprint, as specified in utls"`
 	DNSCache          string `ooni:"Add 'DOMAIN IP...' to cache"`
 	DNSHTTPHost       string `ooni:"Force using specific HTTP Host header for DNS requests"`
 	DNSTLSServerName  string `ooni:"Force TLS to using a specific SNI for encrypted DNS requests"`

internal/engine/netx/dialer/dialer.go

@@ -81,7 +81,7 @@ func New(config *Config, resolver Resolver) Dialer {
 		d = &saverConnDialer{Dialer: d, Saver: config.ReadWriteSaver}
 	}
 	d = &netxlite.DialerResolver{Resolver: resolver, Dialer: d}
-	d = &proxyDialer{ProxyURL: config.ProxyURL, Dialer: d}
+	d = &ProxyDialer{ProxyURL: config.ProxyURL, Dialer: d}
 	if config.ContextByteCounting {
 		d = &byteCounterDialer{Dialer: d}
 	}

internal/engine/netx/dialer/dialer_test.go

@@ -28,7 +28,7 @@ func TestNewCreatesTheExpectedChain(t *testing.T) {
 	if !ok {
 		t.Fatal("not a byteCounterDialer")
 	}
-	pd, ok := bcd.Dialer.(*proxyDialer)
+	pd, ok := bcd.Dialer.(*ProxyDialer)
 	if !ok {
 		t.Fatal("not a proxyDialer")
 	}

internal/engine/netx/dialer/proxy.go

@@ -9,10 +9,10 @@ import (
 	"golang.org/x/net/proxy"
 )
 
-// proxyDialer is a dialer that uses a proxy. If the ProxyURL is not configured, this
+// ProxyDialer is a dialer that uses a proxy. If the ProxyURL is not configured, this
 // dialer is a passthrough for the next Dialer in chain. Otherwise, it will internally
 // create a SOCKS5 dialer that will connect to the proxy using the underlying Dialer.
-type proxyDialer struct {
+type ProxyDialer struct {
 	Dialer
 	ProxyURL *url.URL
 }
@@ -21,7 +21,7 @@ type proxyDialer struct {
 var ErrProxyUnsupportedScheme = errors.New("proxy: unsupported scheme")
 
 // DialContext implements Dialer.DialContext
-func (d *proxyDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+func (d *ProxyDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
 	url := d.ProxyURL
 	if url == nil {
 		return d.Dialer.DialContext(ctx, network, address)
@@ -35,7 +35,7 @@ func (d *proxyDialer) DialContext(ctx context.Context, network, address string)
 	return d.dial(ctx, child, network, address)
 }
 
-func (d *proxyDialer) dial(
+func (d *ProxyDialer) dial(
 	ctx context.Context, child proxy.Dialer, network, address string) (net.Conn, error) {
 	cd := child.(proxy.ContextDialer) // will work
 	return cd.DialContext(ctx, network, address)

internal/engine/netx/dialer/proxy_test.go

@@ -13,7 +13,7 @@ import (
 
 func TestProxyDialerDialContextNoProxyURL(t *testing.T) {
 	expected := errors.New("mocked error")
-	d := &proxyDialer{
+	d := &ProxyDialer{
 		Dialer: &netxmocks.Dialer{
 			MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
 				return nil, expected
@@ -30,7 +30,7 @@ func TestProxyDialerDialContextNoProxyURL(t *testing.T) {
 }
 
 func TestProxyDialerDialContextInvalidScheme(t *testing.T) {
-	d := &proxyDialer{
+	d := &ProxyDialer{
 		ProxyURL: &url.URL{Scheme: "antani"},
 	}
 	conn, err := d.DialContext(context.Background(), "tcp", "www.google.com:443")
@@ -44,7 +44,7 @@ func TestProxyDialerDialContextInvalidScheme(t *testing.T) {
 
 func TestProxyDialerDialContextWithEOF(t *testing.T) {
 	const expect = "10.0.0.1:9050"
-	d := &proxyDialer{
+	d := &ProxyDialer{
 		Dialer: &netxmocks.Dialer{
 			MockDialContext: func(ctx context.Context, network string, address string) (net.Conn, error) {
 				if address != expect {

internal/engine/netx/httptransport/roundtripper.go

@@ -0,0 +1,141 @@
+package httptransport
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"golang.org/x/net/http2"
+)
+
+func newRoundtripper(txp *http.Transport, tlsdialer TLSDialer, tlsconfig *tls.Config) RoundTripper {
+	// we have to assume that this is a netxlite.TLSDialer, because we need access to its TLSHandshaker
+	handshaker := tlsdialer.(*netxlite.TLSDialer).TLSHandshaker
+	return &roundTripper{underlyingTransport: txp, DialTLS: tlsdialer.DialTLSContext, Handshaker: handshaker, tlsconfig: tlsconfig}
+}
+
+// roundTripper is a wrapper around the system transport
+type roundTripper struct {
+	sync.Mutex
+	ctx                 context.Context
+	DialTLS             func(ctx context.Context, network string, address string) (net.Conn, error)
+	Handshaker          netxlite.TLSHandshaker
+	tlsconfig           *tls.Config
+	transport           http.RoundTripper // this will be either http.Transport or http2.Transport
+	underlyingTransport *http.Transport
+}
+
+func (rt *roundTripper) CloseIdleConnections() {
+	rt.underlyingTransport.CloseIdleConnections()
+}
+
+func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	// determine transport type to use for this Roundtrip
+	rt.Lock()
+	rt.transport = nil
+	rt.Unlock()
+
+	if err := rt.getTransport(req); err != nil {
+		return nil, err
+	}
+	return rt.transport.RoundTrip(req)
+}
+
+var errTransportCreated = errors.New("used ALPN to determine transport type")
+
+func (rt *roundTripper) getTransport(req *http.Request) error {
+	scheme := strings.ToLower(req.URL.Scheme)
+	switch scheme {
+	case "http":
+		// HTTP 1.x w/o TLS
+		rt.transport = rt.underlyingTransport
+		return nil
+	case "https":
+	default:
+		return errors.New("invalid scheme")
+	}
+	ctx := req.Context()
+	_, err := rt.dialTLSContext(ctx, "tcp", getDialTLSAddr(req.URL))
+	switch err {
+	case errTransportCreated: // intended behavior
+		return nil
+	case nil:
+		return errors.New("dialTLS returned no error when determining transport")
+	default:
+		return err
+	}
+}
+
+func (rt *roundTripper) dialTLSContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	rt.Lock() // we are updating state
+	defer rt.Unlock()
+
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+	if rt.transport != nil {
+		// transport is already determined: use standard DialTLSContext
+		return rt.DialTLS(ctx, network, addr)
+	}
+	// connect
+	conn, err := net.Dial(network, addr)
+	if err != nil {
+		return nil, err
+	}
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+	// set TLS config
+	cfg := rt.tlsconfig
+	if cfg == nil {
+		cfg = new(tls.Config)
+	}
+	if cfg.ServerName == "" {
+		cfg.ServerName = host
+	}
+	cfg.NextProtos = []string{"h2", "http/1.1"}
+
+	// TLS handshake
+	_, state, err := rt.Handshaker.Handshake(ctx, conn, cfg)
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+	// use ALPN to decide which Transport to use
+	switch state.NegotiatedProtocol {
+	case "h2":
+		// HTTP 2 + TLS.
+		rt.ctx = ctx // there is no DialTLSContext in http2.Transport so we have to remember it in roundTripper
+		rt.transport = &http2.Transport{
+			DialTLS:            rt.dialTLSHTTP2,
+			TLSClientConfig:    rt.underlyingTransport.TLSClientConfig,
+			DisableCompression: rt.underlyingTransport.DisableCompression,
+		}
+	default:
+		// assume HTTP 1.x + TLS.
+		rt.transport = rt.underlyingTransport
+	}
+	return nil, errTransportCreated
+}
+
+// dialTLSHTTP2 fits the signature of http2.Transport.DialTLS
+func (rt *roundTripper) dialTLSHTTP2(network, addr string, cfg *tls.Config) (net.Conn, error) {
+	return rt.dialTLSContext(rt.ctx, network, addr)
+}
+
+func getDialTLSAddr(u *url.URL) string {
+	host, port, err := net.SplitHostPort(u.Host)
+	if err == nil {
+		return net.JoinHostPort(host, port)
+	}
+	return net.JoinHostPort(u.Host, u.Scheme)
+}