Skip to content

Commit

Permalink
Handle http/2
Browse files Browse the repository at this point in the history
There's an [open thread](refraction-networking/utls#16) in uTLS about dial support for http/2 servers. The tldr is that by returning a utls.Connection (versus a tls.Connection ) in the dial function it fails the test specified in [root handler](https://github.com/golang/go/blob/ea1437a8cdf6bb3c2d2447833a5d06dbd75f7ae4/src/net/http/transport.go#L1496) and therefore doesn't keep track of the connection state. This results in the regular connection attempting to request http/2 connection through an http/1 protocol.

We work around this through meek's re-implementation of the roundtrip
handler that dynamically routes between http/1 and http/2.
  • Loading branch information
piercefreeman committed Oct 13, 2022
1 parent 5ab2612 commit c278bc8
Show file tree
Hide file tree
Showing 32 changed files with 2,282 additions and 97 deletions.
16 changes: 10 additions & 6 deletions proxy_benchmarks/assets/proxies/gomitmproxy-mimic/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,17 @@ module proxy_benchmarks/gomitmproxymimic
go 1.18

require (
github.com/AdguardTeam/golibs v0.4.0 // indirect
github.com/AdguardTeam/gomitmproxy v0.2.1 // indirect
github.com/AdguardTeam/golibs v0.4.0
github.com/AdguardTeam/gomitmproxy v0.2.1
github.com/pkg/errors v0.9.1
github.com/refraction-networking/utls v1.1.3
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458
)

require (
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/refraction-networking/utls v1.1.3 // indirect
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
golang.org/x/text v0.3.7 // indirect
)
9 changes: 6 additions & 3 deletions proxy_benchmarks/assets/proxies/gomitmproxy-mimic/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,19 @@ golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2Vt
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 h1:MgJ6t2zo8v0tbmLCueaCbF1RM+TtB0rs3Lv8DGtOIpY=
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
38 changes: 2 additions & 36 deletions proxy_benchmarks/assets/proxies/gomitmproxy-mimic/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ import (
"sync"
"time"

mimic "github.com/refraction-networking/utls"

"github.com/AdguardTeam/gomitmproxy/proxyutil"

"github.com/AdguardTeam/golibs/log"
Expand Down Expand Up @@ -56,40 +54,8 @@ type Proxy struct {
func NewProxy(config Config) *Proxy {
proxy := &Proxy{
Config: config,
transport: &http.Transport{
// This forces http.Transport to not upgrade requests to HTTP/2
// TODO: Remove when HTTP/2 can be supported
TLSNextProto: make(map[string]func(string, *tls.Conn) http.RoundTripper),
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: tlsHandshakeTimeout,
ExpectContinueTimeout: time.Second,
TLSClientConfig: &tls.Config{
GetClientCertificate: func(info *tls.CertificateRequestInfo) (certificate *tls.Certificate, e error) {
// We purposefully cause an error here so that the http.Transport.RoundTrip method failed
// In this case we'll receive the error and will be able to add the host to invalidTLSHosts
return nil, errClientCertRequested
},
},
// @pierce - This is this only logic that we're overriding in this fork - it should be possible
// to integrate this with the shipping package. This isn't trivial however because Proxy.transport
// is a private struct variable so it's not visible by third party packages like `main`.
DialTLS: func(network, addr string) (net.Conn, error) {
conn, err := net.Dial(network, addr)
if err != nil {
return nil, err
}
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
config := &mimic.Config{ServerName: host}
uconn := mimic.UClient(conn, config, mimic.HelloChrome_Auto)
if err := uconn.Handshake(); err != nil {
return nil, err
}
return uconn, nil
},
},
// https://github.com/refraction-networking/utls/issues/16
transport: newRoundTripper(),
timeout: defaultTimeout,
invalidTLSHosts: map[string]bool{},
closing: make(chan bool),
Expand Down
163 changes: 163 additions & 0 deletions proxy_benchmarks/assets/proxies/gomitmproxy-mimic/proxy/transport.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
/*
* Copyright (c) 2019 Yawning Angel <yawning at schwanenlied dot me>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

package gomitmproxy

import (
"crypto/tls"
"errors"
"fmt"
"net"
"net/http"
"net/url"
"strings"
"sync"

utls "github.com/refraction-networking/utls"
"golang.org/x/net/http2"
)

var errProtocolNegotiated = errors.New("meek_lite: protocol negotiated")

type roundTripper struct {
sync.Mutex

transport http.RoundTripper

initConn net.Conn
}

func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
// Note: This isn't protected with a lock, since the meeklite ioWorker
// serializes RoundTripper requests.
//
// This also assumes that req.URL.Host will remain constant for the
// lifetime of the roundTripper, which is a valid assumption for meeklite.
if rt.transport == nil {
if err := rt.getTransport(req); err != nil {
return nil, err
}
}
return rt.transport.RoundTrip(req)
}

func (rt *roundTripper) getTransport(req *http.Request) error {
switch strings.ToLower(req.URL.Scheme) {
case "http":
rt.transport = &http.Transport{Dial: net.Dial}
return nil
case "https":
default:
return fmt.Errorf("meek_lite: invalid URL scheme: '%v'", req.URL.Scheme)
}

_, err := rt.dialTLS("tcp", getDialTLSAddr(req.URL))
switch err {
case errProtocolNegotiated:
case nil:
// Should never happen.
panic("meek_lite: dialTLS returned no error when determining transport")
default:
return err
}

return nil
}

func (rt *roundTripper) dialTLS(network, addr string) (net.Conn, error) {
// Unlike rt.transport, this is protected by a critical section
// since past the initial manual call from getTransport, the HTTP
// client will be the caller.
rt.Lock()
defer rt.Unlock()

// If we have the connection from when we determined the HTTPS
// transport to use, return that.
if conn := rt.initConn; conn != nil {
rt.initConn = nil
return conn, nil
}

rawConn, err := net.Dial(network, addr)
if err != nil {
return nil, err
}

var host string
if host, _, err = net.SplitHostPort(addr); err != nil {
host = addr
}

// TODO: Make this configurable. What "works" is host dependent.
// * HelloChrome_Auto - Failures in a stand alone testcase against google.com
// * HelloFirefox_Auto - Fails with the azure bridge, incompatible group.
// * HelloIOS_Auto - Seems to work.
//
// Since HelloChrome_Auto works with azure, that's what'll be used for
// now, since that's what the overwelming vast majority of people will
// use.
conn := utls.UClient(rawConn, &utls.Config{ServerName: host}, utls.HelloChrome_Auto)
if err = conn.Handshake(); err != nil {
conn.Close()
return nil, err
}

if rt.transport != nil {
return conn, nil
}

// No http.Transport constructed yet, create one based on the results
// of ALPN.
switch conn.ConnectionState().NegotiatedProtocol {
case http2.NextProtoTLS:
// The remote peer is speaking HTTP 2 + TLS.
rt.transport = &http2.Transport{DialTLS: rt.dialTLSHTTP2}
default:
// Assume the remote peer is speaking HTTP 1.x + TLS.
rt.transport = &http.Transport{DialTLS: rt.dialTLS}
}

// Stash the connection just established for use servicing the
// actual request (should be near-immediate).
rt.initConn = conn

return nil, errProtocolNegotiated
}

func (rt *roundTripper) dialTLSHTTP2(network, addr string, cfg *tls.Config) (net.Conn, error) {
return rt.dialTLS(network, addr)
}

func getDialTLSAddr(u *url.URL) string {
host, port, err := net.SplitHostPort(u.Host)
if err == nil {
return net.JoinHostPort(host, port)
}

return net.JoinHostPort(u.Host, u.Scheme)
}

func newRoundTripper() http.RoundTripper {
return &roundTripper{}
}

func init() {
// Attempt to increase compatibility, there's an encrypted link
// underneath, and this doesn't (shouldn't) affect the external
// fingerprint.
utls.EnableWeakCiphers()
}
2 changes: 1 addition & 1 deletion proxy_benchmarks/assets/proxies/goproxy-mimic/cert.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"crypto/tls"
"crypto/x509"

"github.com/elazarl/goproxy"
goproxy "proxy_benchmarks/goproxymimic/proxy"
)

func setCA(caCert string, caKey string) error {
Expand Down
11 changes: 8 additions & 3 deletions proxy_benchmarks/assets/proxies/goproxy-mimic/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,16 @@ module proxy_benchmarks/goproxymimic

go 1.18

require (
github.com/elazarl/goproxy v0.0.0-20220901064549-fbd10ff4f5a1
github.com/refraction-networking/utls v1.1.3
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458
)

require (
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/elazarl/goproxy v0.0.0-20220901064549-fbd10ff4f5a1 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/refraction-networking/utls v1.1.3 // indirect
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
golang.org/x/text v0.3.7 // indirect
)
8 changes: 7 additions & 1 deletion proxy_benchmarks/assets/proxies/goproxy-mimic/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/elazarl/goproxy v0.0.0-20220901064549-fbd10ff4f5a1 h1:ecIiM5NYeEOhy5trm8xel6wpUhYH+QWteUKnwcbCMl4=
github.com/elazarl/goproxy v0.0.0-20220901064549-fbd10ff4f5a1/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
Expand All @@ -12,11 +13,16 @@ golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2Vt
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 h1:MgJ6t2zo8v0tbmLCueaCbF1RM+TtB0rs3Lv8DGtOIpY=
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
36 changes: 4 additions & 32 deletions proxy_benchmarks/assets/proxies/goproxy-mimic/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,8 @@ import (
"regexp"
"strconv"

mimic "github.com/refraction-networking/utls"

"github.com/elazarl/goproxy"
//"github.com/elazarl/goproxy"
goproxy "proxy_benchmarks/goproxymimic/proxy"
)

func orPanic(err error) {
Expand All @@ -25,33 +24,6 @@ func orPanic(err error) {
}
}

func NewProxy() *goproxy.ProxyHttpServer {
/*
* Create a new proxy with
* @pierce - primary modification for the mimic
*/
proxy := goproxy.NewProxyHttpServer()

proxy.Tr.DialTLS = func(network, addr string) (net.Conn, error) {
conn, err := net.Dial(network, addr)
if err != nil {
return nil, err
}
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
config := &mimic.Config{ServerName: host}
uconn := mimic.UClient(conn, config, mimic.HelloChrome_Auto)
if err := uconn.Handshake(); err != nil {
return nil, err
}
return uconn, nil
}

return proxy
}

func main() {
var (
verbose = flag.Bool("v", true, "should every proxy request be logged to stdout")
Expand All @@ -62,15 +34,15 @@ func main() {
// Set our own CA instead of the one that's default bundled with the proxy
setCA("ca.crt", "ca.key")

proxy := NewProxy()
proxy := goproxy.NewProxyHttpServer()
proxy.Verbose = *verbose

// Our other implementations cache the certificates for some length of time, so we do the
// same here for equality in benchmarking
proxy.CertStore = NewOptimizedCertStore()

if proxy.Verbose {
log.Printf("Server starting up! - configured to listen on http interface %d - mimic", *port)
log.Printf("Server starting up! - configured to listen on http interface %d", *port)
}

proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
Expand Down
Loading

0 comments on commit c278bc8

Please sign in to comment.