refactor(netemx): use QAEnvOptionNetStack for ScenarioRoleDNSOverHTTPS (

ooni#1229) This diff continues the refactoring of netemx to use a single mechanism to create all the possible kind of servers. While there, this diff removes the limitation that we cannot create more than a single server per IP address, which was one of the reasons why we started this refactoring process. While there, make sure we have full coverage of netemx. ## Checklist - [x] I have read the [contribution guidelines](https://github.com/ooni/probe-cli/blob/master/CONTRIBUTING.md) - [x] reference issue for this pull request: ooni/probe#1803 - [x] if you changed anything related to how experiments work and you need to reflect these changes in the ooni/spec repository, please link to the related ooni/spec pull request: N/A - [x] if you changed code inside an experiment, make sure you bump its version number: N/A
Murphy-OrangeMud · Feb 13, 2024 · 04440e8 · 04440e8
1 parent 93c189d
commit 04440e8
Show file tree

Hide file tree

Showing 16 changed files with 142 additions and 33 deletions.
diff --git a/internal/experiment/telegram/telegram_test.go b/internal/experiment/telegram/telegram_test.go
@@ -280,7 +280,7 @@ func configureDNSWithDefaults(config *netem.DNSConfig) {
 func newQAEnvironment(ipaddrs ...string) *netemx.QAEnv {
 	// create a single factory for handling all the requests
 	factory := &netemx.HTTPCleartextServerFactory{
-		Factory: netemx.HTTPHandlerFactoryFunc(func(_ *netem.UNetStack) http.Handler {
+		Factory: netemx.HTTPHandlerFactoryFunc(func(env netemx.NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 			// we create an empty mux, which should cause a 404 for each webpage, which seems what
 			// the servers used by telegram DC do as of 2023-07-11
 			return http.NewServeMux()

diff --git a/internal/netemx/dnsoverhttps.go b/internal/netemx/dnsoverhttps.go
@@ -8,13 +8,11 @@ import (
 )
 
 // DNSOverHTTPSHandlerFactory is a [QAEnvHTTPHandlerFactory] for [testingx.GeoIPHandlerUbuntu].
-type DNSOverHTTPSHandlerFactory struct {
-	Config *netem.DNSConfig
-}
+type DNSOverHTTPSHandlerFactory struct{}
 
 var _ HTTPHandlerFactory = &DNSOverHTTPSHandlerFactory{}
 
 // NewHandler implements QAEnvHTTPHandlerFactory.
-func (f *DNSOverHTTPSHandlerFactory) NewHandler(_ *netem.UNetStack) http.Handler {
-	return &testingx.DNSOverHTTPSHandler{Config: f.Config}
+func (f *DNSOverHTTPSHandlerFactory) NewHandler(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
+	return &testingx.DNSOverHTTPSHandler{Config: env.OtherResolversConfig()}
 }
diff --git a/internal/netemx/example_test.go b/internal/netemx/example_test.go
@@ -403,3 +403,44 @@ func Example_ubuntuGeoIPWithInternetScenario() {
 	// Output:
 	// <?xml version="1.0" encoding="UTF-8"?><Response><Ip>130.192.91.211</Ip></Response>
 }
+
+// This example shows how the [InternetScenario] defines a public blockpage server.
+func Example_examplePublicBlockpage() {
+	env := netemx.MustNewScenario(netemx.InternetScenario)
+	defer env.Close()
+
+	env.Do(func() {
+		client := netxlite.NewHTTPClientStdlib(log.Log)
+
+		req, err := http.NewRequest("GET", "https://"+netemx.AddressPublicBlockpage+"/", nil)
+		if err != nil {
+			log.Fatalf("http.NewRequest failed: %s", err.Error())
+		}
+
+		resp, err := client.Do(req)
+		if err != nil {
+			log.Fatalf("client.Do failed: %s", err.Error())
+		}
+		defer resp.Body.Close()
+		body, err := netxlite.ReadAllContext(req.Context(), resp.Body)
+		if err != nil {
+			log.Fatalf("netxlite.ReadAllContext failed: %s", err.Error())
+		}
+
+		fmt.Printf("%+v\n", string(body))
+	})
+
+	// Output:
+	// <!doctype html>
+	// <html>
+	// <head>
+	// 	<title>Access Denied</title>
+	// </head>
+	// <body>
+	// <div>
+	// 	<h1>Access Denied</h1>
+	// 	<p>This request cannot be served in your jurisdiction.</p>
+	// </div>
+	// </body>
+	// </html>
+}
diff --git a/internal/netemx/geoip.go b/internal/netemx/geoip.go
@@ -15,6 +15,6 @@ type GeoIPHandlerFactoryUbuntu struct {
 var _ HTTPHandlerFactory = &GeoIPHandlerFactoryUbuntu{}
 
 // NewHandler implements QAEnvHTTPHandlerFactory.
-func (f *GeoIPHandlerFactoryUbuntu) NewHandler(_ *netem.UNetStack) http.Handler {
+func (f *GeoIPHandlerFactoryUbuntu) NewHandler(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 	return &testingx.GeoIPHandlerUbuntu{ProbeIP: f.ProbeIP}
 }
diff --git a/internal/netemx/http.go b/internal/netemx/http.go
@@ -12,17 +12,17 @@ import (
 
 // HTTPHandlerFactory constructs an [http.Handler].
 type HTTPHandlerFactory interface {
-	NewHandler(stack *netem.UNetStack) http.Handler
+	NewHandler(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler
 }
 
 // HTTPHandlerFactoryFunc allows a func to become an [HTTPHandlerFactory].
-type HTTPHandlerFactoryFunc func(stack *netem.UNetStack) http.Handler
+type HTTPHandlerFactoryFunc func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler
 
 var _ HTTPHandlerFactory = HTTPHandlerFactoryFunc(nil)
 
 // NewHandler implements HTTPHandlerFactory.
-func (fx HTTPHandlerFactoryFunc) NewHandler(stack *netem.UNetStack) http.Handler {
-	return fx(stack)
+func (fx HTTPHandlerFactoryFunc) NewHandler(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
+	return fx(env, stack)
 }
 
 // HTTPCleartextServerFactory implements [NetStackServerFactory] for cleartext HTTP.
@@ -39,9 +39,10 @@ type HTTPCleartextServerFactory struct {
 var _ NetStackServerFactory = &HTTPCleartextServerFactory{}
 
 // MustNewServer implements NetStackServerFactory.
-func (f *HTTPCleartextServerFactory) MustNewServer(_ NetStackServerFactoryEnv, stack *netem.UNetStack) NetStackServer {
+func (f *HTTPCleartextServerFactory) MustNewServer(env NetStackServerFactoryEnv, stack *netem.UNetStack) NetStackServer {
 	return &httpCleartextServer{
 		closers: []io.Closer{},
+		env:     env,
 		factory: f.Factory,
 		mu:      sync.Mutex{},
 		ports:   f.Ports,
@@ -51,6 +52,7 @@ func (f *HTTPCleartextServerFactory) MustNewServer(_ NetStackServerFactoryEnv, s
 
 type httpCleartextServer struct {
 	closers []io.Closer
+	env     NetStackServerFactoryEnv
 	factory HTTPHandlerFactory
 	mu      sync.Mutex
 	ports   []int
@@ -80,7 +82,7 @@ func (srv *httpCleartextServer) MustStart() {
 	srv.mu.Lock()
 
 	// create the handler
-	handler := srv.factory.NewHandler(srv.unet)
+	handler := srv.factory.NewHandler(srv.env, srv.unet)
 
 	// create the listening address
 	ipAddr := net.ParseIP(srv.unet.IPAddress())

diff --git a/internal/netemx/http3.go b/internal/netemx/http3.go
@@ -29,9 +29,10 @@ type HTTP3ServerFactory struct {
 var _ NetStackServerFactory = &HTTP3ServerFactory{}
 
 // MustNewServer implements NetStackServerFactory.
-func (f *HTTP3ServerFactory) MustNewServer(_ NetStackServerFactoryEnv, stack *netem.UNetStack) NetStackServer {
+func (f *HTTP3ServerFactory) MustNewServer(env NetStackServerFactoryEnv, stack *netem.UNetStack) NetStackServer {
 	return &http3Server{
 		closers:   []io.Closer{},
+		env:       env,
 		factory:   f.Factory,
 		mu:        sync.Mutex{},
 		ports:     f.Ports,
@@ -42,6 +43,7 @@ func (f *HTTP3ServerFactory) MustNewServer(_ NetStackServerFactoryEnv, stack *ne
 
 type http3Server struct {
 	closers   []io.Closer
+	env       NetStackServerFactoryEnv
 	factory   HTTPHandlerFactory
 	mu        sync.Mutex
 	ports     []int
@@ -72,7 +74,7 @@ func (srv *http3Server) MustStart() {
 	srv.mu.Lock()
 
 	// create the handler
-	handler := srv.factory.NewHandler(srv.unet)
+	handler := srv.factory.NewHandler(srv.env, srv.unet)
 
 	// create the listening address
 	ipAddr := net.ParseIP(srv.unet.IPAddress())

diff --git a/internal/netemx/http3_test.go b/internal/netemx/http3_test.go
@@ -15,7 +15,7 @@ func TestHTTP3ServerFactory(t *testing.T) {
 	t.Run("when using the TLSConfig provided by netem", func(t *testing.T) {
 		env := MustNewQAEnv(
 			QAEnvOptionNetStack(AddressWwwExampleCom, &HTTP3ServerFactory{
-				Factory: HTTPHandlerFactoryFunc(func(_ *netem.UNetStack) http.Handler {
+				Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 					return ExampleWebPageHandler()
 				}),
 				Ports:     []int{443},
@@ -54,7 +54,7 @@ func TestHTTP3ServerFactory(t *testing.T) {
 
 		env := MustNewQAEnv(
 			QAEnvOptionNetStack(AddressWwwExampleCom, &HTTP3ServerFactory{
-				Factory: HTTPHandlerFactoryFunc(func(_ *netem.UNetStack) http.Handler {
+				Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 					return ExampleWebPageHandler()
 				}),
 				Ports:     []int{443},

diff --git a/internal/netemx/http_test.go b/internal/netemx/http_test.go
@@ -14,7 +14,7 @@ import (
 func TestHTTPCleartextServerFactory(t *testing.T) {
 	env := MustNewQAEnv(
 		QAEnvOptionNetStack(AddressWwwExampleCom, &HTTPCleartextServerFactory{
-			Factory: HTTPHandlerFactoryFunc(func(_ *netem.UNetStack) http.Handler {
+			Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 				return ExampleWebPageHandler()
 			}),
 			Ports: []int{80},

diff --git a/internal/netemx/https.go b/internal/netemx/https.go
@@ -28,9 +28,10 @@ type HTTPSecureServerFactory struct {
 var _ NetStackServerFactory = &HTTPSecureServerFactory{}
 
 // MustNewServer implements NetStackServerFactory.
-func (f *HTTPSecureServerFactory) MustNewServer(_ NetStackServerFactoryEnv, stack *netem.UNetStack) NetStackServer {
+func (f *HTTPSecureServerFactory) MustNewServer(env NetStackServerFactoryEnv, stack *netem.UNetStack) NetStackServer {
 	return &httpSecureServer{
 		closers:   []io.Closer{},
+		env:       env,
 		factory:   f.Factory,
 		mu:        sync.Mutex{},
 		ports:     f.Ports,
@@ -41,6 +42,7 @@ func (f *HTTPSecureServerFactory) MustNewServer(_ NetStackServerFactoryEnv, stac
 
 type httpSecureServer struct {
 	closers   []io.Closer
+	env       NetStackServerFactoryEnv
 	factory   HTTPHandlerFactory
 	mu        sync.Mutex
 	ports     []int
@@ -71,7 +73,7 @@ func (srv *httpSecureServer) MustStart() {
 	srv.mu.Lock()
 
 	// create the handler
-	handler := srv.factory.NewHandler(srv.unet)
+	handler := srv.factory.NewHandler(srv.env, srv.unet)
 
 	// create the listening address
 	ipAddr := net.ParseIP(srv.unet.IPAddress())

diff --git a/internal/netemx/https_test.go b/internal/netemx/https_test.go
@@ -15,7 +15,7 @@ func TestHTTPSecureServerFactory(t *testing.T) {
 	t.Run("when using the TLSConfig provided by netem", func(t *testing.T) {
 		env := MustNewQAEnv(
 			QAEnvOptionNetStack(AddressWwwExampleCom, &HTTPSecureServerFactory{
-				Factory: HTTPHandlerFactoryFunc(func(_ *netem.UNetStack) http.Handler {
+				Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 					return ExampleWebPageHandler()
 				}),
 				Ports:     []int{443},
@@ -54,7 +54,7 @@ func TestHTTPSecureServerFactory(t *testing.T) {
 
 		env := MustNewQAEnv(
 			QAEnvOptionNetStack(AddressWwwExampleCom, &HTTPSecureServerFactory{
-				Factory: HTTPHandlerFactoryFunc(func(_ *netem.UNetStack) http.Handler {
+				Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 					return ExampleWebPageHandler()
 				}),
 				Ports:     []int{443},

diff --git a/internal/netemx/ooapi.go b/internal/netemx/ooapi.go
@@ -15,7 +15,7 @@ type OOAPIHandlerFactory struct{}
 var _ HTTPHandlerFactory = &OOAPIHandlerFactory{}
 
 // NewHandler implements QAEnvHTTPHandlerFactory.
-func (*OOAPIHandlerFactory) NewHandler(_ *netem.UNetStack) http.Handler {
+func (*OOAPIHandlerFactory) NewHandler(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 	return &OOAPIHandler{}
 }
 

diff --git a/internal/netemx/oohelperd.go b/internal/netemx/oohelperd.go
@@ -20,7 +20,7 @@ type OOHelperDFactory struct{}
 var _ HTTPHandlerFactory = &OOHelperDFactory{}
 
 // NewHandler implements QAEnvHTTPHandlerFactory.NewHandler.
-func (f *OOHelperDFactory) NewHandler(unet *netem.UNetStack) http.Handler {
+func (f *OOHelperDFactory) NewHandler(env NetStackServerFactoryEnv, unet *netem.UNetStack) http.Handler {
 	netx := netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: unet}}
 	handler := oohelperd.NewHandler()
 

diff --git a/internal/netemx/qaenv.go b/internal/netemx/qaenv.go
@@ -316,7 +316,7 @@ func (env *QAEnv) mustNewHTTPServers(config *qaEnvConfig) (closables []io.Closer
 		))
 
 		// create HTTP, HTTPS and HTTP/3 servers for this stack
-		handler := factory.NewHandler(stack)
+		handler := factory.NewHandler(env, stack)
 		closables = append(closables, env.mustCreateAllHTTPServers(stack, handler, addr)...)
 	}
 	return

diff --git a/internal/netemx/scenario.go b/internal/netemx/scenario.go
@@ -110,10 +110,8 @@ var InternetScenario = []*ScenarioDomainAddresses{{
 func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv {
 	var opts []QAEnvOption
 
-	// TODO(bassosimone): it's currently a bottleneck that the same server cannot be _at the
-	// same time_ both a DNS over cleartext and a DNS over HTTPS server.
-	//
-	// As a result, the code below for initializing $stuff is more complex than it should.
+	// TODO(bassosimone): we removed the bottleneck that we could not have more than one
+	// server per IP address, so it's time to take advantage of this below.
 
 	// create a common configuration for DoH servers
 	dohConfig := netem.NewDNSConfig()
@@ -131,8 +129,10 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv {
 		switch sad.Role {
 		case ScenarioRoleDNSOverHTTPS:
 			for _, addr := range sad.Addresses {
-				opts = append(opts, QAEnvOptionHTTPServer(addr, &DNSOverHTTPSHandlerFactory{
-					Config: dohConfig,
+				opts = append(opts, QAEnvOptionNetStack(addr, &HTTPSecureServerFactory{
+					Factory:   &DNSOverHTTPSHandlerFactory{},
+					Ports:     []int{443},
+					TLSConfig: nil, // use netem's default
 				}))
 			}
 

diff --git a/internal/netemx/web.go b/internal/netemx/web.go
@@ -60,7 +60,7 @@ func ExampleWebPageHandler() http.Handler {
 // ExampleWebPageHandlerFactory returns a webpage similar to example.org's one when the domain is
 // www.example.{com,org} and redirects to www.example.{com,org} when it is example.{com,org}.
 func ExampleWebPageHandlerFactory() HTTPHandlerFactory {
-	return HTTPHandlerFactoryFunc(func(_ *netem.UNetStack) http.Handler {
+	return HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 		return ExampleWebPageHandler()
 	})
 }
@@ -85,7 +85,7 @@ const Blockpage = `<!doctype html>
 
 // BlockpageHandlerFactory returns a blockpage regardless of the incoming domain.
 func BlockpageHandlerFactory() HTTPHandlerFactory {
-	return HTTPHandlerFactoryFunc(func(_ *netem.UNetStack) http.Handler {
+	return HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			w.Header().Add("Alt-Svc", `h3=":443"`)
 			w.Header().Add("Date", "Thu, 24 Aug 2023 14:35:29 GMT")

diff --git a/internal/netemx/web_test.go b/internal/netemx/web_test.go
@@ -0,0 +1,64 @@
+package netemx
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+)
+
+func TestExampleWebPageHandler(t *testing.T) {
+	t.Run("we're redirected if the host is example.com", func(t *testing.T) {
+		req := &http.Request{
+			URL:   &url.URL{Path: "/"},
+			Body:  http.NoBody,
+			Close: false,
+			Host:  "example.com",
+		}
+		rr := httptest.NewRecorder()
+		handler := ExampleWebPageHandler()
+		handler.ServeHTTP(rr, req)
+		result := rr.Result()
+		if result.StatusCode != http.StatusPermanentRedirect {
+			t.Fatal("unexpected status code", result.StatusCode)
+		}
+		if loc := result.Header.Get("Location"); loc != "https://www.example.com/" {
+			t.Fatal("unexpected location", loc)
+		}
+	})
+
+	t.Run("we're redirected if the host is example.org", func(t *testing.T) {
+		req := &http.Request{
+			URL:   &url.URL{Path: "/"},
+			Body:  http.NoBody,
+			Close: false,
+			Host:  "example.org",
+		}
+		rr := httptest.NewRecorder()
+		handler := ExampleWebPageHandler()
+		handler.ServeHTTP(rr, req)
+		result := rr.Result()
+		if result.StatusCode != http.StatusPermanentRedirect {
+			t.Fatal("unexpected status code", result.StatusCode)
+		}
+		if loc := result.Header.Get("Location"); loc != "https://www.example.org/" {
+			t.Fatal("unexpected location", loc)
+		}
+	})
+
+	t.Run("we get a 400 for an unknown host", func(t *testing.T) {
+		req := &http.Request{
+			URL:   &url.URL{Path: "/"},
+			Body:  http.NoBody,
+			Close: false,
+			Host:  "antani.xyz",
+		}
+		rr := httptest.NewRecorder()
+		handler := ExampleWebPageHandler()
+		handler.ServeHTTP(rr, req)
+		result := rr.Result()
+		if result.StatusCode != http.StatusBadRequest {
+			t.Fatal("unexpected status code", result.StatusCode)
+		}
+	})
+}