Bug 1986228: NE-310 e2e test for HSTS

pkg/manifests/manifests.go

@@ -69,6 +69,9 @@ const (
 	// instance.
 	DefaultIngressControllerName = "default"
 
+	// ClusterIngressConfigName is the name of the cluster Ingress Config
+	ClusterIngressConfigName = "cluster"
+
 	NamespaceManifest                = "manifests/00-namespace.yaml"
 	CustomResourceDefinitionManifest = "manifests/00-custom-resource-definition.yaml"
 )

test/e2e/hsts_policy_test.go

@@ -0,0 +1,157 @@
+// +build e2e
+
+package e2e
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	configv1 "github.com/openshift/api/config/v1"
+
+	corev1 "k8s.io/api/core/v1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+// Helper function for int32 pointers
+func intPtr(s int32) *int32 {
+	return &s
+}
+
+func TestHstsPolicyWorks(t *testing.T) {
+	icName := types.NamespacedName{Namespace: operatorNamespace, Name: "hsts-policy"}
+	domain := icName.Name + "." + dnsConfig.Spec.BaseDomain
+
+	// Setup a Required HSTS Policy for the ingress config
+	maxAgePolicy := configv1.MaxAgePolicy{LargestMaxAge: intPtr(99999), SmallestMaxAge: intPtr(0)}
+	domainPatterns := []string{domain}
+	hstsPolicy := configv1.RequiredHSTSPolicy{
+		DomainPatterns:          domainPatterns, // this policy will only validate routes with hosts in the DomainPatterns
+		PreloadPolicy:           configv1.RequirePreloadPolicy,
+		IncludeSubDomainsPolicy: configv1.RequireIncludeSubDomains,
+		MaxAge:                  maxAgePolicy,
+	}
+
+	ing := &configv1.Ingress{}
+	appsDomain := ""
+
+	// Update the ingress config with the new HSTS policy, and include its apps domain
+	if err := wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
+		// Get the ingress config
+		if err := kclient.Get(context.TODO(), clusterConfigName, ing); err != nil {
+			t.Logf("Get ingress config failed: %v, retrying...", err)
+			return false, nil
+		}
+
+		// Update the hsts.DomainPatterns with the ingress spec domain
+		if !domainExists(ing.Spec.Domain, hstsPolicy.DomainPatterns) {
+			hstsPolicy.DomainPatterns = append(hstsPolicy.DomainPatterns, "*."+ing.Spec.Domain)
+			appsDomain = ing.Spec.Domain
+		}
+
+		// Update the ingress config with the new policy
+		ing.Spec.RequiredHSTSPolicies = append(ing.Spec.RequiredHSTSPolicies, hstsPolicy)
+		// Update the ingress config
+		if err := kclient.Update(context.TODO(), ing); err != nil {
+			t.Logf("failed to update ingress config: %v, retrying...", err)
+			return false, nil
+		}
+
+		return true, nil
+
+	}); err != nil {
+		t.Fatalf("failed to update ingress config: %v", err)
+	}
+	defer func() {
+		// Remove the HSTS policies from the ingress config
+		ing.Spec.RequiredHSTSPolicies = nil
+		if err := kclient.Update(context.TODO(), ing); err != nil {
+			t.Fatalf("failed to restore ingress config: %v", err)
+		}
+	}()
+
+	p := ing.Spec.RequiredHSTSPolicies[0]
+	t.Logf("created a RequiredHSTSPolicy with DomainPatterns: %v,\n preload policy: %s,\n includeSubDomains policy: %s,\n largest age: %d,\n smallest age: %d\n", p.DomainPatterns, p.PreloadPolicy, p.IncludeSubDomainsPolicy, *p.MaxAge.LargestMaxAge, *p.MaxAge.SmallestMaxAge)
+
+	// Use the same namespace for route, service, and pod
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "hsts-policy-namespace",
+		},
+	}
+	if err := kclient.Create(context.TODO(), ns); err != nil {
+		t.Fatalf("failed to create namespace: %v", err)
+	}
+	defer func() {
+		// this will cleanup all components in this namespace
+		if err := kclient.Delete(context.TODO(), ns); err != nil {
+			t.Fatalf("failed to delete test namespace %s: %v", ns.Name, err)
+		}
+	}()
+
+	// Create pod
+	echoPod := buildEchoPod("hsts-policy-echo", ns.Name)
+	if err := kclient.Create(context.TODO(), echoPod); err != nil {
+		t.Fatalf("failed to create pod %s/%s: %v", ns, echoPod.Name, err)
+	}
+
+	// Create service
+	echoService := buildEchoService(echoPod.Name, ns.Name, echoPod.ObjectMeta.Labels)
+	if err := kclient.Create(context.TODO(), echoService); err != nil {
+		t.Fatalf("failed to create service %s/%s: %v", echoService.Namespace, echoService.Name, err)
+	}
+
+	// Create a route that should fail the HSTS policy validation
+	t.Logf("creating an invalid route at %s", time.Now().Format(time.StampMilli))
+	invalidRoute := buildRouteWithHSTS("invalid-route", echoPod.Namespace, echoService.Name, "invalid-route."+appsDomain, "max-age=99999999")
+	if err := kclient.Create(context.TODO(), invalidRoute); err == nil {
+		t.Fatalf("failed to reject an invalid route %s/%s, max-age 99999999", invalidRoute.Namespace, invalidRoute.Name)
+	} else {
+		t.Logf("rejected an invalid route at %s: %s/%s with annotation %s: %v", time.Now().Format(time.StampMilli), invalidRoute.Namespace, invalidRoute.Name, invalidRoute.Annotations, err)
+	}
+
+	// Create a route that should pass the HSTS policy validation
+	exampleHeader := "max-age=0;preload;includesubdomains"
+	t.Logf("creating a valid route at %s", time.Now().Format(time.StampMilli))
+	validRoute := buildRouteWithHSTS("valid-route", echoPod.Namespace, echoService.Name, "valid-route."+appsDomain, exampleHeader)
+	if err := kclient.Create(context.TODO(), validRoute); err != nil {
+		t.Fatalf("failed to create a valid route %s/%s: %v", validRoute.Namespace, validRoute.Name, err)
+	} else {
+		t.Logf("created a valid route at %s: %s/%s with annotation %s", time.Now().Format(time.StampMilli), validRoute.Namespace, validRoute.Name, validRoute.Annotations)
+	}
+
+	code := 0
+	// Wait for route service to respond, and when it does, check for the header that should be there
+	if err := wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) {
+		resultHeader, code, _, err := getHttpResponse(t, validRoute)
+		if code != http.StatusOK && err != nil {
+			return false, nil
+		}
+		header := resultHeader.Get("strict-transport-security")
+		if header == "" {
+			t.Logf("request to %s got no HSTS header: %s (%d) %v, trying again....", validRoute.Spec.Host, resultHeader, code, err)
+			return false, nil
+		}
+		if strings.Compare(header, exampleHeader) != 0 {
+			t.Fatalf("request to %s got wrong HSTS header, should be %s, not %s", validRoute.Spec.Host, exampleHeader, header)
+		}
+		t.Logf("request to %s got correct HSTS header: %s", validRoute.Spec.Host, header)
+		return true, nil
+	}); err != nil {
+		t.Fatalf("failed to find service %s for header %s in 5 minutes, http code %d: %v", validRoute.Spec.To.Name, exampleHeader, code, err)
+	}
+}
+
+func domainExists(appsDomain string, patterns []string) bool {
+	for _, domain := range patterns {
+		if domain == appsDomain {
+			return true
+		}
+	}
+	return false
+}

test/e2e/operator_test.go

@@ -89,6 +89,7 @@ var dnsConfig configv1.DNS
 var infraConfig configv1.Infrastructure
 var operatorNamespace = operatorcontroller.DefaultOperatorNamespace
 var defaultName = types.NamespacedName{Namespace: operatorNamespace, Name: manifests.DefaultIngressControllerName}
+var clusterConfigName = types.NamespacedName{Namespace: operatorNamespace, Name: manifests.ClusterIngressConfigName}
 
 func TestMain(m *testing.M) {
 	kubeConfig, err := config.GetConfig()

test/e2e/util.go

@@ -4,23 +4,24 @@ package e2e
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
+	"io/ioutil"
+	"net/http"
 	"testing"
 	"time"
 
+	operatorv1 "github.com/openshift/api/operator/v1"
 	routev1 "github.com/openshift/api/route/v1"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	operatorv1 "github.com/openshift/api/operator/v1"
-
-	appsv1 "k8s.io/api/apps/v1"
-
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
+
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -37,6 +38,7 @@ func buildEchoPod(name, namespace string) *corev1.Pod {
 		Spec: corev1.PodSpec{
 			Containers: []corev1.Container{
 				{
+					// Note that HTTP/1.0 will strip the HSTS response header
 					Args: []string{
 						"TCP4-LISTEN:8080,reuseaddr,fork",
 						`EXEC:'/bin/bash -c \"printf \\\"HTTP/1.0 200 OK\r\n\r\n\\\"; sed -e \\\"/^\r/q\\\"\"'`,
@@ -145,6 +147,20 @@ func buildRoute(name, namespace, serviceName string) *routev1.Route {
 	}
 }
 
+// buildRoute returns a route definition with the specified HSTS annotation.
+// Overwrites Spec.Host and TLS
+func buildRouteWithHSTS(podName, namespace, serviceName, domain, annotation string) *routev1.Route {
+	route := buildRoute(podName, namespace, serviceName)
+	route.Spec.Host = domain
+	route.Spec.TLS = &routev1.TLSConfig{Termination: routev1.TLSTerminationEdge}
+	if route.Annotations == nil {
+		route.Annotations = map[string]string{}
+	}
+	route.Annotations["haproxy.router.openshift.io/hsts_header"] = annotation
+
+	return route
+}
+
 func getIngressController(t *testing.T, client client.Client, name types.NamespacedName, timeout time.Duration) (*operatorv1.IngressController, error) {
 	t.Helper()
 	ic := operatorv1.IngressController{}
@@ -174,3 +190,41 @@ func getDeployment(t *testing.T, client client.Client, name types.NamespacedName
 	}
 	return &dep, nil
 }
+
+// Send HTTPS Request to a route and check what is returned in the Response
+func getHttpResponse(t *testing.T, route *routev1.Route) (http.Header, int, string, error) {
+	t.Helper()
+	if len(route.Spec.Host) == 0 {
+		return nil, 0, "", fmt.Errorf("route.Spec.Host is empty, cannot test route")
+	}
+	request, err := http.NewRequest("GET", "https://"+route.Spec.Host, nil)
+	if err != nil {
+		return nil, 0, "", fmt.Errorf("error creating HTTPS request %v: %v", request, err)
+	}
+
+	// Send the HTTPS request
+	timeout, _ := time.ParseDuration("10s")
+	client := &http.Client{
+		Timeout: timeout,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	// Get the response
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, 0, "", fmt.Errorf("error receiving HTTP response %v", err)
+	}
+	// Close response body
+	defer response.Body.Close()
+
+	// Read response body
+	bodyBytes, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return response.Header, response.StatusCode, "", fmt.Errorf("error reading HTTP response body: %v", err)
+	}
+	body := string(bodyBytes)
+
+	return response.Header, response.StatusCode, body, nil
+}