Skip to content
Permalink
Browse files

Add an ENV to control ipv6 behavior in the router

This patch adds an environent variable called ROUTER_IP_V4_V6_MODE
which must be set to "v4" (the default), "v6", or "v4v6" to control
whether the router to binds to IPv4, IPv6, or both.

Note that when set to 'v6' or 'v4v6', the X-Forwarded-For and
Forwarded http headers will be in IPv6 form (even when the connection
was an IPv4 one, if set to 'v4v6').

Fixes bug 1471255 (https://bugzilla.redhat.com/show_bug.cgi?id=1471255)
  • Loading branch information...
knobunc committed Jul 16, 2017
1 parent 271fb1c commit 7821bd5070fb167dfe455d017d338e22673d4cf3
@@ -6,6 +6,8 @@
{{- define "/var/lib/haproxy/conf/haproxy.config" }}
{{- $workingDir := .WorkingDir }}
{{- $defaultDestinationCA := .DefaultDestinationCA }}
{{- $router_ip_v4_v6_mode := env "ROUTER_IP_V4_V6_MODE" "v4" }}


{{/* A bunch of regular expressions. Each should be wrapped in (?:) so that it is safe to include bare */}}
{{/* quadPattern: Match a quad in an IP address; e.g. 123 */}}
@@ -163,7 +165,14 @@ listen stats :1936

{{ if .BindPorts -}}
frontend public
bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80" }} v4v6 {{ if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }}
{{ if eq "v4v6" $router_ip_v4_v6_mode }}
bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} v4v6
{{- else if eq "v6" $router_ip_v4_v6_mode }}
bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} v6only
{{- else }}
bind :{{env "ROUTER_SERVICE_HTTP_PORT" "80"}}
{{- end }}
{{- if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }}
mode http
tcp-request inspect-delay 5s
tcp-request content accept if HTTP
@@ -199,7 +208,14 @@ frontend public
# determined by the next backend in the chain which may be an app backend (passthrough termination) or a backend
# that terminates encryption in this router (edge)
frontend public_ssl
bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v4v6 {{ if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }}
{{ if eq "v4v6" $router_ip_v4_v6_mode }}
bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v4v6
{{- else if eq "v6" $router_ip_v4_v6_mode }}
bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v6only
{{- else }}
bind :{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}}
{{- end }}
{{- if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }}
tcp-request inspect-delay 5s
tcp-request content accept if { req_ssl_hello_type 1 }

@@ -394,7 +410,13 @@ backend be_secure:{{$cfgIdx}}
http-request set-header X-Forwarded-Port %[dst_port]
http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
http-request set-header X-Forwarded-Proto https if { ssl_fc }
{{- if matchPattern "v6" $router_ip_v4_v6_mode }}
# See the quoting rules in https://tools.ietf.org/html/rfc7239 for IPv6 addresses (v4 addresses get translated to v6 when in hybrid mode)
http-request set-header Forwarded for="[%[src]]";host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)]
{{- else }}
http-request set-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)]
{{- end }}

{{- if not (matchPattern "true|TRUE" (index $cfg.Annotations "haproxy.router.openshift.io/disable_cookies")) }}
cookie {{$cfg.RoutingKeyName}} insert indirect nocache httponly
{{- if and (or (eq $cfg.TLSTermination "edge") (eq $cfg.TLSTermination "reencrypt")) (ne $cfg.InsecureEdgeTerminationPolicy "Allow") }} secure
@@ -0,0 +1,132 @@
package images

import (
"bufio"
"fmt"
"net/http"
"strings"
"time"

g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"

kapierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
e2e "k8s.io/kubernetes/test/e2e/framework"

exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("[Conformance][networking][router] router headers", func() {
defer g.GinkgoRecover()
var (
configPath = exutil.FixturePath("testdata", "router-http-echo-server.yaml")
oc = exutil.NewCLI("router-headers", exutil.KubeConfigPath())

routerIP, routerNs string
)

g.BeforeEach(func() {
svc, err := oc.AdminKubeClient().Core().Services("default").Get("router", metav1.GetOptions{})
if kapierrs.IsNotFound(err) {
g.Skip("no router installed on the cluster")
return
}
o.Expect(err).NotTo(o.HaveOccurred())
routerIP = svc.Spec.ClusterIP
routerNs = oc.KubeFramework().Namespace.Name
})

g.Describe("The HAProxy router", func() {
g.It("should set Forwarded headers appropriately", func() {
defer func() {
// This should be done if the test fails but
// for now always dump the logs.
// if g.CurrentGinkgoTestDescription().Failed
dumpRouterHeadersLogs(oc, g.CurrentGinkgoTestDescription().FullTestText)
}()
oc.SetOutputDir(exutil.TestContext.OutputDir)
ns := oc.KubeFramework().Namespace.Name
execPodName := exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")
defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, metav1.NewDeleteOptions(1)) }()

g.By(fmt.Sprintf("creating an http echo server from a config file %q", configPath))

err := oc.Run("create").Args("-f", configPath).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

var clientIP string
err = wait.Poll(time.Second, changeTimeoutSeconds*time.Second, func() (bool, error) {
pod, err := oc.KubeFramework().ClientSet.Core().Pods(ns).Get("execpod", metav1.GetOptions{})
if err != nil {
return false, err
}
if len(pod.Status.PodIP) == 0 {
return false, nil
}

clientIP = pod.Status.PodIP
return true, nil
})
o.Expect(err).NotTo(o.HaveOccurred())

// router expected to listen on port 80
routerURL := fmt.Sprintf("http://%s", routerIP)

g.By("waiting for the healthz endpoint to respond")
healthzURI := fmt.Sprintf("http://%s:1936/healthz", routerIP)
err = waitForRouterOKResponseExec(ns, execPodName, healthzURI, routerIP, changeTimeoutSeconds)
o.Expect(err).NotTo(o.HaveOccurred())

host := "router-headers.example.com"
g.By(fmt.Sprintf("waiting for the route to become active"))
err = waitForRouterOKResponseExec(ns, execPodName, routerURL, host, changeTimeoutSeconds)
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("making a request and reading back the echoed headers"))
var payload string
payload, err = getRoutePayloadExec(ns, execPodName, routerURL, host)
o.Expect(err).NotTo(o.HaveOccurred())

// The trailing \n is being stripped, so add it back
payload = payload + "\n"

// parse the echoed request
reader := bufio.NewReader(strings.NewReader(payload))
req, err := http.ReadRequest(reader)
o.Expect(err).NotTo(o.HaveOccurred())

// check that the header is what we expect
g.By(fmt.Sprintf("inspecting the echoed headers"))
ffHeader := req.Header.Get("X-Forwarded-For")
if ffHeader != clientIP {
e2e.Failf("Unexpected header: '%s' (expected %s); All headers: %#v", ffHeader, clientIP, req.Header)
}
})
})
})

func dumpRouterHeadersLogs(oc *exutil.CLI, name string) {
log, _ := e2e.GetPodLogs(oc.AdminKubeClient(), oc.KubeFramework().Namespace.Name, "router-headers", "router")
e2e.Logf("Weighted Router test %s logs:\n %s", name, log)
}

func getRoutePayloadExec(ns, execPodName, url, host string) (string, error) {
cmd := fmt.Sprintf(`
set -e
payload=$( curl -s --header 'Host: %s' %q ) || rc=$?
if [[ "${rc:-0}" -eq 0 ]]; then
printf "${payload}"
exit 0
else
echo "error ${rc}" 1>&2
exit 1
fi
`, host, url)
output, err := e2e.RunHostCmd(ns, execPodName, cmd)
if err != nil {
return "", fmt.Errorf("host command failed: %v\n%s", err, output)
}
return output, nil
}

Some generated files are not rendered by default. Learn more.

@@ -0,0 +1,56 @@
apiVersion: v1
kind: List
metadata: {}
items:
- apiVersion: v1
kind: DeploymentConfig
metadata:
name: router-http-echo
spec:
replicas: 1
selector:
app: router-http-echo
deploymentconfig: router-http-echo
strategy:
type: Rolling
template:
metadata:
labels:
app: router-http-echo
deploymentconfig: router-http-echo
spec:
containers:
- image: openshift/origin-base
name: router-http-echo
command:
- /usr/bin/socat
- TCP4-LISTEN:8676,reuseaddr,fork
- EXEC:'perl -e \"print qq(HTTP/1.0 200 OK\r\n\r\n); while (<>) { print; last if /^\r/}\"'
ports:
- containerPort: 8676
protocol: TCP
dnsPolicy: ClusterFirst
restartPolicy: Always
securityContext: {}
- apiVersion: v1
kind: Service
metadata:
name: router-http-echo
labels:
app: router-http-echo
spec:
selector:
app: router-http-echo
ports:
- port: 8676
name: router-http-echo
protocol: TCP
- apiVersion: v1
kind: Route
metadata:
name: router-http-echo
spec:
host: router-headers.example.com
to:
kind: Service
name: router-http-echo

0 comments on commit 7821bd5

Please sign in to comment.
You can’t perform that action at this time.