Apply annotation to service and add e2e tests

voyagermesh · Nov 13, 2017 · 2afc49f · 2afc49f
1 parent e872862
commit 2afc49f
Show file tree

Hide file tree

Showing 10 changed files with 197 additions and 66 deletions.
diff --git a/apis/voyager/v1beta1/annotations.go b/apis/voyager/v1beta1/annotations.go
@@ -84,19 +84,9 @@ const (
 	AcceptProxy = EngressKey + "/" + "accept-proxy"
 
 	// Enforces use of the PROXY protocol over any connection established to this server.
+	// Possible values are "v1", "v2", "v2-ssl" and "v2-ssl-cn"
 	SendProxy = EngressKey + "/" + "send-proxy"
 
-	// Enforces use of the PROXYv2 protocol over any connection established to this server.
-	SendProxyV2 = EngressKey + "/" + "send-proxy-v2"
-
-	// Enforces use of the PROXYv2 protocol over any connection established to this server,
-	// sending SSL Certificate information.
-	SendProxyV2SSL = EngressKey + "/" + "send-proxy-v2-ssl"
-
-	// Enforces use of the PROXYv2 protocol over any connection established to this server,
-	// sending SSL Certificate information and, if present, the client's certficate CN.
-	SendProxyV2SSLCN = EngressKey + "/" + "send-proxy-v2-ssl-cn"
-
 	// Annotations applied to resources offshoot from an ingress
 	OriginAPISchema = EngressKey + "/" + "origin-api-schema" // APISchema = {APIGroup}/{APIVersion}
 	OriginName      = EngressKey + "/" + "origin-name"
@@ -263,6 +253,26 @@ const (
 	ACMEServerURL        = "ACME_SERVER_URL"
 )
 
+type ProxyProtocolVersion string
+
+const (
+	proxyProtocolV1      ProxyProtocolVersion = "v1"
+	proxyProtocolV2      ProxyProtocolVersion = "v2"
+	proxyProtocolV2SSL   ProxyProtocolVersion = "v2-ssl"
+	proxyProtocolV2SSLCN ProxyProtocolVersion = "v2-ssl-cn"
+)
+
+func ProxyProtocolCommand(version string) string {
+	switch ProxyProtocolVersion(version) {
+	case proxyProtocolV1:
+		return "send-proxy"
+	case proxyProtocolV2, proxyProtocolV2SSL, proxyProtocolV2SSLCN:
+		return "send-proxy-" + version
+	default:
+		return ""
+	}
+}
+
 func (r Ingress) OffshootName() string {
 	return VoyagerPrefix + r.Name
 }
@@ -486,26 +496,6 @@ func (r Ingress) AcceptProxy() bool {
 	return v
 }
 
-func (r Ingress) SendProxy() bool {
-	v, _ := GetBool(r.Annotations, SendProxy)
-	return v
-}
-
-func (r Ingress) SendProxyV2() bool {
-	v, _ := GetBool(r.Annotations, SendProxyV2)
-	return v
-}
-
-func (r Ingress) SendProxyV2SSL() bool {
-	v, _ := GetBool(r.Annotations, SendProxyV2SSL)
-	return v
-}
-
-func (r Ingress) SendProxyV2SSLCN() bool {
-	v, _ := GetBool(r.Annotations, SendProxyV2SSLCN)
-	return v
-}
-
 var timeoutDefaults = map[string]string{
 	// Maximum time to wait for a connection attempt to a server to succeed.
 	"connect": "50s",

diff --git a/hack/docker/voyager/templates/default-backend.cfg b/hack/docker/voyager/templates/default-backend.cfg
@@ -23,13 +23,13 @@ backend {{ .DefaultBackend.Name }}
 	{{- range $e := .DefaultBackend.Endpoints }}
 	{{- if $e.ExternalName }}
 	{{- if $e.UseDNSResolver }}
-	server {{ $e.Name }} {{ $e.ExternalName }}:{{ $e.Port -}} {{ if $e.DNSResolver }} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} resolvers {{ $e.DNSResolver }} resolve-prefer ipv4 {{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if .SendProxy }}send-proxy{{ end }} {{ if .SendProxyV2 }}send-proxy-v2{{ end }} {{ if .SendProxyV2SSL }}send-proxy-v2-ssl{{ end }} {{ if .SendProxyV2SSLCN }}send-proxy-v2-ssl-cn{{ end }}
+	server {{ $e.Name }} {{ $e.ExternalName }}:{{ $e.Port -}} {{ if $e.DNSResolver }} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} resolvers {{ $e.DNSResolver }} resolve-prefer ipv4 {{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.SendProxy }}{{ $e.SendProxy }}{{ end }}
 	{{- else if not $.DefaultBackend.BackendRules }}
 	acl https ssl_fc
 	http-request redirect location https://{{$e.ExternalName}}:{{ $e.Port }} code 301 if https
 	http-request redirect location http://{{$e.ExternalName}}:{{ $e.Port }} code 301 unless https
 	{{ end -}}
 	{{- else }}
-	server {{ $e.Name }} {{ $e.IP }}:{{ $e.Port -}} {{ if $e.MaxConnections }} maxconn {{ $e.MaxConnections }} {{ end -}} {{ if $e.Weight }} weight {{ $e.Weight }}{{ end -}} {{ if $.DefaultBackend.Sticky }} cookie {{ $e.Name }}{{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} {{ if .SendProxy }}send-proxy{{ end }} {{ if .SendProxyV2 }}send-proxy-v2{{ end }} {{ if .SendProxyV2SSL }}send-proxy-v2-ssl{{ end }} {{ if .SendProxyV2SSLCN }}send-proxy-v2-ssl-cn{{ end }}
+	server {{ $e.Name }} {{ $e.IP }}:{{ $e.Port -}} {{ if $e.MaxConnections }} maxconn {{ $e.MaxConnections }} {{ end -}} {{ if $e.Weight }} weight {{ $e.Weight }}{{ end -}} {{ if $.DefaultBackend.Sticky }} cookie {{ $e.Name }}{{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} {{ if $e.SendProxy }}{{ $e.SendProxy }}{{ end }}
 	{{ end -}}
 	{{ end -}}
diff --git a/hack/docker/voyager/templates/http-backend.cfg b/hack/docker/voyager/templates/http-backend.cfg
@@ -25,12 +25,12 @@ backend {{ $path.Backend.Name }}
 	{{- range $index, $e := $path.Backend.Endpoints }}
 	{{- if $e.ExternalName }}
 	{{- if $e.UseDNSResolver }}
-	server {{ $e.Name }} {{ $e.ExternalName }}:{{ $e.Port -}} {{ if $e.DNSResolver }} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} resolvers {{ $e.DNSResolver }} resolve-prefer ipv4 {{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if .SendProxy }}send-proxy{{ end }} {{ if .SendProxyV2 }}send-proxy-v2{{ end }} {{ if .SendProxyV2SSL }}send-proxy-v2-ssl{{ end }} {{ if .SendProxyV2SSLCN }}send-proxy-v2-ssl-cn{{ end }}
+	server {{ $e.Name }} {{ $e.ExternalName }}:{{ $e.Port -}} {{ if $e.DNSResolver }} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} resolvers {{ $e.DNSResolver }} resolve-prefer ipv4 {{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.SendProxy }}{{ $e.SendProxy }}{{ end }}
 	{{- else if not $path.Backend.BackendRules }}
 	http-request redirect location {{ if $.OffloadSSL }}https://{{ else }}http://{{ end }}{{$e.ExternalName}}:{{ $e.Port }} code 301
 	{{- end }}
 	{{- else }}
-	server {{ $e.Name }} {{ $e.IP }}:{{ $e.Port -}} {{ if $e.MaxConnections }} maxconn {{ $e.MaxConnections }} {{ end -}} {{ if $e.Weight }} weight {{ $e.Weight }} {{ end -}} {{ if $path.Backend.Sticky }} cookie {{ backend_hash $e.Name $index $path.Backend.StickyCookieHash }} {{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} {{ if .SendProxy }}send-proxy{{ end }} {{ if .SendProxyV2 }}send-proxy-v2{{ end }} {{ if .SendProxyV2SSL }}send-proxy-v2-ssl{{ end }} {{ if .SendProxyV2SSLCN }}send-proxy-v2-ssl-cn{{ end }}
+	server {{ $e.Name }} {{ $e.IP }}:{{ $e.Port -}} {{ if $e.MaxConnections }} maxconn {{ $e.MaxConnections }} {{ end -}} {{ if $e.Weight }} weight {{ $e.Weight }} {{ end -}} {{ if $path.Backend.Sticky }} cookie {{ backend_hash $e.Name $index $path.Backend.StickyCookieHash }} {{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} {{ if $e.SendProxy }}{{ $e.SendProxy }}{{ end }}
 	{{ end -}}
 	{{ end }}
 {{ end -}}

diff --git a/hack/docker/voyager/templates/tcp-backend.cfg b/hack/docker/voyager/templates/tcp-backend.cfg
@@ -12,8 +12,8 @@ backend {{ .Backend.Name }}
 
 	{{- range $e := .Backend.Endpoints }}
 	{{- if $e.ExternalName }}
-	server {{ $e.Name }} {{ $e.ExternalName }}:{{ $e.Port -}} {{ if $e.DNSResolver }} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} resolvers {{ $e.DNSResolver }} resolve-prefer ipv4{{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if .SendProxy }}send-proxy{{ end }} {{ if .SendProxyV2 }}send-proxy-v2{{ end }} {{ if .SendProxyV2SSL }}send-proxy-v2-ssl{{ end }} {{ if .SendProxyV2SSLCN }}send-proxy-v2-ssl-cn{{ end }}
+	server {{ $e.Name }} {{ $e.ExternalName }}:{{ $e.Port -}} {{ if $e.DNSResolver }} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} resolvers {{ $e.DNSResolver }} resolve-prefer ipv4{{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.SendProxy }}{{ $e.SendProxy }}{{ end }}
 	{{- else }}
-	server {{ $e.Name }} {{ $e.IP }}:{{ $e.Port -}} {{ if $e.MaxConnections }} maxconn {{ $e.MaxConnections }} {{ end -}} {{ if $e.Weight }} weight {{ $e.Weight }}{{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} {{ if .SendProxy }}send-proxy{{ end }} {{ if .SendProxyV2 }}send-proxy-v2{{ end }} {{ if .SendProxyV2SSL }}send-proxy-v2-ssl{{ end }} {{ if .SendProxyV2SSLCN }}send-proxy-v2-ssl-cn{{ end }}
+	server {{ $e.Name }} {{ $e.IP }}:{{ $e.Port -}} {{ if $e.MaxConnections }} maxconn {{ $e.MaxConnections }} {{ end -}} {{ if $e.Weight }} weight {{ $e.Weight }}{{ end -}} {{ if $e.TLSOption }} {{ $e.TLSOption }} {{ end -}} {{ if $e.CheckHealth }} check {{- if $e.CheckHealthPort }} port {{ $e.CheckHealthPort }} {{- end }} {{- end }} {{ if $e.SendProxy }}{{ $e.SendProxy }}{{ end }}
 	{{ end -}}
 	{{ end -}}
diff --git a/pkg/haproxy/types.go b/pkg/haproxy/types.go
@@ -40,10 +40,6 @@ type SharedInfo struct {
 	MaxConnections        int
 	ForceMatchServicePort bool
 	Limit                 *Limit
-	SendProxy             bool
-	SendProxyV2           bool
-	SendProxyV2SSL        bool
-	SendProxyV2SSLCN      bool
 	SSLRedirect           bool
 }
 
@@ -141,8 +137,8 @@ type Endpoint struct {
 	DNSResolver     string
 	CheckHealth     bool
 	CheckHealthPort string
-
-	TLSOption string
+	TLSOption       string
+	SendProxy       string
 }
 
 type UserList struct {

diff --git a/pkg/ingress/parser.go b/pkg/ingress/parser.go
@@ -147,6 +147,7 @@ func (c *controller) getEndpoints(svc *core.Service, servicePort *core.ServicePo
 							ep.CheckHealth = true
 							ep.CheckHealthPort = svc.Annotations[api.CheckHealthPort]
 						}
+						ep.SendProxy = api.ProxyProtocolCommand(svc.Annotations[api.SendProxy])
 					}
 
 					eps = append(eps, ep)
@@ -247,17 +248,6 @@ func (c *controller) generateConfig() error {
 		si.AcceptProxy = true
 	}
 
-	if c.Ingress.SendProxyV2SSLCN() {
-		si.SendProxyV2SSLCN = true
-	} else if c.Ingress.SendProxyV2SSL() {
-		si.SendProxyV2SSL = true
-	} else if c.Ingress.SendProxyV2() {
-		si.SendProxyV2 = true
-	} else if c.Ingress.SendProxy() {
-		si.SendProxy = true
-	}
-
-
 	userLists := make(map[string]haproxy.UserList)
 	var globalBasic *haproxy.BasicAuth
 	if c.Ingress.BasicAuthEnabled() {

diff --git a/test/e2e/ingress_ops.go b/test/e2e/ingress_ops.go
@@ -1015,7 +1015,7 @@ var _ = Describe("IngressOperations", func() {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(eps)).Should(BeNumerically(">=", 1))
 
-				err = f.Ingress.DoHTTP(5, "", ing, eps, "GET", "/testpath/ok", func(r *testserverclient.Response) bool {
+				err = f.Ingress.DoHTTP(5, "", ing, eps, "GET", "/testpath/ok", func(r *client.Response) bool {
 					return Expect(r.Status).Should(Equal(http.StatusOK)) &&
 						Expect(r.Method).Should(Equal("GET")) &&
 						Expect(r.Path).Should(Equal("/testpath/ok"))
@@ -1061,7 +1061,7 @@ var _ = Describe("IngressOperations", func() {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(eps)).Should(BeNumerically(">=", 1))
 
-				err = f.Ingress.DoHTTP(5, "", ing, eps, "GET", "/testpath/ok", func(r *testserverclient.Response) bool {
+				err = f.Ingress.DoHTTP(5, "", ing, eps, "GET", "/testpath/ok", func(r *client.Response) bool {
 					return Expect(r.Status).Should(Equal(http.StatusOK)) &&
 						Expect(r.Method).Should(Equal("GET")) &&
 						Expect(r.Path).Should(Equal("/testpath/ok"))
@@ -1110,7 +1110,7 @@ var _ = Describe("IngressOperations", func() {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(eps)).Should(BeNumerically(">=", 1))
 
-				err = f.Ingress.DoHTTP(5, "", ing, eps, "GET", "/testpath/ok", func(r *testserverclient.Response) bool {
+				err = f.Ingress.DoHTTP(5, "", ing, eps, "GET", "/testpath/ok", func(r *client.Response) bool {
 					return Expect(r.Status).Should(Equal(http.StatusOK)) &&
 						Expect(r.Method).Should(Equal("GET")) &&
 						Expect(r.Path).Should(Equal("/testpath/ok"))
@@ -1159,7 +1159,7 @@ var _ = Describe("IngressOperations", func() {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(eps)).Should(BeNumerically(">=", 1))
 
-				err = f.Ingress.DoHTTPStatus(5, ing, eps, "GET", "/testpath/ok", func(r *testserverclient.Response) bool {
+				err = f.Ingress.DoHTTPStatus(5, ing, eps, "GET", "/testpath/ok", func(r *client.Response) bool {
 					return Expect(r.Status).Should(Equal(http.StatusServiceUnavailable))
 				})
 				Expect(err).NotTo(HaveOccurred())
@@ -1205,7 +1205,7 @@ var _ = Describe("IngressOperations", func() {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(eps)).Should(BeNumerically(">=", 1))
 
-				err = f.Ingress.DoHTTPStatus(5, ing, eps, "GET", "/testpath/ok", func(r *testserverclient.Response) bool {
+				err = f.Ingress.DoHTTPStatus(5, ing, eps, "GET", "/testpath/ok", func(r *client.Response) bool {
 					return Expect(r.Status).Should(Equal(http.StatusServiceUnavailable))
 				})
 				Expect(err).NotTo(HaveOccurred())

diff --git a/test/e2e/ingress_tcp.go b/test/e2e/ingress_tcp.go
@@ -253,4 +253,80 @@ var _ = Describe("IngressTCP", func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 	})
+
+	Describe("Proxy Protocol", func() {
+		Describe("With version 1", func() {
+			BeforeEach(func() {
+				meta, err := f.Ingress.CreateResourceWithSendProxy("v1")
+				Expect(err).NotTo(HaveOccurred())
+				ing.Spec.Rules = []api.IngressRule{
+					{
+						IngressRuleValue: api.IngressRuleValue{
+							TCP: &api.TCPIngressRuleValue{
+								Port: intstr.FromInt(4001),
+								Backend: api.IngressBackend{
+									ServiceName: meta.Name,
+									ServicePort: intstr.FromInt(6767),
+								},
+							},
+						},
+					},
+				}
+			})
+			It("Should test decoded proxy-protocol header", func() {
+				By("Getting HTTP endpoints")
+				eps, err := f.Ingress.GetHTTPEndpoints(ing)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(eps)).Should(BeNumerically(">=", 1))
+
+				svc, err := f.Ingress.GetOffShootService(ing)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(svc.Spec.Ports)).Should(Equal(1))
+				Expect(svc.Spec.Ports[0].Port).To(Equal(int32(4001)))
+
+				By("Checking tcp response")
+				err = f.Ingress.DoTCP(framework.NoRetry, ing, eps, func(r *client.Response) bool {
+					return r.Proxy.Version == 1
+				})
+				Expect(err).NotTo(HaveOccurred())
+			})
+		})
+
+		Describe("With version 2", func() {
+			BeforeEach(func() {
+				meta, err := f.Ingress.CreateResourceWithSendProxy("v2")
+				Expect(err).NotTo(HaveOccurred())
+				ing.Spec.Rules = []api.IngressRule{
+					{
+						IngressRuleValue: api.IngressRuleValue{
+							TCP: &api.TCPIngressRuleValue{
+								Port: intstr.FromInt(4001),
+								Backend: api.IngressBackend{
+									ServiceName: meta.Name,
+									ServicePort: intstr.FromInt(6767),
+								},
+							},
+						},
+					},
+				}
+			})
+			It("Should test decoded proxy-protocol header", func() {
+				By("Getting HTTP endpoints")
+				eps, err := f.Ingress.GetHTTPEndpoints(ing)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(eps)).Should(BeNumerically(">=", 1))
+
+				svc, err := f.Ingress.GetOffShootService(ing)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(svc.Spec.Ports)).Should(Equal(1))
+				Expect(svc.Spec.Ports[0].Port).To(Equal(int32(4001)))
+
+				By("Checking tcp response")
+				err = f.Ingress.DoTCP(framework.NoRetry, ing, eps, func(r *client.Response) bool {
+					return r.Proxy.Version == 2
+				})
+				Expect(err).NotTo(HaveOccurred())
+			})
+		})
+	})
 })
diff --git a/test/framework/ingress_suite.go b/test/framework/ingress_suite.go
@@ -20,7 +20,7 @@ import (
 )
 
 const (
-	testServerImage = "appscode/test-server:2.2"
+	testServerImage = "appscode/test-server:2.3"
 )
 
 var (