From 3b6806c2bf1169a8873899349c232814bd3cf542 Mon Sep 17 00:00:00 2001 From: sadlil Date: Wed, 3 May 2017 16:11:48 +0600 Subject: [PATCH 1/4] Ingress Hostname based traffic forwarding --- hack/make.py | 6 +- pkg/controller/ingress/parser.go | 37 +++-- pkg/controller/ingress/parser_test.go | 28 ++++ test/e2e/ingress.go | 106 ++++++++++++++ test/e2e/test_ingress_values.go | 132 +++++++++++++++++- test/test-server/server.go | 2 + test/test-server/setup.sh | 2 +- test/test-server/testserverclient/http.go | 1 + test/test-server/yaml/controller.yaml | 5 + .../appscode/k8s-addons/api/voyager_types.go | 20 +++ 10 files changed, 321 insertions(+), 18 deletions(-) diff --git a/hack/make.py b/hack/make.py index 46275e8e1..7f465da20 100755 --- a/hack/make.py +++ b/hack/make.py @@ -163,7 +163,7 @@ def test(type, *args): elif type == 'clean': e2e_test_clean() else: - print '{test unit|minikube|e2e|clean}' + print '{test unit|minikube|e2e}' def unit_test(): die(call(libbuild.GOC + ' test -v ./cmd/... ./pkg/... -args -v=3 -verbose=true -mode=unit')) @@ -180,10 +180,6 @@ def integration(args): st = ' '.join(args) die(call(libbuild.GOC + ' test -v ./test/integration/... -timeout 10h -args -v=3 -verbose=true -mode=e2e -in-cluster=true ' + st)) -def e2e_test_clean(): - die(call('./test/hack/cleanup.sh')) - - def default(): gen() fmt() diff --git a/pkg/controller/ingress/parser.go b/pkg/controller/ingress/parser.go index e440abf9d..8ca288916 100644 --- a/pkg/controller/ingress/parser.go +++ b/pkg/controller/ingress/parser.go @@ -27,7 +27,7 @@ func (lbc *EngressController) parse() error { return nil } -func (lbc *EngressController) serviceEndpoints(name string, port intstr.IntOrString) ([]*Endpoint, error) { +func (lbc *EngressController) serviceEndpoints(name string, port intstr.IntOrString, hostNames []string) ([]*Endpoint, error) { log.Infoln("getting endpoints for ", lbc.Config.Namespace, name, "port", port) // the following lines giving support to @@ -48,10 +48,10 @@ func (lbc *EngressController) serviceEndpoints(name string, port intstr.IntOrStr if !ok { return nil, errors.New().WithMessage("service port unavaiable").NotFound() } - return lbc.getEndpoints(service, p) + return lbc.getEndpoints(service, p, hostNames) } -func (lbc *EngressController) getEndpoints(s *kapi.Service, servicePort *kapi.ServicePort) (eps []*Endpoint, err error) { +func (lbc *EngressController) getEndpoints(s *kapi.Service, servicePort *kapi.ServicePort, hostNames []string) (eps []*Endpoint, err error) { ep, err := lbc.EndpointStore.GetServiceEndpoints(s) if err != nil { return nil, errors.New().WithCause(err).Internal() @@ -86,17 +86,32 @@ func (lbc *EngressController) getEndpoints(s *kapi.Service, servicePort *kapi.Se log.Infoln("targert port", targetPort) for _, epAddress := range ss.Addresses { - eps = append(eps, &Endpoint{ - Name: "server-" + epAddress.IP, - IP: epAddress.IP, - Port: targetPort, - }) + if isIncludeAbleAddress(hostNames, epAddress.Hostname) { + eps = append(eps, &Endpoint{ + Name: "server-" + epAddress.IP, + IP: epAddress.IP, + Port: targetPort, + }) + } } } } return } +func isIncludeAbleAddress(hostNames []string, hostName string) bool { + if len(hostNames) <= 0 { + return true + } + + for _, name := range hostNames { + if strings.EqualFold(name, hostName) { + return true + } + } + return false +} + func (lbc *EngressController) generateTemplate() error { log.Infoln("Generating Ingress template.") ctx, err := Context(lbc.Parsed) @@ -154,7 +169,7 @@ func (lbc *EngressController) parseSpec() { lbc.Options.Ports = make([]int, 0) if lbc.Config.Spec.Backend != nil { log.Debugln("generating defulat backend", lbc.Config.Spec.Backend.RewriteRule, lbc.Config.Spec.Backend.HeaderRule) - eps, _ := lbc.serviceEndpoints(lbc.Config.Spec.Backend.ServiceName, lbc.Config.Spec.Backend.ServicePort) + eps, _ := lbc.serviceEndpoints(lbc.Config.Spec.Backend.ServiceName, lbc.Config.Spec.Backend.ServicePort, lbc.Config.Spec.Backend.HostNames) lbc.Parsed.DefaultBackend = &Backend{ Name: "default-backend", Endpoints: eps, @@ -185,7 +200,7 @@ func (lbc *EngressController) parseSpec() { AclMatch: svc.Path, } - eps, err := lbc.serviceEndpoints(svc.Backend.ServiceName, svc.Backend.ServicePort) + eps, err := lbc.serviceEndpoints(svc.Backend.ServiceName, svc.Backend.ServicePort, svc.Backend.HostNames) def.Backends = &Backend{ Name: "backend-" + rand.Characters(5), Endpoints: eps, @@ -216,7 +231,7 @@ func (lbc *EngressController) parseSpec() { ALPNOptions: parseALPNOptions(tcpSvc.ALPN), } log.Infoln(tcpSvc.Backend.ServiceName, tcpSvc.Backend.ServicePort) - eps, err := lbc.serviceEndpoints(tcpSvc.Backend.ServiceName, tcpSvc.Backend.ServicePort) + eps, err := lbc.serviceEndpoints(tcpSvc.Backend.ServiceName, tcpSvc.Backend.ServicePort, tcpSvc.Backend.HostNames) def.Backends = &Backend{ Name: "backend-" + rand.Characters(5), Endpoints: eps, diff --git a/pkg/controller/ingress/parser_test.go b/pkg/controller/ingress/parser_test.go index 19829837f..b307463f2 100644 --- a/pkg/controller/ingress/parser_test.go +++ b/pkg/controller/ingress/parser_test.go @@ -43,3 +43,31 @@ func TestALPNOptions(t *testing.T) { assert.Equal(t, k, parseALPNOptions(v)) } } + +func TestIsIncludeAbleOptions(t *testing.T) { + dataTable := map[string]map[bool][]string{ + "web-0": { + true: { + "web", + "web-0", + }, + }, + + "web-1": { + true: {}, + }, + + "web-2": { + false: { + "web", + "web-0", + }, + }, + } + + for k, v := range dataTable { + for key, val := range v { + assert.Equal(t, key, isIncludeAbleAddress(val, k)) + } + } +} diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 61a3e07c0..ad342f5b7 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -888,3 +888,109 @@ func (ing *IngressTestSuit) TestIngressCoreIngress() error { } return nil } + +func (ing *IngressTestSuit) TestIngressHostNames() error { + headlessSvc, err := ing.t.KubeClient.Core().Services("default").Create(testStatefulSetSvc) + if err != nil { + return err + } + defer func() { + if ing.t.Config.Cleanup { + ing.t.KubeClient.Core().Services("default").Delete(headlessSvc.Name, nil) + } + }() + + ss, err := ing.t.KubeClient.Apps().StatefulSets("default").Create(testServerStatefulSet) + if err != nil { + return err + } + defer func() { + if ing.t.Config.Cleanup { + ing.t.KubeClient.Apps().StatefulSets("default").Delete(ss.Name, nil) + } + }() + + baseIngress := &aci.Ingress{ + ObjectMeta: api.ObjectMeta{ + Name: testIngressName(), + Namespace: TestNamespace, + }, + Spec: aci.ExtendedIngressSpec{ + Rules: []aci.ExtendedIngressRule{ + { + ExtendedIngressRuleValue: aci.ExtendedIngressRuleValue{ + HTTP: &aci.HTTPExtendedIngressRuleValue{ + Paths: []aci.HTTPExtendedIngressPath{ + { + Path: "/testpath", + Backend: aci.ExtendedIngressBackend{ + HostNames: []string{testServerStatefulSet.Name + "-0"}, + ServiceName: headlessSvc.Name, + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err = ing.t.ExtensionClient.Ingress(baseIngress.Namespace).Create(baseIngress) + if err != nil { + return err + } + defer func() { + if ing.t.Config.Cleanup { + ing.t.ExtensionClient.Ingress(baseIngress.Namespace).Delete(baseIngress.Name) + } + }() + + // Wait sometime to loadbalancer be opened up. + time.Sleep(time.Second * 10) + var svc *api.Service + for i := 0; i < maxRetries; i++ { + svc, err = ing.t.KubeClient.Core().Services(baseIngress.Namespace).Get(ingress.VoyagerPrefix + baseIngress.Name) + if err == nil { + break + } + time.Sleep(time.Second * 5) + log.Infoln("Waiting for service to be created") + } + if err != nil { + return err + } + log.Infoln("Service Created for loadbalancer, Checking for service endpoints") + for i := 0; i < maxRetries; i++ { + _, err = ing.t.KubeClient.Core().Endpoints(svc.Namespace).Get(svc.Name) + if err == nil { + break + } + time.Sleep(time.Second * 5) + log.Infoln("Waiting for endpoints to be created") + } + if err != nil { + return err + } + + serverAddr, err := ing.getURLs(baseIngress) + if err != nil { + return err + } + time.Sleep(time.Second * 30) + log.Infoln("Loadbalancer created, calling http endpoints, Total", len(serverAddr)) + for _, url := range serverAddr { + resp, err := testserverclient.NewTestHTTPClient(url).Method("GET").Path("/testpath").DoWithRetry(50) + if err != nil { + return errors.New().WithCause(err).WithMessage("Failed to connect with server").Internal() + } + log.Infoln("Response", *resp) + if resp.Method != http.MethodGet { + return errors.New().WithMessage("Method did not matched").Internal() + } + if resp.PodName != ss.Name+"-0" { + return errors.New().WithMessage("PodName did not matched").Internal() + } + } + return nil +} diff --git a/test/e2e/test_ingress_values.go b/test/e2e/test_ingress_values.go index db6a4e790..b27ae5f54 100644 --- a/test/e2e/test_ingress_values.go +++ b/test/e2e/test_ingress_values.go @@ -2,6 +2,7 @@ package e2e import ( "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/util/intstr" ) @@ -81,7 +82,136 @@ var testServerRc = &api.ReplicationController{ Containers: []api.Container{ { Name: "server", - Image: "appscode/test-server:1.0", + Image: "appscode/test-server:1.1", + Env: []api.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &api.EnvVarSource{ + FieldRef: &api.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + Ports: []api.ContainerPort{ + { + Name: "http-1", + ContainerPort: 8080, + }, + { + Name: "http-2", + ContainerPort: 8989, + }, + { + Name: "http-3", + ContainerPort: 9090, + }, + { + Name: "tcp-1", + ContainerPort: 4343, + }, + { + Name: "tcp-2", + ContainerPort: 4545, + }, + { + Name: "tcp-3", + ContainerPort: 5656, + }, + }, + }, + }, + }, + }, + }, +} + +var testStatefulSetSvc = &api.Service{ + ObjectMeta: api.ObjectMeta{ + Name: "ss-svc", + Namespace: TestNamespace, + Labels: map[string]string{ + "app": "e2e-test", + }, + }, + Spec: api.ServiceSpec{ + ClusterIP: "None", + Ports: []api.ServicePort{ + { + Name: "http-1", + Port: 80, + TargetPort: intstr.FromInt(8080), + Protocol: "TCP", + }, + { + Name: "http-2", + Port: 8989, + TargetPort: intstr.FromInt(8989), + Protocol: "TCP", + }, + { + Name: "http-3", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: "TCP", + }, + { + Name: "tcp-1", + Port: 4343, + TargetPort: intstr.FromInt(4343), + Protocol: "TCP", + }, + { + Name: "tcp-2", + Port: 4545, + TargetPort: intstr.FromInt(4545), + Protocol: "TCP", + }, + { + Name: "tcp-3", + Port: 5656, + TargetPort: intstr.FromInt(5656), + Protocol: "TCP", + }, + }, + Selector: map[string]string{ + "app": "e2e-test", + }, + }, +} + +var testServerStatefulSet = &apps.StatefulSet{ + ObjectMeta: api.ObjectMeta{ + Name: "test-ss", + Namespace: TestNamespace, + Labels: map[string]string{ + "app": "e2e-test", + }, + }, + Spec: apps.StatefulSetSpec{ + Replicas: 3, + ServiceName: testStatefulSetSvc.Name, + Template: api.PodTemplateSpec{ + ObjectMeta: api.ObjectMeta{ + Labels: map[string]string{ + "app": "e2e-test", + }, + }, + Spec: api.PodSpec{ + Containers: []api.Container{ + { + Name: "server", + Image: "appscode/test-server:1.1", + Env: []api.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &api.EnvVarSource{ + FieldRef: &api.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, Ports: []api.ContainerPort{ { Name: "http-1", diff --git a/test/test-server/server.go b/test/test-server/server.go index 06b7ec7ec..76441b992 100644 --- a/test/test-server/server.go +++ b/test/test-server/server.go @@ -13,6 +13,7 @@ import ( type Response struct { Type string `json:"type,omitempty"` Host string `json:"host,omitempty"` + PodName string `json:"podName,omitempty"` ServerPort string `json:"serverPort,omitempty"` Path string `json:"path,omitempty"` Method string `json:"method,omitempty"` @@ -27,6 +28,7 @@ type HttpServerHandler struct { func (h HttpServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { resp := &Response{ Type: "http", + PodName: os.Getenv("POD_NAME"), Host: r.Host, ServerPort: h.port, Path: r.URL.Path, diff --git a/test/test-server/setup.sh b/test/test-server/setup.sh index f24b40e0f..f2e4c4f22 100755 --- a/test/test-server/setup.sh +++ b/test/test-server/setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERSION=1.0 +VERSION=1.1 build() { rm -rf dist/* diff --git a/test/test-server/testserverclient/http.go b/test/test-server/testserverclient/http.go index d4b914bc7..0165375a7 100644 --- a/test/test-server/testserverclient/http.go +++ b/test/test-server/testserverclient/http.go @@ -18,6 +18,7 @@ type httpClient struct { type Response struct { Status int `json:"-"` Type string `json:"type,omitempty"` + PodName string `json:"podName,omitempty"` Host string `json:"host,omitempty"` ServerPort string `json:"serverPort,omitempty"` Path string `json:"path,omitempty"` diff --git a/test/test-server/yaml/controller.yaml b/test/test-server/yaml/controller.yaml index c24051d73..20995c1bb 100644 --- a/test/test-server/yaml/controller.yaml +++ b/test/test-server/yaml/controller.yaml @@ -16,6 +16,11 @@ spec: containers: - name: server image: appscode/test-server:1.0 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: http-1 containerPort: 8080 diff --git a/vendor/github.com/appscode/k8s-addons/api/voyager_types.go b/vendor/github.com/appscode/k8s-addons/api/voyager_types.go index 0011804c9..9107ffeed 100644 --- a/vendor/github.com/appscode/k8s-addons/api/voyager_types.go +++ b/vendor/github.com/appscode/k8s-addons/api/voyager_types.go @@ -171,6 +171,15 @@ type HTTPExtendedIngressPath struct { } type IngressBackend struct { + // Host names to forward traffic to. If empty traffic will be + // forwarded to all subsets instance. + // If set only matched hosts will get the traffic. + // This is an handy way to send traffic to Specific + // StatefulSet pod. + // IE. Setting [web-0] will send traffic to only web-0 host + // for this StatefulSet, https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#creating-a-statefulset + HostNames []string `json:"hostNames,omitempty"` + // Specifies the name of the referenced service. ServiceName string `json:"serviceName,omitempty"` @@ -180,6 +189,17 @@ type IngressBackend struct { // ExtendedIngressBackend describes all endpoints for a given service and port. type ExtendedIngressBackend struct { + // TODO (@sadlil) Consider Embedding IngressBackend. + + // Host names to forward traffic to. If empty traffic will be + // forwarded to all subsets instance. + // If set only matched hosts will get the traffic. + // This is an handy way to send traffic to Specific + // StatefulSet pod. + // IE. Setting [web-0] will send traffic to only web-0 host + // for this StatefulSet, https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#creating-a-statefulset + HostNames []string `json:"hostNames,omitempty"` + // Specifies the name of the referenced service. ServiceName string `json:"serviceName,omitempty"` From 28dd3a116bf1050d9c89016b05ea755ad949bf02 Mon Sep 17 00:00:00 2001 From: sadlil Date: Wed, 3 May 2017 16:44:57 +0600 Subject: [PATCH 2/4] Update Doc --- docs/user-guide/component/ingress/README.md | 5 +- .../component/ingress/statefulset-pod.md | 100 ++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 docs/user-guide/component/ingress/statefulset-pod.md diff --git a/docs/user-guide/component/ingress/README.md b/docs/user-guide/component/ingress/README.md index 6876e37a8..eaa121920 100644 --- a/docs/user-guide/component/ingress/README.md +++ b/docs/user-guide/component/ingress/README.md @@ -25,7 +25,8 @@ hosting. This plugin also support configurable application ports with all the fe - [Cross namespace routing support](named-virtual-hosting.md), - [URL and Request Header Re-writing](header-rewrite.md), - [Wildcard Name based virtual hosting](named-virtual-hosting.md), - - Persistent sessions, Loadbalancer stats. + - Persistent sessions, Loadbalancer stats, + - [Route Traffic to StatefulSet Pods Based on Host Name](statefulset-pod.md) ### Comparison with Kubernetes | Feauture | Kube Ingress | AppsCode Ingress | @@ -38,6 +39,7 @@ hosting. This plugin also support configurable application ports with all the fe | URL and Header rewriting | :x: | :white_check_mark: | | Wildcard name virtual hosting | :x: | :white_check_mark: | | Loadbalancer statistics | :x: | :white_check_mark: | +| Route Traffic to StatefulSet Pods Based on Host Name | :x: | :white_check_mark: | ## AppsCode Ingress Flow Typically, services and pods have IPs only routable by the cluster network. All traffic that ends up at an @@ -126,6 +128,7 @@ same ingress resource. Learn more by reading the certificate doc. - [URL and Header Rewriting](header-rewrite.md) - [TCP Loadbalancing](tcp.md) - [TLS Termination](tls.md) +- [Route Traffic to StatefulSet Pods Based on Host Name](statefulset-pod.md) ## Example diff --git a/docs/user-guide/component/ingress/statefulset-pod.md b/docs/user-guide/component/ingress/statefulset-pod.md new file mode 100644 index 000000000..39c3e98cf --- /dev/null +++ b/docs/user-guide/component/ingress/statefulset-pod.md @@ -0,0 +1,100 @@ +### Forward Traffic to StatefulSet +There is the regular way to forward traffic to StatefulSet. Create a service with the pods label selector as +selector, and use the service name as Backend ServiceName. By following: + +``` +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: web +spec: + serviceName: "nginx-set" + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.8 + ports: + - containerPort: 80 + name: web +---- +apiVersion: v1 +kind: Service +metadata: + name: nginx-set + labels: + app: nginx +spec: + ports: + - port: 80 + name: web + clusterIP: None + selector: + app: nginx +``` + +Create another service for StatefulSets pods with selector. +``` +apiVersion: v1 +kind: Service +metadata: + name: nginx-service + labels: + app: nginx +spec: + ports: + - port: 80 + name: web + selector: + app: nginx + +``` + +And Use the service in the ingress Backend service name, as: +``` +backend: + serviceName: nginx-service + servicePort: '80' +``` + +That will forward traffic to your StatefulSets Pods. + + +#### Forward Traffic to specific Pods of StatefulSet +There is a way to send traffic to all or specific pod of a StatefulSet using voyager. You can set +`hostNames` field in `Backend`, traffic will only forwarded to those pods. + +For Example the above StatefulSet will create two pod. +``` +web-0 +web-1 +``` +Those are the host names. + +Now Create a ingress that will only forward traffic to web-0 +```yaml +apiVersion: appscode.com/v1beta1 +kind: Ingress +metadata: + name: test-ingress + namespace: default +spec: + rules: + - host: appscode.example.com + http: + paths: + - path: '/testPath' + backend: + hostNames: + - web-0 + serviceName: nginx-set #! There is no extra service. This + servicePort: '80' # is the Statefulset's Headless Service +``` + +Viola. Now all `/testPath` traffic will be sent to pod web-0 only. There is no extra service also. +The StatefulSet's Headless Service is enough. +By using all the hostNames You can forward traffic to all pod. \ No newline at end of file From 095e73bd2483315e16f0666d6b3dc7a3fdf5ee0f Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Wed, 3 May 2017 03:53:50 -0700 Subject: [PATCH 3/4] Update statefulset-pod.md --- docs/user-guide/component/ingress/statefulset-pod.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/user-guide/component/ingress/statefulset-pod.md b/docs/user-guide/component/ingress/statefulset-pod.md index 39c3e98cf..b834583fa 100644 --- a/docs/user-guide/component/ingress/statefulset-pod.md +++ b/docs/user-guide/component/ingress/statefulset-pod.md @@ -96,5 +96,4 @@ spec: ``` Viola. Now all `/testPath` traffic will be sent to pod web-0 only. There is no extra service also. -The StatefulSet's Headless Service is enough. -By using all the hostNames You can forward traffic to all pod. \ No newline at end of file +The StatefulSet's Headless Service is enough. By using all the hostNames You can forward traffic to all pods. From a0dc933d37ff4066d53ea2629f07cec283762a29 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Wed, 3 May 2017 03:55:15 -0700 Subject: [PATCH 4/4] Update parser.go --- pkg/controller/ingress/parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/ingress/parser.go b/pkg/controller/ingress/parser.go index 8ca288916..9915a2737 100644 --- a/pkg/controller/ingress/parser.go +++ b/pkg/controller/ingress/parser.go @@ -86,7 +86,7 @@ func (lbc *EngressController) getEndpoints(s *kapi.Service, servicePort *kapi.Se log.Infoln("targert port", targetPort) for _, epAddress := range ss.Addresses { - if isIncludeAbleAddress(hostNames, epAddress.Hostname) { + if isForwardable(hostNames, epAddress.Hostname) { eps = append(eps, &Endpoint{ Name: "server-" + epAddress.IP, IP: epAddress.IP, @@ -99,7 +99,7 @@ func (lbc *EngressController) getEndpoints(s *kapi.Service, servicePort *kapi.Se return } -func isIncludeAbleAddress(hostNames []string, hostName string) bool { +func isForwardable(hostNames []string, hostName string) bool { if len(hostNames) <= 0 { return true }