Adds e2e firewall tests for LoadBalancer service, ingress, and e2e cluster

test/e2e/BUILD

@@ -51,6 +51,7 @@ go_library(
         "federation-replicaset.go",
         "federation-util.go",
         "federation-util-14.go",
+        "firewall.go",
         "garbage_collector.go",
         "generated_clientset.go",
         "gke_local_ssd.go",

test/e2e/firewall.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api/v1"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+	"k8s.io/kubernetes/pkg/master/ports"
+	"k8s.io/kubernetes/pkg/util/sets"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	firewallTimeoutDefault = 3 * time.Minute
+	firewallTestTcpTimeout = time.Duration(1 * time.Second)
+	// Set ports outside of 30000-32767, 80 and 8080 to avoid being whitelisted by the e2e cluster
+	firewallTestHttpPort = int32(29999)
+	firewallTestUdpPort  = int32(29998)
+)
+
+var _ = framework.KubeDescribe("Firewall rule", func() {
+	var firewall_test_name = "firewall-test"
+	f := framework.NewDefaultFramework(firewall_test_name)
+
+	var cs clientset.Interface
+	var cloudConfig framework.CloudConfig
+	var gceCloud *gcecloud.GCECloud
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce")
+		cs = f.ClientSet
+		cloudConfig = framework.TestContext.CloudConfig
+		gceCloud = cloudConfig.Provider.(*gcecloud.GCECloud)
+	})
+
+	// This test takes around 4 minutes to run
+	It("[Slow] [Serial] should create valid firewall rules for LoadBalancer type service", func() {
+		ns := f.Namespace.Name
+		// This source ranges is just used to examine we have exact same things on LB firewall rules
+		firewallTestSourceRanges := []string{"0.0.0.0/1", "128.0.0.0/1"}
+		serviceName := "firewall-test-loadbalancer"
+
+		jig := NewServiceTestJig(cs, serviceName)
+		nodesNames := jig.GetNodesNames(maxNodesForEndpointsTests)
+		if len(nodesNames) <= 0 {
+			framework.Failf("Expect at least 1 node, got: %v", nodesNames)
+		}
+		nodesSet := sets.NewString(nodesNames...)
+
+		// OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE
+		By("Creating a LoadBalancer type service with onlyLocal annotation")
+		svc := jig.createOnlyLocalLoadBalancerService(ns, serviceName,
+			loadBalancerCreateTimeoutDefault, false, func(svc *v1.Service) {
+				svc.Spec.Ports = []v1.ServicePort{{Protocol: "TCP", Port: firewallTestHttpPort}}
+				svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges
+			})
+		defer func() {
+			jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) {
+				svc.Spec.Type = v1.ServiceTypeNodePort
+				svc.Spec.LoadBalancerSourceRanges = nil
+			})
+			Expect(cs.Core().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred())
+		}()
+		svcExternalIP := svc.Status.LoadBalancer.Ingress[0].IP
+
+		By("Checking if service's firewall rules are correct")
+		nodeTags := framework.GetInstanceTags(cloudConfig, nodesNames[0])
+		expFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items)
+		fw, err := gceCloud.GetFirewall(expFw.Name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(framework.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+
+		By(fmt.Sprintf("Creating netexec pods on at most %v nodes", maxNodesForEndpointsTests))
+		for i, nodeName := range nodesNames {
+			podName := fmt.Sprintf("netexec%v", i)
+			jig.LaunchNetexecPodOnNode(f, nodeName, podName, firewallTestHttpPort, firewallTestUdpPort, true)
+			defer func() {
+				framework.Logf("Cleaning up the netexec pod: %v", podName)
+				Expect(cs.Core().Pods(ns).Delete(podName, nil)).NotTo(HaveOccurred())
+			}()
+		}
+
+		// Send requests from outside of the cluster because internal traffic is whitelisted
+		By("Accessing the external service ip from outside, all non-master nodes should be reached")
+		Expect(testHitNodesFromOutside(svcExternalIP, firewallTestHttpPort, firewallTimeoutDefault, nodesSet)).NotTo(HaveOccurred())
+
+		// Check if there are overlapping tags on the firewall that extend beyond just the vms in our cluster
+		// by removing the tag on one vm and make sure it doesn't get any traffic. This is an imperfect
+		// simulation, we really want to check that traffic doesn't reach a vm outside the GKE cluster, but
+		// that's much harder to do in the current e2e framework.
+		By("Removing tags from one of the nodes")
+		nodesSet.Delete(nodesNames[0])
+		removedTags := framework.SetInstanceTags(cloudConfig, nodesNames[0], []string{})
+		defer func() {
+			By("Adding tags back to the node and wait till the traffic is recovered")
+			nodesSet.Insert(nodesNames[0])
+			framework.SetInstanceTags(cloudConfig, nodesNames[0], removedTags)
+			// Make sure traffic is recovered before exit
+			Expect(testHitNodesFromOutside(svcExternalIP, firewallTestHttpPort, firewallTimeoutDefault, nodesSet)).NotTo(HaveOccurred())
+		}()
+
+		By("Accessing serivce through the external ip and examine got no response from the node without tags")
+		Expect(testHitNodesFromOutsideWithCount(svcExternalIP, firewallTestHttpPort, firewallTimeoutDefault, nodesSet, 15)).NotTo(HaveOccurred())
+	})
+
+	It("should have correct firewall rules for e2e cluster", func() {
+		By("Gathering firewall related information")
+		masterTags := framework.GetInstanceTags(cloudConfig, cloudConfig.MasterName)
+		Expect(len(masterTags.Items)).Should(Equal(1))
+
+		nodes := framework.GetReadySchedulableNodesOrDie(cs)
+		if len(nodes.Items) <= 0 {
+			framework.Failf("Expect at least 1 node, got: %v", len(nodes.Items))
+		}
+		nodeTags := framework.GetInstanceTags(cloudConfig, nodes.Items[0].Name)
+		Expect(len(nodeTags.Items)).Should(Equal(1))
+
+		By("Checking if e2e firewall rules are correct")
+		for _, expFw := range framework.GetE2eFirewalls(cloudConfig.MasterName, masterTags.Items[0], nodeTags.Items[0], cloudConfig.Network) {
+			fw, err := gceCloud.GetFirewall(expFw.Name)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(framework.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+		}
+
+		By("Checking well known ports on master and nodes are not exposed externally")
+		nodeAddrs := framework.NodeAddresses(nodes, v1.NodeExternalIP)
+		Expect(len(nodeAddrs)).NotTo(BeZero())
+		masterAddr := framework.GetMasterAddress(cs)
+		flag, _ := testNotReachableHTTPTimeout(masterAddr, ports.ControllerManagerPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+		flag, _ = testNotReachableHTTPTimeout(masterAddr, ports.SchedulerPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+		flag, _ = testNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+		flag, _ = testNotReachableHTTPTimeout(nodeAddrs[0], ports.KubeletReadOnlyPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+		flag, _ = testNotReachableHTTPTimeout(nodeAddrs[0], ports.ProxyStatusPort, firewallTestTcpTimeout)
+		Expect(flag).To(BeTrue())
+	})
+})

test/e2e/framework/BUILD

@@ -13,6 +13,7 @@ go_library(
         "cleanup.go",
         "exec_util.go",
         "federation_util.go",
+        "firewall_util.go",
         "framework.go",
         "get-kubemark-resource-usage.go",
         "google_compute.go",
@@ -100,6 +101,7 @@ go_library(
         "//vendor:github.com/spf13/viper",
         "//vendor:golang.org/x/crypto/ssh",
         "//vendor:golang.org/x/net/websocket",
+        "//vendor:google.golang.org/api/compute/v1",
         "//vendor:google.golang.org/api/googleapi",
         "//vendor:gopkg.in/yaml.v2",
         "//vendor:k8s.io/client-go/kubernetes",