From d9a1446d95415db48137979a89282c3fdf2607f6 Mon Sep 17 00:00:00 2001 From: Anuj Sharma Date: Mon, 6 Apr 2020 00:06:34 +0530 Subject: [PATCH 1/3] #403 added test for default deny network conn --- benchmarks/e2e/config/config.go | 4 +- .../default_deny_net_conn.go | 124 ++++++++++++++++++ benchmarks/e2e/tests/e2e.go | 1 + benchmarks/go.sum | 1 + 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go diff --git a/benchmarks/e2e/config/config.go b/benchmarks/e2e/config/config.go index 57261f785..807400c7d 100644 --- a/benchmarks/e2e/config/config.go +++ b/benchmarks/e2e/config/config.go @@ -31,7 +31,7 @@ func (c *BenchmarkConfig) GetValidTenant() (TenantSpec, error) { return c.TenantB, nil } -func (c *BenchmarkConfig) ValidateTenant(t TenantSpec) (error) { +func (c *BenchmarkConfig) ValidateTenant(t TenantSpec) error { if c == nil { return errors.New("Please fill in a valid/non-empty config.yaml") } @@ -41,4 +41,4 @@ func (c *BenchmarkConfig) ValidateTenant(t TenantSpec) (error) { } return errors.New("Given tenant does not match with TenantSpec") -} +} diff --git a/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go b/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go new file mode 100644 index 000000000..1c92ed93b --- /dev/null +++ b/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go @@ -0,0 +1,124 @@ +package default_deny_net_conn + +import ( + "fmt" + "time" + + "github.com/onsi/ginkgo" + configutil "sigs.k8s.io/multi-tenancy/benchmarks/e2e/config" + "k8s.io/kubernetes/test/e2e/framework" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + imageutils "k8s.io/kubernetes/test/utils/image" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/apimachinery/pkg/util/uuid" +) + +const ( + expectedVal = "command terminated with exit code 1" +) + +func MakeSpecPod(name string, Namespace string) (*v1.Pod) { + podSpec := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: Namespace, + Labels: map[string]string {"run": "my-nginx"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: name, + Image: imageutils.GetE2EImage(imageutils.Nginx), + }, + }, + RestartPolicy: v1.RestartPolicyAlways, + }, + } + return podSpec +} + +func CreateServiceSpec(serviceName, externalName string, isHeadless bool, selector map[string]string) *v1.Service { + headlessService := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Labels: map[string]string {"run": "my-nginx"}, + }, + Spec: v1.ServiceSpec{ + Selector: selector, + }, + } + if externalName != "" { + headlessService.Spec.Type = v1.ServiceTypeExternalName + headlessService.Spec.ExternalName = externalName + } else { + headlessService.Spec.Ports = []v1.ServicePort{ + {Port: 80, Name: "http", Protocol: v1.ProtocolTCP}, + } + } + if isHeadless { + headlessService.Spec.ClusterIP = "None" + } + return headlessService +} + + +var _ = framework.KubeDescribe("Tenants should have explicit control over ingress connections for their workloads", func() { + var config *configutil.BenchmarkConfig + var tenantA, tenantB string + var namespaceFlag = "-n" + var err error + var labels = map[string]string {"run": "my-nginx"} + var name = "security-context-" + string(uuid.NewUUID()) + var url string + + ginkgo.BeforeEach(func() { + config, err = configutil.ReadConfig(configutil.ConfigPath) + framework.ExpectNoError(err) + + err = config.ValidateTenant(config.TenantA) + framework.ExpectNoError(err) + + err = config.ValidateTenant(config.TenantB) + framework.ExpectNoError(err) + + tenantA = configutil.GetContextFromKubeconfig(config.TenantA.Kubeconfig) + tenantB = configutil.GetContextFromKubeconfig(config.TenantB.Kubeconfig) + + url = "http://" + name + "." + config.TenantA.Namespace + }) + + ginkgo.It("Tenant cannot connect to the pod or services of other tenant", func() { + ginkgo.By(fmt.Sprintf("Tenant %s cannot connect to the service in the %s namespace", tenantB, tenantA)) + + kclientTenantA := configutil.NewKubeClientWithKubeconfig(config.TenantA.Kubeconfig) + + // Making nginx pod in TenantA + pod := MakeSpecPod(name, config.TenantA.Namespace) + _, err = kclientTenantA.CoreV1().Pods(config.TenantA.Namespace).Create(pod) + framework.ExpectNoError(err) + + // Making a service in TenantA to expose the nginx pod + svc := CreateServiceSpec(name, "", false, labels) + _, err = kclientTenantA.CoreV1().Services(config.TenantA.Namespace).Create(svc) + framework.ExpectNoError(err) + + kclientTenantB := configutil.NewKubeClientWithKubeconfig(config.TenantB.Kubeconfig) + + // Making nginx pod in TenantB to connect to service in TenantA + testpod := e2epod.MakeSecPod(config.TenantB.Namespace, nil, nil, false, "", false, false, nil, nil) + _, err = kclientTenantB.CoreV1().Pods(config.TenantB.Namespace).Create(testpod) + framework.ExpectNoError(err) + + // Wget the service exposed Url from the TenantB pod bash + _, errNew := framework.LookForString(expectedVal, time.Minute, func() string { + _, err := framework.RunKubectl(namespaceFlag, config.TenantB.Namespace, "exec", "-it", testpod.ObjectMeta.Name, "--", "wget" ,"--timeout=5" ,"-O" ,"-", url) + return err.Error() + }) + framework.ExpectNoError(errNew) + }) +}) \ No newline at end of file diff --git a/benchmarks/e2e/tests/e2e.go b/benchmarks/e2e/tests/e2e.go index 05bc93c28..8aec112d6 100644 --- a/benchmarks/e2e/tests/e2e.go +++ b/benchmarks/e2e/tests/e2e.go @@ -23,6 +23,7 @@ import ( _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/create_role_bindings" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_other_tenant_resources" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_host_ipc" + _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/default_deny_net_conn" ) // RunE2ETests runs the multi-tenancy benchmark tests diff --git a/benchmarks/go.sum b/benchmarks/go.sum index b7e09eb99..a52b8877e 100644 --- a/benchmarks/go.sum +++ b/benchmarks/go.sum @@ -666,6 +666,7 @@ modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= +sigs.k8s.io/multi-tenancy v0.0.0-20200511084551-34a25e2335ba h1:mxTTbwfGpJkOQc+XkbPDHtv7OAz/i0mOR9XD/hmpQPk= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= From d47e7363fdf97f78e642be769647b536fea9fce8 Mon Sep 17 00:00:00 2001 From: Anuj Sharma Date: Mon, 6 Apr 2020 00:10:42 +0530 Subject: [PATCH 2/3] Updated comment --- .../e2e/tests/default_deny_net_conn/default_deny_net_conn.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go b/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go index 1c92ed93b..483164ffd 100644 --- a/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go +++ b/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go @@ -109,7 +109,7 @@ var _ = framework.KubeDescribe("Tenants should have explicit control over ingres kclientTenantB := configutil.NewKubeClientWithKubeconfig(config.TenantB.Kubeconfig) - // Making nginx pod in TenantB to connect to service in TenantA + // Making busybox pod in TenantB to connect to service in TenantA testpod := e2epod.MakeSecPod(config.TenantB.Namespace, nil, nil, false, "", false, false, nil, nil) _, err = kclientTenantB.CoreV1().Pods(config.TenantB.Namespace).Create(testpod) framework.ExpectNoError(err) @@ -121,4 +121,4 @@ var _ = framework.KubeDescribe("Tenants should have explicit control over ingres }) framework.ExpectNoError(errNew) }) -}) \ No newline at end of file +}) From 5d69ce8d085fce49388aba3b4acaf6dc24223937 Mon Sep 17 00:00:00 2001 From: Anuj Sharma Date: Mon, 11 May 2020 23:30:14 +0530 Subject: [PATCH 3/3] added PL selector --- .../default_deny_net_conn.go | 33 +++++++++---------- benchmarks/e2e/tests/e2e.go | 7 ++-- benchmarks/go.sum | 1 - 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go b/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go index 483164ffd..df0bff7b6 100644 --- a/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go +++ b/benchmarks/e2e/tests/default_deny_net_conn/default_deny_net_conn.go @@ -5,20 +5,20 @@ import ( "time" "github.com/onsi/ginkgo" - configutil "sigs.k8s.io/multi-tenancy/benchmarks/e2e/config" - "k8s.io/kubernetes/test/e2e/framework" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - imageutils "k8s.io/kubernetes/test/utils/image" - e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + imageutils "k8s.io/kubernetes/test/utils/image" + configutil "sigs.k8s.io/multi-tenancy/benchmarks/e2e/config" ) const ( expectedVal = "command terminated with exit code 1" ) -func MakeSpecPod(name string, Namespace string) (*v1.Pod) { +func MakeSpecPod(name string, Namespace string) *v1.Pod { podSpec := &v1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -27,13 +27,13 @@ func MakeSpecPod(name string, Namespace string) (*v1.Pod) { ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: Namespace, - Labels: map[string]string {"run": "my-nginx"}, + Labels: map[string]string{"run": "my-nginx"}, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: name, - Image: imageutils.GetE2EImage(imageutils.Nginx), + Name: name, + Image: imageutils.GetE2EImage(imageutils.Nginx), }, }, RestartPolicy: v1.RestartPolicyAlways, @@ -45,8 +45,8 @@ func MakeSpecPod(name string, Namespace string) (*v1.Pod) { func CreateServiceSpec(serviceName, externalName string, isHeadless bool, selector map[string]string) *v1.Service { headlessService := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: map[string]string {"run": "my-nginx"}, + Name: serviceName, + Labels: map[string]string{"run": "my-nginx"}, }, Spec: v1.ServiceSpec{ Selector: selector, @@ -66,13 +66,12 @@ func CreateServiceSpec(serviceName, externalName string, isHeadless bool, select return headlessService } - -var _ = framework.KubeDescribe("Tenants should have explicit control over ingress connections for their workloads", func() { +var _ = framework.KubeDescribe("[PL1] [PL2] [PL3] Tenants should have explicit control over ingress connections for their workloads", func() { var config *configutil.BenchmarkConfig var tenantA, tenantB string var namespaceFlag = "-n" var err error - var labels = map[string]string {"run": "my-nginx"} + var labels = map[string]string{"run": "my-nginx"} var name = "security-context-" + string(uuid.NewUUID()) var url string @@ -94,21 +93,21 @@ var _ = framework.KubeDescribe("Tenants should have explicit control over ingres ginkgo.It("Tenant cannot connect to the pod or services of other tenant", func() { ginkgo.By(fmt.Sprintf("Tenant %s cannot connect to the service in the %s namespace", tenantB, tenantA)) - + kclientTenantA := configutil.NewKubeClientWithKubeconfig(config.TenantA.Kubeconfig) // Making nginx pod in TenantA pod := MakeSpecPod(name, config.TenantA.Namespace) _, err = kclientTenantA.CoreV1().Pods(config.TenantA.Namespace).Create(pod) framework.ExpectNoError(err) - + // Making a service in TenantA to expose the nginx pod svc := CreateServiceSpec(name, "", false, labels) _, err = kclientTenantA.CoreV1().Services(config.TenantA.Namespace).Create(svc) framework.ExpectNoError(err) kclientTenantB := configutil.NewKubeClientWithKubeconfig(config.TenantB.Kubeconfig) - + // Making busybox pod in TenantB to connect to service in TenantA testpod := e2epod.MakeSecPod(config.TenantB.Namespace, nil, nil, false, "", false, false, nil, nil) _, err = kclientTenantB.CoreV1().Pods(config.TenantB.Namespace).Create(testpod) @@ -116,7 +115,7 @@ var _ = framework.KubeDescribe("Tenants should have explicit control over ingres // Wget the service exposed Url from the TenantB pod bash _, errNew := framework.LookForString(expectedVal, time.Minute, func() string { - _, err := framework.RunKubectl(namespaceFlag, config.TenantB.Namespace, "exec", "-it", testpod.ObjectMeta.Name, "--", "wget" ,"--timeout=5" ,"-O" ,"-", url) + _, err := framework.RunKubectl(namespaceFlag, config.TenantB.Namespace, "exec", "-it", testpod.ObjectMeta.Name, "--", "wget", "--timeout=5", "-O", "-", url) return err.Error() }) framework.ExpectNoError(errNew) diff --git a/benchmarks/e2e/tests/e2e.go b/benchmarks/e2e/tests/e2e.go index 8aec112d6..6a50a86f4 100644 --- a/benchmarks/e2e/tests/e2e.go +++ b/benchmarks/e2e/tests/e2e.go @@ -9,10 +9,9 @@ import ( ginkgowrapper "k8s.io/kubernetes/test/e2e/framework/ginkgowrapper" // test sources - _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_cluster_resources" - _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/configure_ns_quotas" - _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_privileged_containers" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_add_capabilities" + _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_cluster_resources" + _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_host_ipc" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_host_pid" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_multitenant_resources" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_ns_quotas" @@ -21,8 +20,6 @@ import ( _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/configure_ns_object_quotas" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/configure_ns_quotas" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/create_role_bindings" - _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_other_tenant_resources" - _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/block_host_ipc" _ "sigs.k8s.io/multi-tenancy/benchmarks/e2e/tests/default_deny_net_conn" ) diff --git a/benchmarks/go.sum b/benchmarks/go.sum index a52b8877e..b7e09eb99 100644 --- a/benchmarks/go.sum +++ b/benchmarks/go.sum @@ -666,7 +666,6 @@ modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/multi-tenancy v0.0.0-20200511084551-34a25e2335ba h1:mxTTbwfGpJkOQc+XkbPDHtv7OAz/i0mOR9XD/hmpQPk= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=