Skip to content

Commit

Permalink
UPSTREAM: 111789: Update Netpol e2e tests to use framework CreateName…
Browse files Browse the repository at this point in the history 
…space

The main purpose of this change is to update the e2e Netpol tests to use
the srandard CreateNamespace function from the Framework. Before this
change, a custom Namespace creation function was used, with the
following consequences:

* Pod security admission settings had to be enforced locally (not using
  the centralized mechanism)
* the custom function was brittle, not waiting for default Namespace
  ServiceAccount creation, causing tests to fail in some infrastructures
* tests were not benefiting from standard framework capabilities:
  Namespace name generation, automatic Namespace deletion, etc.

As part of this change, we also do the following:

* clearly decouple responsibilities between the Model, which defines the
  K8s objects to be created, and the KubeManager, which has access to
  runtime information (actual Namespace names after their creation by
  the framework, Service IPs, etc.)
* simplify / clean-up tests and remove as much unneeded logic / funtions
  as possible for easier long-term maintenance
* remove the useFixedNamespaces compile-time constant switch, which
  aimed at re-using existing K8s resources across test cases. The
  reasons: a) it is currently broken as setting it to true causes most
  tests to panic on the master branch, b) it is not a good idea to have
  some switch like this which changes the behavior of the tests and is
  never exercised in CI, c) it cannot possibly work as different test
  cases have different Model requirements (e.g., the protocols list can
  differ) and hence different K8s resource requirements.

For kubernetes#108298

Signed-off-by: Antonin Bas <abas@vmware.com>
  • Loading branch information
@antoninbas @soltysh
antoninbas authored and soltysh committed Aug 26, 2022
1 parent a6797a6 commit 8aadd1d
Show file tree
Hide file tree
Showing 6 changed files with 464 additions and 609 deletions.
131 changes: 66 additions & 65 deletions test/e2e/network/netpol/kubemanager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
admissionapi "k8s.io/pod-security-admission/api"
)

// probeConnectivityArgs is set of arguments for a probeConnectivity
Expand All @@ -45,65 +44,81 @@ type probeConnectivityArgs struct {
timeoutSeconds int
}

// TestPod represents an actual running pod. For each Pod defined by the model,
// there will be a corresponding TestPod. TestPod includes some runtime info
// (namespace name, service IP) which is not available in the model.
type TestPod struct {
Namespace string
Name string
ContainerName string
ServiceIP string
}

func (pod TestPod) PodString() PodString {
return NewPodString(pod.Namespace, pod.Name)
}

// kubeManager provides a convenience interface to kube functionality that we leverage for polling NetworkPolicy connections.
// Its responsibilities are:
// - creating resources (pods, deployments, namespaces, services, network policies)
// - modifying and cleaning up resources
type kubeManager struct {
framework *framework.Framework
clientSet clientset.Interface
framework *framework.Framework
clientSet clientset.Interface
namespaceNames []string
allPods []TestPod
allPodStrings []PodString
dnsDomain string
}

// newKubeManager is a utility function that wraps creation of the kubeManager instance.
func newKubeManager(framework *framework.Framework) *kubeManager {
func newKubeManager(framework *framework.Framework, dnsDomain string) *kubeManager {
return &kubeManager{
framework: framework,
clientSet: framework.ClientSet,
dnsDomain: dnsDomain,
}
}

// initializeCluster checks the state of the cluster, creating or updating namespaces and deployments as needed.
func (k *kubeManager) initializeCluster(model *Model) error {
// initializeCluster initialized the cluster, creating namespaces pods and services as needed.
func (k *kubeManager) initializeClusterFromModel(model *Model) error {
var createdPods []*v1.Pod
for _, ns := range model.Namespaces {
_, err := k.createNamespace(ns.Spec())
// no labels needed, we just need the default kubernetes.io/metadata.name label
namespace, err := k.framework.CreateNamespace(ns.BaseName, nil)
if err != nil {
return err
}
namespaceName := namespace.Name
k.namespaceNames = append(k.namespaceNames, namespaceName)

for _, pod := range ns.Pods {
framework.Logf("creating/updating pod %s/%s", ns.Name, pod.Name)
framework.Logf("creating pod %s/%s with matching service", namespaceName, pod.Name)

// note that we defer the logic of pod (i.e. node selector) specifics to the model
// which is aware of linux vs windows pods
kubePod, err := k.createPod(pod.KubePod())
kubePod, err := k.createPod(pod.KubePod(namespaceName))
if err != nil {
return err
}

createdPods = append(createdPods, kubePod)
svc, err := k.createService(pod.Service())
svc, err := k.createService(pod.Service(namespaceName))
if err != nil {
return err
}
if netutils.ParseIPSloppy(svc.Spec.ClusterIP) == nil {
return fmt.Errorf("empty IP address found for service %s/%s", svc.Namespace, svc.Name)
}
pod.ServiceIP = svc.Spec.ClusterIP
}
}

for _, podString := range model.AllPodStrings() {
k8sPod, err := k.getPod(podString.Namespace(), podString.PodName())
if err != nil {
return err
}
if k8sPod == nil {
return fmt.Errorf("unable to find pod in ns %s with key/val pod=%s", podString.Namespace(), podString.PodName())
}
err = e2epod.WaitForPodNameRunningInNamespace(k.clientSet, k8sPod.Name, k8sPod.Namespace)
if err != nil {
return fmt.Errorf("unable to wait for pod %s/%s: %w", podString.Namespace(), podString.PodName(), err)
k.allPods = append(k.allPods, TestPod{
Namespace: kubePod.Namespace,
Name: kubePod.Name,
ContainerName: pod.Containers[0].Name(),
ServiceIP: svc.Spec.ClusterIP,
})
k.allPodStrings = append(k.allPodStrings, NewPodString(kubePod.Namespace, kubePod.Name))

}
}

Expand All @@ -117,6 +132,22 @@ func (k *kubeManager) initializeCluster(model *Model) error {
return nil
}

func (k *kubeManager) AllPods() []TestPod {
return k.allPods
}

func (k *kubeManager) AllPodStrings() []PodString {
return k.allPodStrings
}

func (k *kubeManager) DNSDomain() string {
return k.dnsDomain
}

func (k *kubeManager) NamespaceNames() []string {
return k.namespaceNames
}

// getPod gets a pod by namespace and name.
func (k *kubeManager) getPod(ns string, name string) (*v1.Pod, error) {
kubePod, err := k.clientSet.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
Expand Down Expand Up @@ -174,16 +205,6 @@ func (k *kubeManager) executeRemoteCommand(namespace string, pod string, contain
})
}

// createNamespace is a convenience function for namespace setup.
func (k *kubeManager) createNamespace(ns *v1.Namespace) (*v1.Namespace, error) {
enforcePodSecurityBaseline(ns)
createdNamespace, err := k.clientSet.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
if err != nil {
return nil, fmt.Errorf("unable to create namespace %s: %w", ns.Name, err)
}
return createdNamespace, nil
}

// createService is a convenience function for service setup.
func (k *kubeManager) createService(service *v1.Service) (*v1.Service, error) {
ns := service.Namespace
Expand All @@ -209,8 +230,8 @@ func (k *kubeManager) createPod(pod *v1.Pod) (*v1.Pod, error) {
}

// cleanNetworkPolicies is a convenience function for deleting network policies before startup of any new test.
func (k *kubeManager) cleanNetworkPolicies(namespaces []string) error {
for _, ns := range namespaces {
func (k *kubeManager) cleanNetworkPolicies() error {
for _, ns := range k.namespaceNames {
framework.Logf("deleting policies in %s ..........", ns)
l, err := k.clientSet.NetworkingV1().NetworkPolicies(ns).List(context.TODO(), metav1.ListOptions{})
if err != nil {
Expand Down Expand Up @@ -258,36 +279,16 @@ func (k *kubeManager) getNamespace(ns string) (*v1.Namespace, error) {
return selectedNameSpace, nil
}

// setNamespaceLabels sets the labels for a namespace object in kubernetes.
func (k *kubeManager) setNamespaceLabels(ns string, labels map[string]string) error {
selectedNameSpace, err := k.getNamespace(ns)
if err != nil {
return err
}
selectedNameSpace.ObjectMeta.Labels = labels
enforcePodSecurityBaseline(selectedNameSpace)
_, err = k.clientSet.CoreV1().Namespaces().Update(context.TODO(), selectedNameSpace, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("unable to update namespace %s: %w", ns, err)
// getProbeTimeoutSeconds returns a timeout for how long the probe should work before failing a check, and takes windows heuristics into account, where requests can take longer sometimes.
func getProbeTimeoutSeconds() int {
timeoutSeconds := 1
if framework.NodeOSDistroIs("windows") {
timeoutSeconds = 3
}
return nil
}

// deleteNamespaces removes a namespace from kubernetes.
func (k *kubeManager) deleteNamespaces(namespaces []string) error {
for _, ns := range namespaces {
err := k.clientSet.CoreV1().Namespaces().Delete(context.TODO(), ns, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("unable to delete namespace %s: %w", ns, err)
}
}
return nil
return timeoutSeconds
}

func enforcePodSecurityBaseline(ns *v1.Namespace) {
if len(ns.ObjectMeta.Labels) == 0 {
ns.ObjectMeta.Labels = make(map[string]string)
}
// TODO(https://github.com/kubernetes/kubernetes/issues/108298): route namespace creation via framework.Framework.CreateNamespace
ns.ObjectMeta.Labels[admissionapi.EnforceLevelLabel] = string(admissionapi.LevelBaseline)
// getWorkers returns the number of workers suggested to run when testing.
func getWorkers() int {
return 3
}

0 comments on commit 8aadd1d

Please sign in to comment.