Remove security enforcement label from namespace in SaaS (#531)

In SaaS we recently introduced a label that indicates to prevent certain
actions (security related), see related [slack thread
](https://camunda.slack.com/archives/CT702EPFH/p1714475896279759)


In order to make sure that our experiments, and actions are successful
in SaaS
we need to make sure that reconciliation is paused and the
security enforcement label is removed from the corresponding
target namespace. That is now always done when creating a client for a
SaaS environment.

After doing so we can get further privileges, that are needed for
actions like putting stress on the CPU, network partition, etc.

go-chaos/internal/helper_test.go

@@ -133,3 +133,14 @@ func (c K8Client) CreateStatefulSetWithLabelsAndName(t *testing.T, selector *met
 
 	require.NoError(t, err)
 }
+
+func (c *K8Client) createSaaSNamespace(t *testing.T) {
+	namespace := v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   c.GetCurrentNamespace(),
+			Labels: map[string]string{"pod-security.kubernetes.io/enforce": "true"},
+		},
+	}
+	_, err := c.Clientset.CoreV1().Namespaces().Create(context.TODO(), &namespace, metav1.CreateOptions{})
+	require.NoError(t, err)
+}

go-chaos/internal/k8helper.go

@@ -61,13 +61,31 @@ func createK8Client(settings KubernetesSettings) (K8Client, error) {
 
 	if client.SaaSEnv {
 		LogVerbose("Running experiment in SaaS environment.")
+		err = prepareSaaSTargetCluster(client)
+		if err != nil {
+			return K8Client{}, err
+		}
 	} else {
 		LogVerbose("Running experiment in self-managed environment.")
 	}
 
 	return client, nil
 }
 
+func prepareSaaSTargetCluster(client K8Client) error {
+	LogVerbose("Pausing reconciliation preventive.")
+	err := client.PauseReconciliation()
+	if err != nil {
+		return err
+	}
+
+	err = client.disableSaaSNamespaceSecurityLabel()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func internalCreateClient(settings KubernetesSettings) (K8Client, error) {
 	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
 		&clientcmd.ClientConfigLoadingRules{ExplicitPath: settings.kubeConfigPath},

go-chaos/internal/labels.go

@@ -15,6 +15,7 @@
 package internal
 
 import (
+	"context"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 )
@@ -84,3 +85,16 @@ func (c K8Client) getWorkerLabels() string {
 	}
 	return labels.Set(labelSelector.MatchLabels).String()
 }
+
+func (c K8Client) disableSaaSNamespaceSecurityLabel() error {
+	ns, err := c.Clientset.CoreV1().Namespaces().Get(context.TODO(), c.GetCurrentNamespace(), metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	LogVerbose("Removing namespace label: 'pod-security.kubernetes.io/enforce' to allow further privileges.")
+	delete(ns.Labels, "pod-security.kubernetes.io/enforce")
+
+	_, err = c.Clientset.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{})
+	return err
+}

go-chaos/internal/labels_test.go

@@ -15,6 +15,9 @@
 package internal
 
 import (
+	"context"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -63,3 +66,18 @@ func Test_shouldGetSaasGatewayLabels(t *testing.T) {
 	// then
 	assert.Equal(t, expected, actual, "Labels should be equal")
 }
+
+func Test_shouldRemoveNamespaceLabel(t *testing.T) {
+	// given
+	k8Client := CreateFakeClient()
+	k8Client.createSaaSNamespace(t)
+
+	// when
+	err := k8Client.disableSaaSNamespaceSecurityLabel()
+
+	// then
+	require.NoError(t, err)
+	namespace, err := k8Client.Clientset.CoreV1().Namespaces().Get(context.TODO(), k8Client.GetCurrentNamespace(), metav1.GetOptions{})
+	require.NoError(t, err)
+	assert.Empty(t, namespace.Labels)
+}