Skip to content

Commit

Permalink
Merge pull request #16 from snapp-incubator/cleanup
Browse files Browse the repository at this point in the history
implemented cleanup especially for sidecars
  • Loading branch information
sinamna committed Dec 12, 2023
2 parents b8a4207 + f69090f commit 1f39bd7
Show file tree
Hide file tree
Showing 13 changed files with 443 additions and 60 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ package basic_authenticator
import (
"context"
"github.com/go-logr/logr"
"github.com/opdev/subreconciler"
authenticatorv1alpha1 "github.com/snapp-incubator/simple-authenticator/api/v1alpha1"
"github.com/snapp-incubator/simple-authenticator/internal/config"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
Expand Down Expand Up @@ -59,6 +61,19 @@ func (r *BasicAuthenticatorReconciler) Reconcile(ctx context.Context, req ctrl.R
r.logger.Info("reconcile triggered")
r.logger.Info(req.String())
r.initVars(req)

basicAuthenticator := &authenticatorv1alpha1.BasicAuthenticator{}
switch err := r.Get(ctx, req.NamespacedName, basicAuthenticator); {
case errors.IsNotFound(err):
return r.Cleanup(ctx, req)
case err != nil:
r.logger.Error(err, "failed to fetch object")
return subreconciler.Evaluate(subreconciler.Requeue())
default:
if basicAuthenticator.ObjectMeta.DeletionTimestamp != nil {
return r.Cleanup(ctx, req)
}
}
return r.Provision(ctx, req)
}

Expand Down
180 changes: 180 additions & 0 deletions internal/controller/basic_authenticator/cleanup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
package basic_authenticator

import (
"context"
"errors"
"github.com/opdev/subreconciler"
"github.com/snapp-incubator/simple-authenticator/api/v1alpha1"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

func (r *BasicAuthenticatorReconciler) Cleanup(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// Do the actual reconcile work
subRecs := []subreconciler.FnWithRequest{
r.setDeletionStatus,
r.removeInjectedContainers,
r.removeCleanupFinalizer,
}
for _, rec := range subRecs {
result, err := rec(ctx, req)
if subreconciler.ShouldHaltOrRequeue(result, err) {
return subreconciler.Evaluate(result, err)
}
}

return subreconciler.Evaluate(subreconciler.DoNotRequeue())
}
func (r *BasicAuthenticatorReconciler) setDeletionStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}
basicAuthenticator.Status.State = StatusDeleting

if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "Failed to update status while cleaning")
return subreconciler.RequeueWithError(err)
}
return subreconciler.ContinueReconciling()
}
func (r *BasicAuthenticatorReconciler) removeInjectedContainers(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}

if basicAuthenticator.Spec.Type != "sidecar" {
return subreconciler.ContinueReconciling()
}
basicAuthLabel := map[string]string{
basicAuthenticatorNameLabel: basicAuthenticator.Name,
}
deployments, err := getTargetDeployment(ctx, basicAuthenticator, r.Client, basicAuthLabel)
if err != nil {
r.logger.Error(err, "failed to get target deployments to clean up")
return subreconciler.RequeueWithError(err)
}
configmaps, err := getTargetConfigmapNames(ctx, basicAuthenticator, r.Client, basicAuthLabel)
if err != nil {
r.logger.Error(err, "failed to get target configmap to clean up")
return subreconciler.RequeueWithError(err)
}
secrets, err := getTargetSecretName(ctx, basicAuthenticator, r.Client, basicAuthLabel)
if err != nil {
r.logger.Error(err, "failed to get target secret to clean up")
return subreconciler.RequeueWithError(err)
}
r.logger.Info("debug", "configmap", configmaps, "secret", secrets)

cleanupDeployments := removeInjectedResources(deployments, secrets, configmaps)
for _, deploy := range cleanupDeployments {
if err := r.Update(ctx, deploy); err != nil {
r.logger.Error(err, "failed to add update cleaned up deployments")
return subreconciler.RequeueWithError(err)
}
}
return subreconciler.ContinueReconciling()
}
func (r *BasicAuthenticatorReconciler) removeCleanupFinalizer(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}
if controllerutil.ContainsFinalizer(basicAuthenticator, basicAuthenticatorFinalizer) {
if ok := controllerutil.RemoveFinalizer(basicAuthenticator, basicAuthenticatorFinalizer); !ok {
r.logger.Error(errors.New("finalizer not updated"), "Failed to remove finalizer for BasicAuthenticator")
return subreconciler.Requeue()
}
}

if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "Failed to remove finalizer for BasicAuthenticator")
return subreconciler.RequeueWithError(err)
}
return subreconciler.ContinueReconciling()
}

func getTargetDeployment(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]*appsv1.Deployment, error) {
var deploymentList appsv1.DeploymentList
if err := k8Client.List(
ctx,
&deploymentList,
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
return nil, err
}

resultDeployments := make([]*appsv1.Deployment, 0)
for _, deploy := range deploymentList.Items {
resultDeployments = append(resultDeployments, &deploy)
}
return resultDeployments, nil
}
func getTargetConfigmapNames(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]string, error) {
var configMapList v1.ConfigMapList
if err := k8Client.List(
ctx,
&configMapList,
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
return nil, err
}
resultConfigMaps := make([]string, 0)
for _, cm := range configMapList.Items {
resultConfigMaps = append(resultConfigMaps, cm.Name)
}
return resultConfigMaps, nil
}
func getTargetSecretName(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]string, error) {
var secretList v1.SecretList
if err := k8Client.List(
ctx,
&secretList,
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
return nil, err
}

resultSecrets := make([]string, 0)
for _, sec := range secretList.Items {
resultSecrets = append(resultSecrets, sec.Name)
}
return resultSecrets, nil
}
func removeInjectedResources(deployments []*appsv1.Deployment, secrets []string, configmap []string) []*appsv1.Deployment {
for _, deploy := range deployments {
containers := make([]v1.Container, 0)
for _, container := range deploy.Spec.Template.Spec.Containers {
if container.Name != nginxDefaultContainerName {
containers = append(containers, container)
}
}
deploy.Spec.Template.Spec.Containers = containers
volumes := make([]v1.Volume, 0)
for _, vol := range deploy.Spec.Template.Spec.Volumes {
if !existsInList(secrets, vol.Name) && !existsInList(configmap, vol.Name) {
volumes = append(volumes, vol)
}
}
deploy.Spec.Template.Spec.Volumes = volumes
if deploy.Labels != nil {
delete(deploy.Labels, basicAuthenticatorNameLabel)
}
}
return deployments
}

func existsInList(strList []string, targetStr string) bool {
for _, val := range strList {
if val == targetStr {
return true
}
}
return false
}
7 changes: 4 additions & 3 deletions internal/controller/basic_authenticator/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ package basic_authenticator
const (
nginxDefaultImageAddress = "nginx:1.25.3"
nginxDefaultContainerName = "nginx"
SecretAnnotation = "authenticator.snappcloud.io/secret.name"
ConfigmapAnnotation = "authenticator.snappcloud.io/configmap.name"
basicAuthenticatorNameLabel = "basicauthenticator.snappcloud.io/name"
basicAuthenticatorFinalizer = "basicauthenticator.snappcloud.io/finalizer"
ExternallyManaged = "basicauthenticator.snappcloud.io/externally.managed"
ConfigMountPath = "/etc/nginx/conf.d"
Expand All @@ -15,7 +14,6 @@ const (
template = `server {
listen AUTHENTICATOR_PORT;
location / {
resolver 8.8.8.8;
auth_basic "basic authentication area";
auth_basic_user_file "FILE_PATH";
proxy_pass http://APP_SERVICE:APP_PORT;
Expand All @@ -25,4 +23,7 @@ const (
proxy_set_header X-Forwarded-Proto $scheme;
}
}`
StatusAvailable = "Available"
StatusReconciling = "Reconciling"
StatusDeleting = "Deleting"
)
69 changes: 58 additions & 11 deletions internal/controller/basic_authenticator/provision.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,20 @@ import (
"reflect"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

// Provision provisions the required resources for the basicAuthenticator object
func (r *BasicAuthenticatorReconciler) Provision(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// Do the actual reconcile work
subProvisioner := []subreconciler.FnWithRequest{
r.setReconcilingStatus,
r.addCleanupFinalizer,
r.ensureSecret,
r.ensureConfigmap,
r.ensureDeployment,
r.ensureService,
r.setAvailableStatus,
}
for _, provisioner := range subProvisioner {
result, err := provisioner(ctx, req)
Expand All @@ -33,6 +37,37 @@ func (r *BasicAuthenticatorReconciler) Provision(ctx context.Context, req ctrl.R

return subreconciler.Evaluate(subreconciler.DoNotRequeue())
}
func (r *BasicAuthenticatorReconciler) setReconcilingStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}

basicAuthenticator.Status.State = StatusReconciling
if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "failed to update status")
return subreconciler.Requeue()
}
return subreconciler.ContinueReconciling()
}

func (r *BasicAuthenticatorReconciler) addCleanupFinalizer(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}
if !controllerutil.ContainsFinalizer(basicAuthenticator, basicAuthenticatorFinalizer) {
if objUpdated := controllerutil.AddFinalizer(basicAuthenticator, basicAuthenticatorFinalizer); objUpdated {
if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "failed to add basicAuthenticator finalizer")
return subreconciler.Requeue()
}
}
}
return subreconciler.ContinueReconciling()
}

func (r *BasicAuthenticatorReconciler) getLatestBasicAuthenticator(ctx context.Context, req ctrl.Request, basicAuthenticator *v1alpha1.BasicAuthenticator) (*ctrl.Result, error) {
err := r.Get(ctx, req.NamespacedName, basicAuthenticator)
Expand All @@ -51,7 +86,7 @@ func (r *BasicAuthenticatorReconciler) ensureSecret(ctx context.Context, req ctr
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return r, err
return subreconciler.RequeueWithError(err)
}
r.credentialName = basicAuthenticator.Spec.CredentialsSecretRef
var credentialSecret corev1.Secret
Expand Down Expand Up @@ -118,7 +153,7 @@ func (r *BasicAuthenticatorReconciler) ensureConfigmap(ctx context.Context, req
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return r, err
return subreconciler.RequeueWithError(err)
}

authenticatorConfig := createNginxConfigmap(basicAuthenticator)
Expand Down Expand Up @@ -160,7 +195,7 @@ func (r *BasicAuthenticatorReconciler) ensureDeployment(ctx context.Context, req
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return r, err
return subreconciler.RequeueWithError(err)
}

if r.configMapName == "" {
Expand All @@ -182,7 +217,7 @@ func (r *BasicAuthenticatorReconciler) ensureService(ctx context.Context, req ct
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return r, err
return subreconciler.RequeueWithError(err)
}
if r.deploymentLabel == nil {
return subreconciler.ContinueReconciling()
Expand Down Expand Up @@ -217,6 +252,22 @@ func (r *BasicAuthenticatorReconciler) ensureService(ctx context.Context, req ct
}
return subreconciler.ContinueReconciling()
}

func (r *BasicAuthenticatorReconciler) setAvailableStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
basicAuthenticator := &v1alpha1.BasicAuthenticator{}

if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
return subreconciler.RequeueWithError(err)
}

basicAuthenticator.Status.State = StatusAvailable
if err := r.Update(ctx, basicAuthenticator); err != nil {
r.logger.Error(err, "failed to update status")
return subreconciler.Requeue()
}
return subreconciler.ContinueReconciling()
}

func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context.Context, req ctrl.Request, basicAuthenticator *v1alpha1.BasicAuthenticator, authenticatorConfigName, secretName string) (*ctrl.Result, error) {

newDeployment := createNginxDeployment(basicAuthenticator, authenticatorConfigName, secretName, r.CustomConfig)
Expand Down Expand Up @@ -253,6 +304,7 @@ func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context
replica, err := r.acquireTargetReplica(ctx, basicAuthenticator)
if err != nil {
r.logger.Error(err, "failed to acquire target replica using adaptiveScale")
return subreconciler.RequeueWithError(err)
}
targetReplica = &replica
}
Expand All @@ -262,16 +314,15 @@ func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context

foundDeployment.Spec = newDeployment.Spec
foundDeployment.Spec.Replicas = targetReplica

err := r.Update(ctx, foundDeployment)
err = r.Update(ctx, foundDeployment)
if err != nil {
r.logger.Error(err, "failed to update deployment")
return subreconciler.RequeueWithError(err)
}
}
r.logger.Info("updating ready replicas")
basicAuthenticator.Status.ReadyReplicas = int(foundDeployment.Status.ReadyReplicas)
err := r.Status().Update(ctx, basicAuthenticator)
err = r.Status().Update(ctx, basicAuthenticator)
if err != nil {
r.logger.Error(err, "failed to update basic authenticator status")
return subreconciler.RequeueWithError(err)
Expand All @@ -287,10 +338,6 @@ func (r *BasicAuthenticatorReconciler) createSidecarAuthenticator(ctx context.Co
return subreconciler.RequeueWithError(err)
}
for _, deploy := range deploymentsToUpdate {
if err := ctrl.SetControllerReference(basicAuthenticator, deploy, r.Scheme); err != nil {
r.logger.Error(err, "failed to set injected deployment owner")
return subreconciler.RequeueWithError(err)
}
err := r.Update(ctx, deploy)
if err != nil {
r.logger.Error(err, "failed to update injected deployments")
Expand Down

0 comments on commit 1f39bd7

Please sign in to comment.