add possibility to overwrite the yawollet requeue time with yawol-clo…

…ud-controller; make it possible to set the yawollet requeuetime to a maximum of 170s

charts/yawol-controller/templates/yawol-controller.yaml

@@ -100,6 +100,9 @@ spec:
         args:
           - -leader-elect
           - -enable-loadbalancermachine-controller
+          {{- if .Values.yawolletRequeueTime }}
+          - -yawollet-requeue-time={{ .Values.yawolletRequeueTime }}
+          {{- end }}
           {{- if .Values.openstackTimeout }}
           - -openstack-timeout={{ .Values.openstackTimeout }}
           {{- end }}

charts/yawol-controller/values.yaml

@@ -61,6 +61,7 @@ resources:
 
 #yawolClassName: debug
 #openstackTimeout: 20s
+#yawolletRequeueTime: 60
 
 # the name of the Kubernetes secret that contains the .openrc file contents
 # with the correct permissions to connect to the OpenStack API

cmd/yawol-controller/main.go

@@ -63,6 +63,8 @@ func main() {
 	var lbSetController bool
 	var lbMachineController bool
 
+	var yawolletRequeueTime int
+
 	var openstackTimeout time.Duration
 
 	// settings for leases
@@ -95,6 +97,11 @@ func main() {
 	flag.BoolVar(&lbMachineController, "enable-loadbalancermachine-controller", false,
 		"Enable loadbalancer-machine controller manager. ")
 
+	flag.IntVar(&yawolletRequeueTime, "yawollet-requeue-time", 0,
+		"yawollet requeue time in seconds for reconcile if object was successful reconciled. "+
+			"Values less than 5 are set to 5 and greater than 170 are set to 170. "+
+			"If unset the default from yawollet is used.")
+
 	flag.DurationVar(&openstackTimeout, "openstack-timeout", 20*time.Second, "Timeout for all requests against Openstack.")
 
 	flag.IntVar(&leasesDurationInt, "leases-duration", 60,
@@ -256,19 +263,20 @@ func main() {
 		}
 
 		if err := (&loadbalancermachine.LoadBalancerMachineReconciler{
-			Client:           loadBalancerMachineMgr.GetClient(),
-			WorkerCount:      concurrentWorkersPerReconciler,
-			APIHost:          loadBalancerMachineMgr.GetConfig().Host,
-			CACert:           loadBalancerMachineMgr.GetConfig().CAData,
-			Log:              ctrl.Log.WithName("controller").WithName("LoadBalancerMachine"),
-			Recorder:         loadBalancerMachineMgr.GetEventRecorderFor("LoadBalancerMachine"),
-			RecorderLB:       loadBalancerMachineMgr.GetEventRecorderFor("yawol-service"),
-			Scheme:           loadBalancerMachineMgr.GetScheme(),
-			APIEndpoint:      apiEndpoint,
-			Metrics:          &helpermetrics.LoadBalancerMachineMetrics,
-			OpenstackTimeout: openstackTimeout,
-			DiscoveryClient:  discoveryClient,
-			RateLimiter:      rateLimiter,
+			Client:              loadBalancerMachineMgr.GetClient(),
+			WorkerCount:         concurrentWorkersPerReconciler,
+			APIHost:             loadBalancerMachineMgr.GetConfig().Host,
+			CACert:              loadBalancerMachineMgr.GetConfig().CAData,
+			Log:                 ctrl.Log.WithName("controller").WithName("LoadBalancerMachine"),
+			Recorder:            loadBalancerMachineMgr.GetEventRecorderFor("LoadBalancerMachine"),
+			RecorderLB:          loadBalancerMachineMgr.GetEventRecorderFor("yawol-service"),
+			Scheme:              loadBalancerMachineMgr.GetScheme(),
+			APIEndpoint:         apiEndpoint,
+			Metrics:             &helpermetrics.LoadBalancerMachineMetrics,
+			OpenstackTimeout:    openstackTimeout,
+			YawolletRequeueTime: yawolletRequeueTime,
+			DiscoveryClient:     discoveryClient,
+			RateLimiter:         rateLimiter,
 		}).SetupWithManager(loadBalancerMachineMgr); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "LoadBalancerMachine")
 			os.Exit(1)

cmd/yawollet/main.go

@@ -74,8 +74,8 @@ func main() {
 	flag.StringVar(&loadbalancerMachineName, "loadbalancer-machine-name", "", "Name of lbm object.")
 	flag.StringVar(&listenAddress, "listen-address", "", "Address that envoy should listen.")
 	flag.StringVar(&listenInterface, "listen-interface", "", "Interface that envoy should listen on. Ignored if listen-address is set.")
-	flag.IntVar(&requeueTime, "requeue-time", 30, "Requeue Time for reconcile if object was successful reconciled. "+
-		"Values less than 5 are set to 5 and greater than 50 are set to 50")
+	flag.IntVar(&requeueTime, "requeue-time", 30, "Requeue Time in seconds for reconcile if object was successful reconciled. "+
+		"Values less than 5 are set to 5 and greater than 170 are set to 170")
 
 	flag.StringVar(&keepalivedStatsFile, "keepalived-stats-file", "/tmp/keepalived.stats",
 		"Stats file for keepalived (default: /tmp/keepalived.stats). "+
@@ -101,8 +101,8 @@ func main() {
 	// force requeue time between 5 and 50 (to be inside the heartbeat#time of yawol-controller)
 	if requeueTime < 5 {
 		requeueTime = 5
-	} else if requeueTime > 50 {
-		requeueTime = 50
+	} else if requeueTime > 170 {
+		requeueTime = 170
 	}
 
 	// set listen address

controllers/yawol-cloud-controller/targetcontroller/node_controller.go

@@ -5,8 +5,6 @@ import (
 	"encoding/json"
 	"reflect"
 	"regexp"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"strings"
 
 	yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
@@ -19,6 +17,8 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
 
 // NodeReconciler reconciles service Objects with type LoadBalancer

controllers/yawol-cloud-controller/targetcontroller/node_controller_test.go

@@ -3,13 +3,13 @@ package targetcontroller
 import (
 	"context"
 	"fmt"
-	predicatesEvent "sigs.k8s.io/controller-runtime/pkg/event"
 	"strings"
 	"time"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	predicatesEvent "sigs.k8s.io/controller-runtime/pkg/event"
 
 	yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
 	"github.com/stackitcloud/yawol/internal/helper"

controllers/yawol-controller/loadbalancermachine/loadbalancermachine_controller.go

@@ -51,19 +51,20 @@ var ipv4RegexC, _ = regexp.Compile(ipv4Regex)
 // LoadBalancerMachineReconciler reconciles service Objects with type LoadBalancer
 type LoadBalancerMachineReconciler struct { //nolint:revive // naming from kubebuilder
 	client.Client
-	APIHost           string
-	CACert            []byte
-	Log               logr.Logger
-	Scheme            *runtime.Scheme
-	Recorder          record.EventRecorder
-	RecorderLB        record.EventRecorder
-	APIEndpoint       string
-	Metrics           *helpermetrics.LoadBalancerMachineMetricList
-	getOsClientForIni os.GetOSClientFunc
-	WorkerCount       int
-	OpenstackTimeout  time.Duration
-	DiscoveryClient   *discovery.DiscoveryClient
-	RateLimiter       ratelimiter.RateLimiter
+	APIHost             string
+	CACert              []byte
+	Log                 logr.Logger
+	Scheme              *runtime.Scheme
+	Recorder            record.EventRecorder
+	RecorderLB          record.EventRecorder
+	APIEndpoint         string
+	Metrics             *helpermetrics.LoadBalancerMachineMetricList
+	getOsClientForIni   os.GetOSClientFunc
+	WorkerCount         int
+	OpenstackTimeout    time.Duration
+	YawolletRequeueTime int
+	DiscoveryClient     *discovery.DiscoveryClient
+	RateLimiter         ratelimiter.RateLimiter
 }
 
 // Reconcile Reconciles a LoadBalancerMachine
@@ -677,6 +678,7 @@ func (r *LoadBalancerMachineReconciler) reconcileServer(
 		loadbalancer,
 		loadBalancerMachine,
 		vip,
+		r.YawolletRequeueTime,
 	)
 
 	var srv *servers.Server

controllers/yawol-controller/loadbalancerset/loadbalancerset_controller.go

@@ -72,31 +72,31 @@ func (r *LoadBalancerSetReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{}, err
 	}
 
-	readyMachines := make([]yawolv1beta1.LoadBalancerMachine, 0)
-	notReadyMachines := make([]yawolv1beta1.LoadBalancerMachine, 0)
-	deletedMachines := make([]yawolv1beta1.LoadBalancerMachine, 0)
+	var (
+		readyMachineCount   int
+		deletedMachineCount int
+	)
+
 	for i := range childMachines.Items {
 		if childMachines.Items[i].DeletionTimestamp != nil {
-			deletedMachines = append(deletedMachines, childMachines.Items[i])
+			deletedMachineCount++
 			continue
 		}
 		if isMachineReady(childMachines.Items[i]) {
-			readyMachines = append(readyMachines, childMachines.Items[i])
+			readyMachineCount++
 			continue
 		}
-		notReadyMachines = append(notReadyMachines, childMachines.Items[i])
 	}
 
-	if res, err := r.reconcileStatus(ctx, &set, readyMachines); err != nil || res.Requeue || res.RequeueAfter != 0 {
+	if res, err := r.reconcileStatus(ctx, &set, readyMachineCount); err != nil || res.Requeue || res.RequeueAfter != 0 {
 		return res, err
 	}
 
 	if res, err := r.reconcileReplicas(
 		ctx,
 		&set,
-		deletedMachines,
-		notReadyMachines,
-		readyMachines,
+		childMachines.Items,
+		deletedMachineCount,
 	); err != nil || res.Requeue || res.RequeueAfter != 0 {
 		return res, err
 	}
@@ -145,7 +145,7 @@ func (r *LoadBalancerSetReconciler) deletionRoutine(
 func (r *LoadBalancerSetReconciler) reconcileStatus(
 	ctx context.Context,
 	set *yawolv1beta1.LoadBalancerSet,
-	readyMachines []yawolv1beta1.LoadBalancerMachine,
+	readyMachinesCount int,
 ) (ctrl.Result, error) {
 	// Write replicas into status
 	if set.Status.Replicas == nil || *set.Status.Replicas != set.Spec.Replicas {
@@ -159,8 +159,8 @@ func (r *LoadBalancerSetReconciler) reconcileStatus(
 	}
 
 	// Write ready replicas into status
-	if set.Status.ReadyReplicas == nil || *set.Status.ReadyReplicas != len(readyMachines) {
-		replicas := len(readyMachines)
+	if set.Status.ReadyReplicas == nil || *set.Status.ReadyReplicas != readyMachinesCount {
+		replicas := readyMachinesCount
 		if err := r.patchLoadBalancerSetStatus(ctx, set, yawolv1beta1.LoadBalancerSetStatus{
 			ReadyReplicas: &replicas,
 		}); err != nil {
@@ -175,39 +175,49 @@ func (r *LoadBalancerSetReconciler) reconcileStatus(
 func (r *LoadBalancerSetReconciler) reconcileReplicas(
 	ctx context.Context,
 	set *yawolv1beta1.LoadBalancerSet,
-	deletedMachines, notReadyMachines, readyMachines []yawolv1beta1.LoadBalancerMachine,
+	machines []yawolv1beta1.LoadBalancerMachine,
+	deletedMachineCount int,
 ) (ctrl.Result, error) {
-	var currentReplicas, desiredReplicas, deletedReplicas int
-	desiredReplicas = set.Spec.Replicas
-	currentReplicas = len(deletedMachines) + len(notReadyMachines) + len(readyMachines)
-	deletedReplicas = len(deletedMachines)
-
-	if currentReplicas < desiredReplicas {
+	if len(machines) < set.Spec.Replicas {
 		if err := r.createMachine(ctx, set); err != nil {
 			return ctrl.Result{}, err
 		}
 		return ctrl.Result{RequeueAfter: time.Second * 2}, nil
 	}
 
-	if desiredReplicas < currentReplicas-deletedReplicas {
-		// Delete not ready machines first
-		if len(notReadyMachines) > 0 {
-			if err := r.deleteMachine(ctx, &notReadyMachines[0]); err != nil {
-				return ctrl.Result{}, err
-			}
-			return ctrl.Result{RequeueAfter: time.Second * 2}, nil
+	if len(machines) < len(machines)-deletedMachineCount {
+		machineForDeletion := findFirstMachineForDeletion(machines)
+
+		if err := r.deleteMachine(ctx, &machineForDeletion); err != nil {
+			return ctrl.Result{}, err
 		}
+		return ctrl.Result{RequeueAfter: time.Second * 2}, nil
+	}
 
-		// Delete ready machines
-		if len(readyMachines) > 0 {
-			if err := r.deleteMachine(ctx, &readyMachines[0]); err != nil {
-				return ctrl.Result{}, err
+	return ctrl.Result{}, nil
+}
+
+// findFirstMachineForDeletion returns a machine which should be deleted first in case of a scale down.
+// returns unready machines first
+// returns machine which is not keepalived master
+// returns the first machine
+func findFirstMachineForDeletion(machines []yawolv1beta1.LoadBalancerMachine) yawolv1beta1.LoadBalancerMachine {
+	for i := range machines {
+		if !isMachineReady(machines[i]) {
+			return machines[i]
+		}
+	}
+
+	for i := range machines {
+		for _, c := range *machines[i].Status.Conditions {
+			if string(c.Type) == string(helper.KeepalivedMaster) &&
+				string(c.Status) != string(helper.ConditionFalse) {
+				return machines[i]
 			}
-			return ctrl.Result{RequeueAfter: time.Second * 2}, nil
 		}
 	}
 
-	return ctrl.Result{}, nil
+	return machines[0]
 }
 
 func (r *LoadBalancerSetReconciler) patchLoadBalancerSetStatus(
@@ -266,20 +276,20 @@ func shouldMachineBeDeleted(machine yawolv1beta1.LoadBalancerMachine) (bool, err
 
 // Decides whether the machine is ready or not
 // False if not all conditions are set
-// False if LastHeartbeatTime is older than 60sec
+// False if LastHeartbeatTime is older than 180sec
 // False if ConfigReady, EnvoyReady or EnvoyUpToDate are false
 func isMachineReady(machine yawolv1beta1.LoadBalancerMachine) bool {
-	before60seconds := v1.Time{Time: time.Now().Add(-60 * time.Second)}
+	before180seconds := v1.Time{Time: time.Now().Add(-180 * time.Second)}
 
 	// not ready if no conditions are available
-	if machine.Status.Conditions == nil || len(*machine.Status.Conditions) < 3 {
+	if machine.Status.Conditions == nil || len(*machine.Status.Conditions) < 6 {
 		return false
 	}
 
 	// As soon as a conditions are set
 	if machine.Status.Conditions != nil {
 		for _, condition := range *machine.Status.Conditions {
-			if condition.LastHeartbeatTime.Before(&before60seconds) {
+			if condition.LastHeartbeatTime.Before(&before180seconds) {
 				return false
 			}
 			if helper.LoadBalancerSetConditionIsFalse(condition) {