Bug 1743194: For nodes that have been deleted during upgrade, recreate them

pkg/k8shandler/deployment.go

@@ -295,6 +295,17 @@ func (node *deploymentNode) waitForNodeLeaveCluster() (error, bool) {
 	return err, (err == nil)
 }
 
+func (node *deploymentNode) isMissing() bool {
+	getNode := &apps.Deployment{}
+	if getErr := node.client.Get(context.TODO(), types.NamespacedName{Name: node.name(), Namespace: node.self.Namespace}, getNode); getErr != nil {
+		if errors.IsNotFound(getErr) {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (node *deploymentNode) restart(upgradeStatus *api.ElasticsearchNodeStatus) {
 
 	if upgradeStatus.UpgradeStatus.UnderUpgrade != v1.ConditionTrue {
@@ -351,6 +362,11 @@ func (node *deploymentNode) restart(upgradeStatus *api.ElasticsearchNodeStatus)
 
 	if upgradeStatus.UpgradeStatus.UpgradePhase == api.NodeRestarting {
 
+		// if the node doesn't exist -- create it
+		if node.isMissing() {
+			node.create()
+		}
+
 		if err := node.setReplicaCount(1); err != nil {
 			logrus.Warnf("Unable to scale up %v", node.name())
 			return

pkg/k8shandler/nodetypefactory.go

@@ -19,6 +19,7 @@ type NodeTypeInterface interface {
 	name() string
 	updateReference(node NodeTypeInterface)
 	delete()
+	isMissing() bool
 }
 
 // NodeTypeFactory is a factory to construct either statefulset or deployment

pkg/k8shandler/persistentvolumeclaims.go

@@ -4,20 +4,34 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/sirupsen/logrus"
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func createOrUpdatePersistentVolumeClaim(pvc v1.PersistentVolumeClaimSpec, newName, namespace string, client client.Client) error {
 
-	claim := createPersistentVolumeClaim(newName, namespace, pvc)
-	err := client.Create(context.TODO(), claim)
-	if err != nil {
-		if !errors.IsAlreadyExists(err) {
-			return fmt.Errorf("Unable to create PVC: %v", err)
+	// for some reason if the PVC already exists but creating it again would violate
+	// quota we get an error regarding quota not that it already exists
+	// so check to see if it already exists
+	claim := &v1.PersistentVolumeClaim{}
+
+	if getErr := client.Get(context.TODO(), types.NamespacedName{Name: newName, Namespace: namespace}, claim); getErr != nil {
+		if errors.IsNotFound(getErr) {
+			claim = createPersistentVolumeClaim(newName, namespace, pvc)
+			err := client.Create(context.TODO(), claim)
+			if err != nil {
+				if !errors.IsAlreadyExists(err) {
+					return fmt.Errorf("Unable to create PVC: %v", err)
+				}
+			}
+		} else {
+			logrus.Debugf("Could not get PVC %v: %v", newName, getErr)
+			return getErr
 		}
 	}
 

pkg/k8shandler/statefulset.go

@@ -219,6 +219,17 @@ func (node *statefulSetNode) replicaCount() (int32, error) {
 	return desired.Status.Replicas, nil
 }
 
+func (node *statefulSetNode) isMissing() bool {
+	getNode := &apps.StatefulSet{}
+	if getErr := node.client.Get(context.TODO(), types.NamespacedName{Name: node.name(), Namespace: node.self.Namespace}, getNode); getErr != nil {
+		if errors.IsNotFound(getErr) {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (node *statefulSetNode) restart(upgradeStatus *api.ElasticsearchNodeStatus) {
 
 	if upgradeStatus.UpgradeStatus.UnderUpgrade != v1.ConditionTrue {
@@ -254,6 +265,12 @@ func (node *statefulSetNode) restart(upgradeStatus *api.ElasticsearchNodeStatus)
 
 	if upgradeStatus.UpgradeStatus.UpgradePhase == api.NodeRestarting {
 
+		// if the node doesn't exist -- create it
+		// TODO: we can skip this logic after
+		if node.isMissing() {
+			node.create()
+		}
+
 		ordinal, err := node.partition()
 		if err != nil {
 			logrus.Infof("Unable to get node ordinal value: %v", err)