fix:hang for serverUpdate failed (#1659)

Co-authored-by: guozhi.li <guozhi.li@daocloud.io> Co-authored-by: Harshavardhana <harsha@minio.io>
minio · Jun 27, 2023 · 5b71f04 · 5b71f04
1 parent 9562a29
commit 5b71f04
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 44 deletions.
diff --git a/examples/kustomization/tenant-certmanager-kes/vault.yaml b/examples/kustomization/tenant-certmanager-kes/vault.yaml
@@ -30,7 +30,7 @@ spec:
     spec:
       containers:
         - name: vault
-          image: vault:latest
+          image: vault:1.13.3
           imagePullPolicy: "IfNotPresent"
           env:
             - name: SECRET_SHARES

diff --git a/examples/vault/deployment.yaml b/examples/vault/deployment.yaml
@@ -30,7 +30,7 @@ spec:
     spec:
       containers:
         - name: vault
-          image: vault:latest
+          image: vault:1.13.3
           imagePullPolicy: "IfNotPresent"
           env:
             - name: SECRET_SHARES

diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go
@@ -26,8 +26,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/minio/madmin-go/v2"
 	"github.com/minio/operator/pkg/common"
-
 	xcerts "github.com/minio/pkg/certs"
 
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -1090,20 +1090,17 @@ func (c *Controller) syncHandler(key string) error {
 			tenantName, images[0], tenant.Spec.Image)
 
 		latest, err := c.fetchArtifacts(tenant)
+		defer c.removeArtifacts()
 		if err != nil {
-			_ = c.removeArtifacts()
 			return err
 		}
-
 		updateURL, err := tenant.UpdateURL(latest, fmt.Sprintf("http://operator.%s.svc.%s:%s%s",
 			miniov2.GetNSFromFile(),
 			miniov2.GetClusterDomain(),
 			common.UpgradeServerPort,
 			common.WebhookAPIUpdate,
 		))
 		if err != nil {
-			_ = c.removeArtifacts()
-
 			err = fmt.Errorf("Unable to get canonical update URL for Tenant '%s', failed with %v", tenantName, err)
 			if _, terr := c.updateTenantStatus(ctx, tenant, err.Error(), totalAvailableReplicas); terr != nil {
 				return terr
@@ -1118,49 +1115,45 @@ func (c *Controller) syncHandler(key string) error {
 
 		us, err := adminClnt.ServerUpdate(ctx, updateURL)
 		if err != nil {
-			_ = c.removeArtifacts()
-
-			err = fmt.Errorf("Tenant '%s' MinIO update failed with %w", tenantName, err)
-			if _, terr := c.updateTenantStatus(ctx, tenant, err.Error(), totalAvailableReplicas); terr != nil {
-				return terr
-			}
-
-			// Update failed, nothing needs to be changed in the container
-			return err
-		}
-
-		if us.CurrentVersion != us.UpdatedVersion {
-			// In case the upgrade is from an older version to RELEASE.2021-07-27T02-40-15Z (which introduced
-			// MinIO server integrated with Console), we need to delete the old console deployment and service.
-			// We do this only when MinIO server is successfully updated.
-			unifiedConsoleReleaseTime, _ := miniov2.ReleaseTagToReleaseTime("RELEASE.2021-07-27T02-40-15Z")
-			newVer, err := miniov2.ReleaseTagToReleaseTime(us.UpdatedVersion)
-			if err != nil {
-				klog.Errorf("Unsupported release tag on new image, server updated but might leave dangling console deployment %v", err)
+			if madmin.ToErrorResponse(err).Code != "MethodNotAllowed" {
+				if _, terr := c.updateTenantStatus(ctx, tenant, err.Error(), totalAvailableReplicas); terr != nil {
+					return terr
+				}
+				// Update failed, nothing needs to be changed in the container
 				return err
 			}
-			consoleDeployment, err := c.deploymentLister.Deployments(tenant.Namespace).Get(tenant.ConsoleDeploymentName())
-			if unifiedConsoleReleaseTime.Before(newVer) && consoleDeployment != nil && err == nil {
-				if err := c.deleteOldConsoleDeployment(ctx, tenant, consoleDeployment.Name); err != nil {
+			c.RegisterEvent(ctx, tenant, corev1.EventTypeWarning, "Inplace update is disabled, falling back to performing only statefulset update.", fmt.Sprintf("Tenant %s", tenant.Name))
+		}
+		if err == nil {
+			if us.CurrentVersion != us.UpdatedVersion {
+				// In case the upgrade is from an older version to RELEASE.2021-07-27T02-40-15Z (which introduced
+				// MinIO server integrated with Console), we need to delete the old console deployment and service.
+				// We do this only when MinIO server is successfully updated.
+				unifiedConsoleReleaseTime, _ := miniov2.ReleaseTagToReleaseTime("RELEASE.2021-07-27T02-40-15Z")
+				newVer, err := miniov2.ReleaseTagToReleaseTime(us.UpdatedVersion)
+				if err != nil {
+					klog.Errorf("Unsupported release tag on new image, server updated but might leave dangling console deployment %v", err)
 					return err
 				}
+				consoleDeployment, err := c.deploymentLister.Deployments(tenant.Namespace).Get(tenant.ConsoleDeploymentName())
+				if unifiedConsoleReleaseTime.Before(newVer) && consoleDeployment != nil && err == nil {
+					if err := c.deleteOldConsoleDeployment(ctx, tenant, consoleDeployment.Name); err != nil {
+						return err
+					}
+				}
+				klog.Infof("Tenant '%s' MinIO updated successfully from: %s, to: %s successfully",
+					tenantName, us.CurrentVersion, us.UpdatedVersion)
+			} else {
+				msg := fmt.Sprintf("Tenant '%s' MinIO is already running the most recent version of %s",
+					tenantName,
+					us.CurrentVersion)
+				klog.Info(msg)
+				if _, terr := c.updateTenantStatus(ctx, tenant, msg, totalAvailableReplicas); terr != nil {
+					return err
+				}
+				return nil
 			}
-			klog.Infof("Tenant '%s' MinIO updated successfully from: %s, to: %s successfully",
-				tenantName, us.CurrentVersion, us.UpdatedVersion)
-		} else {
-			msg := fmt.Sprintf("Tenant '%s' MinIO is already running the most recent version of %s",
-				tenantName,
-				us.CurrentVersion)
-			klog.Info(msg)
-			if _, terr := c.updateTenantStatus(ctx, tenant, msg, totalAvailableReplicas); terr != nil {
-				return err
-			}
-			return nil
 		}
-
-		// clean the local directory
-		_ = c.removeArtifacts()
-
 		for i, pool := range tenant.Spec.Pools {
 			// Now proceed to make the yaml changes for the tenant statefulset.
 			ss := statefulsets.NewPool(&statefulsets.NewPoolArgs{