[Feature] [Platform] Update Failed Releases (#2002)

Author: ajanikow

.golangci.yaml

@@ -195,6 +195,8 @@ linters-settings:
         pkg: github.com/arangodb/kube-arangodb/pkg/util/constants
       - alias: tcache
         pkg: github.com/arangodb/kube-arangodb/pkg/util/tests/cache
+      - alias: helmRelease
+        pkg: helm.sh/helm/v3/pkg/release
       - alias: apps
         pkg: k8s.io/api/apps/v1
       - alias: batch

CHANGELOG.md

@@ -6,6 +6,7 @@
 - (Bugfix) (Platform) Fix topology for Gateways
 - (Feature) (Platform) Dump CLI switch to Services
 - (Feature) (Platform) Fix ImagePullSecrets Merge
+- (Feature) (Platform) Update Failed Releases
 
 ## [1.3.2](https://github.com/arangodb/kube-arangodb/tree/1.3.2) (2025-11-20)
 - (Bugfix) (Platform) Increase memory limit for Inventory

integrations/scheduler/v2/definition/helpers.go

@@ -22,7 +22,7 @@ package definition
 
 import (
 	"helm.sh/helm/v3/pkg/action"
-	"helm.sh/helm/v3/pkg/release"
+	helmRelease "helm.sh/helm/v3/pkg/release"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/selection"
@@ -31,50 +31,50 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/helm"
 )
 
-func (i SchedulerV2ReleaseInfoStatus) AsHelmStatus() release.Status {
+func (i SchedulerV2ReleaseInfoStatus) AsHelmStatus() helmRelease.Status {
 	switch i {
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNKNOWN_UNSPECIFIED:
-		return release.StatusUnknown
+		return helmRelease.StatusUnknown
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_DEPLOYED:
-		return release.StatusDeployed
+		return helmRelease.StatusDeployed
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNINSTALLED:
-		return release.StatusUninstalled
+		return helmRelease.StatusUninstalled
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_SUPERSEDED:
-		return release.StatusSuperseded
+		return helmRelease.StatusSuperseded
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_FAILED:
-		return release.StatusFailed
+		return helmRelease.StatusFailed
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNINSTALLING:
-		return release.StatusUninstalling
+		return helmRelease.StatusUninstalling
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGINSTALL:
-		return release.StatusPendingInstall
+		return helmRelease.StatusPendingInstall
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGUPGRADE:
-		return release.StatusPendingUpgrade
+		return helmRelease.StatusPendingUpgrade
 	case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGROLLBACK:
-		return release.StatusPendingRollback
+		return helmRelease.StatusPendingRollback
 	default:
-		return release.StatusUnknown
+		return helmRelease.StatusUnknown
 	}
 }
 
-func FromHelmStatus(in release.Status) SchedulerV2ReleaseInfoStatus {
+func FromHelmStatus(in helmRelease.Status) SchedulerV2ReleaseInfoStatus {
 	switch in {
-	case release.StatusUnknown:
+	case helmRelease.StatusUnknown:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNKNOWN_UNSPECIFIED
-	case release.StatusDeployed:
+	case helmRelease.StatusDeployed:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_DEPLOYED
-	case release.StatusUninstalled:
+	case helmRelease.StatusUninstalled:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNINSTALLED
-	case release.StatusSuperseded:
+	case helmRelease.StatusSuperseded:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_SUPERSEDED
-	case release.StatusFailed:
+	case helmRelease.StatusFailed:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_FAILED
-	case release.StatusUninstalling:
+	case helmRelease.StatusUninstalling:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNINSTALLING
-	case release.StatusPendingInstall:
+	case helmRelease.StatusPendingInstall:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGINSTALL
-	case release.StatusPendingUpgrade:
+	case helmRelease.StatusPendingUpgrade:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGUPGRADE
-	case release.StatusPendingRollback:
+	case helmRelease.StatusPendingRollback:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGROLLBACK
 	default:
 		return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNKNOWN_UNSPECIFIED

pkg/apis/platform/v1alpha1/service_status_release.go

@@ -21,7 +21,7 @@
 package v1alpha1
 
 import (
-	"helm.sh/helm/v3/pkg/release"
+	helmRelease "helm.sh/helm/v3/pkg/release"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -33,7 +33,7 @@ type ArangoPlatformServiceStatusRelease struct {
 }
 
 type ArangoPlatformServiceStatusReleaseInfo struct {
-	FirstDeployed *meta.Time     `json:"first_deployed,omitempty"`
-	LastDeployed  *meta.Time     `json:"last_deployed,omitempty"`
-	Status        release.Status `json:"status,omitempty"`
+	FirstDeployed *meta.Time         `json:"first_deployed,omitempty"`
+	LastDeployed  *meta.Time         `json:"last_deployed,omitempty"`
+	Status        helmRelease.Status `json:"status,omitempty"`
 }

pkg/apis/platform/v1beta1/service_status_release.go

@@ -21,8 +21,10 @@
 package v1beta1
 
 import (
-	"helm.sh/helm/v3/pkg/release"
+	helmRelease "helm.sh/helm/v3/pkg/release"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
 )
 
 type ArangoPlatformServiceStatusRelease struct {
@@ -32,8 +34,35 @@ type ArangoPlatformServiceStatusRelease struct {
 	Info    ArangoPlatformServiceStatusReleaseInfo `json:"info"`
 }
 
+func (a *ArangoPlatformServiceStatusRelease) Compare(o *ArangoPlatformServiceStatusRelease) bool {
+	if a == nil && o == nil {
+		return true
+	}
+	if a == nil || o == nil {
+		return false
+	}
+
+	return a.Name == o.Name &&
+		a.Version == o.Version &&
+		a.Hash == o.Hash &&
+		a.Info.Compare(&o.Info)
+}
+
 type ArangoPlatformServiceStatusReleaseInfo struct {
-	FirstDeployed *meta.Time     `json:"first_deployed,omitempty"`
-	LastDeployed  *meta.Time     `json:"last_deployed,omitempty"`
-	Status        release.Status `json:"status,omitempty"`
+	FirstDeployed *meta.Time         `json:"first_deployed,omitempty"`
+	LastDeployed  *meta.Time         `json:"last_deployed,omitempty"`
+	Status        helmRelease.Status `json:"status,omitempty"`
+}
+
+func (a *ArangoPlatformServiceStatusReleaseInfo) Compare(o *ArangoPlatformServiceStatusReleaseInfo) bool {
+	if a == nil && o == nil {
+		return true
+	}
+	if a == nil || o == nil {
+		return false
+	}
+
+	return util.TimeCompareEqualPointer(a.FirstDeployed, o.FirstDeployed) &&
+		util.TimeCompareEqualPointer(a.LastDeployed, o.LastDeployed) &&
+		a.Status == o.Status
 }

pkg/handlers/platform/service/handler.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"helm.sh/helm/v3/pkg/action"
+	helmRelease "helm.sh/helm/v3/pkg/release"
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -413,7 +414,41 @@ func (h *handler) HandleRelease(ctx context.Context, item operation.Item, extens
 		return true, operator.Reconcile("Release Upgraded")
 	}
 
-	return false, nil
+	if s := extractReleaseStatus(release, expectedChecksum); !s.Compare(status.Release) {
+		logger.WrapObj(item).Info("Release Update")
+		status.Release = s
+		return true, operator.Reconcile("Release Updated")
+	}
+
+	switch status.Release.Info.Status {
+	case helmRelease.StatusDeployed:
+		return false, nil
+
+	case helmRelease.StatusFailed:
+		// Try to upgrade
+		logger.WrapObj(item).Info("Upgrade Helm Release")
+
+		_, err = h.helm.Upgrade(ctx, extension.GetName(), helm.Chart(status.ChartInfo.Definition), helm.Values(status.Values), func(in *action.Upgrade) {
+			in.Namespace = extension.GetNamespace()
+
+			in.Labels = labels.GetLabels(status.Deployment.GetName(), status.Chart.GetName())
+
+			in.Timeout = 20 * time.Minute
+		})
+		if err != nil {
+			h.eventRecorder.Warning(extension, "Release Upgrade Failed", "Release upgrade failed: %s", err.Error())
+
+			return false, err
+		}
+
+		status.Release = extractReleaseStatus(release, expectedChecksum)
+
+		h.eventRecorder.Normal(extension, "Release Upgraded", "Release upgraded with version %d on chart %s (%s)", status.Release.Version, status.ChartInfo.Details.Name, status.ChartInfo.Details.Version)
+
+		return true, operator.Reconcile("Release Upgraded")
+	}
+
+	return false, operator.Stop("Invalid release status: %s", status.Release.Info.Status)
 }
 
 func extractReleaseStatus(in *helm.Release, hash string) *platformApi.ArangoPlatformServiceStatusRelease {

pkg/util/k8sutil/helm/types.go

@@ -25,7 +25,7 @@ import (
 	"time"
 
 	"helm.sh/helm/v3/pkg/chart"
-	"helm.sh/helm/v3/pkg/release"
+	helmRelease "helm.sh/helm/v3/pkg/release"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -34,7 +34,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 )
 
-func fromHelmRelease(in *release.Release) (Release, error) {
+func fromHelmRelease(in *helmRelease.Release) (Release, error) {
 	var r Release
 
 	if in == nil {
@@ -88,7 +88,7 @@ func fromHelmReleaseChartMetadata(in *chart.Metadata) *ReleaseChartMetadata {
 	return &r
 }
 
-func fromHelmReleaseInfo(in *release.Info) (ReleaseInfo, error) {
+func fromHelmReleaseInfo(in *helmRelease.Info) (ReleaseInfo, error) {
 	var r ReleaseInfo
 
 	if in == nil {
@@ -191,13 +191,13 @@ func (r *ReleaseChartMetadata) GetVersion() string {
 }
 
 type ReleaseInfo struct {
-	FirstDeployed time.Time      `json:"first_deployed,omitempty"`
-	LastDeployed  time.Time      `json:"last_deployed,omitempty"`
-	Deleted       time.Time      `json:"deleted,omitempty"`
-	Description   string         `json:"description,omitempty"`
-	Status        release.Status `json:"status,omitempty"`
-	Notes         string         `json:"notes,omitempty"`
-	Resources     Resources      `json:"resources,omitempty"`
+	FirstDeployed time.Time          `json:"first_deployed,omitempty"`
+	LastDeployed  time.Time          `json:"last_deployed,omitempty"`
+	Deleted       time.Time          `json:"deleted,omitempty"`
+	Description   string             `json:"description,omitempty"`
+	Status        helmRelease.Status `json:"status,omitempty"`
+	Notes         string             `json:"notes,omitempty"`
+	Resources     Resources          `json:"resources,omitempty"`
 }
 
 type Resources []Resource