diff --git a/.golangci.yaml b/.golangci.yaml index 59d103bdf..ca3f9dd44 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -195,6 +195,8 @@ linters-settings: pkg: github.com/arangodb/kube-arangodb/pkg/util/constants - alias: tcache pkg: github.com/arangodb/kube-arangodb/pkg/util/tests/cache + - alias: helmRelease + pkg: helm.sh/helm/v3/pkg/release - alias: apps pkg: k8s.io/api/apps/v1 - alias: batch diff --git a/CHANGELOG.md b/CHANGELOG.md index 869c3b536..44f5f99a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - (Bugfix) (Platform) Fix topology for Gateways - (Feature) (Platform) Dump CLI switch to Services - (Feature) (Platform) Fix ImagePullSecrets Merge +- (Feature) (Platform) Update Failed Releases ## [1.3.2](https://github.com/arangodb/kube-arangodb/tree/1.3.2) (2025-11-20) - (Bugfix) (Platform) Increase memory limit for Inventory diff --git a/integrations/scheduler/v2/definition/helpers.go b/integrations/scheduler/v2/definition/helpers.go index 2c9914faa..9c61ec292 100644 --- a/integrations/scheduler/v2/definition/helpers.go +++ b/integrations/scheduler/v2/definition/helpers.go @@ -22,7 +22,7 @@ package definition import ( "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/release" + helmRelease "helm.sh/helm/v3/pkg/release" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/selection" @@ -31,50 +31,50 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/helm" ) -func (i SchedulerV2ReleaseInfoStatus) AsHelmStatus() release.Status { +func (i SchedulerV2ReleaseInfoStatus) AsHelmStatus() helmRelease.Status { switch i { case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNKNOWN_UNSPECIFIED: - return release.StatusUnknown + return helmRelease.StatusUnknown case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_DEPLOYED: - return release.StatusDeployed + return helmRelease.StatusDeployed case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNINSTALLED: - return release.StatusUninstalled + return helmRelease.StatusUninstalled case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_SUPERSEDED: - return release.StatusSuperseded + return helmRelease.StatusSuperseded case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_FAILED: - return release.StatusFailed + return helmRelease.StatusFailed case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNINSTALLING: - return release.StatusUninstalling + return helmRelease.StatusUninstalling case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGINSTALL: - return release.StatusPendingInstall + return helmRelease.StatusPendingInstall case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGUPGRADE: - return release.StatusPendingUpgrade + return helmRelease.StatusPendingUpgrade case SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGROLLBACK: - return release.StatusPendingRollback + return helmRelease.StatusPendingRollback default: - return release.StatusUnknown + return helmRelease.StatusUnknown } } -func FromHelmStatus(in release.Status) SchedulerV2ReleaseInfoStatus { +func FromHelmStatus(in helmRelease.Status) SchedulerV2ReleaseInfoStatus { switch in { - case release.StatusUnknown: + case helmRelease.StatusUnknown: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNKNOWN_UNSPECIFIED - case release.StatusDeployed: + case helmRelease.StatusDeployed: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_DEPLOYED - case release.StatusUninstalled: + case helmRelease.StatusUninstalled: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNINSTALLED - case release.StatusSuperseded: + case helmRelease.StatusSuperseded: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_SUPERSEDED - case release.StatusFailed: + case helmRelease.StatusFailed: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_FAILED - case release.StatusUninstalling: + case helmRelease.StatusUninstalling: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNINSTALLING - case release.StatusPendingInstall: + case helmRelease.StatusPendingInstall: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGINSTALL - case release.StatusPendingUpgrade: + case helmRelease.StatusPendingUpgrade: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGUPGRADE - case release.StatusPendingRollback: + case helmRelease.StatusPendingRollback: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_PENDINGROLLBACK default: return SchedulerV2ReleaseInfoStatus_SCHEDULER_V2_RELEASE_INFO_STATUS_UNKNOWN_UNSPECIFIED diff --git a/pkg/apis/platform/v1alpha1/service_status_release.go b/pkg/apis/platform/v1alpha1/service_status_release.go index adda6f215..af7482d29 100644 --- a/pkg/apis/platform/v1alpha1/service_status_release.go +++ b/pkg/apis/platform/v1alpha1/service_status_release.go @@ -21,7 +21,7 @@ package v1alpha1 import ( - "helm.sh/helm/v3/pkg/release" + helmRelease "helm.sh/helm/v3/pkg/release" meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -33,7 +33,7 @@ type ArangoPlatformServiceStatusRelease struct { } type ArangoPlatformServiceStatusReleaseInfo struct { - FirstDeployed *meta.Time `json:"first_deployed,omitempty"` - LastDeployed *meta.Time `json:"last_deployed,omitempty"` - Status release.Status `json:"status,omitempty"` + FirstDeployed *meta.Time `json:"first_deployed,omitempty"` + LastDeployed *meta.Time `json:"last_deployed,omitempty"` + Status helmRelease.Status `json:"status,omitempty"` } diff --git a/pkg/apis/platform/v1beta1/service_status_release.go b/pkg/apis/platform/v1beta1/service_status_release.go index a3c54022c..5d765f872 100644 --- a/pkg/apis/platform/v1beta1/service_status_release.go +++ b/pkg/apis/platform/v1beta1/service_status_release.go @@ -21,8 +21,10 @@ package v1beta1 import ( - "helm.sh/helm/v3/pkg/release" + helmRelease "helm.sh/helm/v3/pkg/release" meta "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/kube-arangodb/pkg/util" ) type ArangoPlatformServiceStatusRelease struct { @@ -32,8 +34,35 @@ type ArangoPlatformServiceStatusRelease struct { Info ArangoPlatformServiceStatusReleaseInfo `json:"info"` } +func (a *ArangoPlatformServiceStatusRelease) Compare(o *ArangoPlatformServiceStatusRelease) bool { + if a == nil && o == nil { + return true + } + if a == nil || o == nil { + return false + } + + return a.Name == o.Name && + a.Version == o.Version && + a.Hash == o.Hash && + a.Info.Compare(&o.Info) +} + type ArangoPlatformServiceStatusReleaseInfo struct { - FirstDeployed *meta.Time `json:"first_deployed,omitempty"` - LastDeployed *meta.Time `json:"last_deployed,omitempty"` - Status release.Status `json:"status,omitempty"` + FirstDeployed *meta.Time `json:"first_deployed,omitempty"` + LastDeployed *meta.Time `json:"last_deployed,omitempty"` + Status helmRelease.Status `json:"status,omitempty"` +} + +func (a *ArangoPlatformServiceStatusReleaseInfo) Compare(o *ArangoPlatformServiceStatusReleaseInfo) bool { + if a == nil && o == nil { + return true + } + if a == nil || o == nil { + return false + } + + return util.TimeCompareEqualPointer(a.FirstDeployed, o.FirstDeployed) && + util.TimeCompareEqualPointer(a.LastDeployed, o.LastDeployed) && + a.Status == o.Status } diff --git a/pkg/handlers/platform/service/handler.go b/pkg/handlers/platform/service/handler.go index 93e2a7aee..080730c4e 100644 --- a/pkg/handlers/platform/service/handler.go +++ b/pkg/handlers/platform/service/handler.go @@ -25,6 +25,7 @@ import ( "time" "helm.sh/helm/v3/pkg/action" + helmRelease "helm.sh/helm/v3/pkg/release" apiErrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -413,7 +414,41 @@ func (h *handler) HandleRelease(ctx context.Context, item operation.Item, extens return true, operator.Reconcile("Release Upgraded") } - return false, nil + if s := extractReleaseStatus(release, expectedChecksum); !s.Compare(status.Release) { + logger.WrapObj(item).Info("Release Update") + status.Release = s + return true, operator.Reconcile("Release Updated") + } + + switch status.Release.Info.Status { + case helmRelease.StatusDeployed: + return false, nil + + case helmRelease.StatusFailed: + // Try to upgrade + logger.WrapObj(item).Info("Upgrade Helm Release") + + _, err = h.helm.Upgrade(ctx, extension.GetName(), helm.Chart(status.ChartInfo.Definition), helm.Values(status.Values), func(in *action.Upgrade) { + in.Namespace = extension.GetNamespace() + + in.Labels = labels.GetLabels(status.Deployment.GetName(), status.Chart.GetName()) + + in.Timeout = 20 * time.Minute + }) + if err != nil { + h.eventRecorder.Warning(extension, "Release Upgrade Failed", "Release upgrade failed: %s", err.Error()) + + return false, err + } + + status.Release = extractReleaseStatus(release, expectedChecksum) + + h.eventRecorder.Normal(extension, "Release Upgraded", "Release upgraded with version %d on chart %s (%s)", status.Release.Version, status.ChartInfo.Details.Name, status.ChartInfo.Details.Version) + + return true, operator.Reconcile("Release Upgraded") + } + + return false, operator.Stop("Invalid release status: %s", status.Release.Info.Status) } func extractReleaseStatus(in *helm.Release, hash string) *platformApi.ArangoPlatformServiceStatusRelease { diff --git a/pkg/util/k8sutil/helm/types.go b/pkg/util/k8sutil/helm/types.go index aa0abf544..66b1ce3bf 100644 --- a/pkg/util/k8sutil/helm/types.go +++ b/pkg/util/k8sutil/helm/types.go @@ -25,7 +25,7 @@ import ( "time" "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/release" + helmRelease "helm.sh/helm/v3/pkg/release" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -34,7 +34,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/errors" ) -func fromHelmRelease(in *release.Release) (Release, error) { +func fromHelmRelease(in *helmRelease.Release) (Release, error) { var r Release if in == nil { @@ -88,7 +88,7 @@ func fromHelmReleaseChartMetadata(in *chart.Metadata) *ReleaseChartMetadata { return &r } -func fromHelmReleaseInfo(in *release.Info) (ReleaseInfo, error) { +func fromHelmReleaseInfo(in *helmRelease.Info) (ReleaseInfo, error) { var r ReleaseInfo if in == nil { @@ -191,13 +191,13 @@ func (r *ReleaseChartMetadata) GetVersion() string { } type ReleaseInfo struct { - FirstDeployed time.Time `json:"first_deployed,omitempty"` - LastDeployed time.Time `json:"last_deployed,omitempty"` - Deleted time.Time `json:"deleted,omitempty"` - Description string `json:"description,omitempty"` - Status release.Status `json:"status,omitempty"` - Notes string `json:"notes,omitempty"` - Resources Resources `json:"resources,omitempty"` + FirstDeployed time.Time `json:"first_deployed,omitempty"` + LastDeployed time.Time `json:"last_deployed,omitempty"` + Deleted time.Time `json:"deleted,omitempty"` + Description string `json:"description,omitempty"` + Status helmRelease.Status `json:"status,omitempty"` + Notes string `json:"notes,omitempty"` + Resources Resources `json:"resources,omitempty"` } type Resources []Resource