From 0adcad7596a88c0236a6e7a6bcd1bda81a452e68 Mon Sep 17 00:00:00 2001 From: vishal Date: Fri, 25 Sep 2020 09:58:02 -0400 Subject: [PATCH] Rollback on update fail --- pkg/operator/resources/batchapi/api.go | 32 +++++++++++++++-- pkg/operator/resources/realtimeapi/api.go | 34 +++++++++++++++++-- pkg/operator/resources/trafficsplitter/api.go | 32 +++++++++++++++-- 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/pkg/operator/resources/batchapi/api.go b/pkg/operator/resources/batchapi/api.go index 6c6cad3194..41ff139fe3 100644 --- a/pkg/operator/resources/batchapi/api.go +++ b/pkg/operator/resources/batchapi/api.go @@ -77,11 +77,19 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string) (*spec.API, string, err = applyK8sResources(api, prevVirtualService) if err != nil { - return nil, "", err + rollBackErr := rollBack(prevVirtualService) + if rollBackErr != nil { + return nil, "", err + } + return nil, "update failed; rollbacked to previous deployment", nil } if err := operator.UpdateAPIGatewayK8s(prevVirtualService, api, true); err != nil { - return nil, "", err + rollBackErr := rollBack(prevVirtualService) + if rollBackErr != nil { + return nil, "", err + } + return nil, "update failed; rollbacked to previous deployment", nil } return api, fmt.Sprintf("updated %s", api.Resource.UserString()), nil @@ -90,6 +98,26 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string) (*spec.API, string, return api, fmt.Sprintf("%s is up to date", api.Resource.UserString()), nil } +func rollBack(prevVirtualService *istioclientnetworking.VirtualService) error { + prevAPIID := prevVirtualService.Labels["apiID"] + prevAPIName := prevVirtualService.Labels["apiName"] + + prevAPI, err := operator.DownloadAPISpec(prevAPIName, prevAPIID) + if err != nil { + return err + } + + if err := applyK8sResources(prevAPI, prevVirtualService); err != nil { + return err + } + + if err := operator.UpdateAPIGatewayK8s(prevVirtualService, prevAPI, false); err != nil { + return err + } + + return nil +} + func DeleteAPI(apiName string, keepCache bool) error { // best effort deletion, so don't handle error yet virtualService, vsErr := config.K8s.GetVirtualService(operator.K8sName(apiName)) diff --git a/pkg/operator/resources/realtimeapi/api.go b/pkg/operator/resources/realtimeapi/api.go index a322870e8e..c7adcc71c2 100644 --- a/pkg/operator/resources/realtimeapi/api.go +++ b/pkg/operator/resources/realtimeapi/api.go @@ -65,7 +65,7 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*spec.A } if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { - go deleteK8sResources(api.Name) + return nil, "", err } err = operator.AddAPIToAPIGateway(*api.Networking.Endpoint, api.Networking.APIGateway, false) @@ -98,10 +98,18 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*spec.A } if err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil { - return nil, "", err + rollBackErr := rollBack(prevDeployment, prevService, prevVirtualService) + if rollBackErr != nil { + return nil, "", err + } + return nil, "update failed; rollbacked to previous deployment", nil } if err := operator.UpdateAPIGatewayK8s(prevVirtualService, api, false); err != nil { - return nil, "", err + rollBackErr := rollBack(prevDeployment, prevService, prevVirtualService) + if rollBackErr != nil { + return nil, "", err + } + return nil, "update failed; rollbacked to previous deployment", nil } return api, fmt.Sprintf("updating %s", api.Resource.UserString()), nil } @@ -117,6 +125,26 @@ func UpdateAPI(apiConfig *userconfig.API, projectID string, force bool) (*spec.A return api, fmt.Sprintf("%s is up to date", api.Resource.UserString()), nil } +func rollBack(prevDeployment *kapps.Deployment, prevService *kcore.Service, prevVirtualService *istioclientnetworking.VirtualService) error { + prevAPIID := prevVirtualService.Labels["apiID"] + prevAPIName := prevVirtualService.Labels["apiName"] + + prevAPI, err := operator.DownloadAPISpec(prevAPIName, prevAPIID) + if err != nil { + return err + } + + if err := applyK8sResources(prevAPI, prevDeployment, prevService, prevVirtualService); err != nil { + return err + } + + if err := operator.UpdateAPIGatewayK8s(prevVirtualService, prevAPI, false); err != nil { + return err + } + + return nil +} + func RefreshAPI(apiName string, force bool) (string, error) { prevDeployment, err := config.K8s.GetDeployment(operator.K8sName(apiName)) if err != nil { diff --git a/pkg/operator/resources/trafficsplitter/api.go b/pkg/operator/resources/trafficsplitter/api.go index d634e21ec7..5e30cace82 100644 --- a/pkg/operator/resources/trafficsplitter/api.go +++ b/pkg/operator/resources/trafficsplitter/api.go @@ -59,10 +59,18 @@ func UpdateAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) return nil, "", errors.Wrap(err, "upload api spec") } if err := applyK8sVirtualService(api, prevVirtualService); err != nil { - return nil, "", err + rollBackErr := rollBack(prevVirtualService) + if rollBackErr != nil { + return nil, "", err + } + return nil, "update failed; rollbacked to previous deployment", nil } if err := operator.UpdateAPIGatewayK8s(prevVirtualService, api, false); err != nil { - return nil, "", err + rollBackErr := rollBack(prevVirtualService) + if rollBackErr != nil { + return nil, "", err + } + return nil, "update failed; rollbacked to previous deployment", nil } return api, fmt.Sprintf("updated %s", api.Resource.UserString()), nil } @@ -70,6 +78,26 @@ func UpdateAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) return api, fmt.Sprintf("%s is up to date", api.Resource.UserString()), nil } +func rollBack(prevVirtualService *istioclientnetworking.VirtualService) error { + prevAPIID := prevVirtualService.Labels["apiID"] + prevAPIName := prevVirtualService.Labels["apiName"] + + prevAPI, err := operator.DownloadAPISpec(prevAPIName, prevAPIID) + if err != nil { + return err + } + + if err := applyK8sVirtualService(prevAPI, prevVirtualService); err != nil { + return err + } + + if err := operator.UpdateAPIGatewayK8s(prevVirtualService, prevAPI, false); err != nil { + return err + } + + return nil +} + func DeleteAPI(apiName string, keepCache bool) error { // best effort deletion, so don't handle error yet virtualService, vsErr := config.K8s.GetVirtualService(operator.K8sName(apiName))