diff --git a/api/bean/ConfigMapAndSecret.go b/api/bean/ConfigMapAndSecret.go index d35d58b8241..b600ecd5ef8 100644 --- a/api/bean/ConfigMapAndSecret.go +++ b/api/bean/ConfigMapAndSecret.go @@ -19,6 +19,7 @@ package bean import ( "encoding/json" + "github.com/devtron-labs/devtron/util" ) type ConfigMapRootJson struct { @@ -61,3 +62,10 @@ func (configSecret ConfigSecretMap) GetDataMap() (map[string]string, error) { err := json.Unmarshal(configSecret.Data, &datamap) return datamap, err } +func (configSecretJson ConfigSecretJson) GetDereferencedSecrets() []ConfigSecretMap { + return util.GetDeReferencedArray(configSecretJson.Secrets) +} + +func (configSecretJson *ConfigSecretJson) SetReferencedSecrets(secrets []ConfigSecretMap) { + configSecretJson.Secrets = util.GetReferencedArray(secrets) +} diff --git a/api/k8s/capacity/k8sCapacityRestHandler.go b/api/k8s/capacity/k8sCapacityRestHandler.go index 81974080570..56d0714a1b5 100644 --- a/api/k8s/capacity/k8sCapacityRestHandler.go +++ b/api/k8s/capacity/k8sCapacityRestHandler.go @@ -61,7 +61,7 @@ func (handler *K8sCapacityRestHandlerImpl) GetClusterListRaw(w http.ResponseWrit return } token := r.Header.Get("token") - clusters, err := handler.clusterService.FindAll() + clusters, err := handler.clusterService.FindAllExceptVirtual() if err != nil { handler.logger.Errorw("error in getting all clusters", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -83,7 +83,6 @@ func (handler *K8sCapacityRestHandlerImpl) GetClusterListRaw(w http.ResponseWrit Id: cluster.Id, Name: cluster.ClusterName, ErrorInConnection: cluster.ErrorInConnecting, - IsVirtualCluster: cluster.IsVirtualCluster, } clusterDetailList = append(clusterDetailList, clusterDetail) } @@ -102,7 +101,7 @@ func (handler *K8sCapacityRestHandlerImpl) GetClusterListWithDetail(w http.Respo return } token := r.Header.Get("token") - clusters, err := handler.clusterService.FindAll() + clusters, err := handler.clusterService.FindAllExceptVirtual() if err != nil { handler.logger.Errorw("error in getting all clusters", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index dce1a87c23f..5427d8b0d71 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -1192,45 +1192,50 @@ func (handler CoreAppRestHandlerImpl) deleteApp(ctx context.Context, appId int, // delete all CD pipelines for app starts cdPipelines, err := handler.pipelineBuilder.GetCdPipelinesForApp(appId) - if err != nil { + if err != nil && err != pg.ErrNoRows { handler.logger.Errorw("service err, GetCdPipelines in DeleteApp", "err", err, "appId", appId) return err } - - for _, cdPipeline := range cdPipelines.Pipelines { - cdPipelineDeleteRequest := &bean.CDPatchRequest{ - AppId: appId, - UserId: userId, - Action: bean.CD_DELETE, - ForceDelete: true, - NonCascadeDelete: false, - Pipeline: cdPipeline, - } - _, err = handler.pipelineBuilder.PatchCdPipelines(cdPipelineDeleteRequest, ctx) - if err != nil { - handler.logger.Errorw("err in deleting cd pipeline in DeleteApp", "err", err, "payload", cdPipelineDeleteRequest) - return err + if err != pg.ErrNoRows { + for _, cdPipeline := range cdPipelines.Pipelines { + cdPipelineDeleteRequest := &bean.CDPatchRequest{ + AppId: appId, + UserId: userId, + Action: bean.CD_DELETE, + ForceDelete: true, + NonCascadeDelete: false, + Pipeline: cdPipeline, + } + _, err = handler.pipelineBuilder.PatchCdPipelines(cdPipelineDeleteRequest, ctx) + if err != nil { + handler.logger.Errorw("err in deleting cd pipeline in DeleteApp", "err", err, "payload", cdPipelineDeleteRequest) + return err + } } + } // delete all CD pipelines for app ends // delete all CI pipelines for app starts ciPipelines, err := handler.pipelineBuilder.GetCiPipeline(appId) - if err != nil { + if err != nil && err != pg.ErrNoRows { handler.logger.Errorw("service err, GetCiPipelines in DeleteApp", "err", err, "appId", appId) return err } - for _, ciPipeline := range ciPipelines.CiPipelines { - ciPipelineDeleteRequest := &bean.CiPatchRequest{ - AppId: appId, - UserId: userId, - Action: bean.DELETE, - CiPipeline: ciPipeline, - } - _, err := handler.pipelineBuilder.PatchCiPipeline(ciPipelineDeleteRequest) - if err != nil { - handler.logger.Errorw("err in deleting ci pipeline in DeleteApp", "err", err, "payload", ciPipelineDeleteRequest) - return err + if err != pg.ErrNoRows { + + for _, ciPipeline := range ciPipelines.CiPipelines { + ciPipelineDeleteRequest := &bean.CiPatchRequest{ + AppId: appId, + UserId: userId, + Action: bean.DELETE, + CiPipeline: ciPipeline, + } + _, err := handler.pipelineBuilder.PatchCiPipeline(ciPipelineDeleteRequest) + if err != nil { + handler.logger.Errorw("err in deleting ci pipeline in DeleteApp", "err", err, "payload", ciPipelineDeleteRequest) + return err + } } } // delete all CI pipelines for app ends @@ -1551,16 +1556,37 @@ func (handler CoreAppRestHandlerImpl) createWorkflows(ctx context.Context, appId //Creating workflow ends //Creating CI pipeline starts - ciPipelineId, err := handler.createCiPipeline(appId, userId, workflowId, workflow.CiPipeline) + ciPipeline, err := handler.createCiPipeline(appId, userId, workflowId, workflow.CiPipeline) if err != nil { + err1 := handler.appWorkflowService.DeleteAppWorkflow(workflowId, userId) + if err1 != nil { + handler.logger.Errorw("service err, DeleteAppWorkflow ") + return err1, http.StatusInternalServerError + } handler.logger.Errorw("err in saving ci pipelines", err, "appId", appId) return err, http.StatusInternalServerError } //Creating CI pipeline ends //Creating CD pipeline starts - err = handler.createCdPipelines(ctx, appId, userId, workflowId, ciPipelineId, workflow.CdPipelines) + err = handler.createCdPipelines(ctx, appId, userId, workflowId, ciPipeline.Id, workflow.CdPipelines) if err != nil { + ciPipelineDeleteRequest := &bean.CiPatchRequest{ + AppId: appId, + UserId: userId, + Action: bean.DELETE, + CiPipeline: ciPipeline, + } + _, err1 := handler.pipelineBuilder.PatchCiPipeline(ciPipelineDeleteRequest) + if err1 != nil { + handler.logger.Errorw("err in deleting ci pipeline in DeleteApp", "err", err, "payload", ciPipelineDeleteRequest) + return err1, http.StatusInternalServerError + } + err1 = handler.appWorkflowService.DeleteAppWorkflow(workflowId, userId) + if err1 != nil { + handler.logger.Errorw("service err, DeleteAppWorkflow ") + return err1, http.StatusInternalServerError + } handler.logger.Errorw("err in saving cd pipelines", err, "appId", appId) return err, http.StatusInternalServerError } @@ -1590,13 +1616,13 @@ func (handler CoreAppRestHandlerImpl) createWorkflowInDb(workflowName string, ap return savedAppWf.Id, nil } -func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, workflowId int, ciPipelineData *appBean.CiPipelineDetails) (int, error) { +func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, workflowId int, ciPipelineData *appBean.CiPipelineDetails) (*bean.CiPipeline, error) { // if ci pipeline is of external type, then throw error as we are not supporting it as of now if ciPipelineData.ParentCiPipeline == 0 && ciPipelineData.ParentAppId == 0 && ciPipelineData.IsExternal { err := errors.New("external ci pipeline creation is not supported yet") handler.logger.Error("external ci pipeline creation is not supported yet") - return 0, err + return nil, err } // build ci pipeline materials starts @@ -1613,13 +1639,13 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, } if err != nil { handler.logger.Errorw("service err, FindByAppIdAndCheckoutPath in CreateWorkflows", "err", err, "appId", appId) - return 0, err + return nil, err } if gitMaterial == nil { err = errors.New("gitMaterial is nil") handler.logger.Errorw("gitMaterial is nil", "checkoutPath", ciMaterial.CheckoutPath) - return 0, err + return nil, err } ciMaterialRequest := &bean.CiMaterial{ @@ -1664,10 +1690,10 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, res, err := handler.pipelineBuilder.PatchCiPipeline(ciPipelineRequest) if err != nil { handler.logger.Errorw("service err, PatchCiPipelines", "err", err, "appId", appId) - return 0, err + return nil, err } - return res.CiPipelines[0].Id, nil + return res.CiPipelines[0], nil } func (handler CoreAppRestHandlerImpl) createCdPipelines(ctx context.Context, appId int, userId int32, workflowId int, ciPipelineId int, cdPipelines []*appBean.CdPipelineDetails) error { diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 38fd34b6810..d4b68e7b928 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -40,6 +40,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDeploymentTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" "github.com/devtron-labs/devtron/pkg/attributes" @@ -199,6 +200,12 @@ func InitializeApp() (*App, error) { wire.Bind(new(dockerRegistryRepository.DockerRegistryIpsConfigRepository), new(*dockerRegistryRepository.DockerRegistryIpsConfigRepositoryImpl)), dockerRegistryRepository.NewOCIRegistryConfigRepositoryImpl, wire.Bind(new(dockerRegistryRepository.OCIRegistryConfigRepository), new(*dockerRegistryRepository.OCIRegistryConfigRepositoryImpl)), + + // chart group repository layer wire injection started + repository3.NewChartGroupDeploymentRepositoryImpl, + wire.Bind(new(repository3.ChartGroupDeploymentRepository), new(*repository3.ChartGroupDeploymentRepositoryImpl)), + // chart group repository layer wire injection ended + // end: docker registry wire set injection ) return &App{}, nil diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 959fbf6239b..b48223587eb 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -274,6 +274,7 @@ func InitializeApp() (*App, error) { appStoreValuesServiceImpl := service2.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) appStoreValuesRestHandlerImpl := appStoreValues.NewAppStoreValuesRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreValuesServiceImpl) appStoreValuesRouterImpl := appStoreValues.NewAppStoreValuesRouterImpl(appStoreValuesRestHandlerImpl) + chartGroupDeploymentRepositoryImpl := repository4.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) clusterInstalledAppsRepositoryImpl := repository4.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) installedAppVersionHistoryRepositoryImpl := repository4.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) @@ -282,7 +283,7 @@ func InitializeApp() (*App, error) { return nil, err } pubSubClientServiceImpl := pubsub_lib.NewPubSubClientServiceImpl(sugaredLogger) - appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, pubSubClientServiceImpl) + appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, pubSubClientServiceImpl) appStoreDeploymentRestHandlerImpl := appStoreDeployment.NewAppStoreDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmUserServiceImpl, attributesServiceImpl) appStoreDeploymentRouterImpl := appStoreDeployment.NewAppStoreDeploymentRouterImpl(appStoreDeploymentRestHandlerImpl) chartProviderServiceImpl := chartProvider.NewChartProviderServiceImpl(sugaredLogger, chartRepoRepositoryImpl, chartRepositoryServiceImpl, dockerArtifactStoreRepositoryImpl, ociRegistryConfigRepositoryImpl) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index b9329335d48..d713b5751cc 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -72,7 +72,7 @@ type CiWorkflow struct { CiBuildType string `sql:"ci_build_type"` EnvironmentId int `sql:"environment_id"` ReferenceCiWorkflowId int `sql:"ref_ci_workflow_id"` - ParentCiWorkFlowId int `sql:"parent_ci_workflow_id"` + ParentCiWorkFlowId int `sql:"parent_ci_workflow_id"` CiPipeline *CiPipeline } diff --git a/internal/sql/repository/security/ImageScanDeployInfoRepository.go b/internal/sql/repository/security/ImageScanDeployInfoRepository.go index 6f421e774fd..cea243d14b0 100644 --- a/internal/sql/repository/security/ImageScanDeployInfoRepository.go +++ b/internal/sql/repository/security/ImageScanDeployInfoRepository.go @@ -145,13 +145,13 @@ func (impl ImageScanDeployInfoRepositoryImpl) FetchListingGroupByObject(size int } func (impl ImageScanDeployInfoRepositoryImpl) FetchByAppIdAndEnvId(appId int, envId int, objectType []string) (*ImageScanDeployInfo, error) { - var model ImageScanDeployInfo - err := impl.dbConnection.Model(&model). + var model *ImageScanDeployInfo + err := impl.dbConnection.Model(model). Where("scan_object_meta_id = ?", appId). Where("env_id = ?", envId).Where("object_type in (?)", pg.In(objectType)). Order("created_on desc").Limit(1). Select() - return &model, err + return model, err } func (impl ImageScanDeployInfoRepositoryImpl) FindByTypeMetaAndTypeId(scanObjectMetaId int, objectType string) (*ImageScanDeployInfo, error) { @@ -185,7 +185,8 @@ func (impl ImageScanDeployInfoRepositoryImpl) scanListQueryWithoutObject(request } query = query + " INNER JOIN environment env on env.id=info.env_id" query = query + " INNER JOIN cluster clus on clus.id=env.cluster_id" - query = query + " WHERE info.scan_object_meta_id > 0 and env.active=true and info.image_scan_execution_history_id[1] != -1" + query = query + " LEFT JOIN app ap on ap.id = info.scan_object_meta_id and info.object_type='app' WHERE ap.active=true" + query = query + " AND info.scan_object_meta_id > 0 and env.active=true and info.image_scan_execution_history_id[1] != -1 " if len(deployInfoIds) > 0 { ids := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(deployInfoIds)), ","), "[]") query = query + " AND info.id IN (" + ids + ")" diff --git a/internal/util/MergeUtil.go b/internal/util/MergeUtil.go index df6ca22fd4e..3f252bb9d15 100644 --- a/internal/util/MergeUtil.go +++ b/internal/util/MergeUtil.go @@ -19,11 +19,11 @@ package util import ( "encoding/json" - "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/util" jsonpatch "github.com/evanphx/json-patch" "go.uber.org/zap" + "golang.org/x/exp/slices" ) type MergeUtil struct { @@ -99,8 +99,6 @@ func (m MergeUtil) ConfigMapMerge(appLevelConfigMapJson string, envLevelConfigMa appLevelConfigMap := bean.ConfigMapJson{} envLevelConfigMap := bean.ConfigMapJson{} configResponse := bean.ConfigMapJson{} - commonMaps := map[string]bean.ConfigSecretMap{} - var finalMaps []bean.ConfigSecretMap if appLevelConfigMapJson != "" { err = json.Unmarshal([]byte(appLevelConfigMapJson), &appLevelConfigMap) if err != nil { @@ -117,20 +115,7 @@ func (m MergeUtil) ConfigMapMerge(appLevelConfigMapJson string, envLevelConfigMa configResponse.Enabled = true } - for _, item := range envLevelConfigMap.Maps { - commonMaps[item.Name] = item - } - for _, item := range appLevelConfigMap.Maps { - if _, ok := commonMaps[item.Name]; ok { - //ignoring this value as override from configB - } else { - commonMaps[item.Name] = item - } - } - for _, v := range commonMaps { - finalMaps = append(finalMaps, v) - } - configResponse.Maps = finalMaps + configResponse.Maps = mergeConfigMapsAndSecrets(envLevelConfigMap.Maps, appLevelConfigMap.Maps) byteData, err := json.Marshal(configResponse) if err != nil { m.Logger.Debugw("error in marshal ", "err", err) @@ -140,67 +125,75 @@ func (m MergeUtil) ConfigMapMerge(appLevelConfigMapJson string, envLevelConfigMa func (m MergeUtil) ConfigSecretMerge(appLevelSecretJson string, envLevelSecretJson string, chartMajorVersion int, chartMinorVersion int, isJob bool) (data string, err error) { appLevelSecret := bean.ConfigSecretJson{} - envLevelSecret := bean.ConfigSecretJson{} - secretResponse := bean.ConfigSecretJson{} - commonSecrets := map[string]*bean.ConfigSecretMap{} - var finalMaps []*bean.ConfigSecretMap if appLevelSecretJson != "" { err = json.Unmarshal([]byte(appLevelSecretJson), &appLevelSecret) if err != nil { m.Logger.Debugw("error in Unmarshal ", "appLevelSecretJson", appLevelSecretJson, "envLevelSecretJson", envLevelSecretJson, "err", err) } } + envLevelSecret := bean.ConfigSecretJson{} if envLevelSecretJson != "" { err = json.Unmarshal([]byte(envLevelSecretJson), &envLevelSecret) if err != nil { m.Logger.Debugw("error in Unmarshal ", "appLevelSecretJson", appLevelSecretJson, "envLevelSecretJson", envLevelSecretJson, "err", err) } } + secretResponse := bean.ConfigSecretJson{} if len(appLevelSecret.Secrets) > 0 || len(envLevelSecret.Secrets) > 0 { secretResponse.Enabled = true } - for _, item := range envLevelSecret.Secrets { - commonSecrets[item.Name] = item + finalCMCS := mergeConfigMapsAndSecrets(envLevelSecret.GetDereferencedSecrets(), appLevelSecret.GetDereferencedSecrets()) + for _, finalMap := range finalCMCS { + finalMap = m.processExternalSecrets(finalMap, chartMajorVersion, chartMinorVersion, isJob) } - for _, item := range appLevelSecret.Secrets { + secretResponse.SetReferencedSecrets(finalCMCS) + byteData, err := json.Marshal(secretResponse) + if err != nil { + m.Logger.Debugw("error in marshal ", "err", err) + } + return string(byteData), err +} + +func mergeConfigMapsAndSecrets(envLevelCMCS []bean.ConfigSecretMap, appLevelSecretCMCS []bean.ConfigSecretMap) []bean.ConfigSecretMap { + envCMCSNames := make([]string, 0) + var finalCMCS []bean.ConfigSecretMap + for _, item := range envLevelCMCS { + envCMCSNames = append(envCMCSNames, item.Name) + } + for _, item := range appLevelSecretCMCS { //else ignoring this value as override from configB - if _, ok := commonSecrets[item.Name]; !ok { - commonSecrets[item.Name] = item + if !slices.Contains(envCMCSNames, item.Name) { + finalCMCS = append(finalCMCS, item) } } + for _, item := range envLevelCMCS { + finalCMCS = append(finalCMCS, item) + } + return finalCMCS +} - for _, item := range commonSecrets { - if item.ExternalType == util.AWSSecretsManager || item.ExternalType == util.AWSSystemManager || item.ExternalType == util.HashiCorpVault { - if item.SecretData != nil && ((chartMajorVersion <= 3 && chartMinorVersion < 8) || isJob) { - var es []map[string]interface{} - esNew := make(map[string]interface{}) - err = json.Unmarshal(item.SecretData, &es) - if err != nil { - m.Logger.Debugw("error in Unmarshal ", "appLevelSecretJson", appLevelSecretJson, "envLevelSecretJson", envLevelSecretJson, "err", err) - } - for _, item := range es { - keyProp := item["name"].(string) - valueProp := item["key"] - esNew[keyProp] = valueProp - } - byteData, err := json.Marshal(esNew) - if err != nil { - m.Logger.Debugw("error in marshal ", "err", err) - } - item.Data = byteData - item.SecretData = nil +func (m MergeUtil) processExternalSecrets(secret bean.ConfigSecretMap, chartMajorVersion int, chartMinorVersion int, isJob bool) bean.ConfigSecretMap { + if secret.ExternalType == util.AWSSecretsManager || secret.ExternalType == util.AWSSystemManager || secret.ExternalType == util.HashiCorpVault { + if secret.SecretData != nil && ((chartMajorVersion <= 3 && chartMinorVersion < 8) || isJob) { + var es []map[string]interface{} + esNew := make(map[string]interface{}) + err := json.Unmarshal(secret.SecretData, &es) + if err != nil { + m.Logger.Debugw("error in Unmarshal ", "SecretData", secret.SecretData, "external secret", es, "err", err) } + for _, item := range es { + keyProp := item["name"].(string) + valueProp := item["key"] + esNew[keyProp] = valueProp + } + byteData, err := json.Marshal(esNew) + if err != nil { + m.Logger.Debugw("error in marshal ", "err", err) + } + secret.Data = byteData + secret.SecretData = nil } } - - for _, v := range commonSecrets { - finalMaps = append(finalMaps, v) - } - secretResponse.Secrets = finalMaps - byteData, err := json.Marshal(secretResponse) - if err != nil { - m.Logger.Debugw("error in marshal ", "err", err) - } - return string(byteData), err + return secret } diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 05622bd4ee3..88459bd5001 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -2008,7 +2008,7 @@ func (impl *AppServiceImpl) MarkImageScanDeployed(appId int, envId int, imageDig var ids []int ids = append(ids, executionHistory.Id) - ot, err := impl.imageScanDeployInfoRepository.FindByTypeMetaAndTypeId(appId, security.ScanObjectType_APP) //todo insure this touple unique in db + ot, err := impl.imageScanDeployInfoRepository.FetchByAppIdAndEnvId(appId, envId, []string{security.ScanObjectType_APP}) if err != nil && err != pg.ErrNoRows { return err } else if err == pg.ErrNoRows && isScanEnabled { diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index b7dd35f1b65..6478bd7c5b3 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -53,6 +53,9 @@ type AppCloneServiceImpl struct { pipelineStageService pipeline.PipelineStageService ciTemplateService pipeline.CiTemplateService appRepository app2.AppRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + pipelineRepository pipelineConfig.PipelineRepository + appWorkflowRepository appWorkflow2.AppWorkflowRepository } func NewAppCloneServiceImpl(logger *zap.SugaredLogger, @@ -65,7 +68,8 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, propertiesConfigService pipeline.PropertiesConfigService, ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, pipelineStageService pipeline.PipelineStageService, ciTemplateService pipeline.CiTemplateService, - appRepository app2.AppRepository) *AppCloneServiceImpl { + appRepository app2.AppRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, + pipelineRepository pipelineConfig.PipelineRepository, appWorkflowRepository appWorkflow2.AppWorkflowRepository) *AppCloneServiceImpl { return &AppCloneServiceImpl{ logger: logger, pipelineBuilder: pipelineBuilder, @@ -78,6 +82,9 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, pipelineStageService: pipelineStageService, ciTemplateService: ciTemplateService, appRepository: appRepository, + ciPipelineRepository: ciPipelineRepository, + pipelineRepository: pipelineRepository, + appWorkflowRepository: appWorkflowRepository, } } @@ -90,6 +97,16 @@ type CloneRequest struct { AppType helper.AppType `json:"appType"` } +type CreateWorkflowMappingDto struct { + oldAppId int + newAppId int + userId int32 + newWfId int + gitMaterialMapping map[int]int + isSameProject bool + externalCiPipelineId int +} + func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { //validate template app templateApp, err := impl.appRepository.FindById(createReq.TemplateId) @@ -312,6 +329,7 @@ func (impl *AppCloneServiceImpl) CreateCiTemplate(oldAppId, newAppId int, userId UserId: userId, BeforeDockerBuild: refCiConf.BeforeDockerBuild, AfterDockerBuild: refCiConf.AfterDockerBuild, + ScanEnabled: refCiConf.ScanEnabled, } res, err := impl.pipelineBuilder.CreateCiPipeline(ciConfRequest) @@ -586,6 +604,7 @@ func (impl *AppCloneServiceImpl) CreateWf(oldAppId, newAppId int, userId int32, return nil, err } impl.logger.Debugw("workflow found", "wf", refAppWFs) + for _, refAppWF := range refAppWFs { thisWf := appWorkflow.AppWorkflowDto{ Id: 0, @@ -594,24 +613,38 @@ func (impl *AppCloneServiceImpl) CreateWf(oldAppId, newAppId int, userId int32, AppWorkflowMappingDto: nil, //first create new mapping then add it UserId: userId, } + thisWf, err = impl.appWorkflowService.CreateAppWorkflow(thisWf) + if err != nil { + impl.logger.Errorw("error in creating workflow without external-ci", "err", err) + return nil, err + } isExternalCiPresent := false for _, awm := range refAppWF.AppWorkflowMappingDto { if awm.Type == appWorkflow2.WEBHOOK { isExternalCiPresent = true + break } } - - if !isExternalCiPresent { - thisWf, err = impl.appWorkflowService.CreateAppWorkflow(thisWf) - impl.logger.Debugw("workflow found", thisWf) + createWorkflowMappingDto := CreateWorkflowMappingDto{ + newAppId: newAppId, + oldAppId: oldAppId, + newWfId: thisWf.Id, + userId: userId, + } + var externalCiPipelineId int + if isExternalCiPresent { + externalCiPipelineId, err = impl.createExternalCiAndAppWorkflowMapping(createWorkflowMappingDto) if err != nil { - impl.logger.Errorw("errir in creating workflow without extenal-ci", "err", err) + impl.logger.Errorw("error in createExternalCiAndAppWorkflowMapping", "err", err) return nil, err } } + createWorkflowMappingDto.gitMaterialMapping = gitMaterialMapping + createWorkflowMappingDto.isSameProject = isSameProject + createWorkflowMappingDto.externalCiPipelineId = externalCiPipelineId - err = impl.createWfMappings(refAppWF.AppWorkflowMappingDto, oldAppId, newAppId, userId, thisWf.Id, gitMaterialMapping, ctx, isSameProject) + err = impl.createWfInstances(refAppWF.AppWorkflowMappingDto, createWorkflowMappingDto, ctx) if err != nil { impl.logger.Errorw("error in creating workflow mapping", "err", err) return nil, err @@ -620,7 +653,28 @@ func (impl *AppCloneServiceImpl) CreateWf(oldAppId, newAppId int, userId int32, return nil, nil } -func (impl *AppCloneServiceImpl) createWfMappings(refWfMappings []appWorkflow.AppWorkflowMappingDto, oldAppId, newAppId int, userId int32, thisWfId int, gitMaterialMapping map[int]int, ctx context.Context, isSameProject bool) error { +func (impl *AppCloneServiceImpl) createExternalCiAndAppWorkflowMapping(createWorkflowMappingDto CreateWorkflowMappingDto) (int, error) { + dbConnection := impl.pipelineRepository.GetConnection() + tx, err := dbConnection.Begin() + if err != nil { + impl.logger.Errorw("error in beginning transaction", "err", err) + return 0, err + } + // Rollback tx on error. + defer tx.Rollback() + externalCiPipelineId, err := impl.pipelineBuilder.CreateExternalCiAndAppWorkflowMapping(createWorkflowMappingDto.newAppId, createWorkflowMappingDto.newWfId, createWorkflowMappingDto.userId, tx) + if err != nil { + impl.logger.Errorw("error in creating new external ci pipeline and new app workflow mapping", "refAppId", createWorkflowMappingDto.oldAppId, "newAppId", createWorkflowMappingDto.newAppId, "err", err) + return 0, err + } + err = tx.Commit() + if err != nil { + return 0, err + } + return externalCiPipelineId, nil +} + +func (impl *AppCloneServiceImpl) createWfInstances(refWfMappings []appWorkflow.AppWorkflowMappingDto, createWorkflowMappingDto CreateWorkflowMappingDto, ctx context.Context) error { impl.logger.Debugw("wf mapping cloning", "refWfMappings", refWfMappings) var ciMapping []appWorkflow.AppWorkflowMappingDto var cdMappings []appWorkflow.AppWorkflowMappingDto @@ -636,23 +690,30 @@ func (impl *AppCloneServiceImpl) createWfMappings(refWfMappings []appWorkflow.Ap return fmt.Errorf("unsupported wf type: %s", appWf.Type) } } - refApp, err := impl.pipelineBuilder.GetApp(oldAppId) + sourceToNewPipelineIdMapping := make(map[int]int) + refApp, err := impl.pipelineBuilder.GetApp(createWorkflowMappingDto.oldAppId) + if err != nil { + impl.logger.Errorw("error in getting app from refAppId", "refAppId", createWorkflowMappingDto.oldAppId) + return err + } if len(webhookMappings) > 0 { - if isSameProject { + if createWorkflowMappingDto.isSameProject { for _, refwebhookMappings := range cdMappings { cdCloneReq := &cloneCdPipelineRequest{ - refCdPipelineId: refwebhookMappings.ComponentId, - refAppId: oldAppId, - appId: newAppId, - userId: userId, - ciPipelineId: 0, - appWfId: thisWfId, - refAppName: refApp.AppName, + refCdPipelineId: refwebhookMappings.ComponentId, + refAppId: createWorkflowMappingDto.oldAppId, + appId: createWorkflowMappingDto.newAppId, + userId: createWorkflowMappingDto.userId, + ciPipelineId: 0, + appWfId: createWorkflowMappingDto.newWfId, + refAppName: refApp.AppName, + sourceToNewPipelineId: sourceToNewPipelineIdMapping, + externalCiPipelineId: createWorkflowMappingDto.externalCiPipelineId, } pipeline, err := impl.CreateCdPipeline(cdCloneReq, ctx) impl.logger.Debugw("cd pipeline created", "pipeline", pipeline) if err != nil { - impl.logger.Errorw("error in getting cd-pipeling", "err", err) + impl.logger.Errorw("error in getting cd-pipeline", "refAppId", createWorkflowMappingDto.oldAppId, "newAppId", createWorkflowMappingDto.newAppId, "err", err) return err } } @@ -666,7 +727,7 @@ func (impl *AppCloneServiceImpl) createWfMappings(refWfMappings []appWorkflow.Ap impl.logger.Warn("no ci pipeline found") return nil } else if len(ciMapping) != 1 { - impl.logger.Warn("more than one cd pipeline not supported") + impl.logger.Warn("more than one ci pipeline not supported") return nil } @@ -678,12 +739,12 @@ func (impl *AppCloneServiceImpl) createWfMappings(refWfMappings []appWorkflow.Ap impl.logger.Debugw("creating ci", "ref", refCiMapping) cloneCiPipelineRequest := &cloneCiPipelineRequest{ - refAppId: oldAppId, + refAppId: createWorkflowMappingDto.oldAppId, refCiPipelineId: refCiMapping.ComponentId, - userId: userId, - appId: newAppId, - wfId: thisWfId, - gitMaterialMapping: gitMaterialMapping, + userId: createWorkflowMappingDto.userId, + appId: createWorkflowMappingDto.newAppId, + wfId: createWorkflowMappingDto.newWfId, + gitMaterialMapping: createWorkflowMappingDto.gitMaterialMapping, refAppName: refApp.AppName, } ci, err = impl.CreateCiPipeline(cloneCiPipelineRequest) @@ -693,16 +754,18 @@ func (impl *AppCloneServiceImpl) createWfMappings(refWfMappings []appWorkflow.Ap } impl.logger.Debugw("ci created", "ci", ci) } - if isSameProject { + + if createWorkflowMappingDto.isSameProject { for _, refCdMapping := range cdMappings { cdCloneReq := &cloneCdPipelineRequest{ - refCdPipelineId: refCdMapping.ComponentId, - refAppId: oldAppId, - appId: newAppId, - userId: userId, - ciPipelineId: ci.CiPipelines[0].Id, - appWfId: thisWfId, - refAppName: refApp.AppName, + refCdPipelineId: refCdMapping.ComponentId, + refAppId: createWorkflowMappingDto.oldAppId, + appId: createWorkflowMappingDto.newAppId, + userId: createWorkflowMappingDto.userId, + ciPipelineId: ci.CiPipelines[0].Id, + appWfId: createWorkflowMappingDto.newWfId, + refAppName: refApp.AppName, + sourceToNewPipelineId: sourceToNewPipelineIdMapping, } pipeline, err := impl.CreateCdPipeline(cdCloneReq, ctx) if err != nil { @@ -813,6 +876,7 @@ func (impl *AppCloneServiceImpl) CreateCiPipeline(req *cloneCiPipelineRequest) ( PreBuildStage: preStageDetail, PostBuildStage: postStageDetail, EnvironmentId: refCiPipeline.EnvironmentId, + ScanEnabled: refCiPipeline.ScanEnabled, PipelineType: refCiPipeline.PipelineType, }, AppId: req.appId, @@ -867,13 +931,15 @@ func (impl *AppCloneServiceImpl) CreateCiPipeline(req *cloneCiPipelineRequest) ( } type cloneCdPipelineRequest struct { - refCdPipelineId int - refAppId int - appId int - userId int32 - ciPipelineId int - appWfId int - refAppName string + refCdPipelineId int + refAppId int + appId int + userId int32 + ciPipelineId int + appWfId int + refAppName string + sourceToNewPipelineId map[int]int + externalCiPipelineId int } func (impl *AppCloneServiceImpl) CreateCdPipeline(req *cloneCdPipelineRequest, ctx context.Context) (*bean.CdPipelines, error) { @@ -891,6 +957,7 @@ func (impl *AppCloneServiceImpl) CreateCdPipeline(req *cloneCdPipelineRequest, c if refCdPipeline == nil { return nil, fmt.Errorf("no cd pipeline found") } + refCdPipeline.SourceToNewPipelineId = req.sourceToNewPipelineId pipelineName := refCdPipeline.Name if strings.HasPrefix(pipelineName, req.refAppName) { pipelineName = strings.Replace(pipelineName, req.refAppName+"-", "", 1) @@ -921,35 +988,6 @@ func (impl *AppCloneServiceImpl) CreateCdPipeline(req *cloneCdPipelineRequest, c deploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_HELM } - if refCdPipeline.ParentPipelineType == "WEBHOOK" { - cdPipeline := &bean.CDPipelineConfigObject{ - Id: 0, - EnvironmentId: refCdPipeline.EnvironmentId, - CiPipelineId: 0, - TriggerType: refCdPipeline.TriggerType, - Name: pipelineName, - Strategies: refCdPipeline.Strategies, - Namespace: refCdPipeline.Namespace, - AppWorkflowId: 0, - DeploymentTemplate: refCdPipeline.DeploymentTemplate, - PreStage: refCdPipeline.PreStage, //FIXME - PostStage: refCdPipeline.PostStage, - PreStageConfigMapSecretNames: refCdPipeline.PreStageConfigMapSecretNames, - PostStageConfigMapSecretNames: refCdPipeline.PostStageConfigMapSecretNames, - RunPostStageInEnv: refCdPipeline.RunPostStageInEnv, - RunPreStageInEnv: refCdPipeline.RunPreStageInEnv, - DeploymentAppType: refCdPipeline.DeploymentAppType, - ParentPipelineId: 0, - ParentPipelineType: refCdPipeline.ParentPipelineType, - } - cdPipelineReq := &bean.CdPipelines{ - Pipelines: []*bean.CDPipelineConfigObject{cdPipeline}, - AppId: req.appId, - UserId: req.userId, - } - cdPipelineRes, err := impl.pipelineBuilder.CreateCdPipelines(cdPipelineReq, ctx) - return cdPipelineRes, err - } cdPipeline := &bean.CDPipelineConfigObject{ Id: 0, EnvironmentId: refCdPipeline.EnvironmentId, @@ -969,6 +1007,15 @@ func (impl *AppCloneServiceImpl) CreateCdPipeline(req *cloneCdPipelineRequest, c DeploymentAppType: deploymentAppType, PreDeployStage: refCdPipeline.PreDeployStage, PostDeployStage: refCdPipeline.PostDeployStage, + SourceToNewPipelineId: refCdPipeline.SourceToNewPipelineId, + RefPipelineId: refCdPipeline.Id, + ParentPipelineType: refCdPipeline.ParentPipelineType, + } + if refCdPipeline.ParentPipelineType == "WEBHOOK" { + cdPipeline.CiPipelineId = 0 + cdPipeline.ParentPipelineId = req.externalCiPipelineId + } else if refCdPipeline.ParentPipelineType != appWorkflow.CI_PIPELINE_TYPE { + cdPipeline.ParentPipelineId = refCdPipeline.ParentPipelineId } cdPipelineReq := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{cdPipeline}, diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index 99b0788d933..385505a74ef 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -91,6 +91,7 @@ func GetDeploymentServiceTypeConfig() (*DeploymentServiceTypeConfig, error) { type AppStoreDeploymentServiceImpl struct { logger *zap.SugaredLogger installedAppRepository repository.InstalledAppRepository + chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository clusterRepository.EnvironmentRepository clusterInstalledAppsRepository repository.ClusterInstalledAppsRepository @@ -110,7 +111,7 @@ type AppStoreDeploymentServiceImpl struct { } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository.InstalledAppRepository, - appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository clusterRepository.EnvironmentRepository, + chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository clusterRepository.EnvironmentRepository, clusterInstalledAppsRepository repository.ClusterInstalledAppsRepository, appRepository app.AppRepository, appStoreDeploymentHelmService appStoreDeploymentTool.AppStoreDeploymentHelmService, appStoreDeploymentArgoCdService appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService, environmentService cluster.EnvironmentService, @@ -122,6 +123,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep appStoreDeploymentServiceImpl := &AppStoreDeploymentServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, + chartGroupDeploymentRepository: chartGroupDeploymentRepository, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, environmentRepository: environmentRepository, clusterInstalledAppsRepository: clusterInstalledAppsRepository, @@ -765,6 +767,19 @@ func (impl AppStoreDeploymentServiceImpl) DeleteInstalledApp(ctx context.Context } } + // soft delete chart-group deployment + chartGroupDeployment, err := impl.chartGroupDeploymentRepository.FindByInstalledAppId(model.Id) + if err != nil { + impl.logger.Errorw("error while fetching chart group deployment", "error", err) + return nil, err + } + chartGroupDeployment.Deleted = true + _, err = impl.chartGroupDeploymentRepository.Update(chartGroupDeployment, tx) + if err != nil { + impl.logger.Errorw("error while updating chart group deployment", "error", err) + return nil, err + } + if util2.IsBaseStack() || util2.IsHelmApp(app.AppOfferingMode) || util.IsHelmApp(model.DeploymentAppType) { // there might be a case if helm release gets uninstalled from helm cli. //in this case on deleting the app from API, it should not give error as it should get deleted from db, otherwise due to delete error, db does not get clean diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 9bf392abe55..29919a091eb 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -552,6 +552,9 @@ type CDPipelineConfigObject struct { ManifestStorageType string `json:"manifestStorageType"` PreDeployStage *bean.PipelineStageDto `json:"preDeployStage,omitempty"` PostDeployStage *bean.PipelineStageDto `json:"postDeployStage,omitempty"` + SourceToNewPipelineId map[int]int `json:"sourceToNewPipelineId,omitempty"` + RefPipelineId int `json:"refPipelineId,omitempty"` + ExternalCiPipelineId int `json:"externalCiPipelineId,omitempty"` } type PreStageConfigMapSecretNames struct { diff --git a/pkg/cluster/ClusterCronService.go b/pkg/cluster/ClusterCronService.go index d35d91191a2..5af22409603 100644 --- a/pkg/cluster/ClusterCronService.go +++ b/pkg/cluster/ClusterCronService.go @@ -46,7 +46,7 @@ func (impl *ClusterCronServiceImpl) GetAndUpdateClusterConnectionStatus() { defer impl.logger.Debug("stopped cluster connection status fetch thread") //getting all clusters - clusters, err := impl.clusterService.FindAll() + clusters, err := impl.clusterService.FindAllExceptVirtual() if err != nil { impl.logger.Errorw("error in getting all clusters", "err", err) return diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index 990772950de..b7aab982148 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -159,6 +159,7 @@ type ClusterService interface { FindOne(clusterName string) (*ClusterBean, error) FindOneActive(clusterName string) (*ClusterBean, error) FindAll() ([]*ClusterBean, error) + FindAllExceptVirtual() ([]*ClusterBean, error) FindAllWithoutConfig() ([]*ClusterBean, error) FindAllActive() ([]ClusterBean, error) DeleteFromDb(bean *ClusterBean, userId int32) error @@ -356,6 +357,19 @@ func (impl *ClusterServiceImpl) FindAll() ([]*ClusterBean, error) { return beans, nil } +func (impl *ClusterServiceImpl) FindAllExceptVirtual() ([]*ClusterBean, error) { + models, err := impl.clusterRepository.FindAllActiveExceptVirtual() + if err != nil { + return nil, err + } + var beans []*ClusterBean + for _, model := range models { + bean := GetClusterBean(model) + beans = append(beans, &bean) + } + return beans, nil +} + func (impl *ClusterServiceImpl) FindAllActive() ([]ClusterBean, error) { models, err := impl.clusterRepository.FindAllActive() if err != nil { diff --git a/pkg/cluster/ClusterServiceExtended.go b/pkg/cluster/ClusterServiceExtended.go index 8b62eb76816..1564867bce9 100644 --- a/pkg/cluster/ClusterServiceExtended.go +++ b/pkg/cluster/ClusterServiceExtended.go @@ -71,11 +71,7 @@ func (impl *ClusterServiceImplExtended) FindAllWithoutConfig() ([]*ClusterBean, return beans, nil } -func (impl *ClusterServiceImplExtended) FindAll() ([]*ClusterBean, error) { - beans, err := impl.ClusterServiceImpl.FindAll() - if err != nil { - return nil, err - } +func (impl *ClusterServiceImplExtended) GetClusterFullModeDTO(beans []*ClusterBean) ([]*ClusterBean, error) { //devtron full mode logic var clusterIds []int for _, cluster := range beans { @@ -143,6 +139,22 @@ func (impl *ClusterServiceImplExtended) FindAll() ([]*ClusterBean, error) { return beans, nil } +func (impl *ClusterServiceImplExtended) FindAll() ([]*ClusterBean, error) { + beans, err := impl.ClusterServiceImpl.FindAll() + if err != nil { + return nil, err + } + return impl.GetClusterFullModeDTO(beans) +} + +func (impl *ClusterServiceImplExtended) FindAllExceptVirtual() ([]*ClusterBean, error) { + beans, err := impl.ClusterServiceImpl.FindAll() + if err != nil { + return nil, err + } + return impl.GetClusterFullModeDTO(beans) +} + func (impl *ClusterServiceImplExtended) Update(ctx context.Context, bean *ClusterBean, userId int32) (*ClusterBean, error) { isGitOpsConfigured, err1 := impl.gitOpsRepository.IsGitOpsConfigured() if err1 != nil { diff --git a/pkg/cluster/repository/ClusterRepository.go b/pkg/cluster/repository/ClusterRepository.go index fa57ff0ba82..5fe9941e93e 100644 --- a/pkg/cluster/repository/ClusterRepository.go +++ b/pkg/cluster/repository/ClusterRepository.go @@ -50,6 +50,7 @@ type ClusterRepository interface { FindOneActive(clusterName string) (*Cluster, error) FindAll() ([]Cluster, error) FindAllActive() ([]Cluster, error) + FindAllActiveExceptVirtual() ([]Cluster, error) FindById(id int) (*Cluster, error) FindByIds(id []int) ([]Cluster, error) Update(model *Cluster) error @@ -127,6 +128,16 @@ func (impl ClusterRepositoryImpl) FindAllActive() ([]Cluster, error) { return clusters, err } +func (impl ClusterRepositoryImpl) FindAllActiveExceptVirtual() ([]Cluster, error) { + var clusters []Cluster + err := impl.dbConnection. + Model(&clusters). + Where("active=?", true). + Where("is_virtual_cluster=?", false). + Select() + return clusters, err +} + func (impl ClusterRepositoryImpl) FindById(id int) (*Cluster, error) { cluster := &Cluster{} err := impl.dbConnection. diff --git a/pkg/k8s/capacity/bean/bean.go b/pkg/k8s/capacity/bean/bean.go index 138aa699530..95e35c86440 100644 --- a/pkg/k8s/capacity/bean/bean.go +++ b/pkg/k8s/capacity/bean/bean.go @@ -66,7 +66,6 @@ type ClusterCapacityDetail struct { ServerVersion string `json:"serverVersion,omitempty"` Cpu *ResourceDetailObject `json:"cpu"` Memory *ResourceDetailObject `json:"memory"` - IsVirtualCluster bool `json:"isVirtualCluster"` } type NodeCapacityDetail struct { diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index 1bef44b9dce..97da7376b0f 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -67,9 +67,7 @@ func (impl *K8sCapacityServiceImpl) GetClusterCapacityDetailList(ctx context.Con for _, cluster := range clusters { clusterCapacityDetail := &bean.ClusterCapacityDetail{} var err error - if cluster.IsVirtualCluster { - clusterCapacityDetail.IsVirtualCluster = cluster.IsVirtualCluster - } else if len(cluster.ErrorInConnecting) > 0 { + if len(cluster.ErrorInConnecting) > 0 { clusterCapacityDetail.ErrorInConnection = cluster.ErrorInConnecting } else { clusterCapacityDetail, err = impl.GetClusterCapacityDetail(ctx, cluster, true) diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index e826c793d8e..0fa74ed661b 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -107,6 +107,7 @@ type CdPipelineConfigService interface { GetEnvironmentListForAutocompleteFilter(envName string, clusterIds []int, offset int, size int, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool), ctx context.Context) (*cluster.ResourceGroupingResponse, error) IsGitopsConfigured() (bool, error) RegisterInACD(gitOpsRepoName string, chartGitAttr *util.ChartGitAttribute, userId int32, ctx context.Context) error + CreateExternalCiAndAppWorkflowMapping(appId, appWorkflowId int, userId int32, tx *pg.Tx) (int, error) } type CdPipelineConfigServiceImpl struct { @@ -1383,15 +1384,7 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a } // Rollback tx on error. defer tx.Rollback() - if pipeline.AppWorkflowId == 0 && pipeline.ParentPipelineType == "WEBHOOK" { - externalCiPipeline := &pipelineConfig.ExternalCiPipeline{ - AppId: app.Id, - AccessToken: "", - Active: true, - AuditLog: sql.AuditLog{CreatedBy: userId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: userId}, - } - externalCiPipeline, err = impl.ciPipelineRepository.SaveExternalCi(externalCiPipeline, tx) wf := &appWorkflow.AppWorkflow{ Name: fmt.Sprintf("wf-%d-%s", app.Id, util2.Generate(4)), AppId: app.Id, @@ -1400,21 +1393,15 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a } savedAppWf, err := impl.appWorkflowRepository.SaveAppWorkflowWithTx(wf, tx) if err != nil { - impl.logger.Errorw("err", err) + impl.logger.Errorw("error in saving app workflow", "appId", app.Id, "err", err) return 0, err } - appWorkflowMap := &appWorkflow.AppWorkflowMapping{ - AppWorkflowId: savedAppWf.Id, - ComponentId: externalCiPipeline.Id, - Type: "WEBHOOK", - Active: true, - AuditLog: sql.AuditLog{CreatedBy: userId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: userId}, - } - appWorkflowMap, err = impl.appWorkflowRepository.SaveAppWorkflowMapping(appWorkflowMap, tx) + externalCiPipelineId, err := impl.CreateExternalCiAndAppWorkflowMapping(app.Id, savedAppWf.Id, userId, tx) if err != nil { + impl.logger.Errorw("error in creating new external ci pipeline and new app workflow mapping", "appId", app.Id, "err", err) return 0, err } - pipeline.ParentPipelineId = externalCiPipeline.Id + pipeline.ParentPipelineId = externalCiPipelineId pipeline.AppWorkflowId = savedAppWf.Id } @@ -1431,9 +1418,12 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a //TODO: mark as created in our db pipelineId, err := impl.ciCdPipelineOrchestrator.CreateCDPipelines(pipeline, app.Id, userId, tx, app.AppName) if err != nil { - impl.logger.Errorw("error in ") + impl.logger.Errorw("error in creating cd pipeline", "appId", app.Id, "pipeline", pipeline) return 0, err } + if pipeline.RefPipelineId > 0 { + pipeline.SourceToNewPipelineId[pipeline.RefPipelineId] = pipelineId + } //adding pipeline to workflow _, err = impl.appWorkflowRepository.FindByIdAndAppId(pipeline.AppWorkflowId, app.Id) @@ -1443,12 +1433,16 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a if pipeline.AppWorkflowId > 0 { var parentPipelineId int var parentPipelineType string + if pipeline.ParentPipelineId == 0 { parentPipelineId = pipeline.CiPipelineId parentPipelineType = "CI_PIPELINE" } else { parentPipelineId = pipeline.ParentPipelineId parentPipelineType = pipeline.ParentPipelineType + if pipeline.ParentPipelineType != appWorkflow.WEBHOOK && pipeline.RefPipelineId > 0 && len(pipeline.SourceToNewPipelineId) > 0 { + parentPipelineId = pipeline.SourceToNewPipelineId[pipeline.ParentPipelineId] + } } appWorkflowMap := &appWorkflow.AppWorkflowMapping{ AppWorkflowId: pipeline.AppWorkflowId, @@ -1817,3 +1811,30 @@ func (impl *CdPipelineConfigServiceImpl) BulkDeleteCdPipelines(impactedPipelines return respDtos } + +func (impl *CdPipelineConfigServiceImpl) CreateExternalCiAndAppWorkflowMapping(appId, appWorkflowId int, userId int32, tx *pg.Tx) (int, error) { + externalCiPipeline := &pipelineConfig.ExternalCiPipeline{ + AppId: appId, + AccessToken: "", + Active: true, + AuditLog: sql.AuditLog{CreatedBy: userId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: userId}, + } + externalCiPipeline, err := impl.ciPipelineRepository.SaveExternalCi(externalCiPipeline, tx) + if err != nil { + impl.logger.Errorw("error in saving external ci", "appId", appId, "err", err) + return 0, err + } + appWorkflowMap := &appWorkflow.AppWorkflowMapping{ + AppWorkflowId: appWorkflowId, + ComponentId: externalCiPipeline.Id, + Type: "WEBHOOK", + Active: true, + AuditLog: sql.AuditLog{CreatedBy: userId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: userId}, + } + appWorkflowMap, err = impl.appWorkflowRepository.SaveAppWorkflowMapping(appWorkflowMap, tx) + if err != nil { + impl.logger.Errorw("error in saving app workflow mapping for external ci", "appId", appId, "appWorkflowId", appWorkflowId, "externalCiPipelineId", externalCiPipeline.Id, "err", err) + return 0, err + } + return externalCiPipeline.Id, nil +} diff --git a/pkg/pipeline/repository/PipelineStageRepository.go b/pkg/pipeline/repository/PipelineStageRepository.go index 56ce81b3303..360a69840b4 100644 --- a/pkg/pipeline/repository/PipelineStageRepository.go +++ b/pkg/pipeline/repository/PipelineStageRepository.go @@ -29,7 +29,7 @@ const ( PIPELINE_STAGE_STEP_VARIABLE_VALUE_TYPE_GLOBAL PipelineStageStepVariableValueType = "GLOBAL" PIPELINE_STAGE_STEP_CONDITION_TYPE_SKIP PipelineStageStepConditionType = "SKIP" PIPELINE_STAGE_STEP_CONDITION_TYPE_TRIGGER PipelineStageStepConditionType = "TRIGGER" - PIPELINE_STAGE_STEP_CONDITION_TYPE_SUCCESS PipelineStageStepConditionType = "SUCCESS" + PIPELINE_STAGE_STEP_CONDITION_TYPE_SUCCESS PipelineStageStepConditionType = "PASS" PIPELINE_STAGE_STEP_CONDITION_TYPE_FAIL PipelineStageStepConditionType = "FAIL" PIPELINE_STAGE_STEP_VARIABLE_FORMAT_TYPE_STRING PipelineStageStepVariableFormatType = "STRING" PIPELINE_STAGE_STEP_VARIABLE_FORMAT_TYPE_NUMBER PipelineStageStepVariableFormatType = "NUMBER" diff --git a/scripts/sql/181_remove_index_image_scan_deploy_info_down.sql b/scripts/sql/181_remove_index_image_scan_deploy_info_down.sql new file mode 100644 index 00000000000..26e070b821d --- /dev/null +++ b/scripts/sql/181_remove_index_image_scan_deploy_info_down.sql @@ -0,0 +1,2 @@ +DROP index image_scan_deploy_info; +CREATE UNIQUE INDEX image_scan_deploy_info_unique ON public.image_scan_deploy_info USING btree (scan_object_meta_id, object_type); diff --git a/scripts/sql/181_remove_index_image_scan_deploy_info_up.sql b/scripts/sql/181_remove_index_image_scan_deploy_info_up.sql new file mode 100644 index 00000000000..369711433e1 --- /dev/null +++ b/scripts/sql/181_remove_index_image_scan_deploy_info_up.sql @@ -0,0 +1,2 @@ +DROP index image_scan_deploy_info_unique; +CREATE INDEX image_scan_deploy_info ON public.image_scan_deploy_info USING btree (scan_object_meta_id, object_type); diff --git a/wire_gen.go b/wire_gen.go index 83a2e9aaf95..6e761e3f7d3 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -392,19 +392,19 @@ func InitializeApp() (*App, error) { return nil, err } appStatusServiceImpl := appStatus2.NewAppStatusServiceImpl(appStatusRepositoryImpl, sugaredLogger, enforcerImpl, enforcerUtilImpl) + chartGroupDeploymentRepositoryImpl := repository3.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) refChartProxyDir := _wireRefChartProxyDirValue appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, refChartProxyDir, gitFactory, gitOpsConfigRepositoryImpl) ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, chartTemplateServiceImpl, refChartProxyDir, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, applicationServiceClientImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, globalEnvVariables, installedAppRepositoryImpl, tokenCache, argoUserServiceImpl, gitOpsConfigRepositoryImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl) - chartGroupDeploymentRepositoryImpl := repository3.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, chartTemplateServiceImpl, gitFactory, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, gitOpsConfigRepositoryImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl) deploymentServiceTypeConfig, err := service.GetDeploymentServiceTypeConfig() if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, pubSubClientServiceImpl) + appStoreDeploymentServiceImpl := service.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, pubSubClientServiceImpl) k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImplExtended) manifestPushConfigRepositoryImpl := repository11.NewManifestPushConfigRepository(sugaredLogger, db) gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl) @@ -505,7 +505,7 @@ func InitializeApp() (*App, error) { deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl) - appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl) + appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, appWorkflowRepositoryImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) deploymentTemplateServiceImpl := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, appListingRepositoryImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sUtil, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl) imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger)