From c026885b8a23941c961e2fa99f6efbe3d70e72a3 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 21 Dec 2023 12:54:27 +0530 Subject: [PATCH 1/5] fix: CD stage trigger for external CI --- internal/sql/repository/CiArtifactRepository.go | 2 +- pkg/pipeline/WorkflowDagExecutor.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 93001355e40..ed563882642 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -590,7 +590,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineV2(cdPipelineId int } func GetCiMaterialInfo(materialInfo string, source string) ([]CiMaterialInfo, error) { - if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" && source != "post_ci" && source != "pre_cd" && source != "post_cd" { + if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" && source != "post_ci" && source != "pre_cd" && source != "post_cd" && source != "ext" { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []CiMaterialInfo diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 3efe3788253..3827c1be792 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -1766,7 +1766,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor cdStageWorkflowRequest.DockerRegistryType = string(ciTemplate.DockerRegistry.RegistryType) cdStageWorkflowRequest.DockerRegistryURL = ciTemplate.DockerRegistry.RegistryURL appLabels, err := impl.appLabelRepository.FindAllByAppId(cdPipeline.AppId) - cdStageWorkflowRequest.DockerRegistryId = ciPipeline.CiTemplate.DockerRegistry.Id + cdStageWorkflowRequest.DockerRegistryId = ciTemplate.DockerRegistry.Id if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting labels by appId", "err", err, "appId", cdPipeline.AppId) return nil, err From 118ce4bf36854d2a35fd92771c30b4c969c0e47d Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 2 Jan 2024 11:49:32 +0530 Subject: [PATCH 2/5] feat: handling for DataSource type backward compatibility --- api/restHandler/ExternalCiRestHandler.go | 7 ++++ api/router/pubsub/CiEventHandler.go | 37 +++++++++++++++---- .../sql/repository/AppListingRepository.go | 2 +- .../sql/repository/CiArtifactRepository.go | 25 +++++++------ pkg/deploymentGroup/DeploymentGroupService.go | 2 +- pkg/pipeline/PipelineBuilder.go | 2 +- pkg/pipeline/WebhookService.go | 32 ++++++---------- 7 files changed, 65 insertions(+), 42 deletions(-) diff --git a/api/restHandler/ExternalCiRestHandler.go b/api/restHandler/ExternalCiRestHandler.go index 5ea4a12d28c..9f74b61a68b 100644 --- a/api/restHandler/ExternalCiRestHandler.go +++ b/api/restHandler/ExternalCiRestHandler.go @@ -98,6 +98,13 @@ func (impl ExternalCiRestHandlerImpl) HandleExternalCiWebhook(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } + + err = impl.validator.Struct(ciArtifactReq) + if err != nil { + impl.logger.Errorw("validation err, HandleExternalCiWebhook", "err", err, "payload", ciArtifactReq) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } _, err = impl.webhookService.HandleExternalCiWebhook(externalCiId, ciArtifactReq, impl.checkExternalCiDeploymentAuth) if err != nil { impl.logger.Errorw("service err, HandleExternalCiWebhook", "err", err, "payload", req) diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go index 649e204b2e5..cc3c727ca8e 100644 --- a/api/router/pubsub/CiEventHandler.go +++ b/api/router/pubsub/CiEventHandler.go @@ -30,6 +30,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/util" "go.uber.org/zap" + "gopkg.in/go-playground/validator.v9" "time" ) @@ -53,6 +54,7 @@ type CiEventHandlerImpl struct { logger *zap.SugaredLogger pubsubClient *pubsub.PubSubClientServiceImpl webhookService pipeline.WebhookService + validator *validator.Validate ciEventConfig *CiEventConfig } @@ -80,11 +82,12 @@ type CiCompleteEvent struct { PluginArtifactStage string `json:"pluginArtifactStage"` } -func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, ciEventConfig *CiEventConfig) *CiEventHandlerImpl { +func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, validator *validator.Validate, ciEventConfig *CiEventConfig) *CiEventHandlerImpl { ciEventHandlerImpl := &CiEventHandlerImpl{ logger: logger, pubsubClient: pubsubClient, webhookService: webhookService, + validator: validator, ciEventConfig: ciEventConfig, } err := ciEventHandlerImpl.Subscribe() @@ -133,10 +136,8 @@ func (impl *CiEventHandlerImpl) Subscribe() error { impl.logger.Error("Error while creating request for pipelineID", "pipelineId", ciCompleteEvent.PipelineId, "err", err) return } - resp, err := impl.webhookService.HandleCiSuccessEvent(ciCompleteEvent.PipelineId, request, detail.ImagePushedAt) + resp, err := impl.ValidateAndHandleCiSuccessEvent(ciCompleteEvent.PipelineId, request, detail.ImagePushedAt) if err != nil { - impl.logger.Error("Error while sending event for CI success for pipelineID", "pipelineId", - ciCompleteEvent.PipelineId, "request", request, "err", err) return } impl.logger.Debug("response of handle ci success event for multiple images from plugin", "resp", resp) @@ -145,10 +146,8 @@ func (impl *CiEventHandlerImpl) Subscribe() error { } else { util.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.ciEventConfig.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName) - resp, err := impl.webhookService.HandleCiSuccessEvent(ciCompleteEvent.PipelineId, req, &time.Time{}) + resp, err := impl.ValidateAndHandleCiSuccessEvent(ciCompleteEvent.PipelineId, req, &time.Time{}) if err != nil { - impl.logger.Error("Error while sending event for CI success for pipelineID: ", - ciCompleteEvent.PipelineId, "request: ", req, "error: ", err) return } impl.logger.Debug(resp) @@ -162,6 +161,21 @@ func (impl *CiEventHandlerImpl) Subscribe() error { return nil } +func (impl *CiEventHandlerImpl) ValidateAndHandleCiSuccessEvent(ciPipelineId int, request *pipeline.CiArtifactWebhookRequest, imagePushedAt *time.Time) (int, error) { + validationErr := impl.validator.Struct(request) + if validationErr != nil { + impl.logger.Errorw("validation err, HandleCiSuccessEvent", "err", validationErr, "payload", request) + return 0, validationErr + } + buildArtifactId, err := impl.webhookService.HandleCiSuccessEvent(ciPipelineId, request, imagePushedAt) + if err != nil { + impl.logger.Error("Error while sending event for CI success for pipelineID", + ciPipelineId, "request", request, "error", err) + return 0, err + } + return buildArtifactId, nil +} + func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (*pipeline.CiArtifactWebhookRequest, error) { var ciMaterialInfos []repository.CiMaterialInfo for _, p := range event.CiProjectDetails { @@ -228,6 +242,9 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (* PluginRegistryArtifactDetails: event.PluginRegistryArtifactDetails, PluginArtifactStage: event.PluginArtifactStage, } + if request.DataSource == "" { + request.DataSource = repository.WEBHOOK + } return request, nil } @@ -244,6 +261,9 @@ func (impl *CiEventHandlerImpl) BuildCIArtifactRequestForImageFromCR(imageDetail WorkflowId: &workflowId, IsArtifactUploaded: event.IsArtifactUploaded, } + if request.DataSource == "" { + request.DataSource = repository.WEBHOOK + } return request, nil } @@ -314,5 +334,8 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequestForWebhook(event CiComplet WorkflowId: event.WorkflowId, IsArtifactUploaded: event.IsArtifactUploaded, } + if request.DataSource == "" { + request.DataSource = repository.WEBHOOK + } return request, nil } diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 2b4a9f84f66..a629b07a02a 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -429,7 +429,7 @@ func parseMaterialInfo(materialInfo string, source string) (json.RawMessage, err fmt.Printf("PARSEMATERIALINFO_MATERIAL_RECOVER, materialInfo: %s, source: %s, err: %s \n", materialInfo, source, r) } }() - if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" { + if source != GOCD && source != CI_RUNNER && source != WEBHOOK && source != DEPRICATED_EXT { return nil, fmt.Errorf("datasource: %s not supported", source) } if materialInfo == "" { diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index ed563882642..4d1c8ba7507 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -32,19 +32,22 @@ import ( ) type credentialsSource = string -type artifactsSourceType = string +type ArtifactsSourceType = string const ( GLOBAL_CONTAINER_REGISTRY credentialsSource = "global_container_registry" ) + +// List of possible DataSource Type for an artifact const ( - CI_RUNNER artifactsSourceType = "CI-RUNNER" - WEBHOOK artifactsSourceType = "EXTERNAL" - PRE_CD artifactsSourceType = "pre_cd" - POST_CD artifactsSourceType = "post_cd" - PRE_CI artifactsSourceType = "pre_ci" - POST_CI artifactsSourceType = "post_ci" - GOCD artifactsSourceType = "GOCD" + CI_RUNNER ArtifactsSourceType = "CI-RUNNER" + WEBHOOK ArtifactsSourceType = "EXTERNAL" // Currently in use instead of DEPRICATED_EXT + PRE_CD ArtifactsSourceType = "pre_cd" + POST_CD ArtifactsSourceType = "post_cd" + POST_CI ArtifactsSourceType = "post_ci" + GOCD ArtifactsSourceType = "GOCD" + DEPRICATED_EXT ArtifactsSourceType = "ext" // For backward compatibility + // PRE_CI is not a valid DataSource for an artifact ) type CiArtifactWithExtraData struct { @@ -63,7 +66,7 @@ type CiArtifact struct { Image string `sql:"image,notnull"` ImageDigest string `sql:"image_digest,notnull"` MaterialInfo string `sql:"material_info"` //git material metadata json array string - DataSource string `sql:"data_source,notnull"` // possible values -> (CI_RUNNER,ext,post_ci,pre_cd,post_cd) CI_runner is for normal build ci + DataSource string `sql:"data_source,notnull"` // possible values -> (CI_RUNNER,EXTERNAL,post_ci,pre_cd,post_cd) CI_runner is for normal build ci WorkflowId *int `sql:"ci_workflow_id"` ParentCiArtifact int `sql:"parent_ci_artifact"` ScanEnabled bool `sql:"scan_enabled,notnull"` @@ -479,7 +482,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineAndRunnerType(cdPip // return map of gitUrl:hash func (info *CiArtifact) ParseMaterialInfo() (map[string]string, error) { - if info.DataSource != "GOCD" && info.DataSource != "CI-RUNNER" && info.DataSource != "EXTERNAL" { + if info.DataSource != GOCD && info.DataSource != CI_RUNNER && info.DataSource != WEBHOOK && info.DataSource != DEPRICATED_EXT { return nil, fmt.Errorf("datasource: %s not supported", info.DataSource) } var ciMaterials []*CiMaterialInfo @@ -590,7 +593,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineV2(cdPipelineId int } func GetCiMaterialInfo(materialInfo string, source string) ([]CiMaterialInfo, error) { - if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" && source != "post_ci" && source != "pre_cd" && source != "post_cd" && source != "ext" { + if source != GOCD && source != CI_RUNNER && source != WEBHOOK && source != POST_CI && source != PRE_CD && source != POST_CD && source != DEPRICATED_EXT { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []CiMaterialInfo diff --git a/pkg/deploymentGroup/DeploymentGroupService.go b/pkg/deploymentGroup/DeploymentGroupService.go index 9bf843e19f8..34eaaf70e26 100644 --- a/pkg/deploymentGroup/DeploymentGroupService.go +++ b/pkg/deploymentGroup/DeploymentGroupService.go @@ -610,7 +610,7 @@ func (impl *DeploymentGroupServiceImpl) GetArtifactsByCiPipeline(ciPipelineId in } func (impl *DeploymentGroupServiceImpl) parseMaterialInfo(materialInfo json.RawMessage, source string) (json.RawMessage, error) { - if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" { + if source != repository.GOCD && source != repository.CI_RUNNER && source != repository.WEBHOOK && source != repository.DEPRICATED_EXT { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []repository.CiMaterialInfo diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 50dadefe9d2..d9af26d2f37 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -253,7 +253,7 @@ type ConfigMapSecretsResponse struct { } func parseMaterialInfo(materialInfo json.RawMessage, source string) (json.RawMessage, error) { - if source != repository.GOCD && source != repository.CI_RUNNER && source != repository.WEBHOOK && source != repository.PRE_CD && source != repository.POST_CD && source != repository.POST_CI { + if source != repository.GOCD && source != repository.CI_RUNNER && source != repository.WEBHOOK && source != repository.DEPRICATED_EXT && source != repository.PRE_CD && source != repository.POST_CD && source != repository.POST_CI { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []repository.CiMaterialInfo diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index 6168d00e96d..3884669894f 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -44,17 +44,17 @@ import ( ) type CiArtifactWebhookRequest struct { - Image string `json:"image"` - ImageDigest string `json:"imageDigest"` - MaterialInfo json.RawMessage `json:"materialInfo"` - DataSource string `json:"dataSource"` - PipelineName string `json:"pipelineName"` - WorkflowId *int `json:"workflowId"` - UserId int32 `json:"userId"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - FailureReason string `json:"failureReason"` - PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` //map of registry and array of images generated by Copy container image plugin - PluginArtifactStage string `json:"pluginArtifactStage"` // at which stage of CI artifact was generated by plugin ("pre_ci/post_ci") + Image string `json:"image" validate:"required"` + ImageDigest string `json:"imageDigest"` + MaterialInfo json.RawMessage `json:"materialInfo"` + DataSource repository.ArtifactsSourceType `json:"dataSource" validate:"oneof=CI-RUNNER EXTERNAL pre_cd post_cd post_ci GOCD"` + PipelineName string `json:"pipelineName"` + WorkflowId *int `json:"workflowId"` + UserId int32 `json:"userId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + FailureReason string `json:"failureReason"` + PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` //map of registry and array of images generated by Copy container image plugin + PluginArtifactStage string `json:"pluginArtifactStage"` // at which stage of CI artifact was generated by plugin ("pre_ci/post_ci") } type WebhookService interface { @@ -190,9 +190,6 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C if request.PipelineName == "" { request.PipelineName = pipeline.Name } - if request.DataSource == "" { - request.DataSource = "EXTERNAL" - } if err != nil { impl.logger.Errorw("unable to find pipeline", "name", request.PipelineName, "err", err) return 0, err @@ -213,10 +210,6 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C if !imagePushedAt.IsZero() { createdOn = *imagePushedAt } - if pipeline.PipelineType == bean.CI_JOB && request.Image == "" { - impl.logger.Errorw("empty image artifact found!", "request", request) - return 0, fmt.Errorf("empty image artifact found") - } buildArtifact := &repository.CiArtifact{ Image: request.Image, ImageDigest: request.ImageDigest, @@ -376,9 +369,6 @@ func (impl WebhookServiceImpl) HandleExternalCiWebhook(externalCiId int, request } impl.logger.Infow("request of webhook external ci", "req", request) - if request.DataSource == "" { - request.DataSource = "EXTERNAL" - } materialJson, err := request.MaterialInfo.MarshalJSON() if err != nil { impl.logger.Errorw("unable to marshal material metadata", "err", err) From 9031282ea82888c473edf1c467e2a46a8921375c Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 2 Jan 2024 16:33:18 +0530 Subject: [PATCH 3/5] migration for deprecated DataSourceType --- .../sql/repository/CiArtifactRepository.go | 13 +++++ pkg/pipeline/WorkflowDagExecutor.go | 47 +++++++++++++++++-- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 4d1c8ba7507..03994a9edc7 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -86,6 +86,9 @@ type CiArtifact struct { type CiArtifactRepository interface { Save(artifact *CiArtifact) error Delete(artifact *CiArtifact) error + + // Get returns the CiArtifact of the given id. + // Note: Use Get along with MigrateToWebHookDataSourceType. For webhook artifacts, migration is required for column DataSource from 'ext' to 'EXTERNAL' Get(id int) (artifact *CiArtifact, err error) GetArtifactParentCiAndWorkflowDetailsByIds(ids []int) ([]*CiArtifact, error) GetByWfId(wfId int) (artifact *CiArtifact, err error) @@ -109,6 +112,8 @@ type CiArtifactRepository interface { GetArtifactsByDataSourceAndComponentId(dataSource string, componentId int) ([]CiArtifact, error) FindCiArtifactByImagePaths(images []string) ([]CiArtifact, error) + // MigrateToWebHookDataSourceType is used for backward compatibility. It'll migrate the deprecated DataSource type + MigrateToWebHookDataSourceType(id int) error UpdateLatestTimestamp(artifactIds []int) error } @@ -135,6 +140,14 @@ func (impl CiArtifactRepositoryImpl) SaveAll(artifacts []*CiArtifact) ([]*CiArti return artifacts, err } +func (impl CiArtifactRepositoryImpl) MigrateToWebHookDataSourceType(id int) error { + _, err := impl.dbConnection.Model(&CiArtifact{}). + Set("data_source = ?", WEBHOOK). + Where("id = ?", id). + Update() + return err +} + func (impl CiArtifactRepositoryImpl) UpdateLatestTimestamp(artifactIds []int) error { if len(artifactIds) == 0 { impl.logger.Debug("UpdateLatestTimestamp empty list of artifacts, not updating") diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 3827c1be792..c9dde1d0e41 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -919,6 +919,13 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(cdStageCompleteE if err != nil { return err } + // Migration of deprecated DataSource Type + if ciArtifact.DataSource == repository.DEPRICATED_EXT { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(ciArtifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", ciArtifact.Id) + } + } PreCDArtifacts, err := impl.SavePluginArtifacts(ciArtifact, cdStageCompleteEvent.PluginRegistryArtifactDetails, pipeline.Id, repository.PRE_CD, cdStageCompleteEvent.TriggeredBy) if err != nil { impl.logger.Errorw("error in saving plugin artifacts", "err", err) @@ -1278,6 +1285,13 @@ func (impl *WorkflowDagExecutorImpl) TriggerPostStage(cdWf *pipelineConfig.CdWor return err } } + // Migration of deprecated DataSource Type + if cdWf.CiArtifact.DataSource == repository.DEPRICATED_EXT { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(cdWf.CiArtifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", cdWf.CiArtifact.Id) + } + } //checking vulnerability for the selected image isVulnerable, err := impl.GetArtifactVulnerabilityStatus(cdWf.CiArtifact, pipeline, context.Background()) if err != nil { @@ -1442,7 +1456,13 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor if err != nil { return nil, err } - + // Migration of deprecated DataSource Type + if artifact.DataSource == repository.DEPRICATED_EXT { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } ciMaterialInfo, err := repository.GetCiMaterialInfo(artifact.MaterialInfo, artifact.DataSource) if err != nil { impl.logger.Errorw("parsing error", "err", err) @@ -2326,6 +2346,13 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value impl.logger.Errorw("error in getting CiArtifact", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) return 0, err } + // Migration of deprecated DataSource Type + if artifact.DataSource == repository.DEPRICATED_EXT { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPreStage") err = impl.TriggerPreStage(ctx, nil, artifact, cdPipeline, overrideRequest.UserId, false, 0) span.End() @@ -2398,6 +2425,13 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value impl.logger.Errorw("error in getting ciArtifact, ManualCdTrigger", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) return 0, err } + // Migration of deprecated DataSource Type + if artifact.DataSource == repository.DEPRICATED_EXT { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } isVulnerable, err := impl.GetArtifactVulnerabilityStatus(artifact, cdPipeline, ctx) if err != nil { impl.logger.Errorw("error in getting Artifact vulnerability status, ManualCdTrigger", "err", err) @@ -2619,14 +2653,21 @@ func (impl *WorkflowDagExecutorImpl) subscribeTriggerBulkAction() error { impl.cdWorkflowRepository.UpdateWorkFlow(wf) return } - artefact, err := impl.ciArtifactRepository.Get(cdWorkflow.CiArtifactId) + artifact, err := impl.ciArtifactRepository.Get(cdWorkflow.CiArtifactId) if err != nil { impl.logger.Errorw("error in fetching artefact", "err", err) wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR impl.cdWorkflowRepository.UpdateWorkFlow(wf) return } - err = impl.triggerStageForBulk(wf, pipeline, artefact, false, false, cdWorkflow.CreatedBy) + // Migration of deprecated DataSource Type + if artifact.DataSource == repository.DEPRICATED_EXT { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } + err = impl.triggerStageForBulk(wf, pipeline, artifact, false, false, cdWorkflow.CreatedBy) if err != nil { impl.logger.Errorw("error in cd trigger ", "err", err) wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR From 9c9bb114bfc037fc33bdd230bc405cc79f1fc518 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Wed, 3 Jan 2024 17:44:05 +0530 Subject: [PATCH 4/5] updated ArtifactsSourceType const --- api/router/pubsub/CiEventHandler.go | 2 ++ .../sql/repository/AppListingRepository.go | 2 +- .../sql/repository/CiArtifactRepository.go | 19 ++++++++++--------- pkg/deploymentGroup/DeploymentGroupService.go | 2 +- pkg/pipeline/PipelineBuilder.go | 2 +- pkg/pipeline/WorkflowDagExecutor.go | 12 ++++++------ wire_gen.go | 2 +- 7 files changed, 22 insertions(+), 19 deletions(-) diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go index cc3c727ca8e..53f95fc9387 100644 --- a/api/router/pubsub/CiEventHandler.go +++ b/api/router/pubsub/CiEventHandler.go @@ -242,6 +242,7 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (* PluginRegistryArtifactDetails: event.PluginRegistryArtifactDetails, PluginArtifactStage: event.PluginArtifactStage, } + // if DataSource is empty, repository.WEBHOOK is considered as default if request.DataSource == "" { request.DataSource = repository.WEBHOOK } @@ -334,6 +335,7 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequestForWebhook(event CiComplet WorkflowId: event.WorkflowId, IsArtifactUploaded: event.IsArtifactUploaded, } + // if DataSource is empty, repository.WEBHOOK is considered as default if request.DataSource == "" { request.DataSource = repository.WEBHOOK } diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index a629b07a02a..ecff1f94c22 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -429,7 +429,7 @@ func parseMaterialInfo(materialInfo string, source string) (json.RawMessage, err fmt.Printf("PARSEMATERIALINFO_MATERIAL_RECOVER, materialInfo: %s, source: %s, err: %s \n", materialInfo, source, r) } }() - if source != GOCD && source != CI_RUNNER && source != WEBHOOK && source != DEPRICATED_EXT { + if source != GOCD && source != CI_RUNNER && source != WEBHOOK && source != EXT { return nil, fmt.Errorf("datasource: %s not supported", source) } if materialInfo == "" { diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 03994a9edc7..76fefa7ebed 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -40,13 +40,14 @@ const ( // List of possible DataSource Type for an artifact const ( - CI_RUNNER ArtifactsSourceType = "CI-RUNNER" - WEBHOOK ArtifactsSourceType = "EXTERNAL" // Currently in use instead of DEPRICATED_EXT - PRE_CD ArtifactsSourceType = "pre_cd" - POST_CD ArtifactsSourceType = "post_cd" - POST_CI ArtifactsSourceType = "post_ci" - GOCD ArtifactsSourceType = "GOCD" - DEPRICATED_EXT ArtifactsSourceType = "ext" // For backward compatibility + CI_RUNNER ArtifactsSourceType = "CI-RUNNER" + WEBHOOK ArtifactsSourceType = "EXTERNAL" // Currently in use instead of EXT + PRE_CD ArtifactsSourceType = "pre_cd" + POST_CD ArtifactsSourceType = "post_cd" + POST_CI ArtifactsSourceType = "post_ci" + GOCD ArtifactsSourceType = "GOCD" + // deprecated; Handled for backward compatibility + EXT ArtifactsSourceType = "ext" // PRE_CI is not a valid DataSource for an artifact ) @@ -495,7 +496,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineAndRunnerType(cdPip // return map of gitUrl:hash func (info *CiArtifact) ParseMaterialInfo() (map[string]string, error) { - if info.DataSource != GOCD && info.DataSource != CI_RUNNER && info.DataSource != WEBHOOK && info.DataSource != DEPRICATED_EXT { + if info.DataSource != GOCD && info.DataSource != CI_RUNNER && info.DataSource != WEBHOOK && info.DataSource != EXT { return nil, fmt.Errorf("datasource: %s not supported", info.DataSource) } var ciMaterials []*CiMaterialInfo @@ -606,7 +607,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineV2(cdPipelineId int } func GetCiMaterialInfo(materialInfo string, source string) ([]CiMaterialInfo, error) { - if source != GOCD && source != CI_RUNNER && source != WEBHOOK && source != POST_CI && source != PRE_CD && source != POST_CD && source != DEPRICATED_EXT { + if source != GOCD && source != CI_RUNNER && source != WEBHOOK && source != POST_CI && source != PRE_CD && source != POST_CD && source != EXT { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []CiMaterialInfo diff --git a/pkg/deploymentGroup/DeploymentGroupService.go b/pkg/deploymentGroup/DeploymentGroupService.go index 34eaaf70e26..3ceb1357a29 100644 --- a/pkg/deploymentGroup/DeploymentGroupService.go +++ b/pkg/deploymentGroup/DeploymentGroupService.go @@ -610,7 +610,7 @@ func (impl *DeploymentGroupServiceImpl) GetArtifactsByCiPipeline(ciPipelineId in } func (impl *DeploymentGroupServiceImpl) parseMaterialInfo(materialInfo json.RawMessage, source string) (json.RawMessage, error) { - if source != repository.GOCD && source != repository.CI_RUNNER && source != repository.WEBHOOK && source != repository.DEPRICATED_EXT { + if source != repository.GOCD && source != repository.CI_RUNNER && source != repository.WEBHOOK && source != repository.EXT { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []repository.CiMaterialInfo diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index d9af26d2f37..06dc8d03bc0 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -253,7 +253,7 @@ type ConfigMapSecretsResponse struct { } func parseMaterialInfo(materialInfo json.RawMessage, source string) (json.RawMessage, error) { - if source != repository.GOCD && source != repository.CI_RUNNER && source != repository.WEBHOOK && source != repository.DEPRICATED_EXT && source != repository.PRE_CD && source != repository.POST_CD && source != repository.POST_CI { + if source != repository.GOCD && source != repository.CI_RUNNER && source != repository.WEBHOOK && source != repository.EXT && source != repository.PRE_CD && source != repository.POST_CD && source != repository.POST_CI { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []repository.CiMaterialInfo diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index c9dde1d0e41..29bb7f84750 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -920,7 +920,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(cdStageCompleteE return err } // Migration of deprecated DataSource Type - if ciArtifact.DataSource == repository.DEPRICATED_EXT { + if ciArtifact.DataSource == repository.EXT { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(ciArtifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", ciArtifact.Id) @@ -1286,7 +1286,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerPostStage(cdWf *pipelineConfig.CdWor } } // Migration of deprecated DataSource Type - if cdWf.CiArtifact.DataSource == repository.DEPRICATED_EXT { + if cdWf.CiArtifact.DataSource == repository.EXT { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(cdWf.CiArtifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", cdWf.CiArtifact.Id) @@ -1457,7 +1457,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor return nil, err } // Migration of deprecated DataSource Type - if artifact.DataSource == repository.DEPRICATED_EXT { + if artifact.DataSource == repository.EXT { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) @@ -2347,7 +2347,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value return 0, err } // Migration of deprecated DataSource Type - if artifact.DataSource == repository.DEPRICATED_EXT { + if artifact.DataSource == repository.EXT { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) @@ -2426,7 +2426,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value return 0, err } // Migration of deprecated DataSource Type - if artifact.DataSource == repository.DEPRICATED_EXT { + if artifact.DataSource == repository.EXT { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) @@ -2661,7 +2661,7 @@ func (impl *WorkflowDagExecutorImpl) subscribeTriggerBulkAction() error { return } // Migration of deprecated DataSource Type - if artifact.DataSource == repository.DEPRICATED_EXT { + if artifact.DataSource == repository.EXT { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) diff --git a/wire_gen.go b/wire_gen.go index 6d77689ce6a..6b65601e387 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -578,7 +578,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciEventHandlerImpl := pubsub.NewCiEventHandlerImpl(sugaredLogger, pubSubClientServiceImpl, webhookServiceImpl, ciEventConfig) + ciEventHandlerImpl := pubsub.NewCiEventHandlerImpl(sugaredLogger, pubSubClientServiceImpl, webhookServiceImpl, validate, ciEventConfig) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, webhookServiceImpl, ciEventHandlerImpl, validate, userServiceImpl, enforcerImpl, enforcerUtilImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) From e19d74bea133507752aecf30f928895192549526 Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Tue, 16 Jan 2024 17:40:50 +0530 Subject: [PATCH 5/5] updated: ci_artifact.data_source migration condition (#4548) --- internal/sql/repository/CiArtifactRepository.go | 9 +++++++++ pkg/pipeline/WorkflowDagExecutor.go | 12 ++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 83825c751d3..caf53ddb83f 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/sql" + "golang.org/x/exp/slices" "strings" "time" @@ -84,6 +85,14 @@ type CiArtifact struct { sql.AuditLog } +func (c *CiArtifact) IsMigrationRequired() bool { + validDataSourceTypeList := []string{CI_RUNNER, WEBHOOK, PRE_CD, POST_CD, POST_CI, GOCD} + if slices.Contains(validDataSourceTypeList, c.DataSource) { + return false + } + return true +} + type CiArtifactRepository interface { Save(artifact *CiArtifact) error Delete(artifact *CiArtifact) error diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 170cddaaf0c..8f4750cf6cc 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -923,7 +923,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(cdStageCompleteE return err } // Migration of deprecated DataSource Type - if ciArtifact.DataSource == repository.EXT { + if ciArtifact.IsMigrationRequired() { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(ciArtifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", ciArtifact.Id) @@ -1270,7 +1270,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerPostStage(cdWf *pipelineConfig.CdWor } } // Migration of deprecated DataSource Type - if cdWf.CiArtifact.DataSource == repository.EXT { + if cdWf.CiArtifact.IsMigrationRequired() { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(cdWf.CiArtifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", cdWf.CiArtifact.Id) @@ -1441,7 +1441,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor return nil, err } // Migration of deprecated DataSource Type - if artifact.DataSource == repository.EXT { + if artifact.IsMigrationRequired() { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) @@ -2311,7 +2311,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value return 0, err } // Migration of deprecated DataSource Type - if artifact.DataSource == repository.EXT { + if artifact.IsMigrationRequired() { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) @@ -2390,7 +2390,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value return 0, err } // Migration of deprecated DataSource Type - if artifact.DataSource == repository.EXT { + if artifact.IsMigrationRequired() { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) @@ -2625,7 +2625,7 @@ func (impl *WorkflowDagExecutorImpl) subscribeTriggerBulkAction() error { return } // Migration of deprecated DataSource Type - if artifact.DataSource == repository.EXT { + if artifact.IsMigrationRequired() { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id)