Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: CD stage trigger is not working for external CI #4440

Merged
merged 9 commits into from
Jan 18, 2024
Merged

fix: CD stage trigger is not working for external CI #4440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c026885
fix: CD stage trigger for external CI
Ash-exp Dec 21, 2023
133b2d2
Merge branch 'main' into fix-cd-stage-ext-ci
Ash-exp Dec 22, 2023
118ce4b
feat: handling for DataSource type backward compatibility
Ash-exp Jan 2, 2024
9031282
migration for deprecated DataSourceType
Ash-exp Jan 2, 2024
9c9bb11
updated ArtifactsSourceType const
Ash-exp Jan 3, 2024
c064f80
chore: main merge
Ash-exp Jan 9, 2024
bd38b2c
chore: main merge
Ash-exp Jan 15, 2024
e19d74b
updated: ci_artifact.data_source migration condition (#4548)
Ash-exp Jan 16, 2024
864d192
Merge branch 'main' into fix-cd-stage-ext-ci
Ash-exp Jan 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 3 additions & 3 deletions api/appStore/chartProvider/ChartProviderRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ type ChartProviderRestHandler interface {
type ChartProviderRestHandlerImpl struct {
Logger *zap.SugaredLogger
chartProviderService chartProvider.ChartProviderService
validator *validator.Validate
userAuthService user.UserService
enforcer casbin.Enforcer
validator *validator.Validate
userAuthService user.UserService
enforcer casbin.Enforcer
}

func NewChartProviderRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, validator *validator.Validate, chartProviderService chartProvider.ChartProviderService,
Expand Down
6 changes: 3 additions & 3 deletions api/appStore/deployment/AppStoreDeploymentRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@ type AppStoreDeploymentRestHandler interface {
}

type AppStoreDeploymentRestHandlerImpl struct {
Logger *zap.SugaredLogger
userAuthService user.UserService
enforcer casbin.Enforcer
Logger *zap.SugaredLogger
userAuthService user.UserService
enforcer casbin.Enforcer
enforcerUtil rbac.EnforcerUtil
enforcerUtilHelm rbac.EnforcerUtilHelm
appStoreDeploymentService service.AppStoreDeploymentService
Expand Down
6 changes: 3 additions & 3 deletions api/appStore/deployment/CommonDeploymentRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ type CommonDeploymentRestHandler interface {
}

type CommonDeploymentRestHandlerImpl struct {
Logger *zap.SugaredLogger
userAuthService user.UserService
enforcer casbin.Enforcer
Logger *zap.SugaredLogger
userAuthService user.UserService
enforcer casbin.Enforcer
enforcerUtil rbac.EnforcerUtil
enforcerUtilHelm rbac.EnforcerUtilHelm
appStoreDeploymentService service.AppStoreDeploymentService
Expand Down
6 changes: 3 additions & 3 deletions api/deployment/DeploymentConfigRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ type DeploymentConfigRestHandler interface {
}

type DeploymentConfigRestHandlerImpl struct {
Logger *zap.SugaredLogger
userAuthService user.UserService
enforcer casbin.Enforcer
Logger *zap.SugaredLogger
userAuthService user.UserService
enforcer casbin.Enforcer
validator *validator.Validate
refChartDir chartRepoRepository.RefChartDir
chartService chart.ChartService
Expand Down
6 changes: 3 additions & 3 deletions api/k8s/application/k8sApplicationRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,9 @@ type K8sApplicationRestHandlerImpl struct {
validator *validator.Validate
enforcerUtil rbac.EnforcerUtil
enforcerUtilHelm rbac.EnforcerUtilHelm
helmAppService client.HelmAppService
userService user.UserService
k8sCommonService k8s.K8sCommonService
helmAppService client.HelmAppService
userService user.UserService
k8sCommonService k8s.K8sCommonService
}

func NewK8sApplicationRestHandlerImpl(logger *zap.SugaredLogger, k8sApplicationService application2.K8sApplicationService, pump connector.Pump, terminalSessionHandler terminal.TerminalSessionHandler, enforcer casbin.Enforcer, enforcerUtilHelm rbac.EnforcerUtilHelm, enforcerUtil rbac.EnforcerUtil, helmAppService client.HelmAppService, userService user.UserService, k8sCommonService k8s.K8sCommonService, validator *validator.Validate) *K8sApplicationRestHandlerImpl {
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/ChartGroupRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@ const CHART_GROUP_DELETE_SUCCESS_RESP = "Chart group deleted successfully."

type ChartGroupRestHandlerImpl struct {
ChartGroupService service.ChartGroupService
Logger *zap.SugaredLogger
userAuthService user.UserService
enforcer casbin.Enforcer
Logger *zap.SugaredLogger
userAuthService user.UserService
enforcer casbin.Enforcer
validator *validator.Validate
}

Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/CoreAppRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ type CoreAppRestHandler interface {
}

type CoreAppRestHandlerImpl struct {
logger *zap.SugaredLogger
userAuthService user.UserService
validator *validator.Validate
logger *zap.SugaredLogger
userAuthService user.UserService
validator *validator.Validate
enforcerUtil rbac.EnforcerUtil
enforcer casbin.Enforcer
appCrudOperationService app.AppCrudOperationService
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/DeploymentGroupRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ type DeploymentGroupRestHandlerImpl struct {
logger *zap.SugaredLogger
validator *validator.Validate
enforcer casbin.Enforcer
teamService team.TeamService
userAuthService user.UserService
enforcerUtil rbac.EnforcerUtil
teamService team.TeamService
userAuthService user.UserService
enforcerUtil rbac.EnforcerUtil
}

func NewDeploymentGroupRestHandlerImpl(deploymentGroupService deploymentGroup.DeploymentGroupService, logger *zap.SugaredLogger,
Expand Down
7 changes: 7 additions & 0 deletions api/restHandler/ExternalCiRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,13 @@ func (impl ExternalCiRestHandlerImpl) HandleExternalCiWebhook(w http.ResponseWri
common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
return
}

err = impl.validator.Struct(ciArtifactReq)
if err != nil {
impl.logger.Errorw("validation err, HandleExternalCiWebhook", "err", err, "payload", ciArtifactReq)
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
return
}
_, err = impl.webhookService.HandleExternalCiWebhook(externalCiId, ciArtifactReq, impl.checkExternalCiDeploymentAuth, token)
if err != nil {
impl.logger.Errorw("service err, HandleExternalCiWebhook", "err", err, "payload", req)
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/GitHostRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ type GitHostRestHandler interface {

type GitHostRestHandlerImpl struct {
logger *zap.SugaredLogger
gitHostConfig pipeline.GitHostConfig
userAuthService user.UserService
validator *validator.Validate
gitHostConfig pipeline.GitHostConfig
userAuthService user.UserService
validator *validator.Validate
enforcer casbin.Enforcer
gitSensorClient gitSensor.Client
gitProviderConfig pipeline.GitRegistryConfig
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/GitProviderRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ type GitProviderRestHandlerImpl struct {
dockerRegistryConfig pipeline.DockerRegistryConfig
logger *zap.SugaredLogger
gitRegistryConfig pipeline.GitRegistryConfig
dbConfigService pipeline.DbConfigService
userAuthService user.UserService
validator *validator.Validate
dbConfigService pipeline.DbConfigService
userAuthService user.UserService
validator *validator.Validate
enforcer casbin.Enforcer
teamService team.TeamService
deleteServiceFullMode delete2.DeleteServiceFullMode
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/GlobalCMCSRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ type GlobalCMCSRestHandler interface {
}

type GlobalCMCSRestHandlerImpl struct {
logger *zap.SugaredLogger
userAuthService user.UserService
validator *validator.Validate
logger *zap.SugaredLogger
userAuthService user.UserService
validator *validator.Validate
enforcer casbin.Enforcer
globalCMCSService pipeline.GlobalCMCSService
}
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/ImageScanRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ type ImageScanRestHandler interface {

type ImageScanRestHandlerImpl struct {
logger *zap.SugaredLogger
imageScanService security.ImageScanService
userService user.UserService
enforcer casbin.Enforcer
imageScanService security.ImageScanService
userService user.UserService
enforcer casbin.Enforcer
enforcerUtil rbac.EnforcerUtil
environmentService cluster.EnvironmentService
}
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/MigrateDbRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ type MigrateDbRestHandlerImpl struct {
dockerRegistryConfig pipeline.DockerRegistryConfig
logger *zap.SugaredLogger
gitRegistryConfig pipeline.GitRegistryConfig
dbConfigService pipeline.DbConfigService
userAuthService user.UserService
validator *validator.Validate
dbConfigService pipeline.DbConfigService
userAuthService user.UserService
validator *validator.Validate
dbMigrationService pipeline.DbMigrationService
enforcer casbin.Enforcer
}
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/NotificationRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ type NotificationRestHandlerImpl struct {
dockerRegistryConfig pipeline.DockerRegistryConfig
logger *zap.SugaredLogger
gitRegistryConfig pipeline.GitRegistryConfig
dbConfigService pipeline.DbConfigService
userAuthService user.UserService
validator *validator.Validate
dbConfigService pipeline.DbConfigService
userAuthService user.UserService
validator *validator.Validate
notificationService notifier.NotificationConfigService
slackService notifier.SlackNotificationService
webhookService notifier.WebhookNotificationService
Expand Down
8 changes: 4 additions & 4 deletions api/restHandler/PolicyRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ type PolicyRestHandler interface {
}
type PolicyRestHandlerImpl struct {
logger *zap.SugaredLogger
policyService security.PolicyService
userService user2.UserService
userAuthService user2.UserAuthService
enforcer casbin.Enforcer
policyService security.PolicyService
userService user2.UserService
userAuthService user2.UserAuthService
enforcer casbin.Enforcer
enforcerUtil rbac.EnforcerUtil
environmentService cluster.EnvironmentService
}
Expand Down
4 changes: 2 additions & 2 deletions api/restHandler/TelemetryRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ type TelemetryRestHandler interface {
type TelemetryRestHandlerImpl struct {
logger *zap.SugaredLogger
telemetryEventClient telemetry.TelemetryEventClient
enforcer casbin.Enforcer
userService user.UserService
enforcer casbin.Enforcer
userService user.UserService
}

type TelemetryGenericEvent struct {
Expand Down
6 changes: 3 additions & 3 deletions api/restHandler/TestSuitRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ type TestSuitRestHandler interface {
}

type TestSuitRestHandlerImpl struct {
logger *zap.SugaredLogger
userService user.UserService
validator *validator.Validate
logger *zap.SugaredLogger
userService user.UserService
validator *validator.Validate
enforcer casbin.Enforcer
enforcerUtil rbac.EnforcerUtil
config *client.EventClientConfig
Expand Down
39 changes: 32 additions & 7 deletions api/router/pubsub/CiEventHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean"
"github.com/devtron-labs/devtron/util"
"go.uber.org/zap"
"gopkg.in/go-playground/validator.v9"
"time"
)

Expand All @@ -53,6 +54,7 @@ type CiEventHandlerImpl struct {
logger *zap.SugaredLogger
pubsubClient *pubsub.PubSubClientServiceImpl
webhookService pipeline.WebhookService
validator *validator.Validate
ciEventConfig *CiEventConfig
}

Expand Down Expand Up @@ -80,11 +82,12 @@ type CiCompleteEvent struct {
PluginArtifactStage string `json:"pluginArtifactStage"`
}

func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, ciEventConfig *CiEventConfig) *CiEventHandlerImpl {
func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, validator *validator.Validate, ciEventConfig *CiEventConfig) *CiEventHandlerImpl {
ciEventHandlerImpl := &CiEventHandlerImpl{
logger: logger,
pubsubClient: pubsubClient,
webhookService: webhookService,
validator: validator,
ciEventConfig: ciEventConfig,
}
err := ciEventHandlerImpl.Subscribe()
Expand Down Expand Up @@ -133,10 +136,8 @@ func (impl *CiEventHandlerImpl) Subscribe() error {
impl.logger.Error("Error while creating request for pipelineID", "pipelineId", ciCompleteEvent.PipelineId, "err", err)
return
}
resp, err := impl.webhookService.HandleCiSuccessEvent(ciCompleteEvent.PipelineId, request, detail.ImagePushedAt)
resp, err := impl.ValidateAndHandleCiSuccessEvent(ciCompleteEvent.PipelineId, request, detail.ImagePushedAt)
if err != nil {
impl.logger.Error("Error while sending event for CI success for pipelineID", "pipelineId",
ciCompleteEvent.PipelineId, "request", request, "err", err)
return
}
impl.logger.Debug("response of handle ci success event for multiple images from plugin", "resp", resp)
Expand All @@ -145,10 +146,8 @@ func (impl *CiEventHandlerImpl) Subscribe() error {

} else {
util.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.ciEventConfig.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName)
resp, err := impl.webhookService.HandleCiSuccessEvent(ciCompleteEvent.PipelineId, req, &time.Time{})
resp, err := impl.ValidateAndHandleCiSuccessEvent(ciCompleteEvent.PipelineId, req, &time.Time{})
if err != nil {
impl.logger.Error("Error while sending event for CI success for pipelineID: ",
ciCompleteEvent.PipelineId, "request: ", req, "error: ", err)
return
}
impl.logger.Debug(resp)
Expand All @@ -162,6 +161,21 @@ func (impl *CiEventHandlerImpl) Subscribe() error {
return nil
}

func (impl *CiEventHandlerImpl) ValidateAndHandleCiSuccessEvent(ciPipelineId int, request *pipeline.CiArtifactWebhookRequest, imagePushedAt *time.Time) (int, error) {
validationErr := impl.validator.Struct(request)
if validationErr != nil {
impl.logger.Errorw("validation err, HandleCiSuccessEvent", "err", validationErr, "payload", request)
return 0, validationErr
}
buildArtifactId, err := impl.webhookService.HandleCiSuccessEvent(ciPipelineId, request, imagePushedAt)
if err != nil {
impl.logger.Error("Error while sending event for CI success for pipelineID",
ciPipelineId, "request", request, "error", err)
return 0, err
}
return buildArtifactId, nil
}

func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (*pipeline.CiArtifactWebhookRequest, error) {
var ciMaterialInfos []repository.CiMaterialInfo
for _, p := range event.CiProjectDetails {
Expand Down Expand Up @@ -228,6 +242,10 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (*
PluginRegistryArtifactDetails: event.PluginRegistryArtifactDetails,
PluginArtifactStage: event.PluginArtifactStage,
}
// if DataSource is empty, repository.WEBHOOK is considered as default
if request.DataSource == "" {
request.DataSource = repository.WEBHOOK
}
return request, nil
}

Expand All @@ -244,6 +262,9 @@ func (impl *CiEventHandlerImpl) BuildCIArtifactRequestForImageFromCR(imageDetail
WorkflowId: &workflowId,
IsArtifactUploaded: event.IsArtifactUploaded,
}
if request.DataSource == "" {
request.DataSource = repository.WEBHOOK
}
return request, nil
}

Expand Down Expand Up @@ -314,5 +335,9 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequestForWebhook(event CiComplet
WorkflowId: event.WorkflowId,
IsArtifactUploaded: event.IsArtifactUploaded,
}
// if DataSource is empty, repository.WEBHOOK is considered as default
if request.DataSource == "" {
request.DataSource = repository.WEBHOOK
}
return request, nil
}
6 changes: 3 additions & 3 deletions api/terminal/UserTerminalAccessRestHandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ type validShellResponse struct {
type UserTerminalAccessRestHandlerImpl struct {
Logger *zap.SugaredLogger
UserTerminalAccessService clusterTerminalAccess.UserTerminalAccessService
Enforcer casbin.Enforcer
UserService user.UserService
validator *validator.Validate
Enforcer casbin.Enforcer
UserService user.UserService
validator *validator.Validate
}

func NewUserTerminalAccessRestHandlerImpl(logger *zap.SugaredLogger, userTerminalAccessService clusterTerminalAccess.UserTerminalAccessService, Enforcer casbin.Enforcer,
Expand Down
6 changes: 3 additions & 3 deletions client/events/EventBuilder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ type EventSimpleFactoryImpl struct {
ciWorkflowRepository pipelineConfig.CiWorkflowRepository
ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository
ciPipelineRepository pipelineConfig.CiPipelineRepository
pipelineRepository pipelineConfig.PipelineRepository
userRepository repository.UserRepository
ciArtifactRepository repository2.CiArtifactRepository
pipelineRepository pipelineConfig.PipelineRepository
userRepository repository.UserRepository
ciArtifactRepository repository2.CiArtifactRepository
}

func NewEventSimpleFactoryImpl(logger *zap.SugaredLogger, cdWorkflowRepository pipelineConfig.CdWorkflowRepository,
Expand Down
2 changes: 1 addition & 1 deletion internal/sql/repository/AppListingRepository.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,7 @@ func parseMaterialInfo(materialInfo string, source string) (json.RawMessage, err
fmt.Printf("PARSEMATERIALINFO_MATERIAL_RECOVER, materialInfo: %s, source: %s, err: %s \n", materialInfo, source, r)
}
}()
if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" {
if source != GOCD && source != CI_RUNNER && source != WEBHOOK && source != EXT {
return nil, fmt.Errorf("datasource: %s not supported", source)
}
if materialInfo == "" {
Expand Down