diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 5274fdd8da6..969c891307b 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -1672,7 +1672,7 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, ParentCiPipeline: ciPipelineData.ParentCiPipeline, ParentAppId: ciPipelineData.ParentAppId, LinkedCount: ciPipelineData.LinkedCount, - PipelineType: bean.PipelineType(ciPipelineData.PipelineType), + PipelineType: bean2.PipelineType(ciPipelineData.PipelineType), }, } diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index c6a0f97f836..cc56379b8af 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -6,7 +6,7 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/pipeline" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/repository" repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" cron2 "github.com/devtron-labs/devtron/util/cron" @@ -84,7 +84,7 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { CiPipelineMaterial: ciPipelineMaterials, TriggeredBy: 1, InvalidateCache: false, - PipelineType: bean2.CI_JOB, + PipelineType: string(pipelineConfigBean.CI_JOB), } _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) if err != nil { diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index caf53ddb83f..4a805a94f2d 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -85,14 +85,18 @@ type CiArtifact struct { sql.AuditLog } -func (c *CiArtifact) IsMigrationRequired() bool { +func (artifact *CiArtifact) IsMigrationRequired() bool { validDataSourceTypeList := []string{CI_RUNNER, WEBHOOK, PRE_CD, POST_CD, POST_CI, GOCD} - if slices.Contains(validDataSourceTypeList, c.DataSource) { + if slices.Contains(validDataSourceTypeList, artifact.DataSource) { return false } return true } +func (artifact *CiArtifact) IsRegistryCredentialMapped() bool { + return artifact.CredentialsSourceType == GLOBAL_CONTAINER_REGISTRY +} + type CiArtifactRepository interface { Save(artifact *CiArtifact) error Delete(artifact *CiArtifact) error @@ -502,12 +506,12 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineAndRunnerType(cdPip } // return map of gitUrl:hash -func (info *CiArtifact) ParseMaterialInfo() (map[string]string, error) { - if info.DataSource != GOCD && info.DataSource != CI_RUNNER && info.DataSource != WEBHOOK && info.DataSource != EXT { - return nil, fmt.Errorf("datasource: %s not supported", info.DataSource) +func (artifact *CiArtifact) ParseMaterialInfo() (map[string]string, error) { + if artifact.DataSource != GOCD && artifact.DataSource != CI_RUNNER && artifact.DataSource != WEBHOOK && artifact.DataSource != EXT { + return nil, fmt.Errorf("datasource: %s not supported", artifact.DataSource) } var ciMaterials []*CiMaterialInfo - err := json.Unmarshal([]byte(info.MaterialInfo), &ciMaterials) + err := json.Unmarshal([]byte(artifact.MaterialInfo), &ciMaterials) scmMap := map[string]string{} for _, material := range ciMaterials { var url string diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index ff49491d7e0..3fd5eb48e30 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -105,6 +105,8 @@ type CiPipelineRepository interface { FindByIdIncludingInActive(id int) (pipeline *CiPipeline, err error) //find non deleted pipeline FindById(id int) (pipeline *CiPipeline, err error) + // FindOneWithAppData is to be used for fetching minimum data (including app.App) for CiPipeline for the given CiPipeline.Id + FindOneWithAppData(id int) (pipeline *CiPipeline, err error) FindCiEnvMappingByCiPipelineId(ciPipelineId int) (*CiEnvMapping, error) FindParentCiPipelineMapByAppId(appId int) ([]*CiPipeline, []int, error) FindByCiAndAppDetailsById(pipelineId int) (pipeline *CiPipeline, err error) @@ -325,6 +327,18 @@ func (impl CiPipelineRepositoryImpl) FindById(id int) (pipeline *CiPipeline, err return pipeline, err } + +// FindOneWithAppData is to be used for fetching minimum data (including app.App) for CiPipeline for the given CiPipeline.Id +func (impl CiPipelineRepositoryImpl) FindOneWithAppData(id int) (pipeline *CiPipeline, err error) { + pipeline = &CiPipeline{} + err = impl.dbConnection.Model(pipeline). + Column("ci_pipeline.*", "App"). + Where("ci_pipeline.id= ?", id). + Where("ci_pipeline.deleted =? ", false). + Select() + + return pipeline, err +} func (impl CiPipelineRepositoryImpl) FindCiEnvMappingByCiPipelineId(ciPipelineId int) (*CiEnvMapping, error) { ciEnvMapping := &CiEnvMapping{} err := impl.dbConnection.Model(ciEnvMapping). diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 5fde7ef79a4..c746344ff64 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -115,7 +115,7 @@ type CiPipeline struct { BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` LinkedCount int `json:"linkedCount"` - PipelineType PipelineType `json:"pipelineType,omitempty"` + PipelineType bean.PipelineType `json:"pipelineType,omitempty"` ScanEnabled bool `json:"scanEnabled,notnull"` AppWorkflowId int `json:"appWorkflowId,omitempty"` PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty"` @@ -138,14 +138,14 @@ type DockerConfigOverride struct { } type CiPipelineMin struct { - Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request - IsExternal bool `json:"isExternal,omitempty"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - PipelineType PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request + IsExternal bool `json:"isExternal,omitempty"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + PipelineType bean.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type CiScript struct { @@ -189,14 +189,6 @@ const ( //DEACTIVATE //pause/deactivate this pipeline ) -const ( - NORMAL PipelineType = "NORMAL" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - const ( CASCADE_DELETE int = iota NON_CASCADE_DELETE @@ -286,12 +278,12 @@ type CiPatchRequest struct { IsJob bool `json:"-"` IsCloneJob bool `json:"isCloneJob,omitempty"` - ParentCDPipeline int `json:"parentCDPipeline"` - DeployEnvId int `json:"deployEnvId"` - SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` - SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` - SwitchFromCiPipelineType PipelineType `json:"-"` - SwitchToCiPipelineType PipelineType `json:"-"` + ParentCDPipeline int `json:"parentCDPipeline"` + DeployEnvId int `json:"deployEnvId"` + SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` + SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` + SwitchFromCiPipelineType bean.PipelineType `json:"-"` + SwitchToCiPipelineType bean.PipelineType `json:"-"` } func (ciPatchRequest CiPatchRequest) IsSwitchCiPipelineRequest() bool { diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index 06e65a8fc83..aef5de5741f 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -88,7 +88,6 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) er } return fmt.Errorf("found vulnerability for image digest %s", cdWf.CiArtifact.ImageDigest) } - cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, triggeredBy) if err != nil { impl.logger.Errorw("error in building wfRequest", "err", err, "runner", runner, "cdWf", cdWf, "pipeline", pipeline) diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 09087406d17..f7e4aef6278 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -16,7 +16,8 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" "github.com/devtron-labs/devtron/pkg/pipeline" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -145,7 +146,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowService.SubmitWorkflow") cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env - cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE + cdStageWorkflowRequest.Type = pipelineConfigBean.CD_WORKFLOW_PIPELINE_TYPE _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) span.End() err = impl.sendPreStageNotification(ctx, cdWf, pipeline) @@ -174,9 +175,9 @@ func (impl *TriggerServiceImpl) SetCopyContainerImagePluginDataInWorkflowRequest if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { var pipelineStageEntityType int if pipelineStage == types.PRE { - pipelineStageEntityType = bean3.EntityTypePreCD + pipelineStageEntityType = pipelineConfigBean.EntityTypePreCD } else { - pipelineStageEntityType = bean3.EntityTypePostCD + pipelineStageEntityType = pipelineConfigBean.EntityTypePostCD } customTagId := -1 var DockerImageTag string @@ -231,7 +232,7 @@ func (impl *TriggerServiceImpl) SetCopyContainerImagePluginDataInWorkflowRequest } if len(savedCIArtifacts) > 0 { // if already present in ci artifact, return "image path already in use error" - return imagePathReservationIds, bean3.ErrImagePathInUse + return imagePathReservationIds, pipelineConfigBean.ErrImagePathInUse } imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, registryDestinationImageMap) if err != nil { @@ -253,6 +254,14 @@ func (impl *TriggerServiceImpl) SetCopyContainerImagePluginDataInWorkflowRequest } func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, triggeredBy int32) (*types.WorkflowRequest, error) { + if cdPipeline.App.Id == 0 { + appModel, err := impl.appRepository.FindById(cdPipeline.AppId) + if err != nil { + impl.logger.Errorw("error in getting app", "appId", cdPipeline.AppId, "err", err) + return nil, err + } + cdPipeline.App = *appModel + } cdWorkflowConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(cdPipeline.Id) if err != nil && !util.IsErrNoRows(err) { return nil, err @@ -277,7 +286,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return nil, err } - var ciProjectDetails []bean3.CiProjectDetails + var ciProjectDetails []pipelineConfigBean.CiProjectDetails var ciPipeline *pipelineConfig.CiPipeline if cdPipeline.CiPipelineId > 0 { ciPipeline, err = impl.ciPipelineRepository.FindById(cdPipeline.CiPipelineId) @@ -285,7 +294,9 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow impl.logger.Errorw("cannot find ciPipelineRequest", "err", err) return nil, err } - + if ciPipeline != nil && util.IsErrNoRows(err) { + ciPipeline.Id = 0 + } for _, m := range ciPipeline.CiPipelineMaterials { // git material should be active in this case if m == nil || m.GitMaterial == nil || !m.GitMaterial.Active { @@ -304,7 +315,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return nil, err } - ciProjectDetail := bean3.CiProjectDetails{ + ciProjectDetail := pipelineConfigBean.CiProjectDetails{ GitRepository: ciMaterialCurrent.Material.GitConfiguration.URL, MaterialName: gitMaterial.Name, CheckoutPath: gitMaterial.CheckoutPath, @@ -312,7 +323,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow SourceType: m.Type, SourceValue: m.Value, Type: string(m.Type), - GitOptions: bean3.GitOptions{ + GitOptions: pipelineConfigBean.GitOptions{ UserName: gitMaterial.GitProvider.UserName, Password: gitMaterial.GitProvider.Password, SshPrivateKey: gitMaterial.GitProvider.SshPrivateKey, @@ -331,7 +342,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return nil, err } ciProjectDetail.CommitTime = commitTime.Format(bean4.LayoutRFC3339) - } else if ciPipeline.PipelineType == bean3.CI_JOB { + } else if ciPipeline.PipelineType == string(pipelineConfigBean.CI_JOB) { // This has been done to resolve unmarshalling issue in ci-runner, in case of no commit time(eg- polling container images) ciProjectDetail.CommitTime = time.Time{}.Format(bean4.LayoutRFC3339) } else { @@ -356,9 +367,9 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow var deployStageWfr pipelineConfig.CdWorkflowRunner var deployStageTriggeredByUserEmail string var pipelineReleaseCounter int - var preDeploySteps []*bean3.StepObject - var postDeploySteps []*bean3.StepObject - var refPluginsData []*bean3.RefPluginObject + var preDeploySteps []*pipelineConfigBean.StepObject + var postDeploySteps []*pipelineConfigBean.StepObject + var refPluginsData []*pipelineConfigBean.RefPluginObject //if pipeline_stage_steps present for pre-CD or post-CD then no need to add stageYaml to cdWorkflowRequest in that //case add PreDeploySteps and PostDeploySteps to cdWorkflowRequest, this is done for backward compatibility pipelineStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(cdPipeline.Id, runner.WorkflowType.WorkflowTypeToStageType()) @@ -375,7 +386,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow //Scope will pick the environment of CD pipeline irrespective of in-cluster mode, //since user sees the environment of the CD pipeline scope := resourceQualifiers.Scope{ - AppId: cdPipeline.App.Id, + AppId: cdPipeline.AppId, EnvId: env.Id, ClusterId: env.ClusterId, SystemMetadata: &resourceQualifiers.SystemMetadata{ @@ -576,36 +587,31 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow extraEnvVariables[CHILD_CD_COUNT] = strconv.Itoa(len(childPipelines)) } + if ciPipeline != nil && ciPipeline.Id > 0 { - extraEnvVariables["APP_NAME"] = ciPipeline.App.AppName - cdStageWorkflowRequest.DockerUsername = ciPipeline.CiTemplate.DockerRegistry.Username - cdStageWorkflowRequest.DockerPassword = ciPipeline.CiTemplate.DockerRegistry.Password - cdStageWorkflowRequest.AwsRegion = ciPipeline.CiTemplate.DockerRegistry.AWSRegion - cdStageWorkflowRequest.DockerConnection = ciPipeline.CiTemplate.DockerRegistry.Connection - cdStageWorkflowRequest.DockerCert = ciPipeline.CiTemplate.DockerRegistry.Cert - cdStageWorkflowRequest.AccessKey = ciPipeline.CiTemplate.DockerRegistry.AWSAccessKeyId - cdStageWorkflowRequest.SecretKey = ciPipeline.CiTemplate.DockerRegistry.AWSSecretAccessKey - cdStageWorkflowRequest.DockerRegistryType = string(ciPipeline.CiTemplate.DockerRegistry.RegistryType) - cdStageWorkflowRequest.DockerRegistryURL = ciPipeline.CiTemplate.DockerRegistry.RegistryURL - cdStageWorkflowRequest.DockerRegistryId = ciPipeline.CiTemplate.DockerRegistry.Id - cdStageWorkflowRequest.CiPipelineType = ciPipeline.PipelineType - } else if cdPipeline.AppId > 0 { - ciTemplate, err := impl.ciTemplateRepository.FindByAppId(cdPipeline.AppId) + sourceCiPipeline, err := impl.getSourceCiPipelineForArtifact(*ciPipeline) if err != nil { + impl.logger.Errorw("error in getting source ciPipeline for artifact", "err", err) return nil, err } - extraEnvVariables["APP_NAME"] = ciTemplate.App.AppName - cdStageWorkflowRequest.DockerUsername = ciTemplate.DockerRegistry.Username - cdStageWorkflowRequest.DockerPassword = ciTemplate.DockerRegistry.Password - cdStageWorkflowRequest.AwsRegion = ciTemplate.DockerRegistry.AWSRegion - cdStageWorkflowRequest.DockerConnection = ciTemplate.DockerRegistry.Connection - cdStageWorkflowRequest.DockerCert = ciTemplate.DockerRegistry.Cert - cdStageWorkflowRequest.AccessKey = ciTemplate.DockerRegistry.AWSAccessKeyId - cdStageWorkflowRequest.SecretKey = ciTemplate.DockerRegistry.AWSSecretAccessKey - cdStageWorkflowRequest.DockerRegistryType = string(ciTemplate.DockerRegistry.RegistryType) - cdStageWorkflowRequest.DockerRegistryURL = ciTemplate.DockerRegistry.RegistryURL + extraEnvVariables["APP_NAME"] = sourceCiPipeline.App.AppName + cdStageWorkflowRequest.CiPipelineType = sourceCiPipeline.PipelineType + buildRegistryConfig, dbErr := impl.getBuildRegistryConfigForArtifact(*sourceCiPipeline, *artifact) + if dbErr != nil { + impl.logger.Errorw("error in getting registry credentials for the artifact", "err", dbErr) + return nil, dbErr + } + adapter.UpdateRegistryDetailsToWrfReq(cdStageWorkflowRequest, buildRegistryConfig) + } else if cdPipeline.AppId > 0 { + // the below flow is used for external ci base pipelines; + extraEnvVariables["APP_NAME"] = cdPipeline.App.AppName + buildRegistryConfig, err := impl.ciTemplateService.GetBaseDockerConfigForCiPipeline(cdPipeline.AppId) + if err != nil { + impl.logger.Errorw("error in getting build configurations", "err", err) + return nil, fmt.Errorf("error in getting build configurations") + } + adapter.UpdateRegistryDetailsToWrfReq(cdStageWorkflowRequest, buildRegistryConfig) appLabels, err := impl.appLabelRepository.FindAllByAppId(cdPipeline.AppId) - cdStageWorkflowRequest.DockerRegistryId = ciTemplate.DockerRegistry.Id if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting labels by appId", "err", err, "appId", cdPipeline.AppId) return nil, err @@ -707,6 +713,102 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return cdStageWorkflowRequest, nil } +/* +getBuildRegistryConfigForArtifact performs the following logic to get Pre/Post CD Registry Credentials: + + 1. CI Build: + It will use the overridden credentials (if any) OR the base application level credentials. + + 2. Link CI: + It will fetch the parent CI pipeline first. + Then will use the CI Build overridden credentials (if any) OR the Source application (App that contains CI Build) level credentials. + + 3. Sync CD: + It will fetch the parent CD pipeline first. + + - CASE CD Pipeline has CI Build as artifact provider: + + Then will use the CI Build overridden credentials (if any) OR the Source application (App that contains CI Build) level credentials. + + - CASE CD Pipeline has Link CI as artifact provider: + + It will fetch the parent CI pipeline of the Link CI first. + Then will use the CI Build overridden credentials (if any) OR the Source application (App that contains CI Build) level credentials. + + 4. Skopeo Plugin: + If any artifact has information about : credentials_source_type(global_container_registry) credentials_source_value(registry_id) + Then we will use the credentials_source_value to derive the credentials. + + 5. Polling plugin: + If the ci_pipeline_type type is CI_JOB + We will always fetch the registry credentials from the ci_template_override table +*/ +func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline pipelineConfig.CiPipeline, artifact repository.CiArtifact) (*types.DockerArtifactStoreBean, error) { + // Handling for Skopeo Plugin + if artifact.IsRegistryCredentialMapped() { + dockerArtifactStore, err := impl.dockerArtifactStoreRepository.FindOne(artifact.CredentialSourceValue) + if util.IsErrNoRows(err) { + impl.logger.Errorw("source artifact registry not found", "registryId", artifact.CredentialSourceValue, "err", err) + return nil, fmt.Errorf("source artifact registry '%s' not found", artifact.CredentialSourceValue) + } else if err != nil { + impl.logger.Errorw("error in fetching artifact info", "err", err) + return nil, err + } + return adapter.GetDockerConfigBean(dockerArtifactStore), nil + } + + // Handling for CI Job + if adapter.IsCIJob(sourceCiPipeline) { + // for bean.CI_JOB the source artifact is always driven from overridden ci template + buildRegistryConfig, err := impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(sourceCiPipeline.Id, sourceCiPipeline.AppId, true) + if err != nil { + impl.logger.Errorw("error in getting build configurations", "err", err) + return nil, fmt.Errorf("error in getting build configurations") + } + return buildRegistryConfig, nil + } + + // Handling for Linked CI + if adapter.IsLinkedCI(sourceCiPipeline) { + parentCiPipeline, err := impl.ciPipelineRepository.FindById(sourceCiPipeline.ParentCiPipeline) + if err != nil { + impl.logger.Errorw("error in finding ciPipeline", "ciPipelineId", sourceCiPipeline.ParentCiPipeline, "err", err) + return nil, err + } + buildRegistryConfig, err := impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(parentCiPipeline.Id, parentCiPipeline.AppId, parentCiPipeline.IsDockerConfigOverridden) + if err != nil { + impl.logger.Errorw("error in getting build configurations", "err", err) + return nil, fmt.Errorf("error in getting build configurations") + } + return buildRegistryConfig, nil + } + + // Handling for Build CI + buildRegistryConfig, err := impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(sourceCiPipeline.Id, sourceCiPipeline.AppId, sourceCiPipeline.IsDockerConfigOverridden) + if err != nil { + impl.logger.Errorw("error in getting build configurations", "err", err) + return nil, fmt.Errorf("error in getting build configurations") + } + return buildRegistryConfig, nil +} + +func (impl *TriggerServiceImpl) getSourceCiPipelineForArtifact(ciPipeline pipelineConfig.CiPipeline) (*pipelineConfig.CiPipeline, error) { + sourceCiPipeline := &ciPipeline + if adapter.IsLinkedCD(ciPipeline) { + sourceCdPipeline, err := impl.pipelineRepository.FindById(ciPipeline.ParentCiPipeline) + if err != nil { + impl.logger.Errorw("error in finding source cdPipeline for linked cd", "cdPipelineId", ciPipeline.ParentCiPipeline, "err", err) + return nil, err + } + sourceCiPipeline, err = impl.ciPipelineRepository.FindOneWithAppData(sourceCdPipeline.CiPipelineId) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in finding ciPipeline for the cd pipeline", "CiPipelineId", sourceCdPipeline.Id, "CiPipelineId", sourceCdPipeline.CiPipelineId, "err", err) + return nil, err + } + } + return sourceCiPipeline, nil +} + func (impl *TriggerServiceImpl) GetArtifactVulnerabilityStatus(artifact *repository.CiArtifact, cdPipeline *pipelineConfig.Pipeline, ctx context.Context) (bool, error) { isVulnerable := false if len(artifact.ImageDigest) > 0 { @@ -760,15 +862,15 @@ func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, return imagePathReservationIds, nil } -func setExtraEnvVariableInDeployStep(deploySteps []*bean3.StepObject, extraEnvVariables map[string]string, webhookAndCiData *gitSensorClient.WebhookAndCiData) { +func setExtraEnvVariableInDeployStep(deploySteps []*pipelineConfigBean.StepObject, extraEnvVariables map[string]string, webhookAndCiData *gitSensorClient.WebhookAndCiData) { for _, deployStep := range deploySteps { for variableKey, variableValue := range extraEnvVariables { if isExtraVariableDynamic(variableKey, webhookAndCiData) && deployStep.StepType == "INLINE" { - extraInputVar := &bean3.VariableObject{ + extraInputVar := &pipelineConfigBean.VariableObject{ Name: variableKey, Format: "STRING", Value: variableValue, - VariableType: bean3.VARIABLE_TYPE_REF_GLOBAL, + VariableType: pipelineConfigBean.VARIABLE_TYPE_REF_GLOBAL, ReferenceVariableName: variableKey, } deployStep.InputVars = append(deployStep.InputVars, extraInputVar) diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 2ca45a2e07b..e05e6046d43 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -17,8 +17,10 @@ import ( "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + repository4 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/security" "github.com/devtron-labs/devtron/internal/util" @@ -106,8 +108,8 @@ type TriggerServiceImpl struct { userService user.UserService gitSensorGrpcClient gitSensorClient.Client config *types.CdConfig - - helmAppService client2.HelmAppService + appRepository appRepository.AppRepository + helmAppService client2.HelmAppService enforcerUtil rbac.EnforcerUtil helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead @@ -125,11 +127,12 @@ type TriggerServiceImpl struct { cdWorkflowRepository pipelineConfig.CdWorkflowRepository ciWorkflowRepository pipelineConfig.CiWorkflowRepository ciArtifactRepository repository3.CiArtifactRepository - ciTemplateRepository pipelineConfig.CiTemplateRepository + ciTemplateService pipeline.CiTemplateService materialRepository pipelineConfig.MaterialRepository appLabelRepository pipelineConfig.AppLabelRepository ciPipelineRepository pipelineConfig.CiPipelineRepository appWorkflowRepository appWorkflow.AppWorkflowRepository + dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository } func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd.CdWorkflowCommonService, @@ -172,11 +175,12 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, ciArtifactRepository repository3.CiArtifactRepository, - ciTemplateRepository pipelineConfig.CiTemplateRepository, + ciTemplateService pipeline.CiTemplateService, materialRepository pipelineConfig.MaterialRepository, appLabelRepository pipelineConfig.AppLabelRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, - appWorkflowRepository appWorkflow.AppWorkflowRepository) (*TriggerServiceImpl, error) { + appWorkflowRepository appWorkflow.AppWorkflowRepository, + dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, cdWorkflowCommonService: cdWorkflowCommonService, @@ -219,11 +223,12 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd cdWorkflowRepository: cdWorkflowRepository, ciWorkflowRepository: ciWorkflowRepository, ciArtifactRepository: ciArtifactRepository, - ciTemplateRepository: ciTemplateRepository, + ciTemplateService: ciTemplateService, materialRepository: materialRepository, appLabelRepository: appLabelRepository, ciPipelineRepository: ciPipelineRepository, appWorkflowRepository: appWorkflowRepository, + dockerArtifactStoreRepository: dockerArtifactStoreRepository, } config, err := types.GetCdConfig() if err != nil { diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index def3c1f8aac..8303fc1db6c 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -30,7 +30,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/bean" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" @@ -504,7 +504,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea } } //map of ciPipelineId and their templateOverrideConfig - ciOverrideTemplateMap := make(map[int]*bean3.CiTemplateBean) + ciOverrideTemplateMap := make(map[int]*pipelineConfigBean.CiTemplateBean) ciTemplateBeanOverrides, err := impl.ciTemplateService.FindTemplateOverrideByAppId(appId) if err != nil { return nil, err @@ -570,14 +570,14 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean.PipelineType(pipeline.PipelineType), + PipelineType: pipelineConfigBean.PipelineType(pipeline.PipelineType), } ciEnvMapping, err := impl.ciPipelineRepository.FindCiEnvMappingByCiPipelineId(pipeline.Id) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching ciEnvMapping", "ciPipelineId ", pipeline.Id, "err", err) return nil, err } - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean3.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) if err != nil && err != pg.ErrNoRows { return nil, err } @@ -714,9 +714,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean.PipelineType(pipeline.PipelineType), + PipelineType: pipelineConfigBean.PipelineType(pipeline.PipelineType), } - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean3.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) if err != nil && err != pg.ErrNoRows { return nil, err } @@ -813,7 +813,7 @@ func (impl *CiPipelineConfigServiceImpl) GetTriggerViewCiPipeline(appId int) (*b return nil, err } - ciOverrideTemplateMap := make(map[int]*bean3.CiTemplateBean) + ciOverrideTemplateMap := make(map[int]*pipelineConfigBean.CiTemplateBean) ciTemplateBeanOverrides, err := impl.ciTemplateService.FindTemplateOverrideByAppId(appId) if err != nil { return nil, err @@ -838,7 +838,7 @@ func (impl *CiPipelineConfigServiceImpl) GetTriggerViewCiPipeline(appId int) (*b ParentCiPipeline: pipeline.ParentCiPipeline, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean.PipelineType(pipeline.PipelineType), + PipelineType: pipelineConfigBean.PipelineType(pipeline.PipelineType), } if ciTemplateBean, ok := ciOverrideTemplateMap[pipeline.Id]; ok { templateOverride := ciTemplateBean.CiTemplateOverride @@ -1171,7 +1171,7 @@ func (impl *CiPipelineConfigServiceImpl) UpdateCiTemplate(updateRequest *bean.Ci } ciBuildConfig.Id = originalCiBuildConfig.Id - ciTemplateBean := &bean3.CiTemplateBean{ + ciTemplateBean := &pipelineConfigBean.CiTemplateBean{ CiTemplate: ciTemplate, CiBuildConfig: ciBuildConfig, UserId: updateRequest.UserId, @@ -1204,7 +1204,7 @@ func (impl *CiPipelineConfigServiceImpl) UpdateCiTemplate(updateRequest *bean.Ci } for _, ciTemplateOverride := range ciTemplateOverrides { if _, ok := ciPipelineIdsMap[ciTemplateOverride.CiPipelineId]; ok { - if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(bean.CI_JOB) { + if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(pipelineConfigBean.CI_JOB) { ciTemplateOverride.DockerRepository = updateRequest.DockerRepository ciTemplateOverride.DockerRegistryId = updateRequest.DockerRegistry _, err = impl.ciTemplateOverrideRepository.Update(ciTemplateOverride) @@ -1236,7 +1236,7 @@ func (impl *CiPipelineConfigServiceImpl) handlePipelineCreate(request *bean.CiPa if pipelineExists { err = &utils.ApiError{Code: "400", HttpStatusCode: 400, UserMessage: "pipeline name already exist"} impl.logger.Errorw("pipeline name already exist", "err", err, "patch cipipeline name", request.CiPipeline.Name) - return nil, fmt.Errorf(bean3.PIPELINE_NAME_ALREADY_EXISTS_ERROR) + return nil, fmt.Errorf(pipelineConfigBean.PIPELINE_NAME_ALREADY_EXISTS_ERROR) } if request.IsSwitchCiPipelineRequest() { @@ -1260,17 +1260,17 @@ func (impl *CiPipelineConfigServiceImpl) PatchCiPipeline(request *bean.CiPatchRe impl.logger.Errorw("err in fetching template for pipeline patch, ", "err", err, "appId", request.AppId) return nil, err } - if request.CiPipeline.PipelineType == bean.CI_JOB { + if request.CiPipeline.PipelineType == pipelineConfigBean.CI_JOB { request.CiPipeline.IsDockerConfigOverridden = true request.CiPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: ciConfig.DockerRegistry, DockerRepository: ciConfig.DockerRepository, - CiBuildConfig: &bean3.CiBuildConfigBean{ + CiBuildConfig: &pipelineConfigBean.CiBuildConfigBean{ Id: 0, GitMaterialId: request.CiPipeline.CiMaterial[0].GitMaterialId, BuildContextGitMaterialId: request.CiPipeline.CiMaterial[0].GitMaterialId, UseRootBuildContext: false, - CiBuildType: bean3.SKIP_BUILD_TYPE, + CiBuildType: pipelineConfigBean.SKIP_BUILD_TYPE, DockerBuildConfig: nil, BuildPackConfig: nil, }, @@ -1390,7 +1390,7 @@ func (impl *CiPipelineConfigServiceImpl) CreateCiPipeline(createRequest *bean.Ci ciTemplate.DockerRepository = createRequest.DockerRepository } - ciTemplateBean := &bean3.CiTemplateBean{ + ciTemplateBean := &pipelineConfigBean.CiTemplateBean{ CiTemplate: ciTemplate, CiBuildConfig: createRequest.CiBuildConfig, } @@ -1453,15 +1453,15 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineMin(appId int, envIds []in var ciPipelineResp []*bean.CiPipelineMin for _, pipeline := range pipelines { parentCiPipeline := pipelineConfig.CiPipeline{} - pipelineType := bean.NORMAL + pipelineType := pipelineConfigBean.NORMAL if pipelineParentCiMap[pipeline.Id] != nil { parentCiPipeline = *pipelineParentCiMap[pipeline.Id] - pipelineType = bean.LINKED + pipelineType = pipelineConfigBean.LINKED } else if pipeline.IsExternal == true { - pipelineType = bean.EXTERNAL - } else if pipeline.PipelineType == string(bean.CI_JOB) { - pipelineType = bean.CI_JOB + pipelineType = pipelineConfigBean.EXTERNAL + } else if pipeline.PipelineType == string(pipelineConfigBean.CI_JOB) { + pipelineType = pipelineConfigBean.CI_JOB } ciPipeline := &bean.CiPipelineMin{ @@ -1644,7 +1644,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineByEnvironment(request reso impl.logger.Errorw("error in fetching templates override", "appIds", appIds, "err", err) return nil, err } - ciOverrideTemplateMap := make(map[int]*bean3.CiTemplateBean) + ciOverrideTemplateMap := make(map[int]*pipelineConfigBean.CiTemplateBean) for _, templateBeanOverride := range ciTemplateBeanOverrides { ciTemplateOverride := templateBeanOverride.CiTemplateOverride ciOverrideTemplateMap[ciTemplateOverride.CiPipelineId] = templateBeanOverride @@ -1678,7 +1678,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineByEnvironment(request reso ExternalCiConfig: externalCiConfig, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean.PipelineType(pipeline.PipelineType), + PipelineType: pipelineConfigBean.PipelineType(pipeline.PipelineType), } parentPipelineAppId, ok := pipelineIdVsAppId[parentCiPipelineId] if ok { diff --git a/pkg/pipeline/BuildPipelineSwitchService.go b/pkg/pipeline/BuildPipelineSwitchService.go index 1eb0f3d3fd8..e325403e7b7 100644 --- a/pkg/pipeline/BuildPipelineSwitchService.go +++ b/pkg/pipeline/BuildPipelineSwitchService.go @@ -4,7 +4,8 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/bean" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/sql" @@ -96,10 +97,10 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req } //get the ciPipeline - var switchFromType bean.PipelineType + var switchFromType pipelineConfigBean.PipelineType var switchFromPipelineId int if request.SwitchFromExternalCiPipelineId != 0 { - switchFromType = bean.EXTERNAL + switchFromType = pipelineConfigBean.EXTERNAL switchFromPipelineId = request.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = request.SwitchFromCiPipelineId @@ -117,7 +118,7 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req return impl.createNewPipelineAndReplaceOldPipelineLinks(request.CiPipeline, ciConfig, switchFromPipelineId, switchFromType, request.UserId) } -func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType bean.PipelineType, userId int32) (*bean.CiConfigRequest, error) { +func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*bean.CiConfigRequest, error) { tx, err := impl.ciPipelineRepository.StartTx() if err != nil { impl.logger.Errorw("error in starting transaction", "switchFromPipelineId", switchFromPipelineId, "switchFromType", switchFromType, "userId", userId, "err", err) @@ -145,7 +146,7 @@ func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipeli } //we don't store ci-pipeline-id in pipeline table for external ci's - if switchFromPipelineId > 0 && switchFromType != bean.EXTERNAL { + if switchFromPipelineId > 0 && switchFromType != pipelineConfigBean.EXTERNAL { // ciPipeline id is being set in res object in the addpipelineToTemplate method. err = impl.pipelineRepository.UpdateOldCiPipelineIdToNewCiPipelineId(tx, switchFromPipelineId, res.CiPipelines[0].Id) if err != nil { @@ -164,7 +165,7 @@ func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipeli // add switchType and remove other id // make constants for error msgs -func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType bean.PipelineType) error { +func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType pipelineConfigBean.PipelineType) error { // this will only allow below conversions // ext -> {ci_job,direct,linked} // direct -> {ci_job,linked} @@ -176,13 +177,13 @@ func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromC } // refer SwitchToExternalCi - if switchToType == bean.EXTERNAL { + if switchToType == pipelineConfigBean.EXTERNAL { return errors.New(string(cannotConvertToExternalCi)) } // we should not check the below logic for external_ci type as builds are not built in devtron and // linked pipelines won't be there as per current external-ci-pipeline architecture - if switchFromCiPipelineId > 0 && switchFromType != bean.EXTERNAL { + if switchFromCiPipelineId > 0 && switchFromType != pipelineConfigBean.EXTERNAL { // old ci_pipeline should not contain any linked ci_pipelines. linkedCiPipelines, err := impl.ciPipelineRepository.FindLinkedCiCount(switchFromCiPipelineId) if err != nil { @@ -228,13 +229,13 @@ func (impl *BuildPipelineSwitchServiceImpl) deleteCiAndItsWorkflowMappings(tx *p return err } -func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType bean.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { +func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { // 1) delete build pipelines // 2) delete app workflowMappings var err error pipelineId := switchFromPipelineId pipelineType := "" - if switchFromType == bean.EXTERNAL { + if switchFromType == pipelineConfigBean.EXTERNAL { err = impl.deleteExternalCi(tx, switchFromPipelineId, userId) pipelineType = appWorkflow.WEBHOOK } else { @@ -315,7 +316,7 @@ func (impl *BuildPipelineSwitchServiceImpl) saveHistoryOfOverriddenTemplate(ciPi impl.logger.Errorw("error in getting ciTemplate ", "err", err, "ciTemplate", ciTemplate.Id) return err } - buildConfig, err := bean3.ConvertDbBuildConfigToBean(ciTemplate.CiBuildConfig) + buildConfig, err := adapter.ConvertDbBuildConfigToBean(ciTemplate.CiBuildConfig) if err != nil { impl.logger.Errorw("error in ConvertDbBuildConfigToBean ", "err", err, "buildConfigId", buildConfig.Id) return err diff --git a/pkg/pipeline/CiBuildConfigService.go b/pkg/pipeline/CiBuildConfigService.go index 4d7862d69e5..f09ff7e98ca 100644 --- a/pkg/pipeline/CiBuildConfigService.go +++ b/pkg/pipeline/CiBuildConfigService.go @@ -3,6 +3,7 @@ package pipeline import ( "errors" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "go.uber.org/zap" "time" @@ -28,7 +29,7 @@ func NewCiBuildConfigServiceImpl(logger *zap.SugaredLogger, ciBuildConfigReposit } func (impl *CiBuildConfigServiceImpl) Save(templateId int, overrideTemplateId int, ciBuildConfigBean *bean.CiBuildConfigBean, userId int32) error { - ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfigBean, userId) + ciBuildConfigEntity, err := adapter.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfigBean, userId) if err != nil { impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, "overrideTemplateId", overrideTemplateId, "ciBuildConfigBean", ciBuildConfigBean, "err", err) @@ -50,7 +51,7 @@ func (impl *CiBuildConfigServiceImpl) UpdateOrSave(templateId int, overrideTempl impl.Logger.Warnw("not updating build config as object is empty", "ciBuildConfig", ciBuildConfig) return nil, nil } - ciBuildConfigEntity, err := bean.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfig, userId) + ciBuildConfigEntity, err := adapter.ConvertBuildConfigBeanToDbEntity(templateId, overrideTemplateId, ciBuildConfig, userId) if err != nil { impl.Logger.Errorw("error occurred while converting build config to db entity", "templateId", templateId, "overrideTemplateId", overrideTemplateId, "ciBuildConfig", ciBuildConfig, "err", err) diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 710dbe05628..2cac226fa9f 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -44,7 +44,7 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/genericNotes" repository3 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" history3 "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" @@ -80,7 +80,7 @@ type CiCdPipelineOrchestrator interface { PatchMaterialValue(createRequest *bean.CiPipeline, userId int32, oldPipeline *pipelineConfig.CiPipeline) (*bean.CiPipeline, error) PatchCiMaterialSource(ciPipeline *bean.CiMaterialPatchRequest, userId int32) (*bean.CiMaterialPatchRequest, error) PatchCiMaterialSourceValue(patchRequest *bean.CiMaterialValuePatchRequest, userId int32, value string, token string, checkAppSpecificAccess func(token, action string, appId int) (bool, error)) (*pipelineConfig.CiPipelineMaterial, error) - CreateCiTemplateBean(ciPipelineId int, dockerRegistryId string, dockerRepository string, gitMaterialId int, ciBuildConfig *bean2.CiBuildConfigBean, userId int32) bean2.CiTemplateBean + CreateCiTemplateBean(ciPipelineId int, dockerRegistryId string, dockerRepository string, gitMaterialId int, ciBuildConfig *pipelineConfigBean.CiBuildConfigBean, userId int32) pipelineConfigBean.CiTemplateBean UpdateCiPipelineMaterials(materialsUpdate []*pipelineConfig.CiPipelineMaterial) error PipelineExists(name string) (bool, error) GetCdPipelinesForApp(appId int) (cdPipelines *bean.CdPipelines, err error) @@ -283,7 +283,7 @@ func (impl CiCdPipelineOrchestratorImpl) validateCiPipelineMaterial(ciPipelineMa func (impl CiCdPipelineOrchestratorImpl) getSkipMessage(ciPipeline *pipelineConfig.CiPipeline) string { switch ciPipeline.PipelineType { - case string(bean.LINKED_CD): + case string(pipelineConfigBean.LINKED_CD): return "“Sync with Environment”" default: return "“Linked Build Pipeline”" @@ -368,7 +368,7 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. //Otherwise deleteIfExists if createRequest.CustomTagObject != nil && len(createRequest.CustomTagObject.TagPattern) > 0 { customTag := bean4.CustomTag{ - EntityKey: bean2.EntityTypeCiPipelineId, + EntityKey: pipelineConfigBean.EntityTypeCiPipelineId, EntityValue: strconv.Itoa(ciPipelineObject.Id), TagPattern: createRequest.CustomTagObject.TagPattern, AutoIncreasingNumber: createRequest.CustomTagObject.CounterX, @@ -380,7 +380,7 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. } } else { customTag := bean4.CustomTag{ - EntityKey: bean2.EntityTypeCiPipelineId, + EntityKey: pipelineConfigBean.EntityTypeCiPipelineId, EntityValue: strconv.Itoa(ciPipelineObject.Id), Enabled: false, } @@ -419,7 +419,7 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. return nil, err } if createRequest.EnvironmentId != 0 { - createJobEnvOverrideRequest := &bean2.CreateJobEnvOverridePayload{ + createJobEnvOverrideRequest := &pipelineConfigBean.CreateJobEnvOverridePayload{ AppId: createRequest.AppId, EnvId: createRequest.EnvironmentId, UserId: userId, @@ -579,7 +579,7 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. templateOverrideReq.Id = savedTemplateOverride.Id templateOverrideReq.CreatedOn = savedTemplateOverride.CreatedOn templateOverrideReq.CreatedBy = savedTemplateOverride.CreatedBy - ciTemplateBean := &bean2.CiTemplateBean{ + ciTemplateBean := &pipelineConfigBean.CiTemplateBean{ CiTemplateOverride: templateOverrideReq, CiBuildConfig: ciBuildConfigBean, UserId: userId, @@ -596,7 +596,7 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. } } else { - ciTemplateBean := &bean2.CiTemplateBean{ + ciTemplateBean := &pipelineConfigBean.CiTemplateBean{ CiTemplateOverride: templateOverrideReq, CiBuildConfig: ciBuildConfigBean, UserId: userId, @@ -614,7 +614,7 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. } } else { - ciTemplateBean := &bean2.CiTemplateBean{ + ciTemplateBean := &pipelineConfigBean.CiTemplateBean{ CiTemplateOverride: &pipelineConfig.CiTemplateOverride{}, CiBuildConfig: nil, UserId: userId, @@ -737,8 +737,8 @@ func (impl CiCdPipelineOrchestratorImpl) DeleteCiPipelineAndCiEnvMappings(tx *pg return err } -func (impl CiCdPipelineOrchestratorImpl) CreateCiTemplateBean(ciPipelineId int, dockerRegistryId string, dockerRepository string, gitMaterialId int, ciBuildConfig *bean2.CiBuildConfigBean, userId int32) bean2.CiTemplateBean { - CiTemplateBean := bean2.CiTemplateBean{ +func (impl CiCdPipelineOrchestratorImpl) CreateCiTemplateBean(ciPipelineId int, dockerRegistryId string, dockerRepository string, gitMaterialId int, ciBuildConfig *pipelineConfigBean.CiBuildConfigBean, userId int32) pipelineConfigBean.CiTemplateBean { + CiTemplateBean := pipelineConfigBean.CiTemplateBean{ CiTemplate: nil, CiTemplateOverride: &pipelineConfig.CiTemplateOverride{ CiPipelineId: ciPipelineId, @@ -761,10 +761,10 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiTemplateBean(ciPipelineId int, } func (impl CiCdPipelineOrchestratorImpl) SaveHistoryOfBaseTemplate(userId int32, pipeline *pipelineConfig.CiPipeline, materials []*pipelineConfig.CiPipelineMaterial) error { - CiTemplateBean := bean2.CiTemplateBean{ + CiTemplateBean := pipelineConfigBean.CiTemplateBean{ CiTemplate: nil, CiTemplateOverride: &pipelineConfig.CiTemplateOverride{}, - CiBuildConfig: &bean2.CiBuildConfigBean{}, + CiBuildConfig: &pipelineConfigBean.CiBuildConfigBean{}, UserId: userId, } err := impl.ciPipelineHistoryService.SaveHistory(pipeline, materials, &CiTemplateBean, repository4.TRIGGER_DELETE) @@ -835,7 +835,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf //If customTagObejct has been passed, save it if !ciPipeline.EnableCustomTag { err := impl.customTagService.DisableCustomTagIfExist(bean4.CustomTag{ - EntityKey: bean2.EntityTypeCiPipelineId, + EntityKey: pipelineConfigBean.EntityTypeCiPipelineId, EntityValue: strconv.Itoa(ciPipeline.Id), }) if err != nil { @@ -843,7 +843,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf } } else if ciPipeline.CustomTagObject != nil && len(ciPipeline.CustomTagObject.TagPattern) != 0 { customTag := &bean4.CustomTag{ - EntityKey: bean2.EntityTypeCiPipelineId, + EntityKey: pipelineConfigBean.EntityTypeCiPipelineId, EntityValue: strconv.Itoa(ciPipeline.Id), TagPattern: ciPipeline.CustomTagObject.TagPattern, AutoIncreasingNumber: ciPipeline.CustomTagObject.CounterX, @@ -868,7 +868,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf } if ciPipeline.EnvironmentId != 0 && !createRequest.IsCloneJob { - createJobEnvOverrideRequest := &bean2.CreateJobEnvOverridePayload{ + createJobEnvOverrideRequest := &pipelineConfigBean.CreateJobEnvOverridePayload{ AppId: createRequest.AppId, EnvId: ciPipeline.EnvironmentId, UserId: createRequest.UserId, @@ -968,7 +968,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf } } - ciTemplateBean := &bean2.CiTemplateBean{} + ciTemplateBean := &pipelineConfigBean.CiTemplateBean{} if ciPipeline.IsDockerConfigOverridden { //creating template override templateOverride := &pipelineConfig.CiTemplateOverride{ @@ -986,7 +986,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf UpdatedOn: time.Now(), }, } - ciTemplateBean = &bean2.CiTemplateBean{ + ciTemplateBean = &pipelineConfigBean.CiTemplateBean{ CiTemplateOverride: templateOverride, CiBuildConfig: ciPipeline.DockerConfigOverride.CiBuildConfig, UserId: createRequest.UserId, @@ -1385,7 +1385,7 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name, description string displayName := name appName := name if appType == helper.Job { - appName = name + "-" + util2.Generate(8) + "J" + bean2.UniquePlaceHolderForAppName + appName = name + "-" + util2.Generate(8) + "J" + pipelineConfigBean.UniquePlaceHolderForAppName } pg := &app2.App{ Active: true, @@ -1700,9 +1700,9 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateCDPipeline(pipelineRequest *bean. func (impl CiCdPipelineOrchestratorImpl) DeleteCdPipeline(pipelineId int, userId int32, tx *pg.Tx) error { return impl.pipelineRepository.Delete(pipelineId, userId, tx) } -func (impl CiCdPipelineOrchestratorImpl) getPipelineIdAndPrePostStageMapping(dbPipelines []*pipelineConfig.Pipeline) (map[int][]*bean2.PipelineStageDto, error) { +func (impl CiCdPipelineOrchestratorImpl) getPipelineIdAndPrePostStageMapping(dbPipelines []*pipelineConfig.Pipeline) (map[int][]*pipelineConfigBean.PipelineStageDto, error) { var err error - pipelineIdAndPrePostStageMapping := make(map[int][]*bean2.PipelineStageDto) + pipelineIdAndPrePostStageMapping := make(map[int][]*pipelineConfigBean.PipelineStageDto) var dbPipelineIds []int for _, pipeline := range dbPipelines { dbPipelineIds = append(dbPipelineIds, pipeline.Id) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 45144bd1c5e..a0b03f2d4a3 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "github.com/devtron-labs/devtron/pkg/infraConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "path/filepath" "strconv" @@ -35,7 +36,7 @@ import ( "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/auth/user" repository1 "github.com/devtron-labs/devtron/pkg/cluster/repository" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -152,7 +153,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) if err != nil { return 0, err } - if trigger.PipelineType == bean2.CI_JOB && len(ciMaterials) != 0 { + if trigger.PipelineType == string(pipelineConfigBean.CI_JOB) && len(ciMaterials) != 0 { ciMaterials = []*pipelineConfig.CiPipelineMaterial{ciMaterials[0]} ciMaterials[0].GitMaterial = nil ciMaterials[0].GitMaterialId = 0 @@ -199,7 +200,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) } // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) - prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, bean2.CiStage, scope) + prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, pipelineConfigBean.CiStage, scope) if err != nil { impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) return 0, err @@ -245,9 +246,9 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) workflowRequest.AppLabels = appLabels workflowRequest.Env = env if isJob { - workflowRequest.Type = bean2.JOB_WORKFLOW_PIPELINE_TYPE + workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE } else { - workflowRequest.Type = bean2.CI_WORKFLOW_PIPELINE_TYPE + workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE } err = impl.executeCiPipeline(workflowRequest) @@ -394,8 +395,8 @@ func (impl *CiServiceImpl) buildDefaultArtifactLocation(ciWorkflowConfig *pipeli return ArtifactLocation } -func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfig *pipelineConfig.CiWorkflowConfig, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*bean2.StepObject, postCiSteps []*bean2.StepObject, refPluginsData []*bean2.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { - var ciProjectDetails []bean2.CiProjectDetails +func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfig *pipelineConfig.CiWorkflowConfig, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { + var ciProjectDetails []pipelineConfigBean.CiProjectDetails commitHashes := trigger.CommitHashes for _, ciMaterial := range ciMaterials { // ignore those materials which have inactive git material @@ -403,7 +404,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. continue } commitHashForPipelineId := commitHashes[ciMaterial.Id] - ciProjectDetail := bean2.CiProjectDetails{ + ciProjectDetail := pipelineConfigBean.CiProjectDetails{ GitRepository: ciMaterial.GitMaterial.Url, MaterialName: ciMaterial.GitMaterial.Name, CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, @@ -416,7 +417,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. Message: commitHashForPipelineId.Message, Type: string(ciMaterial.Type), CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), - GitOptions: bean2.GitOptions{ + GitOptions: pipelineConfigBean.GitOptions{ UserName: ciMaterial.GitMaterial.GitProvider.UserName, Password: ciMaterial.GitMaterial.GitProvider.Password, SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, @@ -460,21 +461,21 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. // building preCiSteps & postCiSteps from them, refPluginsData not needed preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) - refPluginsData = []*bean2.RefPluginObject{} + refPluginsData = []*pipelineConfigBean.RefPluginObject{} } infraConfigScope := &infraConfig.Scope{ AppId: pipeline.AppId, } - infraGetter, err := impl.infraProvider.GetInfraProvider(bean2.CI_WORKFLOW_PIPELINE_TYPE) + infraGetter, err := impl.infraProvider.GetInfraProvider(pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE) if err != nil { - impl.Logger.Errorw("error in getting infra provider", "err", err, "infraProviderType", bean2.CI_WORKFLOW_PIPELINE_TYPE) + impl.Logger.Errorw("error in getting infra provider", "err", err, "infraProviderType", pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE) return nil, err } if isJob { - infraGetter, err = impl.infraProvider.GetInfraProvider(bean2.JOB_WORKFLOW_PIPELINE_TYPE) + infraGetter, err = impl.infraProvider.GetInfraProvider(pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE) if err != nil { - impl.Logger.Errorw("error in getting infra provider", "err", err, "infraProviderType", bean2.JOB_WORKFLOW_PIPELINE_TYPE) + impl.Logger.Errorw("error in getting infra provider", "err", err, "infraProviderType", pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE) return nil, err } } @@ -519,7 +520,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. var dockerfilePath string var dockerRepository string var checkoutPath string - var ciBuildConfigBean *bean2.CiBuildConfigBean + var ciBuildConfigBean *pipelineConfigBean.CiBuildConfigBean dockerRegistry := &repository3.DockerArtifactStore{} if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) @@ -538,7 +539,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. dockerRegistry = ciTemplate.DockerRegistry dockerRepository = ciTemplate.DockerRepository ciBuildConfigEntity := ciTemplate.CiBuildConfig - ciBuildConfigBean, err = bean2.ConvertDbBuildConfigToBean(ciBuildConfigEntity) + ciBuildConfigBean, err = adapter.ConvertDbBuildConfigToBean(ciBuildConfigEntity) if ciBuildConfigBean != nil { ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId } @@ -551,16 +552,16 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. checkoutPath = "./" } var dockerImageTag string - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean2.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) if err != nil && err != pg.ErrNoRows { return nil, err } if customTag.Id != 0 && customTag.Enabled == true { - imagePathReservation, err := impl.customTagService.GenerateImagePath(bean2.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) + imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) if err != nil { - if errors.Is(err, bean2.ErrImagePathInUse) { + if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { savedWf.Status = pipelineConfig.WorkflowFailed - savedWf.Message = bean2.ImageTagUnavailableMessage + savedWf.Message = pipelineConfigBean.ImageTagUnavailableMessage err1 := impl.ciWorkflowRepository.UpdateWorkFlow(savedWf) if err1 != nil { impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") @@ -586,7 +587,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. var imageReservationIds []int if !isJob { registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, - fmt.Sprintf(bean2.ImagePathPattern, + fmt.Sprintf(pipelineConfigBean.ImagePathPattern, dockerRegistry.RegistryURL, dockerRepository, dockerImageTag), @@ -606,7 +607,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } // mergedArgs := string(merged) oldArgs := ciTemplate.Args - ciBuildConfigBean, err = bean2.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) + ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) if err != nil { impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) return nil, errors.New("error while parsing ci build config") @@ -626,13 +627,13 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ciBuildConfigBean.PipelineType = trigger.PipelineType - if ciBuildConfigBean.CiBuildType == bean2.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == bean2.MANAGED_DOCKERFILE_BUILD_TYPE { + if ciBuildConfigBean.CiBuildType == pipelineConfigBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == pipelineConfigBean.MANAGED_DOCKERFILE_BUILD_TYPE { ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) dockerBuildConfig.DockerfilePath = dockerfilePath checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] - } else if ciBuildConfigBean.CiBuildType == bean2.BUILDPACK_BUILD_TYPE { + } else if ciBuildConfigBean.CiBuildType == pipelineConfigBean.BUILDPACK_BUILD_TYPE { buildPackConfig := ciBuildConfigBean.BuildPackConfig checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) } @@ -682,7 +683,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ImageRetryCount: impl.config.ImageRetryCount, ImageRetryInterval: impl.config.ImageRetryInterval, WorkflowExecutor: impl.config.GetWorkflowExecutorType(), - Type: bean2.CI_WORKFLOW_PIPELINE_TYPE, + Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, CiArtifactLastFetch: trigger.CiArtifactLastFetch, RegistryDestinationImageMap: registryDestinationImageMap, RegistryCredentialMap: registryCredentialMap, @@ -771,7 +772,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. return workflowRequest, nil } -func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*bean2.StepObject, postCiSteps []*bean2.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]plugin.RegistryCredentials, string, []int, error) { +func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]plugin.RegistryCredentials, string, []int, error) { var registryDestinationImageMap map[string][]string var registryCredentialMap map[string]plugin.RegistryCredentials var pluginArtifactStage string @@ -802,7 +803,7 @@ func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugi for _, image := range images { if image == buildImagePath { return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, - bean2.ErrImagePathInUse + pipelineConfigBean.ErrImagePathInUse } } } @@ -828,38 +829,38 @@ func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, regis return imagePathReservationIds, nil } -func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*bean2.StepObject { +func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { // before plugin support, few variables were set as env vars in ci-runner // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload - inputVars := []*bean2.VariableObject{ + inputVars := []*pipelineConfigBean.VariableObject{ { Name: "DOCKER_IMAGE_TAG", Format: "STRING", - VariableType: bean2.VARIABLE_TYPE_REF_GLOBAL, + VariableType: pipelineConfigBean.VARIABLE_TYPE_REF_GLOBAL, ReferenceVariableName: "DOCKER_IMAGE_TAG", }, { Name: "DOCKER_REPOSITORY", Format: "STRING", - VariableType: bean2.VARIABLE_TYPE_REF_GLOBAL, + VariableType: pipelineConfigBean.VARIABLE_TYPE_REF_GLOBAL, ReferenceVariableName: "DOCKER_REPOSITORY", }, { Name: "DOCKER_REGISTRY_URL", Format: "STRING", - VariableType: bean2.VARIABLE_TYPE_REF_GLOBAL, + VariableType: pipelineConfigBean.VARIABLE_TYPE_REF_GLOBAL, ReferenceVariableName: "DOCKER_REGISTRY_URL", }, { Name: "DOCKER_IMAGE", Format: "STRING", - VariableType: bean2.VARIABLE_TYPE_REF_GLOBAL, + VariableType: pipelineConfigBean.VARIABLE_TYPE_REF_GLOBAL, ReferenceVariableName: "DOCKER_IMAGE", }, } - var ciSteps []*bean2.StepObject + var ciSteps []*pipelineConfigBean.StepObject for _, dockerBuildScript := range dockerBuildScripts { - ciStep := &bean2.StepObject{ + ciStep := &pipelineConfigBean.StepObject{ Name: dockerBuildScript.Name, Index: dockerBuildScript.Index, Script: dockerBuildScript.Script, diff --git a/pkg/pipeline/CiTemplateService.go b/pkg/pipeline/CiTemplateService.go index 29f9e5d4b79..ae9511a444b 100644 --- a/pkg/pipeline/CiTemplateService.go +++ b/pkg/pipeline/CiTemplateService.go @@ -2,7 +2,10 @@ package pipeline import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -13,6 +16,8 @@ type CiTemplateService interface { FindTemplateOverrideByAppId(appId int) (ciTemplateBeans []*bean.CiTemplateBean, err error) FindTemplateOverrideByCiPipelineIds(ciPipelineIds []int) (ciTemplateBeans []*bean.CiTemplateBean, err error) FindTemplateOverrideByCiPipelineId(ciPipelineId int) (*bean.CiTemplateBean, error) + GetAppliedDockerConfigForCiPipeline(ciPipelineId, appId int, isOverridden bool) (*types.DockerArtifactStoreBean, error) + GetBaseDockerConfigForCiPipeline(appId int) (*types.DockerArtifactStoreBean, error) Update(ciTemplateBean *bean.CiTemplateBean) error FindByAppIds(appIds []int) (map[int]*bean.CiTemplateBean, error) } @@ -72,13 +77,13 @@ func (impl CiTemplateServiceImpl) FindByAppId(appId int) (ciTemplateBean *bean.C return nil, err } ciBuildConfig := ciTemplate.CiBuildConfig - ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(ciBuildConfig) + ciBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBuildConfig) if err != nil { impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", ciBuildConfig, "error", err) } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, "", ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, nil) + ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, "", ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } @@ -134,14 +139,14 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineIds(ciPipeline } func (impl CiTemplateServiceImpl) extractBuildConfigBean(templateOverride *pipelineConfig.CiTemplateOverride) (*bean.CiBuildConfigBean, error) { - ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(templateOverride.CiBuildConfig) + ciBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(templateOverride.CiBuildConfig) if err != nil { impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", templateOverride.CiBuildConfig, "error", err) return nil, err } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", "", nil) + ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(templateOverride.DockerfilePath, "", "", "", "", nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } @@ -161,6 +166,30 @@ func (impl CiTemplateServiceImpl) FindTemplateOverrideByCiPipelineId(ciPipelineI return &bean.CiTemplateBean{CiTemplateOverride: templateOverride, CiBuildConfig: ciBuildConfigBean}, err } +func (impl CiTemplateServiceImpl) GetAppliedDockerConfigForCiPipeline(ciPipelineId, appId int, isOverridden bool) (*types.DockerArtifactStoreBean, error) { + if !isOverridden { + return impl.GetBaseDockerConfigForCiPipeline(appId) + } + templateOverride, err := impl.CiTemplateOverrideRepository.FindByCiPipelineId(ciPipelineId) + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("error in getting ciTemplateOverrides by ciPipelineId", "err", err, "ciPipelineId", ciPipelineId) + return nil, err + } + if util.IsErrNoRows(err) { + return impl.GetBaseDockerConfigForCiPipeline(appId) + } + return adapter.GetDockerConfigBean(templateOverride.DockerRegistry), nil +} + +func (impl CiTemplateServiceImpl) GetBaseDockerConfigForCiPipeline(appId int) (*types.DockerArtifactStoreBean, error) { + ciTemplate, err := impl.CiTemplateRepository.FindByAppId(appId) + if err != nil { + impl.Logger.Errorw("error in getting ciTemplate by appId", "err", err, "appId", appId) + return nil, err + } + return adapter.GetDockerConfigBean(ciTemplate.DockerRegistry), nil +} + func (impl CiTemplateServiceImpl) Update(ciTemplateBean *bean.CiTemplateBean) error { ciTemplate := ciTemplateBean.CiTemplate ciTemplateOverride := ciTemplateBean.CiTemplateOverride @@ -202,13 +231,13 @@ func (impl CiTemplateServiceImpl) FindByAppIds(appIds []int) (map[int]*bean.CiTe ciTemplateMap := make(map[int]*bean.CiTemplateBean) for _, ciTemplate := range ciTemplates { ciBuildConfig := ciTemplate.CiBuildConfig - ciBuildConfigBean, err := bean.ConvertDbBuildConfigToBean(ciBuildConfig) + ciBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBuildConfig) if err != nil { impl.Logger.Errorw("error occurred while converting dbBuildConfig to bean", "ciBuildConfig", ciBuildConfig, "error", err) } if ciBuildConfigBean == nil { - ciBuildConfigBean, err = bean.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, "", ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, nil) + ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(ciTemplate.DockerfilePath, ciTemplate.Args, "", ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, nil) if err != nil { impl.Logger.Errorw("error occurred while parsing ci build config", "err", err) } diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go new file mode 100644 index 00000000000..06994490b98 --- /dev/null +++ b/pkg/pipeline/adapter/adapter.go @@ -0,0 +1,184 @@ +package adapter + +import ( + "encoding/json" + dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/sql" + "time" +) + +func GetDockerConfigBean(dockerRegistry *dockerRegistryRepository.DockerArtifactStore) *types.DockerArtifactStoreBean { + return &types.DockerArtifactStoreBean{ + Id: dockerRegistry.Id, + RegistryType: dockerRegistry.RegistryType, + RegistryURL: dockerRegistry.RegistryURL, + Username: dockerRegistry.Username, + Password: dockerRegistry.Password, + AWSRegion: dockerRegistry.AWSRegion, + Connection: dockerRegistry.Connection, + Cert: dockerRegistry.Cert, + AWSAccessKeyId: dockerRegistry.AWSAccessKeyId, + AWSSecretAccessKey: dockerRegistry.AWSSecretAccessKey, + } +} + +func UpdateRegistryDetailsToWrfReq(cdStageWorkflowRequest *types.WorkflowRequest, dockerRegistry *types.DockerArtifactStoreBean) { + cdStageWorkflowRequest.DockerUsername = dockerRegistry.Username + cdStageWorkflowRequest.DockerPassword = dockerRegistry.Password + cdStageWorkflowRequest.AwsRegion = dockerRegistry.AWSRegion + cdStageWorkflowRequest.DockerConnection = dockerRegistry.Connection + cdStageWorkflowRequest.DockerCert = dockerRegistry.Cert + cdStageWorkflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId + cdStageWorkflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey + cdStageWorkflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) + cdStageWorkflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL + cdStageWorkflowRequest.DockerRegistryId = dockerRegistry.Id +} + +func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ciBuildConfigBean *pipelineConfigBean.CiBuildConfigBean, userId int32) (*pipelineConfig.CiBuildConfig, error) { + buildMetadata := "" + ciBuildType := ciBuildConfigBean.CiBuildType + if ciBuildType == pipelineConfigBean.BUILDPACK_BUILD_TYPE { + buildPackConfigMetadataBytes, err := json.Marshal(ciBuildConfigBean.BuildPackConfig) + if err != nil { + return nil, err + } + buildMetadata = string(buildPackConfigMetadataBytes) + } else if ciBuildType == pipelineConfigBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == pipelineConfigBean.MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildMetadataBytes, err := json.Marshal(ciBuildConfigBean.DockerBuildConfig) + if err != nil { + return nil, err + } + buildMetadata = string(dockerBuildMetadataBytes) + } + ciBuildConfigEntity := &pipelineConfig.CiBuildConfig{ + Id: ciBuildConfigBean.Id, + Type: string(ciBuildType), + CiTemplateId: templateId, + CiTemplateOverrideId: overrideTemplateId, + BuildMetadata: buildMetadata, + AuditLog: sql.AuditLog{UpdatedOn: time.Now(), UpdatedBy: userId}, + UseRootContext: &ciBuildConfigBean.UseRootBuildContext, + } + return ciBuildConfigEntity, nil +} + +func ConvertDbBuildConfigToBean(dbBuildConfig *pipelineConfig.CiBuildConfig) (*pipelineConfigBean.CiBuildConfigBean, error) { + var buildPackConfig *pipelineConfigBean.BuildPackConfig + var dockerBuildConfig *pipelineConfigBean.DockerBuildConfig + var err error + if dbBuildConfig == nil { + return nil, nil + } + ciBuildType := pipelineConfigBean.CiBuildType(dbBuildConfig.Type) + if ciBuildType == pipelineConfigBean.BUILDPACK_BUILD_TYPE { + buildPackConfig, err = convertMetadataToBuildPackConfig(dbBuildConfig.BuildMetadata) + if err != nil { + return nil, err + } + } else if ciBuildType == pipelineConfigBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == pipelineConfigBean.MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildConfig, err = convertMetadataToDockerBuildConfig(dbBuildConfig.BuildMetadata) + if err != nil { + return nil, err + } + } + useRootBuildContext := false + //dbBuildConfig.UseRootContext will be nil if the entry in db never updated before + if dbBuildConfig.UseRootContext == nil || *(dbBuildConfig.UseRootContext) { + useRootBuildContext = true + } + ciBuildConfigBean := &pipelineConfigBean.CiBuildConfigBean{ + Id: dbBuildConfig.Id, + CiBuildType: ciBuildType, + BuildPackConfig: buildPackConfig, + DockerBuildConfig: dockerBuildConfig, + UseRootBuildContext: useRootBuildContext, + } + return ciBuildConfigBean, nil +} + +func convertMetadataToBuildPackConfig(buildConfMetadata string) (*pipelineConfigBean.BuildPackConfig, error) { + buildPackConfig := &pipelineConfigBean.BuildPackConfig{} + err := json.Unmarshal([]byte(buildConfMetadata), buildPackConfig) + return buildPackConfig, err +} + +func convertMetadataToDockerBuildConfig(dockerBuildMetadata string) (*pipelineConfigBean.DockerBuildConfig, error) { + dockerBuildConfig := &pipelineConfigBean.DockerBuildConfig{} + err := json.Unmarshal([]byte(dockerBuildMetadata), dockerBuildConfig) + return dockerBuildConfig, err +} + +func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs string, dockerBuildOptions string, targetPlatform string, ciBuildConfigBean *pipelineConfigBean.CiBuildConfigBean) (*pipelineConfigBean.CiBuildConfigBean, error) { + oldDockerArgs := map[string]string{} + ciLevelDockerArgs := map[string]string{} + dockerBuildOptionsMap := map[string]string{} + if oldArgs != "" { + if err := json.Unmarshal([]byte(oldArgs), &oldDockerArgs); err != nil { + return nil, err + } + } + if ciLevelArgs != "" { + if err := json.Unmarshal([]byte(ciLevelArgs), &ciLevelDockerArgs); err != nil { + return nil, err + } + } + if dockerBuildOptions != "" { + if err := json.Unmarshal([]byte(dockerBuildOptions), &dockerBuildOptionsMap); err != nil { + return nil, err + } + } + //no entry found in ci_build_config table, construct with requested data + if ciBuildConfigBean == nil { + dockerArgs := mergeMap(oldDockerArgs, ciLevelDockerArgs) + ciBuildConfigBean = &pipelineConfigBean.CiBuildConfigBean{ + CiBuildType: pipelineConfigBean.SELF_DOCKERFILE_BUILD_TYPE, + DockerBuildConfig: &pipelineConfigBean.DockerBuildConfig{ + DockerfilePath: dockerfilePath, + Args: dockerArgs, + TargetPlatform: targetPlatform, + DockerBuildOptions: dockerBuildOptionsMap, + BuildContext: "", + }, + //setting true as default + UseRootBuildContext: true, + } + } else if ciBuildConfigBean.CiBuildType == pipelineConfigBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == pipelineConfigBean.MANAGED_DOCKERFILE_BUILD_TYPE { + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerArgs := mergeMap(dockerBuildConfig.Args, ciLevelDockerArgs) + //dockerBuildConfig.DockerfilePath = dockerfilePath + dockerBuildConfig.Args = dockerArgs + } + return ciBuildConfigBean, nil +} + +func mergeMap(oldDockerArgs map[string]string, ciLevelDockerArgs map[string]string) map[string]string { + dockerArgs := make(map[string]string) + for key, value := range oldDockerArgs { + dockerArgs[key] = value + } + for key, value := range ciLevelDockerArgs { + dockerArgs[key] = value + } + return dockerArgs +} + +// IsLinkedCD will return if the pipelineConfig.CiPipeline is a Linked CD +func IsLinkedCD(ci pipelineConfig.CiPipeline) bool { + return ci.ParentCiPipeline != 0 && ci.PipelineType == string(pipelineConfigBean.LINKED_CD) +} + +// IsLinkedCI will return if the pipelineConfig.CiPipeline is a Linked CI +// Currently there are inconsistent values present in PipelineType ("CI_EXTERNAL", "", "LINKED") +// TODO migrate the deprecated values and maintain a consistent PipelineType +func IsLinkedCI(ci pipelineConfig.CiPipeline) bool { + return ci.ParentCiPipeline != 0 && ci.PipelineType != string(pipelineConfigBean.LINKED_CD) +} + +// IsCIJob will return if the pipelineConfig.CiPipeline is a CI JOB +func IsCIJob(ci pipelineConfig.CiPipeline) bool { + return ci.PipelineType == string(pipelineConfigBean.CI_JOB) +} diff --git a/pkg/pipeline/bean/CiBuildConfig.go b/pkg/pipeline/bean/CiBuildConfig.go index f15a0ae26de..3c65a1acff5 100644 --- a/pkg/pipeline/bean/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiBuildConfig.go @@ -1,12 +1,5 @@ package bean -import ( - "encoding/json" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/pkg/sql" - "time" -) - type CiBuildType string const ( @@ -20,6 +13,16 @@ const UniquePlaceHolderForAppName = "$etron" const PIPELINE_NAME_ALREADY_EXISTS_ERROR = "pipeline name already exist" +type PipelineType string + +const ( + NORMAL PipelineType = "NORMAL" + LINKED PipelineType = "LINKED" + EXTERNAL PipelineType = "EXTERNAL" + CI_JOB PipelineType = "CI_JOB" + LINKED_CD PipelineType = "LINKED_CD" +) + type CiBuildConfigBean struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` @@ -53,131 +56,3 @@ type BuildPackConfig struct { Args map[string]string `json:"args"` ProjectPath string `json:"projectPath,omitempty"` } - -func ConvertBuildConfigBeanToDbEntity(templateId int, overrideTemplateId int, ciBuildConfigBean *CiBuildConfigBean, userId int32) (*pipelineConfig.CiBuildConfig, error) { - buildMetadata := "" - ciBuildType := ciBuildConfigBean.CiBuildType - if ciBuildType == BUILDPACK_BUILD_TYPE { - buildPackConfigMetadataBytes, err := json.Marshal(ciBuildConfigBean.BuildPackConfig) - if err != nil { - return nil, err - } - buildMetadata = string(buildPackConfigMetadataBytes) - } else if ciBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { - dockerBuildMetadataBytes, err := json.Marshal(ciBuildConfigBean.DockerBuildConfig) - if err != nil { - return nil, err - } - buildMetadata = string(dockerBuildMetadataBytes) - } - ciBuildConfigEntity := &pipelineConfig.CiBuildConfig{ - Id: ciBuildConfigBean.Id, - Type: string(ciBuildType), - CiTemplateId: templateId, - CiTemplateOverrideId: overrideTemplateId, - BuildMetadata: buildMetadata, - AuditLog: sql.AuditLog{UpdatedOn: time.Now(), UpdatedBy: userId}, - UseRootContext: &ciBuildConfigBean.UseRootBuildContext, - } - return ciBuildConfigEntity, nil -} - -func ConvertDbBuildConfigToBean(dbBuildConfig *pipelineConfig.CiBuildConfig) (*CiBuildConfigBean, error) { - var buildPackConfig *BuildPackConfig - var dockerBuildConfig *DockerBuildConfig - var err error - if dbBuildConfig == nil { - return nil, nil - } - ciBuildType := CiBuildType(dbBuildConfig.Type) - if ciBuildType == BUILDPACK_BUILD_TYPE { - buildPackConfig, err = convertMetadataToBuildPackConfig(dbBuildConfig.BuildMetadata) - if err != nil { - return nil, err - } - } else if ciBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { - dockerBuildConfig, err = convertMetadataToDockerBuildConfig(dbBuildConfig.BuildMetadata) - if err != nil { - return nil, err - } - } - useRootBuildContext := false - //dbBuildConfig.UseRootContext will be nil if the entry in db never updated before - if dbBuildConfig.UseRootContext == nil || *(dbBuildConfig.UseRootContext) { - useRootBuildContext = true - } - ciBuildConfigBean := &CiBuildConfigBean{ - Id: dbBuildConfig.Id, - CiBuildType: ciBuildType, - BuildPackConfig: buildPackConfig, - DockerBuildConfig: dockerBuildConfig, - UseRootBuildContext: useRootBuildContext, - } - return ciBuildConfigBean, nil -} - -func convertMetadataToBuildPackConfig(buildConfMetadata string) (*BuildPackConfig, error) { - buildPackConfig := &BuildPackConfig{} - err := json.Unmarshal([]byte(buildConfMetadata), buildPackConfig) - return buildPackConfig, err -} - -func convertMetadataToDockerBuildConfig(dockerBuildMetadata string) (*DockerBuildConfig, error) { - dockerBuildConfig := &DockerBuildConfig{} - err := json.Unmarshal([]byte(dockerBuildMetadata), dockerBuildConfig) - return dockerBuildConfig, err -} - -func OverrideCiBuildConfig(dockerfilePath string, oldArgs string, ciLevelArgs string, dockerBuildOptions string, targetPlatform string, ciBuildConfigBean *CiBuildConfigBean) (*CiBuildConfigBean, error) { - oldDockerArgs := map[string]string{} - ciLevelDockerArgs := map[string]string{} - dockerBuildOptionsMap := map[string]string{} - if oldArgs != "" { - if err := json.Unmarshal([]byte(oldArgs), &oldDockerArgs); err != nil { - return nil, err - } - } - if ciLevelArgs != "" { - if err := json.Unmarshal([]byte(ciLevelArgs), &ciLevelDockerArgs); err != nil { - return nil, err - } - } - if dockerBuildOptions != "" { - if err := json.Unmarshal([]byte(dockerBuildOptions), &dockerBuildOptionsMap); err != nil { - return nil, err - } - } - //no entry found in ci_build_config table, construct with requested data - if ciBuildConfigBean == nil { - dockerArgs := mergeMap(oldDockerArgs, ciLevelDockerArgs) - ciBuildConfigBean = &CiBuildConfigBean{ - CiBuildType: SELF_DOCKERFILE_BUILD_TYPE, - DockerBuildConfig: &DockerBuildConfig{ - DockerfilePath: dockerfilePath, - Args: dockerArgs, - TargetPlatform: targetPlatform, - DockerBuildOptions: dockerBuildOptionsMap, - BuildContext: "", - }, - //setting true as default - UseRootBuildContext: true, - } - } else if ciBuildConfigBean.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { - dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig - dockerArgs := mergeMap(dockerBuildConfig.Args, ciLevelDockerArgs) - //dockerBuildConfig.DockerfilePath = dockerfilePath - dockerBuildConfig.Args = dockerArgs - } - return ciBuildConfigBean, nil -} - -func mergeMap(oldDockerArgs map[string]string, ciLevelDockerArgs map[string]string) map[string]string { - dockerArgs := make(map[string]string) - for key, value := range oldDockerArgs { - dockerArgs[key] = value - } - for key, value := range ciLevelDockerArgs { - dockerArgs[key] = value - } - return dockerArgs -} diff --git a/pkg/pipeline/bean/workFlowRequestBean.go b/pkg/pipeline/bean/workFlowRequestBean.go index 644239de642..f9f2ed589ac 100644 --- a/pkg/pipeline/bean/workFlowRequestBean.go +++ b/pkg/pipeline/bean/workFlowRequestBean.go @@ -16,8 +16,6 @@ const ( IMAGE_SCANNER_ENDPOINT = "IMAGE_SCANNER_ENDPOINT" ) -const CI_JOB string = "CI_JOB" - type WorkflowPipelineType string const ( diff --git a/pkg/pipeline/history/CiPipelineHistoryService.go b/pkg/pipeline/history/CiPipelineHistoryService.go index db069b665c1..caf0f34b4cc 100644 --- a/pkg/pipeline/history/CiPipelineHistoryService.go +++ b/pkg/pipeline/history/CiPipelineHistoryService.go @@ -3,6 +3,7 @@ package history import ( "encoding/json" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/sql" @@ -51,7 +52,7 @@ func (impl *CiPipelineHistoryServiceImpl) SaveHistory(pipeline *pipelineConfig.C IsCiTemplateOverriden: true, } if CiTemplateBean.CiBuildConfig != nil { - CiBuildConfigDbEntity, _ := bean.ConvertBuildConfigBeanToDbEntity(ciTemplateId, ciTemplateOverrideId, CiTemplateBean.CiBuildConfig, CiTemplateBean.UserId) + CiBuildConfigDbEntity, _ := adapter.ConvertBuildConfigBeanToDbEntity(ciTemplateId, ciTemplateOverrideId, CiTemplateBean.CiBuildConfig, CiTemplateBean.UserId) CiTemplateOverride.CiBuildConfigId = CiBuildConfigDbEntity.Id CiTemplateOverride.BuildMetaDataType = CiBuildConfigDbEntity.Type CiTemplateOverride.BuildMetadata = CiBuildConfigDbEntity.BuildMetadata diff --git a/pkg/pipeline/history/ciTemplateHistoryService.go b/pkg/pipeline/history/ciTemplateHistoryService.go index 44d3704e17e..b1e51b961ba 100644 --- a/pkg/pipeline/history/ciTemplateHistoryService.go +++ b/pkg/pipeline/history/ciTemplateHistoryService.go @@ -1,6 +1,7 @@ package history import ( + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/sql" @@ -33,7 +34,7 @@ func (impl CiTemplateHistoryServiceImpl) SaveHistory(ciTemplateBean *bean.CiTemp ciTemplateId := 0 ciTemplateOverrideId := 0 - ciBuildConfigDbEntity, err := bean.ConvertBuildConfigBeanToDbEntity(ciTemplateId, ciTemplateOverrideId, ciBuildConfig, ciTemplateBean.UserId) + ciBuildConfigDbEntity, err := adapter.ConvertBuildConfigBeanToDbEntity(ciTemplateId, ciTemplateOverrideId, ciBuildConfig, ciTemplateBean.UserId) materialHistory := &repository.CiTemplateHistory{ CiTemplateId: ciTemplate.Id, diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index ccf9a53a457..bf9fc514a44 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -43,7 +43,7 @@ import ( "time" "github.com/devtron-labs/common-lib/pubsub-lib/model" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" serverBean "github.com/devtron-labs/devtron/pkg/server/bean" @@ -642,7 +642,7 @@ func (impl *WorkflowDagExecutorImpl) handleWebhookExternalCiEvent(artifact *repo // handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) func (impl *WorkflowDagExecutorImpl) deleteCorruptedPipelineStage(pipelineStage *repository4.PipelineStage, triggeredBy int32) (error, bool) { if pipelineStage != nil { - stageReq := &bean3.PipelineStageDto{ + stageReq := &pipelineConfigBean.PipelineStageDto{ Id: pipelineStage.Id, Type: pipelineStage.Type, } @@ -913,7 +913,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext bean5.T IsArtifactUploaded: request.IsArtifactUploaded, AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, } - plugin, err := impl.globalPluginRepository.GetPluginByName(bean3.VULNERABILITY_SCANNING_PLUGIN) + plugin, err := impl.globalPluginRepository.GetPluginByName(pipelineConfigBean.VULNERABILITY_SCANNING_PLUGIN) if err != nil || len(plugin) == 0 { impl.logger.Errorw("error in getting image scanning plugin", "err", err) return 0, err @@ -935,7 +935,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext bean5.T var pluginArtifacts []*repository.CiArtifact for registry, artifacts := range request.PluginRegistryArtifactDetails { for _, image := range artifacts { - if pipeline.PipelineType == bean3.CI_JOB && image == "" { + if pipeline.PipelineType == string(pipelineConfigBean.CI_JOB) && image == "" { continue } pluginArtifact := &repository.CiArtifact{ diff --git a/wire_gen.go b/wire_gen.go index ccec858297b..c8b002c9d84 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -588,7 +588,7 @@ func InitializeApp() (*App, error) { gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger, k8sServiceImpl) manifestPushConfigRepositoryImpl := repository10.NewManifestPushConfigRepository(sugaredLogger, db) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, globalEnvVariables, imageScanResultRepositoryImpl, cvePolicyRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateRepositoryImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, globalEnvVariables, imageScanResultRepositoryImpl, cvePolicyRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl) if err != nil { return nil, err }