diff --git a/api/bean/ValuesOverrideRequest.go b/api/bean/ValuesOverrideRequest.go index 9e23d653584..1e1e56e5a48 100644 --- a/api/bean/ValuesOverrideRequest.go +++ b/api/bean/ValuesOverrideRequest.go @@ -113,6 +113,9 @@ type ArtifactsListFilterOptions struct { PipelineId int StageType WorkflowType + // CiPipelineId is id of ci-pipeline present in the same app-workflow of PipelineId + CiPipelineId int + //parent satge data ParentCdId int ParentId int @@ -126,4 +129,7 @@ type ArtifactsListFilterOptions struct { //pluginStage PluginStage string + + // UseCdStageQueryV2 is to set query version + UseCdStageQueryV2 bool } diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 380dfe8e2e9..93001355e40 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -713,13 +713,22 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByParentCiWorkflowId(parentCiWo func (impl CiArtifactRepositoryImpl) FindArtifactByListFilter(listingFilterOptions *bean.ArtifactsListFilterOptions) ([]CiArtifact, int, error) { var ciArtifactsResp []CiArtifactWithExtraData - var ciArtifacts []CiArtifact + ciArtifacts := make([]CiArtifact, 0) totalCount := 0 finalQuery := BuildQueryForArtifactsForCdStage(*listingFilterOptions) _, err := impl.dbConnection.Query(&ciArtifactsResp, finalQuery) if err == pg.ErrNoRows || len(ciArtifactsResp) == 0 { return ciArtifacts, totalCount, nil } + + if listingFilterOptions.UseCdStageQueryV2 { + for _, cia := range ciArtifactsResp { + totalCount = cia.TotalCount + ciArtifacts = append(ciArtifacts, cia.CiArtifact) + } + return ciArtifacts, totalCount, err + } + artifactIds := make([]int, len(ciArtifactsResp)) for i, af := range ciArtifactsResp { artifactIds[i] = af.Id diff --git a/internal/sql/repository/CiArtifactsListingQueryBuilder.go b/internal/sql/repository/CiArtifactsListingQueryBuilder.go index 29220239f41..2d17fe73d97 100644 --- a/internal/sql/repository/CiArtifactsListingQueryBuilder.go +++ b/internal/sql/repository/CiArtifactsListingQueryBuilder.go @@ -50,6 +50,10 @@ func BuildQueryForArtifactsForCdStage(listingFilterOptions bean.ArtifactsListFil // expected result -> will fetch all successfully deployed artifacts ar parent stage plus its own stage. Along with this it will // also fetch all artifacts generated by plugin at pre_cd or post_cd process (will use data_source in ci artifact table for this) + if listingFilterOptions.UseCdStageQueryV2 { + return buildQueryForArtifactsForCdStageV2(listingFilterOptions) + } + commonQuery := " from ci_artifact LEFT JOIN cd_workflow ON ci_artifact.id = cd_workflow.ci_artifact_id" + " LEFT JOIN cd_workflow_runner ON cd_workflow_runner.cd_workflow_id=cd_workflow.id " + " Where (((cd_workflow_runner.id in (select MAX(cd_workflow_runner.id) OVER (PARTITION BY cd_workflow.ci_artifact_id) FROM cd_workflow_runner inner join cd_workflow on cd_workflow.id=cd_workflow_runner.cd_workflow_id))" + @@ -72,6 +76,36 @@ func BuildQueryForArtifactsForCdStage(listingFilterOptions bean.ArtifactsListFil return finalQuery } +func buildQueryForArtifactsForCdStageV2(listingFilterOptions bean.ArtifactsListFilterOptions) string { + whereCondition := fmt.Sprintf(" WHERE (id IN ("+ + " SELECT DISTINCT(cd_workflow.ci_artifact_id) as ci_artifact_id "+ + " FROM cd_workflow_runner"+ + " INNER JOIN cd_workflow ON cd_workflow.id = cd_workflow_runner.cd_workflow_id "+ + " AND (cd_workflow.pipeline_id = %d OR cd_workflow.pipeline_id = %d)"+ + " WHERE ("+ + " (cd_workflow.pipeline_id = %d AND cd_workflow_runner.workflow_type = '%s')"+ + " OR"+ + " (cd_workflow.pipeline_id = %d"+ + " AND cd_workflow_runner.workflow_type = '%s'"+ + " AND cd_workflow_runner.status IN ('Healthy','Succeeded')"+ + " )"+ + " ) ) ", listingFilterOptions.PipelineId, listingFilterOptions.ParentId, listingFilterOptions.PipelineId, listingFilterOptions.StageType, listingFilterOptions.ParentId, listingFilterOptions.ParentStageType) + + whereCondition = fmt.Sprintf(" %s OR (ci_artifact.component_id = %d AND ci_artifact.data_source= '%s' ))", whereCondition, listingFilterOptions.ParentId, listingFilterOptions.PluginStage) + if listingFilterOptions.SearchString != EmptyLikeRegex { + whereCondition = whereCondition + fmt.Sprintf(" AND ci_artifact.image LIKE '%s' ", listingFilterOptions.SearchString) + } + if len(listingFilterOptions.ExcludeArtifactIds) > 0 { + whereCondition = whereCondition + fmt.Sprintf(" AND ( ci_artifact.id NOT IN (%s))", helper.GetCommaSepratedString(listingFilterOptions.ExcludeArtifactIds)) + } + + selectQuery := fmt.Sprintf(" SELECT ci_artifact.* ,COUNT(id) OVER() AS total_count " + + " FROM ci_artifact") + ordeyByAndPaginated := fmt.Sprintf(" ORDER BY id DESC LIMIT %d OFFSET %d ", listingFilterOptions.Limit, listingFilterOptions.Offset) + finalQuery := selectQuery + whereCondition + ordeyByAndPaginated + return finalQuery +} + func BuildQueryForArtifactsForRollback(listingFilterOptions bean.ArtifactsListFilterOptions) string { commonQuery := " FROM cd_workflow_runner cdwr " + " INNER JOIN cd_workflow cdw ON cdw.id=cdwr.cd_workflow_id " + diff --git a/pkg/pipeline/AppArtifactManager.go b/pkg/pipeline/AppArtifactManager.go index 1ded1d4c131..56840f6824a 100644 --- a/pkg/pipeline/AppArtifactManager.go +++ b/pkg/pipeline/AppArtifactManager.go @@ -25,6 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean2 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/user" "github.com/go-pg/pg" "go.uber.org/zap" @@ -56,6 +57,7 @@ type AppArtifactManagerImpl struct { ciArtifactRepository repository.CiArtifactRepository ciWorkflowRepository pipelineConfig.CiWorkflowRepository pipelineStageService PipelineStageService + config *types.CdConfig cdPipelineConfigService CdPipelineConfigService dockerArtifactRegistry dockerArtifactStoreRegistry.DockerArtifactStoreRepository CiPipelineRepository pipelineConfig.CiPipelineRepository @@ -74,7 +76,10 @@ func NewAppArtifactManagerImpl( dockerArtifactRegistry dockerArtifactStoreRegistry.DockerArtifactStoreRepository, CiPipelineRepository pipelineConfig.CiPipelineRepository, ciTemplateService CiTemplateService) *AppArtifactManagerImpl { - + cdConfig, err := types.GetCdConfig() + if err != nil { + return nil + } return &AppArtifactManagerImpl{ logger: logger, cdWorkflowRepository: cdWorkflowRepository, @@ -84,6 +89,7 @@ func NewAppArtifactManagerImpl( ciWorkflowRepository: ciWorkflowRepository, cdPipelineConfigService: cdPipelineConfigService, pipelineStageService: pipelineStageService, + config: cdConfig, dockerArtifactRegistry: dockerArtifactRegistry, CiPipelineRepository: CiPipelineRepository, ciTemplateService: ciTemplateService, @@ -606,11 +612,14 @@ func (impl *AppArtifactManagerImpl) RetrieveArtifactsByCDPipelineV2(pipeline *pi ciArtifactsResponse := &bean2.CiArtifactResponse{} artifactListingFilterOpts.PipelineId = pipeline.Id + // this will be 0 for external-ci cases, note: do not refer this for external-ci cases + artifactListingFilterOpts.CiPipelineId = pipeline.CiPipelineId artifactListingFilterOpts.ParentId = parentId artifactListingFilterOpts.ParentCdId = parentCdId artifactListingFilterOpts.ParentStageType = parentType artifactListingFilterOpts.StageType = stage artifactListingFilterOpts.SearchString = "%" + artifactListingFilterOpts.SearchString + "%" + artifactListingFilterOpts.UseCdStageQueryV2 = impl.config.UseArtifactListingQueryV2 ciArtifactsRefs, latestWfArtifactId, latestWfArtifactStatus, totalCount, err := impl.BuildArtifactsList(artifactListingFilterOpts) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting artifacts for child cd stage", "err", err, "stage", stage) diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index f17f5fbb654..ff491c91945 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -121,6 +121,7 @@ type CiCdConfig struct { BuildxProvenanceMode string `env:"BUILDX_PROVENANCE_MODE" envDefault:""` //provenance is set to false if this flag is not set ExtBlobStorageCmName string `env:"EXTERNAL_BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm"` ExtBlobStorageSecretName string `env:"EXTERNAL_BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"` + UseArtifactListingQueryV2 bool `env:"USE_ARTIFACT_LISTING_QUERY_V2" envDefault:"true"` } type CiConfig struct {