chore: change CI backend infra (#4251)

* wip

* wip

* wip

* added change ci functionalities from all pipeline to all other pipeline except externalwebhook

* added change ci from any ci-pipeline to external ci

* add nil ptr check

* comments added

* delete ci-pipeline code added

* function renamed

* some refactoring done around ciMaterial

* nil pointer handled

* code refactoring

* wip

* delete functionality fixes

* query fix

* query fix

* fixes

* material delete fix

* fix

* workflow existstence check made conditional

* sending prod env flag in cdpipelines response

* fixes

* linked cd count fix

* made delete templateOverride conditional for linked ci's

* ci change handling for linkedCD type in delete

* fix

* allow switch to external ci in all cases

* remove required tag

* make pipeline creation optional incase of external ci creation

* skip some pipeline creation validations if envId is not set in pipeline

* shadowed pipelineid fix

* sync

* fix

* addressed comments

* block new external-ci creation in existing wf unless it's a switch ci request

* remove unused condition

* remove duplicate flags

* move switch ci pipeline logic to separate function

* fix

* migrartion-linked-cd

* bump script

* bump sql script version

* fix

* duplicate external-ci and wfmapping creation fix

* script number bump

* bump script

* merge

* merge

* pr comments

* extract out switch ciPipeline logic to separate ci pipeline switch service

* removing rbac for switch ci

* review comments incorporated

* refactor

* fix

---------

Co-authored-by: Gireesh Naidu <gireesh@devtron.ai>
Co-authored-by: adi6859 <aditya.ar1909@gmail.com>

Wire.go

@@ -228,6 +228,8 @@ func InitializeApp() (*App, error) {
 
 		pipeline.NewPipelineBuilderImpl,
 		wire.Bind(new(pipeline.PipelineBuilder), new(*pipeline.PipelineBuilderImpl)),
+		pipeline.NewBuildPipelineSwitchServiceImpl,
+		wire.Bind(new(pipeline.BuildPipelineSwitchService), new(*pipeline.BuildPipelineSwitchServiceImpl)),
 		pipeline.NewCiPipelineConfigServiceImpl,
 		wire.Bind(new(pipeline.CiPipelineConfigService), new(*pipeline.CiPipelineConfigServiceImpl)),
 		pipeline.NewCiMaterialConfigServiceImpl,

api/bean/AppView.go

@@ -161,6 +161,7 @@ type DeploymentDetailContainer struct {
 	Deprecated                    bool            `json:"deprecated"`
 	K8sVersion                    string          `json:"k8sVersion"`
 	CiArtifactId                  int             `json:"ciArtifactId"`
+	ParentArtifactId              int             `json:"parentArtifactId"`
 	ClusterId                     int             `json:"clusterId"`
 	DeploymentAppType             string          `json:"deploymentAppType"`
 	CiPipelineId                  int             `json:"-"`

api/restHandler/app/BuildPipelineRestHandler.go

@@ -1101,7 +1101,7 @@ func (handler PipelineConfigRestHandlerImpl) GetCIPipelineById(w http.ResponseWr
 	if handler.appWorkflowService.CheckCdPipelineByCiPipelineId(pipelineId) {
 		for _, envId := range environmentIds {
 			envObject := handler.enforcerUtil.GetEnvRBACNameByCiPipelineIdAndEnvId(pipelineId, envId)
-			if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionUpdate, envObject); !ok {
+			if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionGet, envObject); !ok {
 				common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden)
 				return
 			}

api/restHandler/app/DeploymentPipelineRestHandler.go

@@ -20,6 +20,7 @@ import (
 	resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup"
 	"github.com/devtron-labs/devtron/pkg/resourceQualifiers"
 	"github.com/devtron-labs/devtron/pkg/user/casbin"
+	"github.com/devtron-labs/devtron/pkg/variables/models"
 	util2 "github.com/devtron-labs/devtron/util"
 	"github.com/go-pg/pg"
 	"github.com/gorilla/mux"
@@ -214,11 +215,13 @@ func (handler PipelineConfigRestHandlerImpl) CreateCdPipeline(w http.ResponseWri
 		return
 	}
 	for _, deploymentPipeline := range cdPipeline.Pipelines {
-		object := handler.enforcerUtil.GetAppRBACByAppNameAndEnvId(app.AppName, deploymentPipeline.EnvironmentId)
-		handler.Logger.Debugw("Triggered Request By:", "object", object)
-		if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionCreate, object); !ok {
-			common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden)
-			return
+		if deploymentPipeline.EnvironmentId > 0 {
+			object := handler.enforcerUtil.GetAppRBACByAppNameAndEnvId(app.AppName, deploymentPipeline.EnvironmentId)
+			handler.Logger.Debugw("Triggered Request By:", "object", object)
+			if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionCreate, object); !ok {
+				common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden)
+				return
+			}
 		}
 	}
 	//RBAC
@@ -325,7 +328,12 @@ func (handler PipelineConfigRestHandlerImpl) PatchCdPipeline(w http.ResponseWrit
 	createResp, err := handler.pipelineBuilder.PatchCdPipelines(&cdPipeline, ctx)
 	if err != nil {
 		handler.Logger.Errorw("service err, PatchCdPipeline", "err", err, "payload", cdPipeline)
-		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+
+		if errors.As(err, &models.ValidationError{}) {
+			common.WriteJsonResp(w, err, nil, http.StatusPreconditionFailed)
+		} else {
+			common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		}
 		return
 	}
 	common.WriteJsonResp(w, err, createResp, http.StatusOK)

api/restHandler/app/PipelineConfigRestHandler.go

@@ -64,7 +64,7 @@ import (
 )
 
 type PipelineRestHandlerEnvConfig struct {
-	UseArtifactListApiV2 bool `env:"USE_ARTIFACT_LISTING_API_V2"`
+	UseArtifactListApiV2 bool `env:"USE_ARTIFACT_LISTING_API_V2" envDefault:"true"`
 }
 
 type DevtronAppRestHandler interface {

internal/sql/repository/AppListingRepository.go

@@ -401,6 +401,7 @@ func (impl AppListingRepositoryImpl) deploymentDetailsByAppIdAndEnvId(ctx contex
 		" p.deployment_app_delete_request," +
 		" cia.data_source," +
 		" cia.id as ci_artifact_id," +
+		" cia.parent_ci_artifact as parent_artifact_id," +
 		" cl.k8s_version," +
 		" env.cluster_id," +
 		" env.is_virtual_environment," +

internal/sql/repository/CiArtifactRepository.go

@@ -92,8 +92,9 @@ type CiArtifactRepository interface {
 	GetLatestArtifactTimeByCiPipelineId(ciPipelineId int) (*CiArtifact, error)
 	GetArtifactsByCDPipelineV2(cdPipelineId int) ([]CiArtifact, error)
 	GetArtifactsByCDPipelineAndRunnerType(cdPipelineId int, runnerType bean.WorkflowType) ([]CiArtifact, error)
-	SaveAll(artifacts []*CiArtifact) error
+	SaveAll(artifacts []*CiArtifact) ([]*CiArtifact, error)
 	GetArtifactsByCiPipelineId(ciPipelineId int) ([]CiArtifact, error)
+	GetArtifactsByCiPipelineIds(ciPipelineIds []int) ([]CiArtifact, error)
 	FinDByParentCiArtifactAndCiId(parentCiArtifact int, ciPipelineIds []int) ([]*CiArtifact, error)
 	GetLatest(cdPipelineId int) (int, error)
 	GetByImageDigest(imageDigest string) (artifact *CiArtifact, err error)
@@ -104,6 +105,8 @@ type CiArtifactRepository interface {
 	FindArtifactByListFilter(listingFilterOptions *bean.ArtifactsListFilterOptions) ([]CiArtifact, int, error)
 	GetArtifactsByDataSourceAndComponentId(dataSource string, componentId int) ([]CiArtifact, error)
 	FindCiArtifactByImagePaths(images []string) ([]CiArtifact, error)
+
+	UpdateLatestTimestamp(artifactIds []int) error
 }
 
 type CiArtifactRepositoryImpl struct {
@@ -115,7 +118,7 @@ func NewCiArtifactRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger)
 	return &CiArtifactRepositoryImpl{dbConnection: dbConnection, logger: logger}
 }
 
-func (impl CiArtifactRepositoryImpl) SaveAll(artifacts []*CiArtifact) error {
+func (impl CiArtifactRepositoryImpl) SaveAll(artifacts []*CiArtifact) ([]*CiArtifact, error) {
 	err := impl.dbConnection.RunInTransaction(func(tx *pg.Tx) error {
 		for _, ciArtifact := range artifacts {
 			r, err := tx.Model(ciArtifact).Insert()
@@ -126,6 +129,18 @@ func (impl CiArtifactRepositoryImpl) SaveAll(artifacts []*CiArtifact) error {
 		}
 		return nil
 	})
+	return artifacts, err
+}
+
+func (impl CiArtifactRepositoryImpl) UpdateLatestTimestamp(artifactIds []int) error {
+	if len(artifactIds) == 0 {
+		impl.logger.Debug("UpdateLatestTimestamp empty list of artifacts, not updating")
+		return nil
+	}
+	_, err := impl.dbConnection.Model(&CiArtifact{}).
+		Set("updated_on = ?", time.Now()).
+		Where("id IN (?)", pg.In(artifactIds)).
+		Update()
 	return err
 }
 
@@ -602,6 +617,24 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCiPipelineId(ciPipelineId int
 	return artifacts, err
 }
 
+func (impl CiArtifactRepositoryImpl) GetArtifactsByCiPipelineIds(ciPipelineIds []int) ([]CiArtifact, error) {
+	var artifacts []CiArtifact
+	if len(ciPipelineIds) == 0 {
+		impl.logger.Debug("GetArtifactsByCiPipelineIds empty list of ids, returning empty list of artifacts")
+		return artifacts, nil
+	}
+	err := impl.dbConnection.
+		Model(&artifacts).
+		Column("ci_artifact.*").
+		Join("INNER JOIN ci_pipeline cp on cp.id=ci_artifact.pipeline_id").
+		Where("cp.id in (?)", pg.In(ciPipelineIds)).
+		Where("cp.deleted = ?", false).
+		Order("ci_artifact.id DESC").
+		Select()
+
+	return artifacts, err
+}
+
 func (impl CiArtifactRepositoryImpl) FinDByParentCiArtifactAndCiId(parentCiArtifact int, ciPipelineIds []int) ([]*CiArtifact, error) {
 	var CiArtifacts []*CiArtifact
 	err := impl.dbConnection.

internal/sql/repository/appWorkflow/AppWorkflowRepository.go

@@ -61,6 +61,8 @@ type AppWorkflowRepository interface {
 	FindByCDPipelineIds(cdPipelineIds []int) ([]*AppWorkflowMapping, error)
 	FindByWorkflowIds(workflowIds []int) ([]*AppWorkflowMapping, error)
 	FindMappingByAppIds(appIds []int) ([]*AppWorkflowMapping, error)
+	UpdateParentComponentDetails(tx *pg.Tx, oldComponentId int, oldComponentType string, newComponentId int, newComponentType string) error
+	FindWFMappingByComponent(componentType string, componentId int) (*AppWorkflowMapping, error)
 }
 
 type AppWorkflowRepositoryImpl struct {
@@ -411,6 +413,15 @@ func (impl AppWorkflowRepositoryImpl) FindWFCDMappingByExternalCiId(externalCiId
 		Select()
 	return models, err
 }
+func (impl AppWorkflowRepositoryImpl) FindWFMappingByComponent(componentType string, componentId int) (*AppWorkflowMapping, error) {
+	model := AppWorkflowMapping{}
+	err := impl.dbConnection.Model(&model).
+		Where("type = ?", componentType).
+		Where("component_id = ?", componentId).
+		Where("active = ?", true).
+		Select()
+	return &model, err
+}
 
 func (impl AppWorkflowRepositoryImpl) FindWFCDMappingByExternalCiIdByIdsIn(externalCiId []int) ([]*AppWorkflowMapping, error) {
 	var models []*AppWorkflowMapping
@@ -461,3 +472,19 @@ func (impl AppWorkflowRepositoryImpl) FindMappingByAppIds(appIds []int) ([]*AppW
 		Select()
 	return appWorkflowsMapping, err
 }
+
+func (impl AppWorkflowRepositoryImpl) UpdateParentComponentDetails(tx *pg.Tx, oldParentId int, oldParentType string, newParentId int, newParentType string) error {
+
+	/*updateQuery := fmt.Sprintf(" UPDATE app_workflow_mapping "+
+		" SET parent_type = (select type from new_app_workflow_mapping),parent_id = (select id from new_app_workflow_mapping) where parent_id = %v and parent_type='%v' and active = true", oldComponentId, oldComponentType)
+
+	finalQuery := withQuery + updateQuery*/
+	_, err := tx.Model((*AppWorkflowMapping)(nil)).
+		Set("parent_type = ?", newParentType).
+		Set("parent_id = ?", newParentId).
+		Where("parent_type = ?", oldParentType).
+		Where("parent_id = ?", oldParentId).
+		Where("active = true").
+		Update()
+	return err
+}

internal/sql/repository/pipelineConfig/CiPipelineRepository.go

@@ -84,6 +84,7 @@ type CiPipelineScript struct {
 }
 
 type CiPipelineRepository interface {
+	sql.TransactionWrapper
 	Save(pipeline *CiPipeline, tx *pg.Tx) error
 	SaveCiEnvMapping(cienvmapping *CiEnvMapping, tx *pg.Tx) error
 	SaveExternalCi(pipeline *ExternalCiPipeline, tx *pg.Tx) (*ExternalCiPipeline, error)
@@ -112,6 +113,7 @@ type CiPipelineRepository interface {
 	FindByName(pipelineName string) (pipeline *CiPipeline, err error)
 	CheckIfPipelineExistsByNameAndAppId(pipelineName string, appId int) (bool, error)
 	FindByParentCiPipelineId(parentCiPipelineId int) ([]*CiPipeline, error)
+	FindByParentIdAndType(parentCiPipelineId int, pipelineType string) ([]*CiPipeline, error)
 
 	FetchParentCiPipelinesForDG() ([]*CiPipelinesMap, error)
 	FetchCiPipelinesForDG(parentId int, childCiPipelineIds []int) (*CiPipeline, int, error)
@@ -125,16 +127,19 @@ type CiPipelineRepository interface {
 	FindAppIdsForCiPipelineIds(pipelineIds []int) (map[int]int, error)
 	GetCiPipelineByArtifactId(artifactId int) (*CiPipeline, error)
 	GetExternalCiPipelineByArtifactId(artifactId int) (*ExternalCiPipeline, error)
+	FindLinkedCiCount(ciPipelineId int) (int, error)
 }
 type CiPipelineRepositoryImpl struct {
 	dbConnection *pg.DB
 	logger       *zap.SugaredLogger
+	*sql.TransactionUtilImpl
 }
 
 func NewCiPipelineRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *CiPipelineRepositoryImpl {
 	return &CiPipelineRepositoryImpl{
-		dbConnection: dbConnection,
-		logger:       logger,
+		dbConnection:        dbConnection,
+		logger:              logger,
+		TransactionUtilImpl: sql.NewTransactionUtilImpl(dbConnection),
 	}
 }
 
@@ -147,6 +152,16 @@ func (impl CiPipelineRepositoryImpl) FindByParentCiPipelineId(parentCiPipelineId
 	return ciPipelines, err
 }
 
+func (impl CiPipelineRepositoryImpl) FindByParentIdAndType(parentCiPipelineId int, pipelineType string) ([]*CiPipeline, error) {
+	var ciPipelines []*CiPipeline
+	err := impl.dbConnection.Model(&ciPipelines).
+		Where("parent_ci_pipeline = ?", parentCiPipelineId).
+		Where("ci_pipeline_type = ?", pipelineType).
+		Where("active = ?", true).
+		Select()
+	return ciPipelines, err
+}
+
 func (impl CiPipelineRepositoryImpl) FindByIdsIn(ids []int) ([]*CiPipeline, error) {
 	var ciPipelines []*CiPipeline
 	err := impl.dbConnection.Model(&ciPipelines).
@@ -552,3 +567,15 @@ func (impl CiPipelineRepositoryImpl) FindCiPipelineByAppIdAndEnvIds(appId int, e
 	_, err := impl.dbConnection.Query(&pipelines, query, appId, pg.In(envIds))
 	return pipelines, err
 }
+
+func (impl CiPipelineRepositoryImpl) FindLinkedCiCount(ciPipelineId int) (int, error) {
+	pipeline := &CiPipeline{}
+	cnt, err := impl.dbConnection.Model(pipeline).
+		Where("parent_ci_pipeline = ?", ciPipelineId).
+		Where("deleted = ?", false).
+		Count()
+	if err == pg.ErrNoRows {
+		return 0, nil
+	}
+	return cnt, err
+}

internal/sql/repository/pipelineConfig/CiWorkflowRepository.go

@@ -82,8 +82,12 @@ type CiWorkflow struct {
 	CiPipeline              *CiPipeline
 }
 
-func (r *CiWorkflow) IsExternalRunInJobType() bool {
-	return r.EnvironmentId != 0
+func (ciWorkflow *CiWorkflow) InProgress() bool {
+	return ciWorkflow.Status == "Running" || ciWorkflow.Status == "Starting"
+}
+
+func (ciWorkflow *CiWorkflow) IsExternalRunInJobType() bool {
+	return ciWorkflow.EnvironmentId != 0
 }
 
 type WorkflowWithArtifact struct {

internal/sql/repository/pipelineConfig/PipelineRepository.go

@@ -28,6 +28,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/sql"
 	"github.com/go-pg/pg"
 	"go.uber.org/zap"
+	"k8s.io/utils/pointer"
 	"strconv"
 	"time"
 )
@@ -112,6 +113,7 @@ type PipelineRepository interface {
 	FindAppAndEnvironmentAndProjectByPipelineIds(pipelineIds []int) (pipelines []*Pipeline, err error)
 	FilterDeploymentDeleteRequestedPipelineIds(cdPipelineIds []int) (map[int]bool, error)
 	FindDeploymentTypeByPipelineIds(cdPipelineIds []int) (map[int]DeploymentObject, error)
+	UpdateOldCiPipelineIdToNewCiPipelineId(tx *pg.Tx, oldCiPipelineId, newCiPipelineId int) error
 }
 
 type CiArtifactDTO struct {
@@ -701,3 +703,14 @@ func (impl PipelineRepositoryImpl) FindDeploymentTypeByPipelineIds(cdPipelineIds
 
 	return pipelineIdsMap, nil
 }
+
+func (impl PipelineRepositoryImpl) UpdateOldCiPipelineIdToNewCiPipelineId(tx *pg.Tx, oldCiPipelineId, newCiPipelineId int) error {
+	newCiPipId := pointer.Int(newCiPipelineId)
+	if newCiPipelineId == 0 {
+		newCiPipId = nil
+	}
+	_, err := tx.Model((*Pipeline)(nil)).Set("ci_pipeline_id = ?", newCiPipId).
+		Where("ci_pipeline_id = ? ", oldCiPipelineId).
+		Where("deleted = ?", false).Update()
+	return err
+}