feat: Plugin for image scanning in Pre/Post step (#4021)

* image scanning plugin

* check for active ci_pipeline_material

* sql script number change

* sql script number change

* image scanning plugin check

* image scanning plugin check

* check for err

* abort print response

* add dockerregistryId

* script number update

* image scanning plugin name

* Image scanner endpoint for both CiCd

* Code review changes

pkg/pipeline/WebhookService.go

@@ -29,7 +29,10 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	util2 "github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/app"
+	"github.com/devtron-labs/devtron/pkg/pipeline/bean"
+	repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository"
 	types2 "github.com/devtron-labs/devtron/pkg/pipeline/types"
+	repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository"
 	"github.com/devtron-labs/devtron/pkg/sql"
 	"github.com/devtron-labs/devtron/util/event"
 	"github.com/go-pg/pg"
@@ -61,17 +64,19 @@ type WebhookService interface {
 }
 
 type WebhookServiceImpl struct {
-	ciArtifactRepository repository.CiArtifactRepository
-	ciConfig             *types2.CiConfig
-	logger               *zap.SugaredLogger
-	ciPipelineRepository pipelineConfig.CiPipelineRepository
-	ciWorkflowRepository pipelineConfig.CiWorkflowRepository
-	appService           app.AppService
-	eventClient          client.EventClient
-	eventFactory         client.EventFactory
-	workflowDagExecutor  WorkflowDagExecutor
-	ciHandler            CiHandler
-	customTagService     CustomTagService
+	ciArtifactRepository    repository.CiArtifactRepository
+	ciConfig                *types2.CiConfig
+	logger                  *zap.SugaredLogger
+	ciPipelineRepository    pipelineConfig.CiPipelineRepository
+	ciWorkflowRepository    pipelineConfig.CiWorkflowRepository
+	appService              app.AppService
+	eventClient             client.EventClient
+	eventFactory            client.EventFactory
+	workflowDagExecutor     WorkflowDagExecutor
+	ciHandler               CiHandler
+	pipelineStageRepository repository2.PipelineStageRepository
+	globalPluginRepository  repository3.GlobalPluginRepository
+	customTagService        CustomTagService
 }
 
 func NewWebhookServiceImpl(
@@ -81,19 +86,23 @@ func NewWebhookServiceImpl(
 	appService app.AppService, eventClient client.EventClient,
 	eventFactory client.EventFactory,
 	ciWorkflowRepository pipelineConfig.CiWorkflowRepository,
-	customTagService CustomTagService,
-	workflowDagExecutor WorkflowDagExecutor, ciHandler CiHandler) *WebhookServiceImpl {
+	workflowDagExecutor WorkflowDagExecutor, ciHandler CiHandler,
+	pipelineStageRepository repository2.PipelineStageRepository,
+	globalPluginRepository repository3.GlobalPluginRepository,
+	customTagService CustomTagService) *WebhookServiceImpl {
 	webhookHandler := &WebhookServiceImpl{
-		ciArtifactRepository: ciArtifactRepository,
-		logger:               logger,
-		ciPipelineRepository: ciPipelineRepository,
-		appService:           appService,
-		eventClient:          eventClient,
-		eventFactory:         eventFactory,
-		ciWorkflowRepository: ciWorkflowRepository,
-		workflowDagExecutor:  workflowDagExecutor,
-		ciHandler:            ciHandler,
-		customTagService:     customTagService,
+		ciArtifactRepository:    ciArtifactRepository,
+		logger:                  logger,
+		ciPipelineRepository:    ciPipelineRepository,
+		appService:              appService,
+		eventClient:             eventClient,
+		eventFactory:            eventFactory,
+		ciWorkflowRepository:    ciWorkflowRepository,
+		workflowDagExecutor:     workflowDagExecutor,
+		ciHandler:               ciHandler,
+		pipelineStageRepository: pipelineStageRepository,
+		globalPluginRepository:  globalPluginRepository,
+		customTagService:        customTagService,
 	}
 	config, err := types2.GetCiConfig()
 	if err != nil {
@@ -212,8 +221,19 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C
 		IsArtifactUploaded: request.IsArtifactUploaded,
 		AuditLog:           sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn},
 	}
-	if pipeline.ScanEnabled {
+	plugin, err := impl.globalPluginRepository.GetPluginByName(bean.VULNERABILITY_SCANNING_PLUGIN)
+	if err != nil || len(plugin) == 0 {
+		impl.logger.Errorw("error in getting image scanning plugin", "err", err)
+		return 0, err
+	}
+	isScanPluginConfigured, err := impl.pipelineStageRepository.CheckPluginExistsInCiPipeline(pipeline.Id, string(repository2.PIPELINE_STAGE_TYPE_POST_CI), plugin[0].Id)
+	if err != nil {
+		impl.logger.Errorw("error in getting ci pipeline plugin", "err", err, "pipelineId", pipeline.Id, "pluginId", plugin[0].Id)
+		return 0, err
+	}
+	if pipeline.ScanEnabled || isScanPluginConfigured {
 		artifact.Scanned = true
+		artifact.ScanEnabled = true
 	}
 	if err = impl.ciArtifactRepository.Save(artifact); err != nil {
 		impl.logger.Errorw("error in saving material", "err", err)

pkg/pipeline/WorkflowDagExecutor.go

@@ -1208,6 +1208,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor
 		cdStageWorkflowRequest.SecretKey = ciPipeline.CiTemplate.DockerRegistry.AWSSecretAccessKey
 		cdStageWorkflowRequest.DockerRegistryType = string(ciPipeline.CiTemplate.DockerRegistry.RegistryType)
 		cdStageWorkflowRequest.DockerRegistryURL = ciPipeline.CiTemplate.DockerRegistry.RegistryURL
+		cdStageWorkflowRequest.DockerRegistryId = ciPipeline.CiTemplate.DockerRegistry.Id
 		cdStageWorkflowRequest.CiPipelineType = ciPipeline.PipelineType
 	} else if cdPipeline.AppId > 0 {
 		ciTemplate, err := impl.CiTemplateRepository.FindByAppId(cdPipeline.AppId)
@@ -1225,6 +1226,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor
 		cdStageWorkflowRequest.DockerRegistryType = string(ciTemplate.DockerRegistry.RegistryType)
 		cdStageWorkflowRequest.DockerRegistryURL = ciTemplate.DockerRegistry.RegistryURL
 		appLabels, err := impl.appLabelRepository.FindAllByAppId(cdPipeline.AppId)
+		cdStageWorkflowRequest.DockerRegistryId = ciPipeline.CiTemplate.DockerRegistry.Id
 		if err != nil && err != pg.ErrNoRows {
 			impl.logger.Errorw("error in getting labels by appId", "err", err, "appId", cdPipeline.AppId)
 			return nil, err

pkg/pipeline/bean/pipelineStage.go

@@ -92,3 +92,7 @@ type PortMap struct {
 	PortOnLocal     int `json:"portOnLocal" validate:"number,gt=0"`
 	PortOnContainer int `json:"portOnContainer" validate:"number,gt=0"`
 }
+
+const (
+	VULNERABILITY_SCANNING_PLUGIN string = "Vulnerability Scanning"
+)

pkg/pipeline/bean/workFlowRequestBean.go

@@ -13,6 +13,7 @@ const (
 	VARIABLE_TYPE_REF_POST_CI = "REF_POST_CI"
 	VARIABLE_TYPE_REF_GLOBAL  = "REF_GLOBAL"
 	VARIABLE_TYPE_REF_PLUGIN  = "REF_PLUGIN"
+	IMAGE_SCANNER_ENDPOINT    = "IMAGE_SCANNER_ENDPOINT"
 )
 
 const CI_JOB string = "CI_JOB"

pkg/pipeline/repository/PipelineStageRepository.go

@@ -154,6 +154,7 @@ type PipelineStageRepository interface {
 	MarkPipelineStageStepsDeletedByStageId(stageId int, updatedBy int32, tx *pg.Tx) error
 	GetAllStepsByStageId(stageId int) ([]*PipelineStageStep, error)
 	GetAllCiPipelineIdsByPluginIdAndStageType(pluginId int, stageType string) ([]int, error)
+	CheckPluginExistsInCiPipeline(pipelineId int, stageType string, pluginId int) (bool, error)
 	GetStepById(stepId int) (*PipelineStageStep, error)
 	MarkStepsDeletedByStageId(stageId int) error
 	MarkStepsDeletedExcludingActiveStepsInUpdateReq(activeStepIdsPresentInReq []int, stageId int) error
@@ -394,6 +395,19 @@ func (impl *PipelineStageRepositoryImpl) GetAllCiPipelineIdsByPluginIdAndStageTy
 	return ciPipelineIds, nil
 }
 
+func (impl *PipelineStageRepositoryImpl) CheckPluginExistsInCiPipeline(pipelineId int, stageType string, pluginId int) (bool, error) {
+	var step PipelineStageStep
+	query := `Select * from pipeline_stage_step pss  
+		INNER JOIN pipeline_stage ps ON ps.id = pss.pipeline_stage_id  
+		where pss.ref_plugin_id = ? and ps.type = ? and pss.deleted = false and ps.deleted = false and ps.ci_pipeline_id= ?;`
+	_, err := impl.dbConnection.Query(&step, query, pluginId, stageType, pipelineId)
+	if err != nil {
+		impl.logger.Errorw("err in getting pipelineStageStep", "err", err, "pluginId", pluginId, "pipelineId", pipelineId, "stageType", stageType)
+		return false, err
+	}
+	return step.Id != 0, nil
+}
+
 func (impl *PipelineStageRepositoryImpl) MarkStepsDeletedByStageId(stageId int) error {
 	var step PipelineStageStep
 	_, err := impl.dbConnection.Model(&step).Set("deleted = ?", true).

pkg/pipeline/types/Workflow.go

@@ -223,10 +223,7 @@ func (workflowRequest *WorkflowRequest) GetWorkflowTypeForWorkflowRequest() stri
 }
 
 func (workflowRequest *WorkflowRequest) getContainerEnvVariables(config *CiCdConfig, workflowJson []byte) (containerEnvVariables []v1.EnvVar) {
-	if workflowRequest.Type == bean.CI_WORKFLOW_PIPELINE_TYPE ||
-		workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE {
-		containerEnvVariables = []v1.EnvVar{{Name: "IMAGE_SCANNER_ENDPOINT", Value: config.ImageScannerEndpoint}}
-	}
+	containerEnvVariables = []v1.EnvVar{{Name: bean.IMAGE_SCANNER_ENDPOINT, Value: config.ImageScannerEndpoint}}
 	eventEnv := v1.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)}
 	inAppLoggingEnv := v1.EnvVar{Name: "IN_APP_LOGGING", Value: strconv.FormatBool(workflowRequest.InAppLoggingEnabled)}
 	containerEnvVariables = append(containerEnvVariables, eventEnv, inAppLoggingEnv)

scripts/sql/184_image_scan_plugin.down.sql

@@ -0,0 +1,6 @@
+DELETE FROM plugin_step_variable WHERE plugin_step_id =(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false);
+DELETE FROM plugin_step WHERE plugin_id=(SELECT id FROM plugin_metadata WHERE name='Vulnerability Scanning');
+DELETE FROM plugin_stage_mapping WHERE plugin_id =(SELECT id FROM plugin_metadata WHERE name='Vulnerability Scanning');
+DELETE FROM pipeline_stage_step_variable WHERE pipeline_stage_step_id in (SELECT id FROM pipeline_stage_step where ref_plugin_id =(SELECT id from plugin_metadata WHERE name ='Vulnerability Scanning'));
+DELETE FROM pipeline_stage_step where ref_plugin_id in (SELECT id from plugin_metadata WHERE name ='Vulnerability Scanning');
+DELETE FROM plugin_metadata WHERE name ='Vulnerability Scanning';

scripts/sql/184_image_scan_plugin.up.sql

@@ -0,0 +1,39 @@
+INSERT INTO "plugin_metadata" ("id", "name", "description","type","icon","deleted", "created_on", "created_by", "updated_on", "updated_by")
+VALUES (nextval('id_seq_plugin_metadata'), 'Vulnerability Scanning','Scan a image','PRESET','https://raw.githubusercontent.com/devtron-labs/devtron/main/assets/ic-plugin-vulnerability-scan.png','f', 'now()', 1, 'now()', 1);
+
+INSERT INTO "plugin_stage_mapping" ("plugin_id","stage_type","created_on", "created_by", "updated_on", "updated_by")
+VALUES ((SELECT id FROM plugin_metadata WHERE name='Vulnerability Scanning'),0,'now()', 1, 'now()', 1);
+
+INSERT INTO "plugin_pipeline_script" ("id", "script", "type","deleted","created_on", "created_by", "updated_on", "updated_by")
+VALUES (nextval('id_seq_plugin_pipeline_script'),
+        '#!/bin/sh
+        echo "IMAGE SCAN"
+        curl -X POST $IMAGE_SCANNER_ENDPOINT/scanner/image -H "Content-Type: application/json" -d "{\"image\": \"$DEST\", \"imageDigest\": \"$DIGEST\", \"pipelineId\" : $PIPELINE_ID, \"userId\":
+ $TRIGGERED_BY, \"dockerRegistryId\": \"$DOCKER_REGISTRY_ID\" }" >/dev/null 2>&1
+        if [ $? != 0 ]
+        then
+        echo -e "\033[1m======== Vulnerability Scanning request failed ========"
+        exit 1
+        fi',
+        'SHELL',
+        'f',
+        'now()',
+        1,
+        'now()',
+        1);
+
+
+
+
+INSERT INTO "plugin_step" ("id", "plugin_id","name","description","index","step_type","script_id","deleted", "created_on", "created_by", "updated_on", "updated_by")
+VALUES (nextval('id_seq_plugin_step'), (SELECT id FROM plugin_metadata WHERE name='Vulnerability Scanning'),'Step 1','Step 1 - Vulnerability Scanning','1','INLINE',(SELECT last_value FROM id_seq_plugin_pipeline_script),'f','now()', 1, 'now()', 1);
+
+
+INSERT INTO "plugin_step_variable" ("id", "plugin_step_id", "name", "format", "description", "is_exposed", "allow_empty_value","variable_type", "value_type", "variable_step_index",reference_variable_name, "deleted", "created_on", "created_by", "updated_on", "updated_by") VALUES
+    (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false), 'DEST','STRING','image dest',false,true,'INPUT','GLOBAL',1 ,'DEST','f','now()', 1, 'now()', 1),
+    (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false), 'DIGEST','STRING','Image Digest',false,true,'INPUT','GLOBAL',1 ,'DIGEST','f','now()', 1, 'now()', 1),
+    (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false), 'PIPELINE_ID','STRING','Pipeline id',false,true,'INPUT','GLOBAL',1 ,'PIPELINE_ID','f','now()', 1, 'now()', 1),
+    (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false), 'TRIGGERED_BY','STRING','triggered by user',false,true,'INPUT','GLOBAL',1 ,'TRIGGERED_BY','f','now()', 1, 'now()', 1),
+    (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false), 'DOCKER_REGISTRY_ID','STRING','docker registry id',false,true,'INPUT','GLOBAL',1 ,'DOCKER_REGISTRY_ID','f','now()', 1, 'now()', 1),
+    (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false), 'IMAGE_SCANNER_ENDPOINT','STRING','image scanner endpoint',false,true,'INPUT','GLOBAL',1 ,'IMAGE_SCANNER_ENDPOINT','f','now()', 1, 'now()', 1);
+