Merge branch 'main' into consumer-idemp-ad-cic

.github/workflows/pr-issue-validator.yaml

@@ -29,6 +29,10 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+
 
     - name: Validate Issue Reference
       env:
@@ -150,3 +154,84 @@ jobs:
             gh pr edit $PRNUM --remove-label "PR:Ready-to-Review"
             exit 1
         fi
+    - name: Check SQL file format and duplicates
+      shell: bash
+      env: 
+          pr_no: ${{ github.event.pull_request.number }}
+          GH_TOKEN: ${{ github.token }}
+      run: |
+        
+        # Fetch the latest changes from the main branch
+        git fetch origin main
+
+        # Get the list of changed files
+        git diff origin/main...HEAD --name-only > diff
+
+        echo "Changed files:"
+        cat diff
+
+        echo "Changed SQL files-:"
+        # Filter SQL files from the list of changed files
+        awk '/scripts\/sql\//' diff
+
+        # Count the number of changed SQL files in the 'scripts/sql' directory
+        count=$(awk '/scripts\/sql\//' diff | wc -l)
+
+        # Check if no SQL files were changed
+        if [[ $count == "0" ]]; then 
+          echo "No SQL files were added, Exiting from this action."
+          exit 0
+        fi
+
+        # Iterate through each changed SQL file
+        for filename in $(awk '/scripts\/sql\//' diff); do
+          echo "Checking File: $filename"
+          
+          # Check if the SQL file name is in the correct format (i.e., it ends with either '.up.sql' or '.down.sql')
+          if [[ "$filename" =~ \.(up|down)\.sql$ ]]; then
+
+            # Print a message that the file name is in the correct format
+            echo "File name: $filename is in the correct format"
+          else
+            # Print an error message
+            echo "Error: The SQL file name is not in the correct format: $filename."
+
+            # Post a comment on a GitHub pull request with the error message
+            gh pr comment $pr_no --body "The SQL file name: $filename is not in the correct format."
+
+            # Exit the script with a non-zero status code
+            exit 1 
+          fi
+
+          # Navigate to the SQL files directory
+          sql_dir="scripts/sql"
+          echo "Current directory: $(pwd)"
+          cd "$sql_dir"
+          echo "SQL files directory: $(pwd)"
+          
+          # Extract the migration number from the SQL file name
+          migration_no=$(echo "$filename" | cut -d "/" -f 3 | cut -d "_" -f 1)
+          echo "Migration Number: $migration_no"
+          
+          # Count the number of files with the same migration number
+          migration_files_present_of_this_no=$(ls | cut -d "_" -f 1 | grep -w -c "$migration_no")
+
+          # Navigate back to the original directory
+          cd ../..
+          
+          # Check the conditions based on the number of files with the same migration number
+          if [[ $migration_files_present_of_this_no == "2" ]]; then
+            echo "All looks good for this migration number."
+          elif [[ $migration_files_present_of_this_no == "1" ]]; then
+            # Only one file is present for this migration number
+            echo "Only single migration file was present for migration no.: $migration_no. either up or down migration is missing! EXITING"
+            gh pr comment $pr_no --body "Error: Only a single migration file was present for this number: $migration_no."
+            exit 1
+          else 
+            # Migration number is repeated
+            echo "Error: Migration number is repeated."
+            gh pr comment $pr_no --body "Error: The SQL file number: $migration_no is duplicated"
+            exit 1
+          fi
+        done
+

pkg/pipeline/CiService.go

@@ -245,6 +245,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error)
 	} else {
 		workflowRequest.Type = bean2.CI_WORKFLOW_PIPELINE_TYPE
 	}
+
 	err = impl.executeCiPipeline(workflowRequest)
 	if err != nil {
 		impl.Logger.Errorw("workflow error", "err", err)
@@ -659,6 +660,8 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.
 		RegistryDestinationImageMap: registryDestinationImageMap,
 		RegistryCredentialMap:       registryCredentialMap,
 		PluginArtifactStage:         pluginArtifactStage,
+		ImageScanMaxRetries:         impl.config.ImageScanMaxRetries,
+		ImageScanRetryDelay:         impl.config.ImageScanRetryDelay,
 	}
 
 	if dockerRegistry != nil {

pkg/pipeline/types/CiCdConfig.go

@@ -65,6 +65,8 @@ type CiCdConfig struct {
 	SkipCreatingEcrRepo              bool                                `env:"SKIP_CREATING_ECR_REPO" envDefault:"false"`
 	MaxCiWorkflowRetries             int                                 `env:"MAX_CI_WORKFLOW_RETRIES" envDefault:"0"`
 	NatsServerHost                   string                              `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222"`
+	ImageScanMaxRetries              int                                 `env:"IMAGE_SCAN_MAX_RETRIES" envDefault:"3"`
+	ImageScanRetryDelay              int                                 `env:"IMAGE_SCAN_RETRY_DELAY" envDefault:"5"`
 	// from CdConfig
 	CdLimitCpu                       string                              `env:"CD_LIMIT_CI_CPU" envDefault:"0.5"`
 	CdLimitMem                       string                              `env:"CD_LIMIT_CI_MEM" envDefault:"3G"`

pkg/pipeline/types/Workflow.go

@@ -126,6 +126,8 @@ type WorkflowRequest struct {
 	RegistryCredentialMap       map[string]plugin.RegistryCredentials `json:"registryCredentialMap"`
 	PluginArtifactStage         string                                `json:"pluginArtifactStage"`
 	PushImageBeforePostCI       bool                                  `json:"pushImageBeforePostCI"`
+	ImageScanMaxRetries         int                                   `json:"imageScanMaxRetries,omitempty"`
+	ImageScanRetryDelay         int                                   `json:"imageScanRetryDelay,omitempty"`
 	Type                        bean.WorkflowPipelineType
 	Pipeline                    *pipelineConfig.Pipeline
 	Env                         *repository.Environment

scripts/sql/211_image_scan_plugin_update.up.sql

@@ -0,0 +1,27 @@
+INSERT INTO "plugin_step_variable" ("id", "plugin_step_id", "name", "format", "description", "is_exposed", "allow_empty_value", "default_value","variable_type", "value_type", "variable_step_index",reference_variable_name, "deleted", "created_on", "created_by", "updated_on", "updated_by") VALUES
+        (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false), 'IMAGE_SCAN_MAX_RETRIES','STRING','image scan max retry count',true,true,'3','INPUT','GLOBAL',1 ,'IMAGE_SCAN_MAX_RETRIES','f','now()', 1, 'now()', 1),
+        (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerability Scanning' and ps."index"=1 and ps.deleted=false), 'IMAGE_SCAN_RETRY_DELAY','STRING','image scan retry delay (in seconds)',true,true,'5','INPUT','GLOBAL',1 ,'IMAGE_SCAN_RETRY_DELAY','f','now()', 1, 'now()', 1);
+
+UPDATE plugin_pipeline_script SET script = '#!/bin/sh
+echo "IMAGE SCAN"
+
+perform_curl_request() {
+    local attempt=1
+    while [ "$attempt" -le "$IMAGE_SCAN_MAX_RETRIES" ]; do
+        response=$(curl -s -w "\n%{http_code}" -X POST $IMAGE_SCANNER_ENDPOINT/scanner/image -H "Content-Type: application/json" -d "{\"image\": \"$DEST\", \"imageDigest\": \"$DIGEST\", \"pipelineId\" : $PIPELINE_ID, \"userId\": $TRIGGERED_BY, \"dockerRegistryId\": \"$DOCKER_REGISTRY_ID\" }")
+        http_status=$(echo "$response" | tail -n1)
+        if [ "$http_status" = "200" ]; then
+            echo "Vulnerability Scanning request successful."
+            return 0
+        else
+            echo "Attempt $attempt: Vulnerability Scanning request failed with HTTP status code $http_status"
+            echo "Response Body: $response"
+            attempt=$((attempt + 1))
+            sleep "$IMAGE_SCAN_RETRY_DELAY"
+        fi
+    done
+    echo -e "\033[1m======== Maximum retries reached. Vulnerability Scanning request failed ========"
+    exit 1
+}
+perform_curl_request'
+WHERE id = (SELECT id FROM plugin_metadata WHERE name = 'Vulnerability Scanning');

scripts/sql/212_dockerslim_plugin.down.sql

@@ -0,0 +1,6 @@
+DELETE FROM plugin_stage_mapping where plugin_id=(SELECT id from plugin_metadata where name='DockerSlim v1.0.0');
+DELETE FROM plugin_step where plugin_id=(SELECT id FROM plugin_metadata WHERE name='DockerSlim v1.0.0');
+DELETE FROM plugin_step_variable where plugin_step_id=(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='DockerSlim v1.0.0' and ps."index"=1 and ps.deleted=false);
+DELETE FROM pipeline_stage_step WHERE name ='DockerSlim v1.0.0';
+DELETE FROM plugin_tag_relation WHERE plugin_id=(SELECT id FROM plugin_metadata WHERE name='DockerSlim v1.0.0');
+DELETE FROM plugin_metadata where name='DockerSlim v1.0.0';

scripts/sql/212_dockerslim_plugin.up.sql

@@ -0,0 +1,58 @@
+INSERT INTO plugin_metadata (id,name,description,type,icon,deleted,created_on,created_by,updated_on,updated_by)
+VALUES (nextval('id_seq_plugin_metadata'),'DockerSlim v1.0.0','This plugin is used to Slim the docker images (Currently this plugin can be used only for docker images not for docker buildx images).','PRESET','https://raw.githubusercontent.com/devtron-labs/devtron/main/assets/dockerslim-plugin-icon.png',false,'now()',1,'now()',1);
+
+INSERT INTO "plugin_tag_relation" ("id", "tag_id", "plugin_id", "created_on", "created_by", "updated_on", "updated_by") 
+VALUES (nextval('id_seq_plugin_tag_relation'), (SELECT id FROM plugin_tag WHERE name='DevSecOps'), (SELECT id FROM plugin_metadata WHERE name='DockerSlim v1.0.0'),'now()', 1, 'now()', 1);
+
+INSERT INTO plugin_stage_mapping (id,plugin_id,stage_type,created_on,created_by,updated_on,updated_by)
+VALUES (nextval('id_seq_plugin_stage_mapping'),(SELECT id from plugin_metadata where name='DockerSlim v1.0.0'), 0,'now()',1,'now()',1);
+
+INSERT INTO "plugin_pipeline_script" ("id", "script","type","deleted","created_on", "created_by", "updated_on", "updated_by")
+VALUES (
+     nextval('id_seq_plugin_pipeline_script'),
+        $$#!/bin/sh
+httpProbe=$(echo "$HTTPProbe" | tr "[:upper:]" "[:lower:]")
+includeFilePath=$IncludePathFile
+
+export tag=$(echo $CI_CD_EVENT | jq --raw-output .commonWorkflowRequest.dockerImageTag)
+export repo=$(echo $CI_CD_EVENT | jq --raw-output .commonWorkflowRequest.dockerRepository)
+export registry=$(echo $CI_CD_EVENT | jq --raw-output .commonWorkflowRequest.dockerRegistryURL)
+
+cd /devtroncd
+
+docker pull dslim/slim
+
+if [ "$httpProbe" == "true" ]; then
+    if [ -n "$includeFilePath" ]; then
+        docker run -i --rm -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD dslim/slim build --http-probe=true --target $repo:$tag --tag $repo:$tag --continue-after=2 --include-path-file $includeFilePath
+    else
+        docker run -i --rm -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD dslim/slim build --http-probe=true --target $repo:$tag --tag $repo:$tag --continue-after=2
+    fi
+elif [ -n "$includeFilePath" ]; then
+    docker run -i --rm -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD dslim/slim build --http-probe=false --target $repo:$tag --tag $repo:$tag --continue-after=2 --include-path-file $includeFilePath
+else
+    docker run -i --rm -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD dslim/slim build --http-probe=false --target $repo:$tag --tag $repo:$tag --continue-after=2
+fi
+
+docker push $registry/$repo:$tag
+
+# Check the exit code of the last command
+if [ $? -eq 0 ]; then
+    echo "-----------***** Success: Docker-slim images built successfully *****-----------"
+else
+    echo "-----------***** Error: Docker-slim build failed, we are pushing original image to the container registry *****-----------"
+fi$$,
+        'SHELL',
+        'f',
+        'now()',
+        1,
+        'now()',
+        1
+);
+
+INSERT INTO "plugin_step" ("id", "plugin_id","name","description","index","step_type","script_id","deleted", "created_on", "created_by", "updated_on", "updated_by")
+VALUES (nextval('id_seq_plugin_step'), (SELECT id FROM plugin_metadata WHERE name='DockerSlim v1.0.0'),'Step 1','Step 1 - DockerSlim','1','INLINE',(SELECT last_value FROM id_seq_plugin_pipeline_script),'f','now()', 1, 'now()', 1);
+
+INSERT INTO plugin_step_variable (id,plugin_step_id,name,format,description,is_exposed,allow_empty_value,default_value,value,variable_type,value_type,previous_step_index,variable_step_index,variable_step_index_in_plugin,reference_variable_name,deleted,created_on,created_by,updated_on,updated_by) 
+VALUES (nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='DockerSlim v1.0.0' and ps."index"=1 and ps.deleted=false),'HTTPProbe','BOOL','Is port expose or not in Dockerfile','t','t',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='DockerSlim v1.0.0' and ps."index"=1 and ps.deleted=false),'IncludePathFile','STRING','File path contains including path for dockerslim build flag --include-path-file','t','t',null,null,'INPUT','NEW',null,1,null,null,'f','now()',1,'now()',1);

scripts/sql/213_eks_cluster_creation.down.sql

@@ -0,0 +1,6 @@
+DELETE FROM plugin_stage_mapping where plugin_id=(SELECT id from plugin_metadata where name='EKS Create Cluster v1.0.0');
+DELETE FROM plugin_step where plugin_id=(SELECT id FROM plugin_metadata WHERE name='EKS Create Cluster v1.0.0');
+DELETE FROM plugin_step_variable where plugin_step_id=(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='EKS Create Cluster v1.0.0' and ps."index"=1 and ps.deleted=false);
+DELETE FROM pipeline_stage_step WHERE name ='EKS Create Cluster v1.0.0';
+DELETE FROM plugin_tag_relation WHERE plugin_id=(SELECT id FROM plugin_metadata WHERE name='EKS Create Cluster v1.0.0');
+DELETE FROM plugin_metadata where name='EKS Create Cluster v1.0.0';