feat: upload and download logs/artifact from blob storage configured in external cluster

pkg/generateManifest/DeployementTemplateService_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	client2 "github.com/devtron-labs/authenticator/client"
+	"github.com/devtron-labs/common-lib/utils/k8s"
 	"github.com/devtron-labs/devtron/api/bean"
 	client "github.com/devtron-labs/devtron/api/helm-app"
 	mocks4 "github.com/devtron-labs/devtron/api/helm-app/mocks"
@@ -16,7 +17,6 @@ import (
 	"github.com/devtron-labs/devtron/pkg/chart/mocks"
 	chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository"
 	mocks5 "github.com/devtron-labs/devtron/pkg/chartRepo/repository/mocks"
-	"github.com/devtron-labs/devtron/util/k8s"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"testing"

pkg/pipeline/CiCdConfig.go

@@ -116,6 +116,8 @@ type CiCdConfig struct {
 	BaseLogLocationPath            string                       `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"`
 	InAppLoggingEnabled            bool                         `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"`
 	BuildxProvenanceMode           string                       `env:"BUILDX_PROVENANCE_MODE" envDefault:""` //provenance is set to false if this flag is not set
+	ExtBlobStorageCmName           string                       `env:"EXTERNAL_BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm"`
+	ExtBlobStorageSecretName       string                       `env:"EXTERNAL_BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"`
 }
 
 type CiConfig struct {

pkg/pipeline/CiService.go

@@ -593,9 +593,12 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.
 		ImageRetryCount:            impl.config.ImageRetryCount,
 		ImageRetryInterval:         impl.config.ImageRetryInterval,
 		WorkflowExecutor:           impl.config.GetWorkflowExecutorType(),
+		ExtBlobStorageSecretName:   impl.config.ExtBlobStorageSecretName,
+		ExtBlobStorageCmName:       impl.config.ExtBlobStorageCmName,
 		Type:                       bean2.CI_WORKFLOW_PIPELINE_TYPE,
 		CiArtifactLastFetch:        trigger.CiArtifactLastFetch,
 	}
+
 	if dockerRegistry != nil {
 
 		workflowRequest.DockerRegistryId = dockerRegistry.Id

pkg/pipeline/WorkflowDagExecutor.go

@@ -995,11 +995,13 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor
 			DataSource:   artifact.DataSource,
 			WorkflowId:   artifact.WorkflowId,
 		},
-		OrchestratorHost:  impl.config.OrchestratorHost,
-		OrchestratorToken: impl.config.OrchestratorToken,
-		CloudProvider:     impl.config.CloudProvider,
-		WorkflowExecutor:  workflowExecutor,
-		RefPlugins:        refPluginsData,
+		OrchestratorHost:         impl.config.OrchestratorHost,
+		OrchestratorToken:        impl.config.OrchestratorToken,
+		CloudProvider:            impl.config.CloudProvider,
+		WorkflowExecutor:         workflowExecutor,
+		RefPlugins:               refPluginsData,
+		ExtBlobStorageSecretName: impl.config.ExtBlobStorageSecretName,
+		ExtBlobStorageCmName:     impl.config.ExtBlobStorageCmName,
 	}
 
 	extraEnvVariables := make(map[string]string)

pkg/pipeline/WorkflowService.go

@@ -243,6 +243,24 @@ func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *Work
 			workflowSecrets = append(workflowSecrets, *secret)
 		}
 	}
+	//internally inducing BlobStorageCmName and BlobStorageSecretName for getting logs, caches and artifacts from
+	//in-cluster configured blob storage, if USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW = false and isExt = true
+	if workflowRequest.UseExternalClusterBlob {
+		blobDetailsConfigMap := bean.ConfigSecretMap{
+			Name:     impl.ciCdConfig.ExtBlobStorageCmName,
+			Type:     "environment",
+			External: true,
+		}
+		workflowConfigMaps = append(workflowConfigMaps, blobDetailsConfigMap)
+
+		blobDetailsSecret := bean.ConfigSecretMap{
+			Name:     impl.ciCdConfig.ExtBlobStorageSecretName,
+			Type:     "environment",
+			External: true,
+		}
+		workflowSecrets = append(workflowSecrets, blobDetailsSecret)
+	}
+
 	return workflowConfigMaps, workflowSecrets, nil
 }
 

pkg/pipeline/WorkflowUtils.go

@@ -366,6 +366,9 @@ type WorkflowRequest struct {
 	WorkflowExecutor         pipelineConfig.WorkflowExecutorType `json:"workflowExecutor"`
 	PrePostDeploySteps       []*bean.StepObject                  `json:"prePostDeploySteps"`
 	CiArtifactLastFetch      time.Time                           `json:"ciArtifactLastFetch"`
+	ExtBlobStorageCmName     string                              `json:"extBlobStorageCmName"`
+	ExtBlobStorageSecretName string                              `json:"extBlobStorageSecretName"`
+	UseExternalClusterBlob   bool                                `json:"useExternalClusterBlob"`
 	Type                     bean.WorkflowPipelineType
 	Pipeline                 *pipelineConfig.Pipeline
 	Env                      *repository2.Environment
@@ -437,6 +440,7 @@ func (workflowRequest *WorkflowRequest) GetBlobStorageLogsKey(config *CiCdConfig
 func (workflowRequest *WorkflowRequest) GetWorkflowJson(config *CiCdConfig) ([]byte, error) {
 	workflowRequest.updateBlobStorageLogsKey(config)
 	workflowRequest.updateExternalRunMetadata()
+	workflowRequest.UseExternalClusterBlob = !workflowRequest.CheckBlobStorageConfig(config) && workflowRequest.IsExtRun
 	workflowJson, err := workflowRequest.getWorkflowJson()
 	if err != nil {
 		return nil, err
@@ -471,10 +475,6 @@ func (workflowRequest *WorkflowRequest) getContainerEnvVariables(config *CiCdCon
 		workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE {
 		containerEnvVariables = []v12.EnvVar{{Name: "IMAGE_SCANNER_ENDPOINT", Value: config.ImageScannerEndpoint}}
 	}
-	if config.CloudProvider == BLOB_STORAGE_S3 && config.BlobStorageS3AccessKey != "" {
-		miniCred := []v12.EnvVar{{Name: "AWS_ACCESS_KEY_ID", Value: config.BlobStorageS3AccessKey}, {Name: "AWS_SECRET_ACCESS_KEY", Value: config.BlobStorageS3SecretKey}}
-		containerEnvVariables = append(containerEnvVariables, miniCred...)
-	}
 	eventEnv := v12.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)}
 	inAppLoggingEnv := v12.EnvVar{Name: "IN_APP_LOGGING", Value: strconv.FormatBool(workflowRequest.InAppLoggingEnabled)}
 	containerEnvVariables = append(containerEnvVariables, eventEnv, inAppLoggingEnv)