From 72d3c3e063d88b2e078eb62c0adfb6643accdc6d Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 18 Oct 2023 11:09:09 +0530 Subject: [PATCH 01/25] blob storage cm secret inducing internally and removed AWS creds set as env variable(will be done in ci-runner) --- pkg/pipeline/CiCdConfig.go | 2 ++ pkg/pipeline/CiService.go | 2 ++ pkg/pipeline/WorkflowDagExecutor.go | 12 +++++++----- pkg/pipeline/WorkflowService.go | 18 ++++++++++++++++++ pkg/pipeline/WorkflowUtils.go | 6 ++---- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/pkg/pipeline/CiCdConfig.go b/pkg/pipeline/CiCdConfig.go index 6cf984bef94..9904e37fd71 100644 --- a/pkg/pipeline/CiCdConfig.go +++ b/pkg/pipeline/CiCdConfig.go @@ -116,6 +116,8 @@ type CiCdConfig struct { BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"` InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"` BuildxProvenanceMode string `env:"BUILDX_PROVENANCE_MODE" envDefault:""` //provenance is set to false if this flag is not set + BlobStorageCmName string `env:"BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm"` + BlobStorageSecretName string `env:"BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"` } type CiConfig struct { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 9ef337900ff..b836198f7b1 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -593,6 +593,8 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ImageRetryCount: impl.config.ImageRetryCount, ImageRetryInterval: impl.config.ImageRetryInterval, WorkflowExecutor: impl.config.GetWorkflowExecutorType(), + BlobStorageSecretName: impl.config.BlobStorageSecretName, + BlobStorageCmName: impl.config.BlobStorageCmName, Type: bean2.CI_WORKFLOW_PIPELINE_TYPE, CiArtifactLastFetch: trigger.CiArtifactLastFetch, } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 193a705b631..f217a619cc1 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -995,11 +995,13 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor DataSource: artifact.DataSource, WorkflowId: artifact.WorkflowId, }, - OrchestratorHost: impl.config.OrchestratorHost, - OrchestratorToken: impl.config.OrchestratorToken, - CloudProvider: impl.config.CloudProvider, - WorkflowExecutor: workflowExecutor, - RefPlugins: refPluginsData, + OrchestratorHost: impl.config.OrchestratorHost, + OrchestratorToken: impl.config.OrchestratorToken, + CloudProvider: impl.config.CloudProvider, + WorkflowExecutor: workflowExecutor, + RefPlugins: refPluginsData, + BlobStorageSecretName: impl.config.BlobStorageSecretName, + BlobStorageCmName: impl.config.BlobStorageCmName, } extraEnvVariables := make(map[string]string) diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 9d25a19c434..6150db95084 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -243,6 +243,24 @@ func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *Work workflowSecrets = append(workflowSecrets, *secret) } } + //internally inducing BlobStorageCmName and BlobStorageSecretName for getting logs, caches and artifacts from + //in-cluster configured blob storage + if workflowRequest.IsExtRun { + blobDetailsConfigMap := bean.ConfigSecretMap{ + Name: impl.ciCdConfig.BlobStorageCmName, + Type: "environment", + External: true, + } + workflowConfigMaps = append(workflowConfigMaps, blobDetailsConfigMap) + + blobDetailsSecret := bean.ConfigSecretMap{ + Name: impl.ciCdConfig.BlobStorageSecretName, + Type: "environment", + External: true, + } + workflowSecrets = append(workflowSecrets, blobDetailsSecret) + } + return workflowConfigMaps, workflowSecrets, nil } diff --git a/pkg/pipeline/WorkflowUtils.go b/pkg/pipeline/WorkflowUtils.go index c020464564d..fff1a80ed0a 100644 --- a/pkg/pipeline/WorkflowUtils.go +++ b/pkg/pipeline/WorkflowUtils.go @@ -366,6 +366,8 @@ type WorkflowRequest struct { WorkflowExecutor pipelineConfig.WorkflowExecutorType `json:"workflowExecutor"` PrePostDeploySteps []*bean.StepObject `json:"prePostDeploySteps"` CiArtifactLastFetch time.Time `json:"ciArtifactLastFetch"` + BlobStorageCmName string `json:"blobStorageCmName"` + BlobStorageSecretName string `json:"blobStorageSecretName"` Type bean.WorkflowPipelineType Pipeline *pipelineConfig.Pipeline Env *repository2.Environment @@ -471,10 +473,6 @@ func (workflowRequest *WorkflowRequest) getContainerEnvVariables(config *CiCdCon workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE { containerEnvVariables = []v12.EnvVar{{Name: "IMAGE_SCANNER_ENDPOINT", Value: config.ImageScannerEndpoint}} } - if config.CloudProvider == BLOB_STORAGE_S3 && config.BlobStorageS3AccessKey != "" { - miniCred := []v12.EnvVar{{Name: "AWS_ACCESS_KEY_ID", Value: config.BlobStorageS3AccessKey}, {Name: "AWS_SECRET_ACCESS_KEY", Value: config.BlobStorageS3SecretKey}} - containerEnvVariables = append(containerEnvVariables, miniCred...) - } eventEnv := v12.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)} inAppLoggingEnv := v12.EnvVar{Name: "IN_APP_LOGGING", Value: strconv.FormatBool(workflowRequest.InAppLoggingEnabled)} containerEnvVariables = append(containerEnvVariables, eventEnv, inAppLoggingEnv) From 7db0738d5e390a00bc544179dafba23d88f51aec Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 18 Oct 2023 23:19:07 +0530 Subject: [PATCH 02/25] some refactoring and UseExternalClusterBlob flag introduced in cicdconfig --- pkg/pipeline/CiCdConfig.go | 4 ++-- pkg/pipeline/CiService.go | 5 +++-- pkg/pipeline/WorkflowDagExecutor.go | 14 +++++++------- pkg/pipeline/WorkflowService.go | 9 +++++---- pkg/pipeline/WorkflowUtils.go | 5 +++-- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/pkg/pipeline/CiCdConfig.go b/pkg/pipeline/CiCdConfig.go index 9904e37fd71..abdc98dd640 100644 --- a/pkg/pipeline/CiCdConfig.go +++ b/pkg/pipeline/CiCdConfig.go @@ -116,8 +116,8 @@ type CiCdConfig struct { BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"` InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"` BuildxProvenanceMode string `env:"BUILDX_PROVENANCE_MODE" envDefault:""` //provenance is set to false if this flag is not set - BlobStorageCmName string `env:"BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm"` - BlobStorageSecretName string `env:"BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"` + ExtBlobStorageCmName string `env:"EXTERNAL_BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm"` + ExtBlobStorageSecretName string `env:"EXTERNAL_BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"` } type CiConfig struct { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index b836198f7b1..d697288bb04 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -593,11 +593,12 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ImageRetryCount: impl.config.ImageRetryCount, ImageRetryInterval: impl.config.ImageRetryInterval, WorkflowExecutor: impl.config.GetWorkflowExecutorType(), - BlobStorageSecretName: impl.config.BlobStorageSecretName, - BlobStorageCmName: impl.config.BlobStorageCmName, + ExtBlobStorageSecretName: impl.config.ExtBlobStorageSecretName, + ExtBlobStorageCmName: impl.config.ExtBlobStorageCmName, Type: bean2.CI_WORKFLOW_PIPELINE_TYPE, CiArtifactLastFetch: trigger.CiArtifactLastFetch, } + if dockerRegistry != nil { workflowRequest.DockerRegistryId = dockerRegistry.Id diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index f217a619cc1..f28dd582632 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -995,13 +995,13 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor DataSource: artifact.DataSource, WorkflowId: artifact.WorkflowId, }, - OrchestratorHost: impl.config.OrchestratorHost, - OrchestratorToken: impl.config.OrchestratorToken, - CloudProvider: impl.config.CloudProvider, - WorkflowExecutor: workflowExecutor, - RefPlugins: refPluginsData, - BlobStorageSecretName: impl.config.BlobStorageSecretName, - BlobStorageCmName: impl.config.BlobStorageCmName, + OrchestratorHost: impl.config.OrchestratorHost, + OrchestratorToken: impl.config.OrchestratorToken, + CloudProvider: impl.config.CloudProvider, + WorkflowExecutor: workflowExecutor, + RefPlugins: refPluginsData, + ExtBlobStorageSecretName: impl.config.ExtBlobStorageSecretName, + ExtBlobStorageCmName: impl.config.ExtBlobStorageCmName, } extraEnvVariables := make(map[string]string) diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 6150db95084..20f8d8647a2 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -116,6 +116,7 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest } func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *WorkflowRequest) (bean3.WorkflowTemplate, error) { + workflowRequest.UseExternalClusterBlob = !workflowRequest.CheckBlobStorageConfig(impl.ciCdConfig) && workflowRequest.IsExtRun workflowJson, err := workflowRequest.GetWorkflowJson(impl.ciCdConfig) if err != nil { impl.Logger.Errorw("error occurred while getting workflow json", "err", err) @@ -244,17 +245,17 @@ func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *Work } } //internally inducing BlobStorageCmName and BlobStorageSecretName for getting logs, caches and artifacts from - //in-cluster configured blob storage - if workflowRequest.IsExtRun { + //in-cluster configured blob storage, if USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW = false and isExt = true + if workflowRequest.UseExternalClusterBlob { blobDetailsConfigMap := bean.ConfigSecretMap{ - Name: impl.ciCdConfig.BlobStorageCmName, + Name: impl.ciCdConfig.ExtBlobStorageCmName, Type: "environment", External: true, } workflowConfigMaps = append(workflowConfigMaps, blobDetailsConfigMap) blobDetailsSecret := bean.ConfigSecretMap{ - Name: impl.ciCdConfig.BlobStorageSecretName, + Name: impl.ciCdConfig.ExtBlobStorageSecretName, Type: "environment", External: true, } diff --git a/pkg/pipeline/WorkflowUtils.go b/pkg/pipeline/WorkflowUtils.go index fff1a80ed0a..161f829c4f7 100644 --- a/pkg/pipeline/WorkflowUtils.go +++ b/pkg/pipeline/WorkflowUtils.go @@ -366,8 +366,9 @@ type WorkflowRequest struct { WorkflowExecutor pipelineConfig.WorkflowExecutorType `json:"workflowExecutor"` PrePostDeploySteps []*bean.StepObject `json:"prePostDeploySteps"` CiArtifactLastFetch time.Time `json:"ciArtifactLastFetch"` - BlobStorageCmName string `json:"blobStorageCmName"` - BlobStorageSecretName string `json:"blobStorageSecretName"` + ExtBlobStorageCmName string `json:"extBlobStorageCmName"` + ExtBlobStorageSecretName string `json:"extBlobStorageSecretName"` + UseExternalClusterBlob bool `json:"useExternalClusterBlob"` Type bean.WorkflowPipelineType Pipeline *pipelineConfig.Pipeline Env *repository2.Environment From a23d3e851b3535454b30e69d17daeb32dd28d9d0 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 20 Oct 2023 15:11:57 +0530 Subject: [PATCH 03/25] added support fr fetching logs and artifacts from external cluster blob storage --- pkg/pipeline/CdHandler.go | 152 +++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 4 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index b2dbb3913bb..60c1568ca35 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -20,6 +20,7 @@ package pipeline import ( "bufio" "context" + "encoding/json" "errors" "fmt" application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" @@ -52,6 +53,8 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" + errors2 "k8s.io/apimachinery/pkg/api/errors" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" "os" "path/filepath" "strconv" @@ -118,9 +121,10 @@ type CdHandlerImpl struct { k8sUtil *k8s.K8sUtil workflowService WorkflowService config *CdConfig + clusterRepository repository2.ClusterRepository } -func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService) *CdHandlerImpl { +func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService, clusterRepository repository2.ClusterRepository) *CdHandlerImpl { cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, @@ -154,6 +158,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, c imageTaggingService: imageTaggingService, k8sUtil: k8sUtil, workflowService: workflowService, + clusterRepository: clusterRepository, } config, err := GetCdConfig() if err != nil { @@ -169,6 +174,29 @@ type ArgoPipelineStatusSyncEvent struct { IsAppStoreApplication bool `json:"isAppStoreApplication"` } +type CmBlobStorageConfig struct { + //AWS credentials + CloudProvider blob_storage.BlobStorageType `json:"BLOB_STORAGE_PROVIDER"` + S3AccessKey string `json:"BLOB_STORAGE_S3_ACCESS_KEY"` + S3Endpoint string `json:"BLOB_STORAGE_S3_ENDPOINT"` + S3EndpointInsecure bool `json:"BLOB_STORAGE_S3_ENDPOINT_INSECURE"` + S3BucketVersioned bool `json:"BLOB_STORAGE_S3_BUCKET_VERSIONED"` + + //Azure credentials + AzureAccountName string `json:"AZURE_ACCOUNT_NAME"` + AzureGatewayUrl string `json:"AZURE_GATEWAY_URL"` + AzureGatewayConnectionInsecure bool `json:"AZURE_GATEWAY_CONNECTION_INSECURE"` +} + +type SecretBlobStorageConfig struct { + //aws + S3SecretKey string `json:"BLOB_STORAGE_S3_SECRET_KEY"` + //gcp + GcpBlobStorageCredentialJson string `json:"BLOB_STORAGE_GCP_CREDENTIALS_JSON"` + //azure + AzureAccountKey string `json:"AZURE_ACCOUNT_KEY"` +} + const NotTriggered string = "Not Triggered" const NotDeployed = "Not Deployed" const WorklowTypeDeploy = "DEPLOY" @@ -909,7 +937,7 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC return nil, nil, errors.New("logs-not-stored-in-repository") } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == WorkflowCancel { impl.Logger.Debugw("pod is not live ", "err", err) - return impl.getLogsFromRepository(pipelineId, cdWorkflow) + return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) } impl.Logger.Errorw("err on fetch workflow logs", "err", err) return nil, nil, err @@ -918,7 +946,7 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC return logReader, cleanUp, err } -func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner) (*bufio.Reader, func() error, error) { +func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.Logger.Debug("getting historic logs") cdConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(pipelineId) @@ -960,6 +988,29 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } + + useExternalBlobStorage := isExt && !impl.config.UseBlobStorageConfigInCdWorkflow + if useExternalBlobStorage { + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + cdLogRequest.CloudProvider = cmConfig.CloudProvider + cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName + cdLogRequest.AzureBlobConfig.AccountKey = secretConfig.AzureAccountKey + + cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson + + cdLogRequest.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + cdLogRequest.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + cdLogRequest.AwsS3BaseConfig.IsInSecure = cmConfig.S3EndpointInsecure + cdLogRequest.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey + + } + impl.Logger.Infow("s3 log req ", "req", cdLogRequest) oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) if err != nil { @@ -970,6 +1021,59 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip return logReader, cleanUp, err } +func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (CmBlobStorageConfig, SecretBlobStorageConfig, error) { + var cmConfig CmBlobStorageConfig + var secretConfig SecretBlobStorageConfig + _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(clusterConfig) + if err != nil { + impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) + return cmConfig, secretConfig, err + } + cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), impl.config.ExtBlobStorageCmName, v12.GetOptions{}) + if err != nil && !errors2.IsNotFound(err) { + impl.Logger.Errorw("error in getting config map in external cluster", "err", err, "blobStorageCmName", impl.config.ExtBlobStorageCmName, "clusterName", clusterConfig.ClusterName) + return cmConfig, secretConfig, err + } + if errors2.IsNotFound(err) { + cmNotFoundErr := fmt.Sprintf("config map: %s not found in namespace: %s", impl.config.ExtBlobStorageCmName, namespace) + return cmConfig, secretConfig, errors.New(cmNotFoundErr) + } + secret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), impl.config.ExtBlobStorageSecretName, v12.GetOptions{}) + if err != nil && !errors2.IsNotFound(err) { + impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.config.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) + return cmConfig, secretConfig, err + } + if errors2.IsNotFound(err) { + secretNotFoundErr := fmt.Sprintf("secret: %s not found in namespace: %s", impl.config.ExtBlobStorageSecretName, namespace) + return cmConfig, secretConfig, errors.New(secretNotFoundErr) + } + if cm.Data != nil && secret != nil { + cmDataJson, err := json.Marshal(cm.Data) + if err != nil { + fmt.Println("error marshalling external blob storage cm data to json:", err) + return cmConfig, secretConfig, err + } + err = json.Unmarshal(cmDataJson, &cmConfig) + if err != nil { + fmt.Println("error unmarshalling external blob storage cm json to struct:", err) + return cmConfig, secretConfig, err + } + + secretDataJson, err := json.Marshal(secret.Data) + if err != nil { + fmt.Println("error marshalling external blob storage secret data to json:", err) + return cmConfig, secretConfig, err + } + + err = json.Unmarshal(secretDataJson, &secretConfig) + if err != nil { + fmt.Println("error unmarshalling external blob storage secret json to struct:", err) + return cmConfig, secretConfig, err + } + } + return cmConfig, secretConfig, nil +} + func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (WorkflowResponse, error) { workflowR, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil && err != pg.ErrNoRows { @@ -1061,7 +1165,27 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) return nil, err } - + var clusterBean cluster.ClusterBean + if wfr.CdWorkflow.Pipeline.Environment.Id != 0 { + clusterFromDb, err := impl.clusterRepository.FindById(wfr.CdWorkflow.Pipeline.Environment.ClusterId) + if err != nil { + impl.Logger.Errorw("unable to fetch cluster by cluster Id", "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId, "err", err) + return nil, err + } + clusterBean = cluster.GetClusterBean(*clusterFromDb) + } + clusterConfig, err := clusterBean.GetClusterConfig() + if err != nil { + impl.Logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) + return nil, err + } + var isExtCluster bool + if wfr.WorkflowType == PRE { + isExtCluster = wfr.CdWorkflow.Pipeline.RunPreStageInEnv + } else if wfr.WorkflowType == POST { + isExtCluster = wfr.CdWorkflow.Pipeline.RunPostStageInEnv + } + useExternalBlobStorage := isExtCluster && !impl.config.UseBlobStorageConfigInCdWorkflow if !wfr.BlobStorageEnabled { return nil, errors.New("logs-not-stored-in-repository") } @@ -1111,6 +1235,26 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i AwsS3BaseConfig: awsS3BaseConfig, GcpBlobBaseConfig: gcpBlobBaseConfig, } + if useExternalBlobStorage { + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request.StorageType = cmConfig.CloudProvider + + request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + request.AwsS3BaseConfig.IsInSecure = cmConfig.S3EndpointInsecure + request.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey + + request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName + request.AzureBlobBaseConfig.AccountKey = secretConfig.AzureAccountKey + + request.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson + } _, numBytes, err := blobStorageService.Get(request) if err != nil { impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) From 70cf43ab9eaa1d08c48b9ef25d459bc159308de8 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 20 Oct 2023 15:16:41 +0530 Subject: [PATCH 04/25] wire --- wire_gen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wire_gen.go b/wire_gen.go index 6e761e3f7d3..07362974c1f 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -503,7 +503,7 @@ func InitializeApp() (*App, error) { linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterRepositoryImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, appWorkflowRepositoryImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) From 06ccbd12d201cd7f7c0140ecb4d77a3f32024482 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 20 Oct 2023 16:56:39 +0530 Subject: [PATCH 05/25] minor fix --- pkg/pipeline/WorkflowService.go | 1 - pkg/pipeline/WorkflowUtils.go | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 485adf55300..7f1e5ff11d7 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -116,7 +116,6 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest } func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *WorkflowRequest) (bean3.WorkflowTemplate, error) { - workflowRequest.UseExternalClusterBlob = !workflowRequest.CheckBlobStorageConfig(impl.ciCdConfig) && workflowRequest.IsExtRun workflowJson, err := workflowRequest.GetWorkflowJson(impl.ciCdConfig) if err != nil { impl.Logger.Errorw("error occurred while getting workflow json", "err", err) diff --git a/pkg/pipeline/WorkflowUtils.go b/pkg/pipeline/WorkflowUtils.go index 161f829c4f7..3078d16ba27 100644 --- a/pkg/pipeline/WorkflowUtils.go +++ b/pkg/pipeline/WorkflowUtils.go @@ -440,6 +440,7 @@ func (workflowRequest *WorkflowRequest) GetBlobStorageLogsKey(config *CiCdConfig func (workflowRequest *WorkflowRequest) GetWorkflowJson(config *CiCdConfig) ([]byte, error) { workflowRequest.updateBlobStorageLogsKey(config) workflowRequest.updateExternalRunMetadata() + workflowRequest.UseExternalClusterBlob = !workflowRequest.CheckBlobStorageConfig(config) && workflowRequest.IsExtRun workflowJson, err := workflowRequest.getWorkflowJson() if err != nil { return nil, err From 0915bd27c906231deb2762dd66ea209cf3ebdec5 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 20 Oct 2023 16:59:29 +0530 Subject: [PATCH 06/25] import fix --- pkg/generateManifest/DeployementTemplateService_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/generateManifest/DeployementTemplateService_test.go b/pkg/generateManifest/DeployementTemplateService_test.go index ad97dcc5b68..3552db8fce1 100644 --- a/pkg/generateManifest/DeployementTemplateService_test.go +++ b/pkg/generateManifest/DeployementTemplateService_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" client2 "github.com/devtron-labs/authenticator/client" + "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/api/helm-app" mocks4 "github.com/devtron-labs/devtron/api/helm-app/mocks" @@ -16,7 +17,6 @@ import ( "github.com/devtron-labs/devtron/pkg/chart/mocks" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" mocks5 "github.com/devtron-labs/devtron/pkg/chartRepo/repository/mocks" - "github.com/devtron-labs/devtron/util/k8s" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "testing" From 92557ad22e1473d597d1ba1ffc7e4de3871aa3f8 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 20 Oct 2023 19:08:34 +0530 Subject: [PATCH 07/25] code refactoring --- pkg/pipeline/CdHandler.go | 119 ++++++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 45 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 826365852bc..e5b5e472f7b 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -180,13 +180,27 @@ type CmBlobStorageConfig struct { CloudProvider blob_storage.BlobStorageType `json:"BLOB_STORAGE_PROVIDER"` S3AccessKey string `json:"BLOB_STORAGE_S3_ACCESS_KEY"` S3Endpoint string `json:"BLOB_STORAGE_S3_ENDPOINT"` - S3EndpointInsecure bool `json:"BLOB_STORAGE_S3_ENDPOINT_INSECURE"` - S3BucketVersioned bool `json:"BLOB_STORAGE_S3_BUCKET_VERSIONED"` + S3EndpointInsecure string `json:"BLOB_STORAGE_S3_ENDPOINT_INSECURE"` + S3BucketVersioned string `json:"BLOB_STORAGE_S3_BUCKET_VERSIONED"` //Azure credentials AzureAccountName string `json:"AZURE_ACCOUNT_NAME"` AzureGatewayUrl string `json:"AZURE_GATEWAY_URL"` - AzureGatewayConnectionInsecure bool `json:"AZURE_GATEWAY_CONNECTION_INSECURE"` + AzureGatewayConnectionInsecure string `json:"AZURE_GATEWAY_CONNECTION_INSECURE"` +} + +func (c CmBlobStorageConfig) PopulateWithK8sExtBlobCmData(cm map[string]string) (CmBlobStorageConfig, error) { + cmDataJson, err := json.Marshal(cm) + if err != nil { + fmt.Println("error marshalling external blob storage cm data to json:", err) + return c, err + } + err = json.Unmarshal(cmDataJson, &c) + if err != nil { + fmt.Println("error unmarshalling external blob storage cm json to struct:", err) + return c, err + } + return c, nil } type SecretBlobStorageConfig struct { @@ -198,6 +212,20 @@ type SecretBlobStorageConfig struct { AzureAccountKey string `json:"AZURE_ACCOUNT_KEY"` } +func (s SecretBlobStorageConfig) PopulateWithK8sExtBlobSecretData(secret map[string][]byte) (SecretBlobStorageConfig, error) { + cmDataJson, err := json.Marshal(secret) + if err != nil { + fmt.Println("error marshalling external blob storage secret data to json:", err) + return s, err + } + err = json.Unmarshal(cmDataJson, &s) + if err != nil { + fmt.Println("error unmarshalling external blob storage secret json to struct:", err) + return s, err + } + return s, nil +} + const NotTriggered string = "Not Triggered" const NotDeployed = "Not Deployed" const WorklowTypeDeploy = "DEPLOY" @@ -1002,17 +1030,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) return nil, nil, err } - cdLogRequest.CloudProvider = cmConfig.CloudProvider - cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName - cdLogRequest.AzureBlobConfig.AccountKey = secretConfig.AzureAccountKey - - cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson - - cdLogRequest.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey - cdLogRequest.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - cdLogRequest.AwsS3BaseConfig.IsInSecure = cmConfig.S3EndpointInsecure - cdLogRequest.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey - + cdLogRequest = assignNewBlobStorageConfigInCdLogRequest(&cdLogRequest, cmConfig, secretConfig) } impl.Logger.Infow("s3 log req ", "req", cdLogRequest) @@ -1024,6 +1042,21 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip logReader := bufio.NewReader(oldLogsStream) return logReader, cleanUp, err } +func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig CmBlobStorageConfig, secretConfig SecretBlobStorageConfig) BuildLogRequest { + cdLogRequest.CloudProvider = cmConfig.CloudProvider + cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName + cdLogRequest.AzureBlobConfig.AccountKey = secretConfig.AzureAccountKey + + cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson + + cdLogRequest.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + cdLogRequest.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + cdLogRequest.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey + isEndpointInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + cdLogRequest.AwsS3BaseConfig.IsInSecure = isEndpointInSecure + + return *cdLogRequest +} func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (CmBlobStorageConfig, SecretBlobStorageConfig, error) { var cmConfig CmBlobStorageConfig @@ -1052,29 +1085,23 @@ func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(cluster return cmConfig, secretConfig, errors.New(secretNotFoundErr) } if cm.Data != nil && secret != nil { - cmDataJson, err := json.Marshal(cm.Data) + cmConfig, err = cmConfig.PopulateWithK8sExtBlobCmData(cm.Data) if err != nil { - fmt.Println("error marshalling external blob storage cm data to json:", err) + fmt.Println("error marshalling external blob storage cm data to struct:", err) return cmConfig, secretConfig, err } - err = json.Unmarshal(cmDataJson, &cmConfig) - if err != nil { - fmt.Println("error unmarshalling external blob storage cm json to struct:", err) - return cmConfig, secretConfig, err - } - - secretDataJson, err := json.Marshal(secret.Data) + secretConfig, err = secretConfig.PopulateWithK8sExtBlobSecretData(secret.Data) if err != nil { - fmt.Println("error marshalling external blob storage secret data to json:", err) - return cmConfig, secretConfig, err - } - - err = json.Unmarshal(secretDataJson, &secretConfig) - if err != nil { - fmt.Println("error unmarshalling external blob storage secret json to struct:", err) + fmt.Println("error marshalling external blob storage secret data to struct:", err) return cmConfig, secretConfig, err } } + if cm.Data == nil { + return cmConfig, secretConfig, errors.New("Data field not found in config map") + } + if secret.Data == nil { + return cmConfig, secretConfig, errors.New("Data field not found in secret") + } return cmConfig, secretConfig, nil } @@ -1169,26 +1196,13 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) return nil, err } - var clusterBean cluster.ClusterBean - if wfr.CdWorkflow.Pipeline.Environment.Id != 0 { - clusterFromDb, err := impl.clusterRepository.FindById(wfr.CdWorkflow.Pipeline.Environment.ClusterId) - if err != nil { - impl.Logger.Errorw("unable to fetch cluster by cluster Id", "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId, "err", err) - return nil, err - } - clusterBean = cluster.GetClusterBean(*clusterFromDb) - } - clusterConfig, err := clusterBean.GetClusterConfig() - if err != nil { - impl.Logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) - return nil, err - } var isExtCluster bool if wfr.WorkflowType == PRE { isExtCluster = wfr.CdWorkflow.Pipeline.RunPreStageInEnv } else if wfr.WorkflowType == POST { isExtCluster = wfr.CdWorkflow.Pipeline.RunPostStageInEnv } + //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well useExternalBlobStorage := isExtCluster && !impl.config.UseBlobStorageConfigInCdWorkflow if !wfr.BlobStorageEnabled { return nil, errors.New("logs-not-stored-in-repository") @@ -1240,6 +1254,20 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i GcpBlobBaseConfig: gcpBlobBaseConfig, } if useExternalBlobStorage { + var clusterBean cluster.ClusterBean + if wfr.CdWorkflow.Pipeline.Environment.Id != 0 { + clusterFromDb, err := impl.clusterRepository.FindById(wfr.CdWorkflow.Pipeline.Environment.ClusterId) + if err != nil { + impl.Logger.Errorw("unable to fetch cluster by cluster Id", "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId, "err", err) + return nil, err + } + clusterBean = cluster.GetClusterBean(*clusterFromDb) + } + clusterConfig, err := clusterBean.GetClusterConfig() + if err != nil { + impl.Logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) + return nil, err + } //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds //from them else return. cmConfig, secretConfig, err := impl.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) @@ -1251,8 +1279,9 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - request.AwsS3BaseConfig.IsInSecure = cmConfig.S3EndpointInsecure request.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey + isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + request.AwsS3BaseConfig.IsInSecure = isInSecure request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName request.AzureBlobBaseConfig.AccountKey = secretConfig.AzureAccountKey From bc249fddea82c9484a9999ce023d62a757c36c9c Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 24 Oct 2023 18:35:45 +0530 Subject: [PATCH 08/25] info comment --- pkg/pipeline/CdHandler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index e5b5e472f7b..ac498d1cc10 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -1043,6 +1043,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip return logReader, cleanUp, err } func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig CmBlobStorageConfig, secretConfig SecretBlobStorageConfig) BuildLogRequest { + //TODO maybe we need to put a check here because at time of parsing for other blob values can be empty cdLogRequest.CloudProvider = cmConfig.CloudProvider cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName cdLogRequest.AzureBlobConfig.AccountKey = secretConfig.AzureAccountKey @@ -1275,6 +1276,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) return nil, err } + impl.Logger.Infow("cmConfig from ext cluster: ", cmConfig, "secret from ext cluster: ", secretConfig) request.StorageType = cmConfig.CloudProvider request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey From cb174cb3489008d8716f7dd01163c9035e89aae3 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 25 Oct 2023 01:06:36 +0530 Subject: [PATCH 09/25] fix --- pkg/pipeline/CdHandler.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ac498d1cc10..ccbdc2b7804 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -177,16 +177,22 @@ type ArgoPipelineStatusSyncEvent struct { type CmBlobStorageConfig struct { //AWS credentials - CloudProvider blob_storage.BlobStorageType `json:"BLOB_STORAGE_PROVIDER"` - S3AccessKey string `json:"BLOB_STORAGE_S3_ACCESS_KEY"` - S3Endpoint string `json:"BLOB_STORAGE_S3_ENDPOINT"` - S3EndpointInsecure string `json:"BLOB_STORAGE_S3_ENDPOINT_INSECURE"` - S3BucketVersioned string `json:"BLOB_STORAGE_S3_BUCKET_VERSIONED"` + CloudProvider blob_storage.BlobStorageType `json:"BLOB_STORAGE_PROVIDER"` + S3AccessKey string `json:"BLOB_STORAGE_S3_ACCESS_KEY"` + S3Endpoint string `json:"BLOB_STORAGE_S3_ENDPOINT"` + S3EndpointInsecure string `json:"BLOB_STORAGE_S3_ENDPOINT_INSECURE"` + S3BucketVersioned string `json:"BLOB_STORAGE_S3_BUCKET_VERSIONED"` + CdDefaultBuildLogsBucket string `json:"DEFAULT_BUILD_LOGS_BUCKET" ` + CdDefaultCdLogsBucketRegion string `json:"DEFAULT_CD_LOGS_BUCKET_REGION" ` + DefaultCacheBucket string `json:"DEFAULT_CACHE_BUCKET"` + DefaultCacheBucketRegion string `json:"DEFAULT_CACHE_BUCKET_REGION"` //Azure credentials AzureAccountName string `json:"AZURE_ACCOUNT_NAME"` AzureGatewayUrl string `json:"AZURE_GATEWAY_URL"` AzureGatewayConnectionInsecure string `json:"AZURE_GATEWAY_CONNECTION_INSECURE"` + AzureBlobContainerCiLog string `json:"AZURE_BLOB_CONTAINER_CI_LOG"` + AzureBlobContainerCiCache string `json:"AZURE_BLOB_CONTAINER_CI_CACHE"` } func (c CmBlobStorageConfig) PopulateWithK8sExtBlobCmData(cm map[string]string) (CmBlobStorageConfig, error) { @@ -1047,14 +1053,20 @@ func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmC cdLogRequest.CloudProvider = cmConfig.CloudProvider cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName cdLogRequest.AzureBlobConfig.AccountKey = secretConfig.AzureAccountKey + cdLogRequest.AzureBlobConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson + cdLogRequest.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket cdLogRequest.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey cdLogRequest.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint cdLogRequest.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey isEndpointInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) cdLogRequest.AwsS3BaseConfig.IsInSecure = isEndpointInSecure + cdLogRequest.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + cdLogRequest.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + cdLogRequest.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned return *cdLogRequest } @@ -1269,6 +1281,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) return nil, err } + impl.Logger.Infow("external cluster config ", clusterConfig) //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds //from them else return. cmConfig, secretConfig, err := impl.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) @@ -1284,11 +1297,17 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i request.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) request.AwsS3BaseConfig.IsInSecure = isInSecure + request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName request.AzureBlobBaseConfig.AccountKey = secretConfig.AzureAccountKey + request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog request.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson + request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket } _, numBytes, err := blobStorageService.Get(request) if err != nil { From 1da47a855ab82e09463764259cba8bf0689f5afc Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 25 Oct 2023 01:21:14 +0530 Subject: [PATCH 10/25] comments added --- pkg/pipeline/CdHandler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ccbdc2b7804..c155d14f6d4 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -1115,6 +1115,7 @@ func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(cluster if secret.Data == nil { return cmConfig, secretConfig, errors.New("Data field not found in secret") } + impl.Logger.Infow("ext cluster cloud provider: ", cmConfig.CloudProvider) return cmConfig, secretConfig, nil } @@ -1281,7 +1282,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) return nil, err } - impl.Logger.Infow("external cluster config ", clusterConfig) + impl.Logger.Infow("external cluster config: ", clusterConfig.Host) //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds //from them else return. cmConfig, secretConfig, err := impl.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) @@ -1289,7 +1290,6 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) return nil, err } - impl.Logger.Infow("cmConfig from ext cluster: ", cmConfig, "secret from ext cluster: ", secretConfig) request.StorageType = cmConfig.CloudProvider request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey From 74165a89e36a8f833cf0536dfbf5529e29aa2a71 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 25 Oct 2023 01:40:54 +0530 Subject: [PATCH 11/25] comments fix --- pkg/pipeline/CdHandler.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index c155d14f6d4..95602b54ca1 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -1115,7 +1115,7 @@ func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(cluster if secret.Data == nil { return cmConfig, secretConfig, errors.New("Data field not found in secret") } - impl.Logger.Infow("ext cluster cloud provider: ", cmConfig.CloudProvider) + impl.Logger.Infow("fetching cm and secret from external cluster cloud provider", "cloud provider: ", cmConfig.CloudProvider) return cmConfig, secretConfig, nil } @@ -1282,7 +1282,6 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) return nil, err } - impl.Logger.Infow("external cluster config: ", clusterConfig.Host) //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds //from them else return. cmConfig, secretConfig, err := impl.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) From 8ef0a28fc3c2dd3deb3aef347a7fe496854918ae Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 25 Oct 2023 02:35:18 +0530 Subject: [PATCH 12/25] decodeSecretKey --- pkg/pipeline/CdHandler.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 95602b54ca1..86082f60f4b 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -20,6 +20,7 @@ package pipeline import ( "bufio" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -1068,6 +1069,10 @@ func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmC s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) cdLogRequest.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned + cdLogRequest.AzureBlobConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) + cdLogRequest.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) + cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + return *cdLogRequest } @@ -1115,7 +1120,7 @@ func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(cluster if secret.Data == nil { return cmConfig, secretConfig, errors.New("Data field not found in secret") } - impl.Logger.Infow("fetching cm and secret from external cluster cloud provider", "cloud provider: ", cmConfig.CloudProvider) + impl.Logger.Infow("fetching cm and secret from external cluster cloud provider", "ext cluster config: ", cmConfig) return cmConfig, secretConfig, nil } @@ -1307,6 +1312,10 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i request.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) + request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) + request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) } _, numBytes, err := blobStorageService.Get(request) if err != nil { @@ -1324,6 +1333,14 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i return file, nil } +func decodeSecretKey(secretKey string) string { + decodedKey, err := base64.StdEncoding.DecodeString(secretKey) + if err != nil { + fmt.Println("error decoding base64 key:", err) + } + return string(decodedKey) +} + func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pipelineConfig.CdWorkflowWithArtifact { workflow := pipelineConfig.CdWorkflowWithArtifact{} if wfr.Id > 0 { From 24152a18ec3659d8557108efe8f86b57848c9b5f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 25 Oct 2023 11:53:55 +0530 Subject: [PATCH 13/25] code review oncorporation l1 --- pkg/pipeline/CdHandler.go | 87 ++++++++++++++-------------- pkg/pipeline/bean/CloudHelperBean.go | 1 + 2 files changed, 43 insertions(+), 45 deletions(-) create mode 100644 pkg/pipeline/bean/CloudHelperBean.go diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 86082f60f4b..8f2b907b6a9 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -196,18 +196,18 @@ type CmBlobStorageConfig struct { AzureBlobContainerCiCache string `json:"AZURE_BLOB_CONTAINER_CI_CACHE"` } -func (c CmBlobStorageConfig) PopulateWithK8sExtBlobCmData(cm map[string]string) (CmBlobStorageConfig, error) { +func (c *CmBlobStorageConfig) PopulateWithK8sExtBlobCmData(cm map[string]string) error { cmDataJson, err := json.Marshal(cm) if err != nil { fmt.Println("error marshalling external blob storage cm data to json:", err) - return c, err + return err } err = json.Unmarshal(cmDataJson, &c) if err != nil { fmt.Println("error unmarshalling external blob storage cm json to struct:", err) - return c, err + return err } - return c, nil + return nil } type SecretBlobStorageConfig struct { @@ -219,18 +219,18 @@ type SecretBlobStorageConfig struct { AzureAccountKey string `json:"AZURE_ACCOUNT_KEY"` } -func (s SecretBlobStorageConfig) PopulateWithK8sExtBlobSecretData(secret map[string][]byte) (SecretBlobStorageConfig, error) { +func (s *SecretBlobStorageConfig) PopulateWithK8sExtBlobSecretData(secret map[string][]byte) error { cmDataJson, err := json.Marshal(secret) if err != nil { fmt.Println("error marshalling external blob storage secret data to json:", err) - return s, err + return err } err = json.Unmarshal(cmDataJson, &s) if err != nil { fmt.Println("error unmarshalling external blob storage secret json to struct:", err) - return s, err + return err } - return s, nil + return nil } const NotTriggered string = "Not Triggered" @@ -1049,19 +1049,18 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip logReader := bufio.NewReader(oldLogsStream) return logReader, cleanUp, err } -func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig CmBlobStorageConfig, secretConfig SecretBlobStorageConfig) BuildLogRequest { - //TODO maybe we need to put a check here because at time of parsing for other blob values can be empty +func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig *CmBlobStorageConfig, secretConfig *SecretBlobStorageConfig) BuildLogRequest { cdLogRequest.CloudProvider = cmConfig.CloudProvider cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName - cdLogRequest.AzureBlobConfig.AccountKey = secretConfig.AzureAccountKey + cdLogRequest.AzureBlobConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) cdLogRequest.AzureBlobConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog - cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson + cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) cdLogRequest.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket cdLogRequest.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey cdLogRequest.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - cdLogRequest.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey + cdLogRequest.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) isEndpointInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) cdLogRequest.AwsS3BaseConfig.IsInSecure = isEndpointInSecure cdLogRequest.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket @@ -1069,16 +1068,12 @@ func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmC s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) cdLogRequest.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned - cdLogRequest.AzureBlobConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) - cdLogRequest.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) - cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) - return *cdLogRequest } -func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (CmBlobStorageConfig, SecretBlobStorageConfig, error) { - var cmConfig CmBlobStorageConfig - var secretConfig SecretBlobStorageConfig +func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*CmBlobStorageConfig, *SecretBlobStorageConfig, error) { + cmConfig := &CmBlobStorageConfig{} + secretConfig := &SecretBlobStorageConfig{} _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(clusterConfig) if err != nil { impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) @@ -1103,12 +1098,12 @@ func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(cluster return cmConfig, secretConfig, errors.New(secretNotFoundErr) } if cm.Data != nil && secret != nil { - cmConfig, err = cmConfig.PopulateWithK8sExtBlobCmData(cm.Data) + err = cmConfig.PopulateWithK8sExtBlobCmData(cm.Data) if err != nil { fmt.Println("error marshalling external blob storage cm data to struct:", err) return cmConfig, secretConfig, err } - secretConfig, err = secretConfig.PopulateWithK8sExtBlobSecretData(secret.Data) + err = secretConfig.PopulateWithK8sExtBlobSecretData(secret.Data) if err != nil { fmt.Println("error marshalling external blob storage secret data to struct:", err) return cmConfig, secretConfig, err @@ -1221,7 +1216,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i } else if wfr.WorkflowType == POST { isExtCluster = wfr.CdWorkflow.Pipeline.RunPostStageInEnv } - //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well + //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db as well useExternalBlobStorage := isExtCluster && !impl.config.UseBlobStorageConfigInCdWorkflow if !wfr.BlobStorageEnabled { return nil, errors.New("logs-not-stored-in-repository") @@ -1294,28 +1289,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) return nil, err } - request.StorageType = cmConfig.CloudProvider - - request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey - request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - request.AwsS3BaseConfig.Passkey = secretConfig.S3SecretKey - isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) - request.AwsS3BaseConfig.IsInSecure = isInSecure - request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion - s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) - request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned - - request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName - request.AzureBlobBaseConfig.AccountKey = secretConfig.AzureAccountKey - request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog - - request.GcpBlobBaseConfig.CredentialFileJsonData = secretConfig.GcpBlobStorageCredentialJson - request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - - request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) - request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) - request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + request = assignNewBlobStorageConfigInRequest(request, cmConfig, secretConfig) } _, numBytes, err := blobStorageService.Get(request) if err != nil { @@ -1333,6 +1307,29 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i return file, nil } +func assignNewBlobStorageConfigInRequest(request *blob_storage.BlobStorageRequest, cmConfig *CmBlobStorageConfig, secretConfig *SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { + request.StorageType = cmConfig.CloudProvider + + request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) + isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + request.AwsS3BaseConfig.IsInSecure = isInSecure + request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned + + request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName + request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) + request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog + + request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + return request +} + func decodeSecretKey(secretKey string) string { decodedKey, err := base64.StdEncoding.DecodeString(secretKey) if err != nil { diff --git a/pkg/pipeline/bean/CloudHelperBean.go b/pkg/pipeline/bean/CloudHelperBean.go new file mode 100644 index 00000000000..828510c8e84 --- /dev/null +++ b/pkg/pipeline/bean/CloudHelperBean.go @@ -0,0 +1 @@ +package bean From c4dc66e75ffe6c12f382481171ae76efe2eff9c1 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 25 Oct 2023 11:59:22 +0530 Subject: [PATCH 14/25] code review incorporation l2 --- pkg/pipeline/CdHandler.go | 68 ++-------------------------- pkg/pipeline/bean/CloudHelperBean.go | 63 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 63 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 8f2b907b6a9..c9d8cb1c63b 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -21,7 +21,6 @@ import ( "bufio" "context" "encoding/base64" - "encoding/json" "errors" "fmt" application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" @@ -176,63 +175,6 @@ type ArgoPipelineStatusSyncEvent struct { IsAppStoreApplication bool `json:"isAppStoreApplication"` } -type CmBlobStorageConfig struct { - //AWS credentials - CloudProvider blob_storage.BlobStorageType `json:"BLOB_STORAGE_PROVIDER"` - S3AccessKey string `json:"BLOB_STORAGE_S3_ACCESS_KEY"` - S3Endpoint string `json:"BLOB_STORAGE_S3_ENDPOINT"` - S3EndpointInsecure string `json:"BLOB_STORAGE_S3_ENDPOINT_INSECURE"` - S3BucketVersioned string `json:"BLOB_STORAGE_S3_BUCKET_VERSIONED"` - CdDefaultBuildLogsBucket string `json:"DEFAULT_BUILD_LOGS_BUCKET" ` - CdDefaultCdLogsBucketRegion string `json:"DEFAULT_CD_LOGS_BUCKET_REGION" ` - DefaultCacheBucket string `json:"DEFAULT_CACHE_BUCKET"` - DefaultCacheBucketRegion string `json:"DEFAULT_CACHE_BUCKET_REGION"` - - //Azure credentials - AzureAccountName string `json:"AZURE_ACCOUNT_NAME"` - AzureGatewayUrl string `json:"AZURE_GATEWAY_URL"` - AzureGatewayConnectionInsecure string `json:"AZURE_GATEWAY_CONNECTION_INSECURE"` - AzureBlobContainerCiLog string `json:"AZURE_BLOB_CONTAINER_CI_LOG"` - AzureBlobContainerCiCache string `json:"AZURE_BLOB_CONTAINER_CI_CACHE"` -} - -func (c *CmBlobStorageConfig) PopulateWithK8sExtBlobCmData(cm map[string]string) error { - cmDataJson, err := json.Marshal(cm) - if err != nil { - fmt.Println("error marshalling external blob storage cm data to json:", err) - return err - } - err = json.Unmarshal(cmDataJson, &c) - if err != nil { - fmt.Println("error unmarshalling external blob storage cm json to struct:", err) - return err - } - return nil -} - -type SecretBlobStorageConfig struct { - //aws - S3SecretKey string `json:"BLOB_STORAGE_S3_SECRET_KEY"` - //gcp - GcpBlobStorageCredentialJson string `json:"BLOB_STORAGE_GCP_CREDENTIALS_JSON"` - //azure - AzureAccountKey string `json:"AZURE_ACCOUNT_KEY"` -} - -func (s *SecretBlobStorageConfig) PopulateWithK8sExtBlobSecretData(secret map[string][]byte) error { - cmDataJson, err := json.Marshal(secret) - if err != nil { - fmt.Println("error marshalling external blob storage secret data to json:", err) - return err - } - err = json.Unmarshal(cmDataJson, &s) - if err != nil { - fmt.Println("error unmarshalling external blob storage secret json to struct:", err) - return err - } - return nil -} - const NotTriggered string = "Not Triggered" const NotDeployed = "Not Deployed" const WorklowTypeDeploy = "DEPLOY" @@ -1049,7 +991,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip logReader := bufio.NewReader(oldLogsStream) return logReader, cleanUp, err } -func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig *CmBlobStorageConfig, secretConfig *SecretBlobStorageConfig) BuildLogRequest { +func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) BuildLogRequest { cdLogRequest.CloudProvider = cmConfig.CloudProvider cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName cdLogRequest.AzureBlobConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) @@ -1071,9 +1013,9 @@ func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmC return *cdLogRequest } -func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*CmBlobStorageConfig, *SecretBlobStorageConfig, error) { - cmConfig := &CmBlobStorageConfig{} - secretConfig := &SecretBlobStorageConfig{} +func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { + cmConfig := &bean2.CmBlobStorageConfig{} + secretConfig := &bean2.SecretBlobStorageConfig{} _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(clusterConfig) if err != nil { impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) @@ -1307,7 +1249,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i return file, nil } -func assignNewBlobStorageConfigInRequest(request *blob_storage.BlobStorageRequest, cmConfig *CmBlobStorageConfig, secretConfig *SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { +func assignNewBlobStorageConfigInRequest(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { request.StorageType = cmConfig.CloudProvider request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey diff --git a/pkg/pipeline/bean/CloudHelperBean.go b/pkg/pipeline/bean/CloudHelperBean.go index 828510c8e84..757a7d2c905 100644 --- a/pkg/pipeline/bean/CloudHelperBean.go +++ b/pkg/pipeline/bean/CloudHelperBean.go @@ -1 +1,64 @@ package bean + +import ( + "encoding/json" + "fmt" + "github.com/devtron-labs/common-lib/blob-storage" +) + +type CmBlobStorageConfig struct { + //AWS credentials + CloudProvider blob_storage.BlobStorageType `json:"BLOB_STORAGE_PROVIDER"` + S3AccessKey string `json:"BLOB_STORAGE_S3_ACCESS_KEY"` + S3Endpoint string `json:"BLOB_STORAGE_S3_ENDPOINT"` + S3EndpointInsecure string `json:"BLOB_STORAGE_S3_ENDPOINT_INSECURE"` + S3BucketVersioned string `json:"BLOB_STORAGE_S3_BUCKET_VERSIONED"` + CdDefaultBuildLogsBucket string `json:"DEFAULT_BUILD_LOGS_BUCKET" ` + CdDefaultCdLogsBucketRegion string `json:"DEFAULT_CD_LOGS_BUCKET_REGION" ` + DefaultCacheBucket string `json:"DEFAULT_CACHE_BUCKET"` + DefaultCacheBucketRegion string `json:"DEFAULT_CACHE_BUCKET_REGION"` + + //Azure credentials + AzureAccountName string `json:"AZURE_ACCOUNT_NAME"` + AzureGatewayUrl string `json:"AZURE_GATEWAY_URL"` + AzureGatewayConnectionInsecure string `json:"AZURE_GATEWAY_CONNECTION_INSECURE"` + AzureBlobContainerCiLog string `json:"AZURE_BLOB_CONTAINER_CI_LOG"` + AzureBlobContainerCiCache string `json:"AZURE_BLOB_CONTAINER_CI_CACHE"` +} + +func (c *CmBlobStorageConfig) PopulateWithK8sExtBlobCmData(cm map[string]string) error { + cmDataJson, err := json.Marshal(cm) + if err != nil { + fmt.Println("error marshalling external blob storage cm data to json:", err) + return err + } + err = json.Unmarshal(cmDataJson, &c) + if err != nil { + fmt.Println("error unmarshalling external blob storage cm json to struct:", err) + return err + } + return nil +} + +type SecretBlobStorageConfig struct { + //aws + S3SecretKey string `json:"BLOB_STORAGE_S3_SECRET_KEY"` + //gcp + GcpBlobStorageCredentialJson string `json:"BLOB_STORAGE_GCP_CREDENTIALS_JSON"` + //azure + AzureAccountKey string `json:"AZURE_ACCOUNT_KEY"` +} + +func (s *SecretBlobStorageConfig) PopulateWithK8sExtBlobSecretData(secret map[string][]byte) error { + cmDataJson, err := json.Marshal(secret) + if err != nil { + fmt.Println("error marshalling external blob storage secret data to json:", err) + return err + } + err = json.Unmarshal(cmDataJson, &s) + if err != nil { + fmt.Println("error unmarshalling external blob storage secret json to struct:", err) + return err + } + return nil +} From 430c93f34de138ea9d9159a9b2d0ca6b33ceca94 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 26 Oct 2023 15:34:29 +0530 Subject: [PATCH 15/25] code review incorporation l3 --- pkg/pipeline/CdHandler.go | 68 ++------------------------------------ pkg/pipeline/CloudUtils.go | 62 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 65 deletions(-) create mode 100644 pkg/pipeline/CloudUtils.go diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index c9d8cb1c63b..b751b577559 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -20,7 +20,6 @@ package pipeline import ( "bufio" "context" - "encoding/base64" "errors" "fmt" application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" @@ -53,7 +52,6 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" - errors2 "k8s.io/apimachinery/pkg/api/errors" v12 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "os" @@ -969,7 +967,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } - + //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db useExternalBlobStorage := isExt && !impl.config.UseBlobStorageConfigInCdWorkflow if useExternalBlobStorage { //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds @@ -991,27 +989,6 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip logReader := bufio.NewReader(oldLogsStream) return logReader, cleanUp, err } -func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) BuildLogRequest { - cdLogRequest.CloudProvider = cmConfig.CloudProvider - cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName - cdLogRequest.AzureBlobConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) - cdLogRequest.AzureBlobConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog - - cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) - cdLogRequest.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - - cdLogRequest.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey - cdLogRequest.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - cdLogRequest.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) - isEndpointInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) - cdLogRequest.AwsS3BaseConfig.IsInSecure = isEndpointInSecure - cdLogRequest.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - cdLogRequest.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion - s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) - cdLogRequest.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned - - return *cdLogRequest -} func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { cmConfig := &bean2.CmBlobStorageConfig{} @@ -1022,23 +999,15 @@ func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(cluster return cmConfig, secretConfig, err } cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), impl.config.ExtBlobStorageCmName, v12.GetOptions{}) - if err != nil && !errors2.IsNotFound(err) { + if err != nil { impl.Logger.Errorw("error in getting config map in external cluster", "err", err, "blobStorageCmName", impl.config.ExtBlobStorageCmName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err } - if errors2.IsNotFound(err) { - cmNotFoundErr := fmt.Sprintf("config map: %s not found in namespace: %s", impl.config.ExtBlobStorageCmName, namespace) - return cmConfig, secretConfig, errors.New(cmNotFoundErr) - } secret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), impl.config.ExtBlobStorageSecretName, v12.GetOptions{}) - if err != nil && !errors2.IsNotFound(err) { + if err != nil { impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.config.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err } - if errors2.IsNotFound(err) { - secretNotFoundErr := fmt.Sprintf("secret: %s not found in namespace: %s", impl.config.ExtBlobStorageSecretName, namespace) - return cmConfig, secretConfig, errors.New(secretNotFoundErr) - } if cm.Data != nil && secret != nil { err = cmConfig.PopulateWithK8sExtBlobCmData(cm.Data) if err != nil { @@ -1249,37 +1218,6 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i return file, nil } -func assignNewBlobStorageConfigInRequest(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { - request.StorageType = cmConfig.CloudProvider - - request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey - request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) - isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) - request.AwsS3BaseConfig.IsInSecure = isInSecure - request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion - s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) - request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned - - request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName - request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) - request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog - - request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) - request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - - return request -} - -func decodeSecretKey(secretKey string) string { - decodedKey, err := base64.StdEncoding.DecodeString(secretKey) - if err != nil { - fmt.Println("error decoding base64 key:", err) - } - return string(decodedKey) -} - func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pipelineConfig.CdWorkflowWithArtifact { workflow := pipelineConfig.CdWorkflowWithArtifact{} if wfr.Id > 0 { diff --git a/pkg/pipeline/CloudUtils.go b/pkg/pipeline/CloudUtils.go new file mode 100644 index 00000000000..a45be7bdad8 --- /dev/null +++ b/pkg/pipeline/CloudUtils.go @@ -0,0 +1,62 @@ +package pipeline + +import ( + "encoding/base64" + "fmt" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "strconv" +) + +func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) BuildLogRequest { + cdLogRequest.CloudProvider = cmConfig.CloudProvider + cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName + cdLogRequest.AzureBlobConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) + cdLogRequest.AzureBlobConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog + + cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + cdLogRequest.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + cdLogRequest.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + cdLogRequest.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + cdLogRequest.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) + isEndpointInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + cdLogRequest.AwsS3BaseConfig.IsInSecure = isEndpointInSecure + cdLogRequest.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + cdLogRequest.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + cdLogRequest.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned + + return *cdLogRequest +} + +func assignNewBlobStorageConfigInRequest(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { + request.StorageType = cmConfig.CloudProvider + + request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) + isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + request.AwsS3BaseConfig.IsInSecure = isInSecure + request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned + + request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName + request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) + request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog + + request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + return request +} + +func decodeSecretKey(secretKey string) string { + decodedKey, err := base64.StdEncoding.DecodeString(secretKey) + if err != nil { + fmt.Println("error decoding base64 key:", err) + } + return string(decodedKey) +} From 5f68d00444f91bf70a4106924e4a347cdd838f09 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 26 Oct 2023 20:09:29 +0530 Subject: [PATCH 16/25] code review incorporation l4 --- .../pipelineConfig/CdWorfkflowRepository.go | 13 ++++ pkg/cluster/ClusterService.go | 18 ++++++ pkg/pipeline/BlobUtils.go | 40 ++++++++++++ pkg/pipeline/CdHandler.go | 54 +++++++--------- pkg/pipeline/CiLogService.go | 22 +++++++ pkg/pipeline/CiService.go | 2 - pkg/pipeline/CloudUtils.go | 62 ------------------- pkg/pipeline/WorkflowDagExecutor.go | 12 ++-- pkg/pipeline/WorkflowUtils.go | 19 +++--- pkg/pipeline/bean/CloudHelperBean.go | 5 +- wire_gen.go | 2 +- 11 files changed, 134 insertions(+), 115 deletions(-) create mode 100644 pkg/pipeline/BlobUtils.go delete mode 100644 pkg/pipeline/CloudUtils.go diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 352b4266120..b1f7edc07ec 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -100,6 +100,9 @@ const ( WorkflowSucceeded = "Succeeded" WorkflowTimedOut = "TimedOut" WorkflowUnableToFetchState = "UnableToFetch" + WorkflowTypeDeploy = "DEPLOY" + WorkflowTypePre = "PRE" + WorkflowTypePost = "POST" ) func (a WorkflowStatus) String() string { @@ -166,6 +169,16 @@ type CdWorkflowRunner struct { sql.AuditLog } +func (c *CdWorkflowRunner) IsExternalRun() bool { + var isExtCluster bool + if c.WorkflowType == WorkflowTypePre { + isExtCluster = c.CdWorkflow.Pipeline.RunPreStageInEnv + } else if c.WorkflowType == WorkflowTypePost { + isExtCluster = c.CdWorkflow.Pipeline.RunPostStageInEnv + } + return isExtCluster +} + type CiPipelineMaterialResponse struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId"` diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index b7aab982148..d99873d4e49 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -180,6 +180,8 @@ type ClusterService interface { ConnectClustersInBatch(clusters []*ClusterBean, clusterExistInDb bool) ConvertClusterBeanToCluster(clusterBean *ClusterBean, userId int32) *repository.Cluster ConvertClusterBeanObjectToCluster(bean *ClusterBean) *v1alpha1.Cluster + + GetClusterConfigByClusterId(clusterId int) (*k8s.ClusterConfig, error) } type ClusterServiceImpl struct { @@ -1095,3 +1097,19 @@ func (impl ClusterServiceImpl) ConvertClusterBeanObjectToCluster(bean *ClusterBe } return cl } + +func (impl ClusterServiceImpl) GetClusterConfigByClusterId(clusterId int) (*k8s.ClusterConfig, error) { + clusterBean := &ClusterBean{} + clusterBean, err := impl.FindById(clusterId) + if err != nil { + impl.logger.Errorw("error in getting clusterBean", "err", err, "clusterId", clusterId) + return nil, err + } + rq := *clusterBean + clusterConfig, err := rq.GetClusterConfig() + if err != nil { + impl.logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) + return nil, err + } + return clusterConfig, nil +} diff --git a/pkg/pipeline/BlobUtils.go b/pkg/pipeline/BlobUtils.go new file mode 100644 index 00000000000..779a10af22a --- /dev/null +++ b/pkg/pipeline/BlobUtils.go @@ -0,0 +1,40 @@ +package pipeline + +import ( + "encoding/base64" + "fmt" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "strconv" +) + +func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { + request.StorageType = cmConfig.CloudProvider + + request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) + isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + request.AwsS3BaseConfig.IsInSecure = isInSecure + request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned + + request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName + request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) + request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog + + request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + return request +} + +func decodeSecretKey(secretKey string) string { + decodedKey, err := base64.StdEncoding.DecodeString(secretKey) + if err != nil { + fmt.Println("error decoding base64 key:", err) + } + return string(decodedKey) +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index b751b577559..8902c3c9a99 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -120,10 +120,10 @@ type CdHandlerImpl struct { k8sUtil *k8s.K8sUtil workflowService WorkflowService config *CdConfig - clusterRepository repository2.ClusterRepository + clusterService cluster.ClusterService } -func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService, clusterRepository repository2.ClusterRepository) *CdHandlerImpl { +func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService, clusterService cluster.ClusterService) *CdHandlerImpl { cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, @@ -157,7 +157,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, c imageTaggingService: imageTaggingService, k8sUtil: k8sUtil, workflowService: workflowService, - clusterRepository: clusterRepository, + clusterService: clusterService, } config, err := GetCdConfig() if err != nil { @@ -926,7 +926,7 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC } func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs") + impl.Logger.Debug("getting historic logs", "pipelineId", pipelineId) cdConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(pipelineId) if err != nil && !util.IsErrNoRows(err) { @@ -967,8 +967,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } - //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db - useExternalBlobStorage := isExt && !impl.config.UseBlobStorageConfigInCdWorkflow + useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) if useExternalBlobStorage { //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds //from them else return. @@ -977,7 +976,8 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) return nil, nil, err } - cdLogRequest = assignNewBlobStorageConfigInCdLogRequest(&cdLogRequest, cmConfig, secretConfig) + rq := &cdLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) } impl.Logger.Infow("s3 log req ", "req", cdLogRequest) @@ -989,7 +989,10 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip logReader := bufio.NewReader(oldLogsStream) return logReader, cleanUp, err } - +func isExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { + //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db + return isExternalRun && !useBlobStorageConfigInCdWorkflow +} func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { cmConfig := &bean2.CmBlobStorageConfig{} secretConfig := &bean2.SecretBlobStorageConfig{} @@ -1008,23 +1011,24 @@ func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(cluster impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.config.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err } - if cm.Data != nil && secret != nil { - err = cmConfig.PopulateWithK8sExtBlobCmData(cm.Data) + //for IAM configured in S3 in external cluster, get logs/artifact will not work + if cm.Data != nil && secret.Data != nil { + err = cmConfig.SetCmBlobStorageConfig(cm.Data) if err != nil { fmt.Println("error marshalling external blob storage cm data to struct:", err) return cmConfig, secretConfig, err } - err = secretConfig.PopulateWithK8sExtBlobSecretData(secret.Data) + err = secretConfig.SetSecretBlobStorageConfig(secret.Data) if err != nil { fmt.Println("error marshalling external blob storage secret data to struct:", err) return cmConfig, secretConfig, err } } if cm.Data == nil { - return cmConfig, secretConfig, errors.New("Data field not found in config map") + fmt.Println("Data field not found in config map") } if secret.Data == nil { - return cmConfig, secretConfig, errors.New("Data field not found in secret") + fmt.Println("Data field not found in secret") } impl.Logger.Infow("fetching cm and secret from external cluster cloud provider", "ext cluster config: ", cmConfig) return cmConfig, secretConfig, nil @@ -1121,14 +1125,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) return nil, err } - var isExtCluster bool - if wfr.WorkflowType == PRE { - isExtCluster = wfr.CdWorkflow.Pipeline.RunPreStageInEnv - } else if wfr.WorkflowType == POST { - isExtCluster = wfr.CdWorkflow.Pipeline.RunPostStageInEnv - } - //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db as well - useExternalBlobStorage := isExtCluster && !impl.config.UseBlobStorageConfigInCdWorkflow + useExternalBlobStorage := isExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) if !wfr.BlobStorageEnabled { return nil, errors.New("logs-not-stored-in-repository") } @@ -1179,18 +1176,9 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i GcpBlobBaseConfig: gcpBlobBaseConfig, } if useExternalBlobStorage { - var clusterBean cluster.ClusterBean - if wfr.CdWorkflow.Pipeline.Environment.Id != 0 { - clusterFromDb, err := impl.clusterRepository.FindById(wfr.CdWorkflow.Pipeline.Environment.ClusterId) - if err != nil { - impl.Logger.Errorw("unable to fetch cluster by cluster Id", "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId, "err", err) - return nil, err - } - clusterBean = cluster.GetClusterBean(*clusterFromDb) - } - clusterConfig, err := clusterBean.GetClusterConfig() + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) if err != nil { - impl.Logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) return nil, err } //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds @@ -1200,7 +1188,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) return nil, err } - request = assignNewBlobStorageConfigInRequest(request, cmConfig, secretConfig) + request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) } _, numBytes, err := blobStorageService.Get(request) if err != nil { diff --git a/pkg/pipeline/CiLogService.go b/pkg/pipeline/CiLogService.go index 52430cfa1b0..12c34f2eaf2 100644 --- a/pkg/pipeline/CiLogService.go +++ b/pkg/pipeline/CiLogService.go @@ -27,6 +27,7 @@ import ( "k8s.io/client-go/kubernetes" "os" "path/filepath" + "strconv" ) type CiLogService interface { @@ -54,6 +55,26 @@ type BuildLogRequest struct { MinioEndpoint string } +func (r *BuildLogRequest) SetBuildLogRequest(cmConfig *bean.CmBlobStorageConfig, secretConfig *bean.SecretBlobStorageConfig) { + r.CloudProvider = cmConfig.CloudProvider + r.AzureBlobConfig.AccountName = cmConfig.AzureAccountName + r.AzureBlobConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) + r.AzureBlobConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog + + r.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + r.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + r.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + r.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + r.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) + isEndpointInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + r.AwsS3BaseConfig.IsInSecure = isEndpointInSecure + r.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + r.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + r.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned +} + func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil *k8s.K8sUtil) (*CiLogServiceImpl, error) { _, _, clientSet, err := k8sUtil.GetK8sInClusterConfigAndClients() if err != nil { @@ -96,6 +117,7 @@ func (impl *CiLogServiceImpl) FetchRunningWorkflowLogs(ciLogRequest BuildLogRequ return podLogs, cleanUpFunc, nil } +// TODO Prakash CiHandler also calls this func, apply the same logic as present in cdHandler func (impl *CiLogServiceImpl) FetchLogs(baseLogLocationPathConfig string, logRequest BuildLogRequest) (*os.File, func() error, error) { tempFile := baseLogLocationPathConfig tempFile = filepath.Clean(filepath.Join(tempFile, logRequest.PodName+".log")) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index d697288bb04..da834903353 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -593,8 +593,6 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ImageRetryCount: impl.config.ImageRetryCount, ImageRetryInterval: impl.config.ImageRetryInterval, WorkflowExecutor: impl.config.GetWorkflowExecutorType(), - ExtBlobStorageSecretName: impl.config.ExtBlobStorageSecretName, - ExtBlobStorageCmName: impl.config.ExtBlobStorageCmName, Type: bean2.CI_WORKFLOW_PIPELINE_TYPE, CiArtifactLastFetch: trigger.CiArtifactLastFetch, } diff --git a/pkg/pipeline/CloudUtils.go b/pkg/pipeline/CloudUtils.go deleted file mode 100644 index a45be7bdad8..00000000000 --- a/pkg/pipeline/CloudUtils.go +++ /dev/null @@ -1,62 +0,0 @@ -package pipeline - -import ( - "encoding/base64" - "fmt" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "strconv" -) - -func assignNewBlobStorageConfigInCdLogRequest(cdLogRequest *BuildLogRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) BuildLogRequest { - cdLogRequest.CloudProvider = cmConfig.CloudProvider - cdLogRequest.AzureBlobConfig.AccountName = cmConfig.AzureAccountName - cdLogRequest.AzureBlobConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) - cdLogRequest.AzureBlobConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog - - cdLogRequest.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) - cdLogRequest.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - - cdLogRequest.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey - cdLogRequest.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - cdLogRequest.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) - isEndpointInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) - cdLogRequest.AwsS3BaseConfig.IsInSecure = isEndpointInSecure - cdLogRequest.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - cdLogRequest.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion - s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) - cdLogRequest.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned - - return *cdLogRequest -} - -func assignNewBlobStorageConfigInRequest(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { - request.StorageType = cmConfig.CloudProvider - - request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey - request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) - isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) - request.AwsS3BaseConfig.IsInSecure = isInSecure - request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion - s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) - request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned - - request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName - request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) - request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog - - request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) - request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - - return request -} - -func decodeSecretKey(secretKey string) string { - decodedKey, err := base64.StdEncoding.DecodeString(secretKey) - if err != nil { - fmt.Println("error decoding base64 key:", err) - } - return string(decodedKey) -} diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index ded63c6ae3c..1a66caafe50 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -1134,13 +1134,11 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor DataSource: artifact.DataSource, WorkflowId: artifact.WorkflowId, }, - OrchestratorHost: impl.config.OrchestratorHost, - OrchestratorToken: impl.config.OrchestratorToken, - CloudProvider: impl.config.CloudProvider, - WorkflowExecutor: workflowExecutor, - RefPlugins: refPluginsData, - ExtBlobStorageSecretName: impl.config.ExtBlobStorageSecretName, - ExtBlobStorageCmName: impl.config.ExtBlobStorageCmName, + OrchestratorHost: impl.config.OrchestratorHost, + OrchestratorToken: impl.config.OrchestratorToken, + CloudProvider: impl.config.CloudProvider, + WorkflowExecutor: workflowExecutor, + RefPlugins: refPluginsData, } extraEnvVariables := make(map[string]string) diff --git a/pkg/pipeline/WorkflowUtils.go b/pkg/pipeline/WorkflowUtils.go index 3078d16ba27..6344bd0557e 100644 --- a/pkg/pipeline/WorkflowUtils.go +++ b/pkg/pipeline/WorkflowUtils.go @@ -366,13 +366,12 @@ type WorkflowRequest struct { WorkflowExecutor pipelineConfig.WorkflowExecutorType `json:"workflowExecutor"` PrePostDeploySteps []*bean.StepObject `json:"prePostDeploySteps"` CiArtifactLastFetch time.Time `json:"ciArtifactLastFetch"` - ExtBlobStorageCmName string `json:"extBlobStorageCmName"` - ExtBlobStorageSecretName string `json:"extBlobStorageSecretName"` - UseExternalClusterBlob bool `json:"useExternalClusterBlob"` - Type bean.WorkflowPipelineType - Pipeline *pipelineConfig.Pipeline - Env *repository2.Environment - AppLabels map[string]string + + UseExternalClusterBlob bool `json:"useExternalClusterBlob"` + Type bean.WorkflowPipelineType + Pipeline *pipelineConfig.Pipeline + Env *repository2.Environment + AppLabels map[string]string } type CiCdTriggerEvent struct { @@ -394,6 +393,10 @@ func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() { } } +func (workflowRequest *WorkflowRequest) updateUseExternalClusterBlob(config *CiCdConfig) { + workflowRequest.UseExternalClusterBlob = !workflowRequest.CheckBlobStorageConfig(config) && workflowRequest.IsExtRun +} + func (workflowRequest *WorkflowRequest) CheckBlobStorageConfig(config *CiCdConfig) bool { switch workflowRequest.Type { case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: @@ -440,7 +443,7 @@ func (workflowRequest *WorkflowRequest) GetBlobStorageLogsKey(config *CiCdConfig func (workflowRequest *WorkflowRequest) GetWorkflowJson(config *CiCdConfig) ([]byte, error) { workflowRequest.updateBlobStorageLogsKey(config) workflowRequest.updateExternalRunMetadata() - workflowRequest.UseExternalClusterBlob = !workflowRequest.CheckBlobStorageConfig(config) && workflowRequest.IsExtRun + workflowRequest.updateUseExternalClusterBlob(config) workflowJson, err := workflowRequest.getWorkflowJson() if err != nil { return nil, err diff --git a/pkg/pipeline/bean/CloudHelperBean.go b/pkg/pipeline/bean/CloudHelperBean.go index 757a7d2c905..879c2afa758 100644 --- a/pkg/pipeline/bean/CloudHelperBean.go +++ b/pkg/pipeline/bean/CloudHelperBean.go @@ -26,7 +26,7 @@ type CmBlobStorageConfig struct { AzureBlobContainerCiCache string `json:"AZURE_BLOB_CONTAINER_CI_CACHE"` } -func (c *CmBlobStorageConfig) PopulateWithK8sExtBlobCmData(cm map[string]string) error { +func (c *CmBlobStorageConfig) SetCmBlobStorageConfig(cm map[string]string) error { cmDataJson, err := json.Marshal(cm) if err != nil { fmt.Println("error marshalling external blob storage cm data to json:", err) @@ -49,7 +49,8 @@ type SecretBlobStorageConfig struct { AzureAccountKey string `json:"AZURE_ACCOUNT_KEY"` } -func (s *SecretBlobStorageConfig) PopulateWithK8sExtBlobSecretData(secret map[string][]byte) error { +// input secret data contains encoded bytes +func (s *SecretBlobStorageConfig) SetSecretBlobStorageConfig(secret map[string][]byte) error { cmDataJson, err := json.Marshal(secret) if err != nil { fmt.Println("error marshalling external blob storage secret data to json:", err) diff --git a/wire_gen.go b/wire_gen.go index 0f60fb8b10a..088517c72a8 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -505,7 +505,7 @@ func InitializeApp() (*App, error) { linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterRepositoryImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, appWorkflowRepositoryImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) From 23f2bbbc0f438017a242bd4a9e9c78cb111d4656 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 26 Oct 2023 23:07:17 +0530 Subject: [PATCH 17/25] code review incorporation l5 --- Wire.go | 3 + .../pipelineConfig/CiWorkflowRepository.go | 5 + pkg/cluster/ClusterService.go | 19 +++- pkg/pipeline/BlobStorageConfigService.go | 102 ++++++++++++++++++ pkg/pipeline/BlobUtils.go | 40 ------- pkg/pipeline/CdHandler.go | 49 +-------- pkg/pipeline/CiHandler.go | 57 +++++++++- wire_gen.go | 5 +- 8 files changed, 188 insertions(+), 92 deletions(-) create mode 100644 pkg/pipeline/BlobStorageConfigService.go delete mode 100644 pkg/pipeline/BlobUtils.go diff --git a/Wire.go b/Wire.go index 086152ec434..3e7d97105fc 100644 --- a/Wire.go +++ b/Wire.go @@ -565,6 +565,9 @@ func InitializeApp() (*App, error) { pipeline.NewCdHandlerImpl, wire.Bind(new(pipeline.CdHandler), new(*pipeline.CdHandlerImpl)), + pipeline.NewBlobStorageConfigServiceImpl, + wire.Bind(new(pipeline.BlobStorageConfigService), new(*pipeline.BlobStorageConfigServiceImpl)), + pipeline.NewWorkflowDagExecutorImpl, wire.Bind(new(pipeline.WorkflowDagExecutor), new(*pipeline.WorkflowDagExecutorImpl)), appClone.NewAppCloneServiceImpl, diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index d713b5751cc..4e97bb06d52 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -19,6 +19,7 @@ package pipelineConfig import ( "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/go-pg/pg" "go.uber.org/zap" "time" @@ -76,6 +77,10 @@ type CiWorkflow struct { CiPipeline *CiPipeline } +func (r *CiWorkflow) IsJobType() bool { + return r.CiPipeline.App.AppType == helper.Job +} + type WorkflowWithArtifact struct { Id int `json:"id"` Name string `json:"name"` diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index d99873d4e49..dac1674321a 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -182,6 +182,7 @@ type ClusterService interface { ConvertClusterBeanObjectToCluster(bean *ClusterBean) *v1alpha1.Cluster GetClusterConfigByClusterId(clusterId int) (*k8s.ClusterConfig, error) + GetClusterConfigByEnvId(envId int) (*k8s.ClusterConfig, error) } type ClusterServiceImpl struct { @@ -1099,10 +1100,24 @@ func (impl ClusterServiceImpl) ConvertClusterBeanObjectToCluster(bean *ClusterBe } func (impl ClusterServiceImpl) GetClusterConfigByClusterId(clusterId int) (*k8s.ClusterConfig, error) { - clusterBean := &ClusterBean{} clusterBean, err := impl.FindById(clusterId) if err != nil { - impl.logger.Errorw("error in getting clusterBean", "err", err, "clusterId", clusterId) + impl.logger.Errorw("error in getting clusterBean by cluster id", "err", err, "clusterId", clusterId) + return nil, err + } + rq := *clusterBean + clusterConfig, err := rq.GetClusterConfig() + if err != nil { + impl.logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) + return nil, err + } + return clusterConfig, nil +} + +func (impl ClusterServiceImpl) GetClusterConfigByEnvId(envId int) (*k8s.ClusterConfig, error) { + clusterBean, err := impl.environmentService.FindClusterByEnvId(envId) + if err != nil { + impl.logger.Errorw("error in getting clusterBean by envId", "err", err, "envId", envId) return nil, err } rq := *clusterBean diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go new file mode 100644 index 00000000000..0f1ce5a7146 --- /dev/null +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -0,0 +1,102 @@ +package pipeline + +import ( + "context" + "encoding/base64" + "fmt" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils/k8s" + bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "go.uber.org/zap" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strconv" +) + +type BlobStorageConfigService interface { + FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) +} +type BlobStorageConfigServiceImpl struct { + Logger *zap.SugaredLogger + k8sUtil *k8s.K8sUtil + ciCdConfig *CiCdConfig +} + +func NewBlobStorageConfigServiceImpl(Logger *zap.SugaredLogger, k8sUtil *k8s.K8sUtil, ciCdConfig *CiCdConfig) *BlobStorageConfigServiceImpl { + return &BlobStorageConfigServiceImpl{ + Logger: Logger, + k8sUtil: k8sUtil, + ciCdConfig: ciCdConfig, + } +} + +func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { + cmConfig := &bean2.CmBlobStorageConfig{} + secretConfig := &bean2.SecretBlobStorageConfig{} + _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(&k8s.ClusterConfig{}) + if err != nil { + impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) + return cmConfig, secretConfig, err + } + cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), impl.ciCdConfig.ExtBlobStorageCmName, v12.GetOptions{}) + if err != nil { + impl.Logger.Errorw("error in getting config map in external cluster", "err", err, "blobStorageCmName", impl.ciCdConfig.ExtBlobStorageCmName, "clusterName", clusterConfig.ClusterName) + return cmConfig, secretConfig, err + } + secret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), impl.ciCdConfig.ExtBlobStorageSecretName, v12.GetOptions{}) + if err != nil { + impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.ciCdConfig.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) + return cmConfig, secretConfig, err + } + //for IAM configured in S3 in external cluster, get logs/artifact will not work + if cm.Data != nil && secret.Data != nil { + err = cmConfig.SetCmBlobStorageConfig(cm.Data) + if err != nil { + fmt.Println("error marshalling external blob storage cm data to struct:", err) + return cmConfig, secretConfig, err + } + err = secretConfig.SetSecretBlobStorageConfig(secret.Data) + if err != nil { + fmt.Println("error marshalling external blob storage secret data to struct:", err) + return cmConfig, secretConfig, err + } + } + if cm.Data == nil { + fmt.Println("Data field not found in config map") + } + if secret.Data == nil { + fmt.Println("Data field not found in secret") + } + impl.Logger.Infow("fetching cm and secret from external cluster cloud provider", "ext cluster config: ", cmConfig) + return cmConfig, secretConfig, nil +} + +func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { + request.StorageType = cmConfig.CloudProvider + + request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) + isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + request.AwsS3BaseConfig.IsInSecure = isInSecure + request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned + + request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName + request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) + request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog + + request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + return request +} + +func decodeSecretKey(secretKey string) string { + decodedKey, err := base64.StdEncoding.DecodeString(secretKey) + if err != nil { + fmt.Println("error decoding base64 key:", err) + } + return string(decodedKey) +} diff --git a/pkg/pipeline/BlobUtils.go b/pkg/pipeline/BlobUtils.go deleted file mode 100644 index 779a10af22a..00000000000 --- a/pkg/pipeline/BlobUtils.go +++ /dev/null @@ -1,40 +0,0 @@ -package pipeline - -import ( - "encoding/base64" - "fmt" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "strconv" -) - -func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { - request.StorageType = cmConfig.CloudProvider - - request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey - request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint - request.AwsS3BaseConfig.Passkey = decodeSecretKey(secretConfig.S3SecretKey) - isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) - request.AwsS3BaseConfig.IsInSecure = isInSecure - request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion - s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) - request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned - - request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName - request.AzureBlobBaseConfig.AccountKey = decodeSecretKey(secretConfig.AzureAccountKey) - request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog - - request.GcpBlobBaseConfig.CredentialFileJsonData = decodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) - request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket - - return request -} - -func decodeSecretKey(secretKey string) string { - decodedKey, err := base64.StdEncoding.DecodeString(secretKey) - if err != nil { - fmt.Println("error decoding base64 key:", err) - } - return string(decodedKey) -} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 8902c3c9a99..da92dbc3e71 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -52,7 +52,6 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" - v12 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "os" "path/filepath" @@ -121,9 +120,10 @@ type CdHandlerImpl struct { workflowService WorkflowService config *CdConfig clusterService cluster.ClusterService + blobConfigStorageService BlobStorageConfigService } -func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService, clusterService cluster.ClusterService) *CdHandlerImpl { +func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService) *CdHandlerImpl { cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, @@ -158,6 +158,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, c k8sUtil: k8sUtil, workflowService: workflowService, clusterService: clusterService, + blobConfigStorageService: blobConfigStorageService, } config, err := GetCdConfig() if err != nil { @@ -971,7 +972,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip if useExternalBlobStorage { //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds //from them else return. - cmConfig, secretConfig, err := impl.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) if err != nil { impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) return nil, nil, err @@ -993,46 +994,6 @@ func isExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWo //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db return isExternalRun && !useBlobStorageConfigInCdWorkflow } -func (impl *CdHandlerImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { - cmConfig := &bean2.CmBlobStorageConfig{} - secretConfig := &bean2.SecretBlobStorageConfig{} - _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(clusterConfig) - if err != nil { - impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) - return cmConfig, secretConfig, err - } - cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), impl.config.ExtBlobStorageCmName, v12.GetOptions{}) - if err != nil { - impl.Logger.Errorw("error in getting config map in external cluster", "err", err, "blobStorageCmName", impl.config.ExtBlobStorageCmName, "clusterName", clusterConfig.ClusterName) - return cmConfig, secretConfig, err - } - secret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), impl.config.ExtBlobStorageSecretName, v12.GetOptions{}) - if err != nil { - impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.config.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) - return cmConfig, secretConfig, err - } - //for IAM configured in S3 in external cluster, get logs/artifact will not work - if cm.Data != nil && secret.Data != nil { - err = cmConfig.SetCmBlobStorageConfig(cm.Data) - if err != nil { - fmt.Println("error marshalling external blob storage cm data to struct:", err) - return cmConfig, secretConfig, err - } - err = secretConfig.SetSecretBlobStorageConfig(secret.Data) - if err != nil { - fmt.Println("error marshalling external blob storage secret data to struct:", err) - return cmConfig, secretConfig, err - } - } - if cm.Data == nil { - fmt.Println("Data field not found in config map") - } - if secret.Data == nil { - fmt.Println("Data field not found in secret") - } - impl.Logger.Infow("fetching cm and secret from external cluster cloud provider", "ext cluster config: ", cmConfig) - return cmConfig, secretConfig, nil -} func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (WorkflowResponse, error) { workflowR, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) @@ -1183,7 +1144,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i } //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds //from them else return. - cmConfig, secretConfig, err := impl.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) if err != nil { impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) return nil, err diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 5350f37080a..f6361609ef6 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -107,9 +107,11 @@ type CiHandlerImpl struct { imageTaggingService ImageTaggingService config *CiConfig k8sCommonService k8s2.K8sCommonService + clusterService cluster.ClusterService + blobConfigStorageService BlobStorageConfigService } -func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, appListingRepository repository.AppListingRepository, K8sUtil *k8s.K8sUtil, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository3.EnvironmentRepository, imageTaggingService ImageTaggingService, k8sCommonService k8s2.K8sCommonService) *CiHandlerImpl { +func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, appListingRepository repository.AppListingRepository, K8sUtil *k8s.K8sUtil, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository3.EnvironmentRepository, imageTaggingService ImageTaggingService, k8sCommonService k8s2.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, ciService: ciService, @@ -131,6 +133,8 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline envRepository: envRepository, imageTaggingService: imageTaggingService, k8sCommonService: k8sCommonService, + clusterService: clusterService, + blobConfigStorageService: blobConfigStorageService, } config, err := GetCiConfig() if err != nil { @@ -797,7 +801,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC return nil, nil, errors.New("logs-not-stored-in-repository") } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == WorkflowCancel { impl.Logger.Errorw("err", "err", err) - return impl.getLogsFromRepository(pipelineId, ciWorkflow) + return impl.getLogsFromRepository(pipelineId, ciWorkflow, clusterConfig, isExt) } impl.Logger.Errorw("err", "err", err) return nil, nil, err @@ -806,7 +810,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC return logReader, cleanUp, err } -func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { +func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.Logger.Debug("getting historic logs") ciConfig, err := impl.ciWorkflowRepository.FindConfigByPipelineId(pipelineId) @@ -851,6 +855,18 @@ func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pip CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } + useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -866,7 +882,8 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) return nil, err } - + //TODO discuss this with subhashish, isExt is not saved in db so how will we know ext type in jobs case + useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) if !ciWorkflow.BlobStorageEnabled { return nil, errors.New("logs-not-stored-in-repository") } @@ -923,6 +940,21 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i AwsS3BaseConfig: awsS3BaseConfig, GcpBlobBaseConfig: gcpBlobBaseConfig, } + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByEnvId(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByEnvId, error in fetching clusterConfig by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } _, numBytes, err := blobStorageService.Get(request) if err != nil { impl.Logger.Errorw("error occurred while downloading file", "request", request) @@ -982,6 +1014,23 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } + useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByEnvId(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByEnvId, error in fetching clusterConfig by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) logs, err := ioutil.ReadFile(logsFile.Name()) if err != nil { diff --git a/wire_gen.go b/wire_gen.go index 088517c72a8..d85749f1846 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -498,14 +498,15 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sUtil, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl) + blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sUtil, ciCdConfig) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sUtil, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl) gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, clientImpl) dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) appListingViewBuilderImpl := app2.NewAppListingViewBuilderImpl(sugaredLogger) linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, appWorkflowRepositoryImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) From 734cc55af063c1b9c79271063f4f146a171e41ff Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 26 Oct 2023 23:19:41 +0530 Subject: [PATCH 18/25] code review incorporation l6 --- pkg/pipeline/WorkflowService.go | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 7f1e5ff11d7..e4fa8948ba9 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -243,26 +243,31 @@ func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *Work workflowSecrets = append(workflowSecrets, *secret) } } + //internally inducing BlobStorageCmName and BlobStorageSecretName for getting logs, caches and artifacts from //in-cluster configured blob storage, if USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW = false and isExt = true if workflowRequest.UseExternalClusterBlob { - blobDetailsConfigMap := bean.ConfigSecretMap{ - Name: impl.ciCdConfig.ExtBlobStorageCmName, - Type: "environment", - External: true, - } - workflowConfigMaps = append(workflowConfigMaps, blobDetailsConfigMap) - - blobDetailsSecret := bean.ConfigSecretMap{ - Name: impl.ciCdConfig.ExtBlobStorageSecretName, - Type: "environment", - External: true, - } - workflowSecrets = append(workflowSecrets, blobDetailsSecret) + workflowConfigMaps, workflowSecrets = impl.addExtBlobStorageCmCsInResponse(workflowConfigMaps, workflowSecrets) } return workflowConfigMaps, workflowSecrets, nil } +func (impl *WorkflowServiceImpl) addExtBlobStorageCmCsInResponse(workflowConfigMaps []bean.ConfigSecretMap, workflowSecrets []bean.ConfigSecretMap) ([]bean.ConfigSecretMap, []bean.ConfigSecretMap) { + blobDetailsConfigMap := bean.ConfigSecretMap{ + Name: impl.ciCdConfig.ExtBlobStorageCmName, + Type: "environment", + External: true, + } + workflowConfigMaps = append(workflowConfigMaps, blobDetailsConfigMap) + + blobDetailsSecret := bean.ConfigSecretMap{ + Name: impl.ciCdConfig.ExtBlobStorageSecretName, + Type: "environment", + External: true, + } + workflowSecrets = append(workflowSecrets, blobDetailsSecret) + return workflowConfigMaps, workflowSecrets +} func (impl *WorkflowServiceImpl) updateBlobStorageConfig(workflowRequest *WorkflowRequest, workflowTemplate *bean3.WorkflowTemplate) { workflowTemplate.BlobStorageConfigured = workflowRequest.BlobStorageConfigured && (workflowRequest.CheckBlobStorageConfig(impl.ciCdConfig) || !workflowRequest.IsExtRun) From bd3e83d7ad12b61aaeddc29a1dda86d4ce85243c Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 27 Oct 2023 11:27:03 +0530 Subject: [PATCH 19/25] minor fix for job tyoe --- .../sql/repository/pipelineConfig/CiWorkflowRepository.go | 5 ++--- pkg/pipeline/CiHandler.go | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index d4fec2330de..9b1477aa249 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -19,7 +19,6 @@ package pipelineConfig import ( "fmt" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/go-pg/pg" "go.uber.org/zap" "time" @@ -78,8 +77,8 @@ type CiWorkflow struct { CiPipeline *CiPipeline } -func (r *CiWorkflow) IsJobType() bool { - return r.CiPipeline.App.AppType == helper.Job +func (r *CiWorkflow) IsExternalRunInJobType() bool { + return r.EnvironmentId != 0 } type WorkflowWithArtifact struct { diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 27bf95db963..f2e381e4c88 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -885,7 +885,7 @@ func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pip CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) + useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) if useExternalBlobStorage { //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds //from them else return. @@ -912,8 +912,7 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) return nil, err } - //TODO discuss this with subhashish, isExt is not saved in db so how will we know ext type in jobs case - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) + useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) if !ciWorkflow.BlobStorageEnabled { return nil, errors.New("logs-not-stored-in-repository") } @@ -1014,7 +1013,6 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, return nil, err } } - if ciConfig.LogsBucket == "" { ciConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() } @@ -1044,7 +1042,7 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) + useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) if useExternalBlobStorage { clusterConfig, err := impl.clusterService.GetClusterConfigByEnvId(ciWorkflow.EnvironmentId) if err != nil { From c616e88de6100814f0020c58f328368041417cff Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 27 Oct 2023 12:14:37 +0530 Subject: [PATCH 20/25] minor fix --- pkg/cluster/ClusterService.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index dac1674321a..992b377518f 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -1115,15 +1115,14 @@ func (impl ClusterServiceImpl) GetClusterConfigByClusterId(clusterId int) (*k8s. } func (impl ClusterServiceImpl) GetClusterConfigByEnvId(envId int) (*k8s.ClusterConfig, error) { - clusterBean, err := impl.environmentService.FindClusterByEnvId(envId) + envBean, err := impl.environmentService.FindById(envId) if err != nil { - impl.logger.Errorw("error in getting clusterBean by envId", "err", err, "envId", envId) + impl.logger.Errorw("error in getting envBean by envId", "err", err, "envId", envId) return nil, err } - rq := *clusterBean - clusterConfig, err := rq.GetClusterConfig() + clusterConfig, err := impl.GetClusterConfigByClusterId(envBean.ClusterId) if err != nil { - impl.logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) + impl.logger.Errorw("error in getting cluster config by env id", "err", err, "envId", envId) return nil, err } return clusterConfig, nil From e25d2fcd166910b6ef897ecaa5fe471c6af6b7b4 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 27 Oct 2023 12:26:30 +0530 Subject: [PATCH 21/25] code review incorporation --- pkg/pipeline/BlobStorageConfigService.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go index 0f1ce5a7146..97483f0db6e 100644 --- a/pkg/pipeline/BlobStorageConfigService.go +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -32,17 +32,19 @@ func NewBlobStorageConfigServiceImpl(Logger *zap.SugaredLogger, k8sUtil *k8s.K8s func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { cmConfig := &bean2.CmBlobStorageConfig{} secretConfig := &bean2.SecretBlobStorageConfig{} + ctx := context.Background() + opts := v12.GetOptions{} _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(&k8s.ClusterConfig{}) if err != nil { impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) return cmConfig, secretConfig, err } - cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), impl.ciCdConfig.ExtBlobStorageCmName, v12.GetOptions{}) + cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(ctx, impl.ciCdConfig.ExtBlobStorageCmName, opts) if err != nil { impl.Logger.Errorw("error in getting config map in external cluster", "err", err, "blobStorageCmName", impl.ciCdConfig.ExtBlobStorageCmName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err } - secret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), impl.ciCdConfig.ExtBlobStorageSecretName, v12.GetOptions{}) + secret, err := kubeClient.CoreV1().Secrets(namespace).Get(ctx, impl.ciCdConfig.ExtBlobStorageSecretName, opts) if err != nil { impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.ciCdConfig.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err From 67cdff22928535770134abcefceb17eb6032f6a1 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 27 Oct 2023 12:34:31 +0530 Subject: [PATCH 22/25] code review incorporation --- pkg/pipeline/BlobStorageConfigService.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go index 97483f0db6e..6ac4a546feb 100644 --- a/pkg/pipeline/BlobStorageConfigService.go +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -34,17 +34,19 @@ func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternal secretConfig := &bean2.SecretBlobStorageConfig{} ctx := context.Background() opts := v12.GetOptions{} + extBlobStorageCmName := impl.ciCdConfig.ExtBlobStorageCmName + extBlobStorageSecretName := impl.ciCdConfig.ExtBlobStorageSecretName _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(&k8s.ClusterConfig{}) if err != nil { impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) return cmConfig, secretConfig, err } - cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(ctx, impl.ciCdConfig.ExtBlobStorageCmName, opts) + cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(ctx, extBlobStorageCmName, opts) if err != nil { impl.Logger.Errorw("error in getting config map in external cluster", "err", err, "blobStorageCmName", impl.ciCdConfig.ExtBlobStorageCmName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err } - secret, err := kubeClient.CoreV1().Secrets(namespace).Get(ctx, impl.ciCdConfig.ExtBlobStorageSecretName, opts) + secret, err := kubeClient.CoreV1().Secrets(namespace).Get(ctx, extBlobStorageSecretName, opts) if err != nil { impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.ciCdConfig.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err From 7f82b16a01e17810bb1ec51b35c774775af9eba9 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 27 Oct 2023 12:40:47 +0530 Subject: [PATCH 23/25] code review incorporation --- pkg/pipeline/BlobStorageConfigService.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go index 6ac4a546feb..c700ff16bcc 100644 --- a/pkg/pipeline/BlobStorageConfigService.go +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -32,21 +32,22 @@ func NewBlobStorageConfigServiceImpl(Logger *zap.SugaredLogger, k8sUtil *k8s.K8s func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { cmConfig := &bean2.CmBlobStorageConfig{} secretConfig := &bean2.SecretBlobStorageConfig{} - ctx := context.Background() - opts := v12.GetOptions{} - extBlobStorageCmName := impl.ciCdConfig.ExtBlobStorageCmName - extBlobStorageSecretName := impl.ciCdConfig.ExtBlobStorageSecretName _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(&k8s.ClusterConfig{}) if err != nil { impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) return cmConfig, secretConfig, err } - cm, err := kubeClient.CoreV1().ConfigMaps(namespace).Get(ctx, extBlobStorageCmName, opts) + cv1 := kubeClient.CoreV1() + ctx := context.Background() + opts := v12.GetOptions{} + cmName := impl.ciCdConfig.ExtBlobStorageCmName + secretName := impl.ciCdConfig.ExtBlobStorageSecretName + cm, err := cv1.ConfigMaps(namespace).Get(ctx, cmName, opts) if err != nil { impl.Logger.Errorw("error in getting config map in external cluster", "err", err, "blobStorageCmName", impl.ciCdConfig.ExtBlobStorageCmName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err } - secret, err := kubeClient.CoreV1().Secrets(namespace).Get(ctx, extBlobStorageSecretName, opts) + secret, err := cv1.Secrets(namespace).Get(ctx, secretName, opts) if err != nil { impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.ciCdConfig.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) return cmConfig, secretConfig, err From 2bf4838357003a7658d7879d5e280e263596af8a Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 30 Oct 2023 19:18:58 +0530 Subject: [PATCH 24/25] minor fix after debugging error --- pkg/pipeline/BlobStorageConfigService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go index c700ff16bcc..20bd9a58c5e 100644 --- a/pkg/pipeline/BlobStorageConfigService.go +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -32,7 +32,7 @@ func NewBlobStorageConfigServiceImpl(Logger *zap.SugaredLogger, k8sUtil *k8s.K8s func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { cmConfig := &bean2.CmBlobStorageConfig{} secretConfig := &bean2.SecretBlobStorageConfig{} - _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(&k8s.ClusterConfig{}) + _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(clusterConfig) if err != nil { impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) return cmConfig, secretConfig, err From b5c05e44846fe4574efc794d3c399e09c05087be Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 31 Oct 2023 18:07:19 +0530 Subject: [PATCH 25/25] common lib version upgrade from 0.0.3 to 0.0.4 --- go.mod | 2 +- go.sum | 4 ++-- .../common-lib/blob-storage/AwsS3Blob.go | 2 ++ .../common-lib/blob-storage/BlobUtils.go | 16 ++++++++++++++++ vendor/modules.txt | 2 +- 5 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go diff --git a/go.mod b/go.mod index e7e5df8f94c..79e2bd138a3 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.31 - github.com/devtron-labs/common-lib v0.0.3 + github.com/devtron-labs/common-lib v0.0.4 github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 diff --git a/go.sum b/go.sum index 7cd7f50f86c..8315b81d4e6 100644 --- a/go.sum +++ b/go.sum @@ -293,8 +293,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.31 h1:CEMLek3JnMuH9ULsC6BHNJr+NiyGzBd4lgdSxH2IGnc= github.com/devtron-labs/authenticator v0.4.31/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.3 h1:9I1czMTU6+IeNpIPBydVibTRu8H9JQmsNi7JXTEgI/8= -github.com/devtron-labs/common-lib v0.0.3/go.mod h1:x6OdUIo2z9kxXtBfz7fJEfD4s8kiAtEmlApozOf7ECM= +github.com/devtron-labs/common-lib v0.0.4 h1:kKdhQnzy9cK+fDdcbh2JmjLGfzRF67VqtsbTHKyK47Q= +github.com/devtron-labs/common-lib v0.0.4/go.mod h1:x6OdUIo2z9kxXtBfz7fJEfD4s8kiAtEmlApozOf7ECM= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go b/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go index 6c41651ddfe..a574f8e0e85 100644 --- a/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go +++ b/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go @@ -29,7 +29,9 @@ func (impl *AwsS3Blob) UploadBlob(request *BlobStorageRequest, err error) error if s3BaseConfig.Region != "" { cmdArgs = append(cmdArgs, "--region", s3BaseConfig.Region) } + command := exec.Command("aws", cmdArgs...) + setAWSEnvironmentVariables(s3BaseConfig, command) err = utils.RunCommand(command) return err } diff --git a/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go b/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go new file mode 100644 index 00000000000..70f77826410 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go @@ -0,0 +1,16 @@ +package blob_storage + +import ( + "fmt" + "os" + "os/exec" +) + +func setAWSEnvironmentVariables(s3Config *AwsS3BaseConfig, command *exec.Cmd) { + if s3Config.AccessKey != "" && s3Config.Passkey != "" { + command.Env = append(os.Environ(), + fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", s3Config.AccessKey), + fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", s3Config.Passkey), + ) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 5b4d496363c..1b3b37b6dce 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -349,7 +349,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.3 +# github.com/devtron-labs/common-lib v0.0.4 ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/pubsub-lib