From 21773febd93acd2648af3987d16b401947d51d25 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Tue, 22 Aug 2023 13:04:21 +0530 Subject: [PATCH 1/5] separate node selector for external cluster --- pkg/pipeline/CdConfig.go | 32 +++++++++++++++++++++++-------- pkg/pipeline/CdWorkflowService.go | 19 ++++++++++++------ 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/pkg/pipeline/CdConfig.go b/pkg/pipeline/CdConfig.go index 382035aec04..a41a0745fb1 100644 --- a/pkg/pipeline/CdConfig.go +++ b/pkg/pipeline/CdConfig.go @@ -19,11 +19,10 @@ package pipeline import ( "fmt" + "github.com/caarlos0/env" blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "strings" - - "github.com/caarlos0/env" ) type CdConfig struct { @@ -32,12 +31,15 @@ type CdConfig struct { ReqCpu string `env:"CD_REQ_CI_CPU" envDefault:"0.5"` ReqMem string `env:"CD_REQ_CI_MEM" envDefault:"3G"` TaintKey string `env:"CD_NODE_TAINTS_KEY" envDefault:"dedicated"` + ExternalTaintKey string `env:"EXTERNAL_CD_NODE_TAINTS_KEY" envDefault:""` WorkflowServiceAccount string `env:"CD_WORKFLOW_SERVICE_ACCOUNT" envDefault:"cd-runner"` DefaultBuildLogsKeyPrefix string `env:"DEFAULT_BUILD_LOGS_KEY_PREFIX" ` DefaultArtifactKeyPrefix string `env:"DEFAULT_CD_ARTIFACT_KEY_LOCATION" ` TaintValue string `env:"CD_NODE_TAINTS_VALUE" envDefault:"ci"` + ExternalTaintValue string `env:"EXTERNAL_CD_NODE_TAINTS_VALUE" envDefault:""` DefaultBuildLogsBucket string `env:"DEFAULT_BUILD_LOGS_BUCKET" ` NodeLabelSelector []string `env:"CD_NODE_LABEL_SELECTOR"` + ExternalNodeLabelSelector []string `env:"EXTERNAL_CD_NODE_LABEL_SELECTOR"` CdArtifactLocationFormat string `env:"CD_ARTIFACT_LOCATION_FORMAT" envDefault:"%d/%d.zip"` DefaultNamespace string `env:"DEFAULT_CD_NAMESPACE"` DefaultImage string `env:"DEFAULT_CI_IMAGE"` @@ -47,6 +49,7 @@ type CdConfig struct { OrchestratorHost string `env:"ORCH_HOST" envDefault:"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats"` OrchestratorToken string `env:"ORCH_TOKEN" envDefault:""` NodeLabel map[string]string + ExternalNodeLabel map[string]string CloudProvider blob_storage.BlobStorageType `env:"BLOB_STORAGE_PROVIDER" envDefault:"S3"` BlobStorageEnabled bool `env:"BLOB_STORAGE_ENABLED" envDefault:"false"` BlobStorageS3AccessKey string `env:"BLOB_STORAGE_S3_ACCESS_KEY"` @@ -74,17 +77,30 @@ type CdConfig struct { func GetCdConfig() (*CdConfig, error) { cfg := &CdConfig{} err := env.Parse(cfg) - cfg.NodeLabel = make(map[string]string) - for _, l := range cfg.NodeLabelSelector { + if err != nil { + return nil, err + } + err = assignNodeLabelSelector(cfg.NodeLabel, cfg.NodeLabelSelector) + if err != nil { + return nil, err + } + err = assignNodeLabelSelector(cfg.ExternalNodeLabel, cfg.ExternalNodeLabelSelector) + if err != nil { + return nil, err + } + return cfg, err +} + +func assignNodeLabelSelector(label map[string]string, labelSelector []string) error { + for _, l := range labelSelector { if l == "" { continue } kv := strings.Split(l, "=") if len(kv) != 2 { - return nil, fmt.Errorf("invalid ci node label selector %s, it must be in form key=value, key2=val2", kv) + return fmt.Errorf("invalid ci node label selector %s, it must be in form key=value, key2=val2", kv) } - cfg.NodeLabel[kv[0]] = kv[1] + label[kv[0]] = kv[1] } - - return cfg, err + return nil } diff --git a/pkg/pipeline/CdWorkflowService.go b/pkg/pipeline/CdWorkflowService.go index 5c982095901..43d9083b146 100644 --- a/pkg/pipeline/CdWorkflowService.go +++ b/pkg/pipeline/CdWorkflowService.go @@ -238,17 +238,24 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq workflowTemplate.Secrets = workflowSecrets workflowTemplate.ServiceAccountName = impl.cdConfig.WorkflowServiceAccount - workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.TaintKey: impl.cdConfig.TaintValue} - workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.TaintKey, Value: impl.cdConfig.TaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}} + if workflowRequest.IsExtRun { + workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.ExternalTaintKey: impl.cdConfig.ExternalTaintValue} + workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.ExternalTaintKey, Value: impl.cdConfig.ExternalTaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}} + if len(impl.cdConfig.ExternalNodeLabel) > 0 { + workflowTemplate.NodeSelector = impl.cdConfig.ExternalNodeLabel + } + } else { + workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.TaintKey: impl.cdConfig.TaintValue} + workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.TaintKey, Value: impl.cdConfig.TaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}} + if len(impl.cdConfig.NodeLabel) > 0 { + workflowTemplate.NodeSelector = impl.cdConfig.NodeLabel + } + } workflowTemplate.Volumes = ExtractVolumesFromCmCs(workflowConfigMaps, workflowSecrets) workflowTemplate.ArchiveLogs = storageConfigured workflowTemplate.ArchiveLogs = workflowTemplate.ArchiveLogs && !ciCdTriggerEvent.CdRequest.InAppLoggingEnabled workflowTemplate.RestartPolicy = v12.RestartPolicyNever - if len(impl.cdConfig.NodeLabel) > 0 { - workflowTemplate.NodeSelector = impl.cdConfig.NodeLabel - } - limitCpu := impl.cdConfig.LimitCpu limitMem := impl.cdConfig.LimitMem reqCpu := impl.cdConfig.ReqCpu From cf29519bec22c8bc3dbbca397f3cf8dade8f0686 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Wed, 23 Aug 2023 12:22:43 +0530 Subject: [PATCH 2/5] refactor cdConfig --- pkg/pipeline/CdConfig.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/pipeline/CdConfig.go b/pkg/pipeline/CdConfig.go index a41a0745fb1..298f31bd5d2 100644 --- a/pkg/pipeline/CdConfig.go +++ b/pkg/pipeline/CdConfig.go @@ -80,27 +80,28 @@ func GetCdConfig() (*CdConfig, error) { if err != nil { return nil, err } - err = assignNodeLabelSelector(cfg.NodeLabel, cfg.NodeLabelSelector) + cfg.NodeLabel, err = assignNodeLabelSelector(cfg.NodeLabelSelector) if err != nil { return nil, err } - err = assignNodeLabelSelector(cfg.ExternalNodeLabel, cfg.ExternalNodeLabelSelector) + cfg.ExternalNodeLabel, err = assignNodeLabelSelector(cfg.ExternalNodeLabelSelector) if err != nil { return nil, err } return cfg, err } -func assignNodeLabelSelector(label map[string]string, labelSelector []string) error { +func assignNodeLabelSelector(labelSelector []string) (map[string]string, error) { + label := make(map[string]string) for _, l := range labelSelector { if l == "" { continue } kv := strings.Split(l, "=") if len(kv) != 2 { - return fmt.Errorf("invalid ci node label selector %s, it must be in form key=value, key2=val2", kv) + return nil, fmt.Errorf("invalid ci node label selector %s, it must be in form key=value, key2=val2", kv) } label[kv[0]] = kv[1] } - return nil + return label, nil } From ad2651c633241cdec70bea12f04415e741d02f9e Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Wed, 23 Aug 2023 12:58:36 +0530 Subject: [PATCH 3/5] refactor cdConfig --- pkg/pipeline/CdConfig.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/CdConfig.go b/pkg/pipeline/CdConfig.go index 298f31bd5d2..89591b3201d 100644 --- a/pkg/pipeline/CdConfig.go +++ b/pkg/pipeline/CdConfig.go @@ -31,12 +31,12 @@ type CdConfig struct { ReqCpu string `env:"CD_REQ_CI_CPU" envDefault:"0.5"` ReqMem string `env:"CD_REQ_CI_MEM" envDefault:"3G"` TaintKey string `env:"CD_NODE_TAINTS_KEY" envDefault:"dedicated"` - ExternalTaintKey string `env:"EXTERNAL_CD_NODE_TAINTS_KEY" envDefault:""` + ExternalTaintKey string `env:"EXTERNAL_CD_NODE_TAINTS_KEY" envDefault:"dedicated"` WorkflowServiceAccount string `env:"CD_WORKFLOW_SERVICE_ACCOUNT" envDefault:"cd-runner"` DefaultBuildLogsKeyPrefix string `env:"DEFAULT_BUILD_LOGS_KEY_PREFIX" ` DefaultArtifactKeyPrefix string `env:"DEFAULT_CD_ARTIFACT_KEY_LOCATION" ` TaintValue string `env:"CD_NODE_TAINTS_VALUE" envDefault:"ci"` - ExternalTaintValue string `env:"EXTERNAL_CD_NODE_TAINTS_VALUE" envDefault:""` + ExternalTaintValue string `env:"EXTERNAL_CD_NODE_TAINTS_VALUE" envDefault:"ci"` DefaultBuildLogsBucket string `env:"DEFAULT_BUILD_LOGS_BUCKET" ` NodeLabelSelector []string `env:"CD_NODE_LABEL_SELECTOR"` ExternalNodeLabelSelector []string `env:"EXTERNAL_CD_NODE_LABEL_SELECTOR"` From 4e73b49cf88280c7c66dd2bdae31e606405b6657 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Wed, 23 Aug 2023 15:54:02 +0530 Subject: [PATCH 4/5] check for empty taintKey --- pkg/pipeline/CdWorkflowService.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/CdWorkflowService.go b/pkg/pipeline/CdWorkflowService.go index 43d9083b146..e1338cc63c2 100644 --- a/pkg/pipeline/CdWorkflowService.go +++ b/pkg/pipeline/CdWorkflowService.go @@ -239,13 +239,17 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq workflowTemplate.ServiceAccountName = impl.cdConfig.WorkflowServiceAccount if workflowRequest.IsExtRun { - workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.ExternalTaintKey: impl.cdConfig.ExternalTaintValue} + if impl.cdConfig.ExternalTaintKey != "" { + workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.ExternalTaintKey: impl.cdConfig.ExternalTaintValue} + } workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.ExternalTaintKey, Value: impl.cdConfig.ExternalTaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}} if len(impl.cdConfig.ExternalNodeLabel) > 0 { workflowTemplate.NodeSelector = impl.cdConfig.ExternalNodeLabel } } else { - workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.TaintKey: impl.cdConfig.TaintValue} + if impl.cdConfig.TaintKey != "" { + workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.TaintKey: impl.cdConfig.TaintValue} + } workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.TaintKey, Value: impl.cdConfig.TaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}} if len(impl.cdConfig.NodeLabel) > 0 { workflowTemplate.NodeSelector = impl.cdConfig.NodeLabel From 7ab78d6dd2982e5249ba3d3f115f14f167a0d20c Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 24 Aug 2023 13:02:53 +0530 Subject: [PATCH 5/5] use external node flag --- pkg/pipeline/CdConfig.go | 1 + pkg/pipeline/CdWorkflowService.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/pipeline/CdConfig.go b/pkg/pipeline/CdConfig.go index 89591b3201d..0fb7f47443c 100644 --- a/pkg/pipeline/CdConfig.go +++ b/pkg/pipeline/CdConfig.go @@ -32,6 +32,7 @@ type CdConfig struct { ReqMem string `env:"CD_REQ_CI_MEM" envDefault:"3G"` TaintKey string `env:"CD_NODE_TAINTS_KEY" envDefault:"dedicated"` ExternalTaintKey string `env:"EXTERNAL_CD_NODE_TAINTS_KEY" envDefault:"dedicated"` + UseExternalNode bool `env:"USE_EXTERNAL_NODE" envDefault:"false"` WorkflowServiceAccount string `env:"CD_WORKFLOW_SERVICE_ACCOUNT" envDefault:"cd-runner"` DefaultBuildLogsKeyPrefix string `env:"DEFAULT_BUILD_LOGS_KEY_PREFIX" ` DefaultArtifactKeyPrefix string `env:"DEFAULT_CD_ARTIFACT_KEY_LOCATION" ` diff --git a/pkg/pipeline/CdWorkflowService.go b/pkg/pipeline/CdWorkflowService.go index e1338cc63c2..eb3155ee1e7 100644 --- a/pkg/pipeline/CdWorkflowService.go +++ b/pkg/pipeline/CdWorkflowService.go @@ -238,7 +238,7 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq workflowTemplate.Secrets = workflowSecrets workflowTemplate.ServiceAccountName = impl.cdConfig.WorkflowServiceAccount - if workflowRequest.IsExtRun { + if workflowRequest.IsExtRun && impl.cdConfig.UseExternalNode { if impl.cdConfig.ExternalTaintKey != "" { workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.ExternalTaintKey: impl.cdConfig.ExternalTaintValue} }