Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: pre-cd pod not getting scheduled when node affinity is not present in external cluster. #3806

Merged
merged 5 commits into from
Aug 28, 2023
Merged

fix: pre-cd pod not getting scheduled when node affinity is not present in external cluster. #3806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
21773fe
separate node selector for external cluster
Ashish-devtron Aug 22, 2023
cf29519
refactor cdConfig
Ashish-devtron Aug 23, 2023
ad2651c
refactor cdConfig
Ashish-devtron Aug 23, 2023
4e73b49
check for empty taintKey
Ashish-devtron Aug 23, 2023
7ab78d6
use external node flag
Ashish-devtron Aug 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
32 changes: 25 additions & 7 deletions pkg/pipeline/CdConfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,10 @@ package pipeline

import (
"fmt"
"github.com/caarlos0/env"
blob_storage "github.com/devtron-labs/common-lib/blob-storage"
"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
"strings"

"github.com/caarlos0/env"
)

type CdConfig struct {
Expand All @@ -32,12 +31,16 @@ type CdConfig struct {
ReqCpu string `env:"CD_REQ_CI_CPU" envDefault:"0.5"`
ReqMem string `env:"CD_REQ_CI_MEM" envDefault:"3G"`
TaintKey string `env:"CD_NODE_TAINTS_KEY" envDefault:"dedicated"`
ExternalTaintKey string `env:"EXTERNAL_CD_NODE_TAINTS_KEY" envDefault:"dedicated"`
UseExternalNode bool `env:"USE_EXTERNAL_NODE" envDefault:"false"`
WorkflowServiceAccount string `env:"CD_WORKFLOW_SERVICE_ACCOUNT" envDefault:"cd-runner"`
DefaultBuildLogsKeyPrefix string `env:"DEFAULT_BUILD_LOGS_KEY_PREFIX" `
DefaultArtifactKeyPrefix string `env:"DEFAULT_CD_ARTIFACT_KEY_LOCATION" `
TaintValue string `env:"CD_NODE_TAINTS_VALUE" envDefault:"ci"`
ExternalTaintValue string `env:"EXTERNAL_CD_NODE_TAINTS_VALUE" envDefault:"ci"`
DefaultBuildLogsBucket string `env:"DEFAULT_BUILD_LOGS_BUCKET" `
NodeLabelSelector []string `env:"CD_NODE_LABEL_SELECTOR"`
ExternalNodeLabelSelector []string `env:"EXTERNAL_CD_NODE_LABEL_SELECTOR"`
CdArtifactLocationFormat string `env:"CD_ARTIFACT_LOCATION_FORMAT" envDefault:"%d/%d.zip"`
DefaultNamespace string `env:"DEFAULT_CD_NAMESPACE"`
DefaultImage string `env:"DEFAULT_CI_IMAGE"`
Expand All @@ -47,6 +50,7 @@ type CdConfig struct {
OrchestratorHost string `env:"ORCH_HOST" envDefault:"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats"`
OrchestratorToken string `env:"ORCH_TOKEN" envDefault:""`
NodeLabel map[string]string
ExternalNodeLabel map[string]string
CloudProvider blob_storage.BlobStorageType `env:"BLOB_STORAGE_PROVIDER" envDefault:"S3"`
BlobStorageEnabled bool `env:"BLOB_STORAGE_ENABLED" envDefault:"false"`
BlobStorageS3AccessKey string `env:"BLOB_STORAGE_S3_ACCESS_KEY"`
Expand Down Expand Up @@ -74,17 +78,31 @@ type CdConfig struct {
func GetCdConfig() (*CdConfig, error) {
cfg := &CdConfig{}
err := env.Parse(cfg)
cfg.NodeLabel = make(map[string]string)
for _, l := range cfg.NodeLabelSelector {
if err != nil {
return nil, err
}
cfg.NodeLabel, err = assignNodeLabelSelector(cfg.NodeLabelSelector)
if err != nil {
return nil, err
}
cfg.ExternalNodeLabel, err = assignNodeLabelSelector(cfg.ExternalNodeLabelSelector)
if err != nil {
return nil, err
}
return cfg, err
}

func assignNodeLabelSelector(labelSelector []string) (map[string]string, error) {
label := make(map[string]string)
for _, l := range labelSelector {
if l == "" {
continue
}
kv := strings.Split(l, "=")
if len(kv) != 2 {
return nil, fmt.Errorf("invalid ci node label selector %s, it must be in form key=value, key2=val2", kv)
}
cfg.NodeLabel[kv[0]] = kv[1]
label[kv[0]] = kv[1]
}

return cfg, err
return label, nil
}
23 changes: 17 additions & 6 deletions pkg/pipeline/CdWorkflowService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,17 +238,28 @@ func (impl *CdWorkflowServiceImpl) SubmitWorkflow(workflowRequest *CdWorkflowReq
workflowTemplate.Secrets = workflowSecrets

workflowTemplate.ServiceAccountName = impl.cdConfig.WorkflowServiceAccount
workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.TaintKey: impl.cdConfig.TaintValue}
workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.TaintKey, Value: impl.cdConfig.TaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}}
if workflowRequest.IsExtRun && impl.cdConfig.UseExternalNode {
if impl.cdConfig.ExternalTaintKey != "" {
workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.ExternalTaintKey: impl.cdConfig.ExternalTaintValue}
}
workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.ExternalTaintKey, Value: impl.cdConfig.ExternalTaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}}
if len(impl.cdConfig.ExternalNodeLabel) > 0 {
workflowTemplate.NodeSelector = impl.cdConfig.ExternalNodeLabel
}
} else {
if impl.cdConfig.TaintKey != "" {
workflowTemplate.NodeSelector = map[string]string{impl.cdConfig.TaintKey: impl.cdConfig.TaintValue}
}
workflowTemplate.Tolerations = []v12.Toleration{{Key: impl.cdConfig.TaintKey, Value: impl.cdConfig.TaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}}
if len(impl.cdConfig.NodeLabel) > 0 {
workflowTemplate.NodeSelector = impl.cdConfig.NodeLabel
}
}
workflowTemplate.Volumes = ExtractVolumesFromCmCs(workflowConfigMaps, workflowSecrets)
workflowTemplate.ArchiveLogs = storageConfigured
workflowTemplate.ArchiveLogs = workflowTemplate.ArchiveLogs && !ciCdTriggerEvent.CdRequest.InAppLoggingEnabled
workflowTemplate.RestartPolicy = v12.RestartPolicyNever

if len(impl.cdConfig.NodeLabel) > 0 {
workflowTemplate.NodeSelector = impl.cdConfig.NodeLabel
}

limitCpu := impl.cdConfig.LimitCpu
limitMem := impl.cdConfig.LimitMem
reqCpu := impl.cdConfig.ReqCpu
Expand Down