diff --git a/engine/hatchery/marathon/marathon.go b/engine/hatchery/marathon/marathon.go index b8ab60dd4f..485c29ab6c 100644 --- a/engine/hatchery/marathon/marathon.go +++ b/engine/hatchery/marathon/marathon.go @@ -198,8 +198,13 @@ func (h *HatcheryMarathon) CanSpawn(model *sdk.Model, jobID int64, requirements log.Info("CanSpawn> Error on h.marathonClient.Deployments() : %s", errd) return false } - // Do not DOS marathon, if deployment queue is longer than MarathonMaxConcurrentSpawn (default 10) - if h.Config.MarathonMaxConcurrentSpawn > 0 && len(deployments) >= h.Config.MarathonMaxConcurrentSpawn { + // Do not DOS marathon, if deployment queue is longer than MaxConcurrentProvisioning (default 10) + maxProvisionning := h.Configuration().Provision.MaxConcurrentProvisioning + if maxProvisionning == 0 { + maxProvisionning = 10 + } + + if len(deployments) >= maxProvisionning { log.Info("CanSpawn> %d item in deployment queue, waiting", len(deployments)) return false } diff --git a/engine/hatchery/marathon/types.go b/engine/hatchery/marathon/types.go index a139bc368c..685aa01bb5 100644 --- a/engine/hatchery/marathon/types.go +++ b/engine/hatchery/marathon/types.go @@ -24,9 +24,6 @@ type HatcheryConfiguration struct { // MarathonPassword "marathon-password" MarathonPassword string `mapstructure:"password" toml:"password" default:"" commented:"false" comment:"Marathon Password, you need a marathon User to use it"` - // MarathonMaxConcurrentSpawn "marathon-max-deployments" - MarathonMaxConcurrentSpawn int `mapstructure:"maxConcurrentSpawn" toml:"maxConcurrentSpawn" default:"10" commented:"false" comment:"Max elements in deployment queue. The hatchery will wait less than x elements in deployment queue before spawning new CDS worker."` - // MarathonLabelsStr "marathon-labels" MarathonLabels string `mapstructure:"labels" toml:"labels" default:"" commented:"false" comment:"Use this option if you want to add labels on workers spawned by this hatchery.\n Format: MarathonLabels = \"A_LABEL=value-of-label,B_LABEL=value-of-label-b\""` diff --git a/sdk/hatchery/hatchery.go b/sdk/hatchery/hatchery.go index 88d503aca8..f70bb4203b 100644 --- a/sdk/hatchery/hatchery.go +++ b/sdk/hatchery/hatchery.go @@ -37,12 +37,13 @@ type CommonConfiguration struct { MaxHeartbeatFailures int `toml:"maxHeartbeatFailures" default:"10" comment:"Maximum allowed consecutives failures on heatbeat routine"` } `toml:"api"` Provision struct { - Disabled bool `toml:"disabled" default:"false" comment:"Disabled provisioning. Format:true or false"` - Frequency int `toml:"frequency" default:"30" comment:"Check provisioning each n Seconds"` - MaxWorker int `toml:"maxWorker" default:"10" comment:"Maximum allowed simultaneous workers"` - GraceTimeQueued int `toml:"graceTimeQueued" default:"4" comment:"if worker is queued less than this value (seconds), hatchery does not take care of it"` - RegisterFrequency int `toml:"registerFrequency" default:"60" comment:"Check if some worker model have to be registered each n Seconds"` - WorkerLogsOptions struct { + Disabled bool `toml:"disabled" default:"false" comment:"Disabled provisioning. Format:true or false"` + Frequency int `toml:"frequency" default:"30" comment:"Check provisioning each n Seconds"` + MaxWorker int `toml:"maxWorker" default:"10" comment:"Maximum allowed simultaneous workers"` + MaxConcurrentProvisioning int `toml:"maxConcurrentProvisioning" default:"10" comment:"Maximum allowed simultaneous workers provisioning"` + GraceTimeQueued int `toml:"graceTimeQueued" default:"4" comment:"if worker is queued less than this value (seconds), hatchery does not take care of it"` + RegisterFrequency int `toml:"registerFrequency" default:"60" comment:"Check if some worker model have to be registered each n Seconds"` + WorkerLogsOptions struct { Graylog struct { Host string `toml:"host" comment:"Example: thot.ovh.com"` Port int `toml:"port" comment:"Example: 12202"` @@ -130,8 +131,13 @@ func receiveJob(h Interface, isWorkflowJob bool, execGroups []sdk.Group, jobID i return false, false, nil } + maxProvisionning := h.Configuration().Provision.MaxConcurrentProvisioning + if maxProvisionning == 0 { + maxProvisionning = 10 + } + n := atomic.LoadInt64(nRoutines) - if n > 10 { + if int(n) > maxProvisionning { log.Info("too many routines in same time %d", n) return false, false, nil }