Skip to content

Commit

Permalink
soft limit memory by options
Browse files Browse the repository at this point in the history
  • Loading branch information
@CMGS
CMGS committed Jul 30, 2018
1 parent 53bfab4 commit d5573b5
Show file tree
Hide file tree
Showing 10 changed files with 302 additions and 280 deletions.
5 changes: 3 additions & 2 deletions cluster/calcium/create_container.go
View file
Expand Up @@ -300,9 +300,9 @@ func (c *Calcium) makeContainerOptions(index int, quota types.CPUMap, opts *type

var resource enginecontainer.Resources
if favor == scheduler.CPU_PRIOR {
resource = makeCPUPriorSetting(c.config.Scheduler.ShareBase, quota, opts.Memory)
resource = makeCPUPriorSetting(c.config.Scheduler.ShareBase, quota, opts.Memory, opts.SoftLimit)
} else if favor == scheduler.MEMORY_PRIOR {
resource = makeMemoryPriorSetting(opts.Memory, opts.CPUQuota)
resource = makeMemoryPriorSetting(opts.Memory, opts.CPUQuota, opts.SoftLimit)
} else {
return nil, nil, nil, "", fmt.Errorf("favor not support %s", favor)
}
Expand Down Expand Up @@ -357,6 +357,7 @@ func (c *Calcium) createAndStartContainer(
Hook: opts.Entrypoint.Hook,
Privileged: opts.Entrypoint.Privileged,
Engine: node.Engine,
SoftLimit: opts.SoftLimit,
}
createContainerMessage := &types.CreateContainerMessage{
Podname: container.Podname,
Expand Down
25 changes: 17 additions & 8 deletions cluster/calcium/helper.go
View file
Expand Up @@ -41,18 +41,22 @@ func (c *Calcium) Lock(ctx context.Context, name string, timeout int) (lock.Dist
}

// create container begin
func makeMemoryPriorSetting(memory int64, cpu float64) enginecontainer.Resources {
func makeMemoryPriorSetting(memory int64, cpu float64, softlimit bool) enginecontainer.Resources {
resource := enginecontainer.Resources{}
if cpu > 0 {
resource.CPUPeriod = cluster.CPUPeriodBase
resource.CPUQuota = int64(cpu * float64(cluster.CPUPeriodBase))
}
resource.Memory = memory
resource.MemorySwap = memory
if softlimit {
resource.MemoryReservation = memory
} else {
resource.Memory = memory
resource.MemorySwap = memory
}
return resource
}

func makeCPUPriorSetting(shareBase int, quota types.CPUMap, memory int64) enginecontainer.Resources {
func makeCPUPriorSetting(shareBase int, quota types.CPUMap, memory int64, softlimit bool) enginecontainer.Resources {
// calculate CPUShares and CPUSet
// scheduler won't return more than 1 share quota
// so the smallest share is the share numerator
Expand All @@ -66,11 +70,16 @@ func makeCPUPriorSetting(shareBase int, quota types.CPUMap, memory int64) engine
}
cpuShares := int64(float64(shareQuota) / float64(shareBase) * float64(cluster.CPUShareBase))
cpuSetCpus := strings.Join(cpuIDs, ",")
log.Debugf("[makeCPUPriorSetting] CPU core %v CPU share %v Memory soft limit %v", cpuSetCpus, cpuShares, memory)
log.Debugf("[makeCPUPriorSetting] CPU core %v CPU share %v Memory limit %v", cpuSetCpus, cpuShares, memory)
resource := enginecontainer.Resources{
CPUShares: cpuShares,
CpusetCpus: cpuSetCpus,
MemoryReservation: memory,
CPUShares: cpuShares,
CpusetCpus: cpuSetCpus,
}
if softlimit {
resource.MemoryReservation = memory
} else {
resource.Memory = memory
resource.MemorySwap = memory
}
return resource
}
Expand Down
1 change: 1 addition & 0 deletions cluster/calcium/mock_test.go
View file
Expand Up @@ -488,6 +488,7 @@ func initMockConfig() {
Nodename: updatenodename,
Name: "hello_hi_123",
CPU: coretypes.CPUMap{"0": 10},
Quota: 1.0,
Memory: appmemory,
}
rContainers = append(rContainers, rContainer)
Expand Down
37 changes: 14 additions & 23 deletions cluster/calcium/realloc.go
View file
Expand Up @@ -6,7 +6,6 @@ import (
"sync"

enginecontainer "github.com/docker/docker/api/types/container"
"github.com/projecteru2/core/cluster"
"github.com/projecteru2/core/scheduler"
"github.com/projecteru2/core/types"
log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -160,43 +159,35 @@ func (c *Calcium) doUpdateContainerWithMemoryPrior(
cpu float64, memory int64) {

for _, container := range containers {
containerJSON, err := container.Inspect(ctx)
if err != nil {
log.Errorf("[doUpdateContainerWithMemoryPrior] get container failed %v", err)
ch <- &types.ReallocResourceMessage{ContainerID: containerJSON.ID, Success: false}
continue
}
newCPU := container.Quota + cpu
newMemory := container.Memory + memory

cpuQuota := int64(cpu * float64(cluster.CPUPeriodBase))
newCPUQuota := containerJSON.HostConfig.CPUQuota + cpuQuota
newMemory := containerJSON.HostConfig.Memory + memory
// 内存不能低于 4MB
if newCPUQuota <= 0 || newMemory <= minMemory {
log.Warnf("[doUpdateContainerWithMemoryPrior] new resource invaild %s, %d, %d", containerJSON.ID, newCPUQuota, newMemory)
ch <- &types.ReallocResourceMessage{ContainerID: containerJSON.ID, Success: false}
if newCPU <= 0 || newMemory <= minMemory {
log.Errorf("[doUpdateContainerWithMemoryPrior] new resource invaild %s, %f, %d", container.ID, newCPU, newMemory)
ch <- &types.ReallocResourceMessage{ContainerID: container.ID, Success: false}
continue
}
newCPU := float64(newCPUQuota) / float64(cluster.CPUPeriodBase)
log.Debugf("[doUpdateContainerWithMemoryPrior] quota:%d, cpu: %f, mem: %d", newCPUQuota, newCPU, newMemory)
log.Debugf("[doUpdateContainerWithMemoryPrior] cpu: %f, mem: %d", newCPU, newMemory)

// CPUQuota not cpu
newResource := makeMemoryPriorSetting(newMemory, newCPU)
newResource := makeMemoryPriorSetting(newMemory, newCPU, container.SoftLimit)
updateConfig := enginecontainer.UpdateConfig{Resources: newResource}
if err := updateContainer(ctx, containerJSON.ID, node, updateConfig); err != nil {
log.Errorf("[doUpdateContainerWithMemoryPrior] update container failed %v, %s", err, containerJSON.ID)
ch <- &types.ReallocResourceMessage{ContainerID: containerJSON.ID, Success: false}
if err := updateContainer(ctx, container.ID, node, updateConfig); err != nil {
log.Errorf("[doUpdateContainerWithMemoryPrior] update container failed %v, %s", err, container.ID)
ch <- &types.ReallocResourceMessage{ContainerID: container.ID, Success: false}
// 如果是增加内存，失败的时候应该把内存还回去
if memory > 0 {
if err := c.store.UpdateNodeResource(ctx, podname, node.Name, types.CPUMap{}, memory, "+"); err != nil {
log.Errorf("[doUpdateContainerWithMemoryPrior] failed to set mem back %s", containerJSON.ID)
log.Errorf("[doUpdateContainerWithMemoryPrior] failed to set mem back %s", container.ID)
}
}
continue
}
// 如果是要降低内存，当执行成功的时候需要把内存还回去
if memory < 0 {
if err := c.store.UpdateNodeResource(ctx, podname, node.Name, types.CPUMap{}, -memory, "+"); err != nil {
log.Errorf("[doUpdateContainerWithMemoryPrior] failed to set mem back %s", containerJSON.ID)
log.Errorf("[doUpdateContainerWithMemoryPrior] failed to set mem back %s", container.ID)
}
}

Expand All @@ -208,7 +199,7 @@ func (c *Calcium) doUpdateContainerWithMemoryPrior(
ch <- &types.ReallocResourceMessage{ContainerID: container.ID, Success: false}
return
}
ch <- &types.ReallocResourceMessage{ContainerID: containerJSON.ID, Success: true}
ch <- &types.ReallocResourceMessage{ContainerID: container.ID, Success: true}
}
}

Expand Down Expand Up @@ -337,7 +328,7 @@ func (c *Calcium) doReallocContainersWithCPUPrior(
containers := nodeContainers[node]
for index, container := range containers {
cpuPlan := cpuset[index]
resource := makeCPUPriorSetting(c.config.Scheduler.ShareBase, cpuPlan, requireMemory)
resource := makeCPUPriorSetting(c.config.Scheduler.ShareBase, cpuPlan, requireMemory, container.SoftLimit)
updateConfig := enginecontainer.UpdateConfig{Resources: resource}
if err := updateContainer(ctx, container.ID, node, updateConfig); err != nil {
log.Errorf("[doReallocContainersWithCPUPrior] update container failed %v", err)
Expand Down

0 comments on commit d5573b5

Please sign in to comment.