Skip to content
Permalink
Browse files
Updated K8s Scheduler to set Resource Limits and Requests (#3664)
What was previously used for K8s request is now used as a container limit.
There is an optional setting to either set or not set the K8s request to the same value.
  • Loading branch information
nicknezis committed Jan 18, 2021
1 parent 0f0ba18 commit 1a0d198ea301fd6346e0b1b6b7600c834874d8a0
Showing 6 changed files with 55 additions and 6 deletions.
@@ -161,6 +161,7 @@ spec:
-D heron.class.packing.algorithm=org.apache.heron.packing.binpacking.FirstFitDecreasingPacking
-D heron.class.repacking.algorithm=org.apache.heron.packing.binpacking.FirstFitDecreasingPacking
{{- end }}
-D heron.kubernetes.resource.request.mode={{ .Values.topologyResourceRequestMode }}
envFrom:
- configMapRef:
name: {{ .Release.Name }}-tools-config
@@ -42,6 +42,10 @@ heron:
# set to `-` to set base-url to the default k8s proxy URL
# set to `null` to remove the use of base_url
url: "-"

# Can be EQUAL_TO_LIMIT or NOT_SET
topologyResourceRequestMode: EQUAL_TO_LIMIT

# Topologies uploader
uploader:
class: dlog # s3
@@ -32,3 +32,9 @@ heron.scheduler.is.service: False

# docker repo for executor
heron.executor.docker.image: 'heron/heron:latest'

# logic to be used for calculating the Kubernetes resource request and limits
# value can be "NOT_SET", "EQUAL_TO_LIMIT"
# "NOT_SET" will not set a K8s request
# "EQUAL_TO_LIMIT" will set the K8s request to the same values as the K8s limit
heron.kubernetes.resource.request.mode: EQUAL_TO_LIMIT
@@ -33,6 +33,26 @@ public final class KubernetesContext extends Context {
public static final String HERON_KUBERNETES_SCHEDULER_IMAGE_PULL_POLICY =
"heron.kubernetes.scheduler.imagePullPolicy";

public enum KubernetesResourceRequestMode {
/**
* The Kubernetes Request will not be set by the Heron Kubernetes Scheduler.
* The generated pods will use the Kubernetes default values.
*/
NOT_SET,
/**
* The Kubernetes Pod Resource Request will be set to the same values as
* provided in the Resource Limit. This mode effectively guarantees the
* cpu and memory will be reserved.
*/
EQUAL_TO_LIMIT;
}
/**
* This config item is used to determine how to configure the K8s Resource Request.
* The format of this flag is the string encoded values of the
* underlying KubernetesRequestMode value.
*/
public static final String KUBERNETES_RESOURCE_REQUEST_MODE =
"heron.kubernetes.resource.request.mode";

public static final String HERON_KUBERNETES_VOLUME_NAME = "heron.kubernetes.volume.name";
public static final String HERON_KUBERNETES_VOLUME_TYPE = "heron.kubernetes.volume.type";
@@ -87,6 +107,11 @@ public static boolean hasImagePullPolicy(Config config) {
return isNotEmpty(imagePullPolicy);
}

public static KubernetesResourceRequestMode getKubernetesRequestMode(Config config) {
return KubernetesResourceRequestMode.valueOf(
config.getStringValue(KUBERNETES_RESOURCE_REQUEST_MODE));
}

static String getVolumeType(Config config) {
return config.getStringValue(HERON_KUBERNETES_VOLUME_TYPE);
}
@@ -76,4 +76,5 @@ Resource getContainerResource(PackingPlan packingPlan) {
abstract boolean killTopology();

abstract boolean restart(int shardId);

}
@@ -483,12 +483,24 @@ private V1Container getContainer(List<String> executorCommand, Resource resource

// set container resources
final V1ResourceRequirements resourceRequirements = new V1ResourceRequirements();
final Map<String, Quantity> requests = new HashMap<>();
requests.put(KubernetesConstants.MEMORY,
Quantity.fromString(KubernetesUtils.Megabytes(resource.getRam())));
requests.put(KubernetesConstants.CPU,
Quantity.fromString(Double.toString(roundDecimal(resource.getCpu(), 3))));
resourceRequirements.setRequests(requests);
// Set the Kubernetes container resource limit
final Map<String, Quantity> limits = new HashMap<>();
limits.put(KubernetesConstants.MEMORY,
Quantity.fromString(KubernetesUtils.Megabytes(
resource.getRam())));
limits.put(KubernetesConstants.CPU,
Quantity.fromString(Double.toString(roundDecimal(
resource.getCpu(), 3))));
resourceRequirements.setLimits(limits);
KubernetesContext.KubernetesResourceRequestMode requestMode =
KubernetesContext.getKubernetesRequestMode(configuration);
// Set the Kubernetes container resource request
if (requestMode == KubernetesContext.KubernetesResourceRequestMode.EQUAL_TO_LIMIT) {
LOG.log(Level.CONFIG, "Setting K8s Request equal to Limit");
resourceRequirements.setRequests(limits);
} else {
LOG.log(Level.CONFIG, "Not setting K8s request because config was NOT_SET");
}
container.setResources(resourceRequirements);

// set container ports

0 comments on commit 1a0d198

Please sign in to comment.