Skip to content
This repository has been archived by the owner on Mar 3, 2023. It is now read-only.

Commit

Permalink
Merge branch '3707-Config-Map-Support-dev' into 3707-ConfigMap-Support
Browse files Browse the repository at this point in the history
  • Loading branch information
@surahman
surahman committed Oct 26, 2021
2 parents 2ddc067 + 7611638 commit b2d219c
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 140 deletions.
120 changes: 50 additions & 70 deletions heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java
View file
Expand Up @@ -56,7 +56,6 @@
import io.kubernetes.client.openapi.apis.AppsV1Api;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1ConfigMapList;
import io.kubernetes.client.openapi.models.V1Container;
import io.kubernetes.client.openapi.models.V1ContainerPort;
import io.kubernetes.client.openapi.models.V1EnvVar;
Expand Down Expand Up @@ -754,32 +753,6 @@ public static double roundDecimal(double value, int places) {
return Math.round(value * scale) / scale;
}

@VisibleForTesting
protected Pair<String, String> getPodTemplateLocation() {
final String podTemplateConfigMapName = KubernetesContext
.getPodTemplateConfigMapName(getConfiguration());

if (podTemplateConfigMapName == null) {
return null;
}

try {
final int splitPoint = podTemplateConfigMapName.indexOf(".");
final String configMapName = podTemplateConfigMapName.substring(0, splitPoint);
final String podTemplateName = podTemplateConfigMapName.substring(splitPoint + 1);

if (configMapName.isEmpty() || podTemplateName.isEmpty()) {
throw new IllegalArgumentException("Empty ConfigMap or Pod Template name");
}

return new Pair<>(configMapName, podTemplateName);
} catch (IndexOutOfBoundsException | IllegalArgumentException e) {
final String message = "Invalid ConfigMap and/or Pod Template name";
KubernetesUtils.logExceptionWithDetails(LOG, message, e);
throw new TopologySubmissionException(message);
}
}

@VisibleForTesting
protected V1PodTemplateSpec loadPodFromTemplate() {
final Pair<String, String> podTemplateConfigMapName = getPodTemplateLocation();
Expand All @@ -791,76 +764,83 @@ protected V1PodTemplateSpec loadPodFromTemplate() {
}

if (isPodTemplateDisabled) {
throw new TopologySubmissionException("Custom executor Pod Templates are disabled");
throw new TopologySubmissionException("Custom Pod Templates are disabled");
}

final String configMapName = podTemplateConfigMapName.first;
final String podTemplateName = podTemplateConfigMapName.second;

// Attempt to locate ConfigMap with provided Pod Template name.
try {
List<V1ConfigMap> configMapLists = getConfigMaps();
if (configMapLists == null) {
throw new ApiException("No ConfigMaps returned by K8s client");
V1ConfigMap configMap = getConfigMap(configMapName);
if (configMap == null) {
throw new ApiException(
String.format("K8s client unable to locate ConfigMap '%s'", configMapName));
}

// Probe ConfigMaps for the specified Pod Template name.
for (V1ConfigMap configMap : configMapLists) {
// kubectl will set the name filed in metadata automatically for ConfigMaps using the
// parameters on the command line. Hence, name field in the metadata cannot be null.
if (configMap == null || !configMap.getMetadata().getName().equals(configMapName)) {
continue;
}

final Map<String, String> configMapData = configMap.getData();
if (configMapData != null && configMapData.containsKey(podTemplateName)) {
// NullPointerException when Pod Template is empty.
V1PodTemplateSpec podTemplate = ((V1PodTemplate)
Yaml.load(configMapData.get(podTemplateName))).getTemplate();
LOG.log(Level.INFO, String.format("Configuring cluster with the %s.%s Pod Template",
configMapName, podTemplateName));
return podTemplate;
}
final Map<String, String> configMapData = configMap.getData();
if (configMapData != null && configMapData.containsKey(podTemplateName)) {
// NullPointerException when Pod Template is empty.
V1PodTemplateSpec podTemplate = ((V1PodTemplate)
Yaml.load(configMapData.get(podTemplateName))).getTemplate();
LOG.log(Level.INFO, String.format("Configuring cluster with the %s.%s Pod Template",
configMapName, podTemplateName));
return podTemplate;
}

// Failure to locate Pod Template with provided name.
throw new ApiException(String.format("Failed to locate Pod Template %s in ConfigMap %s",
throw new ApiException(String.format("Failed to locate Pod Template '%s' in ConfigMap '%s'",
podTemplateName, configMapName));
} catch (ApiException e) {
KubernetesUtils.logExceptionWithDetails(LOG, e.getMessage(), e);
throw new TopologySubmissionException(e.getMessage());
} catch (IOException | ClassCastException | NullPointerException e) {
final String message = String.format("Error parsing Pod Template %s in ConfigMap %s",
final String message = String.format("Error parsing Pod Template '%s' in ConfigMap '%s'",
podTemplateName, configMapName);
KubernetesUtils.logExceptionWithDetails(LOG, message, e);
throw new TopologySubmissionException(message);
}
}

@VisibleForTesting
protected List<V1ConfigMap> getConfigMaps() {
protected Pair<String, String> getPodTemplateLocation() {
final String podTemplateConfigMapName = KubernetesContext
.getPodTemplateConfigMapName(getConfiguration());

if (podTemplateConfigMapName == null) {
return null;
}

try {
V1ConfigMapList configMapList = coreClient
.listNamespacedConfigMap(
getNamespace(),
null,
null,
null,
null,
null,
null,
null,
null,
null,
null);

if (configMapList == null) {
throw new ApiException("No ConfigMaps returned by K8s client");
final int splitPoint = podTemplateConfigMapName.indexOf(".");
final String configMapName = podTemplateConfigMapName.substring(0, splitPoint);
final String podTemplateName = podTemplateConfigMapName.substring(splitPoint + 1);

if (configMapName.isEmpty() || podTemplateName.isEmpty()) {
throw new IllegalArgumentException("Empty ConfigMap or Pod Template name");
}

return configMapList.getItems();
return new Pair<>(configMapName, podTemplateName);
} catch (IndexOutOfBoundsException | IllegalArgumentException e) {
final String message = "Invalid ConfigMap and/or Pod Template name";
KubernetesUtils.logExceptionWithDetails(LOG, message, e);
throw new TopologySubmissionException(message);
}
}

@VisibleForTesting
protected V1ConfigMap getConfigMap(String configMapName) {
try {
return coreClient.readNamespacedConfigMap(
configMapName,
getNamespace(),
null,
null,
null);
} catch (ApiException e) {
KubernetesUtils.logExceptionWithDetails(LOG, "Error retrieving ConfigMaps", e);
throw new TopologySubmissionException(e.getMessage());
final String message = "Error retrieving ConfigMaps";
KubernetesUtils.logExceptionWithDetails(LOG, message, e);
throw new TopologySubmissionException(String.format("%s:%n%s", message, e.getMessage()));
}
}
}

0 comments on commit b2d219c

Please sign in to comment.