Skip to content

Commit

Permalink
Consistent Kubernetes service object naming (#399)
Browse files Browse the repository at this point in the history 
Resolves spring-cloud/spring-cloud-skipper#953
  • Loading branch information
@chrisjs
chrisjs committed Sep 23, 2020
1 parent 166339b commit 04e9195
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 44 deletions.
10 changes: 10 additions & 0 deletions ...in/java/org/springframework/cloud/deployer/spi/kubernetes/AbstractKubernetesDeployer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ public class AbstractKubernetesDeployer {
protected static final String SPRING_APP_KEY = "spring-app-id";
protected static final String SPRING_MARKER_KEY = "role";
protected static final String SPRING_MARKER_VALUE = "spring-app";
protected static final String APP_NAME_PROPERTY_KEY = AppDeployer.PREFIX + "appName";
protected static final String APP_NAME_KEY = "spring-application-name";

private static final String SERVER_PORT_KEY = "server.port";

Expand Down Expand Up @@ -115,6 +117,14 @@ Map<String, String> createIdMap(String appId, AppDeploymentRequest request) {
map.put(SPRING_GROUP_KEY, groupId);
}
map.put(SPRING_DEPLOYMENT_KEY, appId);

// un-versioned app name provided by skipper
String appName = request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY);

if (StringUtils.hasText(appName)) {
map.put(APP_NAME_KEY, appName);
}

return map;
}

Expand Down
142 changes: 98 additions & 44 deletions src/main/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesAppDeployer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,47 +139,13 @@ public void undeploy(String appId) {

throw new IllegalStateException(String.format("App '%s' is not deployed", appId));
}
List<Service> apps = client.services().withLabel(SPRING_APP_KEY, appId).list().getItems();
if (apps != null) {
for (Service app : apps) {
String appIdToDelete = app.getMetadata().getName();
logger.debug(String.format("Deleting Resources for: %s", appIdToDelete));

Service svc = client.services().withName(appIdToDelete).get();
try {
if (svc != null && "LoadBalancer".equals(svc.getSpec().getType())) {
int tries = 0;
int maxWait = properties.getMinutesToWaitForLoadBalancer() * 6; // we check 6 times per minute
while (tries++ < maxWait) {
if (svc.getStatus() != null && svc.getStatus().getLoadBalancer() != null
&& svc.getStatus().getLoadBalancer().getIngress() != null && svc.getStatus()
.getLoadBalancer().getIngress().isEmpty()) {
if (tries % 6 == 0) {
logger.warn("Waiting for LoadBalancer to complete before deleting it ...");
}
logger.debug(String.format("Waiting for LoadBalancer, try %d", tries));
try {
Thread.sleep(10000L);
}
catch (InterruptedException e) {
}
svc = client.services().withName(appIdToDelete).get();
}
else {
break;
}
}
logger.debug(String.format("LoadBalancer Ingress: %s",
svc.getStatus().getLoadBalancer().getIngress().toString()));
}

deleteAllObjects(appIdToDelete);
}
catch (RuntimeException e) {
logger.error(e.getMessage(), e);
throw e;
}
}
try {
deleteAllObjects(appId);
}
catch (RuntimeException e) {
logger.error(e.getMessage(), e);
throw e;
}
}

Expand Down Expand Up @@ -325,9 +291,9 @@ protected void createStatefulSet(AppDeploymentRequest request) {
String statefulSetInitContainerImageName = this.deploymentPropertiesResolver.getStatefulSetInitContainerImageName(kubernetesDeployerProperties);

podSpec.getInitContainers().add(createStatefulSetInitContainer(statefulSetInitContainerImageName));

Map<String, String> deploymentLabels= this.deploymentPropertiesResolver.getDeploymentLabels(request.getDeploymentProperties());

StatefulSetSpec spec = new StatefulSetSpecBuilder().withNewSelector().addToMatchLabels(idMap)
.addToMatchLabels(SPRING_MARKER_KEY, SPRING_MARKER_VALUE).endSelector()
.withVolumeClaimTemplates(persistentVolumeClaimBuilder.build()).withServiceName(appId)
Expand Down Expand Up @@ -402,15 +368,69 @@ protected void createService(AppDeploymentRequest request) {
servicePorts.addAll(addAdditionalServicePorts(additionalServicePorts));
}

spec.withSelector(idMap).addAllToPorts(servicePorts);
spec.addAllToPorts(servicePorts);

Map<String, String> annotations = this.deploymentPropertiesResolver.getServiceAnnotations(request.getDeploymentProperties());

client.services().createNew().withNewMetadata().withName(appId)
String serviceName = getServiceName(request, appId);

// if called from skipper, use unique selectors to maintain connectivity
// between service and pods that are being brought up/down
if (request.getDeploymentProperties().containsKey(APP_NAME_PROPERTY_KEY)) {
spec.withSelector(Collections.singletonMap(APP_NAME_KEY,
request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY)));

String groupId = request.getDeploymentProperties().get(AppDeployer.GROUP_PROPERTY_KEY);

if (groupId != null) {
spec.addToSelector(SPRING_GROUP_KEY, groupId);
}
} else {
spec.withSelector(idMap);
}

client.services().createOrReplaceWithNew().withNewMetadata().withName(serviceName)
.withLabels(idMap).withAnnotations(annotations).addToLabels(SPRING_MARKER_KEY, SPRING_MARKER_VALUE)
.endMetadata().withSpec(spec.build()).done();
}

// logic to support using un-versioned service names when called from skipper
private String getServiceName(AppDeploymentRequest request, String appId) {
String appName = request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY);

// if we have an un-versioned app name from skipper
if(StringUtils.hasText(appName)) {
String serviceName = formatServiceName(request, appName);

// need to check if a versioned service exists to maintain backwards compat..
// version number itself isn't checked on as it could be different if create or upgrade
// which we don't know at runtime....
List<Service> services = client.services().withLabel(SPRING_DEPLOYMENT_KEY).list().getItems();

for (Service service : services) {
String serviceMetadataName = service.getMetadata().getName();

if(serviceMetadataName.startsWith(serviceName + "-v")) {
return appId;
}
}

return serviceName;
}

// no un-versioned app name provided, maybe not called from skipper, return whatever is in appId
return appId;
}

private String formatServiceName(AppDeploymentRequest request, String appName) {
String groupId = request.getDeploymentProperties().get(AppDeployer.GROUP_PROPERTY_KEY);

String serviceName = groupId == null ? String.format("%s", appName)
: String.format("%s-%s", groupId, appName);

return serviceName.replace('.', '-').toLowerCase();
}

private Set<ServicePort> addAdditionalServicePorts(String additionalServicePorts) {
Set<ServicePort> ports = new HashSet<>();

Expand Down Expand Up @@ -463,6 +483,7 @@ private String setIndexProperty(String name) {
private void deleteAllObjects(String appIdToDelete) {
Map<String, String> labels = Collections.singletonMap(SPRING_APP_KEY, appIdToDelete);

waitForLoadBalancerReady(labels);
deleteService(labels);
deleteDeployment(labels);
deleteStatefulSet(labels);
Expand Down Expand Up @@ -520,4 +541,37 @@ private void deletePvc(Map<String, String> labels) {
logger.debug(String.format("PVC deleted for: %s - %b", labels, pvcsDeleted));
}
}

private void waitForLoadBalancerReady(Map<String, String> labels) {
List<Service> services = client.services().withLabels(labels).list().getItems();

if (!services.isEmpty()) {
Service svc = services.get(0);
if (svc != null && "LoadBalancer".equals(svc.getSpec().getType())) {
int tries = 0;
int maxWait = properties.getMinutesToWaitForLoadBalancer() * 6; // we check 6 times per minute
while (tries++ < maxWait) {
if (svc.getStatus() != null && svc.getStatus().getLoadBalancer() != null
&& svc.getStatus().getLoadBalancer().getIngress() != null && svc.getStatus()
.getLoadBalancer().getIngress().isEmpty()) {
if (tries % 6 == 0) {
logger.warn("Waiting for LoadBalancer to complete before deleting it ...");
}
logger.debug(String.format("Waiting for LoadBalancer, try %d", tries));
try {
Thread.sleep(10000L);
}
catch (InterruptedException e) {
}
svc = client.services().withLabels(labels).list().getItems().get(0);
}
else {
break;
}
}
logger.debug(String.format("LoadBalancer Ingress: %s",
svc.getStatus().getLoadBalancer().getIngress().toString()));
}
}
}
}

0 comments on commit 04e9195

Please sign in to comment.