Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Consistent Kubernetes service object naming #399

Merged
merged 1 commit into from
Sep 23, 2020
Merged

Consistent Kubernetes service object naming #399

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions ...in/java/org/springframework/cloud/deployer/spi/kubernetes/AbstractKubernetesDeployer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ public class AbstractKubernetesDeployer {
protected static final String SPRING_APP_KEY = "spring-app-id";
protected static final String SPRING_MARKER_KEY = "role";
protected static final String SPRING_MARKER_VALUE = "spring-app";
protected static final String APP_NAME_PROPERTY_KEY = AppDeployer.PREFIX + "appName";
protected static final String APP_NAME_KEY = "spring-application-name";

private static final String SERVER_PORT_KEY = "server.port";

Expand Down Expand Up @@ -115,6 +117,14 @@ Map<String, String> createIdMap(String appId, AppDeploymentRequest request) {
map.put(SPRING_GROUP_KEY, groupId);
}
map.put(SPRING_DEPLOYMENT_KEY, appId);

// un-versioned app name provided by skipper
String appName = request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY);

if (StringUtils.hasText(appName)) {
map.put(APP_NAME_KEY, appName);
}

return map;
}

Expand Down
142 changes: 98 additions & 44 deletions src/main/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesAppDeployer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,47 +139,13 @@ public void undeploy(String appId) {

throw new IllegalStateException(String.format("App '%s' is not deployed", appId));
}
List<Service> apps = client.services().withLabel(SPRING_APP_KEY, appId).list().getItems();
if (apps != null) {
for (Service app : apps) {
String appIdToDelete = app.getMetadata().getName();
logger.debug(String.format("Deleting Resources for: %s", appIdToDelete));

Service svc = client.services().withName(appIdToDelete).get();
try {
if (svc != null && "LoadBalancer".equals(svc.getSpec().getType())) {
int tries = 0;
int maxWait = properties.getMinutesToWaitForLoadBalancer() * 6; // we check 6 times per minute
while (tries++ < maxWait) {
if (svc.getStatus() != null && svc.getStatus().getLoadBalancer() != null
&& svc.getStatus().getLoadBalancer().getIngress() != null && svc.getStatus()
.getLoadBalancer().getIngress().isEmpty()) {
if (tries % 6 == 0) {
logger.warn("Waiting for LoadBalancer to complete before deleting it ...");
}
logger.debug(String.format("Waiting for LoadBalancer, try %d", tries));
try {
Thread.sleep(10000L);
}
catch (InterruptedException e) {
}
svc = client.services().withName(appIdToDelete).get();
}
else {
break;
}
}
logger.debug(String.format("LoadBalancer Ingress: %s",
svc.getStatus().getLoadBalancer().getIngress().toString()));
}

deleteAllObjects(appIdToDelete);
}
catch (RuntimeException e) {
logger.error(e.getMessage(), e);
throw e;
}
}
try {
deleteAllObjects(appId);
}
catch (RuntimeException e) {
logger.error(e.getMessage(), e);
throw e;
}
}

Expand Down Expand Up @@ -325,9 +291,9 @@ protected void createStatefulSet(AppDeploymentRequest request) {
String statefulSetInitContainerImageName = this.deploymentPropertiesResolver.getStatefulSetInitContainerImageName(kubernetesDeployerProperties);

podSpec.getInitContainers().add(createStatefulSetInitContainer(statefulSetInitContainerImageName));

Map<String, String> deploymentLabels= this.deploymentPropertiesResolver.getDeploymentLabels(request.getDeploymentProperties());

StatefulSetSpec spec = new StatefulSetSpecBuilder().withNewSelector().addToMatchLabels(idMap)
.addToMatchLabels(SPRING_MARKER_KEY, SPRING_MARKER_VALUE).endSelector()
.withVolumeClaimTemplates(persistentVolumeClaimBuilder.build()).withServiceName(appId)
Expand Down Expand Up @@ -402,15 +368,69 @@ protected void createService(AppDeploymentRequest request) {
servicePorts.addAll(addAdditionalServicePorts(additionalServicePorts));
}

spec.withSelector(idMap).addAllToPorts(servicePorts);
spec.addAllToPorts(servicePorts);

Map<String, String> annotations = this.deploymentPropertiesResolver.getServiceAnnotations(request.getDeploymentProperties());

client.services().createNew().withNewMetadata().withName(appId)
String serviceName = getServiceName(request, appId);

// if called from skipper, use unique selectors to maintain connectivity
// between service and pods that are being brought up/down
if (request.getDeploymentProperties().containsKey(APP_NAME_PROPERTY_KEY)) {
spec.withSelector(Collections.singletonMap(APP_NAME_KEY,
request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY)));

String groupId = request.getDeploymentProperties().get(AppDeployer.GROUP_PROPERTY_KEY);

if (groupId != null) {
spec.addToSelector(SPRING_GROUP_KEY, groupId);
}
} else {
spec.withSelector(idMap);
}

client.services().createOrReplaceWithNew().withNewMetadata().withName(serviceName)
.withLabels(idMap).withAnnotations(annotations).addToLabels(SPRING_MARKER_KEY, SPRING_MARKER_VALUE)
.endMetadata().withSpec(spec.build()).done();
}

// logic to support using un-versioned service names when called from skipper
private String getServiceName(AppDeploymentRequest request, String appId) {
String appName = request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY);

// if we have an un-versioned app name from skipper
if(StringUtils.hasText(appName)) {
String serviceName = formatServiceName(request, appName);

// need to check if a versioned service exists to maintain backwards compat..
// version number itself isn't checked on as it could be different if create or upgrade
// which we don't know at runtime....
List<Service> services = client.services().withLabel(SPRING_DEPLOYMENT_KEY).list().getItems();

for (Service service : services) {
String serviceMetadataName = service.getMetadata().getName();

if(serviceMetadataName.startsWith(serviceName + "-v")) {
return appId;
}
}

return serviceName;
}

// no un-versioned app name provided, maybe not called from skipper, return whatever is in appId
return appId;
}

private String formatServiceName(AppDeploymentRequest request, String appName) {
String groupId = request.getDeploymentProperties().get(AppDeployer.GROUP_PROPERTY_KEY);

String serviceName = groupId == null ? String.format("%s", appName)
: String.format("%s-%s", groupId, appName);

return serviceName.replace('.', '-').toLowerCase();
}

private Set<ServicePort> addAdditionalServicePorts(String additionalServicePorts) {
Set<ServicePort> ports = new HashSet<>();

Expand Down Expand Up @@ -463,6 +483,7 @@ private String setIndexProperty(String name) {
private void deleteAllObjects(String appIdToDelete) {
Map<String, String> labels = Collections.singletonMap(SPRING_APP_KEY, appIdToDelete);

waitForLoadBalancerReady(labels);
deleteService(labels);
deleteDeployment(labels);
deleteStatefulSet(labels);
Expand Down Expand Up @@ -520,4 +541,37 @@ private void deletePvc(Map<String, String> labels) {
logger.debug(String.format("PVC deleted for: %s - %b", labels, pvcsDeleted));
}
}

private void waitForLoadBalancerReady(Map<String, String> labels) {
List<Service> services = client.services().withLabels(labels).list().getItems();

if (!services.isEmpty()) {
Service svc = services.get(0);
if (svc != null && "LoadBalancer".equals(svc.getSpec().getType())) {
int tries = 0;
int maxWait = properties.getMinutesToWaitForLoadBalancer() * 6; // we check 6 times per minute
while (tries++ < maxWait) {
if (svc.getStatus() != null && svc.getStatus().getLoadBalancer() != null
&& svc.getStatus().getLoadBalancer().getIngress() != null && svc.getStatus()
.getLoadBalancer().getIngress().isEmpty()) {
if (tries % 6 == 0) {
logger.warn("Waiting for LoadBalancer to complete before deleting it ...");
}
logger.debug(String.format("Waiting for LoadBalancer, try %d", tries));
try {
Thread.sleep(10000L);
}
catch (InterruptedException e) {
}
svc = client.services().withLabels(labels).list().getItems().get(0);
}
else {
break;
}
}
logger.debug(String.format("LoadBalancer Ingress: %s",
svc.getStatus().getLoadBalancer().getIngress().toString()));
}
}
}
}