Skip to content

Commit 4e7d993

Browse files
jandadavjandadav
authored
feat: Distributed authentication load balancing (#1602)
* parametrize debug header filter size Signed-off-by: jandadav <janda.david@gmail.com> * scan factory only once Signed-off-by: jandadav <janda.david@gmail.com> * Caching service client skeleton Signed-off-by: jandadav <janda.david@gmail.com> * fix incorrect test Signed-off-by: jandadav <janda.david@gmail.com> * parametrize client Signed-off-by: jandadav <janda.david@gmail.com> * enhance LoadBalancerCache with remote cache Signed-off-by: jandadav <janda.david@gmail.com> * fix load balancer cache storage Signed-off-by: jandadav <janda.david@gmail.com> * Call against local gw Signed-off-by: jandadav <janda.david@gmail.com> * prefix lb keys Signed-off-by: jandadav <janda.david@gmail.com> * debug logging Signed-off-by: jandadav <janda.david@gmail.com> * doc Signed-off-by: jandadav <janda.david@gmail.com> * javadoc Signed-off-by: jandadav <janda.david@gmail.com> * Sonar and cleanup Signed-off-by: jandadav <janda.david@gmail.com> * Conditional remote cache Signed-off-by: jandadav <janda.david@gmail.com> * Attempt to fix flaky test Signed-off-by: jandadav <janda.david@gmail.com>
1 parent 6c7b36e commit 4e7d993

File tree

16 files changed

+506
-51
lines changed

16 files changed

+506
-51
lines changed

‎apiml-common/src/main/java/org/zowe/apiml/product/gateway/GatewayConfigProperties.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020
public class GatewayConfigProperties {
2121

2222
private String scheme;
23+
24+
/**
25+
* Format: hostname:port
26+
*/
2327
private String hostname;
2428

2529
}

‎gateway-service/src/main/java/org/zowe/apiml/gateway/cache/CachingServiceClient.java

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
* This program and the accompanying materials are made available under the terms of the
3+
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
4+
* https://www.eclipse.org/legal/epl-v20.html
5+
*
6+
* SPDX-License-Identifier: EPL-2.0
7+
*
8+
* Copyright Contributors to the Zowe Project.
9+
*/
10+
11+
package org.zowe.apiml.gateway.cache;
12+
13+
import com.fasterxml.jackson.annotation.JsonCreator;
14+
import com.fasterxml.jackson.annotation.JsonInclude;
15+
import lombok.Data;
16+
import lombok.RequiredArgsConstructor;
17+
import org.springframework.beans.factory.annotation.Value;
18+
import org.springframework.http.*;
19+
import org.springframework.web.client.RestClientException;
20+
import org.springframework.web.client.RestTemplate;
21+
22+
23+
/**
24+
* Client for interaction with Caching Service
25+
* Supports basic CRUD operations
26+
* Assumes calling caching service through Gateway. Uses rest template with client certificate
27+
* as Gateway will forward the certificates in headers to caching service, which in turn uses this
28+
* as a distinguishing factor to store the keys.
29+
*
30+
*/
31+
@SuppressWarnings({"squid:S1192"}) // literals are repeating in debug logs only
32+
public class CachingServiceClient {
33+
34+
private final RestTemplate restTemplate;
35+
private final String gatewayProtocolHostPort;
36+
@Value("${apiml.cachingServiceClient.apiPath}")
37+
private static final String CACHING_API_PATH = "/cachingservice/api/v1/cache"; //NOSONAR parametrization provided by @Value annotation
38+
39+
public CachingServiceClient(RestTemplate restTemplate, String gatewayProtocolHostPort) {
40+
if (gatewayProtocolHostPort == null || gatewayProtocolHostPort.isEmpty()) {
41+
throw new IllegalStateException("gatewayProtocolHostPort has to have value in format <protocol>://<host>:<port> and not be null");
42+
}
43+
if (restTemplate == null) {
44+
throw new IllegalStateException("RestTemplate instance cannot be null");
45+
}
46+
this.restTemplate = restTemplate;
47+
this.gatewayProtocolHostPort = gatewayProtocolHostPort;
48+
49+
}
50+
51+
/**
52+
* Creates {@link KeyValue} in Caching Service.
53+
* @param kv {@link KeyValue} to store
54+
* @throws CachingServiceClientException when http response from caching is not 2xx, such as connect exception or cache conflict
55+
*/
56+
57+
public void create(KeyValue kv) throws CachingServiceClientException {
58+
try {
59+
restTemplate.exchange(gatewayProtocolHostPort + CACHING_API_PATH, HttpMethod.POST, new HttpEntity<KeyValue>(kv, new HttpHeaders()), String.class);
60+
} catch (RestClientException e) {
61+
throw new CachingServiceClientException("Unable to create keyValue: " + kv.toString() + ", caused by: " + e.getMessage(), e);
62+
}
63+
}
64+
65+
66+
/**
67+
* Reads {@link KeyValue} from Caching Service
68+
* @param key Key to read
69+
* @return {@link KeyValue}
70+
* @throws CachingServiceClientException when http response from caching is not 2xx, such as connect exception or 404 key not found in cache
71+
*/
72+
public KeyValue read(String key) throws CachingServiceClientException {
73+
try {
74+
ResponseEntity<KeyValue> response = restTemplate.exchange(gatewayProtocolHostPort + CACHING_API_PATH + "/" + key, HttpMethod.GET, new HttpEntity<KeyValue>(null, new HttpHeaders()), KeyValue.class);
75+
if (response != null && response.hasBody()) { //NOSONAR tests return null
76+
return response.getBody();
77+
} else {
78+
throw new CachingServiceClientException("Unable to read key: " + key + ", caused by response from caching service is null or has no body");
79+
}
80+
} catch (RestClientException e) {
81+
throw new CachingServiceClientException("Unable to read key: " + key + ", caused by: " + e.getMessage(), e);
82+
}
83+
}
84+
85+
/**
86+
* Updates {@link KeyValue} in Caching Service
87+
* @param kv {@link KeyValue} to update
88+
* @throws CachingServiceClientException when http response from caching is not 2xx, such as connect exception or 404 key not found in cache
89+
*/
90+
public void update(KeyValue kv) throws CachingServiceClientException {
91+
try {
92+
restTemplate.exchange(gatewayProtocolHostPort + CACHING_API_PATH, HttpMethod.PUT, new HttpEntity<KeyValue>(kv, new HttpHeaders()), String.class);
93+
} catch (RestClientException e) {
94+
throw new CachingServiceClientException("Unable to update keyValue: " + kv.toString() + ", caused by: " + e.getMessage(), e);
95+
}
96+
}
97+
98+
/**
99+
* Deletes {@link KeyValue} from Caching Service
100+
* @param key Key to delete
101+
* @throws CachingServiceClientException when http response from caching is not 2xx, such as connect exception or 404 key not found in cache
102+
*/
103+
public void delete(String key) throws CachingServiceClientException {
104+
try {
105+
restTemplate.exchange(gatewayProtocolHostPort + CACHING_API_PATH + "/" + key, HttpMethod.DELETE, new HttpEntity<KeyValue>(null, new HttpHeaders()), String.class);
106+
} catch (RestClientException e) {
107+
throw new CachingServiceClientException("Unable to delete key: " + key + ", caused by: " + e.getMessage(), e);
108+
}
109+
}
110+
111+
/**
112+
* Data POJO that represents entry in caching service
113+
*/
114+
@RequiredArgsConstructor
115+
@JsonInclude(JsonInclude.Include.NON_EMPTY)
116+
@Data
117+
static class KeyValue {
118+
private final String key;
119+
private final String value;
120+
121+
@JsonCreator
122+
public KeyValue() {
123+
key = "";
124+
value = "";
125+
}
126+
}
127+
128+
public class CachingServiceClientException extends Exception {
129+
130+
public CachingServiceClientException(String message, Throwable cause) {
131+
super(message, cause);
132+
}
133+
134+
public CachingServiceClientException(String message) {
135+
super(message);
136+
}
137+
}
138+
}

‎gateway-service/src/main/java/org/zowe/apiml/gateway/cache/LoadBalancerCache.java

Lines changed: 60 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@
99
*/
1010
package org.zowe.apiml.gateway.cache;
1111

12+
import com.fasterxml.jackson.core.JsonProcessingException;
13+
import com.fasterxml.jackson.databind.ObjectMapper;
14+
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
1215
import lombok.Getter;
16+
import lombok.extern.slf4j.Slf4j;
1317
import org.zowe.apiml.gateway.ribbon.loadbalancer.model.LoadBalancerCacheRecord;
1418

1519
import java.util.Map;
@@ -18,13 +22,23 @@
1822
/**
1923
* Cache for storing Load Balancer related information. The initial goal was to support user based instance load
2024
* balancing
25+
*
26+
* Supports optional CachingServiceClient inject through constructor, which gives acts as remote cache. Remote cache
27+
* entries have preference to local ones.
2128
*/
2229
@Getter
30+
@Slf4j
2331
public class LoadBalancerCache {
24-
private final Map<String, LoadBalancerCacheRecord> cache;
32+
private final Map<String, LoadBalancerCacheRecord> localCache;
33+
private final CachingServiceClient remoteCache;
34+
private final ObjectMapper mapper = new ObjectMapper();
35+
36+
public static final String LOAD_BALANCER_KEY_PREFIX = "lb.";
2537

26-
public LoadBalancerCache() {
27-
cache = new ConcurrentHashMap<>();
38+
public LoadBalancerCache(CachingServiceClient cachingServiceClient) {
39+
this.remoteCache = cachingServiceClient;
40+
localCache = new ConcurrentHashMap<>();
41+
mapper.registerModule(new JavaTimeModule());
2842
}
2943

3044
/**
@@ -35,13 +49,20 @@ public LoadBalancerCache() {
3549
* @param loadBalancerCacheRecord Object containing the selected instance and its creation time
3650
* @return True if storing succeeded, otherwise false
3751
*/
38-
public synchronized boolean store(String user, String service, LoadBalancerCacheRecord loadBalancerCacheRecord) {
39-
try {
40-
cache.put(getKey(user, service), loadBalancerCacheRecord);
41-
return true;
42-
} catch (UnsupportedOperationException | ClassCastException | IllegalArgumentException e) {
43-
return false;
52+
public boolean store(String user, String service, LoadBalancerCacheRecord loadBalancerCacheRecord) {
53+
if (remoteCache != null) {
54+
try {
55+
remoteCache.create(new CachingServiceClient.KeyValue(getKey(user, service), mapper.writeValueAsString(loadBalancerCacheRecord)));
56+
log.debug("Stored record to remote cache for user: {}, service: {}, record: {}", user, service, loadBalancerCacheRecord);
57+
} catch (CachingServiceClient.CachingServiceClientException e) {
58+
log.debug("Failed to store record for user: {}, service: {}, record {}, with exception: {}", user, service, loadBalancerCacheRecord, e);
59+
} catch (JsonProcessingException e) {
60+
log.debug("Failed to serialize record for user: {}, service: {}, record {}, with exception: {}", user, service, loadBalancerCacheRecord, e);
61+
}
4462
}
63+
localCache.put(getKey(user, service), loadBalancerCacheRecord);
64+
log.debug("Stored record to local cache for user: {}, service: {}, record: {}", user, service, loadBalancerCacheRecord);
65+
return true;
4566
}
4667

4768
/**
@@ -51,8 +72,24 @@ public synchronized boolean store(String user, String service, LoadBalancerCache
5172
* @param service Service towards which is the user routed
5273
* @return Retrieved record containing the instance to use for this user and its creation time.
5374
*/
54-
public synchronized LoadBalancerCacheRecord retrieve(String user, String service) {
55-
return cache.get(getKey(user, service));
75+
public LoadBalancerCacheRecord retrieve(String user, String service) {
76+
if (remoteCache != null) {
77+
try {
78+
CachingServiceClient.KeyValue kv = remoteCache.read(getKey(user, service));
79+
if (kv != null) {
80+
LoadBalancerCacheRecord loadBalancerCacheRecord = mapper.readValue(kv.getValue(), LoadBalancerCacheRecord.class);
81+
log.debug("Retrieved record from remote cache for user: {}, service: {}, record: {}", user, service, loadBalancerCacheRecord);
82+
return loadBalancerCacheRecord;
83+
}
84+
} catch (CachingServiceClient.CachingServiceClientException e) {
85+
log.debug("Failed to retrieve record for user: {}, service: {}, with exception: {}", user, service, e);
86+
} catch (JsonProcessingException e) {
87+
log.debug("Failed to deserialize record for user: {}, service: {}, with exception: {}", user, service, e);
88+
}
89+
}
90+
LoadBalancerCacheRecord loadBalancerCacheRecord = localCache.get(getKey(user, service));
91+
log.debug("Retrieved record from local cache for user: {}, service: {}, record: {}", user, service, loadBalancerCacheRecord);
92+
return loadBalancerCacheRecord;
5693
}
5794

5895
/**
@@ -61,11 +98,20 @@ public synchronized LoadBalancerCacheRecord retrieve(String user, String service
6198
* @param user User being routed towards southbound service
6299
* @param service Service towards which is the user routed
63100
*/
64-
public synchronized void delete(String user, String service) {
65-
cache.remove(getKey(user, service));
101+
public void delete(String user, String service) {
102+
if (remoteCache != null) {
103+
try {
104+
remoteCache.delete(getKey(user, service));
105+
log.debug("Deleted record from remote cache for user: {}, service: {}", user, service);
106+
} catch (CachingServiceClient.CachingServiceClientException e) {
107+
log.debug("Failed to deleted record from remote cache for user: {}, service: {}, with exception: {}", user, service, e);
108+
}
109+
}
110+
localCache.remove(getKey(user, service));
111+
log.debug("Deleted record from local cache for user: {}, service: {}", user, service);
66112
}
67113

68114
private String getKey(String user, String service) {
69-
return user + ":" + service;
115+
return LOAD_BALANCER_KEY_PREFIX + user + ":" + service;
70116
}
71117
}

‎gateway-service/src/main/java/org/zowe/apiml/gateway/cache/LoadBalancerCacheBeansConfig.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* This program and the accompanying materials are made available under the terms of the
3+
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
4+
* https://www.eclipse.org/legal/epl-v20.html
5+
*
6+
* SPDX-License-Identifier: EPL-2.0
7+
*
8+
* Copyright Contributors to the Zowe Project.
9+
*/
10+
11+
package org.zowe.apiml.gateway.cache;
12+
13+
import lombok.RequiredArgsConstructor;
14+
import org.springframework.beans.factory.annotation.Qualifier;
15+
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
16+
import org.springframework.context.annotation.Bean;
17+
import org.springframework.context.annotation.Configuration;
18+
import org.springframework.web.client.RestTemplate;
19+
import org.zowe.apiml.product.gateway.GatewayConfigProperties;
20+
21+
/**
22+
* Setup for caching service backed load balancing cache.
23+
*/
24+
@Configuration
25+
@RequiredArgsConstructor
26+
public class LoadBalancerCacheBeansConfig {
27+
28+
private final GatewayConfigProperties gatewayConfigProperties;
29+
30+
@Bean
31+
@ConditionalOnProperty(name = "apiml.loadBalancer.distribute", havingValue = "true")
32+
public CachingServiceClient cachingServiceClient(@Qualifier("restTemplateWithKeystore") RestTemplate restTemplate) {
33+
String gatewayUri = String.format("%s://%s", gatewayConfigProperties.getScheme(), gatewayConfigProperties.getHostname());
34+
return new CachingServiceClient(restTemplate, gatewayUri);
35+
}
36+
37+
@Bean
38+
@ConditionalOnProperty(name = "apiml.loadBalancer.distribute", havingValue = "true")
39+
public LoadBalancerCache loadBalancerCacheWithRemoteCache(CachingServiceClient cachingServiceClient) {
40+
return new LoadBalancerCache(cachingServiceClient);
41+
}
42+
43+
@Bean
44+
@ConditionalOnProperty(name = "apiml.loadBalancer.distribute", havingValue = "false", matchIfMissing = true)
45+
public LoadBalancerCache loadBalancerCacheOnlyLocalCache() {
46+
return new LoadBalancerCache(null);
47+
}
48+
}

‎gateway-service/src/main/java/org/zowe/apiml/gateway/config/CacheConfig.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import org.springframework.core.io.ClassPathResource;
2020
import org.zowe.apiml.cache.CompositeKeyGenerator;
2121
import org.zowe.apiml.cache.CompositeKeyGeneratorWithoutLast;
22-
import org.zowe.apiml.gateway.cache.LoadBalancerCache;
2322
import org.zowe.apiml.util.CacheUtils;
2423

2524
import javax.annotation.PostConstruct;
@@ -77,8 +76,4 @@ public CacheUtils cacheUtils() {
7776
return new CacheUtils();
7877
}
7978

80-
@Bean
81-
public LoadBalancerCache loadBalancerCache() {
82-
return new LoadBalancerCache();
83-
}
8479
}

‎gateway-service/src/main/java/org/zowe/apiml/gateway/context/NamedContextConfiguration.java

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* This program and the accompanying materials are made available under the terms of the
3+
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
4+
* https://www.eclipse.org/legal/epl-v20.html
5+
*
6+
* SPDX-License-Identifier: EPL-2.0
7+
*
8+
* Copyright Contributors to the Zowe Project.
9+
*/
10+
11+
package org.zowe.apiml.gateway.context;
12+
13+
import org.springframework.cloud.context.named.NamedContextFactory;
14+
import org.springframework.context.annotation.Bean;
15+
import org.springframework.context.annotation.Configuration;
16+
import org.zowe.apiml.gateway.ribbon.loadbalancer.LoadBalancerConstants;
17+
import org.zowe.apiml.gateway.ribbon.loadbalancer.LoadBalancingPredicatesRibbonConfig;
18+
19+
@Configuration
20+
public class NamedContextConfiguration {
21+
22+
/**
23+
* Factory for load balancer predicates
24+
*
25+
* Takes in {@link LoadBalancingPredicatesRibbonConfig} which specifies the composition of load
26+
* balancer predicates.
27+
*
28+
* This is static now, but in the future can be wired in as a bean to allow broader extensibility
29+
* There should be only one instance of this factory in main application context
30+
*/
31+
@Bean
32+
public ConfigurableNamedContextFactory<NamedContextFactory.Specification> predicateFactory() {
33+
return new ConfigurableNamedContextFactory<>(LoadBalancingPredicatesRibbonConfig.class, "contextConfiguration",
34+
LoadBalancerConstants.INSTANCE_KEY + LoadBalancerConstants.SERVICEID_KEY);
35+
}
36+
}

‎gateway-service/src/main/java/org/zowe/apiml/gateway/filters/post/DebugHeaderFilter.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.netflix.zuul.context.Debug;
1414
import com.netflix.zuul.context.RequestContext;
1515
import lombok.extern.slf4j.Slf4j;
16+
import org.springframework.beans.factory.annotation.Value;
1617
import org.springframework.stereotype.Component;
1718
import org.zowe.apiml.gateway.ribbon.RequestContextUtils;
1819

@@ -31,6 +32,9 @@
3132
@Slf4j
3233
public class DebugHeaderFilter extends ZuulFilter {
3334

35+
@Value("${zuul.debug.request.debugHeaderLimit:4096}")
36+
private int debugHeaderLimit;
37+
3438
@Override
3539
public String filterType() {
3640
return POST_TYPE;
@@ -48,7 +52,6 @@ public boolean shouldFilter() {
4852

4953
@Override
5054
public Object run() {
51-
int debugHeaderLimit = 4096;
5255

5356
String debug = convertToPrettyPrintString(Debug.getRoutingDebug());
5457
String reqInfo = RequestContext.getCurrentContext().getFilterExecutionSummary().toString();

0 commit comments

Comments
 (0)