Skip to content

Commit a29223a

Browse files
smolnar82smolnar82
authored
KNOX-2680 - Introducing a central repository for CM service discovery configs (#533)
* KNOX-2680 - Filtering CM service discovery by service names listed in the given topology * KNOX-2680 - Introducing a central repository for CM service discovery configs
1 parent 4073660 commit a29223a

File tree

23 files changed

+697
-89
lines changed

23 files changed

+697
-89
lines changed

‎gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import java.nio.file.Files;
4040
import java.nio.file.Paths;
4141
import java.util.ArrayList;
42+
import java.util.Collection;
4243
import java.util.HashMap;
4344
import java.util.List;
4445
import java.util.Locale;
@@ -325,4 +326,9 @@ public Cluster discover(GatewayConfig gatewayConfig, ServiceDiscoveryConfig conf
325326
return cluster;
326327
}
327328

329+
@Override
330+
public Cluster discover(GatewayConfig gwConfig, ServiceDiscoveryConfig config, String clusterName, Collection<String> includedServices) {
331+
throw new UnsupportedOperationException("Filtering Ambari service discovery by service names is not supported!");
332+
}
333+
328334
}

‎gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717
package org.apache.knox.gateway.topology.discovery.ambari;
1818

19+
import org.apache.knox.gateway.config.GatewayConfig;
1920
import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
2021
import org.apache.knox.gateway.topology.discovery.ServiceDiscoveryType;
2122

@@ -26,7 +27,7 @@ public String getType() {
2627
}
2728

2829
@Override
29-
public ServiceDiscovery newInstance() {
30+
public ServiceDiscovery newInstance(GatewayConfig gatewayConfig) {
3031
return new AmbariServiceDiscovery();
3132
}
3233
}

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java‎

Lines changed: 144 additions & 66 deletions
Large diffs are not rendered by default.

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryMessages.java‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,27 @@
2525
@Messages(logger="org.apache.knox.gateway.topology.discovery.cm")
2626
public interface ClouderaManagerServiceDiscoveryMessages {
2727

28+
@Message(level = MessageLevel.INFO, text = "Discovering cluster: {0} ({1})")
29+
void discoveringCluster(String clusterName, String version);
30+
2831
@Message(level = MessageLevel.INFO, text = "Discovered cluster: {0} ({1})")
2932
void discoveredCluster(String clusterName, String version);
3033

3134
@Message(level = MessageLevel.INFO, text = "Performing cluster discovery for \"{0}\"")
3235
void discoveringCluster(String clusterName);
3336

37+
@Message(level = MessageLevel.INFO, text = "Discovering service: {0} ({1}) ...")
38+
void discoveringService(String serviceName, String serviceType);
39+
3440
@Message(level = MessageLevel.INFO, text = "Discovered service: {0} ({1})")
3541
void discoveredService(String serviceName, String serviceType);
3642

43+
@Message(level = MessageLevel.INFO, text = "Skipping service discovery: {0} ({1})")
44+
void skipServiceDiscovery(String serviceName, String serviceType);
45+
46+
@Message(level = MessageLevel.INFO, text = "Discovering service role: {0} ({1}) ...")
47+
void discoveringServiceRole(String roleName, String roleType);
48+
3749
@Message(level = MessageLevel.INFO, text = "Discovered service role: {0} ({1})")
3850
void discoveredServiceRole(String roleName, String roleType);
3951

@@ -74,8 +86,8 @@ void failedToInstantiateJAASConfigurationFileImplementation(String implementatio
7486
void failedToAccessServiceConfigs(String clusterName, @StackTrace(level = MessageLevel.DEBUG) Exception e);
7587

7688
@Message(level = MessageLevel.ERROR,
77-
text = "Failed to access the service role configurations for cluster ({0}) discovery: {1}")
78-
void failedToAccessServiceRoleConfigs(String clusterName, @StackTrace(level = MessageLevel.DEBUG) Exception e);
89+
text = "Failed to access the service role configurations ({0} / {1}) for cluster ({2}) discovery: {3}")
90+
void failedToAccessServiceRoleConfigs(String serviceName, String roleName, String clusterName, @StackTrace(level = MessageLevel.DEBUG) Exception e);
7991

8092
@Message(level = MessageLevel.ERROR,
8193
text = "No address for Cloudera Manager service discovery has been configured.")
@@ -210,4 +222,32 @@ void roleConfigurationPropertyHasChanged(String propertyName,
210222

211223
@Message(level = MessageLevel.ERROR, text = "Failed to configure truststore")
212224
void failedToConfigureTruststore();
225+
226+
@Message(level = MessageLevel.DEBUG, text = "Looking up cluster services from service discovery repository...")
227+
void lookupClusterServicesFromRepository();
228+
229+
@Message(level = MessageLevel.DEBUG, text = "Looking up service configuration from service discovery repository...")
230+
void lookupServiceConfigsFromRepository();
231+
232+
@Message(level = MessageLevel.DEBUG, text = "Looking up roles from service discovery repository...")
233+
void lookupRolesFromRepository();
234+
235+
@Message(level = MessageLevel.DEBUG, text = "Looking up role configuration from service discovery repository...")
236+
void lookupRoleConfigsFromRepository();
237+
238+
@Message(level = MessageLevel.DEBUG, text = "Looking up cluster services from the configured Cloudera Manager discovery endpoint...")
239+
void lookupClusterServicesFromCM();
240+
241+
@Message(level = MessageLevel.DEBUG, text = "Looking up service configuration from the configured Cloudera Manager discovery endpoint...")
242+
void lookupServiceConfigsFromCM();
243+
244+
@Message(level = MessageLevel.DEBUG, text = "Looking up roles from the configured Cloudera Manager discovery endpoint...")
245+
void lookupRolesFromCM();
246+
247+
@Message(level = MessageLevel.DEBUG, text = "Looking up role configuration from the configured Cloudera Manager discovery endpoint...")
248+
void lookupRoleConfigsFromCM();
249+
250+
@Message(level = MessageLevel.DEBUG, text = "Clearing service discovery repository...")
251+
void clearServiceDiscoveryRepository();
252+
213253
}

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryRepository.java‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.knox.gateway.topology.discovery.cm;
18+
19+
import java.time.Duration;
20+
import java.util.List;
21+
import java.util.Map;
22+
import java.util.concurrent.ConcurrentHashMap;
23+
import java.util.stream.Collectors;
24+
25+
import org.apache.commons.lang3.builder.EqualsBuilder;
26+
import org.apache.commons.lang3.builder.HashCodeBuilder;
27+
import org.apache.commons.lang3.builder.ToStringBuilder;
28+
import org.apache.commons.lang3.builder.ToStringStyle;
29+
import org.apache.knox.gateway.config.GatewayConfig;
30+
import org.apache.knox.gateway.topology.discovery.ServiceDiscoveryConfig;
31+
32+
import com.cloudera.api.swagger.model.ApiConfigList;
33+
import com.cloudera.api.swagger.model.ApiRole;
34+
import com.cloudera.api.swagger.model.ApiRoleList;
35+
import com.cloudera.api.swagger.model.ApiService;
36+
import com.cloudera.api.swagger.model.ApiServiceConfig;
37+
import com.github.benmanes.caffeine.cache.Cache;
38+
import com.github.benmanes.caffeine.cache.Caffeine;
39+
40+
class ClouderaManagerServiceDiscoveryRepository {
41+
42+
private static final ClouderaManagerServiceDiscoveryRepository INSTANCE = new ClouderaManagerServiceDiscoveryRepository();
43+
private final Map<RepositoryKey, Cache<ApiService, ServiceDetails>> repository;
44+
private long cacheEntryTTL = GatewayConfig.DEFAULT_CM_SERVICE_DISCOVERY_CACHE_ENTRY_TTL;
45+
46+
private ClouderaManagerServiceDiscoveryRepository() {
47+
this.repository = new ConcurrentHashMap<>();
48+
}
49+
50+
static ClouderaManagerServiceDiscoveryRepository getInstance() {
51+
return INSTANCE;
52+
}
53+
54+
void setCacheEntryTTL(long cacheEntryTTL) {
55+
this.cacheEntryTTL = cacheEntryTTL;
56+
}
57+
58+
void clear() {
59+
repository.clear();
60+
}
61+
62+
void registerCluster(ServiceDiscoveryConfig serviceDiscoveryConfig) {
63+
repository.putIfAbsent(RepositoryKey.of(serviceDiscoveryConfig), Caffeine.newBuilder().expireAfterWrite(Duration.ofSeconds(cacheEntryTTL)).build());
64+
}
65+
66+
void addService(ServiceDiscoveryConfig serviceDiscoveryConfig, ApiService service) {
67+
getClusterServices(serviceDiscoveryConfig).put(service, new ServiceDetails());
68+
}
69+
70+
List<ApiService> getServices(ServiceDiscoveryConfig serviceDiscoveryConfig) {
71+
final Cache<ApiService, ServiceDetails> clusterServices = getClusterServices(serviceDiscoveryConfig);
72+
return clusterServices == null ? null
73+
: clusterServices.asMap().entrySet().stream().filter(entry -> entry.getValue() != null).map(entry -> entry.getKey())
74+
.collect(Collectors.toList());
75+
}
76+
77+
private Cache<ApiService, ServiceDetails> getClusterServices(ServiceDiscoveryConfig serviceDiscoveryConfig) {
78+
return repository.get(RepositoryKey.of(serviceDiscoveryConfig));
79+
}
80+
81+
void addServiceConfig(ServiceDiscoveryConfig serviceDiscoveryConfig, ApiService service, ApiServiceConfig serviceConfig) {
82+
final ServiceDetails serviceDetails = getServiceDetails(serviceDiscoveryConfig, service);
83+
if (serviceDetails != null) {
84+
serviceDetails.setServiceConfig(serviceConfig);
85+
}
86+
}
87+
88+
private ServiceDetails getServiceDetails(ServiceDiscoveryConfig serviceDiscoveryConfig, ApiService service) {
89+
return getClusterServices(serviceDiscoveryConfig).getIfPresent(service);
90+
}
91+
92+
ApiServiceConfig getServiceConfig(ServiceDiscoveryConfig serviceDiscoveryConfig, ApiService service) {
93+
final ServiceDetails serviceDetails = getServiceDetails(serviceDiscoveryConfig, service);
94+
return serviceDetails == null ? null : serviceDetails.getServiceConfig();
95+
}
96+
97+
void addRoles(ServiceDiscoveryConfig serviceDiscoveryConfig, ApiService service, ApiRoleList roles) {
98+
final ServiceDetails serviceDetails = getServiceDetails(serviceDiscoveryConfig, service);
99+
if (serviceDetails != null) {
100+
serviceDetails.addRoles(roles);
101+
}
102+
}
103+
104+
ApiRoleList getRoles(ServiceDiscoveryConfig serviceDiscoveryConfig, ApiService service) {
105+
final ServiceDetails serviceDetails = getServiceDetails(serviceDiscoveryConfig, service);
106+
return serviceDetails == null ? null : serviceDetails.getRoles();
107+
}
108+
109+
void addRoleConfigs(ServiceDiscoveryConfig serviceDiscoveryConfig, ApiService service, ApiRole role, ApiConfigList roleConfigs) {
110+
final ServiceDetails serviceDetails = getServiceDetails(serviceDiscoveryConfig, service);
111+
if (serviceDetails != null) {
112+
serviceDetails.addRoleConfigs(role, roleConfigs);
113+
}
114+
}
115+
116+
ApiConfigList getRoleConfigs(ServiceDiscoveryConfig serviceDiscoveryConfig, ApiService service, ApiRole role) {
117+
final ServiceDetails serviceDetails = getServiceDetails(serviceDiscoveryConfig, service);
118+
return serviceDetails == null ? null : serviceDetails.getRoleConfigs(role);
119+
}
120+
121+
private static final class RepositoryKey {
122+
private final String address;
123+
private final String clusterName;
124+
125+
private RepositoryKey(String address, String clusterName) {
126+
this.address = address;
127+
this.clusterName = clusterName;
128+
}
129+
130+
static RepositoryKey of(ServiceDiscoveryConfig serviceDiscoveryConfig) {
131+
return new RepositoryKey(serviceDiscoveryConfig.getAddress(), serviceDiscoveryConfig.getCluster());
132+
}
133+
134+
String getAddress() {
135+
return address;
136+
}
137+
138+
String getClusterName() {
139+
return clusterName;
140+
}
141+
142+
@Override
143+
public int hashCode() {
144+
return HashCodeBuilder.reflectionHashCode(this);
145+
}
146+
147+
@Override
148+
public boolean equals(Object obj) {
149+
return EqualsBuilder.reflectionEquals(this, obj);
150+
}
151+
152+
@Override
153+
public String toString() {
154+
return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
155+
}
156+
}
157+
158+
private static class ServiceDetails {
159+
private ApiServiceConfig serviceConfig;
160+
private Map<ApiRole, ApiConfigList> roleConfigsMap = new ConcurrentHashMap<>();
161+
162+
public ApiServiceConfig getServiceConfig() {
163+
return serviceConfig;
164+
}
165+
166+
public void setServiceConfig(ApiServiceConfig serviceConfig) {
167+
this.serviceConfig = serviceConfig;
168+
}
169+
170+
public ApiRoleList getRoles() {
171+
ApiRoleList roles = new ApiRoleList();
172+
for (ApiRole role : roleConfigsMap.keySet()) {
173+
roles = roles.addItemsItem(role);
174+
}
175+
return roles;
176+
}
177+
178+
public void addRoles(ApiRoleList roles) {
179+
for (ApiRole role : roles.getItems()) {
180+
roleConfigsMap.put(role, new ApiConfigList());
181+
}
182+
}
183+
184+
public ApiConfigList getRoleConfigs(ApiRole role) {
185+
return roleConfigsMap.get(role);
186+
}
187+
188+
public void addRoleConfigs(ApiRole role, ApiConfigList roleConfigs) {
189+
roleConfigsMap.put(role, roleConfigs);
190+
}
191+
192+
@Override
193+
public int hashCode() {
194+
return HashCodeBuilder.reflectionHashCode(this);
195+
}
196+
197+
@Override
198+
public boolean equals(Object obj) {
199+
return EqualsBuilder.reflectionEquals(this, obj);
200+
}
201+
202+
@Override
203+
public String toString() {
204+
return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
205+
}
206+
207+
}
208+
209+
}

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryType.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717
package org.apache.knox.gateway.topology.discovery.cm;
1818

19+
import org.apache.knox.gateway.config.GatewayConfig;
1920
import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
2021
import org.apache.knox.gateway.topology.discovery.ServiceDiscoveryType;
2122

@@ -30,8 +31,8 @@ public String getType() {
3031
}
3132

3233
@Override
33-
public ServiceDiscovery newInstance() {
34-
return new ClouderaManagerServiceDiscovery();
34+
public ServiceDiscovery newInstance(GatewayConfig gatewayConfig) {
35+
return new ClouderaManagerServiceDiscovery(gatewayConfig);
3536
}
3637

3738
}

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/DiscoveryApiClient.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ public DiscoveryApiClient(ServiceDiscoveryConfig discoveryConfig, AliasService a
5757
configure(aliasService, keystoreService);
5858
}
5959

60+
ServiceDiscoveryConfig getConfig() {
61+
return config;
62+
}
63+
6064
boolean isKerberos() {
6165
return isKerberos;
6266
}

0 commit comments

Comments
 (0)