Skip to content

Commit

Permalink
Dubbo cloud native (#4850)
Browse files Browse the repository at this point in the history
* Polish #4542 : [Enhancement] Adapt the Java standard Event/Listener mechanism

* Polish #4541 : [Feature] Add local File System DynamicConfigurationFactory‘s extension

* Polish #4541 : Bugfix

* Polish #4541 : Optimization

* Polish #4541 : Add the compatibility for PollingWatchService on the some platforms

* Polish #4541 : Add delay publish without ThreadPoolExecutor

* Polish #4541 : Refactor the extension name

* Polish #4541 : Add remove ops

* Polish #4541 : Add testable constructor

* Polish #4541 : Add getConfigGroups method

* Polish #4610 : [Refactor] Refactor the bootstrap module

* Polish #4541 : Fix the nulling URL issue

* Polish #4622 : [Refactor] Refactor ConfigManager

* Polish #4622 : [Refactor] Refactor ConfigManager

* Polish #4622 : Support multiple configcenters

* Polish #4671 : ServiceNameMapping will not map the group, version and protocol

* update referenceCount log (#4683)

Add comments to support multiple shared connections

* Polish /#4687 : Remove the duplicated test code in dubbo-config-spring (#4688)

* #4685  修改代码if判断false问题 if (hasException == false)修改成if (!hasException) (#4695)

* Fixed Service annotation method parameters are not in effect (#4598)

* keep demo simple, and switch to use zookeeper as registry center (#4705)

* keep demo simple, and switch to use zookeeper as registry center

* remove comment

* @reference auto-wires the instance of generic interface #4594 (#4677)

* try to shorten maven output to make travis build pass (#4710)

* use CountDownLatch to check zk registry if establish connection (#4589)

* Minor change

* Rename the extension name of WritableMetadataService

* Polish #4759 : [Refactor] Change the signature of methods of MetadataService #4759

* Merge remote-tracking branch 'upstream/master' into dubbo-cloud-native

# Conflicts:
#	dubbo-all/pom.xml
#	dubbo-bom/pom.xml
#	dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/AbstractInterfaceConfig.java
#	dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ApplicationConfig.java
#	dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ConfigCenterConfig.java
#	dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ReferenceConfig.java
#	dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/RegistryConfig.java
#	dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ServiceConfig.java
#	dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ReferenceConfigTest.java
#	dubbo-configcenter/dubbo-configcenter-api/src/main/java/org/apache/dubbo/configcenter/DynamicConfiguration.java
#	dubbo-configcenter/dubbo-configcenter-api/src/test/java/org/apache/dubbo/configcenter/mock/MockDynamicConfiguration.java
#	dubbo-configcenter/dubbo-configcenter-consul/src/main/java/org/apache/dubbo/configcenter/consul/ConsulDynamicConfiguration.java
#	dubbo-configcenter/dubbo-configcenter-etcd/src/test/java/org/apache/dubbo/configcenter/support/etcd/EtcdDynamicConfigurationTest.java
#	dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosDynamicConfiguration.java
#	dubbo-configcenter/dubbo-configcenter-nacos/src/test/java/org/apache/dubbo/configcenter/support/nacos/NacosDynamicConfigurationTest.java
#	dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfiguration.java
#	dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/definition/model/MethodDefinition.java
#	dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/identifier/MetadataIdentifier.java
#	dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReport.java
#	dubbo-metadata/dubbo-metadata-api/src/test/java/org/apache/dubbo/metadata/report/identifier/MetadataIdentifierTest.java
#	dubbo-metadata/dubbo-metadata-definition-protobuf/src/main/java/org/apache/dubbo/metadata/definition/protobuf/ProtobufTypeBuilder.java
#	dubbo-metadata/dubbo-metadata-definition-protobuf/src/test/java/org/apache/dubbo/metadata/definition/protobuf/ProtobufTypeBuilderTest.java
#	dubbo-metadata/pom.xml
#	dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/integration/AbstractConfiguratorListener.java
#	dubbo-registry/dubbo-registry-nacos/src/main/java/org/apache/dubbo/registry/nacos/NacosRegistry.java
#	dubbo-registry/dubbo-registry-nacos/src/main/java/org/apache/dubbo/registry/nacos/NacosRegistryFactory.java
#	dubbo-rpc/dubbo-rpc-xml/src/main/java/org/apache/dubbo/xml/rpc/protocol/xmlrpc/XmlRpcProtocol.java

* Polish #3984 : Add the implementation of Page<ServiceInstance> getInstances(String serviceName, int offset, int pageSize, boolean healthyOnly)

* Code merge

* Fix the cases

* Merge remote-tracking branch 'upstream/cloud-native' into dubbo-cloud-native

# Conflicts:
#	dubbo-bootstrap/src/test/java/org/apache/dubbo/bootstrap/DubboServiceProviderBootstrap.java
#	dubbo-metadata/dubbo-metadata-definition-protobuf/pom.xml
#	dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/support/ServiceOrientedRegistryTest.java
#	dubbo-registry/dubbo-registry-consul/src/main/java/org/apache/dubbo/registry/consul/ConsulServiceDiscoveryFactory.java
#	dubbo-registry/dubbo-registry-etcd3/src/main/java/org/apache/dubbo/registry/etcd/EtcdServiceDiscovery.java

* Refactor ConfigManager

* Refactor ConfigManager

* Resolve the issues on ConfigManager

* Refactor and add test-cases for ConfigManager

* Polish #4774 : [Feature] Dubbo Cloud Native - To Support in Spring

* Polish #4808 : [Feature] Add the registered/unregistered event mechanism ShutdownHook

* Polish #4807 : [Feature] Add the callback mechanism ShutdownHook #4807

* Polish #4813 : [Feature] add Prioritized implementation for ServiceInstanceCustomizer

* Polish #4815 : [Feature] Add the ServiceLoader for Dubbo's services or components

* Polish #4815 : [Feature] Add the ServiceLoader for Dubbo's services or components

* Polish #4813 : [Feature] add Prioritized implementation for ServiceInstanceCustomizer

* Polish #4807 : Add sort implementation

* Refactor

* Refactor

* Polish #4845 : [Feature] Enhance the Event-Publishing feature to original ServiceDiscovery

* Merge remote-tracking branch 'upstream/cloud-native' into dubbo-cloud-native

# Conflicts:
#	dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/DefaultServiceDiscoveryFactory.java
#	dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/ServiceDiscoveryRegistry.java
  • Loading branch information
mercyblitz authored Aug 16, 2019
1 parent d9cc75f commit 2a93324
Show file tree
Hide file tree
Showing 11 changed files with 180 additions and 197 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,15 @@
import org.apache.dubbo.config.utils.ReferenceConfigCache;
import org.apache.dubbo.event.EventDispatcher;
import org.apache.dubbo.event.EventListener;
import org.apache.dubbo.event.GenericEventListener;
import org.apache.dubbo.metadata.MetadataService;
import org.apache.dubbo.metadata.MetadataServiceExporter;
import org.apache.dubbo.metadata.WritableMetadataService;
import org.apache.dubbo.metadata.report.MetadataReportInstance;
import org.apache.dubbo.registry.client.AbstractServiceDiscoveryFactory;
import org.apache.dubbo.registry.client.DefaultServiceInstance;
import org.apache.dubbo.registry.client.ServiceDiscovery;
import org.apache.dubbo.registry.client.ServiceInstance;
import org.apache.dubbo.registry.client.event.ServiceDiscoveryInitializingEvent;

import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -65,6 +66,7 @@
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
Expand All @@ -77,6 +79,7 @@
import static org.apache.dubbo.common.config.ConfigurationUtils.parseProperties;
import static org.apache.dubbo.common.config.configcenter.DynamicConfiguration.getDynamicConfiguration;
import static org.apache.dubbo.common.constants.CommonConstants.METADATA_REMOTE;
import static org.apache.dubbo.common.function.ThrowableAction.execute;
import static org.apache.dubbo.common.utils.StringUtils.isNotEmpty;
import static org.apache.dubbo.config.context.ConfigManager.getInstance;
import static org.apache.dubbo.metadata.WritableMetadataService.getExtension;
Expand All @@ -88,7 +91,7 @@
*
* @since 2.7.4
*/
public class DubboBootstrap implements Lifecycle {
public class DubboBootstrap extends GenericEventListener implements Lifecycle {

public static final String DEFAULT_REGISTRY_ID = "REGISTRY#DEFAULT";

Expand Down Expand Up @@ -137,10 +140,22 @@ public class DubboBootstrap implements Lifecycle {

private volatile MetadataServiceExporter metadataServiceExporter;

private SortedSet<ServiceDiscovery> serviceDiscoveries = new ConcurrentSkipListSet<>();

public DubboBootstrap() {
DubboShutdownHook.getDubboShutdownHook().register();
}

/**
* Store the {@link ServiceDiscovery} instances into {@link ServiceDiscoveryInitializingEvent}
*
* @param event {@link ServiceDiscoveryInitializingEvent}
* @see {@linkplan org.apache.dubbo.registry.client.EventPublishingServiceDiscovery}
*/
public void onServiceDiscoveryInitializing(ServiceDiscoveryInitializingEvent event) {
serviceDiscoveries.add(event.getSource());
}

/**
* Set only register provider or not
*
Expand Down Expand Up @@ -440,6 +455,8 @@ public DubboBootstrap initialize() {

initMetadataServiceExporter();

initEventListener();

initialized = true;

if (logger.isInfoEnabled()) {
Expand All @@ -451,22 +468,74 @@ public DubboBootstrap initialize() {
return this;
}

/**
* Initialize {@link MetadataService} from {@link WritableMetadataService}'s extension
*/
private void initMetadataService() {
this.metadataService = getExtension(isDefaultMetadataStorageType());
private void startConfigCenter() {
Collection<ConfigCenterConfig> configCenters = configManager.getConfigCenters();

if (CollectionUtils.isNotEmpty(configCenters)) {
CompositeDynamicConfiguration compositeDynamicConfiguration = new CompositeDynamicConfiguration();
for (ConfigCenterConfig configCenter : configCenters) {
configCenter.refresh();
compositeDynamicConfiguration.addConfiguration(prepareEnvironment(configCenter));
}
Environment.getInstance().setDynamicConfiguration(compositeDynamicConfiguration);
}
configManager.refreshAll();
}

private void startMetadataReport() {
ApplicationConfig applicationConfig = configManager.getApplication().orElseThrow(
() -> new IllegalStateException("There's no ApplicationConfig specified.")
);

String metadataType = applicationConfig.getMetadata();
// FIXME, multiple metadata config support.
Collection<MetadataReportConfig> metadataReportConfigs = configManager.getMetadataConfigs();
if (CollectionUtils.isEmpty(metadataReportConfigs)) {
if (METADATA_REMOTE.equals(metadataType)) {
throw new IllegalStateException("No MetadataConfig found, you must specify the remote Metadata Center address when set 'metadata=remote'.");
}
return;
}
MetadataReportConfig metadataReportConfig = metadataReportConfigs.iterator().next();
if (!metadataReportConfig.isValid()) {
return;
}

MetadataReportInstance.init(metadataReportConfig.toUrl());
}

/**
* Initialize {@link MetadataServiceExporter}
* For compatibility purpose, use registry as the default config center when the registry protocol is zookeeper and
* there's no config center specified explicitly.
*/
private void initMetadataServiceExporter() {
this.metadataServiceExporter = new ConfigurableMetadataServiceExporter()
.setApplicationConfig(getApplication())
.setRegistries(configManager.getRegistries())
.setProtocols(configManager.getProtocols())
.metadataService(metadataService);
private void useRegistryAsConfigCenterIfNecessary() {
// we use the loading status of DynamicConfiguration to decide whether ConfigCenter has been initiated.
if (Environment.getInstance().getDynamicConfiguration().isPresent()) {
return;
}

if (CollectionUtils.isNotEmpty(configManager.getConfigCenters())) {
return;
}

configManager.getDefaultRegistries().stream()
.filter(registryConfig -> registryConfig.getUseAsConfigCenter() == null || registryConfig.getUseAsConfigCenter())
.forEach(registryConfig -> {
String protocol = registryConfig.getProtocol();
String id = "config-center-" + protocol + "-" + registryConfig.getPort();
ConfigCenterConfig cc = new ConfigCenterConfig();
cc.setId(id);
cc.setParameters(registryConfig.getParameters() == null ?
new HashMap<>() :
new HashMap<>(registryConfig.getParameters()));
cc.getParameters().put(CLIENT_KEY, registryConfig.getClient());
cc.setProtocol(registryConfig.getProtocol());
cc.setAddress(registryConfig.getAddress());
cc.setNamespace(registryConfig.getGroup());
cc.setHighestPriority(false);
configManager.addConfigCenter(cc);
});
startConfigCenter();
}

private void loadRemoteConfigs() {
Expand Down Expand Up @@ -503,42 +572,35 @@ private void loadRemoteConfigs() {
configManager.addProtocols(tmpProtocols);
}


/**
* For compatibility purpose, use registry as the default config center when the registry protocol is zookeeper and
* there's no config center specified explicitly.
* Initialize {@link MetadataService} from {@link WritableMetadataService}'s extension
*/
private void useRegistryAsConfigCenterIfNecessary() {
// we use the loading status of DynamicConfiguration to decide whether ConfigCenter has been initiated.
if (Environment.getInstance().getDynamicConfiguration().isPresent()) {
return;
}
private void initMetadataService() {
this.metadataService = getExtension(isDefaultMetadataStorageType());
}

if (CollectionUtils.isNotEmpty(configManager.getConfigCenters())) {
return;
}
/**
* Initialize {@link MetadataServiceExporter}
*/
private void initMetadataServiceExporter() {
this.metadataServiceExporter = new ConfigurableMetadataServiceExporter()
.setApplicationConfig(getApplication())
.setRegistries(configManager.getRegistries())
.setProtocols(configManager.getProtocols())
.metadataService(metadataService);
}

configManager.getDefaultRegistries().stream()
.filter(registryConfig -> registryConfig.getUseAsConfigCenter() == null || registryConfig.getUseAsConfigCenter())
.forEach(registryConfig -> {
String protocol = registryConfig.getProtocol();
String id = "config-center-" + protocol + "-" + registryConfig.getPort();
ConfigCenterConfig cc = new ConfigCenterConfig();
cc.setId(id);
cc.setParameters(registryConfig.getParameters() == null ?
new HashMap<>() :
new HashMap<>(registryConfig.getParameters()));
cc.getParameters().put(CLIENT_KEY, registryConfig.getClient());
cc.setProtocol(registryConfig.getProtocol());
cc.setAddress(registryConfig.getAddress());
cc.setNamespace(registryConfig.getGroup());
cc.setHighestPriority(false);
configManager.addConfigCenter(cc);
});
startConfigCenter();
/**
* Initialize {@link EventListener}
*/
private void initEventListener() {
// Add current instance into listeners
addEventListener(this);
}

private Collection<ServiceDiscovery> getServiceDiscoveries() {
return AbstractServiceDiscoveryFactory.getDiscoveries();
return serviceDiscoveries;
}

/**
Expand Down Expand Up @@ -631,8 +693,8 @@ public boolean isInitialized() {
public boolean isStarted() {
return started;
}

/* serve for builder apis, begin */

private ApplicationBuilder createApplicationBuilder(String name) {
return new ApplicationBuilder().name(name);
}
Expand Down Expand Up @@ -660,43 +722,7 @@ private ProviderBuilder createProviderBuilder(String id) {
private ConsumerBuilder createConsumerBuilder(String id) {
return new ConsumerBuilder().id(id);
}

/* serve for builder apis, end */
private void startMetadataReport() {
ApplicationConfig applicationConfig = configManager.getApplication().orElseThrow(
() -> new IllegalStateException("There's no ApplicationConfig specified.")
);

String metadataType = applicationConfig.getMetadata();
// FIXME, multiple metadata config support.
Collection<MetadataReportConfig> metadataReportConfigs = configManager.getMetadataConfigs();
if (CollectionUtils.isEmpty(metadataReportConfigs)) {
if (METADATA_REMOTE.equals(metadataType)) {
throw new IllegalStateException("No MetadataConfig found, you must specify the remote Metadata Center address when set 'metadata=remote'.");
}
return;
}
MetadataReportConfig metadataReportConfig = metadataReportConfigs.iterator().next();
if (!metadataReportConfig.isValid()) {
return;
}

MetadataReportInstance.init(metadataReportConfig.toUrl());
}

private void startConfigCenter() {
Collection<ConfigCenterConfig> configCenters = configManager.getConfigCenters();

if (CollectionUtils.isNotEmpty(configCenters)) {
CompositeDynamicConfiguration compositeDynamicConfiguration = new CompositeDynamicConfiguration();
for (ConfigCenterConfig configCenter : configCenters) {
configCenter.refresh();
compositeDynamicConfiguration.addConfiguration(prepareEnvironment(configCenter));
}
Environment.getInstance().setDynamicConfiguration(compositeDynamicConfiguration);
}
configManager.refreshAll();
}

private DynamicConfiguration prepareEnvironment(ConfigCenterConfig configCenter) {
if (configCenter.isValid()) {
Expand Down Expand Up @@ -777,7 +803,7 @@ private void registerServiceInstance() {

int port = exportedURL.getPort();

ServiceInstance serviceInstance = initServiceInstance(serviceName, host, port);
ServiceInstance serviceInstance = createServiceInstance(serviceName, host, port);

getServiceDiscoveries().forEach(serviceDiscovery -> serviceDiscovery.register(serviceInstance));
}
Expand All @@ -801,42 +827,15 @@ private URL selectMetadataServiceExportedURL() {
return selectedURL;
}

/**
* Use rest protocol if there's one, otherwise, choose the first one available.
*
* @return
*/
private String findOneProtocolForServiceInstance(Set<String> protocols) {
String result = null;
for (String protocol : protocols) {
if ("rest".equalsIgnoreCase(protocol)) {
result = protocol;
break;
}
}

if (result == null) {
for (String protocol : protocols) {
if (!"injvm".equalsIgnoreCase(protocol) && "registry".equalsIgnoreCase(protocol)) {
result = protocol;
break;
}
}
}
return result;
}

private void unregisterServiceInstance() {

if (serviceInstance != null) {
getServiceDiscoveries().forEach(serviceDiscovery -> {
serviceDiscovery.unregister(serviceInstance);
});
}

}

private ServiceInstance initServiceInstance(String serviceName, String host, int port) {
private ServiceInstance createServiceInstance(String serviceName, String host, int port) {
this.serviceInstance = new DefaultServiceInstance(serviceName, host, port);
setMetadataStorageType(serviceInstance, isDefaultMetadataStorageType());
return this.serviceInstance;
Expand All @@ -850,6 +849,8 @@ public void destroy() {

destroyReferences();

destroyServiceDiscoveries();

clear();

release();
Expand All @@ -871,8 +872,20 @@ private void destroyReferences() {
}
}

private void destroyServiceDiscoveries() {
getServiceDiscoveries().forEach(serviceDiscovery -> {
execute(() -> {
serviceDiscovery.destroy();
});
});
if (logger.isDebugEnabled()) {
logger.debug(NAME + "'s all ServiceDiscoveries have been destroyed.");
}
}

private void clear() {
clearConfigs();
clearServiceDiscoveries();
}

private void clearConfigs() {
Expand All @@ -882,6 +895,13 @@ private void clearConfigs() {
}
}

private void clearServiceDiscoveries() {
serviceDiscoveries.clear();
if (logger.isDebugEnabled()) {
logger.debug(NAME + "'s serviceDiscoveries have been clear.");
}
}

private void release() {
executeMutually(() -> {
while (awaited.compareAndSet(false, true)) {
Expand Down
Loading

0 comments on commit 2a93324

Please sign in to comment.