[SCB-1106]isolation must make sure one of instances is available

[SCB-1104]fix review comments

handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationDiscoveryFilter.java

@@ -90,7 +90,15 @@ public DiscoveryTreeNode discovery(DiscoveryContext context, DiscoveryTreeNode p
         filteredServers.put(key, instance);
       }
     }
-    DiscoveryTreeNode child = new DiscoveryTreeNode().data(filteredServers);
+
+    DiscoveryTreeNode child = new DiscoveryTreeNode();
+    if (filteredServers.isEmpty() && DynamicPropertyFactory.getInstance()
+        .getBooleanProperty("servicecomb.loadbalance.filter.isolation.emptyInstanceProtectionEnabled", false).get()) {
+      LOGGER.warn("All servers have been isolated, allow one of them based on load balance rule.");
+      child.data(instances);
+    } else {
+      child.data(filteredServers);
+    }
     parent.child("filterred", child);
     return child;
   }

handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java

@@ -215,6 +215,142 @@ public void testZoneAwareAndIsolationFilterWorks() throws Exception {
     Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9091");
   }
 
+  @Test
+  public void testZoneAwareAndIsolationFilterWorksEmptyInstanceProtectionEnabled() throws Exception {
+    ArchaiusUtils.setProperty("servicecomb.loadbalance.filter.isolation.emptyInstanceProtectionEnabled", "true");
+    ReferenceConfig referenceConfig = Mockito.mock(ReferenceConfig.class);
+    OperationMeta operationMeta = Mockito.mock(OperationMeta.class);
+    SchemaMeta schemaMeta = Mockito.mock(SchemaMeta.class);
+    when(operationMeta.getSchemaMeta()).thenReturn(schemaMeta);
+    MicroserviceMeta microserviceMeta = Mockito.mock(MicroserviceMeta.class);
+    when(schemaMeta.getMicroserviceMeta()).thenReturn(microserviceMeta);
+    when(schemaMeta.getMicroserviceName()).thenReturn("testMicroserviceName");
+    when(microserviceMeta.getAppId()).thenReturn("testApp");
+    when(referenceConfig.getVersionRule()).thenReturn("0.0.0+");
+    when(referenceConfig.getTransport()).thenReturn("rest");
+    Invocation invocation = new Invocation(referenceConfig, operationMeta, new Object[0]);
+
+    InstanceCacheManager instanceCacheManager = Mockito.mock(InstanceCacheManager.class);
+    ServiceRegistry serviceRegistry = Mockito.mock(ServiceRegistry.class);
+    TransportManager transportManager = Mockito.mock(TransportManager.class);
+    Transport transport = Mockito.mock(Transport.class);
+    ArchaiusUtils.setProperty("servicecomb.loadbalance.filter.operation.enabled", "false");
+
+    // set up data
+    MicroserviceInstance myself = new MicroserviceInstance();
+    DataCenterInfo info = new DataCenterInfo();
+    info.setName("test");
+    info.setRegion("test-Region");
+    info.setAvailableZone("test-zone");
+    myself.setDataCenterInfo(info);
+
+    MicroserviceInstance allmatchInstance = new MicroserviceInstance();
+    info = new DataCenterInfo();
+    info.setName("test");
+    info.setRegion("test-Region");
+    info.setAvailableZone("test-zone");
+    List<String> allMatchEndpoint = new ArrayList<>();
+    allMatchEndpoint.add("rest://localhost:9090");
+    allmatchInstance.setEndpoints(allMatchEndpoint);
+    allmatchInstance.setDataCenterInfo(info);
+    allmatchInstance.setInstanceId("allmatchInstance");
+
+    MicroserviceInstance regionMatchInstance = new MicroserviceInstance();
+    info = new DataCenterInfo();
+    info.setName("test");
+    info.setRegion("test-Region");
+    info.setAvailableZone("test-zone2");
+    List<String> regionMatchEndpoint = new ArrayList<>();
+    regionMatchEndpoint.add("rest://localhost:9091");
+    regionMatchInstance.setEndpoints(regionMatchEndpoint);
+    regionMatchInstance.setDataCenterInfo(info);
+    regionMatchInstance.setInstanceId("regionMatchInstance");
+
+    MicroserviceInstance noneMatchInstance = new MicroserviceInstance();
+    info = new DataCenterInfo();
+    info.setName("test");
+    info.setRegion("test-Region2");
+    info.setAvailableZone("test-zone2");
+    List<String> noMatchEndpoint = new ArrayList<>();
+    noMatchEndpoint.add("rest://localhost:9092");
+    noneMatchInstance.setEndpoints(noMatchEndpoint);
+    noneMatchInstance.setDataCenterInfo(info);
+    noneMatchInstance.setInstanceId("noneMatchInstance");
+
+    Map<String, MicroserviceInstance> data = new HashMap<>();
+    DiscoveryTreeNode parent = new DiscoveryTreeNode().name("parent").data(data);
+    CseContext.getInstance().setTransportManager(transportManager);
+
+    RegistryUtils.setServiceRegistry(serviceRegistry);
+
+    when(serviceRegistry.getMicroserviceInstance()).thenReturn(myself);
+    when(serviceRegistry.getInstanceCacheManager()).thenReturn(instanceCacheManager);
+    when(instanceCacheManager.getOrCreateVersionedCache("testApp", "testMicroserviceName", "0.0.0+"))
+        .thenReturn(parent);
+    when(transportManager.findTransport("rest")).thenReturn(transport);
+
+    LoadbalanceHandler handler = null;
+    LoadBalancer loadBalancer = null;
+    ServiceCombServer server = null;
+
+    handler = new LoadbalanceHandler();
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server, null);
+
+    data.put("noneMatchInstance", noneMatchInstance);
+    parent.cacheVersion(1);
+    handler = new LoadbalanceHandler();
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9092");
+
+    data.put("regionMatchInstance", regionMatchInstance);
+    parent.cacheVersion(parent.cacheVersion() + 1);
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9091");
+
+    data.put("allmatchInstance", allmatchInstance);
+    parent.cacheVersion(parent.cacheVersion() + 1);
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9090");
+
+    ServiceCombLoadBalancerStats.INSTANCE.markSuccess(server);
+    ServiceCombLoadBalancerStats.INSTANCE.markSuccess(server);
+    ServiceCombLoadBalancerStats.INSTANCE.markSuccess(server);
+    ServiceCombLoadBalancerStats.INSTANCE.markSuccess(server);
+    ServiceCombLoadBalancerStats.INSTANCE.markFailure(server);
+
+    //if errorThresholdPercentage is 0,that means errorThresholdPercentage is not active.
+    ArchaiusUtils.setProperty("servicecomb.loadbalance.isolation.errorThresholdPercentage", "0");
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9090");
+
+    //if errorThresholdPercentage greater than 0, it will activate.
+    ArchaiusUtils.setProperty("servicecomb.loadbalance.isolation.errorThresholdPercentage", "20");
+    ArchaiusUtils.setProperty("servicecomb.loadbalance.isolation.minIsolationTime", "10");
+    ServiceCombServer server2 = server;
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9090");
+    ServiceCombLoadBalancerStats.INSTANCE.markSuccess(server2);
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9090");
+    TimeUnit.MILLISECONDS.sleep(20);
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9090");
+    ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2);
+    ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2);
+    loadBalancer = handler.getOrCreateLoadBalancer(invocation);
+    server = loadBalancer.chooseServer(invocation);
+    Assert.assertEquals(server.getEndpoint().getEndpoint(), "rest://localhost:9090");
+  }
+
   @Test
   public void testZoneAwareAndIsolationFilterUsingMockedInvocationWorks() throws Exception {
     Invocation invocation = new NonSwaggerInvocation("testApp", "testMicroserviceName", "0.0.0+", (inv, aysnc) -> {

service-registry/src/main/java/org/apache/servicecomb/serviceregistry/discovery/InstanceStatusDiscoveryFilter.java

@@ -22,19 +22,14 @@
 
 import org.apache.servicecomb.serviceregistry.api.registry.MicroserviceInstance;
 import org.apache.servicecomb.serviceregistry.api.registry.MicroserviceInstanceStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import com.netflix.config.DynamicPropertyFactory;
 
 public class InstanceStatusDiscoveryFilter extends AbstractDiscoveryFilter {
-  private static final Logger LOGGER = LoggerFactory.getLogger(InstanceStatusDiscoveryFilter.class);
-
   private static final String UP_INSTANCES = "upInstances";
 
   @Override
   public int getOrder() {
-    // OperationInstancesDiscoveryFilter should execute first
     return -20000;
   }