Implement TestBestPossibleExternalViewVerifier apache#1321

TestBestPossibleExternalViewVerifier would have a default COOL_DOWN period
before verifyByPolling. Due to apache#526, currently we have randow sleep in the
test before verifyByPolling. We intend to use TestBestPossibeVerifier
instead in our unit test.  While keep using BestPossibleExternalViewVerifier
in production code.

helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java

@@ -105,7 +105,7 @@ public BestPossibleExternalViewVerifier(RealmAwareZkClient zkClient, String clus
     _dataProvider = new ResourceControllerDataProvider();
   }
 
-  private BestPossibleExternalViewVerifier(RealmAwareZkClient zkClient, String clusterName,
+  protected BestPossibleExternalViewVerifier(RealmAwareZkClient zkClient, String clusterName,
       Map<String, Map<String, String>> errStates, Set<String> resources,
       Set<String> expectLiveInstances) {
     super(zkClient, clusterName);
@@ -224,12 +224,15 @@ public boolean verifyByZkCallback(long timeout) {
   @Override
   protected synchronized boolean verifyState() {
     try {
+      LOG.debug("Verifier start verifyState at {}", System.currentTimeMillis());
       PropertyKey.Builder keyBuilder = _accessor.keyBuilder();
 
       _dataProvider.requireFullRefresh();
       _dataProvider.refresh(_accessor);
       _dataProvider.setClusterEventId("ClusterStateVerifier");
 
+      LOG.debug("Verifier finished dataProvider full refresh at {}", System.currentTimeMillis());
+
       Map<String, IdealState> idealStates = new HashMap<>(_dataProvider.getIdealStates());
 
       // filter out all resources that use Task state model
@@ -245,13 +248,17 @@ protected synchronized boolean verifyState() {
           return false;
         }
       }
+
+      LOG.debug("Verifier finished live instances at {}", System.currentTimeMillis());
 
       Map<String, ExternalView> extViews =
           _accessor.getChildValuesMap(keyBuilder.externalViews(), true);
       if (extViews == null) {
         extViews = Collections.emptyMap();
       }
 
+      LOG.debug("Verifier finished retrieving external views at {}", System.currentTimeMillis());
+
       // Filter resources if requested
       if (_resources != null && !_resources.isEmpty()) {
         idealStates.keySet().retainAll(_resources);
@@ -295,6 +302,8 @@ protected synchronized boolean verifyState() {
       }
 
       for (String resourceName : idealStates.keySet()) {
+        LOG.debug("verifier start verifiying resource {} at {}", resourceName, System.currentTimeMillis());
+
         IdealState is = idealStates.get(resourceName);
         ExternalView extView = extViews.get(resourceName);
         if (extView == null) {
@@ -319,6 +328,8 @@ protected synchronized boolean verifyState() {
         }
 
         boolean result = verifyExternalView(extView, bpStateMap, stateModelDef);
+        LOG.debug("verifier finished verifiying resource {} at {}", resourceName, System.currentTimeMillis());
+
         if (!result) {
           if (LOG.isDebugEnabled()) {
             LOG.debug("verifyExternalView fails for " + resourceName + "! ExternalView: " + extView

helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/TestBestPossibleExternalViewVerifier.java

@@ -0,0 +1,4 @@
+package org.apache.helix.tools.ClusterVerifiers;
+
+public class TestBestPossibleExternalViewVerifier {
+}

helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java

@@ -19,6 +19,7 @@
  * under the License.
  */
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
@@ -197,8 +198,10 @@ public boolean verifyByPolling(long timeout, long period) {
         }
         TimeUnit.MILLISECONDS.sleep(period);
       } while ((System.currentTimeMillis() - start) <= timeout);
+      LOG.error("verifier timeout out with timeout {},. stack trace {} ",
+          timeout, Arrays.asList(Thread.currentThread().getStackTrace()));
     } catch (Exception e) {
-      LOG.error("Exception in verifier", e);
+      LOG.error("Exception in verifier {}", e);
     }
     return false;
   }
@@ -234,10 +237,14 @@ protected boolean verifyByCallback(long timeout, List<ClusterVerifyTrigger> trig
         if (!success) {
           // make a final try if timeout
           success = verifyState();
+          if (!success) {
+            LOG.error("verifyByCallback failed due to timeout, with stack trace {}",
+                Arrays.asList(Thread.currentThread().getStackTrace()));
+          }
         }
       }
     } catch (Exception e) {
-      LOG.error("Exception in verifier", e);
+      LOG.error("Exception in verifier {}", e);
     }
 
     // clean up