add artifial delay to start in DiscoveryService and remove ClusterCha…

…ngeEvent workaround

When a node starts up and elects itself as master then the first cluster
state might not be sent to the other nodes because
- discovery starts
- node elects itself and applies cluster state 1 locally
- does not send it to the other nodes because the lifecycle is not started
  yet in DiscoveryService.publish() (that is onlty set after doStart() is called)

As a result the data node only receives the second cluster state. State
persistence is not disabled. If the cluster state does not contain the index
(for example because the the master was started with empty data folder and
does not contain old cluster state) then the data node interprets the fact that the index
is missing from the new clsuter state as a delete and so deletes all shards.

src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java

@@ -106,9 +106,6 @@ public List<String> indicesCreated() {
      * Returns the indices deleted in this event
      */
     public List<String> indicesDeleted() {
-        if (newMaster()) {
-            return ImmutableList.of();
-        }
         if (previousState == null) {
             return ImmutableList.of();
         }
@@ -169,16 +166,4 @@ public boolean nodesAdded() {
     public boolean nodesChanged() {
         return nodesRemoved() || nodesAdded();
     }
-
-    public boolean newMaster() {
-        String oldMaster = previousState().getNodes().masterNodeId();
-        String newMaster = state().getNodes().masterNodeId();
-        if (oldMaster == null && newMaster == null) {
-            return false;
-        }
-        if (oldMaster == null && newMaster != null) {
-            return true;
-        }
-        return previousState().getNodes().masterNodeId().equals(state().getNodes().masterNodeId()) == false;
-    }
 }

src/main/java/org/elasticsearch/discovery/DiscoveryService.java

@@ -34,6 +34,8 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import static java.lang.Thread.sleep;
+
 /**
  *
  */
@@ -83,6 +85,12 @@ protected void doStart() throws ElasticsearchException {
         discovery.addListener(initialStateListener);
         discovery.start();
         logger.info(discovery.nodeDescription());
+        try {
+            //NOCOMMIT
+            sleep(1000);
+        } catch (InterruptedException e) {
+
+        }
     }
 
     public void waitForInitialState() {