Skip to content

Commit

Permalink
Merge pull request #14089 from brwe/fix-test-access-denied
Browse files Browse the repository at this point in the history 
fix access denied for shard deletion with WindowsFS

We randomly use the WindowsFS mock file system to simulate that
windows does not delete files if they are opened by some other
process and instead gives you java.io.IOException: access denied.
In the tests we also check if the shard was deleted while it is
being deleted. This check loads the mata data of the index
and therefore might hold on to a file while the node
is trying to delete it and deletion will fail then.

Instead we should just check if the directory was removed.

closes #13758
  • Loading branch information
@brwe
brwe committed Oct 19, 2015
1 parent ad4c94d commit e1a7cc2
Showing 1 changed file with 37 additions and 39 deletions.
76 changes: 37 additions & 39 deletions core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,22 @@

package org.elasticsearch.gateway;

import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.base.Predicate;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.InternalTestCluster;
import org.junit.Test;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashMap;
import java.util.concurrent.Future;

Expand Down Expand Up @@ -71,14 +73,14 @@ public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception {
index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen();
assertIndexInMetaState(node1, index);
assertIndexNotInMetaState(node2, index);
assertIndexDirectoryDeleted(node2, index);
assertIndexInMetaState(masterNode, index);

logger.debug("relocating index...");
client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node2)).get();
client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get();
ensureGreen();
assertIndexNotInMetaState(node1, index);
assertIndexDirectoryDeleted(node1, index);
assertIndexInMetaState(node2, index);
assertIndexInMetaState(masterNode, index);
}
Expand Down Expand Up @@ -150,51 +152,47 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception {
assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN));
}

protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception {
assertMetaState(nodeName, indexName, false);
protected void assertIndexDirectoryDeleted(final String nodeName, final String indexName) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
logger.info("checking if index directory exists...");
assertFalse("Expecting index directory of " + indexName + " to be deleted from node " + nodeName, MetaDataWriteDataNodesIT.this.indexDirectoryExists(nodeName, indexName));
}
}
);
}

protected void assertIndexInMetaState(String nodeName, String indexName) throws Exception {
assertMetaState(nodeName, indexName, true);
protected void assertIndexInMetaState(final String nodeName, final String indexName) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
logger.info("checking if meta state exists...");
try {
assertTrue("Expecting meta state of index " + indexName + " to be on node " + nodeName, MetaDataWriteDataNodesIT.this.getIndicesMetaDataOnNode(nodeName).containsKey(indexName));
} catch (Throwable t) {
logger.info("failed to load meta state", t);
fail("could not load meta state");

}
}
}
);
}


private void assertMetaState(final String nodeName, final String indexName, final boolean shouldBe) throws Exception {
awaitBusy(new Predicate<Object>() {
@Override
public boolean apply(Object o) {
logger.info("checking if meta state exists...");
try {
return shouldBe == metaStateExists(nodeName, indexName);
} catch (Throwable t) {
logger.info("failed to load meta state", t);
// TODO: loading of meta state fails rarely if the state is deleted while we try to load it
// this here is a hack, would be much better to use for example a WatchService
return false;
}
private boolean indexDirectoryExists(String nodeName, String indexName) {
NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName);
for (Path path : nodeEnv.indexPaths(new Index(indexName))) {
if (Files.exists(path)) {
return true;
}
});
boolean inMetaSate = metaStateExists(nodeName, indexName);
if (shouldBe) {
assertTrue("expected " + indexName + " in meta state of node " + nodeName, inMetaSate);
} else {
assertFalse("expected " + indexName + " to not be in meta state of node " + nodeName, inMetaSate);
}
}

private boolean metaStateExists(String nodeName, String indexName) throws Exception {
ImmutableOpenMap<String, IndexMetaData> indices = getIndicesMetaDataOnNode(nodeName);
boolean inMetaSate = false;
for (ObjectObjectCursor<String, IndexMetaData> index : indices) {
inMetaSate = inMetaSate || index.key.equals(indexName);
}
return inMetaSate;
return false;
}

private ImmutableOpenMap<String, IndexMetaData> getIndicesMetaDataOnNode(String nodeName) throws Exception {
GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName);
MetaData nodeMetaData = null;
nodeMetaData = nodeMetaState.loadMetaState();
MetaData nodeMetaData = nodeMetaState.loadMetaState();
return nodeMetaData.getIndices();
}
}

0 comments on commit e1a7cc2

Please sign in to comment.