Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ignore obsolete dangling indices #37824

Next

Ignore obsolete dangling indices

For a non-data, non-master node we now warn about dangling indices and
will otherwise ignore them. This avoids import of old indices with a
following inevitable red cluster status.

Issue #27073
  • Loading branch information...
henningandersen committed Jan 24, 2019
commit c4ba7eb39de684a9c3d03c1b0dda007f527e2d8a
@@ -27,8 +27,10 @@
import org.elasticsearch.cluster.metadata.IndexGraveyard;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
@@ -55,15 +57,17 @@

private static final Logger logger = LogManager.getLogger(DanglingIndicesState.class);

private final Settings settings;
private final NodeEnvironment nodeEnv;
private final MetaStateService metaStateService;
private final LocalAllocateDangledIndices allocateDangledIndices;

private final Map<Index, IndexMetaData> danglingIndices = ConcurrentCollections.newConcurrentMap();

@Inject
public DanglingIndicesState(NodeEnvironment nodeEnv, MetaStateService metaStateService,
public DanglingIndicesState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService,
LocalAllocateDangledIndices allocateDangledIndices, ClusterService clusterService) {
this.settings = settings;
this.nodeEnv = nodeEnv;
this.metaStateService = metaStateService;
this.allocateDangledIndices = allocateDangledIndices;
@@ -132,8 +136,12 @@ void findNewAndAddDanglingIndices(final MetaData metaData) {
final List<IndexMetaData> indexMetaDataList = metaStateService.loadIndicesStates(excludeIndexPathIds::contains);
Map<Index, IndexMetaData> newIndices = new HashMap<>(indexMetaDataList.size());
final IndexGraveyard graveyard = metaData.indexGraveyard();
boolean coordinatingOnly = DiscoveryNode.isDataNode(settings) == false
&& DiscoveryNode.isMasterNode(settings) == false;
for (IndexMetaData indexMetaData : indexMetaDataList) {
if (metaData.hasIndex(indexMetaData.getIndex().getName())) {
if (coordinatingOnly) {
logger.warn("[{}] dangling index ignored for non-data, non-master node", indexMetaData.getIndex());
} else if (metaData.hasIndex(indexMetaData.getIndex().getName())) {
logger.warn("[{}] can not be imported as a dangling index, as index with same name already exists in cluster metadata",
indexMetaData.getIndex());
} else if (graveyard.containsIndex(indexMetaData.getIndex())) {
@@ -26,9 +26,11 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
@@ -158,7 +160,54 @@ public void testDanglingIndicesNotImportedWhenTombstonePresent() throws Exceptio
}
}

public void testDanglingIndicesIgnoredWhenObsolete() throws IOException {
Settings noDataNoMasterSettings = Settings.builder()
.put(Node.NODE_DATA_SETTING.getKey(), false)
.put(Node.NODE_MASTER_SETTING.getKey(), false)
.build();

Settings noDataSettings = Settings.builder()
.put(Node.NODE_DATA_SETTING.getKey(), false)
.build();

Settings noMasterSettings = Settings.builder()
.put(Node.NODE_MASTER_SETTING.getKey(), false)
.build();

verifyDanglingIndicesIgnoredWhenObsolete(noDataNoMasterSettings, 0,
"node.data=false and node.master=false nodes should not detect any dangling indices");
verifyDanglingIndicesIgnoredWhenObsolete(noDataSettings, 1,
"node.data=false and node.master=true nodes should detect dangling indices");
verifyDanglingIndicesIgnoredWhenObsolete(noMasterSettings, 1,
"node.data=true and node.master=false nodes should detect dangling indices");
// also validated by #testDanglingIndicesDiscovery, included for completeness.
verifyDanglingIndicesIgnoredWhenObsolete(Settings.EMPTY, 1,
"node.data=true and node.master=true nodes should detect dangling indices");
}

private void verifyDanglingIndicesIgnoredWhenObsolete(Settings settings, int expected, String reason) throws IOException {
try (NodeEnvironment env = newNodeEnvironment(settings)) {
MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService, settings);

final Settings.Builder testIndexSettings = Settings.builder().put(indexSettings)
.put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(testIndexSettings).build();
metaStateService.writeIndex("test_write", dangledIndex);

assertThat(reason,
danglingState.findNewDanglingIndices(MetaData.builder().build()).size(),
equalTo(expected));
}
}

private DanglingIndicesState createDanglingIndicesState(NodeEnvironment env, MetaStateService metaStateService) {
return new DanglingIndicesState(env, metaStateService, null, mock(ClusterService.class));
return new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null, mock(ClusterService.class));
}

private DanglingIndicesState createDanglingIndicesState(NodeEnvironment env,
MetaStateService metaStateService,
Settings settings) {
return new DanglingIndicesState(settings, env, metaStateService, null, mock(ClusterService.class));
}
}
@@ -40,6 +40,7 @@
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.zen.ElectMasterService;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.mapper.MapperParsingException;
@@ -318,6 +319,40 @@ public boolean clearData(String nodeName) {
assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true));
}

public void testDanglingIndicesIgnoredWhenObsolete() throws Exception {
logger.info("--> starting first node");
internalCluster().startNode();

logger.info("--> indexing a simple document");
client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();

internalCluster().fullRestart(new RestartCallback() {
@Override
public Settings onNodeStopped(String nodeName) throws Exception {
return Settings.builder()
.put(Node.NODE_DATA_SETTING.getKey(), false)
.put(Node.NODE_MASTER_SETTING.getKey(), false)
// avoid waiting for discovery.
.put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0)
.build();
}

@Override
public boolean validateClusterForming() {
return false;
}
});

logger.info("--> starting second node (master)");
internalCluster().startNode();

logger.info("--> verify green status");
ensureGreen();

logger.info("--> verify that the dangling index does not exists");
assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(false));
}

/**
* This test ensures that when an index deletion takes place while a node is offline, when that
* node rejoins the cluster, it deletes the index locally instead of importing it as a dangling index.
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.