Skip to content

Commit

Permalink
Convert empty tier preference check to cluster check (#81895)
Browse files Browse the repository at this point in the history
  • Loading branch information
@joegallo
joegallo committed Jan 10, 2022
1 parent afb07c4 commit e5ef389
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 86 deletions.
45 changes: 45 additions & 0 deletions ...precation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@
import org.apache.logging.log4j.Logger;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.cluster.routing.allocation.DataTier;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
Expand Down Expand Up @@ -1028,4 +1030,47 @@ static DeprecationIssue checkTransientSettingsExistence(ClusterState state) {
}
return null;
}

static DeprecationIssue emptyDataTierPreferenceCheck(ClusterState clusterState) {
if (DataTier.dataNodesWithoutAllDataRoles(clusterState).isEmpty() == false) {
List<String> indices = new ArrayList<>();
for (IndexMetadata indexMetadata : clusterState.metadata().getIndices().values()) {
List<String> tierPreference = DataTier.parseTierList(DataTier.TIER_PREFERENCE_SETTING.get(indexMetadata.getSettings()));
if (tierPreference.isEmpty()) {
String indexName = indexMetadata.getIndex().getName();
indices.add(indexName);
}
}

if (indices.isEmpty() == false) {
// this is a bit of a hassle, but the String sort order puts .someindex before someindex, and we
// don't want to give the users a list of only just all .ds-somebackingindex-blah indices -- on the other
// hand, if that's all that exists, then we don't have much choice. this next little block splits out
// all the leading-dot indices and sorts them *after* all the non-leading-dot indices
Map<Boolean, List<String>> groups = indices.stream().collect(Collectors.partitioningBy(s -> s.startsWith(".")));
List<String> noLeadingPeriod = new ArrayList<>(groups.get(false));
List<String> leadingPeriod = new ArrayList<>(groups.get(true));
Collections.sort(noLeadingPeriod);
Collections.sort(leadingPeriod);
noLeadingPeriod.addAll(leadingPeriod);
indices = noLeadingPeriod;

// if there's more than a few indices, or their names are surprisingly long, then we need to cut off the list.
// this is not ideal, but our message here is displayed unmodified in the UA, so we have to think about this.
StringBuilder builder = new StringBuilder();
Strings.collectionToDelimitedStringWithLimit(indices, ", ", "", "", 256, builder);

return new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"No [" + DataTier.TIER_PREFERENCE + "] is set for indices [" + builder + "].",
"https://ela.st/es-deprecation-7-empty-tier-preference",
"Specify a data tier preference for these indices.",
false,
null
);
}
}
return null;
}

}
6 changes: 3 additions & 3 deletions ...ugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ private DeprecationChecks() {}
ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting,
ClusterDeprecationChecks::checkGeoShapeTemplates,
ClusterDeprecationChecks::checkSparseVectorTemplates,
ClusterDeprecationChecks::checkILMFreezeActions
ClusterDeprecationChecks::checkILMFreezeActions,
ClusterDeprecationChecks::emptyDataTierPreferenceCheck
)
);

Expand Down Expand Up @@ -263,8 +264,7 @@ private static Set<Setting<Boolean>> getAllDeprecatedNodeRolesSettings() {
(clusterState, indexMetadata) -> IndexDeprecationChecks.checkGeoShapeMappings(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.frozenIndexSettingCheck(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.httpContentTypeRequiredSettingCheck(indexMetadata),
(clusterState, indexMetadata) -> IndexDeprecationChecks.mapperDyamicSettingCheck(indexMetadata),
IndexDeprecationChecks::emptyDataTierPreferenceCheck
(clusterState, indexMetadata) -> IndexDeprecationChecks.mapperDyamicSettingCheck(indexMetadata)
)
);

Expand Down
20 changes: 0 additions & 20 deletions ...deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.cluster.routing.allocation.DataTier;
import org.elasticsearch.common.joda.JodaDeprecationPatterns;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
Expand Down Expand Up @@ -729,24 +727,6 @@ static DeprecationIssue frozenIndexSettingCheck(IndexMetadata indexMetadata) {
return null;
}

static DeprecationIssue emptyDataTierPreferenceCheck(ClusterState clusterState, IndexMetadata indexMetadata) {
if (DataTier.dataNodesWithoutAllDataRoles(clusterState).isEmpty() == false) {
final List<String> tierPreference = DataTier.parseTierList(DataTier.TIER_PREFERENCE_SETTING.get(indexMetadata.getSettings()));
if (tierPreference.isEmpty()) {
String indexName = indexMetadata.getIndex().getName();
return new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"No [" + DataTier.TIER_PREFERENCE + "] is set for index [" + indexName + "].",
"https://ela.st/es-deprecation-7-empty-tier-preference",
"Specify a data tier preference for this index.",
false,
null
);
}
}
return null;
}

static DeprecationIssue checkSettingNoReplacement(IndexMetadata indexMetadata, Setting<?> deprecatedSetting, String url) {
return NodeDeprecationChecks.checkSettingNoReplacement(indexMetadata.getSettings(), deprecatedSetting, url);
}
Expand Down
80 changes: 80 additions & 0 deletions ...tion/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,20 @@
*/
package org.elasticsearch.xpack.deprecation;

import org.elasticsearch.Version;
import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.allocation.DataTier;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.collect.ImmutableOpenMap;
Expand All @@ -36,6 +42,7 @@

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand All @@ -48,9 +55,11 @@
import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING;
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS;
import static org.elasticsearch.xpack.deprecation.IndexDeprecationChecksTests.addRandomFields;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.collection.IsIterableContainingInOrder.contains;

public class ClusterDeprecationChecksTests extends ESTestCase {

Expand Down Expand Up @@ -1653,4 +1662,75 @@ public void testComponentTemplateWithBoostedFieldDynamicTemplate() throws IOExce
);
assertEquals(singletonList(expected), issues);
}

public void testEmptyDataTierPreference() {
List<String> indexNames = new ArrayList<>();
indexNames.add(".some_dotted_index");
for (int i = 0; i < 15; i++) {
indexNames.add("test-index-name-" + i);
}

List<IndexMetadata> indices = new ArrayList<>();
for (String indexName : indexNames) {
indices.add(
IndexMetadata.builder(indexName)
.settings(settings(Version.CURRENT).put(DataTier.TIER_PREFERENCE_SETTING.getKey(), " "))
.numberOfShards(randomIntBetween(1, 100))
.numberOfReplicas(randomIntBetween(1, 100))
.build()
);
}

{
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(ClusterState.EMPTY_STATE));
assertThat(issues, empty());
}

{
Metadata.Builder metadata = Metadata.builder();
for (IndexMetadata indexMetadata : indices) {
metadata.put(indexMetadata, false);
}
ClusterState clusterState = ClusterState.builder(clusterStateWithoutAllDataRoles()).metadata(metadata).build();

List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(clusterState));
assertThat(
issues,
contains(
new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"No [index.routing.allocation.include._tier_preference] is set for indices [test-index-name-0, "
+ "test-index-name-1, test-index-name-10, test-index-name-11, test-index-name-12, test-index-name-13, "
+ "test-index-name-14, test-index-name-2, test-index-name-3, test-index-name-4, test-index-name-5, "
+ "test-index-name-6, test-index-name-7, test-index-name-8, ... (16 in total, 2 omitted)].",
"https://ela.st/es-deprecation-7-empty-tier-preference",
"Specify a data tier preference for these indices.",
false,
null
)
)
);
}
}

private static ClusterState clusterStateWithoutAllDataRoles() {
DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
List<DiscoveryNode> nodesList = org.elasticsearch.core.List.of(
new DiscoveryNode(
"name_0",
"node_0",
buildNewFakeTransportAddress(),
org.elasticsearch.core.Map.of(),
org.elasticsearch.core.Set.of(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE),
Version.CURRENT
)
);
for (DiscoveryNode node : nodesList) {
discoBuilder = discoBuilder.add(node);
}
discoBuilder.localNodeId(randomFrom(nodesList).getId());
discoBuilder.masterNodeId(randomFrom(nodesList).getId());

return ClusterState.builder(ClusterState.EMPTY_STATE).nodes(discoBuilder.build()).build();
}
}
63 changes: 0 additions & 63 deletions ...cation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.allocation.DataTier;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
Expand Down Expand Up @@ -1019,66 +1016,6 @@ public void testFrozenIndex() {
);
}

public void testEmptyDataTierPreference() {
Settings.Builder settings = settings(Version.CURRENT);
settings.put(DataTier.TIER_PREFERENCE_SETTING.getKey(), " ");
IndexMetadata indexMetadata = IndexMetadata.builder("test")
.settings(settings)
.numberOfShards(randomIntBetween(1, 100))
.numberOfReplicas(randomIntBetween(1, 100))
.build();

{
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(
INDEX_SETTINGS_CHECKS,
c -> c.apply(ClusterState.EMPTY_STATE, indexMetadata)
);
assertThat(issues, empty());
}

{
ClusterState clusterState = clusterStateWithoutAllDataRoles();
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(
INDEX_SETTINGS_CHECKS,
c -> c.apply(clusterState, indexMetadata)
);
assertThat(
issues,
contains(
new DeprecationIssue(
DeprecationIssue.Level.WARNING,
"No [index.routing.allocation.include._tier_preference] is set for index [test].",
"https://ela.st/es-deprecation-7-empty-tier-preference",
"Specify a data tier preference for this index.",
false,
null
)
)
);
}
}

private static ClusterState clusterStateWithoutAllDataRoles() {
DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
List<DiscoveryNode> nodesList = org.elasticsearch.core.List.of(
new DiscoveryNode(
"name_0",
"node_0",
buildNewFakeTransportAddress(),
org.elasticsearch.core.Map.of(),
org.elasticsearch.core.Set.of(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE),
Version.CURRENT
)
);
for (DiscoveryNode node : nodesList) {
discoBuilder = discoBuilder.add(node);
}
discoBuilder.localNodeId(randomFrom(nodesList).getId());
discoBuilder.masterNodeId(randomFrom(nodesList).getId());

return ClusterState.builder(ClusterState.EMPTY_STATE).nodes(discoBuilder.build()).build();
}

public void testForceMemoryTermDictionary() {
Settings settings = Settings.builder()
.put(Store.FORCE_RAM_TERM_DICT.getKey(), randomBoolean())
Expand Down

0 comments on commit e5ef389

Please sign in to comment.