Skip to content

Commit

Permalink
Adding a critical warning for low watermark exceeded to the deprecati…
Browse files Browse the repository at this point in the history 
…on info API (#83544)
  • Loading branch information
@masseyke
masseyke committed Feb 10, 2022
1 parent 8d10cfe commit 66632cc
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 3 deletions.
69 changes: 67 additions & 2 deletions ...rc/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,18 @@
import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.nodes.TransportNodesAction;
import org.elasticsearch.cluster.ClusterInfo;
import org.elasticsearch.cluster.ClusterInfoService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.DiskUsage;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.plugins.PluginsService;
Expand All @@ -28,6 +33,9 @@
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Locale;

import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING;

public class TransportNodeDeprecationCheckAction extends TransportNodesAction<
NodesDeprecationCheckRequest,
Expand All @@ -38,6 +46,7 @@ public class TransportNodeDeprecationCheckAction extends TransportNodesAction<
private final Settings settings;
private final XPackLicenseState licenseState;
private final PluginsService pluginsService;
private final ClusterInfoService clusterInfoService;
private volatile List<String> skipTheseDeprecations;

@Inject
Expand All @@ -48,7 +57,8 @@ public TransportNodeDeprecationCheckAction(
ClusterService clusterService,
TransportService transportService,
PluginsService pluginsService,
ActionFilters actionFilters
ActionFilters actionFilters,
ClusterInfoService clusterInfoService
) {
super(
NodesDeprecationCheckAction.NAME,
Expand All @@ -64,6 +74,7 @@ public TransportNodeDeprecationCheckAction(
this.settings = settings;
this.pluginsService = pluginsService;
this.licenseState = licenseState;
this.clusterInfoService = clusterInfoService;
skipTheseDeprecations = DeprecationChecks.SKIP_DEPRECATIONS_SETTING.get(settings);
// Safe to register this here because it happens synchronously before the cluster service is started:
clusterService.getClusterSettings()
Expand Down Expand Up @@ -123,8 +134,62 @@ NodesDeprecationCheckAction.NodeResponse nodeOperation(
nodeSettingsChecks,
(c) -> c.apply(filteredNodeSettings, pluginsService.info(), filteredClusterState, licenseState)
);

DeprecationIssue watermarkIssue = checkDiskLowWatermark(
filteredNodeSettings,
filteredClusterState.metadata().settings(),
clusterInfoService.getClusterInfo(),
clusterService.getClusterSettings(),
transportService.getLocalNode().getId()
);
if (watermarkIssue != null) {
issues.add(watermarkIssue);
}
return new NodesDeprecationCheckAction.NodeResponse(transportService.getLocalNode(), issues);
}

static DeprecationIssue checkDiskLowWatermark(
Settings nodeSettings,
Settings dynamicSettings,
ClusterInfo clusterInfo,
ClusterSettings clusterSettings,
String nodeId
) {
DiskUsage usage = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeId);
if (usage != null) {
long freeBytes = usage.getFreeBytes();
double freeDiskPercentage = usage.getFreeDiskAsPercentage();
if (exceedsLowWatermark(nodeSettings, clusterSettings, freeBytes, freeDiskPercentage)
|| exceedsLowWatermark(dynamicSettings, clusterSettings, freeBytes, freeDiskPercentage)) {
return new DeprecationIssue(
DeprecationIssue.Level.CRITICAL,
"Disk usage exceeds low watermark",
"https://ela.st/es-deprecation-7-disk-watermark-exceeded",
String.format(
Locale.ROOT,
"Disk usage exceeds low watermark, which will prevent reindexing indices during upgrade. Get disk usage on "
+ "all nodes below the value specified in %s",
CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey()
),
false,
null
);
}
}
return null;
}

private static boolean exceedsLowWatermark(
Settings settingsToCheck,
ClusterSettings clusterSettings,
long freeBytes,
double freeDiskPercentage
) {
DiskThresholdSettings diskThresholdSettings = new DiskThresholdSettings(settingsToCheck, clusterSettings);
if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLow().getBytes()
|| freeDiskPercentage < diskThresholdSettings.getFreeDiskThresholdLow()) {
return true;
}
return false;
}

}
63 changes: 62 additions & 1 deletion ...st/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@

import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.cluster.ClusterInfo;
import org.elasticsearch.cluster.ClusterInfoService;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.DiskUsage;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.List;
Expand Down Expand Up @@ -58,14 +62,18 @@ public void testNodeOperation() {
Mockito.when(transportService.getLocalNode()).thenReturn(node);
PluginsService pluginsService = Mockito.mock(PluginsService.class);
ActionFilters actionFilters = Mockito.mock(ActionFilters.class);
ClusterInfoService clusterInfoService = Mockito.mock(ClusterInfoService.class);
ClusterInfo clusterInfo = ClusterInfo.EMPTY;
Mockito.when(clusterInfoService.getClusterInfo()).thenReturn(clusterInfo);
TransportNodeDeprecationCheckAction transportNodeDeprecationCheckAction = new TransportNodeDeprecationCheckAction(
nodeSettings,
threadPool,
licenseState,
clusterService,
transportService,
pluginsService,
actionFilters
actionFilters,
clusterInfoService
);
NodesDeprecationCheckAction.NodeRequest nodeRequest = null;
AtomicReference<Settings> visibleNodeSettings = new AtomicReference<>();
Expand Down Expand Up @@ -126,4 +134,57 @@ public void testNodeOperation() {
);
}

public void testCheckDiskLowWatermark() {
Settings nodeSettings = Settings.EMPTY;
Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put("cluster.routing.allocation.disk.watermark.low", "10%");
Settings settingsWithLowWatermark = settingsBuilder.build();
Settings dynamicSettings = settingsWithLowWatermark;
ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
String nodeId = "123";
ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableSpaceUsageBuilder = ImmutableOpenMap.builder();
long totalBytesOnMachine = 100;
long totalBytesFree = 70;
mostAvailableSpaceUsageBuilder.put(nodeId, new DiskUsage(nodeId, "", "", totalBytesOnMachine, totalBytesFree));
ClusterInfo clusterInfo = new ClusterInfo(
ImmutableOpenMap.of(),
mostAvailableSpaceUsageBuilder.build(),
ImmutableOpenMap.of(),
ImmutableOpenMap.of(),
ImmutableOpenMap.of(),
ImmutableOpenMap.of()
);
DeprecationIssue issue = TransportNodeDeprecationCheckAction.checkDiskLowWatermark(
nodeSettings,
dynamicSettings,
clusterInfo,
clusterSettings,
nodeId
);
assertNotNull(issue);
assertEquals("Disk usage exceeds low watermark", issue.getMessage());

// Making sure there's no warning when we clear out the cluster settings:
dynamicSettings = Settings.EMPTY;
issue = TransportNodeDeprecationCheckAction.checkDiskLowWatermark(
nodeSettings,
dynamicSettings,
clusterInfo,
clusterSettings,
nodeId
);
assertNull(issue);

// And make sure there is a warning when the setting is in the node settings but not the cluster settings:
nodeSettings = settingsWithLowWatermark;
issue = TransportNodeDeprecationCheckAction.checkDiskLowWatermark(
nodeSettings,
dynamicSettings,
clusterInfo,
clusterSettings,
nodeId
);
assertNotNull(issue);
assertEquals("Disk usage exceeds low watermark", issue.getMessage());
}
}

0 comments on commit 66632cc

Please sign in to comment.