Frozen default cache size (#71844) (#71967)

This commit adds a default cache size to frozen tier of the greater of
90% and total disk size minus 100 GB.

docs/reference/searchable-snapshots/index.asciidoc

@@ -153,18 +153,42 @@ for search. Many searches will need to retrieve only a small subset of the
 total shard data before returning results.
 
 To mount a searchable snapshot index with the shared cache mount option, you
-must configure the `xpack.searchable.snapshot.shared_cache.size` setting to
-reserve space for the cache on one or more nodes. Indices mounted with the
-shared cache mount option are only allocated to nodes that have this setting
-configured.
+must have one or more nodes with a shared cache available. By default,
+dedicated frozen data tier nodes (nodes with the `data_frozen` role and no other
+data roles) have a shared cache configured using the greater of 90% of total
+disk space and total disk space subtracted a headroom of 100GB.
+
+Using a dedicated frozen tier is highly recommended for production use. If you
+do not have a dedicated frozen tier, you must configure the
+`xpack.searchable.snapshot.shared_cache.size` setting to reserve space for the
+cache on one or more nodes. Indices mounted with the shared cache mount option
+are only allocated to nodes that have a shared cache.
 
 [[searchable-snapshots-shared-cache]]
 `xpack.searchable.snapshot.shared_cache.size`::
+(<<static-cluster-setting,Static>>)
+The size of the space reserved for the shared cache, either specified as a
+percentage of total disk space or an absolute <<byte-units,byte value>>.
+Defaults to 90% of total disk space on dedicated frozen data tier nodes,
+otherwise `0b`.
+
+`xpack.searchable.snapshot.shared_cache.size.max_headroom`::
 (<<static-cluster-setting,Static>>, <<byte-units,byte value>>)
-The size of the space reserved for the shared cache. Defaults to `0b`, meaning
-that the node has no shared cache.
+For dedicated frozen tier nodes, the max headroom to maintain. Defaults to 100GB
+on dedicated frozen tier nodes when
+`xpack.searchable.snapshot.shared_cache.size` is not explicitly set, otherwise
+-1 (not set). Can only be set when `xpack.searchable.snapshot.shared_cache.size`
+is set as a percentage.
+
+To illustrate how these settings work in concert let us look at two examples
+when using the default values of the settings on a dedicated frozen node:
+
+* A 4000 GB disk will result in a shared cache sized at 3900 GB. 90% of 4000 GB
+is 3600 GB, leaving 400 GB headroom. The default `max_headroom` of 100 GB
+takes effect, and the result is therefore 3900 GB.
+* A 400 GB disk will result in a shard cache sized at 360 GB.
 
-You can configure the setting in `elasticsearch.yml`:
+You can configure the settings in `elasticsearch.yml`:
 
 [source,yaml]
 ----
@@ -174,7 +198,8 @@ xpack.searchable.snapshot.shared_cache.size: 4TB
 IMPORTANT: Currently, you can configure
 `xpack.searchable.snapshot.shared_cache.size` on any node. However, if the cache size is set on any
 node that does not have the <<data-frozen-node,`data_frozen`>> role, it will be treated as though it
-is set to `0b`.
+is set to `0b`. Additionally, nodes with a shared  cache can only have a single
+<<path-settings,data path>>.
 
 You can set `xpack.searchable.snapshot.shared_cache.size` to any size between a
 couple of gigabytes up to 90% of available disk space. We only recommend larger

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java

@@ -104,6 +104,19 @@ public static boolean isRemoteClusterClient(final Settings settings) {
         return hasRole(settings, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE);
     }
 
+    private static boolean isDedicatedFrozenRoles(Set<DiscoveryNodeRole> roles) {
+        return roles.contains(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE) &&
+            roles.stream().filter(DiscoveryNodeRole::canContainData)
+                .anyMatch(r -> r != DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE) == false;
+    }
+
+    /**
+     * Check if the settings are for a dedicated frozen node, i.e. has frozen role and no other data roles.
+     */
+    public static boolean isDedicatedFrozenNode(final Settings settings) {
+        return isDedicatedFrozenRoles(getRolesFromSettings(settings));
+    }
+
     private final String nodeName;
     private final String nodeId;
     private final String ephemeralId;
@@ -426,6 +439,14 @@ public boolean isRemoteClusterClient() {
         return roles.contains(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE);
     }
 
+    /**
+     * Returns whether or not the node is a frozen only node, i.e., has data frozen role and no other data roles.
+     * @return
+     */
+    public boolean isDedicatedFrozenNode() {
+        return isDedicatedFrozenRoles(getRoles());
+    }
+
     /**
      * Returns a set of all the roles that the node has. The roles are returned in sorted order by the role name.
      * <p>

server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java

@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
@@ -150,7 +149,7 @@ public void onNewInfo(ClusterInfo info) {
             final DiskUsage usage = entry.value;
             final RoutingNode routingNode = routingNodes.node(node);
 
-            if (isFrozenOnlyNode(routingNode)) {
+            if (isDedicatedFrozenNode(routingNode)) {
                 ByteSizeValue total = ByteSizeValue.ofBytes(usage.getTotalBytes());
                 long frozenFloodStageThreshold = diskThresholdSettings.getFreeBytesThresholdFrozenFloodStage(total).getBytes();
                 if (usage.getFreeBytes() < frozenFloodStageThreshold) {
@@ -395,13 +394,11 @@ private static void cleanUpRemovedNodes(ObjectLookupContainer<String> nodesToKee
         }
     }
 
-    private boolean isFrozenOnlyNode(RoutingNode routingNode) {
+    private boolean isDedicatedFrozenNode(RoutingNode routingNode) {
         if (routingNode == null) {
             return false;
         }
         DiscoveryNode node = routingNode.node();
-        return node.getRoles().contains(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE) &&
-            node.getRoles().stream().filter(DiscoveryNodeRole::canContainData)
-                .anyMatch(r -> r != DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE) == false;
+        return node.isDedicatedFrozenNode();
     }
 }

server/src/main/java/org/elasticsearch/common/unit/RelativeByteSizeValue.java

@@ -18,7 +18,7 @@
  */
 public class RelativeByteSizeValue {
 
-    public static final String MAX_HEADROOM_PREFIX = "max_headroom=";
+    public static final RelativeByteSizeValue ZERO = new RelativeByteSizeValue(ByteSizeValue.ZERO);
     private final ByteSizeValue absolute;
     private final RatioValue ratio;
 

server/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.env.NodeEnvironment.NodePath;
 
 import java.io.IOException;
+import java.nio.file.FileStore;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -133,12 +134,16 @@ public static FsInfo.Path getFSInfo(NodePath nodePath) throws IOException {
         // NOTE: we use already cached (on node startup) FileStore and spins
         // since recomputing these once per second (default) could be costly,
         // and they should not change:
-        fsPath.total = adjustForHugeFilesystems(nodePath.fileStore.getTotalSpace());
+        fsPath.total = getTotal(nodePath.fileStore);
         fsPath.free = adjustForHugeFilesystems(nodePath.fileStore.getUnallocatedSpace());
         fsPath.available = adjustForHugeFilesystems(nodePath.fileStore.getUsableSpace());
         fsPath.type = nodePath.fileStore.type();
         fsPath.mount = nodePath.fileStore.toString();
         return fsPath;
     }
 
+    public static long getTotal(FileStore fileStore) throws IOException {
+        return adjustForHugeFilesystems(fileStore.getTotalSpace());
+    }
+
 }

server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleSettingTests.java

@@ -15,6 +15,8 @@
 import java.util.Collections;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.test.NodeRoles.addRoles;
+import static org.elasticsearch.test.NodeRoles.nonDataNode;
 import static org.elasticsearch.test.NodeRoles.onlyRole;
 import static org.elasticsearch.test.NodeRoles.removeRoles;
 import static org.hamcrest.Matchers.hasItem;
@@ -68,4 +70,13 @@ private void runRoleTest(final Predicate<Settings> predicate, final DiscoveryNod
         assertSettingDeprecationsAndWarnings(new Setting<?>[]{role.legacySetting()});
     }
 
+    public void testIsDedicatedFrozenNode() {
+        assertTrue(DiscoveryNode.isDedicatedFrozenNode(onlyRole(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)));
+        assertFalse(DiscoveryNode.isDedicatedFrozenNode(removeRoles(Collections.singleton(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE))));
+        assertTrue(DiscoveryNode.isDedicatedFrozenNode(addRoles(nonDataNode(),
+            org.elasticsearch.common.collect.Set.of(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE))));
+        assertFalse(DiscoveryNode.isDedicatedFrozenNode(
+            addRoles(org.elasticsearch.common.collect.Set.of(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE,
+                DiscoveryNodeRole.DATA_HOT_NODE_ROLE))));
+    }
 }

server/src/test/java/org/elasticsearch/common/unit/RelativeByteSizeValueTests.java

@@ -27,8 +27,7 @@ public void testPercentage() {
     public void testRatio() {
         double value = (double) randomIntBetween(1, 100) / 100;
         RelativeByteSizeValue parsed = RelativeByteSizeValue.parseRelativeByteSizeValue(Double.toString(value), "test");
-        assertThat(parsed.getRatio().getAsRatio(),
-            equalTo(value));
+        assertThat(parsed.getRatio().getAsRatio(), equalTo(value));
         assertThat(parsed.isAbsolute(), is(false));
         assertThat(parsed.isNonZeroSize(), is(true));
     }

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseFrozenSearchableSnapshotsIntegTestCase.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.RatioValue;
 import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 
 public class BaseFrozenSearchableSnapshotsIntegTestCase extends BaseSearchableSnapshotsIntegTestCase {
@@ -27,9 +28,10 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
                 FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(),
                 rarely()
                     ? randomBoolean()
-                        ? new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.KB)
-                        : new ByteSizeValue(randomIntBetween(1, 1000), ByteSizeUnit.BYTES)
-                    : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB)
+                        ? new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.KB).getStringRep()
+                        : new ByteSizeValue(randomIntBetween(1, 1000), ByteSizeUnit.BYTES).getStringRep()
+                    : randomBoolean() ? new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB).getStringRep()
+                    : new RatioValue(randomDoubleBetween(0.0d, 0.1d, false)).toString() // only use up to 0.1% disk to be friendly.
             );
         }
 

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java

@@ -308,6 +308,7 @@ public List<Setting<?>> getSettings() {
             CacheService.SNAPSHOT_CACHE_SYNC_SHUTDOWN_TIMEOUT,
             SearchableSnapshotEnableAllocationDecider.SEARCHABLE_SNAPSHOTS_ALLOCATE_ON_ROLLING_RESTART,
             FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING,
+            FrozenCacheService.SNAPSHOT_CACHE_SIZE_MAX_HEADROOM_SETTING,
             FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING,
             FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING,
             FrozenCacheService.FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING,

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java

@@ -53,7 +53,7 @@ public static class TransportAction extends HandledTransportAction<Request, Froz
         @Inject
         public TransportAction(Settings settings, TransportService transportService, ActionFilters actionFilters) {
             super(NAME, transportService, actionFilters, Request::new);
-            response = new FrozenCacheInfoResponse(SNAPSHOT_CACHE_SIZE_SETTING.get(settings).getBytes() > 0);
+            response = new FrozenCacheInfoResponse(SNAPSHOT_CACHE_SIZE_SETTING.get(settings).isNonZeroSize());
         }
 
         @Override