diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index a7a12f880b49a..3e27ce8460907 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -40,6 +40,7 @@ import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -168,6 +169,18 @@ private void setup(boolean addShutdownHook, Environment environment) throws Boot BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings), BootstrapSettings.CTRLHANDLER_SETTING.get(settings)); + final long cacheSize = SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.get(settings).getBytes(); + final long regionSize = SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(settings).getBytes(); + final int numRegions = Math.toIntExact(cacheSize / regionSize); + final long fileSize = numRegions * regionSize; + if (fileSize > 0) { + try { + Natives.tryCreateCacheFile(environment, fileSize); + } catch (Exception e) { + throw new BootstrapException(e); + } + } + // initialize probes before the security manager is installed initializeProbes(); diff --git a/server/src/main/java/org/elasticsearch/bootstrap/JNAFalloc.java b/server/src/main/java/org/elasticsearch/bootstrap/JNAFalloc.java new file mode 100644 index 0000000000000..49fe8dde23d62 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/bootstrap/JNAFalloc.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.bootstrap; + +import com.sun.jna.Native; +import com.sun.jna.Platform; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.Constants; +import org.elasticsearch.common.Nullable; + +/** + * System specific wrappers of the fallocate system call via JNA for Linux and OSX. + */ +abstract class JNAFalloc { + + private static final Logger logger = LogManager.getLogger(JNAFalloc.class); + + public abstract int fallocate(int fd, long offset, long length); + + @Nullable + public static JNAFalloc falloc() { + try { + if (Constants.MAC_OS_X) { + return OSX.INSTANCE; + } else if (Constants.LINUX) { + return Linux.INSTANCE; + } + } catch (Throwable t) { + logger.warn("unable to link C library. native (falloc) will be disabled.", t); + } + return null; + } + + private static class Linux extends JNAFalloc { + + static final Linux INSTANCE = new Linux(); + + static { + Native.register(Platform.C_LIBRARY_NAME); + } + + @Override + public int fallocate(int fd, long offset, long length) { + final int res = fallocate(fd, 0, offset, length); + return res == 0 ? 0 : Native.getLastError(); + } + + private static native int fallocate(int fd, int mode, long offset, long length); + } + + private static class OSX extends JNAFalloc { + + static final OSX INSTANCE = new OSX(); + + static { + Native.register("c"); + } + + @Override + public int fallocate(int fd, long offset, long length) { + return posix_fallocate(fd, offset, length); + } + + private static native int posix_fallocate(int fd, long offset, long length); + } + +} diff --git a/server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java b/server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java index 90d576b29fdd8..b62f35a0f07ae 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java @@ -14,9 +14,17 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.util.Constants; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.env.Environment; import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.snapshots.SnapshotUtils; +import java.io.FileOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; import java.nio.file.Path; import static org.elasticsearch.bootstrap.JNAKernel32Library.SizeT; @@ -260,4 +268,40 @@ static void tryInstallSystemCallFilter(Path tmpFile) { logger.warn("unable to install syscall filter: ", e); } } + + @SuppressForbidden(reason = "need access to fd on FileOutputStream") + static void fallocateSnapshotCacheFile(Environment environment, long fileSize) throws IOException { + final JNAFalloc falloc = JNAFalloc.falloc(); + if (falloc == null) { + logger.debug("not trying to create a shared cache file using fallocate because native fallocate library could not be loaded."); + return; + } + + Path cacheFile = SnapshotUtils.findCacheSnapshotCacheFilePath(environment, fileSize); + if (cacheFile == null) { + throw new IOException("could not find a directory with adequate free space for cache file"); + } + boolean success = false; + try (FileOutputStream fileChannel = new FileOutputStream(cacheFile.toFile())) { + long currentSize = fileChannel.getChannel().size(); + if (currentSize < fileSize) { + final Field field = fileChannel.getFD().getClass().getDeclaredField("fd"); + field.setAccessible(true); + final int result = falloc.fallocate((int) field.get(fileChannel.getFD()), currentSize, fileSize - currentSize); + if (result == 0) { + success = true; + logger.info("allocated cache file [{}] using fallocate", cacheFile); + } else { + logger.warn("failed to initialize cache file [{}] using fallocate errno [{}]", cacheFile, result); + } + } + } catch (Exception e) { + logger.warn(new ParameterizedMessage("failed to initialize cache file [{}] using fallocate", cacheFile), e); + } finally { + if (success == false) { + // if anything goes wrong, delete the potentially created file to not waste disk space + Files.deleteIfExists(cacheFile); + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Natives.java b/server/src/main/java/org/elasticsearch/bootstrap/Natives.java index b9f072a52e817..e91116a5508ec 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Natives.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Natives.java @@ -10,7 +10,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.env.Environment; +import java.io.IOException; import java.nio.file.Path; /** @@ -132,4 +134,20 @@ static boolean isSystemCallFilterInstalled() { } return JNANatives.LOCAL_SYSTEM_CALL_FILTER; } + + /** + * On Linux, this method tries to create the searchable snapshot frozen cache file using fallocate if JNA is available. This enables + * a much faster creation of the file than the fallback mechanism in the searchable snapshots plugin that will pre-allocate the cache + * file by writing zeros to the file. + * + * @throws IOException on failure to determine free disk space for a data path + */ + public static void tryCreateCacheFile(Environment environment, long fileSize) throws IOException { + if (JNA_AVAILABLE == false) { + logger.warn("cannot use fallocate to create cache file because JNA is not available"); + return; + } + JNANatives.fallocateSnapshotCacheFile(environment, fileSize); + } + } diff --git a/server/src/main/java/org/elasticsearch/env/Environment.java b/server/src/main/java/org/elasticsearch/env/Environment.java index 18b509c6f58bf..8bf2a5fbbd241 100644 --- a/server/src/main/java/org/elasticsearch/env/Environment.java +++ b/server/src/main/java/org/elasticsearch/env/Environment.java @@ -300,6 +300,16 @@ public static FileStore getFileStore(final Path path) throws IOException { return new ESFileStore(Files.getFileStore(path)); } + public static long getUsableSpace(Path path) throws IOException { + long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace(); + + /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */ + if (freeSpaceInBytes < 0) { + freeSpaceInBytes = Long.MAX_VALUE; + } + return freeSpaceInBytes; + } + /** * asserts that the two environments are equivalent for all things the environment cares about (i.e., all but the setting * object which may contain different setting) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java index 58f41ffd9b306..16d479d0f94f8 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java @@ -9,9 +9,14 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -107,4 +112,29 @@ public static List filterIndices(List availableIndices, String[] } return List.copyOf(result); } + + /** + * Tries to find a suitable path to a searchable snapshots shared cache file in the data paths founds in the environment. + * + * @return path for the cache file or {@code null} if none could be found + */ + @Nullable + public static Path findCacheSnapshotCacheFilePath(Environment environment, long fileSize) throws IOException { + Path cacheFile = null; + for (Path path : environment.dataFiles()) { + Files.createDirectories(path); + // TODO: be resilient to this check failing and try next path? + long usableSpace = Environment.getUsableSpace(path); + Path p = path.resolve(SnapshotsService.CACHE_FILE_NAME); + if (Files.exists(p)) { + usableSpace += Files.size(p); + } + // TODO: leave some margin for error here + if (usableSpace > fileSize) { + cacheFile = p; + break; + } + } + return cacheFile; + } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 7fe2ce8caca49..913e99736323e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus; @@ -132,6 +133,26 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus public static final String UPDATE_SNAPSHOT_STATUS_ACTION_NAME = "internal:cluster/snapshot/update_snapshot_status"; + public static final String SHARED_CACHE_SETTINGS_PREFIX = "xpack.searchable.snapshot.shared_cache."; + + public static final Setting SHARED_CACHE_RANGE_SIZE_SETTING = Setting.byteSizeSetting( + SHARED_CACHE_SETTINGS_PREFIX + "range_size", + ByteSizeValue.ofMb(16), // default + Setting.Property.NodeScope + ); + public static final Setting SNAPSHOT_CACHE_REGION_SIZE_SETTING = Setting.byteSizeSetting( + SHARED_CACHE_SETTINGS_PREFIX + "region_size", + SHARED_CACHE_RANGE_SIZE_SETTING, + Setting.Property.NodeScope + ); + public static final Setting SNAPSHOT_CACHE_SIZE_SETTING = Setting.byteSizeSetting( + SHARED_CACHE_SETTINGS_PREFIX + "size", + ByteSizeValue.ZERO, + Setting.Property.NodeScope + ); + + public static final String CACHE_FILE_NAME = "shared_snapshot_cache"; + private final ClusterService clusterService; private final IndexNameExpressionResolver indexNameExpressionResolver; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java index d5e2edf479060..23307d4452856 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java @@ -134,13 +134,8 @@ public ByteSizeValue getMinLocalStorageAvailable() { return minLocalStorageAvailable; } + // non-static indirection to enable mocking in tests long getUsableSpace(Path path) throws IOException { - long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace(); - - /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */ - if (freeSpaceInBytes < 0) { - freeSpaceInBytes = Long.MAX_VALUE; - } - return freeSpaceInBytes; + return Environment.getUsableSpace(path); } } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java index 62efbc0ac72f7..8bab23714d365 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; @@ -72,7 +73,7 @@ protected Settings nodeSettings(int nodeOrdinal) { ); } builder.put( - FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), + SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), rarely() ? randomBoolean() ? new ByteSizeValue(randomIntBetween(0, 10), ByteSizeUnit.KB) @@ -80,14 +81,14 @@ protected Settings nodeSettings(int nodeOrdinal) { : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB) ); builder.put( - FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), + SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), rarely() ? new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB) : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB) ); if (randomBoolean()) { builder.put( - FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(), + SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), rarely() ? new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB) : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB) diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index d3ad5807c7b23..1b76d6b3312b8 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -57,6 +57,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.snapshots.SourceOnlySnapshotRepository; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; @@ -250,9 +251,9 @@ public List> getSettings() { CacheService.SNAPSHOT_CACHE_MAX_FILES_TO_SYNC_AT_ONCE_SETTING, CacheService.SNAPSHOT_CACHE_SYNC_SHUTDOWN_TIMEOUT, SearchableSnapshotEnableAllocationDecider.SEARCHABLE_SNAPSHOTS_ALLOCATE_ON_ROLLING_RESTART, - FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING, - FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING, - FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING, + SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING, + SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING, + SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING, FrozenCacheService.FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING, FrozenCacheService.SNAPSHOT_CACHE_MAX_FREQ_SETTING, FrozenCacheService.SNAPSHOT_CACHE_DECAY_INTERVAL_SETTING, diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheService.java index f9c498545a5e9..94d192cb11973 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheService.java @@ -46,35 +46,19 @@ import java.util.function.LongSupplier; import java.util.function.Predicate; +import static org.elasticsearch.snapshots.SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING; +import static org.elasticsearch.snapshots.SnapshotsService.SHARED_CACHE_SETTINGS_PREFIX; +import static org.elasticsearch.snapshots.SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING; +import static org.elasticsearch.snapshots.SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes; public class FrozenCacheService implements Releasable { - private static final String SETTINGS_PREFIX = "xpack.searchable.snapshot.shared_cache."; - - public static final Setting SNAPSHOT_CACHE_SIZE_SETTING = Setting.byteSizeSetting( - SETTINGS_PREFIX + "size", - ByteSizeValue.ZERO, - Setting.Property.NodeScope - ); - public static final ByteSizeValue MIN_SNAPSHOT_CACHE_RANGE_SIZE = new ByteSizeValue(4, ByteSizeUnit.KB); public static final ByteSizeValue MAX_SNAPSHOT_CACHE_RANGE_SIZE = new ByteSizeValue(Integer.MAX_VALUE, ByteSizeUnit.BYTES); - public static final Setting FROZEN_CACHE_RANGE_SIZE_SETTING = Setting.byteSizeSetting( - SETTINGS_PREFIX + "range_size", - ByteSizeValue.ofMb(16), // default - Setting.Property.NodeScope - ); - - public static final Setting SNAPSHOT_CACHE_REGION_SIZE_SETTING = Setting.byteSizeSetting( - SETTINGS_PREFIX + "region_size", - FROZEN_CACHE_RANGE_SIZE_SETTING, - Setting.Property.NodeScope - ); - public static final Setting FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING = Setting.byteSizeSetting( - SETTINGS_PREFIX + "recovery_range_size", + SHARED_CACHE_SETTINGS_PREFIX + "recovery_range_size", new ByteSizeValue(128, ByteSizeUnit.KB), // default MIN_SNAPSHOT_CACHE_RANGE_SIZE, // min MAX_SNAPSHOT_CACHE_RANGE_SIZE, // max @@ -83,7 +67,7 @@ public class FrozenCacheService implements Releasable { public static final TimeValue MIN_SNAPSHOT_CACHE_DECAY_INTERVAL = TimeValue.timeValueSeconds(1L); public static final Setting SNAPSHOT_CACHE_DECAY_INTERVAL_SETTING = Setting.timeSetting( - SETTINGS_PREFIX + "decay.interval", + SHARED_CACHE_SETTINGS_PREFIX + "decay.interval", TimeValue.timeValueSeconds(60L), // default MIN_SNAPSHOT_CACHE_DECAY_INTERVAL, // min Setting.Property.NodeScope, @@ -91,14 +75,14 @@ public class FrozenCacheService implements Releasable { ); public static final Setting SNAPSHOT_CACHE_MAX_FREQ_SETTING = Setting.intSetting( - SETTINGS_PREFIX + "max_freq", + SHARED_CACHE_SETTINGS_PREFIX + "max_freq", 100, // default 1, // min Setting.Property.NodeScope ); public static final Setting SNAPSHOT_CACHE_MIN_TIME_DELTA_SETTING = Setting.timeSetting( - SETTINGS_PREFIX + "min_time_delta", + SHARED_CACHE_SETTINGS_PREFIX + "min_time_delta", TimeValue.timeValueSeconds(60L), // default TimeValue.timeValueSeconds(0L), // min Setting.Property.NodeScope @@ -157,7 +141,7 @@ public FrozenCacheService(Environment environment, ThreadPool threadPool) { } decayTask = new CacheDecayTask(threadPool, SNAPSHOT_CACHE_DECAY_INTERVAL_SETTING.get(settings)); decayTask.rescheduleIfNecessary(); - this.rangeSize = FROZEN_CACHE_RANGE_SIZE_SETTING.get(settings); + this.rangeSize = SHARED_CACHE_RANGE_SIZE_SETTING.get(settings); this.recoveryRangeSize = FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING.get(settings); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/SharedBytes.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/SharedBytes.java index 82bcb1082c3a4..4dfd9ba195b85 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/SharedBytes.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/SharedBytes.java @@ -15,6 +15,8 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; +import org.elasticsearch.snapshots.SnapshotUtils; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.IOException; import java.nio.ByteBuffer; @@ -33,8 +35,6 @@ public class SharedBytes extends AbstractRefCounted { StandardOpenOption.WRITE, StandardOpenOption.CREATE }; - private static final String CACHE_FILE_NAME = "snap_cache"; - final int numRegions; final long regionSize; @@ -51,29 +51,21 @@ public class SharedBytes extends AbstractRefCounted { final long fileSize = numRegions * regionSize; Path cacheFile = null; if (fileSize > 0) { - for (Path path : environment.dataFiles()) { - // TODO: be resilient to this check failing and try next path? - long usableSpace = getUsableSpace(path); - Path p = path.resolve(CACHE_FILE_NAME); - if (Files.exists(p)) { - usableSpace += Files.size(p); - } - // TODO: leave some margin for error here - if (usableSpace > fileSize) { - cacheFile = p; - break; - } - } + cacheFile = SnapshotUtils.findCacheSnapshotCacheFilePath(environment, fileSize); if (cacheFile == null) { throw new IOException("Could not find a directory with adequate free space for cache file"); } // TODO: maybe make this faster by allocating a larger direct buffer if this is too slow for very large files // We fill either the full file or the bytes between its current size and the desired size once with zeros to fully allocate // the file up front - logger.info("creating shared snapshot cache file [size={}, path={}]", fileSize, cacheFile); final ByteBuffer fillBytes = ByteBuffer.allocate(Channels.WRITE_CHUNK_SIZE); this.fileChannel = FileChannel.open(cacheFile, OPEN_OPTIONS); long written = fileChannel.size(); + if (fileSize < written) { + logger.info("creating shared snapshot cache file [size={}, path={}]", fileSize, cacheFile); + } else if (fileSize == written) { + logger.debug("reusing existing shared snapshot cache file [size={}, path={}]", fileSize, cacheFile); + } fileChannel.position(written); while (written < fileSize) { final int toWrite = Math.toIntExact(Math.min(fileSize - written, Channels.WRITE_CHUNK_SIZE)); @@ -87,23 +79,12 @@ public class SharedBytes extends AbstractRefCounted { } else { this.fileChannel = null; for (Path path : environment.dataFiles()) { - Files.deleteIfExists(path.resolve(CACHE_FILE_NAME)); + Files.deleteIfExists(path.resolve(SnapshotsService.CACHE_FILE_NAME)); } } this.path = cacheFile; } - // TODO: dry up against MLs usage of the same method - private static long getUsableSpace(Path path) throws IOException { - long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace(); - - /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */ - if (freeSpaceInBytes < 0) { - freeSpaceInBytes = Long.MAX_VALUE; - } - return freeSpaceInBytes; - } - @Override protected void closeInternal() { try { diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/FrozenIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/FrozenIndexInputTests.java index 6e55d51e920ae..d943a83c8664f 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/FrozenIndexInputTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/FrozenIndexInputTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.store.StoreFileMetadata; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase; import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; @@ -55,7 +56,7 @@ public void testRandomReads() throws IOException { final ByteSizeValue rangeSize; if (rarely()) { - rangeSize = FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.get(Settings.EMPTY); + rangeSize = SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.get(Settings.EMPTY); } else if (randomBoolean()) { rangeSize = new ByteSizeValue( randomLongBetween(CacheService.MIN_SNAPSHOT_CACHE_RANGE_SIZE.getBytes(), ByteSizeValue.ofKb(8L).getBytes()) @@ -68,7 +69,7 @@ public void testRandomReads() throws IOException { final ByteSizeValue regionSize; if (rarely()) { - regionSize = FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(Settings.EMPTY); + regionSize = SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(Settings.EMPTY); } else if (randomBoolean()) { regionSize = new ByteSizeValue(randomLongBetween(ByteSizeValue.ofKb(1L).getBytes(), ByteSizeValue.ofKb(8L).getBytes())); } else { @@ -83,9 +84,9 @@ public void testRandomReads() throws IOException { } final Settings settings = Settings.builder() - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), regionSize) - .put(FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(), rangeSize) - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize) + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), regionSize) + .put(SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), rangeSize) + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize) .put("path.home", createTempDir()) .build(); final Environment environment = TestEnvironment.newEnvironment(settings); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java index 1b8c5d5c5dfd5..6044db3f31dab 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java @@ -33,6 +33,7 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; @@ -141,13 +142,13 @@ protected FrozenCacheService defaultFrozenCacheService() { protected FrozenCacheService randomFrozenCacheService() { final Settings.Builder cacheSettings = Settings.builder(); if (randomBoolean()) { - cacheSettings.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), randomFrozenCacheSize()); + cacheSettings.put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), randomFrozenCacheSize()); } if (randomBoolean()) { - cacheSettings.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), randomFrozenCacheSize()); + cacheSettings.put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), randomFrozenCacheSize()); } if (randomBoolean()) { - cacheSettings.put(FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(), randomCacheRangeSize()); + cacheSettings.put(SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), randomCacheRangeSize()); } if (randomBoolean()) { cacheSettings.put(FrozenCacheService.FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(), randomCacheRangeSize()); @@ -174,8 +175,8 @@ protected FrozenCacheService createFrozenCacheService(final ByteSizeValue cacheS return new FrozenCacheService( newEnvironment( Settings.builder() - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize) - .put(FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(), cacheRangeSize) + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize) + .put(SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), cacheRangeSize) .build() ), threadPool diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheServiceTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheServiceTests.java index 769b924a24238..bda1258f3bee2 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheServiceTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheServiceTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.cache.CacheKey; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.searchablesnapshots.cache.FrozenCacheService.CacheFileRegion; @@ -28,8 +29,8 @@ public class FrozenCacheServiceTests extends ESTestCase { public void testBasicEviction() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random()); @@ -74,8 +75,8 @@ public void testBasicEviction() throws IOException { public void testAutoEviction() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "200b") - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "200b") + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random()); @@ -111,8 +112,8 @@ public void testAutoEviction() throws IOException { public void testForceEviction() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random()); @@ -140,8 +141,8 @@ public void testForceEviction() throws IOException { public void testDecay() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random());