From ae95ec29967f0326b23920933c5262d856c1e465 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 13 Jun 2023 16:51:36 -0700 Subject: [PATCH 1/5] HDDS-8842. Limit number of open files for snapshot in Snapshot cache --- .../common/src/main/resources/ozone-default.xml | 12 ++++++++++++ .../hadoop/hdds/utils/db/DBStoreBuilder.java | 14 +++++++++++++- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 5 +++++ .../hadoop/ozone/om/OmMetadataManagerImpl.java | 15 +++++++++------ .../apache/hadoop/ozone/om/OmSnapshotManager.java | 11 +++++++++-- .../hadoop/ozone/om/SstFilteringService.java | 2 +- 6 files changed, 49 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 87be7a509a20..c3e254d0b1b5 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3894,6 +3894,18 @@ + + ozone.om.snapshot.db.max.open.files + 100 + OZONE, OM + + Max number of open files for each snapshot db present in the snapshot cache. + This will limit the total number of files opened by a snapshot db thereby limiting the total number of + open file handles by snapshot dbs. + Max total number of open handles = (snapshot cache size * max open files) + + + ozone.om.snapshot.force.full.diff false diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index bed044904e0e..88c78368d173 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -105,6 +105,7 @@ public final class DBStoreBuilder { // this is to track the total size of dbUpdates data since sequence // number in request to avoid increase in heap memory. private long maxDbUpdatesSizeThreshold; + private Integer maxNumberOfOpenFiles = null; /** * Create DBStoreBuilder from a generic DBDefinition. @@ -182,6 +183,17 @@ private void applyDBDefinition(DBDefinition definition) { } } + public DBStoreBuilder setMaxNumberOfOpenFiles(Integer maxNumberOfOpenFiles) { + this.maxNumberOfOpenFiles = maxNumberOfOpenFiles; + return this; + } + + private void setDBOptionsProps(ManagedDBOptions dbOptions) { + if (maxNumberOfOpenFiles != null) { + rocksDBOption.setMaxOpenFiles(maxNumberOfOpenFiles); + } + } + /** * Builds a DBStore instance and returns that. * @@ -200,7 +212,7 @@ public DBStore build() throws IOException { if (rocksDBOption == null) { rocksDBOption = getDefaultDBOptions(tableConfigs); } - + setDBOptions(rocksDBOption); ManagedWriteOptions writeOptions = new ManagedWriteOptions(); writeOptions.setSync(rocksDBConfiguration.getSyncOption()); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 9a7acb02f573..1f8d12be190c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -494,6 +494,11 @@ private OMConfigKeys() { public static final String OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE = "ozone.om.snapshot.diff.max.page.size"; + + public static final String OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES + = "ozone.om.snapshot.db.max.open.files"; + public static final int OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT + = 100; public static final int OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT = 1000; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index f6519bb098ec..cbe1cdea0e7e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -354,14 +354,14 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) lock = new OmReadOnlyLock(); omEpoch = 0; setStore(loadDB(conf, dir, name, true, - java.util.Optional.of(Boolean.TRUE))); + java.util.Optional.of(Boolean.TRUE), Optional.empty())); initializeOmTables(false); } // metadata constructor for snapshots OmMetadataManagerImpl(OzoneConfiguration conf, String snapshotDirName, - boolean isSnapshotInCache) throws IOException { + boolean isSnapshotInCache, int maxOpenFiles) throws IOException { try { lock = new OmReadOnlyLock(); omEpoch = 0; @@ -380,7 +380,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) checkSnapshotDirExist(checkpoint); } setStore(loadDB(conf, metaDir, dbName, false, - java.util.Optional.of(Boolean.TRUE), false, false)); + java.util.Optional.of(Boolean.TRUE), Optional.of(maxOpenFiles), false, false)); initializeOmTables(false); } catch (IOException e) { stop(); @@ -525,22 +525,24 @@ public void start(OzoneConfiguration configuration) throws IOException { public static DBStore loadDB(OzoneConfiguration configuration, File metaDir) throws IOException { return loadDB(configuration, metaDir, OM_DB_NAME, false, - java.util.Optional.empty(), true, true); + java.util.Optional.empty(), Optional.empty(), true, true); } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly, java.util.Optional - disableAutoCompaction) + disableAutoCompaction, + java.util.Optional maxOpenFiles) throws IOException { return loadDB(configuration, metaDir, dbName, readOnly, - disableAutoCompaction, true, true); + disableAutoCompaction, maxOpenFiles, true, true); } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly, java.util.Optional disableAutoCompaction, + java.util.Optional maxOpenFiles, boolean enableCompactionDag, boolean createCheckpointDirs) throws IOException { @@ -557,6 +559,7 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, .setCreateCheckpointDirs(createCheckpointDirs); disableAutoCompaction.ifPresent( dbStoreBuilder::disableDefaultCFAutoCompaction); + maxOpenFiles.ifPresent(dbStoreBuilder::setMaxNumberOfOpenFiles); return addOMTablesAndCodecs(dbStoreBuilder).build(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index cdd39ead3446..c8b99cfdc18f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -80,6 +80,8 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_CLEANUP_SERVICE_RUN_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_CLEANUP_SERVICE_RUN_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_CLEANUP_SERVICE_TIMEOUT; @@ -147,6 +149,7 @@ public final class OmSnapshotManager implements AutoCloseable { "snap-diff-purged-job-table"; private final long diffCleanupServiceInterval; + private final int maxOpenFilesSnapshotDB; private final ManagedColumnFamilyOptions columnFamilyOptions; private final ManagedDBOptions options; private final List columnFamilyDescriptors; @@ -180,7 +183,10 @@ public OmSnapshotManager(OzoneManager ozoneManager) { OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE, OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT ); - + this.maxOpenFilesSnapshotDB = ozoneManager.getConfiguration().getInt( + OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES, + OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT + ); ColumnFamilyHandle snapDiffJobCf; ColumnFamilyHandle snapDiffReportCf; ColumnFamilyHandle snapDiffPurgedJobCf; @@ -317,7 +323,8 @@ public OmSnapshot load(@Nonnull String snapshotTableKey) OMMetadataManager snapshotMetadataManager; try { snapshotMetadataManager = new OmMetadataManagerImpl(conf, - snapshotInfo.getCheckpointDirName(), isSnapshotInCache); + snapshotInfo.getCheckpointDirName(), isSnapshotInCache, + maxOpenFilesSnapshotDB); } catch (IOException e) { LOG.error("Failed to retrieve snapshot: {}", snapshotTableKey); throw e; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index e1190fce00e8..4b4518803770 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -158,7 +158,7 @@ public BackgroundTaskResult call() throws Exception { try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), - dbName, true, Optional.of(Boolean.TRUE), false, false)) { + dbName, true, Optional.of(Boolean.TRUE), Optional.empty(), false, false)) { RocksDatabase db = rdbStore.getDb(); try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { From 12ac936953451c939c934b9c105ad19b97dd9378 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Thu, 15 Jun 2023 23:08:31 -0700 Subject: [PATCH 2/5] Fix checkstyle --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +++- .../java/org/apache/hadoop/ozone/om/SstFilteringService.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index cbe1cdea0e7e..09b48a0bec8e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -380,7 +380,8 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) checkSnapshotDirExist(checkpoint); } setStore(loadDB(conf, metaDir, dbName, false, - java.util.Optional.of(Boolean.TRUE), Optional.of(maxOpenFiles), false, false)); + java.util.Optional.of(Boolean.TRUE), + Optional.of(maxOpenFiles), false, false)); initializeOmTables(false); } catch (IOException e) { stop(); @@ -538,6 +539,7 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, disableAutoCompaction, maxOpenFiles, true, true); } + @SuppressWarnings("checkstyle:parameternumber") public static DBStore loadDB(OzoneConfiguration configuration, File metaDir, String dbName, boolean readOnly, java.util.Optional diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java index 4b4518803770..2ad80844161c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java @@ -158,7 +158,8 @@ public BackgroundTaskResult call() throws Exception { try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl .loadDB(ozoneManager.getConfiguration(), new File(snapshotCheckpointDir), - dbName, true, Optional.of(Boolean.TRUE), Optional.empty(), false, false)) { + dbName, true, Optional.of(Boolean.TRUE), + Optional.empty(), false, false)) { RocksDatabase db = rdbStore.getDb(); try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { From 4116497205d578ad7719be4d56f4783ff07f2fcd Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sat, 17 Jun 2023 12:04:46 -0700 Subject: [PATCH 3/5] HDDS-8842. Address review comments --- .../common/src/main/resources/ozone-default.xml | 1 + .../apache/hadoop/hdds/utils/db/DBStoreBuilder.java | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index c3e254d0b1b5..fe64ea184f97 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3900,6 +3900,7 @@ OZONE, OM Max number of open files for each snapshot db present in the snapshot cache. + Essentially sets `max_open_files` config for RocksDB instances opened for Ozone snapshots. This will limit the total number of files opened by a snapshot db thereby limiting the total number of open file handles by snapshot dbs. Max total number of open handles = (snapshot cache size * max open files) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index 88c78368d173..6287b06f2998 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -183,11 +183,6 @@ private void applyDBDefinition(DBDefinition definition) { } } - public DBStoreBuilder setMaxNumberOfOpenFiles(Integer maxNumberOfOpenFiles) { - this.maxNumberOfOpenFiles = maxNumberOfOpenFiles; - return this; - } - private void setDBOptionsProps(ManagedDBOptions dbOptions) { if (maxNumberOfOpenFiles != null) { rocksDBOption.setMaxOpenFiles(maxNumberOfOpenFiles); @@ -212,7 +207,7 @@ public DBStore build() throws IOException { if (rocksDBOption == null) { rocksDBOption = getDefaultDBOptions(tableConfigs); } - setDBOptions(rocksDBOption); + setDBOptionsProps(rocksDBOption); ManagedWriteOptions writeOptions = new ManagedWriteOptions(); writeOptions.setSync(rocksDBConfiguration.getSyncOption()); @@ -304,6 +299,11 @@ public DBStoreBuilder setProfile(DBProfile prof) { return this; } + public DBStoreBuilder setMaxNumberOfOpenFiles(Integer maxNumberOfOpenFiles) { + this.maxNumberOfOpenFiles = maxNumberOfOpenFiles; + return this; + } + /** * Converts column families and their corresponding options that have been * registered with the builder to a set of {@link TableConfig} objects. From 5a58474cb98689e8af2b779f36e773b261639bbe Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sat, 17 Jun 2023 12:09:46 -0700 Subject: [PATCH 4/5] HDDS-8842. Address review comments --- .../java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java index 6287b06f2998..7e7bdb0eccaa 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java @@ -185,7 +185,7 @@ private void applyDBDefinition(DBDefinition definition) { private void setDBOptionsProps(ManagedDBOptions dbOptions) { if (maxNumberOfOpenFiles != null) { - rocksDBOption.setMaxOpenFiles(maxNumberOfOpenFiles); + dbOptions.setMaxOpenFiles(maxNumberOfOpenFiles); } } From dbaf98f43eb71169bedcfa4aaa18f3ba3c475bef Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 20 Jun 2023 17:17:44 -0700 Subject: [PATCH 5/5] HDDS-8842: Address review comments --- .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index c8b99cfdc18f..ca5637e82bc3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -149,7 +149,7 @@ public final class OmSnapshotManager implements AutoCloseable { "snap-diff-purged-job-table"; private final long diffCleanupServiceInterval; - private final int maxOpenFilesSnapshotDB; + private final int maxOpenSstFilesInSnapshotDb; private final ManagedColumnFamilyOptions columnFamilyOptions; private final ManagedDBOptions options; private final List columnFamilyDescriptors; @@ -183,7 +183,7 @@ public OmSnapshotManager(OzoneManager ozoneManager) { OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE, OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT ); - this.maxOpenFilesSnapshotDB = ozoneManager.getConfiguration().getInt( + this.maxOpenSstFilesInSnapshotDb = ozoneManager.getConfiguration().getInt( OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES, OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT ); @@ -324,7 +324,7 @@ public OmSnapshot load(@Nonnull String snapshotTableKey) try { snapshotMetadataManager = new OmMetadataManagerImpl(conf, snapshotInfo.getCheckpointDirName(), isSnapshotInCache, - maxOpenFilesSnapshotDB); + maxOpenSstFilesInSnapshotDb); } catch (IOException e) { LOG.error("Failed to retrieve snapshot: {}", snapshotTableKey); throw e;