Skip to content

HDDS-8842. [Snapshot] Limit number of open files for each snapshot RocksDB instance #4884

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
prashantpogde merged 5 commits into from
Jun 21, 2023
Merged

HDDS-8842. [Snapshot] Limit number of open files for each snapshot RocksDB instance #4884

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ae95ec2
HDDS-8842. Limit number of open files for snapshot in Snapshot cache
swamirishi Jun 13, 2023
12ac936
Fix checkstyle
smengcl Jun 16, 2023
4116497
HDDS-8842. Address review comments
swamirishi Jun 17, 2023
5a58474
HDDS-8842. Address review comments
swamirishi Jun 17, 2023
dbaf98f
HDDS-8842: Address review comments
swamirishi Jun 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions hadoop-hdds/common/src/main/resources/ozone-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3894,6 +3894,19 @@
</description>
</property>

<property>
<name>ozone.om.snapshot.db.max.open.files</name>
<value>100</value>
<tag>OZONE, OM</tag>
<description>
Max number of open files for each snapshot db present in the snapshot cache.
Essentially sets `max_open_files` config for RocksDB instances opened for Ozone snapshots.
This will limit the total number of files opened by a snapshot db thereby limiting the total number of
open file handles by snapshot dbs.
Max total number of open handles = (snapshot cache size * max open files)
</description>
</property>

<property>
<name>ozone.om.snapshot.force.full.diff</name>
<value>false</value>
Expand Down
14 changes: 13 additions & 1 deletion hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ public final class DBStoreBuilder {
// this is to track the total size of dbUpdates data since sequence
// number in request to avoid increase in heap memory.
private long maxDbUpdatesSizeThreshold;
private Integer maxNumberOfOpenFiles = null;

/**
* Create DBStoreBuilder from a generic DBDefinition.
Expand Down Expand Up @@ -182,6 +183,12 @@ private void applyDBDefinition(DBDefinition definition) {
}
}

private void setDBOptionsProps(ManagedDBOptions dbOptions) {
if (maxNumberOfOpenFiles != null) {
dbOptions.setMaxOpenFiles(maxNumberOfOpenFiles);
}
}

/**
* Builds a DBStore instance and returns that.
*
Expand All @@ -200,7 +207,7 @@ public DBStore build() throws IOException {
if (rocksDBOption == null) {
rocksDBOption = getDefaultDBOptions(tableConfigs);
}

setDBOptionsProps(rocksDBOption);
ManagedWriteOptions writeOptions = new ManagedWriteOptions();
writeOptions.setSync(rocksDBConfiguration.getSyncOption());

Expand Down Expand Up @@ -292,6 +299,11 @@ public DBStoreBuilder setProfile(DBProfile prof) {
return this;
}

public DBStoreBuilder setMaxNumberOfOpenFiles(Integer maxNumberOfOpenFiles) {
this.maxNumberOfOpenFiles = maxNumberOfOpenFiles;
return this;
}

/**
* Converts column families and their corresponding options that have been
* registered with the builder to a set of {@link TableConfig} objects.
Expand Down
5 changes: 5 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,11 @@ private OMConfigKeys() {

public static final String OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE
= "ozone.om.snapshot.diff.max.page.size";

public static final String OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES
= "ozone.om.snapshot.db.max.open.files";
public static final int OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT
Copy link
Contributor

@hemantk-12 hemantk-12 Jun 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

QQ: Should we use -1 here just to make sure if it is not set -1 is used?

Copy link
Contributor Author

@swamirishi swamirishi Jun 21, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is better to have a proper limit? Instead of having no limit, otherwise there would be more failures & more tickets going forward for this.

= 100;
public static final int OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT
= 1000;

Expand Down
17 changes: 11 additions & 6 deletions ...p-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,14 +354,14 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
lock = new OmReadOnlyLock();
omEpoch = 0;
setStore(loadDB(conf, dir, name, true,
java.util.Optional.of(Boolean.TRUE)));
java.util.Optional.of(Boolean.TRUE), Optional.empty()));
initializeOmTables(false);
}


// metadata constructor for snapshots
OmMetadataManagerImpl(OzoneConfiguration conf, String snapshotDirName,
boolean isSnapshotInCache) throws IOException {
boolean isSnapshotInCache, int maxOpenFiles) throws IOException {
try {
lock = new OmReadOnlyLock();
omEpoch = 0;
Expand All @@ -380,7 +380,8 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
checkSnapshotDirExist(checkpoint);
}
setStore(loadDB(conf, metaDir, dbName, false,
java.util.Optional.of(Boolean.TRUE), false, false));
java.util.Optional.of(Boolean.TRUE),
Optional.of(maxOpenFiles), false, false));
initializeOmTables(false);
} catch (IOException e) {
stop();
Expand Down Expand Up @@ -525,22 +526,25 @@ public void start(OzoneConfiguration configuration) throws IOException {
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir)
throws IOException {
return loadDB(configuration, metaDir, OM_DB_NAME, false,
java.util.Optional.empty(), true, true);
java.util.Optional.empty(), Optional.empty(), true, true);
}

public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
String dbName, boolean readOnly,
java.util.Optional<Boolean>
disableAutoCompaction)
disableAutoCompaction,
java.util.Optional<Integer> maxOpenFiles)
throws IOException {
return loadDB(configuration, metaDir, dbName, readOnly,
disableAutoCompaction, true, true);
disableAutoCompaction, maxOpenFiles, true, true);
}

@SuppressWarnings("checkstyle:parameternumber")
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
String dbName, boolean readOnly,
java.util.Optional<Boolean>
disableAutoCompaction,
java.util.Optional<Integer> maxOpenFiles,
boolean enableCompactionDag,
boolean createCheckpointDirs)
throws IOException {
Expand All @@ -557,6 +561,7 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
.setCreateCheckpointDirs(createCheckpointDirs);
disableAutoCompaction.ifPresent(
dbStoreBuilder::disableDefaultCFAutoCompaction);
maxOpenFiles.ifPresent(dbStoreBuilder::setMaxNumberOfOpenFiles);
return addOMTablesAndCodecs(dbStoreBuilder).build();
}

Expand Down
11 changes: 9 additions & 2 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_CLEANUP_SERVICE_RUN_INTERVAL;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_CLEANUP_SERVICE_RUN_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_CLEANUP_SERVICE_TIMEOUT;
Expand Down Expand Up @@ -147,6 +149,7 @@ public final class OmSnapshotManager implements AutoCloseable {
"snap-diff-purged-job-table";

private final long diffCleanupServiceInterval;
private final int maxOpenSstFilesInSnapshotDb;
private final ManagedColumnFamilyOptions columnFamilyOptions;
private final ManagedDBOptions options;
private final List<ColumnFamilyDescriptor> columnFamilyDescriptors;
Expand Down Expand Up @@ -180,7 +183,10 @@ public OmSnapshotManager(OzoneManager ozoneManager) {
OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE,
OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT
);

this.maxOpenSstFilesInSnapshotDb = ozoneManager.getConfiguration().getInt(
OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES,
OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES_DEFAULT
);
ColumnFamilyHandle snapDiffJobCf;
ColumnFamilyHandle snapDiffReportCf;
ColumnFamilyHandle snapDiffPurgedJobCf;
Expand Down Expand Up @@ -317,7 +323,8 @@ public OmSnapshot load(@Nonnull String snapshotTableKey)
OMMetadataManager snapshotMetadataManager;
try {
snapshotMetadataManager = new OmMetadataManagerImpl(conf,
snapshotInfo.getCheckpointDirName(), isSnapshotInCache);
snapshotInfo.getCheckpointDirName(), isSnapshotInCache,
maxOpenSstFilesInSnapshotDb);
} catch (IOException e) {
LOG.error("Failed to retrieve snapshot: {}", snapshotTableKey);
throw e;
Expand Down
3 changes: 2 additions & 1 deletion hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,8 @@ public BackgroundTaskResult call() throws Exception {
try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl
.loadDB(ozoneManager.getConfiguration(),
new File(snapshotCheckpointDir),
dbName, true, Optional.of(Boolean.TRUE), false, false)) {
dbName, true, Optional.of(Boolean.TRUE),
Optional.empty(), false, false)) {
RocksDatabase db = rdbStore.getDb();
try (BootstrapStateHandler.Lock lock =
getBootstrapStateLock().lock()) {
Expand Down