HDDS-15466. Make RocksDB bottommost level compaction options configurable for CLI tools

hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java

@@ -62,6 +62,13 @@ public class RocksDBManualCompaction extends RepairTool {
       description = "Column family name")
   private String columnFamilyName;
 
+  @CommandLine.Option(names = {"--bottommost-level-compaction", "--blc"},
+      description = "BottommostLevelCompaction option for RocksDB compaction." +
+          " Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce)." +
+          " Default: 0 (kSkip).",
+      defaultValue = "0")
+  private int bottommostLevelCompaction;
+
   private String getConsoleReadLineWithFormat() {
     err().printf(WARNING_TO_STOP_SERVICE);
     return getScanner().nextLine().trim();
@@ -96,11 +103,14 @@ public void execute() throws Exception {
             " is not in a column family in DB for the given path.");
       }
 
-      info("Running compaction on " + columnFamilyName);
+      ManagedCompactRangeOptions.BottommostLevelCompaction blcOption =
+          getBottommostLevelCompaction(bottommostLevelCompaction);
+      info("Running compaction on " + columnFamilyName +
+          " with BottommostLevelCompaction=" + blcOption.name());
       long startTime = Time.monotonicNow();
       if (!isDryRun()) {
         ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions();
-        compactOptions.setBottommostLevelCompaction(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce);
+        compactOptions.setBottommostLevelCompaction(blcOption);
         db.get().compactRange(cfh, null, null, compactOptions);
       }
       long duration = Time.monotonicNow() - startTime;
@@ -117,4 +127,15 @@ public void execute() throws Exception {
       IOUtils.closeQuietly(cfHandleList);
     }
   }
+
+  /**
+   * Converts the given rocksId to a
+   * {@link ManagedCompactRangeOptions.BottommostLevelCompaction} enum.
+   * Defaults to kSkip if the value is invalid.
+   */
+  static ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction(int rocksId) {
+    ManagedCompactRangeOptions.BottommostLevelCompaction blc =
+        ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(rocksId);
+    return blc != null ? blc : ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip;
+  }
 }

hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java

@@ -20,6 +20,7 @@
 import java.io.IOException;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions;
 import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
 import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolClientSideImpl;
 import org.apache.hadoop.ozone.repair.RepairTool;
@@ -58,22 +59,43 @@ public class CompactOMDB extends RepairTool {
   )
   private String nodeId;
 
+  @CommandLine.Option(names = {"--bottommost-level-compaction", "--blc"},
+      description = "BottommostLevelCompaction option for RocksDB compaction." +
+          " Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce)." +
+          " Default: 0 (kSkip).",
+      defaultValue = "0")
+  private int bottommostLevelCompaction;
+
   @Override
   public void execute() throws Exception {
 
     OzoneConfiguration conf = getOzoneConf();
     OMNodeDetails omNodeDetails = OMNodeDetails.getOMNodeDetailsFromConf(
         conf, omServiceId, nodeId);
+    ManagedCompactRangeOptions.BottommostLevelCompaction blcOption =
+        getBottommostLevelCompaction(bottommostLevelCompaction);
     if (!isDryRun()) {
       try (OMAdminProtocolClientSideImpl omAdminProtocolClient =
                OMAdminProtocolClientSideImpl.createProxyForSingleOM(conf,
                    UserGroupInformation.getCurrentUser(), omNodeDetails)) {
-        omAdminProtocolClient.compactOMDB(columnFamilyName);
-        info("Compaction request issued for om.db of om node: %s, column-family: %s.", nodeId, columnFamilyName);
+        omAdminProtocolClient.compactOMDB(columnFamilyName, blcOption.getValue());
+        info("Compaction request issued for om.db of om node: %s, column-family: %s," +
+            " BottommostLevelCompaction=%s.", nodeId, columnFamilyName, blcOption.name());
         info("Please check role logs of %s for completion status.", nodeId);
       } catch (IOException ex) {
         error("Couldn't compact column %s. \nException: %s", columnFamilyName, ex);
       }
     }
   }
+
+  /**
+   * Converts the given rocksId to a
+   * {@link ManagedCompactRangeOptions.BottommostLevelCompaction} enum.
+   * Defaults to kSkip if the value is invalid.
+   */
+  static ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction(int rocksId) {
+    ManagedCompactRangeOptions.BottommostLevelCompaction blc =
+        ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(rocksId);
+    return blc != null ? blc : ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip;
+  }
 }

hadoop-ozone/cli-repair/src/test/java/org/apache/hadoop/ozone/repair/ldb/TestLdbRepair.java

@@ -125,7 +125,8 @@ public void testRocksDBManualCompaction() throws Exception {
     CommandLine cmd = new CommandLine(compactionTool);
     String[] args = {
         "--db", dbPath.toString(),
-        "--column-family", TEST_CF_NAME
+        "--column-family", TEST_CF_NAME,
+        "--blc", "2"
     };
     // Pass two "y" inputs - one for user confirmation and the other for warning to stop service
     int exitCode = withTextFromSystemIn("y", "y")

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java

@@ -46,6 +46,18 @@ public interface OMAdminProtocol extends Closeable {
    */
   void compactOMDB(String columnFamily) throws IOException;
 
+  /**
+   * Requests compaction of a column family of om.db with the specified
+   * BottommostLevelCompaction option.
+   *
+   * @param columnFamily              column family name
+   * @param bottommostLevelCompaction rocksId of BottommostLevelCompaction
+   *                                  (0=kSkip, 1=kIfHaveCompactionFilter, 2=kForce)
+   */
+  default void compactOMDB(String columnFamily, int bottommostLevelCompaction) throws IOException {
+    compactOMDB(columnFamily);
+  }
+
   /**
    * Triggers the Snapshot Defragmentation Service to run immediately.
    * @param noWait if true, return immediately without waiting for completion

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java

@@ -217,8 +217,14 @@ public void decommission(OMNodeDetails removeOMNode) throws IOException {
 
   @Override
   public void compactOMDB(String columnFamily) throws IOException {
+    compactOMDB(columnFamily, 0);
+  }
+
+  @Override
+  public void compactOMDB(String columnFamily, int bottommostLevelCompaction) throws IOException {
     CompactRequest compactRequest = CompactRequest.newBuilder()
         .setColumnFamily(columnFamily)
+        .setBottommostLevelCompaction(bottommostLevelCompaction)
         .build();
     CompactResponse response;
     try {

hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto

@@ -72,6 +72,9 @@ message DecommissionOMResponse {
 
 message CompactRequest {
     required string columnFamily = 1;
+  // BottommostLevelCompaction option: 0=kSkip, 1=kIfHaveCompactionFilter, 2=kForce.
+  // Defaults to kSkip (0) if not set.
+  optional int32 bottommostLevelCompaction = 2 [default = 0];
 }
 
 message CompactResponse {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -5643,9 +5643,13 @@ public void checkFeatureEnabled(OzoneManagerVersion feature) throws OMException
   }
 
   public void compactOMDB(String columnFamily) throws IOException {
+    compactOMDB(columnFamily, 0);
+  }
+
+  public void compactOMDB(String columnFamily, int bottommostLevelCompaction) throws IOException {
     checkAdminUserPrivilege("compact column family " + columnFamily);
     CompletableFuture<Void> compactFuture =
-        CompactDBUtil.compactTableAsync(metadataManager, columnFamily);
+        CompactDBUtil.compactTableAsync(metadataManager, columnFamily, bottommostLevelCompaction);
     compactFuture.whenComplete((result, throwable) -> {
       if (throwable == null) {
         LOG.info("Compaction request for column family \"{}\" completed successfully.",

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java

@@ -40,11 +40,19 @@ private CompactDBUtil() {
 
   public static void compactTable(OMMetadataManager omMetadataManager,
                                   String tableName) throws IOException {
+    compactTable(omMetadataManager, tableName, 0);
+  }
+
+  public static void compactTable(OMMetadataManager omMetadataManager,
+      String tableName,
+      int bottommostLevelCompaction) throws IOException {
     long startTime = Time.monotonicNow();
-    LOG.info("Compacting column family: {}", tableName);
+    ManagedCompactRangeOptions.BottommostLevelCompaction blcOption =
+        getBottommostLevelCompaction(bottommostLevelCompaction);
+    LOG.info("Compacting column family: {} with BottommostLevelCompaction={}",
+        tableName, blcOption.name());
     try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) {
-      options.setBottommostLevelCompaction(
-          ManagedCompactRangeOptions.BottommostLevelCompaction.kForce);
+      options.setBottommostLevelCompaction(blcOption);
       options.setExclusiveManualCompaction(true);
       RocksDatabase rocksDatabase =
           ((RDBStore) omMetadataManager.getStore()).getDb();
@@ -63,13 +71,33 @@ public static void compactTable(OMMetadataManager omMetadataManager,
   }
 
   public static CompletableFuture<Void> compactTableAsync(OMMetadataManager metadataManager, String tableName) {
+    return compactTableAsync(metadataManager, tableName, 0);
+  }
+
+  public static CompletableFuture<Void> compactTableAsync(
+      OMMetadataManager metadataManager, String tableName, int bottommostLevelCompaction) {
     return CompletableFuture.runAsync(() -> {
       try {
-        compactTable(metadataManager, tableName);
+        compactTable(metadataManager, tableName, bottommostLevelCompaction);
       } catch (Exception e) {
         LOG.warn("Failed to compact column family: {}", tableName, e);
         throw new CompletionException("Compaction failed for column family: " + tableName, e);
       }
     });
   }
+
+  /**
+   * Converts the given rocksId to a
+   * {@link ManagedCompactRangeOptions.BottommostLevelCompaction} enum.
+   * Defaults to kSkip if the value is invalid.
+   */
+  static ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction(int rocksId) {
+    ManagedCompactRangeOptions.BottommostLevelCompaction blc =
+        ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(rocksId);
+    if (blc == null) {
+      LOG.warn("Invalid BottommostLevelCompaction value: {}. Defaulting to kSkip.", rocksId);
+      return ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip;
+    }
+    return blc;
+  }
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java

@@ -121,7 +121,7 @@ public CompactResponse compactDB(RpcController controller, CompactRequest compac
     try {
       // check if table exists. IOException is thrown if table is not found.
       ozoneManager.getMetadataManager().getStore().getTable(compactRequest.getColumnFamily());
-      ozoneManager.compactOMDB(compactRequest.getColumnFamily());
+      ozoneManager.compactOMDB(compactRequest.getColumnFamily(), compactRequest.getBottommostLevelCompaction());
     } catch (IOException ex) {
       return CompactResponse.newBuilder()
           .setSuccess(false)