HDDS-8940. SST files are missing on optimized snapDiff path.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -606,4 +606,9 @@ private OzoneConsts() {
    */
   public static final String SNAPSHOT_INFO_TABLE = "snapshotInfoTable";
 
+  /**
+   * DB compaction log table name. Referenced in RDBStore.
+   */
+  public static final String COMPACTION_LOG_TABLE =
+      "compactionLogTable";
 }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java

@@ -49,6 +49,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.ozone.OzoneConsts.COMPACTION_LOG_TABLE;
 import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR;
 import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR;
@@ -154,6 +155,14 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions,
         // sequence number as current compaction log filename.
         rocksDBCheckpointDiffer.setCurrentCompactionLog(
             db.getLatestSequenceNumber());
+
+        ColumnFamily compactionLogTableTableCF =
+            db.getColumnFamily(COMPACTION_LOG_TABLE);
+        Preconditions.checkNotNull(compactionLogTableTableCF,
+            "CompactionLogTable column family handle should not be null");
+        rocksDBCheckpointDiffer.setCompactionLogTableCFHandle(
+            compactionLogTableTableCF.getHandle());
+        rocksDBCheckpointDiffer.setActiveRocksDB(db.getManagedRocksDb().get());
         // Load all previous compaction logs
         rocksDBCheckpointDiffer.loadAllCompactionLogs();
       }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java

@@ -18,9 +18,7 @@
 package org.apache.hadoop.hdds.utils.db;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hdds.StringUtils;
-import org.apache.hadoop.hdds.utils.BooleanTriFunction;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedCheckpoint;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions;
@@ -35,6 +33,7 @@
 import org.apache.hadoop.hdds.utils.db.managed.ManagedTransactionLogIterator;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions;
+import org.apache.ozone.rocksdiff.RocksDiffUtils;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.Holder;
@@ -48,7 +47,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -65,6 +63,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.hadoop.hdds.StringUtils.bytes2String;
 import static org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions.closeDeeply;
 import static org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator.managed;
@@ -611,7 +610,7 @@ private ColumnFamilyHandle getColumnFamilyHandle(String cfName)
     assertClose();
     for (ColumnFamilyHandle cf : getCfHandleMap().get(db.get().getName())) {
       try {
-        String table = new String(cf.getName(), StandardCharsets.UTF_8);
+        String table = new String(cf.getName(), UTF_8);
         if (cfName.equals(table)) {
           return cf;
         }
@@ -949,46 +948,46 @@ private int getLastLevel() throws IOException {
   /**
    * Deletes sst files which do not correspond to prefix
    * for given table.
-   * @param prefixPairs, a list of pair (TableName,prefixUsed).
+   * @param prefixPairs, a map of TableName to prefixUsed.
    */
-  public void deleteFilesNotMatchingPrefix(
-      List<Pair<String, String>> prefixPairs,
-      BooleanTriFunction<String, String, String, Boolean> filterFunction)
+  public void deleteFilesNotMatchingPrefix(Map<String, String> prefixPairs)
       throws IOException, RocksDBException {
     assertClose();
     for (LiveFileMetaData liveFileMetaData : getSstFileList()) {
       String sstFileColumnFamily =
           new String(liveFileMetaData.columnFamilyName(),
-              StandardCharsets.UTF_8);
+              UTF_8);
       int lastLevel = getLastLevel();
-      for (Pair<String, String> prefixPair : prefixPairs) {
-        String columnFamily = prefixPair.getKey();
-        String prefixForColumnFamily = prefixPair.getValue();
-        if (!sstFileColumnFamily.equals(columnFamily)) {
-          continue;
-        }
-        // RocksDB #deleteFile API allows only to delete the last level of
-        // SST Files. Any level < last level won't get deleted and
-        // only last file of level 0 can be deleted
-        // and will throw warning in the rocksdb manifest.
-        // Instead, perform the level check here
-        // itself to avoid failed delete attempts for lower level files.
-        if (liveFileMetaData.level() != lastLevel || lastLevel == 0) {
-          continue;
-        }
-        String firstDbKey =
-            new String(liveFileMetaData.smallestKey(), StandardCharsets.UTF_8);
-        String lastDbKey =
-            new String(liveFileMetaData.largestKey(), StandardCharsets.UTF_8);
-        boolean isKeyWithPrefixPresent =
-            filterFunction.apply(firstDbKey, lastDbKey, prefixForColumnFamily);
-        if (!isKeyWithPrefixPresent) {
-          LOG.info("Deleting sst file {} corresponding to column family"
-                  + " {} from db: {}", liveFileMetaData.fileName(),
-              StringUtils.bytes2String(liveFileMetaData.columnFamilyName()),
-              db.get().getName());
-          db.deleteFile(liveFileMetaData);
-        }
+
+      if (!prefixPairs.containsKey(sstFileColumnFamily)) {
+        continue;
+      }
+
+      // RocksDB #deleteFile API allows only to delete the last level of
+      // SST Files. Any level < last level won't get deleted and
+      // only last file of level 0 can be deleted
+      // and will throw warning in the rocksdb manifest.
+      // Instead, perform the level check here
+      // itself to avoid failed delete attempts for lower level files.
+      if (liveFileMetaData.level() != lastLevel || lastLevel == 0) {
+        continue;
+      }
+
+      String prefixForColumnFamily = prefixPairs.get(sstFileColumnFamily);
+      String firstDbKey = new String(liveFileMetaData.smallestKey(), UTF_8);
+      String lastDbKey = new String(liveFileMetaData.largestKey(), UTF_8);
+      boolean isKeyWithPrefixPresent = RocksDiffUtils.isKeyWithPrefixPresent(
+          prefixForColumnFamily, firstDbKey, lastDbKey);
+      if (!isKeyWithPrefixPresent) {
+        LOG.info("Deleting sst file: {} with start key: {} and end key: {} " +
+                "corresponding to column family {} from db: {}. " +
+                "Prefix for the column family: {}.",
+            liveFileMetaData.fileName(),
+            firstDbKey, lastDbKey,
+            StringUtils.bytes2String(liveFileMetaData.columnFamilyName()),
+            db.get().getName(),
+            prefixForColumnFamily);
+        db.deleteFile(liveFileMetaData);
       }
     }
   }

hadoop-hdds/interface-client/src/main/proto/hdds.proto

@@ -479,4 +479,19 @@ message DeletedBlocksTransactionInfo {
     optional int64 containerID = 2;
     repeated int64 localID = 3;
     optional int32 count = 4;
-}
+}
+
+message CompactionFileInfoProto {
+    required string fileName = 1;
+    optional string startKey = 2;
+    optional string endKey = 3;
+    optional string columnFamily = 4;
+}
+
+message CompactionLogEntryProto {
+    required uint64 dbSequenceNumber = 1;
+    required uint64 compactionTime = 2;
+    repeated CompactionFileInfoProto inputFileIntoList = 3;
+    repeated CompactionFileInfoProto outputFileIntoList = 4;
+    optional string compactionReason = 5;
+}

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozone.compaction.log;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+
+/**
+ * Dao to keep SST file information in the compaction log.
+ */
+public class CompactionFileInfo {
+  private final String fileName;
+  private final String startKey;
+  private final String endKey;
+  private final String columnFamily;
+
+  public CompactionFileInfo(String fileName,
+                            String startRange,
+                            String endRange,
+                            String columnFamily) {
+    this.fileName = fileName;
+    this.startKey = startRange;
+    this.endKey = endRange;
+    this.columnFamily = columnFamily;
+  }
+
+  public String getFileName() {
+    return fileName;
+  }
+
+  public String getStartKey() {
+    return startKey;
+  }
+
+  public String getEndKey() {
+    return endKey;
+  }
+
+  public String getColumnFamily() {
+    return columnFamily;
+  }
+
+  public HddsProtos.CompactionFileInfoProto getProtobuf() {
+    HddsProtos.CompactionFileInfoProto.Builder builder =
+        HddsProtos.CompactionFileInfoProto.newBuilder()
+            .setFileName(fileName);
+    if (startKey != null) {
+      builder = builder.setStartKey(startKey);
+    }
+    if (endKey != null) {
+      builder = builder.setStartKey(endKey);
+    }
+    if (columnFamily != null) {
+      builder = builder.setStartKey(columnFamily);
+    }
+    return builder.build();
+  }
+
+  public static CompactionFileInfo getFromProtobuf(
+      HddsProtos.CompactionFileInfoProto proto) {
+    return new CompactionFileInfo(proto.getFileName(), proto.getStartKey(),
+        proto.getEndKey(), proto.getColumnFamily());
+  }
+
+  @Override
+  public String toString() {
+    return String.format("fileName: '%s', startKey: '%s', endKey: '%s'," +
+        " columnFamily: '%s'", fileName, startKey, endKey, columnFamily);
+  }
+}