HDDS-8039. Allow container inspector to run from ozone debug.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java

@@ -48,7 +48,7 @@
 /**
  * Class that manages Containers created on the datanode.
  */
-public class ContainerSet {
+public class ContainerSet implements Iterable<Container<?>> {
 
   private static final Logger LOG = LoggerFactory.getLogger(ContainerSet.class);
 
@@ -201,6 +201,11 @@ public void handleVolumeFailures() {
    * @return {@literal Iterator<Container<?>>}
    */
   public Iterator<Container<?>> getContainerIterator() {
+    return iterator();
+  }
+
+  @Override
+  public Iterator<Container<?>> iterator() {
     return containerMap.values().iterator();
   }
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java

@@ -102,6 +102,10 @@ public String toString() {
 
   private Mode mode;
 
+  public KeyValueContainerMetadataInspector(Mode mode) {
+    this.mode = mode;
+  }
+
   public KeyValueContainerMetadataInspector() {
     mode = Mode.OFF;
   }
@@ -155,10 +159,15 @@ public boolean isReadOnly() {
 
   @Override
   public void process(ContainerData containerData, DatanodeStore store) {
+    process(containerData, store, REPORT_LOG);
+  }
+
+  public String process(ContainerData containerData, DatanodeStore store,
+      Logger log) {
     // If the system property to process container metadata was not
     // specified, or the inspector is unloaded, this method is a no-op.
     if (mode == Mode.OFF) {
-      return;
+      return null;
     }
 
     KeyValueContainerData kvData = null;
@@ -167,7 +176,7 @@ public void process(ContainerData containerData, DatanodeStore store) {
     } else {
       LOG.error("This inspector only works on KeyValueContainers. Inspection " +
           "will not be run for container {}", containerData.getContainerID());
-      return;
+      return null;
     }
 
     JsonObject containerJson = inspectContainer(kvData, store);
@@ -178,14 +187,17 @@ public void process(ContainerData containerData, DatanodeStore store) {
         .serializeNulls()
         .create();
     String jsonReport = gson.toJson(containerJson);
-    if (correct) {
-      REPORT_LOG.trace(jsonReport);
-    } else {
-      REPORT_LOG.error(jsonReport);
+    if (log != null) {
+      if (correct) {
+        log.trace(jsonReport);
+      } else {
+        log.error(jsonReport);
+      }
     }
+    return jsonReport;
   }
 
-  private JsonObject inspectContainer(KeyValueContainerData containerData,
+  static JsonObject inspectContainer(KeyValueContainerData containerData,
       DatanodeStore store) {
 
     JsonObject containerJson = new JsonObject();
@@ -224,7 +236,7 @@ private JsonObject inspectContainer(KeyValueContainerData containerData,
     return containerJson;
   }
 
-  private JsonObject getDBMetadataJson(Table<String, Long> metadataTable,
+  static JsonObject getDBMetadataJson(Table<String, Long> metadataTable,
       KeyValueContainerData containerData) throws IOException {
     JsonObject dBMetadata = new JsonObject();
 
@@ -242,14 +254,13 @@ private JsonObject getDBMetadataJson(Table<String, Long> metadataTable,
     return dBMetadata;
   }
 
-  private JsonObject getAggregateValues(DatanodeStore store,
+  static JsonObject getAggregateValues(DatanodeStore store,
       KeyValueContainerData containerData, String schemaVersion)
       throws IOException {
     JsonObject aggregates = new JsonObject();
 
     long usedBytesTotal = 0;
     long blockCountTotal = 0;
-    long pendingDeleteBlockCountTotal = 0;
     // Count normal blocks.
     try (BlockIterator<BlockData> blockIter =
              store.getBlockIterator(containerData.getContainerID(),
@@ -262,26 +273,33 @@ private JsonObject getAggregateValues(DatanodeStore store,
     }
 
     // Count pending delete blocks.
+    final PendingDelete pendingDelete;
     if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) {
+      long pendingDeleteBlockCountTotal = 0;
+      long pendingDeleteBytes = 0;
       try (BlockIterator<BlockData> blockIter =
                store.getBlockIterator(containerData.getContainerID(),
                    containerData.getDeletingBlockKeyFilter())) {
 
         while (blockIter.hasNext()) {
           blockCountTotal++;
           pendingDeleteBlockCountTotal++;
-          usedBytesTotal += getBlockLength(blockIter.nextBlock());
+          final long bytes = getBlockLength(blockIter.nextBlock());
+          usedBytesTotal += bytes;
+          pendingDeleteBytes += bytes;
         }
       }
+      pendingDelete = new PendingDelete(
+          pendingDeleteBlockCountTotal, pendingDeleteBytes);
     } else if (schemaVersion.equals(OzoneConsts.SCHEMA_V2)) {
       DatanodeStoreSchemaTwoImpl schemaTwoStore =
           (DatanodeStoreSchemaTwoImpl) store;
-      pendingDeleteBlockCountTotal =
-          countPendingDeletesSchemaV2(schemaTwoStore);
+      pendingDelete =
+          countPendingDeletesSchemaV2(schemaTwoStore, containerData);
     } else if (schemaVersion.equals(OzoneConsts.SCHEMA_V3)) {
       DatanodeStoreSchemaThreeImpl schemaThreeStore =
           (DatanodeStoreSchemaThreeImpl) store;
-      pendingDeleteBlockCountTotal =
+      pendingDelete =
           countPendingDeletesSchemaV3(schemaThreeStore, containerData);
     } else {
       throw new IOException("Failed to process deleted blocks for unknown " +
@@ -290,13 +308,12 @@ private JsonObject getAggregateValues(DatanodeStore store,
 
     aggregates.addProperty("blockCount", blockCountTotal);
     aggregates.addProperty("usedBytes", usedBytesTotal);
-    aggregates.addProperty("pendingDeleteBlocks",
-        pendingDeleteBlockCountTotal);
+    pendingDelete.addToJson(aggregates);
 
     return aggregates;
   }
 
-  private JsonObject getChunksDirectoryJson(File chunksDir) throws IOException {
+  static JsonObject getChunksDirectoryJson(File chunksDir) throws IOException {
     JsonObject chunksDirectory = new JsonObject();
 
     chunksDirectory.addProperty("path", chunksDir.getAbsolutePath());
@@ -321,6 +338,9 @@ private boolean checkAndRepair(JsonObject parent,
 
     Table<String, Long> metadataTable = store.getMetadataTable();
 
+    final JsonObject dBMetadata = parent.getAsJsonObject("dBMetadata");
+    final JsonObject aggregates = parent.getAsJsonObject("aggregates");
+
     // Check and repair block count.
     JsonElement blockCountDB = parent.getAsJsonObject("dBMetadata")
         .get(OzoneConsts.BLOCK_COUNT);
@@ -392,6 +412,36 @@ private boolean checkAndRepair(JsonObject parent,
       errors.add(usedBytesError);
     }
 
+    // check and repair if db delete count mismatches delete transaction count.
+    final JsonElement pendingDeleteCountDB = dBMetadata.get(
+        OzoneConsts.PENDING_DELETE_BLOCK_COUNT);
+    final long dbDeleteCount = jsonToLong(pendingDeleteCountDB);
+    final JsonElement pendingDeleteCountAggregate
+        = aggregates.get(PendingDelete.COUNT);
+    final long deleteTransactionCount = jsonToLong(pendingDeleteCountAggregate);
+    if (dbDeleteCount != deleteTransactionCount) {
+      passed = false;
+
+      final BooleanSupplier deleteCountRepairAction = () -> {
+        final String key = containerData.getPendingDeleteBlockCountKey();
+        try {
+          // set delete block count metadata table to delete transaction count
+          metadataTable.put(key, deleteTransactionCount);
+          return true;
+        } catch (IOException ex) {
+          LOG.error("Failed to reset {} for container {}.",
+              key, containerData.getContainerID(), ex);
+        }
+        return false;
+      };
+
+      final JsonObject deleteCountError = buildErrorAndRepair(
+          "dBMetadata." + OzoneConsts.PENDING_DELETE_BLOCK_COUNT,
+          pendingDeleteCountAggregate, pendingDeleteCountDB,
+          deleteCountRepairAction);
+      errors.add(deleteCountError);
+    }
+
     // check and repair chunks dir.
     JsonElement chunksDirPresent = parent.getAsJsonObject("chunksDirectory")
         .get("present");
@@ -421,6 +471,10 @@ private boolean checkAndRepair(JsonObject parent,
     return passed;
   }
 
+  static long jsonToLong(JsonElement e) {
+    return e == null || e.isJsonNull() ? 0 : e.getAsLong();
+  }
+
   private JsonObject buildErrorAndRepair(String property, JsonElement expected,
       JsonElement actual, BooleanSupplier repairAction) {
     JsonObject error = new JsonObject();
@@ -437,41 +491,96 @@ private JsonObject buildErrorAndRepair(String property, JsonElement expected,
     return error;
   }
 
-  private long countPendingDeletesSchemaV2(DatanodeStoreSchemaTwoImpl
-      schemaTwoStore) throws IOException {
+  static class PendingDelete {
+    static final String COUNT = "pendingDeleteBlocks";
+    static final String BYTES = "pendingDeleteBytes";
+
+    private final long count;
+    private final long bytes;
+
+    PendingDelete(long count, long bytes) {
+      this.count = count;
+      this.bytes = bytes;
+    }
+
+    void addToJson(JsonObject json) {
+      json.addProperty(COUNT, count);
+      json.addProperty(BYTES, bytes);
+    }
+  }
+
+  static PendingDelete countPendingDeletesSchemaV2(
+      DatanodeStoreSchemaTwoImpl schemaTwoStore,
+      KeyValueContainerData containerData) throws IOException {
     long pendingDeleteBlockCountTotal = 0;
+    long pendingDeleteBytes = 0;
+
     Table<Long, DeletedBlocksTransaction> delTxTable =
         schemaTwoStore.getDeleteTransactionTable();
+    final Table<String, BlockData> blockDataTable
+        = schemaTwoStore.getBlockDataTable();
+
     try (TableIterator<Long, ? extends Table.KeyValue<Long,
         DeletedBlocksTransaction>> iterator = delTxTable.iterator()) {
       while (iterator.hasNext()) {
         DeletedBlocksTransaction txn = iterator.next().getValue();
+        final List<Long> localIDs = txn.getLocalIDList();
         // In schema 2, pending delete blocks are stored in the
         // transaction object. Since the actual blocks still exist in the
         // block data table with no prefix, they have already been
         // counted towards bytes used and total block count above.
-        pendingDeleteBlockCountTotal += txn.getLocalIDList().size();
+        pendingDeleteBlockCountTotal += localIDs.size();
+        pendingDeleteBytes += computePendingDeleteBytes(
+            localIDs, containerData, blockDataTable);
       }
     }
 
-    return pendingDeleteBlockCountTotal;
+    return new PendingDelete(pendingDeleteBlockCountTotal,
+        pendingDeleteBytes);
+  }
+
+  static long computePendingDeleteBytes(List<Long> localIDs,
+      KeyValueContainerData containerData,
+      Table<String, BlockData> blockDataTable) {
+    long pendingDeleteBytes = 0;
+    for (long id : localIDs) {
+      try {
+        final String blockKey = containerData.getBlockKey(id);
+        final BlockData blockData = blockDataTable.get(blockKey);
+        if (blockData != null) {
+          pendingDeleteBytes += blockData.getSize();
+        }
+      } catch (IOException e) {
+        LOG.error("Failed to get block " + id
+            + " in container " + containerData.getContainerID()
+            + " from blockDataTable", e);
+      }
+    }
+    return pendingDeleteBytes;
   }
 
-  private long countPendingDeletesSchemaV3(
+  static PendingDelete countPendingDeletesSchemaV3(
       DatanodeStoreSchemaThreeImpl schemaThreeStore,
       KeyValueContainerData containerData) throws IOException {
     long pendingDeleteBlockCountTotal = 0;
+    long pendingDeleteBytes = 0;
+    final Table<String, BlockData> blockDataTable
+        = schemaThreeStore.getBlockDataTable();
     try (
         TableIterator<String, ? extends Table.KeyValue<String,
             DeletedBlocksTransaction>>
             iter = schemaThreeStore.getDeleteTransactionTable()
             .iterator(containerData.containerPrefix())) {
       while (iter.hasNext()) {
         DeletedBlocksTransaction delTx = iter.next().getValue();
-        pendingDeleteBlockCountTotal += delTx.getLocalIDList().size();
+        final List<Long> localIDs = delTx.getLocalIDList();
+        pendingDeleteBlockCountTotal += localIDs.size();
+        pendingDeleteBytes += computePendingDeleteBytes(
+            localIDs, containerData, blockDataTable);
       }
-      return pendingDeleteBlockCountTotal;
     }
+    return new PendingDelete(pendingDeleteBlockCountTotal,
+        pendingDeleteBytes);
   }
 
   private static long getBlockLength(BlockData block) {

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerController.java

@@ -196,6 +196,10 @@ public Iterator<Container<?>> getContainers() {
     return containerSet.getContainerIterator();
   }
 
+  public Iterable<Container<?>> getContainerSet() {
+    return containerSet;
+  }
+
   /**
    * Return an iterator of containers which are associated with the specified
    * <code>volume</code>.

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java

@@ -65,6 +65,11 @@ public static Iterable<Object[]> versionParameters() {
         .collect(toList());
   }
 
+  @Override
+  public String toString() {
+    return "schema=" + schemaVersion + ", layout=" + layout;
+  }
+
   public static List<ContainerTestVersionInfo> getLayoutList() {
     return layoutList;
   }

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java

@@ -58,7 +58,7 @@
  */
 public class TestKeyValueContainerIntegrityChecks {
 
-  private static final Logger LOG =
+  static final Logger LOG =
       LoggerFactory.getLogger(TestKeyValueContainerIntegrityChecks.class);
 
   private final ContainerLayoutTestInfo containerLayoutTestInfo;
@@ -75,6 +75,7 @@ public class TestKeyValueContainerIntegrityChecks {
 
   public TestKeyValueContainerIntegrityChecks(
       ContainerTestVersionInfo versionInfo) {
+    LOG.info("new {} for {}", getClass().getSimpleName(), versionInfo);
     this.conf = new OzoneConfiguration();
     ContainerTestVersionInfo.setTestSchemaVersion(
         versionInfo.getSchemaVersion(), conf);