Added Indication of Intermediate Compactions to CompactionConfigurer

core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionConfigurer.java

@@ -20,6 +20,7 @@
 
 import java.util.Collection;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.accumulo.core.client.PluginEnvironment;
 import org.apache.accumulo.core.data.TableId;
@@ -59,6 +60,28 @@ public interface InputParameters {
 
     public Collection<CompactableFile> getInputFiles();
 
+    /**
+     * For user and selector compactions:
+     * <ul>
+     * <li>Returns the selected set of files to be compacted.</li>
+     * <li>When getInputFiles() (inputFiles) and getSelectedFiles() (selectedFiles) are equal, then
+     * this is the final compaction.</li>
+     * <li>When they are not equal, this is an intermediate compaction.</li>
+     * <li>Intermediate compactions are compactions whose resultant RFile will be consumed by
+     * another compaction.</li>
+     * <li>inputFiles and selectedFiles can be compared using: <code>
+     * selectedFiles.equals(inputFiles instanceof Set ? inputFiles : Set.copyOf(inputFiles))
+     * </code></li>
+     * </ul>
+     * For system compactions:
+     * <ul>
+     * <li>There is no selected set of files so the empty set is returned.</li>
+     * </ul>
+     *
+     * @since 3.1
+     */
+    public Set<CompactableFile> getSelectedFiles();
+
     PluginEnvironment getEnvironment();
   }
 

core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintBCInfo.java

@@ -47,8 +47,7 @@ public class PrintBCInfo {
   CryptoService cryptoService = NoCryptoServiceFactory.NONE;
 
   public void printMetaBlockInfo() throws IOException {
-    FSDataInputStream fsin = fs.open(path);
-    try (BCFile.Reader bcfr =
+    try (FSDataInputStream fsin = fs.open(path); BCFile.Reader bcfr =
         new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf, cryptoService)) {
 
       Set<Entry<String,MetaIndexEntry>> es = bcfr.metaIndex.index.entrySet();
@@ -67,6 +66,17 @@ public void printMetaBlockInfo() throws IOException {
     }
   }
 
+  public String getCompressionType() throws IOException {
+    try (FSDataInputStream fsin = fs.open(path); BCFile.Reader bcfr =
+        new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf, cryptoService)) {
+
+      Set<Entry<String,MetaIndexEntry>> es = bcfr.metaIndex.index.entrySet();
+
+      return es.stream().filter(entry -> entry.getKey().equals("RFile.index")).findFirst()
+          .map(entry -> entry.getValue().getCompressionAlgorithm().getName()).orElse(null);
+    }
+  }
+
   static class Opts extends ConfigOpts {
     @Parameter(description = " <file>")
     String file;

core/src/test/java/org/apache/accumulo/core/compaction/ShellCompactCommandConfigurerTest.java

@@ -88,6 +88,11 @@ public Collection<CompactableFile> getInputFiles() {
         return files;
       }
 
+      @Override
+      public Set<CompactableFile> getSelectedFiles() {
+        throw new UnsupportedOperationException();
+      }
+
       @Override
       public PluginEnvironment getEnvironment() {
         return null;

server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java

@@ -134,7 +134,8 @@ enum FileSelectionStatus {
   public interface CompactionHelper {
     Set<StoredTabletFile> selectFiles(SortedMap<StoredTabletFile,DataFileValue> allFiles);
 
-    Map<String,String> getConfigOverrides(Set<CompactableFile> files);
+    Map<String,String> getConfigOverrides(Set<CompactableFile> inputFiles,
+        Set<StoredTabletFile> selectedFiles, CompactionKind kind);
 
   }
 
@@ -1128,8 +1129,8 @@ public ExternalCompactionJob reserveExternalCompaction(CompactionServiceId servi
     var cInfo = ocInfo.orElseThrow();
 
     try {
-      Map<String,String> overrides =
-          CompactableUtils.getOverrides(job.getKind(), tablet, cInfo.localHelper, job.getFiles());
+      Map<String,String> overrides = CompactableUtils.getOverrides(job.getKind(), tablet,
+          cInfo.localHelper, job.getFiles(), cInfo.selectedFiles);
 
       ReferencedTabletFile compactTmpName =
           tablet.getNextDataFilenameForMajc(cInfo.propagateDeletes);

server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java

@@ -81,7 +81,8 @@
 
 public class CompactableUtils {
 
-  static Map<String,String> computeOverrides(Tablet tablet, Set<CompactableFile> files) {
+  static Map<String,String> computeOverrides(Tablet tablet, Set<CompactableFile> inputFiles,
+      Set<StoredTabletFile> selectedFiles, CompactionKind kind) {
     var tconf = tablet.getTableConfiguration();
 
     var configurorClass = tconf.get(Property.TABLE_COMPACTION_CONFIGURER);
@@ -91,11 +92,12 @@ static Map<String,String> computeOverrides(Tablet tablet, Set<CompactableFile> f
 
     var opts = tconf.getAllPropertiesWithPrefixStripped(Property.TABLE_COMPACTION_CONFIGURER_OPTS);
 
-    return computeOverrides(tablet, files, new PluginConfig(configurorClass, opts));
+    return computeOverrides(tablet, inputFiles, selectedFiles,
+        new PluginConfig(configurorClass, opts), kind);
   }
 
-  static Map<String,String> computeOverrides(Tablet tablet, Set<CompactableFile> files,
-      PluginConfig cfg) {
+  static Map<String,String> computeOverrides(Tablet tablet, Set<CompactableFile> inputFiles,
+      Set<StoredTabletFile> selectedFiles, PluginConfig cfg, CompactionKind kind) {
     CompactionConfigurer configurer = CompactableUtils.newInstance(tablet.getTableConfiguration(),
         cfg.getClassName(), CompactionConfigurer.class);
 
@@ -121,7 +123,18 @@ public TableId getTableId() {
     var overrides = configurer.override(new CompactionConfigurer.InputParameters() {
       @Override
       public Collection<CompactableFile> getInputFiles() {
-        return files;
+        return inputFiles;
+      }
+
+      @Override
+      public Set<CompactableFile> getSelectedFiles() {
+        if (kind == CompactionKind.USER || kind == CompactionKind.SELECTOR) {
+          var dataFileSizes = tablet.getDatafileManager().getDatafileSizes();
+          return selectedFiles.stream().map(f -> new CompactableFileImpl(f, dataFileSizes.get(f)))
+              .collect(Collectors.toSet());
+        } else { // kind == CompactionKind.SYSTEM
+          return Collections.emptySet();
+        }
       }
 
       @Override
@@ -266,7 +279,8 @@ public Set<StoredTabletFile> selectFiles(SortedMap<StoredTabletFile,DataFileValu
     }
 
     @Override
-    public Map<String,String> getConfigOverrides(Set<CompactableFile> files) {
+    public Map<String,String> getConfigOverrides(Set<CompactableFile> inputFiles,
+        Set<StoredTabletFile> selectedFiles, CompactionKind kind) {
       return null;
     }
   }
@@ -306,9 +320,11 @@ public Set<StoredTabletFile> selectFiles(SortedMap<StoredTabletFile,DataFileValu
     }
 
     @Override
-    public Map<String,String> getConfigOverrides(Set<CompactableFile> files) {
+    public Map<String,String> getConfigOverrides(Set<CompactableFile> inputFiles,
+        Set<StoredTabletFile> selectedFiles, CompactionKind kind) {
       if (!UserCompactionUtils.isDefault(compactionConfig.getConfigurer())) {
-        return computeOverrides(tablet, files, compactionConfig.getConfigurer());
+        return computeOverrides(tablet, inputFiles, selectedFiles, compactionConfig.getConfigurer(),
+            kind);
       }
 
       return null;
@@ -340,16 +356,17 @@ public static CompactionHelper getHelper(CompactionKind kind, Tablet tablet, Lon
   }
 
   public static Map<String,String> getOverrides(CompactionKind kind, Tablet tablet,
-      CompactionHelper driver, Set<CompactableFile> files) {
+      CompactionHelper driver, Set<CompactableFile> inputFiles,
+      Set<StoredTabletFile> selectedFiles) {
 
     Map<String,String> overrides = null;
 
     if (kind == CompactionKind.USER || kind == CompactionKind.SELECTOR) {
-      overrides = driver.getConfigOverrides(files);
+      overrides = driver.getConfigOverrides(inputFiles, selectedFiles, kind);
     }
 
     if (overrides == null) {
-      overrides = computeOverrides(tablet, files);
+      overrides = computeOverrides(tablet, inputFiles, selectedFiles, kind);
     }
 
     if (overrides == null) {
@@ -380,8 +397,9 @@ static CompactionStats compact(Tablet tablet, CompactionJob job,
       throws IOException, CompactionCanceledException {
     TableConfiguration tableConf = tablet.getTableConfiguration();
 
-    AccumuloConfiguration compactionConfig = getCompactionConfig(tableConf,
-        getOverrides(job.getKind(), tablet, cInfo.localHelper, job.getFiles()));
+    AccumuloConfiguration compactionConfig =
+        getCompactionConfig(tableConf, getOverrides(job.getKind(), tablet, cInfo.localHelper,
+            job.getFiles(), cInfo.selectedFiles));
 
     FileCompactor compactor = new FileCompactor(tablet.getContext(), tablet.getExtent(),
         compactFiles, tmpFileName, cInfo.propagateDeletes, cenv, cInfo.iters, compactionConfig,