[AMORO-2540] Report expiring snapshots event for Iceberg and mixed format

ams/api/pom.xml

@@ -86,6 +86,11 @@
             <artifactId>fastjson</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.immutables</groupId>
+            <artifactId>value</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.iceberg</groupId>
             <artifactId>iceberg-api</artifactId>

ams/api/src/main/java/com/netease/arctic/ams/api/events/EventType.java

@@ -20,5 +20,6 @@
 
 /** Event types define */
 public enum EventType {
-  ICEBERG_REPORT
+  ICEBERG_REPORT,
+  EXPIRE_REPORT
 }

ams/api/src/main/java/com/netease/arctic/ams/api/events/ExpireEvent.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netease.arctic.ams.api.events;
+
+import org.immutables.value.Value;
+
+/**
+ * Amoro automatically maintains datalake metadata files(e.g. expire snapshots, clean orphan
+ * files...), every maintenance process will generate an event which could be reported to {@link
+ * com.netease.arctic.ams.api.events.EventListener EventListener}
+ */
+@Value.Immutable
+public interface ExpireEvent extends TableEvent {
+  /**
+   * The expiring process id
+   *
+   * @return id
+   */
+  long processId();
+
+  /**
+   * Operation of expiring(e.g. expire snapshots)
+   *
+   * @return expiring operation
+   */
+  ExpireOperation operation();
+
+  /**
+   * Details of expiring event
+   *
+   * @return event detail
+   */
+  ExpireResult expireResult();
+
+  /**
+   * Get expiring event result as target specific Class
+   *
+   * @param clazz Subclass of ExpireResult
+   * @return Specific Class
+   * @param <R> Specific Class
+   */
+  default <R extends ExpireResult> R getExpireResultAs(Class<R> clazz) {
+    return clazz.cast(expireResult());
+  }
+}

ams/api/src/main/java/com/netease/arctic/ams/api/events/ExpireOperation.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netease.arctic.ams.api.events;
+
+/** Define various expiration operations */
+public enum ExpireOperation {
+  EXPIRE_SNAPSHOTS,
+  REMOVE_ORPHAN_FILES,
+  REMOVE_DANGLING_FILES,
+  EXPIRE_DATA
+}

ams/api/src/main/java/com/netease/arctic/ams/api/events/ExpireResult.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netease.arctic.ams.api.events;
+
+import java.time.Duration;
+
+/** Expire event details */
+public interface ExpireResult {
+  /**
+   * total duration of the event
+   *
+   * @return duration
+   */
+  Duration totalDuration();
+}

ams/api/src/main/java/com/netease/arctic/ams/api/events/IcebergReportEvent.java

@@ -18,6 +18,7 @@
 
 package com.netease.arctic.ams.api.events;
 
+import com.netease.arctic.ams.api.TableFormat;
 import org.apache.iceberg.metrics.MetricsReport;
 
 /** An event triggered when iceberg report metrics */
@@ -27,16 +28,13 @@
   private final String catalog;
   private final String database;
   private final String table;
-  private final boolean external;
 
-  public IcebergReportEvent(
-      String catalog, String database, String table, boolean external, MetricsReport report) {
+  public IcebergReportEvent(String catalog, String database, String table, MetricsReport report) {
     this.report = report;
     this.timestamp = System.currentTimeMillis();
     this.catalog = catalog;
     this.database = database;
     this.table = table;
-    this.external = external;
   }
 
   @Override
@@ -65,8 +63,8 @@
   }
 
   @Override
-  public boolean isExternal() {
-    return external;
+  public TableFormat format() {
+    return TableFormat.ICEBERG;
   }
 
   /**

ams/api/src/main/java/com/netease/arctic/ams/api/events/TableEvent.java

@@ -18,6 +18,8 @@
 
 package com.netease.arctic.ams.api.events;
 
+import com.netease.arctic.ams.api.TableFormat;
+
 /** An event associated with a table */
 public interface TableEvent extends Event {
 
@@ -43,9 +45,9 @@ public interface TableEvent extends Event {
   String table();
 
   /**
-   * External or Internal Catalog that this event source related to.
+   * Effective table format
    *
-   * @return True if event source from an external catalog.
+   * @return table format
    */
-  boolean isExternal();
+  TableFormat format();
 }

ams/api/src/main/java/com/netease/arctic/ams/api/events/iceberg/ExpireSnapshotsResult.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netease.arctic.ams.api.events.iceberg;
+
+import com.netease.arctic.ams.api.events.ExpireResult;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.immutables.value.Value;
+
+import java.util.Set;
+
+/** Event details for expiring snapshots in native Iceberg format */
+@Value.Immutable
+public abstract class ExpireSnapshotsResult implements ExpireResult {
+  @Value.Default
+  public Set<String> deletedDataFiles() {
+    return Sets.newHashSet();
+  }
+
+  @Value.Default
+  public Set<String> deletedPositionDeleteFiles() {
+    return Sets.newHashSet();
+  }
+
+  @Value.Default
+  public Set<String> deletedEqualityDeleteFiles() {
+    return Sets.newHashSet();
+  }
+
+  public abstract Set<String> deletedMetadataFiles();
+
+  @Value.Default
+  public Set<String> deletedStatisticsFiles() {
+    return Sets.newHashSet();
+  }
+}

ams/api/src/main/java/com/netease/arctic/ams/api/events/iceberg/RemoveOrphanFilesResult.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netease.arctic.ams.api.events.iceberg;
+
+import com.netease.arctic.ams.api.events.ExpireResult;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.immutables.value.Value;
+
+import java.util.Set;
+
+/** Event details for cleaning orphan files in native Iceberg format */
+@Value.Immutable
+public abstract class RemoveOrphanFilesResult implements ExpireResult {
+  @Value.Default
+  public Set<String> removedFiles() {
+    return Sets.newHashSet();
+  }
+
+  @Value.Default
+  public long releasedFileSize() {
+    return 0L;
+  }
+}

ams/api/src/main/java/com/netease/arctic/ams/api/events/mixed/ExpireMixedSnapshotsResult.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netease.arctic.ams.api.events.mixed;
+
+import com.netease.arctic.ams.api.events.ExpireResult;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.immutables.value.Value;
+
+import java.util.Set;
+
+/** Event details for expiring snapshots in mixed format */
+@Value.Immutable
+public abstract class ExpireMixedSnapshotsResult implements ExpireResult {
+  @Value.Default
+  public Set<String> deletedBaseFiles() {
+    return Sets.newHashSet();
+  }
+
+  @Value.Default
+  public Set<String> deletedInsertFiles() {
+    return Sets.newHashSet();
+  }
+
+  @Value.Default
+  public Set<String> deletedPositionDeleteFiles() {
+    return Sets.newHashSet();
+  }
+
+  @Value.Default
+  public Set<String> deletedEqualityDeleteFiles() {
+    return Sets.newHashSet();
+  }
+
+  @Value.Default
+  public Set<String> deletedIcebergEqualityDeleteFiles() {
+    return Sets.newHashSet();
+  }
+
+  public abstract Set<String> deletedMetadataFiles();
+
+  @Value.Default
+  public Set<String> deletedStatisticsFiles() {
+    return Sets.newHashSet();
+  }
+}

ams/server/src/main/java/com/netease/arctic/server/RestCatalogService.java

@@ -373,7 +373,6 @@ public void reportMetrics(Context ctx) {
                   identifier.getCatalog(),
                   identifier.getDatabase(),
                   identifier.getTableName(),
-                  false,
                   metricsRequest.report());
           EventsManager.getInstance().emit(event);
           return null;