HIVE-26363: converting replLogger time format from epoch to UTC (apac…

…he#3541) HIVE-26363: converting replLogger time format from epoch to readable UTC (Rakshith Chandraiah, reviewed by Teddy Choi)
DongWei-4 · Oct 28, 2022 · dca9b25 · dca9b25
1 parent 73e0e88
commit dca9b25
Show file tree

Hide file tree

Showing 25 changed files with 242 additions and 34 deletions.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java
@@ -17,6 +17,10 @@
  */
 package org.apache.hadoop.hive.ql.exec.repl.util;
 
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -82,6 +86,7 @@
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -545,4 +550,11 @@ public static boolean isErrorRecoverable(Throwable e) {
   public static boolean filterTransactionOperations(HiveConf conf) {
     return (conf.getBoolVar(HiveConf.ConfVars.REPL_FILTER_TRANSACTIONS));
   }
+  public  static class TimeSerializer extends JsonSerializer<Long> {
+
+    @Override
+    public void serialize(Long epoch, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
+      jsonGenerator.writeString(Instant.ofEpochSecond(epoch).toString());
+    }
+  }
 }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/AtlasDumpBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/AtlasDumpBegin.java
@@ -17,11 +17,13 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
 import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import org.apache.hive.common.util.SuppressFBWarnings;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
-
 /**
  * AtlasDumpBegin.
  *
@@ -34,6 +36,7 @@ public class AtlasDumpBegin extends ReplState {
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpStartTime;
 
   public AtlasDumpBegin(String dbName) {

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/AtlasDumpEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/AtlasDumpEnd.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
 import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import org.apache.hive.common.util.SuppressFBWarnings;
-import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
  * AtlasDumpEnd.
@@ -33,6 +36,7 @@ public class AtlasDumpEnd extends ReplState {
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpEndTime;
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class BootstrapDumpBegin extends ReplState {
   @JsonProperty
@@ -35,6 +38,7 @@ public class BootstrapDumpBegin extends ReplState {
   private Long estimatedNumFunctions;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpStartTime;
 
   public BootstrapDumpBegin(String dbName, long estimatedNumTables, long estimatedNumFunctions) {

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class BootstrapDumpEnd extends ReplState {
   @JsonProperty
@@ -35,6 +38,7 @@ public class BootstrapDumpEnd extends ReplState {
   private Long actualNumFunctions;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpEndTime;
 
   @JsonProperty

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java
@@ -17,8 +17,11 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class BootstrapDumpFunction extends ReplState {
   @JsonProperty
@@ -31,6 +34,7 @@ public class BootstrapDumpFunction extends ReplState {
   private String functionsDumpProgress;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpTime;
 
   public BootstrapDumpFunction(String dbName, String funcName,

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
 import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
 import org.apache.hadoop.hive.ql.parse.repl.ReplState;
-import com.fasterxml.jackson.annotation.JsonProperty;
 
 public class BootstrapDumpTable extends ReplState {
   @JsonProperty
@@ -35,6 +38,7 @@ public class BootstrapDumpTable extends ReplState {
   private String tablesDumpProgress;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpTime;
 
   public BootstrapDumpTable(String dbName,

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class IncrementalDumpBegin extends ReplState {
   @JsonProperty
@@ -32,6 +35,7 @@ public class IncrementalDumpBegin extends ReplState {
   private Long estimatedNumEvents;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpStartTime;
 
   @JsonProperty

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class IncrementalDumpEnd extends ReplState {
   @JsonProperty
@@ -32,6 +35,7 @@ public class IncrementalDumpEnd extends ReplState {
   private Long actualNumEvents;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpEndTime;
 
   @JsonProperty

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java
@@ -17,8 +17,11 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class IncrementalDumpEvent extends ReplState {
   @JsonProperty
@@ -34,6 +37,7 @@ public class IncrementalDumpEvent extends ReplState {
   private String eventsDumpProgress;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpTime;
 
   public IncrementalDumpEvent(String dbName,

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/RangerDumpBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/RangerDumpBegin.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
 import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import org.apache.hive.common.util.SuppressFBWarnings;
-import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
  * RangerDumpBegin.
@@ -33,6 +36,7 @@ public class RangerDumpBegin extends ReplState {
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpStartTime;
 
   public RangerDumpBegin(String dbName) {

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/RangerDumpEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/RangerDumpEnd.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
 import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import org.apache.hive.common.util.SuppressFBWarnings;
-import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
  * RangerDumpEnd.
@@ -37,6 +40,7 @@ public class RangerDumpEnd extends ReplState {
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long dumpEndTime;
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/AtlasLoadBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/AtlasLoadBegin.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
 import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import org.apache.hive.common.util.SuppressFBWarnings;
-import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
  * Replication state for Atlas Load Begin.
@@ -35,6 +38,7 @@ public class AtlasLoadBegin extends ReplState {
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long loadStartTime;
 
   public AtlasLoadBegin(String sourceDbName, String targetDbName) {

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/AtlasLoadEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/AtlasLoadEnd.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
 import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import org.apache.hive.common.util.SuppressFBWarnings;
-import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
  * Replication state for Atlas Load End.
@@ -39,6 +42,7 @@ public class AtlasLoadEnd extends ReplState {
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long loadEndTime;
 
   @SuppressFBWarnings("URF_UNREAD_FIELD")

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class BootstrapLoadBegin extends ReplState {
   @JsonProperty
@@ -38,6 +41,7 @@ public class BootstrapLoadBegin extends ReplState {
   private Long numFunctions;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long loadStartTime;
 
   public BootstrapLoadBegin(String dbName, String dumpDir, long numTables, long numFunctions) {

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class BootstrapLoadEnd extends ReplState {
   @JsonProperty
@@ -35,6 +38,7 @@ public class BootstrapLoadEnd extends ReplState {
   private Long numFunctions;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long loadEndTime;
 
   @JsonProperty

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java
@@ -17,8 +17,11 @@
  */
 package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
 
-import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
 
 public class BootstrapLoadFunction extends ReplState {
   @JsonProperty
@@ -31,6 +34,7 @@ public class BootstrapLoadFunction extends ReplState {
   private String functionsLoadProgress;
 
   @JsonProperty
+  @JsonSerialize(using = ReplUtils.TimeSerializer.class)
   private Long loadTime;
 
   public BootstrapLoadFunction(String dbName, String funcName,