From c3d3f45e6ffd0884e7f82ce916ecff7713e057ed Mon Sep 17 00:00:00 2001 From: 1fanwang <1fannnw@gmail.com> Date: Tue, 12 May 2026 01:21:37 -0700 Subject: [PATCH] [HUDI-9624] Warn when event-time field is set without watermark tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a user sets `hoodie.payload.event.time.field` but leaves `hoodie.write.track.event.time.watermark` at its default `false`, event-time metadata is silently absent from commit metadata — the user's intent to use event-time semantics is dropped without any signal. Add a warning at write-config build time that names both the configured event-time field and the disabled tracking flag, so the user can opt in explicitly. The check is purely on write-config properties (no meta-client access needed) and runs once per write client construction, matching the existing pattern of advisory warnings in `Builder.validate()`. --- .../apache/hudi/config/HoodieWriteConfig.java | 18 +++++++++ .../hudi/config/TestHoodieWriteConfig.java | 40 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index a663c63dba780..52070ad71651f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -43,6 +43,7 @@ import org.apache.hudi.common.fs.FileSystemRetryConfig; import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; +import org.apache.hudi.common.model.HoodiePayloadProps; import org.apache.hudi.common.model.HoodiePreWriteCleanerPolicy; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecordMerger; @@ -3862,6 +3863,23 @@ private void validate() { checkArgument(lookbackCommits >= 0, String.format("%s must be non-negative, but was %d", ROLLING_METADATA_TIMELINE_LOOKBACK_COMMITS.key(), lookbackCommits)); + + validateEventTimeConfigs(); + } + + private void validateEventTimeConfigs() { + // Event-time field is configured but watermark tracking is off — the field + // will be ignored at commit time. Surface a hint so the user can opt in via + // TRACK_EVENT_TIME_WATERMARK. + String eventTimeFieldName = writeConfig.getString(HoodiePayloadProps.PAYLOAD_EVENT_TIME_FIELD_PROP_KEY); + if (!StringUtils.isNullOrEmpty(eventTimeFieldName) + && !writeConfig.getBooleanOrDefault(TRACK_EVENT_TIME_WATERMARK)) { + log.warn("{}={} is configured but {}={}; event-time watermark metadata will not be tracked. " + + "Set {}=true to record event-time watermark in commit metadata.", + HoodiePayloadProps.PAYLOAD_EVENT_TIME_FIELD_PROP_KEY, eventTimeFieldName, + TRACK_EVENT_TIME_WATERMARK.key(), writeConfig.getBooleanOrDefault(TRACK_EVENT_TIME_WATERMARK), + TRACK_EVENT_TIME_WATERMARK.key()); + } } public HoodieWriteConfig build() { diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java index 6c1d1418b1907..8ebbbbcc9b50c 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java @@ -29,6 +29,7 @@ import org.apache.hudi.common.engine.EngineType; import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; +import org.apache.hudi.common.model.HoodiePayloadProps; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.WriteConcurrencyMode; import org.apache.hudi.common.table.HoodieTableConfig; @@ -40,6 +41,12 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -48,9 +55,11 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -908,4 +917,35 @@ public void testUpdatesStrategyNotOverriddenWhenExplicitlySet() { .build(); assertEquals(customStrategy, writeConfig.getClusteringUpdatesStrategyClass()); } + + @Test + public void testWarnsWhenEventTimeFieldSetWithoutWatermarkTracking() { + Properties props = new Properties(); + props.setProperty(HoodiePayloadProps.PAYLOAD_EVENT_TIME_FIELD_PROP_KEY, "ts"); + // hoodie.write.track.event.time.watermark is not set; default is false. + List captured = new ArrayList<>(); + AbstractAppender appender = new AbstractAppender( + "CaptureAppender", null, null, true, Property.EMPTY_ARRAY) { + @Override + public void append(LogEvent event) { + captured.add(event.toImmutable()); + } + }; + appender.start(); + Logger logger = (Logger) LogManager.getLogger(HoodieWriteConfig.class); + logger.addAppender(appender); + try { + HoodieWriteConfig.newBuilder().withPath("/tmp").withProperties(props).build(); + } finally { + logger.removeAppender(appender); + appender.stop(); + } + boolean foundHint = captured.stream() + .filter(e -> e.getLevel() == Level.WARN) + .map(e -> e.getMessage().getFormattedMessage()) + .anyMatch(msg -> msg.contains(HoodiePayloadProps.PAYLOAD_EVENT_TIME_FIELD_PROP_KEY) + && msg.contains(HoodieWriteConfig.TRACK_EVENT_TIME_WATERMARK.key())); + assertTrue(foundHint, + "Expected warning hint about TRACK_EVENT_TIME_WATERMARK when event-time field is set without tracking"); + } }